#!/usr/bin/env node /** * AgentKits Memory Web Viewer * * Web-based viewer for memory database with hybrid search support. * Uses ProjectMemoryService for vector + text search. * * Usage: * npx @aitytech/agentkits-memory web [--port=1905] * * @module @aitytech/agentkits-memory/cli/web-viewer */ import * as http from 'node:http'; import * as path from 'node:path'; import Database from 'better-sqlite3'; import type { Database as BetterDatabase } from 'better-sqlite3'; import { HybridSearchEngine, LocalEmbeddingsService } from '../index.js'; const args = process.argv.slice(2); const projectDir = process.env.CLAUDE_PROJECT_DIR || process.cwd(); // Embeddings service singleton let _embeddingsService: LocalEmbeddingsService | null = null; function parseArgs(): Record { const parsed: Record = {}; for (const arg of args) { if (arg.startsWith('--')) { const [key, value] = arg.slice(2).split('='); parsed[key] = value ?? true; } } return parsed; } const options = parseArgs(); const PORT = parseInt(options.port as string, 10) || 1905; const dbDir = path.join(projectDir, '.claude/memory'); const dbPath = path.join(dbDir, 'memory.db'); // Singleton database and search engine let _searchEngine: HybridSearchEngine | null = null; let _db: BetterDatabase | null = null; /** * Get direct database access (memory.db) */ function getDatabase(): BetterDatabase { if (_db) return _db; _db = new Database(dbPath); _db.pragma('journal_mode = WAL'); // Ensure all tables exist (web viewer may start before MCP server or hooks) _db.exec(` CREATE TABLE IF NOT EXISTS memory_entries ( id TEXT PRIMARY KEY, key TEXT NOT NULL, content TEXT NOT NULL, type TEXT DEFAULT 'semantic', namespace TEXT DEFAULT 'default', tags TEXT DEFAULT '[]', metadata TEXT DEFAULT '{}', embedding BLOB, session_id TEXT, owner_id TEXT, access_level TEXT DEFAULT 'project', created_at INTEGER NOT NULL DEFAULT (unixepoch('now') * 1000), updated_at INTEGER NOT NULL DEFAULT (unixepoch('now') * 1000), expires_at INTEGER, version INTEGER DEFAULT 1, "references" TEXT DEFAULT '[]', access_count INTEGER DEFAULT 0, last_accessed_at INTEGER NOT NULL DEFAULT (unixepoch('now') * 1000) ); CREATE TABLE IF NOT EXISTS sessions ( id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT UNIQUE NOT NULL, project TEXT NOT NULL, prompt TEXT, started_at INTEGER NOT NULL, ended_at INTEGER, observation_count INTEGER DEFAULT 0, summary TEXT, status TEXT DEFAULT 'active' ); CREATE TABLE IF NOT EXISTS observations ( id TEXT PRIMARY KEY, session_id TEXT NOT NULL, project TEXT NOT NULL, tool_name TEXT NOT NULL, tool_input TEXT, tool_response TEXT, cwd TEXT, timestamp INTEGER NOT NULL, type TEXT, title TEXT, prompt_number INTEGER, files_read TEXT DEFAULT '[]', files_modified TEXT DEFAULT '[]', subtitle TEXT, narrative TEXT, facts TEXT DEFAULT '[]', concepts TEXT DEFAULT '[]', embedding BLOB, FOREIGN KEY (session_id) REFERENCES sessions(session_id) ); CREATE TABLE IF NOT EXISTS user_prompts ( id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL, prompt_number INTEGER NOT NULL, prompt_text TEXT NOT NULL, created_at INTEGER NOT NULL, embedding BLOB, UNIQUE(session_id, prompt_number), FOREIGN KEY (session_id) REFERENCES sessions(session_id) ); CREATE TABLE IF NOT EXISTS session_summaries ( id INTEGER PRIMARY KEY AUTOINCREMENT, session_id TEXT NOT NULL, project TEXT NOT NULL, request TEXT, completed TEXT, files_read TEXT DEFAULT '[]', files_modified TEXT DEFAULT '[]', next_steps TEXT, notes TEXT, prompt_number INTEGER, created_at INTEGER NOT NULL, embedding BLOB, FOREIGN KEY (session_id) REFERENCES sessions(session_id) ); `); // Task queue table (shared with hooks service — used for embed + enrich workers) _db.exec(` CREATE TABLE IF NOT EXISTS task_queue ( id INTEGER PRIMARY KEY AUTOINCREMENT, task_type TEXT NOT NULL, target_table TEXT NOT NULL, target_id TEXT NOT NULL, created_at INTEGER NOT NULL, status TEXT DEFAULT 'pending' ) `); // Migration: add embedding column to existing session tables for (const table of ['observations', 'user_prompts', 'session_summaries']) { try { const cols = _db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>; if (!cols.some(c => c.name === 'embedding')) { _db.exec(`ALTER TABLE ${table} ADD COLUMN embedding BLOB`); } } catch { /* ignore */ } } return _db; } /** * Get or initialize embeddings service */ async function getEmbeddingsService(): Promise { if (_embeddingsService) return _embeddingsService; _embeddingsService = new LocalEmbeddingsService({ cacheDir: path.join(dbDir, 'embeddings-cache'), }); await _embeddingsService.initialize(); return _embeddingsService; } /** * Get or initialize the HybridSearchEngine with embeddings */ async function getSearchEngine(): Promise { if (_searchEngine) return _searchEngine; const db = getDatabase(); const embeddings = await getEmbeddingsService(); // Create embedding generator function const embeddingGenerator = async (text: string): Promise => { const result = await embeddings.embed(text); return result.embedding; }; _searchEngine = new HybridSearchEngine(db, {}, embeddingGenerator); await _searchEngine.initialize(); return _searchEngine; } // ===== Session Hybrid Search ===== /** * Cosine similarity between two Float32Arrays */ function cosineSimilarity(a: Float32Array, b: Float32Array): number { if (a.length !== b.length) return 0; let dot = 0, normA = 0, normB = 0; for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; normA += a[i] * a[i]; normB += b[i] * b[i]; } const denom = Math.sqrt(normA) * Math.sqrt(normB); return denom === 0 ? 0 : dot / denom; } /** * Extract text to embed for a session table row */ function getSessionEmbeddingText( table: 'observations' | 'user_prompts' | 'session_summaries', row: Record ): string { switch (table) { case 'observations': { const parts = [row.title, row.subtitle, row.narrative]; try { const concepts = JSON.parse((row.concepts as string) || '[]'); if (concepts.length > 0) parts.push(concepts.join(', ')); } catch { /* ignore */ } return parts.filter(Boolean).join(' ').trim(); } case 'user_prompts': return ((row.prompt_text as string) || '').trim(); case 'session_summaries': { const parts = [row.request, row.completed, row.next_steps, row.notes]; return parts.filter(Boolean).join(' ').trim(); } } } interface SessionSearchResult { table: 'observations' | 'user_prompts' | 'session_summaries'; id: string | number; sessionId: string; score: number; keywordScore: number; semanticScore: number; time: number; snippet: string; data: Record; } /** * Hybrid search across all session tables (text + vector) */ async function searchSessionsHybrid( db: BetterDatabase, query: string, options: { type?: 'hybrid' | 'text' | 'vector'; limit?: number } = {} ): Promise { const { type = 'hybrid', limit = 30 } = options; const results = new Map(); const queryLower = query.toLowerCase(); // === Text search (LIKE) === if (type === 'hybrid' || type === 'text') { const pattern = `%${query}%`; // Observations const obs = db.prepare(` SELECT * FROM observations WHERE title LIKE ? OR subtitle LIKE ? OR narrative LIKE ? OR tool_name LIKE ? ORDER BY timestamp DESC LIMIT ? `).all(pattern, pattern, pattern, pattern, limit) as Record[]; for (const row of obs) { const text = getSessionEmbeddingText('observations', row); const idx = text.toLowerCase().indexOf(queryLower); const kwScore = idx >= 0 ? Math.max(0.3, 1 - idx / 500) : 0.3; results.set(`obs_${row.id}`, { table: 'observations', id: row.id as string, sessionId: row.session_id as string, score: type === 'text' ? kwScore : kwScore * 0.3, keywordScore: kwScore, semanticScore: 0, time: row.timestamp as number, snippet: text.substring(0, 120), data: { ...row, embedding: undefined }, }); } // User prompts const prompts = db.prepare(` SELECT * FROM user_prompts WHERE prompt_text LIKE ? ORDER BY created_at DESC LIMIT ? `).all(pattern, limit) as Record[]; for (const row of prompts) { const text = (row.prompt_text as string) || ''; const idx = text.toLowerCase().indexOf(queryLower); const kwScore = idx >= 0 ? Math.max(0.3, 1 - idx / 500) : 0.3; results.set(`prompt_${row.id}`, { table: 'user_prompts', id: row.id as number, sessionId: row.session_id as string, score: type === 'text' ? kwScore : kwScore * 0.3, keywordScore: kwScore, semanticScore: 0, time: row.created_at as number, snippet: text.substring(0, 120), data: { ...row, embedding: undefined }, }); } // Session summaries const summaries = db.prepare(` SELECT * FROM session_summaries WHERE request LIKE ? OR completed LIKE ? OR notes LIKE ? OR next_steps LIKE ? ORDER BY created_at DESC LIMIT ? `).all(pattern, pattern, pattern, pattern, limit) as Record[]; for (const row of summaries) { const text = getSessionEmbeddingText('session_summaries', row); const idx = text.toLowerCase().indexOf(queryLower); const kwScore = idx >= 0 ? Math.max(0.3, 1 - idx / 500) : 0.3; results.set(`summary_${row.id}`, { table: 'session_summaries', id: row.id as number, sessionId: row.session_id as string, score: type === 'text' ? kwScore : kwScore * 0.3, keywordScore: kwScore, semanticScore: 0, time: row.created_at as number, snippet: text.substring(0, 120), data: { ...row, embedding: undefined }, }); } } // === Vector search === if ((type === 'hybrid' || type === 'vector') && query.trim()) { try { const embeddingsService = await getEmbeddingsService(); const queryResult = await embeddingsService.embed(query); const queryEmbedding = queryResult.embedding; const tables: Array<{ name: 'observations' | 'user_prompts' | 'session_summaries'; idCol: string; timeCol: string }> = [ { name: 'observations', idCol: 'id', timeCol: 'timestamp' }, { name: 'user_prompts', idCol: 'id', timeCol: 'created_at' }, { name: 'session_summaries', idCol: 'id', timeCol: 'created_at' }, ]; for (const { name, idCol, timeCol } of tables) { const rows = db.prepare( `SELECT * FROM ${name} WHERE embedding IS NOT NULL AND LENGTH(embedding) > 0 ORDER BY ${timeCol} DESC LIMIT 2000` ).all() as Record[]; for (const row of rows) { const embBuffer = row.embedding as Buffer; if (!embBuffer || embBuffer.length === 0) continue; const embedding = new Float32Array( embBuffer.buffer.slice(embBuffer.byteOffset, embBuffer.byteOffset + embBuffer.byteLength) ); const sim = cosineSimilarity(queryEmbedding, embedding); if (sim < 0.1) continue; const prefix = name === 'observations' ? 'obs' : name === 'user_prompts' ? 'prompt' : 'summary'; const key = `${prefix}_${row[idCol]}`; const existing = results.get(key); if (existing) { existing.semanticScore = sim; existing.score = existing.keywordScore * 0.3 + sim * 0.7; } else { const text = getSessionEmbeddingText(name, row); results.set(key, { table: name, id: row[idCol] as string | number, sessionId: row.session_id as string, score: type === 'vector' ? sim : sim * 0.7, keywordScore: 0, semanticScore: sim, time: row[timeCol] as number, snippet: text.substring(0, 120), data: { ...row, embedding: undefined }, }); } } } } catch { // Embeddings not available, fall back to text-only results } } return Array.from(results.values()) .filter(r => r.score >= 0.05) .sort((a, b) => b.score - a.score) .slice(0, limit); } /** * Get database statistics using direct SQL (faster for stats queries) */ function getStats(db: BetterDatabase): { total: number; byNamespace: Record; byType: Record; tokenEconomics: { totalTokens: number; avgTokensPerEntry: number; totalCharacters: number; estimatedSavings: number; }; } { const totalRow = db.prepare('SELECT COUNT(*) as count FROM memory_entries').get() as { count: number }; const total = totalRow?.count || 0; const nsRows = db.prepare('SELECT namespace, COUNT(*) as count FROM memory_entries GROUP BY namespace').all() as { namespace: string; count: number }[]; const byNamespace: Record = {}; for (const row of nsRows) { byNamespace[row.namespace] = row.count; } const typeRows = db.prepare('SELECT type, COUNT(*) as count FROM memory_entries GROUP BY type').all() as { type: string; count: number }[]; const byType: Record = {}; for (const row of typeRows) { byType[row.type] = row.count; } // Calculate token economics const contentRow = db.prepare('SELECT SUM(LENGTH(content)) as total_chars, COUNT(*) as count FROM memory_entries').get() as { total_chars: number; count: number }; const totalCharacters = contentRow?.total_chars || 0; const entryCount = contentRow?.count || 0; // Estimate tokens (~4 chars per token) const totalTokens = Math.ceil(totalCharacters / 4); const avgTokensPerEntry = entryCount > 0 ? Math.ceil(totalTokens / entryCount) : 0; // Estimated savings: if you had to rediscover this info each time // Assume 5x overhead for discovery vs recall const estimatedSavings = totalTokens * 5; return { total, byNamespace, byType, tokenEconomics: { totalTokens, avgTokensPerEntry, totalCharacters, estimatedSavings, }, }; } /** * Result type for getEntries with optional score and embedding info */ interface EntryResult { id: string; key: string; content: string; type: string; namespace: string; tags: string[]; created_at: number; updated_at: number; score?: number; hasEmbedding?: boolean; } /** * Get entries with optional search (standard listing) */ function getEntries( db: BetterDatabase, namespace?: string, limit = 50, offset = 0, search?: string ): EntryResult[] { // Standard query without search if (!search || !search.trim()) { let query = 'SELECT id, key, content, type, namespace, tags, embedding, created_at, updated_at FROM memory_entries'; const conditions: string[] = []; const params: (string | number)[] = []; if (namespace) { conditions.push('namespace = ?'); params.push(namespace); } if (conditions.length > 0) { query += ' WHERE ' + conditions.join(' AND '); } query += ' ORDER BY created_at DESC LIMIT ? OFFSET ?'; params.push(limit, offset); const rows = db.prepare(query).all(...params) as { id: string; key: string; content: string; type: string; namespace: string; tags: string; embedding: Buffer | null; created_at: number; updated_at: number; }[]; return rows.map((row) => ({ id: row.id, key: row.key, content: row.content, type: row.type, namespace: row.namespace, tags: JSON.parse(row.tags || '[]'), created_at: row.created_at, updated_at: row.updated_at, hasEmbedding: !!(row.embedding && row.embedding.length > 0), })); } // Use FTS5 search for better CJK support const sanitizedSearch = search.trim().replace(/"/g, '""'); let ftsQuery = ` SELECT m.id, m.key, m.content, m.type, m.namespace, m.tags, m.embedding, m.created_at, m.updated_at FROM memory_entries m INNER JOIN memory_fts f ON m.id = f.id WHERE memory_fts MATCH '"${sanitizedSearch}"' `; if (namespace) { ftsQuery += ` AND m.namespace = ?`; } ftsQuery += ` ORDER BY m.created_at DESC LIMIT ? OFFSET ?`; try { const params = namespace ? [namespace, limit, offset] : [limit, offset]; const rows = db.prepare(ftsQuery).all(...params) as { id: string; key: string; content: string; type: string; namespace: string; tags: string; embedding: Buffer | null; created_at: number; updated_at: number; }[]; return rows.map((row) => ({ id: row.id, key: row.key, content: row.content, type: row.type, namespace: row.namespace, tags: JSON.parse(row.tags || '[]'), created_at: row.created_at, updated_at: row.updated_at, hasEmbedding: !!(row.embedding && row.embedding.length > 0), })); } catch { // Fallback to LIKE if FTS fails console.warn('[WebViewer] FTS search failed, falling back to LIKE'); let query = 'SELECT id, key, content, type, namespace, tags, embedding, created_at, updated_at FROM memory_entries'; const conditions: string[] = []; const params: (string | number)[] = []; if (namespace) { conditions.push('namespace = ?'); params.push(namespace); } conditions.push('(content LIKE ? OR key LIKE ? OR tags LIKE ?)'); const searchPattern = `%${search}%`; params.push(searchPattern, searchPattern, searchPattern); query += ' WHERE ' + conditions.join(' AND '); query += ' ORDER BY created_at DESC LIMIT ? OFFSET ?'; params.push(limit, offset); const rows = db.prepare(query).all(...params) as { id: string; key: string; content: string; type: string; namespace: string; tags: string; embedding: Buffer | null; created_at: number; updated_at: number; }[]; return rows.map((row) => ({ id: row.id, key: row.key, content: row.content, type: row.type, namespace: row.namespace, tags: JSON.parse(row.tags || '[]'), created_at: row.created_at, updated_at: row.updated_at, hasEmbedding: !!(row.embedding && row.embedding.length > 0), })); } } /** * Search entries using HybridSearchEngine * Supports hybrid (text + vector), text-only, or vector-only search */ async function searchEntries( searchEngine: HybridSearchEngine, query: string, options: { type?: 'hybrid' | 'text' | 'vector'; namespace?: string; limit?: number; } = {} ): Promise { const { type = 'hybrid', namespace, limit = 20 } = options; // Use searchCompact for efficient search with scores const results = await searchEngine.searchCompact(query, { limit, namespace, includeKeyword: type === 'hybrid' || type === 'text', includeSemantic: type === 'hybrid' || type === 'vector', }); // Fetch full entries for the results const db = getDatabase(); const entries: EntryResult[] = []; for (const result of results) { const row = db.prepare(` SELECT id, key, content, type, namespace, tags, embedding, created_at, updated_at FROM memory_entries WHERE id = ? `).get(result.id) as { id: string; key: string; content: string; type: string; namespace: string; tags: string; embedding: Buffer | null; created_at: number; updated_at: number; } | undefined; if (row) { entries.push({ id: row.id, key: row.key, content: row.content, type: row.type, namespace: row.namespace, tags: JSON.parse(row.tags || '[]'), created_at: row.created_at, updated_at: row.updated_at, score: result.score, hasEmbedding: !!(row.embedding && row.embedding.length > 0), }); } } return entries; } function getHTML(): string { return ` AgentKits Memory Viewer
Vector index: loading...
Loading sessions...
`; } async function readBody(req: http.IncomingMessage): Promise { return new Promise((resolve, reject) => { let body = ''; req.on('data', chunk => body += chunk); req.on('end', () => resolve(body)); req.on('error', reject); }); } function handleRequest( req: http.IncomingMessage, res: http.ServerResponse ): void { const url = new URL(req.url || '/', `http://localhost:${PORT}`); const method = req.method || 'GET'; res.setHeader('Content-Type', 'application/json'); try { const db = getDatabase(); // Serve HTML if (url.pathname === '/' && method === 'GET') { res.setHeader('Content-Type', 'text/html'); res.writeHead(200); res.end(getHTML()); return; } // GET stats if (url.pathname === '/api/stats' && method === 'GET') { const stats = getStats(db); res.writeHead(200); res.end(JSON.stringify(stats)); return; } // GET entries (standard listing with optional FTS search) if (url.pathname === '/api/entries' && method === 'GET') { const namespace = url.searchParams.get('namespace') || undefined; const limit = parseInt(url.searchParams.get('limit') || '50', 10); const offset = parseInt(url.searchParams.get('offset') || '0', 10); const search = url.searchParams.get('search') || undefined; const entries = getEntries(db, namespace, limit, offset, search); res.writeHead(200); res.end(JSON.stringify(entries)); return; } // GET hybrid search (new endpoint with vector support) if (url.pathname === '/api/search' && method === 'GET') { const query = url.searchParams.get('q') || ''; const searchType = (url.searchParams.get('type') || 'hybrid') as 'hybrid' | 'text' | 'vector'; const limit = parseInt(url.searchParams.get('limit') || '20', 10); const namespace = url.searchParams.get('namespace') || undefined; getSearchEngine() .then((searchEngine) => searchEntries(searchEngine, query, { type: searchType, namespace, limit })) .then((results) => { res.writeHead(200); res.end(JSON.stringify(results)); }) .catch((error) => { res.writeHead(500); res.end(JSON.stringify({ error: error instanceof Error ? error.message : 'Search failed' })); }); return; } // POST create entry (direct DB for compatibility with existing schema) if (url.pathname === '/api/entries' && method === 'POST') { readBody(req) .then(async (body) => { const data = JSON.parse(body) as { key: string; content: string; type?: string; namespace?: string; tags?: string[]; }; const now = Date.now(); const id = `mem_${now}_${Math.random().toString(36).slice(2, 10)}`; const tags = JSON.stringify(data.tags || []); // Generate embedding for the content let embeddingBuffer: Buffer | null = null; try { const embeddingsService = await getEmbeddingsService(); const result = await embeddingsService.embed(data.content); embeddingBuffer = Buffer.from(result.embedding); } catch (e) { console.warn('[WebViewer] Failed to generate embedding:', e); } db.prepare( `INSERT INTO memory_entries (id, key, content, type, namespace, tags, embedding, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run(id, data.key, data.content, data.type || 'semantic', data.namespace || 'general', tags, embeddingBuffer, now, now); res.writeHead(201); res.end(JSON.stringify({ id, success: true })); }) .catch((error) => { res.writeHead(500); res.end(JSON.stringify({ error: error instanceof Error ? error.message : 'Internal error' })); }); return; } // GET single entry if (url.pathname.startsWith('/api/entry/') && method === 'GET') { const id = url.pathname.split('/').pop(); const row = db.prepare('SELECT * FROM memory_entries WHERE id = ?').get(id) as { id: string; key: string; content: string; type: string; namespace: string; tags: string; embedding: Buffer | null; created_at: number; updated_at: number; } | undefined; if (row) { // Extract embedding info for visualization let embeddingInfo: { hasEmbedding: boolean; dimensions?: number; preview?: number[] } = { hasEmbedding: false, }; if (row.embedding && row.embedding.length > 0) { const embedding = new Float32Array( row.embedding.buffer.slice( row.embedding.byteOffset, row.embedding.byteOffset + row.embedding.byteLength ) ); // Get first 20 values for preview visualization const preview = Array.from(embedding.slice(0, 20)); embeddingInfo = { hasEmbedding: true, dimensions: embedding.length, preview, }; } res.writeHead(200); res.end(JSON.stringify({ id: row.id, key: row.key, content: row.content, type: row.type, namespace: row.namespace, tags: JSON.parse(row.tags || '[]'), created_at: row.created_at, updated_at: row.updated_at, embedding: embeddingInfo, })); } else { res.writeHead(404); res.end(JSON.stringify({ error: 'Entry not found' })); } return; } // PUT update entry (direct DB for full field updates) if (url.pathname.startsWith('/api/entry/') && method === 'PUT') { const id = url.pathname.split('/').pop(); if (!id) { res.writeHead(400); res.end(JSON.stringify({ error: 'Missing entry ID' })); return; } readBody(req) .then(async (body) => { const data = JSON.parse(body) as { key: string; content: string; type: string; namespace: string; tags?: string[]; }; const now = Date.now(); const tags = JSON.stringify(data.tags || []); // Generate embedding for the updated content let embeddingBuffer: Buffer | null = null; try { const embeddingsService = await getEmbeddingsService(); const result = await embeddingsService.embed(data.content); embeddingBuffer = Buffer.from(result.embedding); } catch (e) { console.warn('[WebViewer] Failed to generate embedding:', e); } // Update with embedding const result = db.prepare( `UPDATE memory_entries SET key = ?, content = ?, type = ?, namespace = ?, tags = ?, embedding = ?, updated_at = ? WHERE id = ?` ).run(data.key, data.content, data.type, data.namespace, tags, embeddingBuffer, now, id); if (result.changes > 0) { res.writeHead(200); res.end(JSON.stringify({ success: true })); } else { res.writeHead(404); res.end(JSON.stringify({ error: 'Entry not found' })); } }) .catch((error) => { res.writeHead(500); res.end(JSON.stringify({ error: error instanceof Error ? error.message : 'Internal error' })); }); return; } // DELETE entry (direct DB for compatibility) if (url.pathname.startsWith('/api/entry/') && method === 'DELETE') { const id = url.pathname.split('/').pop(); if (!id) { res.writeHead(400); res.end(JSON.stringify({ error: 'Missing entry ID' })); return; } const result = db.prepare('DELETE FROM memory_entries WHERE id = ?').run(id); if (result.changes > 0) { res.writeHead(200); res.end(JSON.stringify({ success: true })); } else { res.writeHead(404); res.end(JSON.stringify({ error: 'Entry not found' })); } return; } // GET embedding stats if (url.pathname === '/api/embeddings/stats' && method === 'GET') { const totalRow = db.prepare('SELECT COUNT(*) as count FROM memory_entries').get() as { count: number }; const withEmbeddingRow = db.prepare('SELECT COUNT(*) as count FROM memory_entries WHERE embedding IS NOT NULL AND LENGTH(embedding) > 0').get() as { count: number }; res.writeHead(200); res.end(JSON.stringify({ total: totalRow?.count || 0, withEmbedding: withEmbeddingRow?.count || 0, withoutEmbedding: (totalRow?.count || 0) - (withEmbeddingRow?.count || 0), })); return; } // POST batch generate embeddings if (url.pathname === '/api/embeddings/generate' && method === 'POST') { readBody(req) .then(async (body) => { const options = JSON.parse(body || '{}') as { mode?: 'missing' | 'all' }; const mode = options.mode || 'missing'; // Get entries to process const query = mode === 'missing' ? 'SELECT id, content FROM memory_entries WHERE embedding IS NULL OR LENGTH(embedding) = 0' : 'SELECT id, content FROM memory_entries'; const entries = db.prepare(query).all() as { id: string; content: string }[]; if (entries.length === 0) { res.writeHead(200); res.end(JSON.stringify({ processed: 0, success: 0, failed: 0, message: 'No entries to process' })); return; } const embeddingsService = await getEmbeddingsService(); let success = 0; let failed = 0; const updateStmt = db.prepare('UPDATE memory_entries SET embedding = ?, updated_at = ? WHERE id = ?'); for (const entry of entries) { try { const result = await embeddingsService.embed(entry.content); const embeddingBuffer = Buffer.from(result.embedding); updateStmt.run(embeddingBuffer, Date.now(), entry.id); success++; } catch (e) { console.warn(`[WebViewer] Failed to generate embedding for ${entry.id}:`, e); failed++; } } res.writeHead(200); res.end(JSON.stringify({ processed: entries.length, success, failed, message: `Generated embeddings for ${success} entries${failed > 0 ? `, ${failed} failed` : ''}`, })); }) .catch((error) => { res.writeHead(500); res.end(JSON.stringify({ error: error instanceof Error ? error.message : 'Internal error' })); }); return; } // GET session hybrid search if (url.pathname === '/api/sessions/search' && method === 'GET') { const query = url.searchParams.get('q') || ''; const searchType = (url.searchParams.get('type') || 'hybrid') as 'hybrid' | 'text' | 'vector'; const limit = parseInt(url.searchParams.get('limit') || '30', 10); searchSessionsHybrid(db, query, { type: searchType, limit }) .then((results) => { res.writeHead(200); res.end(JSON.stringify(results)); }) .catch((error) => { res.writeHead(500); res.end(JSON.stringify({ error: error instanceof Error ? error.message : 'Search failed' })); }); return; } // GET session embeddings stats if (url.pathname === '/api/sessions/embeddings/stats' && method === 'GET') { try { const stats: Record = {}; for (const table of ['observations', 'user_prompts', 'session_summaries'] as const) { const total = (db.prepare(`SELECT COUNT(*) as c FROM ${table}`).get() as { c: number }).c; const withEmb = (db.prepare(`SELECT COUNT(*) as c FROM ${table} WHERE embedding IS NOT NULL AND LENGTH(embedding) > 0`).get() as { c: number }).c; stats[table] = { total, withEmbedding: withEmb }; } res.writeHead(200); res.end(JSON.stringify(stats)); } catch (error) { res.writeHead(500); res.end(JSON.stringify({ error: String(error) })); } return; } // POST generate session embeddings if (url.pathname === '/api/sessions/embeddings/generate' && method === 'POST') { readBody(req).then(async (body) => { try { const opts = JSON.parse(body || '{}') as { mode?: 'missing' | 'all' }; const mode = opts.mode || 'missing'; const embeddingsService = await getEmbeddingsService(); let totalSuccess = 0, totalFailed = 0; const tableConfigs = [ { name: 'observations' as const, idCol: 'id' }, { name: 'user_prompts' as const, idCol: 'id' }, { name: 'session_summaries' as const, idCol: 'id' }, ]; for (const { name, idCol } of tableConfigs) { const where = mode === 'missing' ? 'WHERE embedding IS NULL OR LENGTH(embedding) = 0' : ''; const rows = db.prepare(`SELECT * FROM ${name} ${where}`).all() as Record[]; const updateStmt = db.prepare(`UPDATE ${name} SET embedding = ? WHERE ${idCol} = ?`); for (const row of rows) { const text = getSessionEmbeddingText(name, row); if (!text) { totalFailed++; continue; } try { const result = await embeddingsService.embed(text); const buffer = Buffer.from(result.embedding); updateStmt.run(buffer, row[idCol]); totalSuccess++; } catch { totalFailed++; } } } res.writeHead(200); res.end(JSON.stringify({ processed: totalSuccess + totalFailed, success: totalSuccess, failed: totalFailed, })); } catch (error) { res.writeHead(500); res.end(JSON.stringify({ error: error instanceof Error ? error.message : 'Generation failed' })); } }).catch(() => { res.writeHead(400); res.end(JSON.stringify({ error: 'Invalid request body' })); }); return; } // GET sessions data (sessions, prompts, summaries) - all in memory.db now if (url.pathname === '/api/sessions' && method === 'GET') { const limit = parseInt(url.searchParams.get('limit') || '20', 10); const offset = parseInt(url.searchParams.get('offset') || '0', 10); const query = url.searchParams.get('q') || ''; // Strip embedding BLOBs and add hasEmbedding flag const stripEmb = (rows: Record[]) => rows.map(r => ({ ...r, hasEmbedding: !!(r.embedding && (r.embedding as Buffer).length > 0), embedding: undefined })); try { let sessions: Record[]; if (query) { const pattern = `%${query}%`; sessions = db.prepare(` SELECT * FROM sessions WHERE session_id LIKE ? OR project LIKE ? OR prompt LIKE ? OR summary LIKE ? ORDER BY started_at DESC LIMIT ? OFFSET ? `).all(pattern, pattern, pattern, pattern, limit, offset) as Record[]; } else { sessions = db.prepare(` SELECT * FROM sessions ORDER BY started_at DESC LIMIT ? OFFSET ? `).all(limit, offset) as Record[]; } // user_prompts let prompts: Record[] = []; try { if (query) { const pattern = `%${query}%`; prompts = stripEmb(db.prepare(` SELECT up.*, s.project FROM user_prompts up JOIN sessions s ON s.session_id = up.session_id WHERE up.prompt_text LIKE ? ORDER BY up.created_at DESC LIMIT ? OFFSET ? `).all(pattern, limit, offset) as Record[]); } else { prompts = stripEmb(db.prepare(` SELECT up.*, s.project FROM user_prompts up JOIN sessions s ON s.session_id = up.session_id ORDER BY up.created_at DESC LIMIT ? OFFSET ? `).all(limit, offset) as Record[]); } } catch { /* table may not exist */ } // session_summaries let summaries: Record[] = []; try { if (query) { const pattern = `%${query}%`; summaries = stripEmb(db.prepare(` SELECT * FROM session_summaries WHERE request LIKE ? OR completed LIKE ? OR notes LIKE ? OR next_steps LIKE ? ORDER BY created_at DESC LIMIT ? OFFSET ? `).all(pattern, pattern, pattern, pattern, limit, offset) as Record[]); } else { summaries = stripEmb(db.prepare(` SELECT * FROM session_summaries ORDER BY created_at DESC LIMIT ? OFFSET ? `).all(limit, offset) as Record[]); } } catch { /* table may not exist */ } res.writeHead(200); res.end(JSON.stringify({ sessions, prompts, summaries })); } catch (error) { res.writeHead(200); res.end(JSON.stringify({ sessions: [], prompts: [], summaries: [], error: String(error) })); } return; } // GET observations from memory.db if (url.pathname === '/api/observations' && method === 'GET') { const limit = parseInt(url.searchParams.get('limit') || '50', 10); const offset = parseInt(url.searchParams.get('offset') || '0', 10); const sessionId = url.searchParams.get('session_id') || undefined; const query = url.searchParams.get('q') || ''; // Strip embedding BLOBs and add hasEmbedding flag const stripEmb = (rows: Record[]) => rows.map(r => ({ ...r, hasEmbedding: !!(r.embedding && (r.embedding as Buffer).length > 0), embedding: undefined })); try { let rows: Record[]; if (query) { const pattern = `%${query}%`; if (sessionId) { rows = db.prepare(` SELECT * FROM observations WHERE session_id = ? AND (tool_name LIKE ? OR title LIKE ? OR subtitle LIKE ? OR narrative LIKE ?) ORDER BY timestamp DESC LIMIT ? OFFSET ? `).all(sessionId, pattern, pattern, pattern, pattern, limit, offset) as Record[]; } else { rows = db.prepare(` SELECT * FROM observations WHERE tool_name LIKE ? OR title LIKE ? OR subtitle LIKE ? OR narrative LIKE ? ORDER BY timestamp DESC LIMIT ? OFFSET ? `).all(pattern, pattern, pattern, pattern, limit, offset) as Record[]; } } else if (sessionId) { rows = db.prepare(` SELECT * FROM observations WHERE session_id = ? ORDER BY timestamp DESC LIMIT ? OFFSET ? `).all(sessionId, limit, offset) as Record[]; } else { rows = db.prepare(` SELECT * FROM observations ORDER BY timestamp DESC LIMIT ? OFFSET ? `).all(limit, offset) as Record[]; } res.writeHead(200); res.end(JSON.stringify(stripEmb(rows))); } catch { res.writeHead(200); res.end(JSON.stringify([])); } return; } // ===== Hook API Endpoints ===== // GET /api/hook/sessions - List hook sessions if (url.pathname === '/api/hook/sessions' && method === 'GET') { const project = url.searchParams.get('project') || undefined; const limit = parseInt(url.searchParams.get('limit') || '20', 10); try { let rows: Record[]; if (project) { rows = db.prepare( 'SELECT * FROM sessions WHERE project = ? ORDER BY started_at DESC LIMIT ?' ).all(project, limit) as Record[]; } else { rows = db.prepare( 'SELECT * FROM sessions ORDER BY started_at DESC LIMIT ?' ).all(limit) as Record[]; } res.writeHead(200); res.end(JSON.stringify(rows)); } catch (error) { res.writeHead(500); res.end(JSON.stringify({ error: String(error) })); } return; } // GET /api/hook/observations - List hook observations if (url.pathname === '/api/hook/observations' && method === 'GET') { const project = url.searchParams.get('project') || undefined; const limit = parseInt(url.searchParams.get('limit') || '50', 10); try { let rows: Record[]; if (project) { rows = db.prepare( 'SELECT id, session_id, project, tool_name, timestamp, type, title, subtitle, narrative, facts, concepts, prompt_number, compressed_summary, is_compressed FROM observations WHERE project = ? ORDER BY timestamp DESC LIMIT ?' ).all(project, limit) as Record[]; } else { rows = db.prepare( 'SELECT id, session_id, project, tool_name, timestamp, type, title, subtitle, narrative, facts, concepts, prompt_number, compressed_summary, is_compressed FROM observations ORDER BY timestamp DESC LIMIT ?' ).all(limit) as Record[]; } res.writeHead(200); res.end(JSON.stringify(rows)); } catch (error) { res.writeHead(500); res.end(JSON.stringify({ error: String(error) })); } return; } // GET /api/hook/session/:id - Session detail with observations and prompts if (url.pathname.startsWith('/api/hook/session/') && method === 'GET') { const sessionId = url.pathname.slice('/api/hook/session/'.length); try { const session = db.prepare('SELECT * FROM sessions WHERE session_id = ?').get(sessionId); if (!session) { res.writeHead(404); res.end(JSON.stringify({ error: 'Session not found' })); return; } const observations = db.prepare( 'SELECT id, tool_name, timestamp, type, title, subtitle, narrative, facts, concepts, prompt_number, compressed_summary, is_compressed FROM observations WHERE session_id = ? ORDER BY timestamp ASC' ).all(sessionId); const prompts = db.prepare( 'SELECT * FROM user_prompts WHERE session_id = ? ORDER BY prompt_number ASC' ).all(sessionId); const summary = db.prepare( 'SELECT * FROM session_summaries WHERE session_id = ? ORDER BY created_at DESC LIMIT 1' ).get(sessionId); res.writeHead(200); res.end(JSON.stringify({ session, observations, prompts, summary })); } catch (error) { res.writeHead(500); res.end(JSON.stringify({ error: String(error) })); } return; } // GET /api/hook/queue/status - Task queue stats if (url.pathname === '/api/hook/queue/status' && method === 'GET') { try { const stats = db.prepare(` SELECT task_type, status, COUNT(*) as count FROM task_queue GROUP BY task_type, status `).all(); const total = (db.prepare('SELECT COUNT(*) as c FROM task_queue').get() as { c: number }).c; res.writeHead(200); res.end(JSON.stringify({ total, breakdown: stats })); } catch (error) { res.writeHead(500); res.end(JSON.stringify({ error: String(error) })); } return; } // POST /api/hook/cleanup - Clean old completed/failed queue tasks if (url.pathname === '/api/hook/cleanup' && method === 'POST') { try { const oneDayAgo = Date.now() - 86400000; const result = db.prepare( "DELETE FROM task_queue WHERE status IN ('completed', 'failed') OR (status = 'processing' AND created_at < ?)" ).run(oneDayAgo); res.writeHead(200); res.end(JSON.stringify({ deleted: result.changes })); } catch (error) { res.writeHead(500); res.end(JSON.stringify({ error: String(error) })); } return; } res.writeHead(404); res.end(JSON.stringify({ error: 'Not found' })); } catch (error) { res.writeHead(500); res.end(JSON.stringify({ error: error instanceof Error ? error.message : 'Internal error' })); } } const server = http.createServer(handleRequest); /** Try to listen on the given port; on EADDRINUSE, pick a random available port. */ function startServer(port: number) { server.listen(port, () => { const addr = server.address(); const actualPort = typeof addr === 'object' && addr ? addr.port : port; console.log(`\n AgentKits Memory Viewer\n`); console.log(` Local: http://localhost:${actualPort}`); console.log(` Database: ${dbPath}\n`); console.log(` Press Ctrl+C to stop\n`); }); server.on('error', (err: NodeJS.ErrnoException) => { if (err.code === 'EADDRINUSE') { console.log(` Port ${port} is in use, finding an available port...`); // Listen on port 0 to let the OS assign a random available port server.listen(0, () => { const addr = server.address(); const actualPort = typeof addr === 'object' && addr ? addr.port : 0; console.log(`\n AgentKits Memory Viewer\n`); console.log(` Local: http://localhost:${actualPort}`); console.log(` Database: ${dbPath}\n`); console.log(` Press Ctrl+C to stop\n`); }); } else { console.error(` Failed to start server: ${err.message}`); process.exit(1); } }); } startServer(PORT); // Graceful shutdown: close server, DB, and embeddings service on SIGINT/SIGTERM function cleanup() { server.close(); if (_db) { try { _db.close(); } catch { /* ignore */ } _db = null; } if (_embeddingsService) { _embeddingsService = null; } if (_searchEngine) { _searchEngine = null; } process.exit(0); } process.on('SIGINT', cleanup); process.on('SIGTERM', cleanup);