#!/usr/bin/env bun /** * Export web store documents to markdown fixtures * * Usage: bun run scripts/export-web-store.ts */ import * as fs from 'node:fs'; import * as path from 'node:path'; interface StoreEntry { type: string; id: string; name: string; url?: string; status: string; } interface StoresData { stores: StoreEntry[]; } interface DocumentRow { id: string; content: string; metadata: string; } function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } function isStoresData(value: unknown): value is StoresData { if (!isRecord(value)) return false; if (!Array.isArray(value['stores'])) return false; return true; } function isDocumentRow(value: unknown): value is DocumentRow { if (!isRecord(value)) return false; return ( typeof value['id'] === 'string' && typeof value['content'] === 'string' && typeof value['metadata'] === 'string' ); } async function main(): Promise { const [storeName, outputDir] = process.argv.slice(2); if (!storeName || !outputDir) { console.error('Usage: bun run scripts/export-web-store.ts '); process.exit(1); } // Find store in stores.json const storesPath = path.join(process.cwd(), '.bluera/bluera-knowledge/data/stores.json'); if (!fs.existsSync(storesPath)) { console.error('stores.json not found'); process.exit(1); } const storesJson: unknown = JSON.parse(fs.readFileSync(storesPath, 'utf-8')); if (!isStoresData(storesJson)) { console.error('Invalid stores.json format'); process.exit(1); } const store = storesJson.stores.find((s) => s.name === storeName); if (!store) { console.error(`Store not found: ${storeName}`); console.error('Available stores:', storesJson.stores.map((s) => s.name).join(', ')); process.exit(1); } if (store.type !== 'web') { console.error(`Store ${storeName} is not a web store (type: ${store.type})`); process.exit(1); } // Load documents from LanceDB const lanceDbPath = path.join(process.cwd(), `.bluera/bluera-knowledge/data/documents_${store.id}.lance`); if (!fs.existsSync(lanceDbPath)) { console.error(`LanceDB not found: ${lanceDbPath}`); process.exit(1); } // Dynamic import for lancedb const lancedb = await import('@lancedb/lancedb'); const db = await lancedb.connect(path.join(process.cwd(), '.bluera/bluera-knowledge/data')); const table = await db.openTable(`documents_${store.id}`); // Query all documents const results = await table.query().select(['id', 'content', 'metadata']).toArray(); // Group chunks by source URL const pageChunks = new Map(); for (const row of results) { if (!isDocumentRow(row)) continue; const metadata: unknown = JSON.parse(row.metadata); if (!isRecord(metadata)) continue; const url = typeof metadata['url'] === 'string' ? metadata['url'] : 'unknown'; const title = typeof metadata['title'] === 'string' ? metadata['title'] : url; if (!pageChunks.has(url)) { pageChunks.set(url, { content: '', title, chunks: [] }); } const page = pageChunks.get(url); if (page) { page.chunks.push(row.content); } } // Create output directory const fullOutputDir = path.resolve(outputDir); if (!fs.existsSync(fullOutputDir)) { fs.mkdirSync(fullOutputDir, { recursive: true }); } // Write each page as a markdown file let fileCount = 0; for (const [url, page] of pageChunks) { // Generate filename from URL const urlPath = new URL(url).pathname.replace(/^\//, '').replace(/\/$/, '') || 'index'; const filename = urlPath.replace(/\//g, '-').replace(/[^a-zA-Z0-9-]/g, '') + '.md'; // Combine chunks (they may overlap, just concatenate for now) const content = `# ${page.title}\n\nSource: ${url}\n\n---\n\n${page.chunks.join('\n\n---\n\n')}`; fs.writeFileSync(path.join(fullOutputDir, filename), content); fileCount++; } console.log(`Exported ${fileCount} pages to ${fullOutputDir}`); } main().catch((err: unknown) => { console.error(err instanceof Error ? err.message : String(err)); process.exit(1); });