/** * gno embed command implementation. * Batch embed chunks into vector storage. * * @module src/cli/commands/embed */ import type { Database } from "bun:sqlite"; import type { EmbeddingPort } from "../../llm/types"; import type { StoreResult } from "../../store/types"; import { getIndexDbPath } from "../../app/constants"; import { type Config, getConfigPaths, isInitialized, loadConfig, } from "../../config"; import { type CliWriteLeaseOptions, type WriteLeaseContention, withCliWriteLease, } from "../../core/write-lease"; import { embedBacklog, prepareEmbeddingBacklog } from "../../embed/backlog"; import { getEmbeddingFingerprint } from "../../embed/fingerprint"; import { addUniqueSamples, chunkRetryKey, embedAndStoreBatch, MAX_EMBED_CHUNK_ATTEMPTS, type EmbedStoreBatchResult, } from "../../embed/retry"; import { findInterruptedStage, formatInterruptedStage, type InterruptedStage, markIndexStageFinished, markIndexStageRunning, readIndexStageState, } from "../../embed/stage-state"; import { countVariantBacklog } from "../../embed/variant-plan"; import { LlmAdapter } from "../../llm/nodeLlamaCpp/adapter"; import { resolveDownloadPolicy } from "../../llm/policy"; import { resolveModelUri } from "../../llm/registry"; import { SqliteAdapter } from "../../store/sqlite/adapter"; import { err, ok } from "../../store/types"; import { type BacklogItem, createVectorIndexPort, createVectorStatsPort, type VectorIndexPort, type VectorStatsPort, } from "../../store/vector"; import { getGlobals } from "../program"; import { createProgressRenderer, createThrottledProgressRenderer, } from "../progress"; // ───────────────────────────────────────────────────────────────────────────── // Types // ───────────────────────────────────────────────────────────────────────────── export interface EmbedOptions extends CliWriteLeaseOptions { /** Override config path */ configPath?: string; /** Index name */ indexName?: string; /** Restrict embedding work to a single collection */ collection?: string; /** Override model URI */ model?: string; /** Batch size for embedding */ batchSize?: number; /** Re-embed all chunks (not just backlog) */ force?: boolean; /** Show what would be done without embedding */ dryRun?: boolean; /** Skip confirmation prompts */ yes?: boolean; /** Output as JSON */ json?: boolean; /** Verbose error logging */ verbose?: boolean; /** Use cached models only (also used by the standalone setup worker). */ offline?: boolean; /** * Resume preamble already handled by the caller (`gno index`). When set * (including `null`), embed() neither re-detects an interrupted stage nor * prints its own preamble; the value is echoed in the result. */ resumedFrom?: InterruptedStage | null; } export type EmbedResult = ( | { success: true; embedded: number; errors: number; /** Chunks deferred by SQLITE_BUSY/LOCKED after retries (fn-127 R6). */ contentionErrors: number; duration: number; model: string; searchAvailable: boolean; errorSamples?: string[]; suggestion?: string; syncError?: string; } | { success: false; error: string; contention?: WriteLeaseContention } ) & { /** Stage a previous run left `running` (fn-132 R4); null when none. */ resumedFrom?: InterruptedStage | null; }; /** * Stage outcome for the persisted marker and the `gno index` receipt: the * embed stage completed only when every chunk it attempted was stored. */ export function embedStageOutcome(result: EmbedResult): "completed" | "failed" { if (!result.success) { return "failed"; } return result.errors === 0 && !result.syncError ? "completed" : "failed"; } // ───────────────────────────────────────────────────────────────────────────── // Helpers // ───────────────────────────────────────────────────────────────────────────── function formatDuration(seconds: number): string { if (seconds < 60) { return `${seconds.toFixed(1)}s`; } const mins = Math.floor(seconds / 60); const secs = seconds % 60; return `${mins}m ${secs.toFixed(0)}s`; } function isDisposedBatchError(message: string): boolean { return message.toLowerCase().includes("object is disposed"); } interface BatchContext { db: import("bun:sqlite").Database; stats: VectorStatsPort; embedPort: EmbeddingPort; vectorIndex: VectorIndexPort; modelUri: string; collection?: string; batchSize: number; force: boolean; showProgress: boolean; totalToEmbed: number; verbose: boolean; recreateEmbedPort?: () => Promise< { ok: true; value: EmbeddingPort } | { ok: false; error: string } >; } type BatchResult = | { ok: true; embedded: number; errors: number; contentionErrors: number; duration: number; errorSamples: string[]; suggestion?: string; } | { ok: false; error: string }; interface Cursor { mirrorHash: string; seq: number; } async function processBatches(ctx: BatchContext): Promise { const startTime = Date.now(); let embedded = 0; let errors = 0; let contentionErrors = 0; const errorSamples: string[] = []; let suggestion: string | undefined; let cursor: Cursor | undefined; const retryQueue = new Map(); const embedFingerprint = getEmbeddingFingerprint({ modelUri: ctx.modelUri, dimensions: ctx.vectorIndex.dimensions, }); const pushErrorSamples = (samples: string[]): void => { addUniqueSamples(errorSamples, samples); }; const enqueueRetryItems = (items: BacklogItem[], attempts: number): void => { for (const item of items) { const key = chunkRetryKey(item); const existing = retryQueue.get(key); retryQueue.set(key, { item, attempts: Math.max(existing?.attempts ?? 0, attempts), }); } }; const writeBatchDiagnostics = ( batch: BacklogItem[], result: EmbedStoreBatchResult ): void => { if (ctx.verbose && result.batchFailed) { const titles = batch .slice(0, 3) .map((item) => item.title ?? item.mirrorHash.slice(0, 8)) .join(", "); process.stderr.write( `\n[embed] Batch fallback (${batch.length} chunks: ${titles}${batch.length > 3 ? "..." : ""}): ${result.batchError ?? "unknown batch error"}\n` ); } if (ctx.verbose && result.errorSamples.length > 0) { for (const sample of result.errorSamples) { process.stderr.write(`\n[embed] Sample failure: ${sample}\n`); } } }; const processStoreBatch = async ( batch: BacklogItem[] ): Promise => { let result = await embedAndStoreBatch({ embedPort: ctx.embedPort, vectorIndex: ctx.vectorIndex, items: batch, modelUri: ctx.modelUri, embedFingerprint, }); if ( ctx.recreateEmbedPort && result.retryItems.length === batch.length && result.batchError && isDisposedBatchError(result.batchError) ) { if (ctx.verbose) { process.stderr.write( "\n[embed] Embedding port disposed; recreating model/contexts and retrying batch once\n" ); } const recreated = await ctx.recreateEmbedPort(); if (recreated.ok) { ctx.embedPort = recreated.value; result = await embedAndStoreBatch({ embedPort: ctx.embedPort, vectorIndex: ctx.vectorIndex, items: batch, modelUri: ctx.modelUri, embedFingerprint, }); if (ctx.verbose && result.embedded > 0) { process.stderr.write("\n[embed] Retry after port reset succeeded\n"); } } } return result; }; const renderProgress = (): void => { if (!ctx.showProgress) { return; } const embeddedDisplay = Math.min(embedded, ctx.totalToEmbed); const completed = Math.min(embedded + errors, ctx.totalToEmbed); const pct = (completed / ctx.totalToEmbed) * 100; const elapsed = (Date.now() - startTime) / 1000; const rate = embedded / Math.max(elapsed, 0.001); const eta = Math.max(0, ctx.totalToEmbed - completed) / Math.max(rate, 0.001); process.stdout.write( `\rEmbedding: ${embeddedDisplay.toLocaleString()}/${ctx.totalToEmbed.toLocaleString()} (${pct.toFixed(1)}%) | ${rate.toFixed(1)} chunks/s | ETA ${formatDuration(eta)}` ); }; const drainRetryQueue = async (): Promise => { if (retryQueue.size === 0) { return 0; } let retryEmbedded = 0; const entries = [...retryQueue.values()].filter( (entry) => entry.attempts < MAX_EMBED_CHUNK_ATTEMPTS ); for (let idx = 0; idx < entries.length; idx += ctx.batchSize) { const slice = entries.slice(idx, idx + ctx.batchSize); for (const entry of slice) { retryQueue.delete(chunkRetryKey(entry.item)); entry.attempts += 1; } const retryResult = await processStoreBatch( slice.map((entry) => entry.item) ); writeBatchDiagnostics( slice.map((entry) => entry.item), retryResult ); pushErrorSamples(retryResult.errorSamples); suggestion ||= retryResult.suggestion; embedded += retryResult.embedded; errors += retryResult.errors; contentionErrors += retryResult.contentionErrors; retryEmbedded += retryResult.embedded; const retryByKey = new Set( retryResult.retryItems.map((item) => chunkRetryKey(item)) ); for (const entry of slice) { if (!retryByKey.has(chunkRetryKey(entry.item))) { continue; } if (entry.attempts >= MAX_EMBED_CHUNK_ATTEMPTS) { errors += 1; } else { retryQueue.set(chunkRetryKey(entry.item), entry); } } renderProgress(); } return retryEmbedded; }; while (embedded + errors < ctx.totalToEmbed) { // Get next batch using seek pagination (cursor-based) const batchResult = ctx.force ? await getActiveChunks(ctx.db, ctx.batchSize, cursor, ctx.collection) : await ctx.stats.getBacklog(ctx.modelUri, embedFingerprint, { limit: ctx.batchSize, after: cursor, collection: ctx.collection, }); if (!batchResult.ok) { return { ok: false, error: batchResult.error.message }; } const batch = batchResult.value; if (batch.length === 0) { break; } // Advance cursor to last item (even on failure, to avoid infinite loops) const lastItem = batch.at(-1); if (lastItem) { cursor = { mirrorHash: lastItem.mirrorHash, seq: lastItem.seq }; } const beforeEmbedded = embedded; const batchStoreResult = await processStoreBatch(batch); writeBatchDiagnostics(batch, batchStoreResult); pushErrorSamples(batchStoreResult.errorSamples); suggestion ||= batchStoreResult.suggestion; embedded += batchStoreResult.embedded; errors += batchStoreResult.errors; contentionErrors += batchStoreResult.contentionErrors; enqueueRetryItems(batchStoreResult.retryItems, 1); if (embedded > beforeEmbedded) { await drainRetryQueue(); } renderProgress(); } await drainRetryQueue(); if (retryQueue.size > 0) { errors += retryQueue.size; pushErrorSamples(["Some chunks failed after same-run retry attempts"]); suggestion ??= "Some chunks failed after retry. Rerun `gno --verbose embed --batch-size 1` to isolate failing chunks."; retryQueue.clear(); } if (ctx.showProgress) { process.stdout.write("\n"); } return { ok: true, embedded, errors, contentionErrors, duration: (Date.now() - startTime) / 1000, errorSamples, suggestion, }; } // ───────────────────────────────────────────────────────────────────────────── // Helpers // ───────────────────────────────────────────────────────────────────────────── interface EmbedContext { config: Config; modelUri: string; store: SqliteAdapter; } /** * Initialize embed context: check init, load config, open store. */ async function initEmbedContext( configPath?: string, indexName?: string, collection?: string, model?: string ): Promise<({ ok: true } & EmbedContext) | { ok: false; error: string }> { const initialized = await isInitialized(configPath); if (!initialized) { return { ok: false, error: "GNO not initialized. Run: gno init" }; } const configResult = await loadConfig(configPath); if (!configResult.ok) { return { ok: false, error: configResult.error.message }; } const config = configResult.value; if ( collection && !config.collections.some((candidate) => candidate.name === collection) ) { return { ok: false, error: `Collection not found: ${collection}` }; } const modelUri = resolveModelUri(config, "embed", model, collection); const store = new SqliteAdapter(); const dbPath = getIndexDbPath(indexName); const paths = getConfigPaths(); store.setConfigPath(configPath ?? paths.configFile); const openResult = await store.open( dbPath, config.ftsTokenizer, config.busyTimeoutMs ); if (!openResult.ok) { return { ok: false, error: openResult.error.message }; } return { ok: true, config, modelUri, store }; } // ───────────────────────────────────────────────────────────────────────────── // Main Command // ───────────────────────────────────────────────────────────────────────────── /** * Execute gno embed command. */ export async function embed(options: EmbedOptions = {}): Promise { const batchSize = options.batchSize ?? 32; const force = options.force ?? false; const dryRun = options.dryRun ?? false; return await withCliWriteLease(options, async () => { // Initialize config and store const initResult = await initEmbedContext( options.configPath, options.indexName, options.collection, options.model ); if (!initResult.ok) { return { success: false, error: initResult.error }; } const { config, modelUri, store } = initResult; // Get raw DB for vector ops (SqliteAdapter always implements SqliteDbProvider) const db = store.getRawDb(); // Resume preamble (fn-132 R4): a stage left `running` by a dead process. // `gno index` detects it once for both stages and hands it in. const resumedFrom = options.resumedFrom === undefined ? findInterruptedStage(readIndexStageState(db)) : options.resumedFrom; if (options.resumedFrom === undefined && resumedFrom && !options.json) { process.stderr.write(`${formatInterruptedStage(resumedFrom)}\n`); } const runEmbedPass = async (): Promise => { let embedPort: EmbeddingPort | null = null; let vectorIndex: VectorIndexPort | null = null; const runEmbedBody = async (): Promise => { // Create stats port for backlog detection const stats: VectorStatsPort = createVectorStatsPort(db); let totalToEmbed = 0; // Create LLM adapter and embedding port with auto-download let offline = options.offline; if (offline === undefined) { offline = getGlobals().offline; } const policy = resolveDownloadPolicy(process.env, { offline, }); // Create progress renderer for model download (throttled to avoid spam) const showDownloadProgress = !options.json && process.stderr.isTTY; const downloadProgress = showDownloadProgress ? createThrottledProgressRenderer(createProgressRenderer()) : undefined; const llm = new LlmAdapter(config); const recreateEmbedPort = async () => { if (embedPort) { await embedPort.dispose(); } await llm.getManager().dispose(modelUri); const recreated = await llm.createEmbeddingPort(modelUri, { egressCollections: options.collection ? [options.collection] : "all", policy, onProgress: downloadProgress ? (progress) => downloadProgress("embed", progress) : undefined, }); if (!recreated.ok) { return { ok: false as const, error: recreated.error.message }; } const initResult = await recreated.value.init(); if (!initResult.ok) { await recreated.value.dispose(); return { ok: false as const, error: initResult.error.message }; } return { ok: true as const, value: recreated.value }; }; const embedResult = await llm.createEmbeddingPort(modelUri, { egressCollections: options.collection ? [options.collection] : "all", policy, onProgress: downloadProgress ? (progress) => downloadProgress("embed", progress) : undefined, }); if (!embedResult.ok) { return { success: false, error: embedResult.error.message }; } embedPort = embedResult.value; // Clear download progress line if shown if (showDownloadProgress) { process.stderr.write("\n"); } const initializedPort = await embedPort.init(); if (!initializedPort.ok) return { success: false, error: initializedPort.error.message }; let dimensions = embedPort.dimensions(); if (!embedPort.getIdentity?.()) { const probeResult = await embedPort.embed("dimension probe"); if (!probeResult.ok) return { success: false, error: probeResult.error.message }; dimensions = probeResult.value.length; } // Create vector index port const vectorResult = await createVectorIndexPort(db, { model: modelUri, dimensions, }); if (!vectorResult.ok) { return { success: false, error: vectorResult.error.message }; } vectorIndex = vectorResult.value; const prepared = await prepareEmbeddingBacklog({ statsPort: stats, embedPort, vectorIndex, modelUri, collection: options.collection, batchSize, force, }); if (!prepared.ok) return { success: false, error: prepared.error.message }; if (prepared.value.variantStore) { totalToEmbed = countVariantBacklog(prepared.value); const startedAt = Date.now(); if (dryRun) return { success: true, embedded: totalToEmbed, errors: 0, contentionErrors: 0, duration: 0, model: modelUri, searchAvailable: prepared.value.variantStore.searchAvailable, errorSamples: [], }; const processed = await embedBacklog({ ...prepared.value, onProgress: !options.json ? (embedded, errors) => process.stderr.write( `\rEmbedded ${embedded}/${totalToEmbed} owners; ${errors} errors` ) : undefined, }); if (!options.json) process.stderr.write("\n"); if (!processed.ok) return { success: false, error: processed.error.message }; return { success: true, ...processed.value, contentionErrors: processed.value.contentionErrors ?? 0, duration: (Date.now() - startedAt) / 1000, model: modelUri, searchAvailable: prepared.value.variantStore.searchAvailable, errorSamples: [], }; } if (force) { const count = await getActiveChunkCount(db, options.collection); if (!count.ok) return { success: false, error: count.error.message }; totalToEmbed = count.value; if (dryRun || !totalToEmbed) return { success: true, embedded: totalToEmbed, errors: 0, contentionErrors: 0, duration: 0, model: modelUri, searchAvailable: vectorIndex.searchAvailable, errorSamples: [], }; } if (!force) { const embedFingerprint = getEmbeddingFingerprint({ modelUri, dimensions, }); const backlogResult = await stats.countBacklog( modelUri, embedFingerprint, { collection: options.collection, } ); if (!backlogResult.ok) { return { success: false, error: backlogResult.error.message }; } totalToEmbed = backlogResult.value; if (totalToEmbed === 0 || dryRun) { return { success: true, embedded: totalToEmbed, errors: 0, contentionErrors: 0, duration: 0, model: modelUri, searchAvailable: vectorIndex.searchAvailable, errorSamples: [], }; } } // Process batches const result = await processBatches({ db, stats, embedPort, vectorIndex, modelUri, collection: options.collection, batchSize, force, showProgress: !options.json, totalToEmbed, verbose: options.verbose ?? false, recreateEmbedPort, }); if (!result.ok) { return { success: false, error: result.error }; } // Sync vec index if any vec0 writes failed (matches embedBacklog behavior) if (vectorIndex.vecDirty) { const syncResult = await vectorIndex.syncVecIndex(); if (syncResult.ok) { const { added, removed } = syncResult.value; if (added > 0 || removed > 0) { if (!options.json) { process.stdout.write( `\n[vec] Synced index: +${added} -${removed}\n` ); } } vectorIndex.vecDirty = false; } else { if (!options.json) { process.stdout.write( `\n[vec] Sync failed: ${syncResult.error.message}\n` ); } return { success: true, embedded: result.embedded, errors: result.errors, contentionErrors: result.contentionErrors, duration: result.duration, model: modelUri, searchAvailable: vectorIndex.searchAvailable, errorSamples: [ ...result.errorSamples, syncResult.error.message, ].slice(0, 5), suggestion: "Vector index sync failed after embedding. Rerun `gno embed` once more. If it repeats, run `gno vec sync`.", syncError: syncResult.error.message, }; } } return { success: true, embedded: result.embedded, errors: result.errors, contentionErrors: result.contentionErrors, duration: result.duration, model: modelUri, searchAvailable: vectorIndex.searchAvailable, errorSamples: result.errorSamples, suggestion: result.suggestion, }; }; try { return await runEmbedBody(); } finally { // Assigned inside runEmbedBody; the cast defeats TS's null narrowing. await (embedPort as EmbeddingPort | null)?.dispose(); } }; // Persisted stage marker: `running` until the pass reports, so a process // that dies here is surfaced by the next run's resume preamble. markIndexStageRunning(db, "embed", { collection: options.collection }); let outcome: "completed" | "failed" = "failed"; try { const result = await runEmbedPass(); outcome = embedStageOutcome(result); return { ...result, resumedFrom }; } finally { markIndexStageFinished(db, "embed", outcome); await store.close(); } }); } // ───────────────────────────────────────────────────────────────────────────── // Helper: Get all active chunks (for --force mode) // ───────────────────────────────────────────────────────────────────────────── function getActiveChunkCount( db: Database, collection?: string ): Promise> { try { const collectionClause = collection ? " AND d.collection = ?" : ""; const result = db .prepare( ` SELECT COUNT(*) as count FROM content_chunks c WHERE EXISTS ( SELECT 1 FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1${collectionClause} ) ` ) .get(...(collection ? [collection] : [])) as { count: number }; return Promise.resolve(ok(result.count)); } catch (e) { return Promise.resolve( err( "QUERY_FAILED", `Failed to count chunks: ${e instanceof Error ? e.message : String(e)}` ) ); } } function getActiveChunks( db: Database, limit: number, after?: { mirrorHash: string; seq: number }, collection?: string ): Promise> { try { const collectionClause = collection ? " AND d.collection = ?" : ""; // Include title for contextual embedding const sql = after ? ` SELECT c.mirror_hash as mirrorHash, c.seq, c.text, (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 ORDER BY d.id LIMIT 1) as title, 'force' as reason FROM content_chunks c WHERE EXISTS ( SELECT 1 FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1${collectionClause} ) AND (c.mirror_hash > ? OR (c.mirror_hash = ? AND c.seq > ?)) ORDER BY c.mirror_hash, c.seq LIMIT ? ` : ` SELECT c.mirror_hash as mirrorHash, c.seq, c.text, (SELECT d.title FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1 ORDER BY d.id LIMIT 1) as title, 'force' as reason FROM content_chunks c WHERE EXISTS ( SELECT 1 FROM documents d WHERE d.mirror_hash = c.mirror_hash AND d.active = 1${collectionClause} ) ORDER BY c.mirror_hash, c.seq LIMIT ? `; const params = after ? [ ...(collection ? [collection] : []), after.mirrorHash, after.mirrorHash, after.seq, limit, ] : [...(collection ? [collection] : []), limit]; const results = db.prepare(sql).all(...params) as BacklogItem[]; return Promise.resolve(ok(results)); } catch (e) { return Promise.resolve( err( "QUERY_FAILED", `Failed to get chunks: ${e instanceof Error ? e.message : String(e)}` ) ); } } // ───────────────────────────────────────────────────────────────────────────── // Format // ───────────────────────────────────────────────────────────────────────────── /** * Format embed result for output. */ export function formatEmbed( result: EmbedResult, options: EmbedOptions ): string { if (!result.success) { return options.json ? JSON.stringify({ error: { code: "RUNTIME", message: result.error } }) : `Error: ${result.error}`; } if (options.json) { return JSON.stringify( { embedded: result.embedded, errors: result.errors, contentionErrors: result.contentionErrors, duration: result.duration, model: result.model, searchAvailable: result.searchAvailable, errorSamples: result.errorSamples ?? [], suggestion: result.suggestion, syncError: result.syncError, }, null, 2 ); } if (options.dryRun) { return `Dry run: would embed ${result.embedded.toLocaleString()} chunks with model ${result.model}`; } if ( result.embedded === 0 && result.errors === 0 && result.contentionErrors === 0 ) { return "No chunks need embedding. All up to date."; } const lines: string[] = []; lines.push( `Embedded ${result.embedded.toLocaleString()} chunks in ${formatDuration(result.duration)}` ); if (result.errors > 0) { lines.push(`${result.errors} chunks failed to embed.`); if ((result.errorSamples?.length ?? 0) > 0) { for (const sample of result.errorSamples ?? []) { lines.push(`Sample error: ${sample}`); } } if (result.suggestion) { lines.push(`Hint: ${result.suggestion}`); } } if (result.contentionErrors > 0) { lines.push( `${result.contentionErrors} chunks deferred by index contention (SQLITE_BUSY) — not embedding failures. Rerun \`gno embed\` when the other writer finishes.` ); } if (!result.searchAvailable) { lines.push( "Warning: sqlite-vec not available. Embeddings stored but KNN search disabled." ); } if (result.syncError) { lines.push(`Vec sync error: ${result.syncError}`); } return lines.join("\n"); }