/** * @module custom_cil-embeddings * @audience installer * @layer api-handler * @stability experimental * * Pass 3 of the CIL ingestion pipeline: generate embeddings for cil_chunk items. * * Vector types per cil_chunk: * - semantic : "chunk_name. macro. micro." — what this chunk IS and DOES * - structural : JSON snapshot of chunk_type, file_path, stability, is_dead_code * - contract : flat-text rendering of input_spec + output_spec + side_effects * * Uses the sanctioned generate() from core embeddings.ts via direct call. * Never writes to the embeddings table directly. * * Actions: * POST ?action=embed_chunk { chunk_id, force_regenerate? } — embed one chunk * POST ?action=embed_project { project_id, force_regenerate?, batch_size? } — embed all chunks in project * POST ?action=embed_approved { project_id, force_regenerate? } — embed only approved chunks * GET ?action=status { project_id } — count embedded vs total * GET ?action=search { q, project_id?, limit? } — semantic search over cil_chunk embeddings */ import { createHandler } from './_shared/middleware' import { adminDb } from './_shared/db' import { generate } from './embeddings' // ─── CONSTANTS ──────────────────────────────────────────────────────────────── const EMBEDDING_MODEL = 'text-embedding-3-small' const DEFAULT_BATCH_SIZE = 20 // chunk_index is used as a stable discriminator per vector type so that the // unique constraint (model_id, document_id, chunk_index) is satisfied when // all three vectors for the same chunk are inserted in a single batch. const VECTOR_TYPE_INDEX: Record = { semantic: 0, structural: 1, contract: 2, } // Embedding model pricing per 1M tokens (USD) — as of mid-2025 const EMBEDDING_PRICES: Record = { 'text-embedding-3-small': 0.02, 'text-embedding-3-large': 0.13, 'text-embedding-ada-002': 0.10, } // ─── CONTENT BUILDERS ───────────────────────────────────────────────────────── // These produce the text that gets embedded for each vector type. // They are intentionally separate from the core embeddings.ts builders so we // can tune them for code intelligence without touching the framework layer. function buildSemanticText(chunk: any): string { const d = chunk.data || {} const parts: string[] = [] if (d.chunk_name) parts.push(d.chunk_name) if (d.chunk_type) parts.push(`[${d.chunk_type}]`) if (d.file_path) parts.push(`in ${d.file_path}`) if (d.macro) parts.push(d.macro) if (d.micro) parts.push(d.micro) if (d.input_spec) parts.push(`Inputs: ${d.input_spec}`) if (d.output_spec) parts.push(`Returns: ${d.output_spec}`) return parts.join('. ') } function buildStructuralText(chunk: any): string { const d = chunk.data || {} return JSON.stringify({ chunk_name: d.chunk_name || '', chunk_type: d.chunk_type || '', file_path: d.file_path || '', stability: d.stability || '', is_dead_code: d.is_dead_code || false, boundary_confidence: d.boundary_confidence || '', doc_confidence: d.doc_confidence || '', review_status: d.review_status || '', }) } function buildContractText(chunk: any): string { const d = chunk.data || {} const parts: string[] = [] if (d.input_spec) parts.push(`INPUTS: ${d.input_spec}`) if (d.output_spec) parts.push(`OUTPUTS: ${d.output_spec}`) if (d.side_effects) parts.push(`SIDE_EFFECTS: ${d.side_effects}`) if (d.macro) parts.push(`PURPOSE: ${d.macro}`) // Include first ~400 chars of actual code content for code-aware search if (d.code_content) parts.push(`CODE:\n${String(d.code_content).slice(0, 400)}`) return parts.join('\n') || `${d.chunk_name || 'unknown'} — no contract yet` } // ─── EMBEDDING HELPER ───────────────────────────────────────────────────────── async function callEmbeddingApi(inputs: string[]): Promise { const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1' if (!apiKey || !inputs.length) throw new Error('OPENAI_API_KEY not configured') const res = await fetch(`${baseUrl}/embeddings`, { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` }, body: JSON.stringify({ model: EMBEDDING_MODEL, input: inputs }), }) if (!res.ok) throw new Error(`Embedding API error ${res.status}: ${await res.text()}`) const j: any = await res.json() return (j.data || []) .sort((a: any, b: any) => a.index - b.index) .map((d: any) => d.embedding) } function contentHash(text: string): string { // Simple FNV-1a for content deduplication (no crypto needed here) let h = 2166136261 for (let i = 0; i < text.length; i++) { h ^= text.charCodeAt(i) h = Math.imul(h, 16777619) } return (h >>> 0).toString(16).padStart(8, '0') } // ─── UPSERT ONE CHUNK'S EMBEDDINGS ─────────────────────────────────────────── async function embedChunk( chunkItem: any, accountId: string, forceRegenerate: boolean ): Promise<{ created: string[]; skipped: string[]; error?: string }> { const chunkId = chunkItem.id const created: string[] = [] const skipped: string[] = [] // Build texts for each vector type const vectors: { type: string; text: string }[] = [ { type: 'semantic', text: buildSemanticText(chunkItem) }, { type: 'structural', text: buildStructuralText(chunkItem) }, { type: 'contract', text: buildContractText(chunkItem) }, ].filter(v => v.text.trim().length > 0) if (!vectors.length) return { created, skipped: ['all:empty'] } // Load existing embeddings for this chunk const { data: existing } = await adminDb .from('embeddings') .select('id, content_hash, metadata') .eq('document_id', chunkId) const existingMap = new Map() for (const row of existing || []) { existingMap.set(row.metadata?.vector_type || '', row) } if (forceRegenerate && (existing || []).length > 0) { await adminDb.from('embeddings').delete().eq('document_id', chunkId) existingMap.clear() } // Filter to only changed vectors const toEmbed = vectors.filter(v => { const hash = contentHash(v.text) const row = existingMap.get(v.type) if (row && row.content_hash === hash) { skipped.push(v.type); return false } return true }) if (!toEmbed.length) return { created, skipped } try { const embeddings = await callEmbeddingApi(toEmbed.map(v => v.text)) const now = new Date().toISOString() for (let i = 0; i < toEmbed.length; i++) { const { type, text } = toEmbed[i] const hash = contentHash(text) const existingRow = existingMap.get(type) // Format as PostgreSQL vector literal const vectorLiteral = `[${embeddings[i].join(',')}]` const row = { account_id: accountId, model_id: EMBEDDING_MODEL, document_id: chunkId, chunk_index: VECTOR_TYPE_INDEX[type] ?? 0, content: text, embedding: vectorLiteral, content_hash: hash, approved_at: now, // CIL chunks are always "approved" — they're internal, not published content revoked_at: null, metadata: { vector_type: type, item_type: 'cil_chunk', kb_type: 'cil_chunk', security_level: 'internal', source_item_id: chunkId, chunk_name: chunkItem.data?.chunk_name || '', file_path: chunkItem.data?.file_path || '', project_id: chunkItem.data?.project_id || '', }, } if (existingRow) { await adminDb.from('embeddings').update(row).eq('id', existingRow.id) } else { await adminDb.from('embeddings').insert(row) } created.push(type) } } catch (err) { return { created, skipped, error: err instanceof Error ? err.message : String(err) } } return { created, skipped } } // ─── SEMANTIC SEARCH ────────────────────────────────────────────────────────── async function semanticSearch( query: string, projectId: string | null, limit: number, accountId: string ): Promise { const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1' if (!apiKey) throw new Error('OPENAI_API_KEY not configured for semantic search') // Embed the query const res = await fetch(`${baseUrl}/embeddings`, { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` }, body: JSON.stringify({ model: EMBEDDING_MODEL, input: [query] }), }) if (!res.ok) throw new Error(`Embedding API error ${res.status}`) const j: any = await res.json() const queryVector = j.data?.[0]?.embedding if (!queryVector) throw new Error('Failed to embed query') const vectorLiteral = `[${queryVector.join(',')}]` // Call match_embeddings RPC with cil_chunk filter const rpcParams: any = { query_embedding: vectorLiteral, match_count: limit, filter_account_ids: [accountId], filter_vector_type: 'semantic', similarity_threshold: 0.2, filter_item_types: ['cil_chunk'], } const { data: matches, error } = await adminDb.rpc('match_embeddings', rpcParams) if (error) throw new Error(`match_embeddings RPC error: ${error.message}`) // Enrich with chunk item data const chunkIds = [...new Set((matches || []).map((m: any) => m.document_id))] if (!chunkIds.length) return [] const { data: chunks } = await adminDb .from('items') .select('id, title, data') .in('id', chunkIds) const chunkMap = new Map((chunks || []).map((c: any) => [c.id, c])) return (matches || []) .filter((m: any) => { if (!projectId) return true const chunk = chunkMap.get(m.document_id) return chunk?.data?.project_id === projectId }) .map((m: any) => { const chunk = chunkMap.get(m.document_id) return { chunk_id: m.document_id, chunk_name: chunk?.data?.chunk_name || chunk?.title || '', file_path: chunk?.data?.file_path || '', macro: chunk?.data?.macro || '', micro: chunk?.data?.micro || '', chunk_type: chunk?.data?.chunk_type || '', doc_confidence: chunk?.data?.doc_confidence || '', similarity: m.similarity, content_snippet: (m.content || '').slice(0, 200), } }) .slice(0, limit) } // ─── RESOLVE ACCOUNT ID ─────────────────────────────────────────────────────── async function getPlatformAccountId(): Promise { const { data } = await adminDb.from('accounts').select('id').eq('slug', 'spine-system').maybeSingle() return data?.id || '' } // ─── HANDLER ────────────────────────────────────────────────────────────────── export const handler = createHandler(async (ctx, body) => { const action = ctx.query?.action const accountId = ctx.accountId || await getPlatformAccountId() // ── COST ESTIMATE — pre-flight token + USD cost projection ─────────────────── if (action === 'estimate') { const projectId = body?.project_id || ctx.query?.project_id if (!projectId) throw new Error('project_id is required') const model = body?.model || ctx.query?.model || EMBEDDING_MODEL const pricePerMillion = EMBEDDING_PRICES[model] ?? EMBEDDING_PRICES[EMBEDDING_MODEL] const filePath = body?.file_path || ctx.query?.file_path // Load all approved chunks (paginate to bypass Supabase 1000-row cap) const PAGE_SIZE = 1000 let offset = 0 const chunks: any[] = [] while (true) { let q = adminDb .from('items') .select('id, data') .filter('data->>kb_type', 'eq', 'cil_chunk') .filter('data->>project_id', 'eq', projectId) .filter('data->>review_status', 'eq', 'approved') .eq('is_active', true) .range(offset, offset + PAGE_SIZE - 1) if (filePath) q = q.filter('data->>file_path', 'eq', filePath) const { data: page, error } = await q if (error) throw new Error(`Failed to load chunks: ${error.message}`) chunks.push(...(page || [])) if ((page || []).length < PAGE_SIZE) break offset += PAGE_SIZE } // Load existing embeddings to identify what's already embedded const chunkIds = (chunks || []).map((c: any) => c.id) const existingDocIds = new Set() const SUPABASE_IN_LIMIT = 500 for (let i = 0; i < chunkIds.length; i += SUPABASE_IN_LIMIT) { const { data: emb } = await adminDb .from('embeddings') .select('document_id') .in('document_id', chunkIds.slice(i, i + SUPABASE_IN_LIMIT)) for (const row of emb || []) existingDocIds.add(row.document_id) } const toEmbed = (chunks || []).filter((c: any) => !existingDocIds.has(c.id)) const alreadyEmbedded = (chunks || []).length - toEmbed.length // Estimate tokens: each chunk produces 3 vectors (semantic, structural, contract) // Use stored token_count if available, otherwise estimate from code_content length let totalTokens = 0 for (const chunk of toEmbed) { const baseTokens = chunk.data?.token_count || Math.ceil((chunk.data?.code_content?.length || 200) / 4) totalTokens += baseTokens * 3 // 3 vector types per chunk } const estimatedCostUsd = (totalTokens / 1_000_000) * pricePerMillion return { project_id: projectId, model, file_path: filePath || null, price_per_1m_tokens: pricePerMillion, total_approved_chunks: (chunks || []).length, chunks_to_embed: toEmbed.length, already_embedded: alreadyEmbedded, vectors_per_chunk: 3, estimated_tokens: totalTokens, estimated_cost_usd: parseFloat(estimatedCostUsd.toFixed(4)), } } // ── EMBED ONE CHUNK ─────────────────────────────────────────────────────────── if (action === 'embed_chunk') { const chunkId = body?.chunk_id || ctx.query?.chunk_id if (!chunkId) throw new Error('chunk_id is required') const { data: chunk, error } = await adminDb .from('items') .select('id, title, data') .eq('id', chunkId) .maybeSingle() if (error || !chunk) throw new Error(`Chunk not found: ${chunkId}`) const result = await embedChunk(chunk, accountId, body?.force_regenerate ?? false) return { chunk_id: chunkId, chunk_name: chunk.data?.chunk_name || '', ...result, } } // ── EMBED ALL CHUNKS IN A PROJECT (paginated, parallel OpenAI calls) ─────────── // // Supports limit + offset so the UI can page through large projects: // POST { project_id, limit: 200, offset: 0 } // POST { project_id, limit: 200, offset: 200 } ... until has_more=false // // Also supports single-file targeting: // POST { project_id, file_path: '.framework/functions/_shared/principal.ts' } // // Within each page all OpenAI batches run in parallel (Promise.all). if (action === 'embed_project' || action === 'embed_approved') { const projectId = body?.project_id || ctx.query?.project_id if (!projectId) throw new Error('project_id is required') const forceRegenerate = body?.force_regenerate ?? false const filePath = body?.file_path || ctx.query?.file_path const pageLimit = Math.min(parseInt(body?.limit || '200', 10), 500) const pageOffset = parseInt(body?.offset || '0', 10) // 1. Get total count first (avoids Supabase's default 1000-row select limit) let countQuery = adminDb .from('items') .select('*', { count: 'exact', head: true }) .filter('data->>kb_type', 'eq', 'cil_chunk') .filter('data->>project_id', 'eq', projectId) .eq('is_active', true) if (action === 'embed_approved') { countQuery = countQuery.filter('data->>review_status', 'eq', 'approved') } if (filePath) { countQuery = countQuery.filter('data->>file_path', 'eq', filePath) } const { count: totalCount, error: countError } = await countQuery if (countError) throw new Error(`Failed to count chunks: ${countError.message}`) const totalChunks = totalCount || 0 // 2. Load just this page from the DB (pushed down to avoid 1000-row cap) let chunkQuery = adminDb .from('items') .select('id, title, data') .filter('data->>kb_type', 'eq', 'cil_chunk') .filter('data->>project_id', 'eq', projectId) .eq('is_active', true) .order('id') .range(pageOffset, pageOffset + pageLimit - 1) if (action === 'embed_approved') { chunkQuery = chunkQuery.filter('data->>review_status', 'eq', 'approved') } if (filePath) { chunkQuery = chunkQuery.filter('data->>file_path', 'eq', filePath) } const { data: pageChunks, error } = await chunkQuery if (error) throw new Error(`Failed to load chunks: ${error.message}`) if (!pageChunks || pageChunks.length === 0) { return { project_id: projectId, total: totalChunks, embeddings_created: 0, embeddings_skipped: 0, errors: [], has_more: false, next_offset: null, done_total: totalChunks } } // 2. Load existing embeddings for just this page's chunks const pageChunkIds = pageChunks.map((c: any) => c.id) const SUPABASE_IN_LIMIT = 500 const existingRows: any[] = [] for (let i = 0; i < pageChunkIds.length; i += SUPABASE_IN_LIMIT) { const { data } = await adminDb .from('embeddings') .select('id, document_id, content_hash, metadata') .in('document_id', pageChunkIds.slice(i, i + SUPABASE_IN_LIMIT)) existingRows.push(...(data || [])) } const existingMap = new Map>() for (const row of existingRows) { const docId = row.document_id if (!existingMap.has(docId)) existingMap.set(docId, new Map()) existingMap.get(docId)!.set(row.metadata?.vector_type || '', row) } // 3. Build (chunk, vectorType, text) triples that need embedding const toEmbed: { chunkId: string; type: string; text: string; existingRowId?: string }[] = [] let totalSkipped = 0 for (const chunk of pageChunks) { const vectors = [ { type: 'semantic', text: buildSemanticText(chunk) }, { type: 'structural', text: buildStructuralText(chunk) }, { type: 'contract', text: buildContractText(chunk) }, ].filter(v => v.text.trim().length > 0) for (const v of vectors) { const hash = contentHash(v.text) const existing = existingMap.get(chunk.id)?.get(v.type) if (!forceRegenerate && existing && existing.content_hash === hash) { totalSkipped++ continue } toEmbed.push({ chunkId: chunk.id, type: v.type, text: v.text, existingRowId: existing?.id }) } } if (toEmbed.length === 0) { const nextOffset = pageOffset + pageLimit return { project_id: projectId, total: totalChunks, embeddings_created: 0, embeddings_skipped: totalSkipped, errors: [], has_more: nextOffset < totalChunks, next_offset: nextOffset < totalChunks ? nextOffset : null, done_total: pageOffset + pageChunks.length, } } // 4. Delete stale embeddings if force_regenerate if (forceRegenerate && pageChunkIds.length > 0) { for (let i = 0; i < pageChunkIds.length; i += SUPABASE_IN_LIMIT) { await adminDb.from('embeddings').delete().in('document_id', pageChunkIds.slice(i, i + SUPABASE_IN_LIMIT)) } for (const item of toEmbed) { item.existingRowId = undefined } } // 5. Call OpenAI in PARALLEL batches of 100 texts each const OPENAI_BATCH = 100 const errors: string[] = [] // Build all batch promises and run concurrently const batchPromises: Promise<{ startIdx: number; vecs: number[][] }>[] = [] for (let i = 0; i < toEmbed.length; i += OPENAI_BATCH) { const startIdx = i const batchTexts = toEmbed.slice(i, i + OPENAI_BATCH).map(e => e.text) batchPromises.push( callEmbeddingApi(batchTexts) .then(vecs => ({ startIdx, vecs })) .catch(err => { errors.push(`OpenAI batch ${startIdx}-${startIdx + OPENAI_BATCH}: ${err instanceof Error ? err.message : String(err)}`) return { startIdx, vecs: batchTexts.map(() => []) } }) ) } const batchResults = await Promise.all(batchPromises) // Reassemble vectors in order const allVectors: number[][] = new Array(toEmbed.length) for (const { startIdx, vecs } of batchResults) { for (let k = 0; k < vecs.length; k++) allVectors[startIdx + k] = vecs[k] } // 6. Batch insert/update embeddings rows const now = new Date().toISOString() const chunkMap = new Map(pageChunks.map((c: any) => [c.id, c])) const rowsToInsert: object[] = [] const rowsToUpdate: { id: string; row: object }[] = [] for (let i = 0; i < toEmbed.length; i++) { const { chunkId, type, text, existingRowId } = toEmbed[i] const vec = allVectors[i] if (!vec || vec.length === 0) continue const chunk = chunkMap.get(chunkId) const row = { account_id: accountId, model_id: EMBEDDING_MODEL, document_id: chunkId, chunk_index: VECTOR_TYPE_INDEX[type] ?? 0, content: text, embedding: `[${vec.join(',')}]`, content_hash: contentHash(text), approved_at: now, revoked_at: null, metadata: { vector_type: type, item_type: 'cil_chunk', kb_type: 'cil_chunk', security_level: 'internal', source_item_id: chunkId, chunk_name: chunk?.data?.chunk_name || '', file_path: chunk?.data?.file_path || '', project_id: projectId, }, } if (existingRowId) rowsToUpdate.push({ id: existingRowId, row }) else rowsToInsert.push(row) } // Insert new rows in parallel batches const INSERT_BATCH = 200 const insertPromises: Promise[] = [] for (let i = 0; i < rowsToInsert.length; i += INSERT_BATCH) { insertPromises.push( adminDb.from('embeddings').insert(rowsToInsert.slice(i, i + INSERT_BATCH)) .then(({ error: e }) => { if (e) errors.push(`embedding insert batch failed: ${e.message}`) }) ) } // Update existing rows in parallel const PARALLEL_UPDATE = 20 const updatePromises: Promise[] = [] for (let i = 0; i < rowsToUpdate.length; i += PARALLEL_UPDATE) { for (const { id, row } of rowsToUpdate.slice(i, i + PARALLEL_UPDATE)) { updatePromises.push( adminDb.from('embeddings').update(row).eq('id', id) .then(({ error: e }) => { if (e) errors.push(`embedding update ${id}: ${e.message}`) }) ) } } await Promise.all([...insertPromises, ...updatePromises]) const nextOffset = pageOffset + pageLimit return { project_id: projectId, file_path: filePath || null, total: totalChunks, embeddings_created: rowsToInsert.length + rowsToUpdate.length, embeddings_skipped: totalSkipped, errors, has_more: nextOffset < totalChunks, next_offset: nextOffset < totalChunks ? nextOffset : null, done_total: pageOffset + pageChunks.length, } } // ── STATUS — embedding coverage for a project ───────────────────────────────── if (action === 'status') { const projectId = body?.project_id || ctx.query?.project_id if (!projectId) throw new Error('project_id is required') const { count: totalChunks } = await adminDb .from('items') .select('id', { count: 'exact' }) .filter('data->>kb_type', 'eq', 'cil_chunk') .filter('data->>project_id', 'eq', projectId) .eq('is_active', true) // Count chunks that have at least one embedding const { data: chunkItems } = await adminDb .from('items') .select('id') .filter('data->>kb_type', 'eq', 'cil_chunk') .filter('data->>project_id', 'eq', projectId) .eq('is_active', true) const chunkIds = (chunkItems || []).map((c: any) => c.id) let embeddedCount = 0 if (chunkIds.length > 0) { const { data: embeddedDocs } = await adminDb .from('embeddings') .select('document_id') .in('document_id', chunkIds) .is('revoked_at', null) embeddedCount = new Set((embeddedDocs || []).map((e: any) => e.document_id)).size } return { project_id: projectId, total_chunks: totalChunks || 0, embedded_chunks: embeddedCount, unembedded_chunks: (totalChunks || 0) - embeddedCount, coverage_pct: totalChunks ? Math.round((embeddedCount / totalChunks) * 100) : 0, } } // ── SEMANTIC SEARCH ─────────────────────────────────────────────────────────── if (action === 'search') { const query = body?.q || ctx.query?.q const projectId = body?.project_id || ctx.query?.project_id || null const limit = parseInt(body?.limit || ctx.query?.limit || '10') if (!query) throw new Error('q is required') const results = await semanticSearch(query, projectId, limit, accountId) return { query, results, count: results.length } } // ── DELETE — remove embeddings for a chunk ──────────────────────────────────── if (action === 'delete') { const chunkId = body?.chunk_id || ctx.query?.chunk_id if (!chunkId) throw new Error('chunk_id is required') const { error } = await adminDb .from('embeddings') .delete() .eq('document_id', chunkId) if (error) throw new Error(`Failed to delete embeddings: ${error.message}`) return { chunk_id: chunkId, deleted: true } } throw new Error(`Unknown action: ${action}. Valid: embed_chunk, embed_project, embed_approved, status, search, delete`) })