import { createHandler } from './_shared/middleware' import { adminDb } from './_shared/db' import { create as adminCreate, update as adminUpdate } from './admin-data' import { generate } from './embeddings' import { resolveTypeId } from './_shared/resolve-ids' // ─── INTERFACES ─────────────────────────────────────────────────────────────── interface ParsedChunk { identifier: string chunk_id: string version: string hash: string macro: string micro: string inputs: Record outputs: string depends_on: string[] depended_by: string[] side_effects: string[] tags: string[] code: string metadata: { chunk_id: string file_path: string line_start: number line_end: number chunk_type: 'function' | 'class' | 'interface' | 'config' | 'object' | 'component' | 'hook' | 'route' purpose: string hash: string dependencies: string[] dependents: string[] } } interface RawChunk { name: string file_path: string line_start: number line_end: number chunk_type: 'function' | 'class' | 'interface' | 'config' | 'component' | 'hook' | 'route' | 'schema' code: string imports?: string[] app_name?: string } interface IngestionRequest { chunks: ParsedChunk[] | RawChunk[] force_update?: boolean app_name?: string } interface IngestionResponse { success: boolean items_created: number items_updated: number items_skipped: number embeddings_generated: number errors: string[] } // ─── ANNOTATED MODE ─────────────────────────────────────────────────────────── async function findExistingByChunkId(typeId: string, chunkId: string): Promise { const { data } = await adminDb .from('items') .select('id, data') .eq('type_id', typeId) .filter('data->>kb_type', 'eq', 'code_chunk') .filter('data->code_metadata->>chunk_id', 'eq', chunkId) .maybeSingle() return data } function buildAnnotatedDescriptionHtml(chunk: ParsedChunk, cleanCode: string): string { const sections: string[] = [] sections.push(`

Purpose
${chunk.macro}

`) if (chunk.micro && chunk.micro !== chunk.macro) { sections.push(`

${chunk.micro}

`) } if (chunk.inputs && Object.keys(chunk.inputs).length > 0) { const rows = Object.entries(chunk.inputs) .map(([name, desc]) => `${name} — ${desc}`) .join('
') sections.push(`

Parameters
${rows}

`) } if (chunk.outputs) { sections.push(`

Returns
${chunk.outputs}

`) } if (chunk.depends_on && chunk.depends_on.length > 0) { const depList = chunk.depends_on.map(d => `${d}`).join(', ') sections.push(`

Dependencies
${depList}

`) } if (chunk.side_effects && chunk.side_effects.length > 0) { sections.push(`

Side Effects
${chunk.side_effects.join('
')}

`) } sections.push( `

Source
${chunk.metadata.file_path} lines ${chunk.metadata.line_start}–${chunk.metadata.line_end}

` ) const escapedCode = cleanCode.trim() .replace(/&/g, '&') .replace(//g, '>') sections.push(`
${escapedCode}
`) return sections.join('\n') } async function handleIngestAnnotated(ctx: any, body: IngestionRequest): Promise { const response: IngestionResponse = { success: true, items_created: 0, items_updated: 0, items_skipped: 0, embeddings_generated: 0, errors: [] } const chunks = body.chunks as ParsedChunk[] const forceUpdate = body.force_update ?? false const typeId = await resolveTypeId('item', 'kb_article') for (const chunk of chunks) { try { if (!chunk.chunk_id || !chunk.code || !chunk.metadata) { response.errors.push( `Invalid chunk: missing chunk_id, code, or metadata for ${chunk.identifier || 'unknown'}` ) continue } const existing = await findExistingByChunkId(typeId, chunk.chunk_id) if (existing && !forceUpdate) { response.items_skipped++ continue } // Extract clean code (remove Spine docblock if present) let cleanCode = chunk.code const docblockMatch = chunk.code.match(/^\/\*\*[\s\S]*?\*\/\s*/) if (docblockMatch) { cleanCode = chunk.code.substring(docblockMatch[0].length) } const title = chunk.identifier .replace(/_/g, ' ') .replace(/\b\w/g, l => l.toUpperCase()) const descriptionHtml = buildAnnotatedDescriptionHtml(chunk, cleanCode) const itemData = { kb_type: 'code_chunk', intent: chunk.macro, cause: chunk.micro, procedure: cleanCode.trim(), error_signatures: chunk.side_effects, code_metadata: { chunk_id: chunk.chunk_id, identifier: chunk.identifier, version: chunk.version, hash: chunk.hash, file_path: chunk.metadata.file_path, line_start: chunk.metadata.line_start, line_end: chunk.metadata.line_end, chunk_type: chunk.metadata.chunk_type, depends_on: chunk.depends_on, depended_by: chunk.depended_by, inputs: chunk.inputs, outputs: chunk.outputs }, tags: chunk.tags, audience: ['developer', 'ai_system'], security_level: 'internal', priority: 'medium', category: 'technical', source_info: { source_type: 'automated_ingestion', author: 'code-ingestion-v1.0', ingestion_timestamp: new Date().toISOString(), original_source: chunk.metadata.file_path } } let itemId: string if (existing) { const ctxWithQuery = { ...ctx, query: { ...ctx.query, entity: 'items', id: existing.id } } await adminUpdate(ctxWithQuery, { title, status: 'published', description: descriptionHtml, data: itemData, is_active: true }) itemId = existing.id response.items_updated++ } else { const result: any = await adminCreate(ctx, { entity: 'items', type_id: typeId, title, status: 'published', description: descriptionHtml, data: itemData, is_active: true }) itemId = result?.id if (!itemId) throw new Error('Failed to create kb_article: no ID returned') response.items_created++ } // Use sanctioned generate — never write to embeddings table directly await generate(ctx, { item_id: itemId, force_regenerate: forceUpdate }) response.embeddings_generated++ } catch (error) { const msg = error instanceof Error ? error.message : String(error) console.error(`Code ingestion (annotated) error for ${(chunk as ParsedChunk).identifier}:`, error) response.errors.push(`${(chunk as ParsedChunk).identifier || 'unknown'}: ${msg}`) } } if ( response.errors.length > 0 && response.items_created === 0 && response.items_updated === 0 && response.items_skipped === 0 ) { response.success = false } return response } // ─── RAW MODE ───────────────────────────────────────────────────────────────── async function generateMacroDescription(code: string, fallback: string): Promise { const apiKey = process.env.OPENAI_API_KEY || process.env.LLM_API_KEY const baseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1' if (!apiKey) return fallback try { const res = await fetch(`${baseUrl}/chat/completions`, { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` }, body: JSON.stringify({ model: process.env.LLM_DEFAULT_MODEL || 'gpt-4o-mini', temperature: 0.2, max_tokens: 60, messages: [ { role: 'system', content: 'You are a code documentation expert. Given a code snippet, write one concise sentence (max 20 words) describing what this function/component does.' }, { role: 'user', content: code } ] }) }) if (!res.ok) return fallback const j: any = await res.json() return j.choices?.[0]?.message?.content?.trim() || fallback } catch { return fallback } } function deriveSyntheticChunkId(filePath: string, name: string): string { return `${filePath}::${name}`.replace(/[^a-zA-Z0-9_:./-]/g, '_') } async function findExistingRawChunk(typeId: string, syntheticChunkId: string): Promise { const { data } = await adminDb .from('items') .select('id, data') .eq('type_id', typeId) .filter('data->>kb_type', 'eq', 'code_chunk') .filter('data->code_metadata->>chunk_id', 'eq', syntheticChunkId) .maybeSingle() return data } async function handleIngestRaw(ctx: any, body: IngestionRequest): Promise { const response: IngestionResponse = { success: true, items_created: 0, items_updated: 0, items_skipped: 0, embeddings_generated: 0, errors: [] } const chunks = body.chunks as RawChunk[] const forceUpdate = body.force_update ?? false const appName = body.app_name || 'unknown' const typeId = await resolveTypeId('item', 'kb_article') for (const chunk of chunks) { try { if (!chunk.name || !chunk.code) { response.errors.push('Invalid raw chunk: missing name or code') continue } const syntheticChunkId = deriveSyntheticChunkId(chunk.file_path || '', chunk.name) const existing = await findExistingRawChunk(typeId, syntheticChunkId) if (existing && !forceUpdate) { response.items_skipped++ continue } // Use LLM to generate a one-sentence macro description; fallback to chunk.name const macro = await generateMacroDescription(chunk.code, chunk.name) const title = chunk.name .replace(/_/g, ' ') .replace(/\b\w/g, l => l.toUpperCase()) const escapedCode = chunk.code .trim() .replace(/&/g, '&') .replace(//g, '>') const descriptionHtml = [ `

Purpose
${macro}

`, `

Source
${chunk.file_path} lines ${chunk.line_start}–${chunk.line_end}

`, `
${escapedCode}
` ].join('\n') const itemData = { kb_type: 'code_chunk', intent: macro, cause: macro, procedure: chunk.code.trim(), chunk_type: chunk.chunk_type, code_metadata: { chunk_id: syntheticChunkId, identifier: chunk.name, file_path: chunk.file_path, line_start: chunk.line_start, line_end: chunk.line_end, chunk_type: chunk.chunk_type, app_name: chunk.app_name || appName, imports: chunk.imports || [] }, tags: [chunk.chunk_type, appName, 'raw-ingestion'].filter(Boolean), audience: ['developer', 'ai_system'], security_level: 'internal', priority: 'medium', category: 'technical', source_info: { source_type: 'automated_ingestion', author: 'raw-code-ingestion-v1.0', ingestion_timestamp: new Date().toISOString(), original_source: chunk.file_path } } let itemId: string if (existing) { const ctxWithQuery = { ...ctx, query: { ...ctx.query, entity: 'items', id: existing.id } } await adminUpdate(ctxWithQuery, { title, status: 'published', description: descriptionHtml, data: itemData, is_active: true }) itemId = existing.id response.items_updated++ } else { const result: any = await adminCreate(ctx, { entity: 'items', type_id: typeId, title, status: 'published', description: descriptionHtml, data: itemData, is_active: true }) itemId = result?.id if (!itemId) throw new Error('Failed to create kb_article: no ID returned') response.items_created++ } // Use sanctioned generate — never write to embeddings table directly await generate(ctx, { item_id: itemId }) response.embeddings_generated++ } catch (error) { const msg = error instanceof Error ? error.message : String(error) console.error(`Code ingestion (raw) error for ${(chunk as RawChunk).name}:`, error) response.errors.push(`${(chunk as RawChunk).name || 'unknown'}: ${msg}`) } } if ( response.errors.length > 0 && response.items_created === 0 && response.items_updated === 0 && response.items_skipped === 0 ) { response.success = false } return response } // ─── HANDLER EXPORT ─────────────────────────────────────────────────────────── export const handler = createHandler(async (ctx, body) => { const action = (ctx as any).query?.action || 'ingest_annotated' if (action === 'ingest_annotated') return await handleIngestAnnotated(ctx, body) if (action === 'ingest_raw') return await handleIngestRaw(ctx, body) throw new Error('Unknown action. Use ingest_annotated or ingest_raw.') })