import { useState } from 'react' import { apiFetch } from '@core/lib/api' import { Button } from '@core/components/ui/button' import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@core/components/ui/card' import { Progress } from '@core/components/ui/progress' import { Alert, AlertDescription } from '@core/components/ui/alert' import { Badge } from '@core/components/ui/badge' import { ScrollArea } from '@core/components/ui/scroll-area' import { Tabs, TabsList, TabsTrigger, TabsContent } from '@core/components/ui/tabs' interface IngestionResponse { success: boolean items_created: number items_updated: number embeddings_generated: number errors: string[] skipped: string[] item_ids?: string[] } interface IngestionStats { total: number processed: number created: number updated: number errors: number } type Step = 'idle' | 'chunks_loaded' | 'ingestion_complete' | 'embeddings_complete' // --- Paste & Ingest sequential ingestion types --- type IngestStatus = 'queued' | 'ingesting' | 'done' | 'failed' interface ChunkWithStatus { name: string file_path: string line_start: number line_end: number chunk_type: string code: string status: IngestStatus error?: string } export default function KBIngestionPage() { const [currentStep, setCurrentStep] = useState('idle') const [isLoading, setIsLoading] = useState(false) const [chunks, setChunks] = useState([]) const [stats, setStats] = useState(null) const [error, setError] = useState(null) const [logs, setLogs] = useState([]) const [ingestFrom, setIngestFrom] = useState('') const [ingestTo, setIngestTo] = useState('') const [ingestedItemIds, setIngestedItemIds] = useState([]) const addLog = (message: string) => { const timestamp = new Date().toLocaleTimeString() setLogs(prev => [...prev, `[${timestamp}] ${message}`]) } const loadChunks = async (): Promise => { try { const response = await apiFetch('/.netlify/functions/custom_cortex-chunks', { method: 'GET', headers: { 'Content-Type': 'application/json', } }) if (!response.ok) { const errorData = await response.json().catch(() => ({ error: 'Unknown error' })) throw new Error(`Failed to load chunks: ${errorData.error || response.statusText}`) } const data = await response.json() return data.data?.chunks || [] } catch (err) { throw new Error(`Could not load chunks: ${err instanceof Error ? err.message : 'Unknown error'}`) } } const ingestChunks = async (chunks: any[]): Promise => { const response = await apiFetch('/api/custom_kb-ingestion?action=ingest', { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ chunks, force_update: false }) }) if (!response.ok) { const errorData = await response.json().catch(() => ({ error: 'Unknown error' })) throw new Error(`Ingestion failed: ${errorData.error || response.statusText}`) } const json = await response.json() return json.data ?? json } const generateEmbeddings = async (itemIds: string[]): Promise => { const response = await apiFetch('/api/custom_kb-embeddings?action=generate', { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ item_ids: itemIds, vector_types: ['semantic', 'structure'], force_regenerate: false }) }) if (!response.ok) { const errorData = await response.json().catch(() => ({ error: 'Unknown error' })) throw new Error(`Embedding generation failed: ${errorData.error || response.statusText}`) } const json = await response.json() return json.data ?? json } const handleLoadChunks = async () => { setIsLoading(true) setError(null) setLogs([]) try { addLog('Loading chunks from file...') const loadedChunks = await loadChunks() addLog(`Loaded ${loadedChunks.length} chunks`) setChunks(loadedChunks) setCurrentStep('chunks_loaded') } catch (err) { const errorMessage = err instanceof Error ? err.message : 'Unknown error' setError(errorMessage) addLog(`Error: ${errorMessage}`) } finally { setIsLoading(false) } } const handleIngestChunks = async () => { setIsLoading(true) setError(null) setLogs([]) try { // Parse range inputs const from = parseInt(ingestFrom) || 1 const to = parseInt(ingestTo) || chunks.length // Validate range if (from < 1 || to < 1 || from > to || from > chunks.length || to > chunks.length) { throw new Error(`Invalid range: Please enter values between 1 and ${chunks.length}, with from ≤ to`) } // Get the subset of chunks (convert to 0-based index) const selectedChunks = chunks.slice(from - 1, to) const recordCount = selectedChunks.length addLog(`Starting ingestion of records ${from} through ${to} (${recordCount} records)...`) const response = await ingestChunks(selectedChunks) const newStats: IngestionStats = { total: recordCount, processed: recordCount, created: response.items_created || 0, updated: response.items_updated || 0, errors: response.errors?.length || 0 } setStats(newStats) setIngestedItemIds(response.item_ids || []) addLog(`Ingestion complete: ${response.items_created || 0} created, ${response.items_updated || 0} updated`) addLog(`Captured ${(response.item_ids || []).length} item IDs for embedding generation`) if (response.errors && response.errors.length > 0) { addLog(`Errors: ${response.errors.join(', ')}`) } setCurrentStep('ingestion_complete') } catch (err) { const errorMessage = err instanceof Error ? err.message : 'Unknown error' setError(errorMessage) addLog(`Error: ${errorMessage}`) } finally { setIsLoading(false) } } const handleGenerateEmbeddings = async () => { setIsLoading(true) setError(null) setLogs([]) try { addLog('Generating embeddings for ingested items...') const itemIds = ingestedItemIds if (itemIds.length === 0) { throw new Error('No item IDs available — run ingestion first') } addLog(`Generating embeddings for ${itemIds.length} items...`) const response = await generateEmbeddings(itemIds) addLog(`Embeddings complete: ${(response as any).embeddings_created || response.embeddings_generated || 0} generated`) setCurrentStep('embeddings_complete') } catch (err) { const errorMessage = err instanceof Error ? err.message : 'Unknown error' setError(errorMessage) addLog(`Error: ${errorMessage}`) } finally { setIsLoading(false) } } return (
Spine Code Chunks Paste & Ingest App Code & Schema KB Code Chunk Ingestion Step-by-step ingestion of parsed code chunks from v2-core functions into the Knowledge Base system. {/* Step 1: Load Chunks */} Step 1: Load Chunk Data Load parsed code chunks from the chunks.json file {currentStep !== 'idle' && (
Chunks Loaded: {chunks.length}
)}
{/* Step 2: Display Chunks */} {currentStep !== 'idle' && chunks.length > 0 && ( Chunk Data Preview {chunks.length} chunks loaded from file
{chunks.slice(0, 10).map((chunk, index) => (
{chunk.identifier}
{chunk.macro}: {chunk.micro}
{chunk.chunk_id} • {chunk.version}
))} {chunks.length > 10 && (
... and {chunks.length - 10} more chunks
)}
)} {/* Step 3: Ingest Chunks */} {currentStep === 'chunks_loaded' && ( Step 2: Ingest Chunks Create KB articles from the loaded chunks
setIngestFrom(e.target.value)} placeholder="1" className="w-20 px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500" /> through setIngestTo(e.target.value)} placeholder={chunks.length.toString()} className="w-20 px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500" />
({(() => { const from = parseInt(ingestFrom) || 1 const to = parseInt(ingestTo) || chunks.length const count = from > 0 && to > 0 && from <= to && to <= chunks.length ? to - from + 1 : 0 return count > 0 ? `${count} records` : 'Invalid range' })()})
{stats && (
Total: {stats.total} Created: {stats.created} Updated: {stats.updated} {stats.errors > 0 && ( Errors: {stats.errors} )}
)}
)} {/* Step 4: Generate Embeddings */} {currentStep === 'ingestion_complete' && stats && stats.created > 0 && ( Step 3: Generate Embeddings Generate vector embeddings for the {stats.created} created KB articles )} {/* Error Display */} {error && ( {error} )} {/* Activity Log */} {logs.length > 0 && ( Activity Log
                      {logs.join('\n')}
                    
)}
) } // --- Paste & Ingest Tab --- type ChunkType = 'function' | 'class' | 'interface' | 'config' | 'component' | 'hook' | 'route' | 'schema' interface FileEntry { id: string filePath: string content: string } interface ParsedRawChunk { name: string file_path: string line_start: number line_end: number chunk_type: ChunkType code: string } function detectExtension(filePath: string): string { return filePath.split('.').pop()?.toLowerCase() || '' } function parseFileIntoChunks(filePath: string, content: string): ParsedRawChunk[] { const ext = detectExtension(filePath) const lines = content.split('\n') // JSON — treat as single chunk if (ext === 'json') { return [{ name: filePath.split('/').pop() || filePath, file_path: filePath, line_start: 1, line_end: lines.length, chunk_type: 'config', code: content, }] } // Markdown — split on ## headings if (ext === 'md' || ext === 'mdx') { const chunks: ParsedRawChunk[] = [] let currentLines: string[] = [] let currentName = filePath.split('/').pop() || filePath let startLine = 1 lines.forEach((line, i) => { if (line.startsWith('## ') && currentLines.length > 0) { chunks.push({ name: currentName, file_path: filePath, line_start: startLine, line_end: i, chunk_type: 'config', code: currentLines.join('\n'), }) currentName = line.replace(/^#+\s*/, '').trim() startLine = i + 1 currentLines = [line] } else { currentLines.push(line) } }) if (currentLines.length > 0) { chunks.push({ name: currentName, file_path: filePath, line_start: startLine, line_end: lines.length, chunk_type: 'config', code: currentLines.join('\n'), }) } return chunks } // TS / TSX / JS / JSX — split on CHUNK_START markers if present, otherwise split on export/function/class declarations if (['ts', 'tsx', 'js', 'jsx'].includes(ext)) { // Try chunk markers first const markerChunks = parseByChunkMarkers(filePath, lines) if (markerChunks.length > 0) return markerChunks // Fallback: split on top-level export/function/class declarations return parseByDeclarations(filePath, lines) } // Fallback — single chunk for anything else return [{ name: filePath.split('/').pop() || filePath, file_path: filePath, line_start: 1, line_end: lines.length, chunk_type: 'config', code: content, }] } function parseByChunkMarkers(filePath: string, lines: string[]): ParsedRawChunk[] { const chunks: ParsedRawChunk[] = [] let inChunk = false let chunkName = '' let startLine = 0 let chunkLines: string[] = [] lines.forEach((line, i) => { const startMatch = line.match(/CHUNK_START:\s*([A-Z0-9_]+)/) const endMatch = line.match(/CHUNK_END:/) if (startMatch) { inChunk = true chunkName = startMatch[1].toLowerCase().replace(/_/g, ' ') startLine = i + 1 chunkLines = [] } else if (endMatch && inChunk) { chunks.push({ name: chunkName, file_path: filePath, line_start: startLine, line_end: i, chunk_type: guessChunkType(chunkLines.join('\n')), code: chunkLines.join('\n'), }) inChunk = false chunkLines = [] } else if (inChunk) { chunkLines.push(line) } }) return chunks } function parseByDeclarations(filePath: string, lines: string[]): ParsedRawChunk[] { const chunks: ParsedRawChunk[] = [] const declarationRe = /^export\s+(async\s+)?(function|class|const|interface|type)\s+(\w+)/ let currentName = '' let startLine = 0 let currentLines: string[] = [] const flush = (endLine: number) => { if (currentName && currentLines.length > 0) { chunks.push({ name: currentName, file_path: filePath, line_start: startLine, line_end: endLine, chunk_type: guessChunkType(currentLines.join('\n')), code: currentLines.join('\n'), }) } } lines.forEach((line, i) => { const m = line.match(declarationRe) if (m) { flush(i) currentName = m[3] startLine = i + 1 currentLines = [line] } else if (currentName) { currentLines.push(line) } }) flush(lines.length) // If nothing matched, return the whole file as one chunk if (chunks.length === 0) { return [{ name: filePath.split('/').pop() || filePath, file_path: filePath, line_start: 1, line_end: lines.length, chunk_type: 'config', code: lines.join('\n'), }] } return chunks } function guessChunkType(code: string): ChunkType { if (/^export\s+(default\s+)?function\s+[A-Z]/.test(code) || /\breturn\s+\(/.test(code)) return 'component' if (/^export\s+(async\s+)?function\s+use[A-Z]/.test(code)) return 'hook' if (/\bclass\b/.test(code)) return 'class' if (/^export\s+interface\b/.test(code)) return 'interface' if (/Route|router|app\.(get|post|patch|delete)/.test(code)) return 'route' return 'function' } function PasteIngestTab() { const [files, setFiles] = useState([{ id: crypto.randomUUID(), filePath: '', content: '' }]) const [appName, setAppName] = useState('spine-framework') const [preview, setPreview] = useState(null) const [isLoading, setIsLoading] = useState(false) const [result, setResult] = useState<{ success: boolean; message: string } | null>(null) // Sequential ingestion state const [chunksWithStatus, setChunksWithStatus] = useState([]) const [isIngestingAll, setIsIngestingAll] = useState(false) const [currentChunkIndex, setCurrentChunkIndex] = useState(null) const [cancelRequested, setCancelRequested] = useState(false) const addFile = () => setFiles(f => [...f, { id: crypto.randomUUID(), filePath: '', content: '' }]) const removeFile = (id: string) => setFiles(f => f.filter(e => e.id !== id)) const updateFile = (id: string, field: keyof Omit, value: string) => setFiles(f => f.map(e => e.id === id ? { ...e, [field]: value } : e)) const handlePreview = () => { const allChunks: ParsedRawChunk[] = [] for (const file of files) { if (!file.filePath.trim() || !file.content.trim()) continue allChunks.push(...parseFileIntoChunks(file.filePath.trim(), file.content.trim())) } setPreview(allChunks) // Initialize chunks with status for sequential ingestion const chunksWithStatus: ChunkWithStatus[] = allChunks.map(chunk => ({ ...chunk, status: 'queued' as IngestStatus })) setChunksWithStatus(chunksWithStatus) setResult(null) } const ingestSingleChunk = async (chunk: ChunkWithStatus): Promise<{ success: boolean; error?: string }> => { try { const response = await apiFetch('/api/custom_code-ingestion?action=ingest_raw', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ chunks: [chunk], app_name: appName }), // Add timeout to avoid hanging signal: AbortSignal.timeout(30000) // 30 seconds }) if (!response.ok) { const errData = await response.json().catch(() => ({ error: response.statusText })) return { success: false, error: errData.error || response.statusText } } return { success: true } } catch (err) { if (err instanceof Error) { if (err.name === 'AbortError') { return { success: false, error: 'Request timed out' } } return { success: false, error: err.message } } return { success: false, error: 'Unknown error' } } } const retryChunk = async (index: number) => { const chunk = chunksWithStatus[index] if (!chunk) return setChunksWithStatus(prev => prev.map((c, idx) => idx === index ? { ...c, status: 'ingesting', error: undefined } : c )) const result = await ingestSingleChunk(chunk) setChunksWithStatus(prev => prev.map((c, idx) => idx === index ? { ...c, status: result.success ? 'done' : 'failed', error: result.error } : c )) } const handleIngest = async () => { if (!chunksWithStatus || chunksWithStatus.length === 0) return setIsLoading(true) setIsIngestingAll(true) setCancelRequested(false) setResult(null) let created = 0 let updated = 0 let skipped = 0 let errors: string[] = [] for (let i = 0; i < chunksWithStatus.length; i++) { if (cancelRequested) break const chunk = chunksWithStatus[i] if (chunk.status === 'done') continue setCurrentChunkIndex(i) setChunksWithStatus(prev => prev.map((c, idx) => idx === i ? { ...c, status: 'ingesting', error: undefined } : c )) const result = await ingestSingleChunk(chunk) if (result.success) { setChunksWithStatus(prev => prev.map((c, idx) => idx === i ? { ...c, status: 'done' } : c )) created += 1 // Simplified: assume all are creates for now } else { setChunksWithStatus(prev => prev.map((c, idx) => idx === i ? { ...c, status: 'failed', error: result.error } : c )) errors.push(`${chunk.name}: ${result.error}`) } // Small delay between chunks to avoid rate limiting await new Promise(r => setTimeout(r, 500)) } setCurrentChunkIndex(null) setIsIngestingAll(false) const totalProcessed = chunksWithStatus.filter(c => c.status === 'done').length setResult({ success: errors.length === 0, message: `Done. ${created} created, ${updated} updated, ${skipped} skipped. ${totalProcessed}/${chunksWithStatus.length} chunks processed.${errors.length ? ' Errors: ' + errors.join('; ') : ''}`, }) setIsLoading(false) } const handleCancel = () => { setCancelRequested(true) setIsIngestingAll(false) setCurrentChunkIndex(null) setIsLoading(false) } return ( Paste & Ingest Paste source files directly. Supports .ts, .tsx, .js, .jsx, .json, .md — split automatically into chunks and ingested as KB code articles. {/* App name */}
setAppName(e.target.value)} placeholder="e.g. spine-framework" className="flex-1 px-3 py-1.5 text-sm border border-border rounded-md bg-background focus:outline-none focus:ring-2 focus:ring-primary" />
{/* File entries */}
{files.map((file, idx) => (
File {idx + 1} updateFile(file.id, 'filePath', e.target.value)} placeholder=".framework/functions/admin-data.ts" className="flex-1 px-3 py-1.5 text-sm border border-border rounded-md bg-background font-mono focus:outline-none focus:ring-2 focus:ring-primary" /> {files.length > 1 && ( )}