import { useEffect, useState } from 'react' import { useParams } from 'react-router-dom' import { apiFetch } from '@core/lib/api' import { useAppPath } from '@core/hooks/useAppPath' import { Card, CardContent, CardHeader, CardTitle } from '@core/components/ui/card' import { Button } from '@core/components/ui/button' import { Badge } from '@core/components/ui/badge' import { Skeleton } from '@core/components/ui/skeleton' import { Input } from '@core/components/ui/input' import { Label } from '@core/components/ui/label' import { Switch } from '@core/components/ui/switch' import { Play, CheckCircle2, XCircle, AlertTriangle, Clock, FileCode2, Code2, GitPullRequest, ChevronRight, RefreshCw, Zap, Sparkles, GitGraph, Settings2, Save, DollarSign, } from 'lucide-react' interface Project { id: string title: string data: { repo_path: string ingestion_status: string file_count: number chunk_count: number total_ingestion_cost_usd: number last_ingested_at: string | null globs?: string[] exclude_globs?: string[] } stats?: { file_count: number chunk_count: number pending_review_count: number } } interface ChunkProposal { id: string title: string data: { file_path: string proposed_name: string line_start: number line_end: number chunk_type: string confidence_score: number review_status: string code_preview: string confidence_reasons: string existing_marker_match: boolean existing_marker_name: string } } interface PassConfig { repo_path: string globs: string exclude_globs: string force_reprocess: boolean } const PASS_STEPS = [ { key: 'pass1', label: 'Pass 1 — Discover Files', icon: FileCode2, description: 'Walk the repo, register cil_file items, compute hashes' }, { key: 'pass2', label: 'Pass 2 — AST Chunk', icon: Code2, description: 'Parse files with AST, propose chunk boundaries, write high-confidence markers' }, { key: 'review', label: 'Review Queue', icon: GitPullRequest, description: 'Review low-confidence boundary proposals before continuing' }, { key: 'pass3', label: 'Pass 3 — AI Document', icon: Sparkles, description: 'Use LLM to fill macro / micro / contract fields — do this before embedding so vectors are rich' }, { key: 'pass4', label: 'Pass 4 — Embed Chunks', icon: Zap, description: 'Generate semantic + structural + contract vector embeddings using the documented fields' }, { key: 'pass4b', label: 'Pass 5 — Call Map', icon: GitGraph, description: 'Extract import + call edges, write cil_imports, cil_calls, cil_tested_by links' }, { key: 'pass5', label: 'Pass 6 — Config Extract', icon: Settings2, description: 'Scan config files, create cil_config items, write cil_configures links' }, ] function ProposalRow({ proposal, selected, onToggle }: { proposal: ChunkProposal selected: boolean onToggle: (id: string) => void }) { const d = proposal.data return (
onToggle(proposal.id)} > {/* Checkbox */}
{selected && }
{/* Content */}
{d.proposed_name || proposal.title} {d.chunk_type || 'unknown'} {Math.round((d.confidence_score || 0) * 100)}% confidence {d.existing_marker_match && ( matches marker )}

{d.file_path} · L{d.line_start}–{d.line_end}

{d.confidence_reasons && (

{d.confidence_reasons}

)} {d.code_preview && (
            {d.code_preview}
          
)}
) } export default function CILIngestionPage() { const { id: projectId } = useParams<{ id: string }>() const appPath = useAppPath() const [project, setProject] = useState(null) const [proposals, setProposals] = useState([]) const [selectedProposalIds, setSelectedProposalIds] = useState>(new Set()) const [bulkProcessing, setBulkProcessing] = useState(false) const [loadingProject, setLoadingProject] = useState(true) const [loadingProposals, setLoadingProposals] = useState(true) const [runningPass, setRunningPass] = useState(null) const [passResults, setPassResults] = useState>({}) const [passProgress, setPassProgress] = useState>({}) const [passConfig, setPassConfig] = useState({ repo_path: '', globs: '**/*.ts,**/*.tsx,**/*.sql', exclude_globs: 'node_modules/**,.git/**,dist/**,build/**,.netlify/**,**/*.min.js,**/*.d.ts', force_reprocess: false, }) const [ingestMode, setIngestMode] = useState<'batch' | 'file'>('batch') const [targetFilePath, setTargetFilePath] = useState('') const [savingConfig, setSavingConfig] = useState(false) const [configSaved, setConfigSaved] = useState(false) const [costEstimate, setCostEstimate] = useState(null) const [loadingEstimate, setLoadingEstimate] = useState(false) const [embeddingEstimate, setEmbeddingEstimate] = useState(null) const [loadingEmbeddingEstimate, setLoadingEmbeddingEstimate] = useState(false) useEffect(() => { if (!projectId) return apiFetch(`/api/custom_cil-project?action=get&id=${projectId}`) .then(r => r.json()) .then(data => { const p = data?.project || data?.data?.project || null setProject(p) // Populate pass config from persisted project settings (always, using defaults if not yet saved) const DEFAULT_GLOBS = '**/*.ts,**/*.tsx,**/*.sql' const DEFAULT_EXCLUDES = 'node_modules/**,.git/**,dist/**,build/**,.netlify/**,**/*.min.js,**/*.d.ts' setPassConfig(c => ({ ...c, repo_path: p?.data?.repo_path || '', globs: p?.data?.globs?.length ? p.data.globs.join(',') : DEFAULT_GLOBS, exclude_globs: p?.data?.exclude_globs?.length ? p.data.exclude_globs.join(',') : DEFAULT_EXCLUDES, })) }) .catch(() => {}) .finally(() => setLoadingProject(false)) }, [projectId]) const saveConfig = async () => { if (!projectId) return setSavingConfig(true) setConfigSaved(false) try { await apiFetch(`/api/custom_cil-project?action=update&id=${projectId}`, { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ id: projectId, // Send only the fields we own — server merges into existing data data: { repo_path: passConfig.repo_path.trim(), globs: passConfig.globs.split(',').map(s => s.trim()).filter(Boolean), exclude_globs: passConfig.exclude_globs.split(',').map(s => s.trim()).filter(Boolean), }, }), }) setConfigSaved(true) setTimeout(() => setConfigSaved(false), 2000) // Refresh project state so UI reflects merged record apiFetch(`/api/custom_cil-project?action=get&id=${projectId}`) .then(r => r.json()) .then(data => { const p = data?.project || data?.data?.project || null; if (p) setProject(p) }) .catch(() => {}) } catch { // silent — user can retry } finally { setSavingConfig(false) } } const fetchProposals = () => { if (!projectId) return setLoadingProposals(true) apiFetch(`/api/custom_cil-project?action=list_proposals&id=${projectId}&status=pending_review`) .then(r => r.json()) .then(res => { // Framework wraps response in { data: { proposals: [...] } } setProposals(res?.data?.proposals || res?.proposals || []) }) .catch(() => {}) .finally(() => setLoadingProposals(false)) } useEffect(() => { fetchProposals() }, [projectId]) const fetchCostEstimate = () => { if (!projectId) return setLoadingEstimate(true) const fileParam = (ingestMode === 'file' && targetFilePath) ? `&file_path=${encodeURIComponent(targetFilePath)}` : '' apiFetch(`/api/custom_cil-doc?action=estimate&project_id=${projectId}${fileParam}`) .then(r => r.json()) .then(res => setCostEstimate(res?.data || res || null)) .catch(() => setCostEstimate(null)) .finally(() => setLoadingEstimate(false)) } const fetchEmbeddingEstimate = () => { if (!projectId) return setLoadingEmbeddingEstimate(true) const fileParam = (ingestMode === 'file' && targetFilePath) ? `&file_path=${encodeURIComponent(targetFilePath)}` : '' apiFetch(`/api/custom_cil-embeddings?action=estimate&project_id=${projectId}${fileParam}`) .then(r => r.json()) .then(res => setEmbeddingEstimate(res?.data || res || null)) .catch(() => setEmbeddingEstimate(null)) .finally(() => setLoadingEmbeddingEstimate(false)) } useEffect(() => { fetchCostEstimate() fetchEmbeddingEstimate() }, [projectId, ingestMode, targetFilePath]) const runPass = async (pass: 'pass1' | 'pass2' | 'pass3' | 'pass4' | 'pass4b' | 'pass5') => { if (!projectId) return setRunningPass(pass) try { let res: Response if (pass === 'pass3') { // Paginated loop — 50 chunks per request, ~20 in parallel each, well within 30s const PAGE = 50 let offset = 0 let totalDocumented = 0, totalSkipped = 0, totalFailed = 0 const allErrors: string[] = [] setPassProgress(p => ({ ...p, pass3: { done: 0, total: 0 } })) while (true) { const pageRes = await apiFetch('/api/custom_cil-doc?action=document', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ project_id: projectId, limit: PAGE, offset, file_path: targetFilePath || undefined }), }) const pageData = await pageRes.json() const d = pageData?.data || pageData totalDocumented += d.documented || 0 totalSkipped += d.skipped || 0 totalFailed += d.failed || 0 if (d.errors?.length) allErrors.push(...d.errors) setPassProgress(p => ({ ...p, pass3: { done: d.done_total || offset + PAGE, total: d.total_chunks || d.total_to_document || 0 } })) if (!d.has_more) break offset = d.next_offset } setPassResults(prev => ({ ...prev, pass3: { documented: totalDocumented, skipped: totalSkipped, failed: totalFailed, errors: allErrors } })) setPassProgress(p => ({ ...p, pass3: null })) fetchCostEstimate() setRunningPass(null) return // skip the generic result-setting below } else if (pass === 'pass4') { // Paginated loop — 200 chunks per request, parallel OpenAI batches within each const PAGE = 200 let offset = 0 let totalCreated = 0, totalSkipped = 0 const allErrors: string[] = [] setPassProgress(p => ({ ...p, pass4: { done: 0, total: 0 } })) while (true) { const pageRes = await apiFetch('/api/custom_cil-embeddings?action=embed_approved', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ project_id: projectId, limit: PAGE, offset, file_path: targetFilePath || undefined }), }) const pageData = await pageRes.json() const d = pageData?.data || pageData totalCreated += d.embeddings_created || 0 totalSkipped += d.embeddings_skipped || 0 if (d.errors?.length) allErrors.push(...d.errors) setPassProgress(p => ({ ...p, pass4: { done: d.done_total || offset + PAGE, total: d.total || 0 } })) if (!d.has_more) break offset = d.next_offset } setPassResults(prev => ({ ...prev, pass4: { embeddings_created: totalCreated, embeddings_skipped: totalSkipped, errors: allErrors } })) setPassProgress(p => ({ ...p, pass4: null })) fetchEmbeddingEstimate() setRunningPass(null) return } else if (pass === 'pass4b') { // Paginated loop — 100 files per request const PAGE = 100 let offset = 0 let totalImports = 0, totalCalls = 0, totalExports = 0 const allErrors: string[] = [] setPassProgress(p => ({ ...p, pass4b: { done: 0, total: 0 } })) while (true) { const pageRes = await apiFetch('/api/custom_cil-callmap?action=map_project', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ project_id: projectId, limit: PAGE, offset, file_path: targetFilePath || undefined }), }) const pageData = await pageRes.json() const d = pageData?.data || pageData totalImports += d.imports_created || 0 totalCalls += d.calls_created || 0 totalExports += d.exports_created || 0 if (d.errors?.length) allErrors.push(...d.errors) setPassProgress(p => ({ ...p, pass4b: { done: d.done_total || offset + PAGE, total: d.total_files || 0 } })) if (!d.has_more) break offset = d.next_offset } setPassResults(prev => ({ ...prev, pass4b: { imports_created: totalImports, calls_created: totalCalls, exports_created: totalExports, errors: allErrors } })) setPassProgress(p => ({ ...p, pass4b: null })) setRunningPass(null) return } else if (pass === 'pass5') { res = await apiFetch('/api/custom_cil-config?action=run', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ project_id: projectId, file_path: targetFilePath || undefined }), }) } else { res = await apiFetch(`/api/custom_cil-ingest?action=${pass}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ project_id: projectId, globs: passConfig.globs.split(',').map(s => s.trim()).filter(Boolean), exclude_globs: passConfig.exclude_globs.split(',').map(s => s.trim()).filter(Boolean), force_reprocess: passConfig.force_reprocess, file_path: targetFilePath || undefined, }), }) } const result = await res.json() setPassResults(prev => ({ ...prev, [pass]: result?.data || result })) fetchProposals() if (pass === 'pass2') { fetchCostEstimate(); fetchEmbeddingEstimate() } } catch (err) { setPassResults(prev => ({ ...prev, [pass]: { error: String(err) } })) } finally { setRunningPass(null) } } const toggleProposalSelection = (id: string) => { setSelectedProposalIds(prev => { const next = new Set(prev) if (next.has(id)) next.delete(id) else next.add(id) return next }) } const toggleSelectAll = () => { if (selectedProposalIds.size === proposals.length) { setSelectedProposalIds(new Set()) } else { setSelectedProposalIds(new Set(proposals.map(p => p.id))) } } const handleBulkApprove = async () => { if (!selectedProposalIds.size || !projectId) return setBulkProcessing(true) const ids = Array.from(selectedProposalIds) const repoPath = passConfig.repo_path || project?.data?.repo_path || '' await Promise.all(ids.map(proposalId => apiFetch(`/api/custom_cil-ingest?action=promote_proposal`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ project_id: projectId, proposal_id: proposalId, repo_path: repoPath }), }).catch(() => // Fallback: mark as approved without disk write apiFetch(`/api/admin-data?id=${proposalId}`, { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ entity: 'items', data: { review_status: 'approved' } }), }) ) )) setSelectedProposalIds(new Set()) setBulkProcessing(false) fetchProposals() } const handleBulkReject = async () => { if (!selectedProposalIds.size) return setBulkProcessing(true) const ids = Array.from(selectedProposalIds) await Promise.all(ids.map(proposalId => apiFetch(`/api/admin-data?id=${proposalId}`, { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ entity: 'items', data: { review_status: 'rejected' } }), }) )) setSelectedProposalIds(new Set()) setBulkProcessing(false) fetchProposals() } if (loadingProject) { return (
) } if (!project) { return (
Project not found.
) } return (
{/* Header */}
Code Intelligence {project.title}

{project.title} — Ingestion

{project.data?.repo_path}

View Graph
{/* Stats row */}
{[ { label: 'Files', value: project.stats?.file_count ?? project.data?.file_count ?? 0 }, { label: 'Chunks', value: project.stats?.chunk_count ?? project.data?.chunk_count ?? 0 }, { label: 'Pending Review', value: proposals.length }, { label: 'Cost', value: `$${(project.data?.total_ingestion_cost_usd ?? 0).toFixed(4)}` }, ].map(s => (
{s.value}
{s.label}
))}
{/* Pass Config */}
Pass Configuration
setPassConfig(c => ({ ...c, repo_path: e.target.value }))} placeholder="/absolute/path/to/repo" className="font-mono text-xs" /> {!passConfig.repo_path && (

Required — absolute path to the repository on this machine

)}
setPassConfig(c => ({ ...c, globs: e.target.value }))} placeholder="**/*.ts,**/*.tsx" className="font-mono text-xs" />
setPassConfig(c => ({ ...c, exclude_globs: e.target.value }))} placeholder="node_modules/**,dist/**" className="font-mono text-xs" />

Patterns saved here are applied to all passes for this project.

{ingestMode === 'batch' ? 'Run passes across the whole project using the globs above.' : 'Run passes against a single file or directory only.'}

Batch setIngestMode(checked ? 'file' : 'batch')} /> Single
{ingestMode === 'file' && (
setTargetFilePath(e.target.value)} placeholder=".framework/functions/_shared/principal.ts" className="font-mono text-xs" />

Path from the project root. Passes 1–6 will filter to this file/dir.

)}
{/* Pass steps */} {PASS_STEPS.map((step, idx) => { // Review queue renders inline between AST chunk and AI Document if (step.key === 'review') { return (
{idx + 1}
{step.label}

{step.description}

{selectedProposalIds.size > 0 && ( <> )} {proposals.length > 0 && ( )}
{loadingProposals ? (
{[1,2,3].map(i => )}
) : proposals.length === 0 ? (
Queue is clear — all boundaries auto-approved or reviewed
) : (

{selectedProposalIds.size > 0 ? `${selectedProposalIds.size} of ${proposals.length} selected` : `${proposals.length} pending`}

{proposals.map(proposal => ( ))}
)}
) } const passKey = step.key as 'pass1' | 'pass2' | 'pass3' | 'pass4' | 'pass4b' | 'pass5' const isRunning = runningPass === passKey return (
{idx + 1}
{step.label}

{step.description}

{/* Progress bars while running paginated passes */} {(['pass3', 'pass4', 'pass4b'] as const).includes(step.key as any) && isRunning && passProgress[step.key] && (

Processing… {passProgress[step.key]!.done} / {passProgress[step.key]!.total || '?'} {step.key === 'pass4b' ? ' files' : ' chunks'}

{passProgress[step.key]!.total > 0 && (
)}
)} {/* Progress bar + cost estimate for Pass 3 (AI Document) */} {step.key === 'pass3' && ( {isRunning && passProgress['pass3'] ? (

Documenting… {passProgress['pass3'].done} / {passProgress['pass3'].total || '?'} chunks

{passProgress['pass3'].total > 0 && (
)}
) : (
{loadingEstimate ? (

Calculating cost estimate…

) : costEstimate ? (

Estimated cost: ${costEstimate.estimated_cost_usd ?? '?'} ({costEstimate.model ?? 'gpt-4o-mini'})

{costEstimate.chunks_to_document ?? 0} chunks to document · {costEstimate.already_documented ?? 0} already done · ~{((costEstimate.estimated_total_tokens ?? 0) / 1000).toFixed(0)}K tokens

) : (

Run Pass 2 first to generate chunks before estimating cost.

)}
)} )} {/* Cost estimate banner for Pass 4 (Embed Chunks) */} {step.key === 'pass4' && !isRunning && ( {loadingEmbeddingEstimate ? (

Calculating embedding cost…

) : embeddingEstimate ? (

Estimated cost: ${embeddingEstimate.estimated_cost_usd ?? '?'} ({embeddingEstimate.model ?? 'text-embedding-3-small'})

{embeddingEstimate.chunks_to_embed ?? 0} chunks × {embeddingEstimate.vectors_per_chunk ?? 3} vectors · {embeddingEstimate.already_embedded ?? 0} already embedded · ~{((embeddingEstimate.estimated_tokens ?? 0) / 1000).toFixed(0)}K tokens

) : (

Run Pass 2 first to generate chunks before estimating cost.

)}
)} {passResults[passKey] && !isRunning && (
                      {JSON.stringify(passResults[passKey], null, 2)}
                    
)} ) })}
) }