import { useEffect, useState } from 'react'
import { useParams } from 'react-router-dom'
import { apiFetch } from '@core/lib/api'
import { useAppPath } from '@core/hooks/useAppPath'
import { Card, CardContent, CardHeader, CardTitle } from '@core/components/ui/card'
import { Button } from '@core/components/ui/button'
import { Badge } from '@core/components/ui/badge'
import { Skeleton } from '@core/components/ui/skeleton'
import { Input } from '@core/components/ui/input'
import { Label } from '@core/components/ui/label'
import { Switch } from '@core/components/ui/switch'
import {
Play,
CheckCircle2,
XCircle,
AlertTriangle,
Clock,
FileCode2,
Code2,
GitPullRequest,
ChevronRight,
RefreshCw,
Zap,
Sparkles,
GitGraph,
Settings2,
Save,
DollarSign,
} from 'lucide-react'
interface Project {
id: string
title: string
data: {
repo_path: string
ingestion_status: string
file_count: number
chunk_count: number
total_ingestion_cost_usd: number
last_ingested_at: string | null
globs?: string[]
exclude_globs?: string[]
}
stats?: {
file_count: number
chunk_count: number
pending_review_count: number
}
}
interface ChunkProposal {
id: string
title: string
data: {
file_path: string
proposed_name: string
line_start: number
line_end: number
chunk_type: string
confidence_score: number
review_status: string
code_preview: string
confidence_reasons: string
existing_marker_match: boolean
existing_marker_name: string
}
}
interface PassConfig {
repo_path: string
globs: string
exclude_globs: string
force_reprocess: boolean
}
const PASS_STEPS = [
{ key: 'pass1', label: 'Pass 1 — Discover Files', icon: FileCode2, description: 'Walk the repo, register cil_file items, compute hashes' },
{ key: 'pass2', label: 'Pass 2 — AST Chunk', icon: Code2, description: 'Parse files with AST, propose chunk boundaries, write high-confidence markers' },
{ key: 'review', label: 'Review Queue', icon: GitPullRequest, description: 'Review low-confidence boundary proposals before continuing' },
{ key: 'pass3', label: 'Pass 3 — AI Document', icon: Sparkles, description: 'Use LLM to fill macro / micro / contract fields — do this before embedding so vectors are rich' },
{ key: 'pass4', label: 'Pass 4 — Embed Chunks', icon: Zap, description: 'Generate semantic + structural + contract vector embeddings using the documented fields' },
{ key: 'pass4b', label: 'Pass 5 — Call Map', icon: GitGraph, description: 'Extract import + call edges, write cil_imports, cil_calls, cil_tested_by links' },
{ key: 'pass5', label: 'Pass 6 — Config Extract', icon: Settings2, description: 'Scan config files, create cil_config items, write cil_configures links' },
]
function ProposalRow({ proposal, selected, onToggle }: {
proposal: ChunkProposal
selected: boolean
onToggle: (id: string) => void
}) {
const d = proposal.data
return (
onToggle(proposal.id)}
>
{/* Checkbox */}
{selected && }
{/* Content */}
{d.proposed_name || proposal.title}
{d.chunk_type || 'unknown'}
{Math.round((d.confidence_score || 0) * 100)}% confidence
{d.existing_marker_match && (
matches marker
)}
{d.file_path} · L{d.line_start}–{d.line_end}
{d.confidence_reasons && (
{d.confidence_reasons}
)}
{d.code_preview && (
{d.code_preview}
)}
)
}
export default function CILIngestionPage() {
const { id: projectId } = useParams<{ id: string }>()
const appPath = useAppPath()
const [project, setProject] = useState(null)
const [proposals, setProposals] = useState([])
const [selectedProposalIds, setSelectedProposalIds] = useState>(new Set())
const [bulkProcessing, setBulkProcessing] = useState(false)
const [loadingProject, setLoadingProject] = useState(true)
const [loadingProposals, setLoadingProposals] = useState(true)
const [runningPass, setRunningPass] = useState(null)
const [passResults, setPassResults] = useState>({})
const [passProgress, setPassProgress] = useState>({})
const [passConfig, setPassConfig] = useState({
repo_path: '',
globs: '**/*.ts,**/*.tsx,**/*.sql',
exclude_globs: 'node_modules/**,.git/**,dist/**,build/**,.netlify/**,**/*.min.js,**/*.d.ts',
force_reprocess: false,
})
const [ingestMode, setIngestMode] = useState<'batch' | 'file'>('batch')
const [targetFilePath, setTargetFilePath] = useState('')
const [savingConfig, setSavingConfig] = useState(false)
const [configSaved, setConfigSaved] = useState(false)
const [costEstimate, setCostEstimate] = useState(null)
const [loadingEstimate, setLoadingEstimate] = useState(false)
const [embeddingEstimate, setEmbeddingEstimate] = useState(null)
const [loadingEmbeddingEstimate, setLoadingEmbeddingEstimate] = useState(false)
useEffect(() => {
if (!projectId) return
apiFetch(`/api/custom_cil-project?action=get&id=${projectId}`)
.then(r => r.json())
.then(data => {
const p = data?.project || data?.data?.project || null
setProject(p)
// Populate pass config from persisted project settings (always, using defaults if not yet saved)
const DEFAULT_GLOBS = '**/*.ts,**/*.tsx,**/*.sql'
const DEFAULT_EXCLUDES = 'node_modules/**,.git/**,dist/**,build/**,.netlify/**,**/*.min.js,**/*.d.ts'
setPassConfig(c => ({
...c,
repo_path: p?.data?.repo_path || '',
globs: p?.data?.globs?.length ? p.data.globs.join(',') : DEFAULT_GLOBS,
exclude_globs: p?.data?.exclude_globs?.length ? p.data.exclude_globs.join(',') : DEFAULT_EXCLUDES,
}))
})
.catch(() => {})
.finally(() => setLoadingProject(false))
}, [projectId])
const saveConfig = async () => {
if (!projectId) return
setSavingConfig(true)
setConfigSaved(false)
try {
await apiFetch(`/api/custom_cil-project?action=update&id=${projectId}`, {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
id: projectId,
// Send only the fields we own — server merges into existing data
data: {
repo_path: passConfig.repo_path.trim(),
globs: passConfig.globs.split(',').map(s => s.trim()).filter(Boolean),
exclude_globs: passConfig.exclude_globs.split(',').map(s => s.trim()).filter(Boolean),
},
}),
})
setConfigSaved(true)
setTimeout(() => setConfigSaved(false), 2000)
// Refresh project state so UI reflects merged record
apiFetch(`/api/custom_cil-project?action=get&id=${projectId}`)
.then(r => r.json())
.then(data => { const p = data?.project || data?.data?.project || null; if (p) setProject(p) })
.catch(() => {})
} catch {
// silent — user can retry
} finally {
setSavingConfig(false)
}
}
const fetchProposals = () => {
if (!projectId) return
setLoadingProposals(true)
apiFetch(`/api/custom_cil-project?action=list_proposals&id=${projectId}&status=pending_review`)
.then(r => r.json())
.then(res => {
// Framework wraps response in { data: { proposals: [...] } }
setProposals(res?.data?.proposals || res?.proposals || [])
})
.catch(() => {})
.finally(() => setLoadingProposals(false))
}
useEffect(() => { fetchProposals() }, [projectId])
const fetchCostEstimate = () => {
if (!projectId) return
setLoadingEstimate(true)
const fileParam = (ingestMode === 'file' && targetFilePath)
? `&file_path=${encodeURIComponent(targetFilePath)}`
: ''
apiFetch(`/api/custom_cil-doc?action=estimate&project_id=${projectId}${fileParam}`)
.then(r => r.json())
.then(res => setCostEstimate(res?.data || res || null))
.catch(() => setCostEstimate(null))
.finally(() => setLoadingEstimate(false))
}
const fetchEmbeddingEstimate = () => {
if (!projectId) return
setLoadingEmbeddingEstimate(true)
const fileParam = (ingestMode === 'file' && targetFilePath)
? `&file_path=${encodeURIComponent(targetFilePath)}`
: ''
apiFetch(`/api/custom_cil-embeddings?action=estimate&project_id=${projectId}${fileParam}`)
.then(r => r.json())
.then(res => setEmbeddingEstimate(res?.data || res || null))
.catch(() => setEmbeddingEstimate(null))
.finally(() => setLoadingEmbeddingEstimate(false))
}
useEffect(() => {
fetchCostEstimate()
fetchEmbeddingEstimate()
}, [projectId, ingestMode, targetFilePath])
const runPass = async (pass: 'pass1' | 'pass2' | 'pass3' | 'pass4' | 'pass4b' | 'pass5') => {
if (!projectId) return
setRunningPass(pass)
try {
let res: Response
if (pass === 'pass3') {
// Paginated loop — 50 chunks per request, ~20 in parallel each, well within 30s
const PAGE = 50
let offset = 0
let totalDocumented = 0, totalSkipped = 0, totalFailed = 0
const allErrors: string[] = []
setPassProgress(p => ({ ...p, pass3: { done: 0, total: 0 } }))
while (true) {
const pageRes = await apiFetch('/api/custom_cil-doc?action=document', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ project_id: projectId, limit: PAGE, offset, file_path: targetFilePath || undefined }),
})
const pageData = await pageRes.json()
const d = pageData?.data || pageData
totalDocumented += d.documented || 0
totalSkipped += d.skipped || 0
totalFailed += d.failed || 0
if (d.errors?.length) allErrors.push(...d.errors)
setPassProgress(p => ({ ...p, pass3: { done: d.done_total || offset + PAGE, total: d.total_chunks || d.total_to_document || 0 } }))
if (!d.has_more) break
offset = d.next_offset
}
setPassResults(prev => ({ ...prev, pass3: { documented: totalDocumented, skipped: totalSkipped, failed: totalFailed, errors: allErrors } }))
setPassProgress(p => ({ ...p, pass3: null }))
fetchCostEstimate()
setRunningPass(null)
return // skip the generic result-setting below
} else if (pass === 'pass4') {
// Paginated loop — 200 chunks per request, parallel OpenAI batches within each
const PAGE = 200
let offset = 0
let totalCreated = 0, totalSkipped = 0
const allErrors: string[] = []
setPassProgress(p => ({ ...p, pass4: { done: 0, total: 0 } }))
while (true) {
const pageRes = await apiFetch('/api/custom_cil-embeddings?action=embed_approved', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ project_id: projectId, limit: PAGE, offset, file_path: targetFilePath || undefined }),
})
const pageData = await pageRes.json()
const d = pageData?.data || pageData
totalCreated += d.embeddings_created || 0
totalSkipped += d.embeddings_skipped || 0
if (d.errors?.length) allErrors.push(...d.errors)
setPassProgress(p => ({ ...p, pass4: { done: d.done_total || offset + PAGE, total: d.total || 0 } }))
if (!d.has_more) break
offset = d.next_offset
}
setPassResults(prev => ({ ...prev, pass4: { embeddings_created: totalCreated, embeddings_skipped: totalSkipped, errors: allErrors } }))
setPassProgress(p => ({ ...p, pass4: null }))
fetchEmbeddingEstimate()
setRunningPass(null)
return
} else if (pass === 'pass4b') {
// Paginated loop — 100 files per request
const PAGE = 100
let offset = 0
let totalImports = 0, totalCalls = 0, totalExports = 0
const allErrors: string[] = []
setPassProgress(p => ({ ...p, pass4b: { done: 0, total: 0 } }))
while (true) {
const pageRes = await apiFetch('/api/custom_cil-callmap?action=map_project', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ project_id: projectId, limit: PAGE, offset, file_path: targetFilePath || undefined }),
})
const pageData = await pageRes.json()
const d = pageData?.data || pageData
totalImports += d.imports_created || 0
totalCalls += d.calls_created || 0
totalExports += d.exports_created || 0
if (d.errors?.length) allErrors.push(...d.errors)
setPassProgress(p => ({ ...p, pass4b: { done: d.done_total || offset + PAGE, total: d.total_files || 0 } }))
if (!d.has_more) break
offset = d.next_offset
}
setPassResults(prev => ({ ...prev, pass4b: { imports_created: totalImports, calls_created: totalCalls, exports_created: totalExports, errors: allErrors } }))
setPassProgress(p => ({ ...p, pass4b: null }))
setRunningPass(null)
return
} else if (pass === 'pass5') {
res = await apiFetch('/api/custom_cil-config?action=run', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ project_id: projectId, file_path: targetFilePath || undefined }),
})
} else {
res = await apiFetch(`/api/custom_cil-ingest?action=${pass}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
project_id: projectId,
globs: passConfig.globs.split(',').map(s => s.trim()).filter(Boolean),
exclude_globs: passConfig.exclude_globs.split(',').map(s => s.trim()).filter(Boolean),
force_reprocess: passConfig.force_reprocess,
file_path: targetFilePath || undefined,
}),
})
}
const result = await res.json()
setPassResults(prev => ({ ...prev, [pass]: result?.data || result }))
fetchProposals()
if (pass === 'pass2') { fetchCostEstimate(); fetchEmbeddingEstimate() }
} catch (err) {
setPassResults(prev => ({ ...prev, [pass]: { error: String(err) } }))
} finally {
setRunningPass(null)
}
}
const toggleProposalSelection = (id: string) => {
setSelectedProposalIds(prev => {
const next = new Set(prev)
if (next.has(id)) next.delete(id)
else next.add(id)
return next
})
}
const toggleSelectAll = () => {
if (selectedProposalIds.size === proposals.length) {
setSelectedProposalIds(new Set())
} else {
setSelectedProposalIds(new Set(proposals.map(p => p.id)))
}
}
const handleBulkApprove = async () => {
if (!selectedProposalIds.size || !projectId) return
setBulkProcessing(true)
const ids = Array.from(selectedProposalIds)
const repoPath = passConfig.repo_path || project?.data?.repo_path || ''
await Promise.all(ids.map(proposalId =>
apiFetch(`/api/custom_cil-ingest?action=promote_proposal`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ project_id: projectId, proposal_id: proposalId, repo_path: repoPath }),
}).catch(() =>
// Fallback: mark as approved without disk write
apiFetch(`/api/admin-data?id=${proposalId}`, {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ entity: 'items', data: { review_status: 'approved' } }),
})
)
))
setSelectedProposalIds(new Set())
setBulkProcessing(false)
fetchProposals()
}
const handleBulkReject = async () => {
if (!selectedProposalIds.size) return
setBulkProcessing(true)
const ids = Array.from(selectedProposalIds)
await Promise.all(ids.map(proposalId =>
apiFetch(`/api/admin-data?id=${proposalId}`, {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ entity: 'items', data: { review_status: 'rejected' } }),
})
))
setSelectedProposalIds(new Set())
setBulkProcessing(false)
fetchProposals()
}
if (loadingProject) {
return (
)
}
if (!project) {
return (
Project not found.
)
}
return (
{/* Header */}
{project.title} — Ingestion
{project.data?.repo_path}
View Graph
{/* Stats row */}
{[
{ label: 'Files', value: project.stats?.file_count ?? project.data?.file_count ?? 0 },
{ label: 'Chunks', value: project.stats?.chunk_count ?? project.data?.chunk_count ?? 0 },
{ label: 'Pending Review', value: proposals.length },
{ label: 'Cost', value: `$${(project.data?.total_ingestion_cost_usd ?? 0).toFixed(4)}` },
].map(s => (
{s.value}
{s.label}
))}
{/* Pass Config */}
Pass Configuration
setPassConfig(c => ({ ...c, repo_path: e.target.value }))}
placeholder="/absolute/path/to/repo"
className="font-mono text-xs"
/>
{!passConfig.repo_path && (
Required — absolute path to the repository on this machine
)}
setPassConfig(c => ({ ...c, globs: e.target.value }))}
placeholder="**/*.ts,**/*.tsx"
className="font-mono text-xs"
/>
{ingestMode === 'batch'
? 'Run passes across the whole project using the globs above.'
: 'Run passes against a single file or directory only.'}
Batch
setIngestMode(checked ? 'file' : 'batch')}
/>
Single
{ingestMode === 'file' && (
)}
{/* Pass steps */}
{PASS_STEPS.map((step, idx) => {
// Review queue renders inline between AST chunk and AI Document
if (step.key === 'review') {
return (
{idx + 1}
{step.label}
{step.description}
{selectedProposalIds.size > 0 && (
<>
>
)}
{proposals.length > 0 && (
)}
{loadingProposals ? (
{[1,2,3].map(i => )}
) : proposals.length === 0 ? (
Queue is clear — all boundaries auto-approved or reviewed
) : (
{selectedProposalIds.size > 0 ? `${selectedProposalIds.size} of ${proposals.length} selected` : `${proposals.length} pending`}
{proposals.map(proposal => (
))}
)}
)
}
const passKey = step.key as 'pass1' | 'pass2' | 'pass3' | 'pass4' | 'pass4b' | 'pass5'
const isRunning = runningPass === passKey
return (
{idx + 1}
{step.label}
{step.description}
{/* Progress bars while running paginated passes */}
{(['pass3', 'pass4', 'pass4b'] as const).includes(step.key as any) && isRunning && passProgress[step.key] && (
Processing… {passProgress[step.key]!.done} / {passProgress[step.key]!.total || '?'}
{step.key === 'pass4b' ? ' files' : ' chunks'}
{passProgress[step.key]!.total > 0 && (
)}
)}
{/* Progress bar + cost estimate for Pass 3 (AI Document) */}
{step.key === 'pass3' && (
{isRunning && passProgress['pass3'] ? (
Documenting… {passProgress['pass3'].done} / {passProgress['pass3'].total || '?'} chunks
{passProgress['pass3'].total > 0 && (
)}
) : (
{loadingEstimate ? (
Calculating cost estimate…
) : costEstimate ? (
Estimated cost: ${costEstimate.estimated_cost_usd ?? '?'} ({costEstimate.model ?? 'gpt-4o-mini'})
{costEstimate.chunks_to_document ?? 0} chunks to document
· {costEstimate.already_documented ?? 0} already done
· ~{((costEstimate.estimated_total_tokens ?? 0) / 1000).toFixed(0)}K tokens
) : (
Run Pass 2 first to generate chunks before estimating cost.
)}
)}
)}
{/* Cost estimate banner for Pass 4 (Embed Chunks) */}
{step.key === 'pass4' && !isRunning && (
{loadingEmbeddingEstimate ? (
Calculating embedding cost…
) : embeddingEstimate ? (
Estimated cost: ${embeddingEstimate.estimated_cost_usd ?? '?'} ({embeddingEstimate.model ?? 'text-embedding-3-small'})
{embeddingEstimate.chunks_to_embed ?? 0} chunks × {embeddingEstimate.vectors_per_chunk ?? 3} vectors
· {embeddingEstimate.already_embedded ?? 0} already embedded
· ~{((embeddingEstimate.estimated_tokens ?? 0) / 1000).toFixed(0)}K tokens
) : (
Run Pass 2 first to generate chunks before estimating cost.
)}
)}
{passResults[passKey] && !isRunning && (
{JSON.stringify(passResults[passKey], null, 2)}
)}
)
})}
)
}