#!/usr/bin/env node /** * cli:sources-ingest — entry point. * * The ONLY writer of the client-sources registry `.smartstack/sources/` * (sibling root of the BA tree, COMMITTED). The model judges, this CLI * fingerprints, dedups, extracts (DOCX/XLSX), scrubs and writes atomically. * * Invocation: * npx --prefer-offline tsx skills/business-analyse/create-sources/cli/ingest/index.ts \ * --spec '{"baRoot":".smartstack/ba","mode":"plan","kind":"file","origin":{"path":"docs/cdc.pdf"}}' \ * [--workdir ] * * - mode=plan → READ-ONLY preview: fingerprint, dedup, allocation, * extraction for the model. Nothing written to the registry. * - mode=write → writes SRC-NNN/source.md + index.json (+ raw/). * - mode=register-blocked → reserves the code with a typed unread status. */ import { parseArgs } from 'node:util' import { copyFileSync, mkdirSync } from 'node:fs' import { basename, join } from 'node:path' import { executeEnvelope, failExecute, failGenerate, generateEnvelope, printEnvelope, } from '../../../../lib/output.js' import { writeFileAtomic } from '../../../../lib/fs.js' import { fingerprintOfContent, loadSourcesRegistry, sourcesRootFor, SOURCE_DOC_FILE, SOURCES_INDEX_FILE, } from '../../../../lib/ba-sources.js' import { validateSpec } from './validate.js' import { detectFormat, extractionFor, readOriginFile, type OriginContent } from './sources.js' import { buildIngest, resolveAllocation, RAW_COPY_MAX_BYTES } from './execute.js' import type { IngestPlanReport, IngestWriteReport } from './types.js' const COMMAND = 'sources-ingest' async function main(): Promise { const { values } = parseArgs({ options: { spec: { type: 'string' }, workdir: { type: 'string' }, }, strict: true, }) if (!values.spec) { printEnvelope(failExecute(COMMAND, ['--spec is required'])) process.exit(1) } let raw: unknown try { raw = JSON.parse(values.spec) } catch { printEnvelope(failExecute(COMMAND, ['Invalid JSON in --spec'])) process.exit(1) } const validation = validateSpec(raw, values.workdir) if (!validation.valid || !validation.spec || !validation.resolvedBaRoot) { printEnvelope(failExecute(COMMAND, validation.errors)) process.exit(1) } const spec = validation.spec const sourcesRoot = sourcesRootFor(validation.resolvedBaRoot) const registry = loadSourcesRegistry(sourcesRoot) const registryWarnings: string[] = [] if (registry.exists && registry.index === null) { registryWarnings.push( `Registre présent mais ${SOURCES_INDEX_FILE} illisible — répare-le avant d'écrire (voir cli/status) : ${registry.reconciliation.issues.join(' ; ')}`, ) } // --- content + fingerprint ----------------------------------------------- const format = detectFormat(spec.kind, validation.resolvedOriginPath) let content: OriginContent | null = null let fingerprint: string if (spec.kind === 'file') { content = readOriginFile(validation.resolvedOriginPath!, format) fingerprint = fingerprintOfContent(content.text ?? content.buffer) } else { fingerprint = fingerprintOfContent((spec.origin as { url: string }).url.trim()) } const today = new Date().toISOString().slice(0, 10) // --- mode=plan: READ-ONLY preview ---------------------------------------- if (spec.mode === 'plan') { const allocation = resolveAllocation(registry.index, fingerprint, spec.as) if (allocation.error) { printEnvelope(failExecute(COMMAND, [allocation.error])) process.exit(1) } const extraction = await extractionFor(format, content, fingerprint) const copyRaw = spec.kind === 'file' && spec.copyRaw !== false const report: IngestPlanReport = { mode: 'plan', kind: spec.kind, format, fingerprint, registryExists: registry.exists, allocation: { kind: allocation.kind, code: allocation.code }, extraction, rawPlan: { copy: copyRaw && (content?.bytes ?? 0) <= RAW_COPY_MAX_BYTES, bytes: content?.bytes, ...(copyRaw && (content?.bytes ?? 0) > RAW_COPY_MAX_BYTES ? { reason: `> cap ${RAW_COPY_MAX_BYTES} octets — rawOmitted` } : {}), }, warnings: registryWarnings, } const nextSteps: string[] = [] if (allocation.kind === 'existing') { nextSteps.push(`Empreinte déjà connue → ${allocation.code} : un mode=write MET À JOUR ce code (jamais un second dossier).`) } if (extraction.status === 'unavailable') { nextSteps.push('Contenu illisible → mode=register-blocked (le code est réservé, le contenu se rattache plus tard via `as`).') } else { nextSteps.push('Lis le contenu, juge (titre, résumé, tags, sections §, extraits verbatim, portée), puis mode=write avec `analysis`.') } printEnvelope( executeEnvelope(COMMAND, { success: true, data: { fingerprint, format, allocation: allocation.code, extraction: extraction.status }, report, warnings: registryWarnings, nextSteps, }), ) process.exit(0) } // --- mode=write | register-blocked: the writer ---------------------------- if (registry.exists && registry.index === null) { printEnvelope(failGenerate(COMMAND, registryWarnings)) process.exit(1) } const fallbackTitle = spec.kind === 'file' ? basename((spec.origin as { path: string }).path) : (spec.origin as { url: string }).url const result = buildIngest({ mode: spec.mode, kind: spec.kind, format, fingerprint, origin: spec.origin, blockedStatus: format === 'docx' || format === 'xlsx' ? 'blocked/unreadable' : 'blocked/needs-export', reason: spec.reason, analysis: spec.analysis, fallbackTitle, today, index: registry.index, asCode: spec.as, supersedes: spec.supersedes, raw: spec.kind === 'file' && content ? { basename: basename((spec.origin as { path: string }).path), bytes: content.bytes, copyRaw: spec.copyRaw !== false, } : undefined, }) if (!result.ok) { printEnvelope(failGenerate(COMMAND, result.errors)) process.exit(1) } const { entry, sourceMd, index, warnings } = result.rendered const code = entry.code const codeDir = join(sourcesRoot, code) const docPath = join(codeDir, SOURCE_DOC_FILE) const indexPath = join(sourcesRoot, SOURCES_INDEX_FILE) const isNew = result.allocation.kind === 'new' mkdirSync(codeDir, { recursive: true }) if (entry.rawFile && content) { const rawDest = join(codeDir, ...entry.rawFile.split('/')) mkdirSync(join(codeDir, 'raw'), { recursive: true }) copyFileSync(validation.resolvedOriginPath!, rawDest) } await writeFileAtomic(docPath, sourceMd) await writeFileAtomic(indexPath, `${JSON.stringify(index, null, 2)}\n`) // The superseded doc's anchor flips too — its source.md must agree with the // index (loadSourcesRegistry cross-checks both). const supersededFiles: string[] = [] if (spec.supersedes) { const oldDoc = registry.docs.find((d) => d.code === spec.supersedes) if (oldDoc) { const flipped = oldDoc.raw.replace( /(