/** * fable-harness.ts — Cost-disciplined headless Fable LLM-as-judge harness. * * This is TIER 2 of the tiered `resolved` oracle (see distill-oracle.ts). When * a trajectory has NO mechanical test spec to execute, we judge whether its * completion actually resolved the task with a headless Fable model — a smarter * proxy than the structural verifier, but still a proxy (provenance tag * `judge:fable`, never presented as ground truth per ADR-169). * * ── MEASURED COST DATA (load-bearing — this file is built around it) ──────── * `claude -p --model claude-fable-5 --output-format json` costs, per call: * • ~$1.56 when launched FROM THE PROJECT DIR — it auto-loads CLAUDE.md and * ~56k cache tokens of project context we do NOT want for judging. * • ~$0.34 from a CLEAN empty cwd with `--append-system-prompt` for the role. * • ~$0.02/item when we BATCH ~20 items into a single call (context amortizes * ~free across the batch). * * Therefore this harness MUST, and does: * 1. run `claude -p` from a FRESH EMPTY TEMP cwd — never the project dir, so * no CLAUDE.md / project context is loaded (the 5x cost driver); * 2. carry the judge/reflect role via `--append-system-prompt`, not a project * system prompt; * 3. BATCH N items per call (default 20) so per-item cost collapses to ~$0.02; * 4. pass `--max-budget-usd` as a hard per-call cap and stop launching batches * once the cumulative measured spend reaches the caller's budget; * 5. be OPT-IN and OFF BY DEFAULT — nothing here runs unless a caller * explicitly constructs the harness AND provides a budget cap. * * SAFETY: constructing this class spends nothing. Only `judgeBatch` / * `reflectFailures` spawn `claude`, and only when `maxBudgetUsd > 0`. The * `spawnClaude` implementation is injectable so tests never touch the real CLI. * * @module services/fable-harness */ export declare const FABLE_COST_MODEL: { /** Measured $/call when launched from the project dir (loads CLAUDE.md). Anti-pattern. */ readonly perCallProjectCwdUsd: 1.56; /** Measured $/call from a clean cwd with --append-system-prompt. */ readonly perCallCleanCwdUsd: 0.34; /** Measured amortized $/item when batching ~20 items per call. */ readonly perItemBatchedUsd: 0.02; /** Default items per `claude -p` call. */ readonly defaultBatchSize: 20; /** The judge model. */ readonly model: "claude-fable-5"; }; /** * Estimate the USD cost of judging `itemCount` items in batches of `batchSize`. * Uses the measured amortized per-item figure; callers use this to size a * budget cap before opting in. */ export declare function estimateFableCostUsd(itemCount: number, batchSize?: number): number; /** One item to judge: did `output` actually resolve `task`? */ export interface JudgeItem { id: string; task: string; output: string; } /** Verdict for a single judged item. */ export interface JudgeResult { id: string; resolved: boolean; /** 0..1 self-reported judge confidence. */ confidence: number; reason: string; } /** One item for reflective failure analysis (GEPA/evolve mutation input). */ export interface ReflectItem { id: string; task: string; output: string; /** Optional signal that this trajectory is believed to have failed. */ failureHint?: string; } /** Reflective diagnosis for a single item (the reflective-mutation SOTA trick). */ export interface ReflectResult { id: string; failureClass: string; diagnosis: string; mutationHint: string; } /** * ADR-316 — a compact, STRUCTURAL-ONLY snapshot of the current coding * session for the statusline's co-pilot advisor tip. Every field here * mirrors funnel/insights.ts's LocalInsightContext — no raw prompt/command/ * file content, ever (same bar as ADR-309, applied to a different, opt-in * data flow). The model sees only what's already surfaced structurally * elsewhere in the statusline. */ export interface CoPilotSnapshot { security?: { status: string; findings?: number; cvesFixed: number; totalCves: number; }; swarm?: { activeAgents: number; maxAgents: number; coordinationActive: boolean; }; gitUncommittedCount?: number; contextPctUsed?: number; } /** A single proactive suggestion for the insight ticker. */ export interface CoPilotTip { /** Short enough for a single statusline row (caller truncates further). */ headline: string; /** One actionable sentence of extra detail. */ detail: string; confidence: number; } /** Result of a raw claude spawn. */ export interface ClaudeSpawnResult { stdout: string; stderr: string; code: number | null; /** Measured spend for this call, parsed from the JSON envelope when present. */ costUsd?: number; } /** * Injectable spawner. Receives the argv (after `claude`), the prompt to pipe to * stdin, and the cwd (a fresh empty temp dir). Default implementation shells out * to the real `claude` CLI; tests inject a fake. */ export type ClaudeSpawnFn = (argv: string[], stdinPrompt: string, cwd: string, opts: { timeoutMs: number; }) => Promise; export interface FableHarnessOptions { /** Model id (default claude-fable-5). */ model?: string; /** Items per `claude -p` call (default 20). */ batchSize?: number; /** * Hard budget cap in USD across all calls this harness makes. REQUIRED to be * > 0 for any spend to happen — 0/undefined means the harness refuses to * spawn (safe default). */ maxBudgetUsd?: number; /** Per-call timeout (default 5 min). */ timeoutMs?: number; /** Injected spawner for tests; defaults to the real `claude` CLI. */ spawnClaude?: ClaudeSpawnFn; } /** * Default `claude -p` spawner. Pipes the prompt via stdin (never as an argv * positional — mirrors the #1852 fix so shell metachars in prompts are never * re-tokenized), runs in the provided (temp) cwd, and returns stdout/stderr. * Parses `total_cost_usd` from the `--output-format json` envelope when present. */ export declare const defaultSpawnClaude: ClaudeSpawnFn; /** Pull `total_cost_usd`/`cost_usd` out of a claude `--output-format json` envelope. */ export declare function parseCostFromEnvelope(stdout: string): number | undefined; export declare class FableHarness { private readonly model; private readonly batchSize; private readonly maxBudgetUsd; private readonly timeoutMs; private readonly spawnClaude; private spentUsd; constructor(opts?: FableHarnessOptions); /** Cumulative measured spend across all calls this harness has made. */ getSpentUsd(): number; /** True when a positive budget cap is configured (a precondition for any spend). */ isEnabled(): boolean; /** * Judge a set of items in batches. Returns one JudgeResult per input id that * the model returned. Items that fall outside the budget, or that the model * omits, are simply absent from the result — the caller (distill-oracle) is * responsible for falling back to the structural proxy for those. * * Spends $0 and returns [] when no budget cap is configured. */ judgeBatch(items: JudgeItem[]): Promise; /** * Reflective failure analysis over items — the second cost-disciplined entry * point, used by GEPA/evolve for mutation hints. Same batching + budget * discipline as judgeBatch. Returns [] when no budget cap is configured. */ reflectFailures(items: ReflectItem[]): Promise; /** * ADR-316 — one proactive co-pilot tip from a structural session snapshot * (no raw prompt/command/file content). A single-item "batch" — same * budget/cwd/parsing discipline as judgeBatch/reflectFailures, just with * exactly one call instead of a loop. Returns null when disabled, over * budget, or the model found nothing worth surfacing (an empty verdict * array is a valid, non-error answer here, not a parse failure). */ adviseCoPilotTip(snapshot: CoPilotSnapshot): Promise; /** Build the argv for a `claude -p` call. Exposed shape for testability. */ buildArgv(systemPrompt: string): string[]; /** * Run one batch: create a FRESH EMPTY temp cwd (critical — no project * context), spawn `claude -p` there with the role via --append-system-prompt, * pipe the batch JSON to stdin, parse the verdict array out of the envelope, * and account the measured spend. */ private runBatch; } /** * Extract the model's verdict array. `claude -p --output-format json` wraps the * assistant text in an envelope `{ result: "", ... }`; the text is itself * the JSON array we asked for. Handle both the enveloped and bare forms, and * arrays fenced in ```json blocks. */ export declare function extractVerdictArray(stdout: string): unknown[]; export default FableHarness; //# sourceMappingURL=fable-harness.d.ts.map