{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../../src/core/evaluation/cli.ts"],"names":[],"mappings":"AAmDA;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,GAAG,MAAM,CAapG;AAED,wBAAsB,sBAAsB,CAC3C,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7B,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAM3C;AAED,wBAAsB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CA0Q9E","sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { mkdir, writeFile } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport chalk from \"chalk\";\nimport { runCavecrew } from \"../cavecrew-runtime.js\";\nimport { compareAgents } from \"./compare-agents.js\";\nimport { checkReleaseGate } from \"./gates.js\";\nimport {\n\taggregateStability,\n\tclusterFailure,\n\tcompareArtifacts,\n\tcreateArtifact,\n\tcreateBaseline,\n\tcreateLiveProviderExecutor,\n\tcreateOpenAiCompatibleProvider,\n\tcreatePruneManifest,\n\tdiscoverEvaluationPacks,\n\tinspectArtifactStore,\n\tlistArtifacts,\n\tlistBaselines,\n\tmergeFailureClusters,\n\tpruneEvaluationStore,\n\treadArtifact,\n\treplayArtifact,\n\trescoreArtifact,\n\tresolveProviderProfile,\n\trunEvaluation,\n\tverifyArtifact,\n\tverifyBaseline,\n\twriteArtifact,\n} from \"./index.js\";\nimport type { EvaluationArtifact, EvaluationReleaseGate, EvaluationVerdict } from \"./types.js\";\n\nconst evaluationRoot = () => join(process.cwd(), \".jensen\", \"evaluations\");\nconst baselineRoot = () => join(evaluationRoot(), \"baselines\");\n\nfunction jsonOutput(args: string[]): boolean {\n\treturn args.includes(\"--json\");\n}\n\nfunction print(value: unknown, json: boolean): void {\n\tconsole.log(\n\t\tjson ? JSON.stringify(value, null, 2) : typeof value === \"string\" ? value : JSON.stringify(value, null, 2),\n\t);\n}\n\nfunction option(args: string[], name: string): string | undefined {\n\tconst index = args.indexOf(name);\n\treturn index === -1 ? undefined : args[index + 1];\n}\n\n/**\n * Map an evaluation verdict to a stable process exit code. JSON and human\n * output share the same result object, so the render mode cannot change the\n * exit status. CI relies on the process exit matching the artifact verdict.\n *\n * 0 -> pass\n * 1 -> fail\n * 2 -> invalid\n * 3 -> cancelled / timed out\n * 4 -> invocation / runtime / internal error\n */\nexport function verdictExitCode(verdict: \"pass\" | \"fail\" | \"invalid\" | \"cancelled\" | \"error\"): number {\n\tswitch (verdict) {\n\t\tcase \"pass\":\n\t\t\treturn 0;\n\t\tcase \"fail\":\n\t\t\treturn 1;\n\t\tcase \"invalid\":\n\t\t\treturn 2;\n\t\tcase \"cancelled\":\n\t\t\treturn 3;\n\t\tdefault:\n\t\t\treturn 4;\n\t}\n}\n\nexport async function runEvaluationSelfProbe(\n\toptions: { root?: string } = {},\n): Promise<{ root: string; marker: string }> {\n\tawait mkdir(options.root ?? process.cwd(), { recursive: true });\n\tconst root = options.root ?? process.cwd();\n\tconst markerPath = join(root, \".jensen-candidate-complete\");\n\tawait writeFile(markerPath, \"candidate\", \"utf8\");\n\treturn { root, marker: markerPath };\n}\n\nexport async function handleEvaluationCommand(args: string[]): Promise<boolean> {\n\tconst isDoctor = args[0] === \"doctor\" && args[1] === \"eval\";\n\tif (args[0] !== \"eval\" && !isDoctor) return false;\n\tconst command = isDoctor ? \"doctor\" : (args[1] ?? \"help\");\n\tconst json = jsonOutput(args);\n\tif (command === \"self-probe\") {\n\t\tprint(await runEvaluationSelfProbe(), json);\n\t\treturn true;\n\t}\n\tif (command === \"help\") {\n\t\tconsole.log(\n\t\t\t`${chalk.bold(\"Usage:\")} jensen eval <packs|scenarios|validate|run|compare-agents|compare|replay|rescore|stability|failures|prune|baseline|gate|doctor>`,\n\t\t);\n\t\treturn true;\n\t}\n\tif (command === \"doctor\") {\n\t\tconst discovered = await discoverEvaluationPacks();\n\t\tconst store = await inspectArtifactStore(evaluationRoot());\n\t\tconst errors = [...discovered.errors, ...store.errors];\n\t\tconst warnings = [...store.warnings];\n\t\tconst status = errors.length ? \"fail\" : warnings.length ? \"warn\" : \"pass\";\n\t\tconst result = {\n\t\t\tname: \"evaluation\",\n\t\t\tstatus,\n\t\t\tpacks: discovered.packs.length,\n\t\t\tscenarios: discovered.scenarios.length,\n\t\t\tstore,\n\t\t\terrors,\n\t\t\twarnings,\n\t\t\texitCode: errors.length ? 2 : warnings.length ? 1 : 0,\n\t\t};\n\t\tprint(result, json);\n\t\tprocess.exitCode = result.exitCode;\n\t\treturn true;\n\t}\n\tconst discovered = await discoverEvaluationPacks();\n\tif (command === \"packs\") {\n\t\tprint(discovered.packs, json);\n\t\treturn true;\n\t}\n\tif (command === \"scenarios\") {\n\t\tprint(discovered.scenarios, json);\n\t\treturn true;\n\t}\n\tif (command === \"validate\") {\n\t\tprint({ valid: discovered.errors.length === 0, errors: discovered.errors }, json);\n\t\tif (discovered.errors.length) process.exitCode = 1;\n\t\treturn true;\n\t}\n\tif (command === \"stability\") {\n\t\tconst artifact = await readArtifact(evaluationRoot(), args[2] ?? \"\");\n\t\tprint(artifact.stability ?? aggregateStability([artifact]), json);\n\t\treturn true;\n\t}\n\tif (command === \"failures\") {\n\t\tconst clusters = mergeFailureClusters(\n\t\t\t(await listArtifacts(evaluationRoot())).flatMap((artifact) => {\n\t\t\t\tconst cluster = clusterFailure(artifact);\n\t\t\t\treturn cluster ? [cluster] : [];\n\t\t\t}),\n\t\t);\n\t\tprint(clusters, json);\n\t\tif (clusters.length > 0) process.exitCode = 1;\n\t\treturn true;\n\t}\n\tif (command === \"prune\") {\n\t\tconst execute = args.includes(\"--execute\");\n\t\tif (!execute && !args.includes(\"--preview\")) throw new Error(\"eval prune requires --preview or --execute\");\n\t\tconst preview = await createPruneManifest(evaluationRoot());\n\t\tprint(execute ? await pruneEvaluationStore(evaluationRoot(), true, undefined, preview) : preview, json);\n\t\treturn true;\n\t}\n\tif (command === \"run\") {\n\t\tconst target = args[2];\n\t\tconst mode = (option(args, \"--mode\") ?? \"fixture\") as \"offline\" | \"fixture\" | \"sandbox\" | \"live\";\n\t\tconst scenarios =\n\t\t\ttarget && discovered.packs.some((pack) => pack.packId === target)\n\t\t\t\t? discovered.scenarios.filter((scenario) =>\n\t\t\t\t\t\tdiscovered.packs.find((pack) => pack.packId === target)?.scenarios.includes(scenario.scenarioId),\n\t\t\t\t\t)\n\t\t\t\t: target\n\t\t\t\t\t? discovered.scenarios.filter((scenario) => scenario.scenarioId === target)\n\t\t\t\t\t: [];\n\t\tif (!scenarios.length) throw new Error(`unknown evaluation scenario or pack: ${target ?? \"\"}`);\n\t\tif (mode === \"live\" && !args.includes(\"--confirm-live\"))\n\t\t\tthrow new Error(\"live evaluation requires --confirm-live\");\n\t\tconst requestedRepeat = option(args, \"--repeat\");\n\t\tconst repeat = requestedRepeat === undefined ? 1 : Number(requestedRepeat);\n\t\tif (!Number.isInteger(repeat) || repeat < 1) throw new Error(\"--repeat must be a positive integer\");\n\t\tconst artifacts: EvaluationArtifact[] = [];\n\t\tfor (const scenario of scenarios) {\n\t\t\tconst repetitions: EvaluationArtifact[] = [];\n\t\t\tfor (let repetition = 0; repetition < repeat; repetition += 1) {\n\t\t\t\tconst artifact = await runEvaluation(scenario, {\n\t\t\t\t\tmode,\n\t\t\t\t\tcandidate: {\n\t\t\t\t\t\tproviderProfile: option(args, \"--provider-profile\") ?? \"fixture\",\n\t\t\t\t\t\tprovider: option(args, \"--provider\") ?? \"fixture\",\n\t\t\t\t\t\tconfiguredModel: option(args, \"--model\") ?? \"deterministic-fixture\",\n\t\t\t\t\t\tseed: repetition,\n\t\t\t\t\t},\n\t\t\t\t\tlive: args.includes(\"--live\") || args.includes(\"--confirm-live\"),\n\t\t\t\t\tconfirmLive: args.includes(\"--confirm-live\"),\n\t\t\t\t\tbudget: {\n\t\t\t\t\t\tmaximumCostUsd: option(args, \"--max-cost-usd\") ? Number(option(args, \"--max-cost-usd\")) : undefined,\n\t\t\t\t\t\tmaximumModelCalls: option(args, \"--max-model-calls\")\n\t\t\t\t\t\t\t? Number(option(args, \"--max-model-calls\"))\n\t\t\t\t\t\t\t: undefined,\n\t\t\t\t\t\tmaximumWallTimeMs: option(args, \"--max-wall-time-ms\")\n\t\t\t\t\t\t\t? Number(option(args, \"--max-wall-time-ms\"))\n\t\t\t\t\t\t\t: undefined,\n\t\t\t\t\t},\n\t\t\t\t\texecutor:\n\t\t\t\t\t\tmode === \"live\"\n\t\t\t\t\t\t\t? (() => {\n\t\t\t\t\t\t\t\t\tconst profile = resolveProviderProfile({\n\t\t\t\t\t\t\t\t\t\tprofileId: option(args, \"--provider-profile\") ?? \"openrouter\",\n\t\t\t\t\t\t\t\t\t\tprovider:\n\t\t\t\t\t\t\t\t\t\t\toption(args, \"--provider\") ?? option(args, \"--provider-profile\") ?? \"openrouter\",\n\t\t\t\t\t\t\t\t\t\tconfiguredModel: option(args, \"--model\") ?? \"unknown\",\n\t\t\t\t\t\t\t\t\t});\n\t\t\t\t\t\t\t\t\treturn createLiveProviderExecutor(profile, createOpenAiCompatibleProvider(profile), {\n\t\t\t\t\t\t\t\t\t\tmaximumCostUsd: option(args, \"--max-cost-usd\")\n\t\t\t\t\t\t\t\t\t\t\t? Number(option(args, \"--max-cost-usd\"))\n\t\t\t\t\t\t\t\t\t\t\t: undefined,\n\t\t\t\t\t\t\t\t\t\tmaximumModelCalls: option(args, \"--max-model-calls\")\n\t\t\t\t\t\t\t\t\t\t\t? Number(option(args, \"--max-model-calls\"))\n\t\t\t\t\t\t\t\t\t\t\t: undefined,\n\t\t\t\t\t\t\t\t\t\tmaximumWallTimeMs: option(args, \"--max-wall-time-ms\")\n\t\t\t\t\t\t\t\t\t\t\t? Number(option(args, \"--max-wall-time-ms\"))\n\t\t\t\t\t\t\t\t\t\t\t: undefined,\n\t\t\t\t\t\t\t\t\t});\n\t\t\t\t\t\t\t\t})()\n\t\t\t\t\t\t\t: undefined,\n\t\t\t\t});\n\t\t\t\tawait writeArtifact(evaluationRoot(), artifact);\n\t\t\t\trepetitions.push(artifact);\n\t\t\t\tartifacts.push(artifact);\n\t\t\t}\n\t\t\tif (repeat > 1) {\n\t\t\t\tconst first = repetitions[0]!;\n\t\t\t\tconst stability = aggregateStability(repetitions);\n\t\t\t\tconst { artifactHash: _artifactHash, ...unsigned } = first;\n\t\t\t\tconst grouped = createArtifact({\n\t\t\t\t\t...unsigned,\n\t\t\t\t\tstability,\n\t\t\t\t\tprovenance: {\n\t\t\t\t\t\t...first.provenance,\n\t\t\t\t\t\tsourceRunIds: repetitions.map((artifact) => artifact.run.evaluationRunId),\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t\tawait writeArtifact(evaluationRoot(), grouped);\n\t\t\t\tartifacts.push(grouped);\n\t\t\t}\n\t\t}\n\t\tprint(\n\t\t\t{\n\t\t\t\tartifacts: artifacts.map((artifact) => ({\n\t\t\t\t\tartifactId: artifact.artifactHash,\n\t\t\t\t\tscenarioId: artifact.scenario.scenarioId,\n\t\t\t\t\tverdict: artifact.verdict,\n\t\t\t\t})),\n\t\t\t},\n\t\t\tjson,\n\t\t);\n\t\t// Process exit must reflect the artifact verdict so CI cannot be fooled\n\t\t// by a completed-but-failing evaluation run.\n\t\tconst worstVerdict = artifacts.reduce<EvaluationVerdict>((worst, artifact) => {\n\t\t\tconst severity: Record<EvaluationVerdict, number> = { pass: 0, fail: 1, invalid: 2, cancelled: 3 };\n\t\t\treturn severity[artifact.verdict] > severity[worst] ? artifact.verdict : worst;\n\t\t}, \"pass\");\n\t\tprocess.exitCode = verdictExitCode(worstVerdict);\n\t\treturn true;\n\t}\n\tif (command === \"compare-agents\") {\n\t\tconst scenarioId = args[2] ?? \"\";\n\t\tconst scenario = discovered.scenarios.find((candidate) => candidate.scenarioId === scenarioId);\n\t\tif (!scenario) throw new Error(`unknown evaluation scenario: ${scenarioId}`);\n\t\tif (option(args, \"--single-agent\") === undefined || option(args, \"--orchestration\") !== \"cavecrew\")\n\t\t\tthrow new Error(\"compare-agents requires --single-agent and --orchestration cavecrew\");\n\t\tconst singleAgent = {\n\t\t\texecute: async () => ({\n\t\t\t\tevents: [{ eventId: randomUUID(), type: \"single-agent.completed\", timestamp: new Date().toISOString() }],\n\t\t\t}),\n\t\t};\n\t\tconst cavecrew = {\n\t\t\texecute: async ({ workspaceRoot }: { workspaceRoot?: string }) => {\n\t\t\t\tif (!workspaceRoot) throw new Error(\"Cavecrew comparison requires a sandbox workspace\");\n\t\t\t\tconst result = await runCavecrew({\n\t\t\t\t\tobjective: scenario.task.prompt,\n\t\t\t\t\tassignments: [scenario.task.prompt],\n\t\t\t\t\tcwd: workspaceRoot,\n\t\t\t\t\tstorageDir: join(workspaceRoot, \".cavecrew\"),\n\t\t\t\t\tfixtures: {\n\t\t\t\t\t\tinvestigator: async () => ({\n\t\t\t\t\t\t\tobjective: scenario.task.prompt,\n\t\t\t\t\t\t\tsummary: \"fixture investigation\",\n\t\t\t\t\t\t\tflow: [],\n\t\t\t\t\t\t\trootCauses: [],\n\t\t\t\t\t\t\trelevantFiles: [],\n\t\t\t\t\t\t\tunknowns: [],\n\t\t\t\t\t\t\trecommendedNextAgent: \"planner\",\n\t\t\t\t\t\t}),\n\t\t\t\t\t\tplanner: async () => ({\n\t\t\t\t\t\t\tscope: [],\n\t\t\t\t\t\t\tnonGoals: [],\n\t\t\t\t\t\t\timplementationSteps: [],\n\t\t\t\t\t\t\tinvariants: [],\n\t\t\t\t\t\t\tfocusedTests: [],\n\t\t\t\t\t\t\tacceptanceCriteria: [],\n\t\t\t\t\t\t\trollbackExpectations: [],\n\t\t\t\t\t\t}),\n\t\t\t\t\t\tbuilder: async () => ({\n\t\t\t\t\t\t\tedits: [],\n\t\t\t\t\t\t\toutput: {\n\t\t\t\t\t\t\t\tobjective: scenario.task.prompt,\n\t\t\t\t\t\t\t\tstatus: \"implemented\",\n\t\t\t\t\t\t\t\tfilesChanged: [],\n\t\t\t\t\t\t\t\tvalidations: [\"fixture\"],\n\t\t\t\t\t\t\t\trollbackState: \"confirmed\",\n\t\t\t\t\t\t\t\tremainingRisks: [],\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t}),\n\t\t\t\t\t\treviewer: async () => ({ verdict: \"pass\", findings: [], missingTests: [], acceptanceGaps: [] }),\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t\treturn {\n\t\t\t\t\tevents: [\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\teventId: randomUUID(),\n\t\t\t\t\t\t\ttype: \"cavecrew.completed\",\n\t\t\t\t\t\t\ttimestamp: new Date().toISOString(),\n\t\t\t\t\t\t\tdetails: { state: result.state, childRunCount: result.childRunIds.length },\n\t\t\t\t\t\t},\n\t\t\t\t\t],\n\t\t\t\t};\n\t\t\t},\n\t\t};\n\t\tconst result = await compareAgents({\n\t\t\tscenario,\n\t\t\tsingleAgent,\n\t\t\tcavecrew,\n\t\t\tmode: \"sandbox\",\n\t\t\torderSeed: Date.now(),\n\t\t});\n\t\tawait writeArtifact(evaluationRoot(), result.singleAgent);\n\t\tawait writeArtifact(evaluationRoot(), result.cavecrew);\n\t\tprint(result.comparison, json);\n\t\treturn true;\n\t}\n\tif (command === \"compare\") {\n\t\tconst baseline = await readArtifact(evaluationRoot(), args[2] ?? \"\");\n\t\tconst candidate = await readArtifact(evaluationRoot(), args[3] ?? \"\");\n\t\tprint(compareArtifacts(baseline, candidate), json);\n\t\treturn true;\n\t}\n\tif (command === \"replay\" || command === \"rescore\") {\n\t\tconst artifact = await readArtifact(evaluationRoot(), args[2] ?? \"\");\n\t\tconst replayed = command === \"replay\" ? replayArtifact(artifact) : rescoreArtifact(artifact);\n\t\tawait writeArtifact(evaluationRoot(), replayed);\n\t\tprint({ artifactId: replayed.artifactHash, originalArtifactId: artifact.artifactHash, artifact: replayed }, json);\n\t\treturn true;\n\t}\n\tif (command === \"baseline\") return handleBaselineCommand(args.slice(2), json);\n\tif (command === \"gate\") return handleGateCommand(args.slice(2), json);\n\tthrow new Error(`unknown eval command: ${command}`);\n}\n\nasync function handleBaselineCommand(args: string[], json: boolean): Promise<boolean> {\n\tconst subcommand = args[0] ?? \"list\";\n\tif (subcommand === \"list\") {\n\t\tprint(await listBaselines(baselineRoot()), json);\n\t\treturn true;\n\t}\n\tif (subcommand === \"verify\") {\n\t\tconst baselineId = args[1] ?? \"\";\n\t\tprint({ baselineId, valid: await verifyBaseline(baselineRoot(), baselineId) }, json);\n\t\treturn true;\n\t}\n\tif (subcommand === \"create\") {\n\t\tconst artifactIds = args.slice(1).filter((arg) => !arg.startsWith(\"--\"));\n\t\tconst artifacts = await Promise.all(artifactIds.map((id) => readArtifact(evaluationRoot(), id)));\n\t\tconst first = artifacts[0];\n\t\tif (!first) throw new Error(\"baseline create requires artifact ids\");\n\t\tconst baseline = await createBaseline(\n\t\t\tbaselineRoot(),\n\t\t\t{\n\t\t\t\tartifactIds,\n\t\t\t\tcreatedAt: new Date().toISOString(),\n\t\t\t\tcandidate: first.candidate,\n\t\t\t\tpackId: option(args, \"--pack\") ?? \"custom\",\n\t\t\t\tpackVersion: option(args, \"--pack-version\") ?? \"1\",\n\t\t\t},\n\t\t\tartifacts,\n\t\t);\n\t\tprint(baseline, json);\n\t\treturn true;\n\t}\n\tthrow new Error(`unknown baseline command: ${subcommand}`);\n}\n\nasync function handleGateCommand(args: string[], json: boolean): Promise<boolean> {\n\tif (args[0] !== \"check\") throw new Error(`unknown gate command: ${args[0] ?? \"\"}`);\n\tconst artifactIds = args.slice(1).filter((arg) => !arg.startsWith(\"--\"));\n\tconst artifacts = await Promise.all(artifactIds.map((id) => readArtifact(evaluationRoot(), id)));\n\tif (artifacts.some((artifact) => !verifyArtifact(artifact))) throw new Error(\"invalid evaluation artifact\");\n\tconst gate: EvaluationReleaseGate = {\n\t\tgateId: \"default\",\n\t\tscenarioPack: option(args, \"--pack\") ?? \"custom\",\n\t\tbaselineId: option(args, \"--baseline\") ?? \"required\",\n\t\trequiredScenarioPasses: artifacts.map((artifact) => artifact.scenario.scenarioId),\n\t\tforbiddenRegressions: [],\n\t\tmaximumCriticalSafetyFailures: 0,\n\t\tmaximumHighSafetyFailures: 0,\n\t\tflakinessPolicy: { rejectNewFlakiness: true, minimumRepetitions: 1 },\n\t};\n\tconst result = checkReleaseGate(gate, artifacts, []);\n\tprint(result, json);\n\tif (!result.passed) process.exitCode = 1;\n\treturn true;\n}\n"]}