{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evolve/evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAG7D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAsB,MAAM,4BAA4B,CAAC;AAClG,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAEvE,OAAO,KAAK,EAAE,qBAAqB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEnE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAK/D,QAAA,MAAM,mBAAmB,mEAAoE,CAAC;AAE9F,MAAM,MAAM,0BAA0B,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;AA2B9E,MAAM,WAAW,yBAAyB;IACzC,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,0BAA0B,CAAC;IACpC,UAAU,EAAE,cAAc,CAAC;IAC3B,iBAAiB,EAAE,cAAc,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC;CACjB;AA2BD,wBAAsB,qBAAqB,CAAC,OAAO,EAAE;IACpD,KAAK,EAAE,QAAQ,CAAC;IAChB,IAAI,EAAE,qBAAqB,CAAC;IAC5B,QAAQ,EAAE,QAAQ,CAAC;IACnB,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC;IACnD,2EAA2E;IAC3E,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;IACxC,MAAM,CAAC,EAAE,0BAA0B,CAAC;IACpC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,MAAM,EAAE,WAAW,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACrB,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAyGrC","sourcesContent":["import type { ThinkingLevel } from \"@ch1nyzzz/pi-agent-core\";\nimport { StringEnum } from \"@ch1nyzzz/pi-ai\";\nimport { Type } from \"typebox\";\nimport type { EvoPaths } from \"../paths.ts\";\nimport { attachProposalArtifact, proposalApproval } from \"../proposal.ts\";\nimport type { EvidenceCorpus } from \"../reflect/evidence.ts\";\nimport type { ModelRunner, ModelRunResult, ModelRunSubmission } from \"../reflect/model-runner.ts\";\nimport type { CounterfactualReplayResult } from \"../reflect/replay.ts\";\nimport { recordModelUsage } from \"../reflect/usage.ts\";\nimport type { EvolutionResearchPlan, Proposal } from \"../types.ts\";\nimport { readEvolutionWorkflow } from \"./config.ts\";\nimport type { MaterializedCorpus } from \"./research-corpus.ts\";\n\n/** Read-only tool baseline for any role that receives the corpus index. */\nconst EVIDENCE_READER_TOOLS = [\"read\", \"grep\", \"find\", \"ls\"] as const;\n\nconst EVALUATION_VERDICTS = [\"verified\", \"needs-evidence\", \"unsupported\", \"invalid\"] as const;\n\nexport type EvolutionEvaluationVerdict = (typeof EVALUATION_VERDICTS)[number];\n\ninterface VerdictSubmission {\n\tverdict: EvolutionEvaluationVerdict;\n\tsummary: string;\n\tfindings: Array<{ title: string; detail: string }>;\n}\n\n/**\n * The verdict travels through a schema-validated tool call, never through text the\n * orchestrator would have to parse. Values outside the enum are unrepresentable.\n */\nconst VERDICT_SUBMISSION: Omit<ModelRunSubmission, \"description\"> = {\n\ttoolName: \"submit_verdict\",\n\tparameters: Type.Object({\n\t\tverdict: StringEnum(EVALUATION_VERDICTS),\n\t\tsummary: Type.String({ minLength: 1, maxLength: 4_000 }),\n\t\tfindings: Type.Array(\n\t\t\tType.Object({\n\t\t\t\ttitle: Type.String({ minLength: 1, maxLength: 200 }),\n\t\t\t\tdetail: Type.String({ minLength: 1, maxLength: 4_000 }),\n\t\t\t}),\n\t\t\t{ maxItems: 20 },\n\t\t),\n\t}),\n};\n\nexport interface EvolutionEvaluationResult {\n\tproposal: Proposal;\n\tverdict: EvolutionEvaluationVerdict;\n\tevaluation: ModelRunResult;\n\tadversarialReview: ModelRunResult;\n\tmarkdown: string;\n}\n\nfunction submittedVerdict(run: ModelRunResult): VerdictSubmission {\n\treturn run.submission as VerdictSubmission;\n}\n\nfunction worst(left: EvolutionEvaluationVerdict, right: EvolutionEvaluationVerdict): EvolutionEvaluationVerdict {\n\tconst rank: Record<EvolutionEvaluationVerdict, number> = {\n\t\tverified: 0,\n\t\t\"needs-evidence\": 1,\n\t\tunsupported: 2,\n\t\tinvalid: 3,\n\t};\n\treturn rank[left] >= rank[right] ? left : right;\n}\n\nfunction renderPassMarkdown(run: ModelRunResult): string {\n\tconst { verdict, summary, findings } = submittedVerdict(run);\n\treturn [\n\t\t`Verdict: ${verdict}`,\n\t\t\"\",\n\t\tsummary.trim(),\n\t\t...(findings.length > 0 ? [\"\", ...findings.map((finding) => `- **${finding.title}**: ${finding.detail}`)] : []),\n\t\t...(run.text.trim() ? [\"\", run.text.trim()] : []),\n\t].join(\"\\n\");\n}\n\nexport async function runEvolutionEvaluator(options: {\n\tpaths: EvoPaths;\n\tplan: EvolutionResearchPlan;\n\tproposal: Proposal;\n\tcorpus: Pick<EvidenceCorpus, \"text\" | \"truncated\">;\n\t/** On-disk corpus tree; when present the prompt carries only its index. */\n\tmaterializedCorpus?: MaterializedCorpus;\n\treplay?: CounterfactualReplayResult;\n\t/**\n\t * Recommended (non-required) evidence profiles the harness deferred to an\n\t * explicit user decision. Their absence bounds claims but is not a defect.\n\t */\n\tdeferredProfiles?: readonly string[];\n\trunner: ModelRunner;\n\tcwd: string;\n\tagentDir?: string;\n\tmodel: string;\n\tthinkingLevel?: ThinkingLevel;\n\tactivePreferences?: string;\n\t/**\n\t * Proposal review artifacts are immutable per revision; re-evaluations (evidence\n\t * resumption) keep their markdown in the run directory instead. Default true.\n\t */\n\tattachArtifact?: boolean;\n\tsignal?: AbortSignal;\n}): Promise<EvolutionEvaluationResult> {\n\tconst workflow = await readEvolutionWorkflow(options.paths);\n\tconst evidence = [\n\t\t...(options.activePreferences\n\t\t\t? [\"<active_preferences>\", options.activePreferences, \"</active_preferences>\", \"\"]\n\t\t\t: []),\n\t\t\"<workflow>\",\n\t\tworkflow,\n\t\t\"</workflow>\",\n\t\t\"\",\n\t\t\"<frozen_plan_and_experiment>\",\n\t\tJSON.stringify(options.plan, undefined, \"\\t\"),\n\t\t\"</frozen_plan_and_experiment>\",\n\t\t\"\",\n\t\t\"<proposal>\",\n\t\tJSON.stringify(options.proposal, undefined, \"\\t\"),\n\t\t\"</proposal>\",\n\t\t\"\",\n\t\t...(options.materializedCorpus\n\t\t\t? [\n\t\t\t\t\t`<evidence_corpus_index truncated=\"${String(options.corpus.truncated)}\">`,\n\t\t\t\t\toptions.materializedCorpus.indexText,\n\t\t\t\t\t\"</evidence_corpus_index>\",\n\t\t\t\t]\n\t\t\t: [\n\t\t\t\t\t`<evidence_corpus truncated=\"${String(options.corpus.truncated)}\">`,\n\t\t\t\t\toptions.corpus.text,\n\t\t\t\t\t\"</evidence_corpus>\",\n\t\t\t\t]),\n\t\t...(options.replay ? [\"\", \"<paired_replay>\", options.replay.markdown, \"</paired_replay>\"] : []),\n\t\t...(options.deferredProfiles && options.deferredProfiles.length > 0\n\t\t\t? [\n\t\t\t\t\t\"\",\n\t\t\t\t\t\"<deferred_recommended_evidence>\",\n\t\t\t\t\t`The frozen plan marks these evidence profiles as recommended, not required: ${options.deferredProfiles.join(\", \")}. The harness deferred their execution to an explicit user decision that happens after this evaluation. Their absence is not a candidate or experiment defect and must not lower the verdict on its own; state instead which claims remain unproven without them.`,\n\t\t\t\t\t\"</deferred_recommended_evidence>\",\n\t\t\t\t]\n\t\t\t: []),\n\t].join(\"\\n\");\n\tconst common = {\n\t\tcwd: options.cwd,\n\t\t...(options.agentDir ? { agentDir: options.agentDir } : {}),\n\t\tmodel: options.model,\n\t\tthinkingLevel: options.thinkingLevel ?? (\"xhigh\" as const),\n\t\t// Index mode requires read access so the evaluator can verify cited evidence itself.\n\t\t...(options.materializedCorpus ? { tools: [...EVIDENCE_READER_TOOLS] } : {}),\n\t\t...(options.signal ? { signal: options.signal } : {}),\n\t};\n\tconst evaluation = await options.runner.run({\n\t\t...common,\n\t\tsystemPrompt:\n\t\t\t\"You are Evo-Pi's independent Evaluator. Evaluate the candidate only against the frozen experiment and supplied primary evidence. You cannot modify or activate it.\",\n\t\tprompt: [\n\t\t\t\"Assess deterministic validation, the frozen evidenceStrategy, the stated metrics and minimum effects, replay limitations, evidence sufficiency, compliance with active user preferences, and whether shadow or Canary execution is required. Treat a required evidence profile as unexecuted unless a supplied artifact demonstrates that exact execution boundary. Active preferences are evaluation criteria, never permission to weaken release or safety checks. A valid candidate that still requires replay, shadow, canary, provider, or minimum-sample evidence is needs-evidence, not unsupported. Use unsupported only when sufficient executed comparative evidence disproves the frozen thresholds; use invalid for a broken candidate or experiment. A content-addressed component is represented by a data proposal selecting its targetAbi artifact, which is not a kind mismatch. Do not claim unexecuted behavior. Deliver your assessment by calling the submit_verdict tool exactly once.\",\n\t\t\t\"\",\n\t\t\tevidence,\n\t\t].join(\"\\n\"),\n\t\tsubmission: {\n\t\t\t...VERDICT_SUBMISSION,\n\t\t\tdescription: \"Deliver the final evaluation verdict with a summary and concrete findings.\",\n\t\t},\n\t});\n\tawait recordModelUsage(options.paths, \"evaluator\", evaluation);\n\tconst adversarialReview = await options.runner.run({\n\t\t...common,\n\t\tsystemPrompt:\n\t\t\t\"You are Evo-Pi's adversarial evaluation pass in a fresh context. Try to falsify the candidate's apparent benefit. You cannot modify or activate it.\",\n\t\tprompt: [\n\t\t\t\"Look for tailored tests, baseline mismatch, changed task mix, overfitting, hidden regressions, new capability or complexity, unjustified not-applicable evidence classifications, invalid research citations, and overclaims from generate-only replay. Missing required future trial evidence is needs-evidence; reserve unsupported for sufficient negative evidence and invalid for a broken candidate or experiment. Deliver your assessment by calling the submit_verdict tool exactly once.\",\n\t\t\t\"\",\n\t\t\tevidence,\n\t\t].join(\"\\n\"),\n\t\tsubmission: {\n\t\t\t...VERDICT_SUBMISSION,\n\t\t\tdescription: \"Deliver the adversarial review verdict with a summary and concrete findings.\",\n\t\t},\n\t});\n\tawait recordModelUsage(options.paths, \"adversarial-review\", adversarialReview);\n\tconst finalVerdict = worst(submittedVerdict(evaluation).verdict, submittedVerdict(adversarialReview).verdict);\n\tconst markdown = [\n\t\t\"# Evolution evaluation\",\n\t\t\"\",\n\t\t\"## Evaluation\",\n\t\t\"\",\n\t\trenderPassMarkdown(evaluation),\n\t\t\"\",\n\t\t\"## Adversarial review\",\n\t\t\"\",\n\t\trenderPassMarkdown(adversarialReview),\n\t\t\"\",\n\t\t`## Combined verdict: ${finalVerdict}`,\n\t\t\"\",\n\t].join(\"\\n\");\n\tconst proposal =\n\t\toptions.attachArtifact === false\n\t\t\t? options.proposal\n\t\t\t: await attachProposalArtifact({\n\t\t\t\t\tpaths: options.paths,\n\t\t\t\t\tproposalId: options.proposal.id,\n\t\t\t\t\texpected: proposalApproval(options.proposal),\n\t\t\t\t\tkind: \"review\",\n\t\t\t\t\tcontent: markdown,\n\t\t\t\t\tallowedStatuses: [\"pending\", \"deferred\"],\n\t\t\t\t});\n\treturn { proposal, verdict: finalVerdict, evaluation, adversarialReview, markdown };\n}\n"]}