{"version":3,"file":"compare-agents.d.ts","sourceRoot":"","sources":["../../../src/core/evaluation/compare-agents.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,kBAAkB,EAAiB,MAAM,aAAa,CAAC;AACrE,OAAO,KAAK,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEnG,MAAM,WAAW,yBAAyB;IACzC,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,WAAW,EAAE,kBAAkB,CAAC;IAChC,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,MAAM,CAAC;IACrC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB;AAmBD,wBAAsB,aAAa,CAAC,KAAK,EAAE,yBAAyB,GAAG,OAAO,CAAC;IAC9E,UAAU,EAAE,wBAAwB,CAAC;IACrC,WAAW,EAAE,kBAAkB,CAAC;IAChC,QAAQ,EAAE,kBAAkB,CAAC;CAC7B,CAAC,CAwDD","sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { type EvaluationExecutor, runEvaluation } from \"./runner.js\";\nimport type { CavecrewComparisonResult, EvaluationArtifact, EvaluationScenario } from \"./types.js\";\n\nexport interface PairedAgentExecutionInput {\n\tscenario: EvaluationScenario;\n\tsingleAgent: EvaluationExecutor;\n\tcavecrew: EvaluationExecutor;\n\tmode: \"fixture\" | \"sandbox\" | \"live\";\n\torderSeed?: number;\n\tproviderProfile?: string;\n}\n\nfunction metric(artifact: EvaluationArtifact, ids: string[]): number {\n\treturn ids.reduce((total, id) => total + (artifact.metrics.find((item) => item.metricId === id)?.value ?? 0), 0);\n}\n\nfunction safety(artifact: EvaluationArtifact): number {\n\treturn artifact.assertions.some(\n\t\t(assertion) =>\n\t\t\t(assertion.severity === \"critical\" || assertion.severity === \"high\") && assertion.status !== \"pass\",\n\t)\n\t\t? 0\n\t\t: 1;\n}\n\nfunction correctness(artifact: EvaluationArtifact): number {\n\treturn artifact.verdict === \"pass\" ? 1 : 0;\n}\n\nexport async function compareAgents(input: PairedAgentExecutionInput): Promise<{\n\tcomparison: CavecrewComparisonResult;\n\tsingleAgent: EvaluationArtifact;\n\tcavecrew: EvaluationArtifact;\n}> {\n\tconst reverse = (input.orderSeed ?? 0) % 2 === 1;\n\tconst first = reverse ? \"cavecrew\" : \"single-agent\";\n\tconst run = async (candidate: \"single-agent\" | \"cavecrew\") =>\n\t\trunEvaluation(input.scenario, {\n\t\t\tmode: input.mode,\n\t\t\texecutor: candidate === \"single-agent\" ? input.singleAgent : input.cavecrew,\n\t\t\tcandidate: {\n\t\t\t\tproviderProfile: `${input.providerProfile ?? \"fixture\"}:${candidate}`,\n\t\t\t\tprovider: candidate,\n\t\t\t\tconfiguredModel: candidate,\n\t\t\t\tresolvedModel: candidate,\n\t\t\t\tseed: input.orderSeed,\n\t\t\t},\n\t\t});\n\tconst firstArtifact = await run(first);\n\tconst secondArtifact = await run(first === \"single-agent\" ? \"cavecrew\" : \"single-agent\");\n\tconst singleAgent = first === \"single-agent\" ? firstArtifact : secondArtifact;\n\tconst cavecrew = first === \"cavecrew\" ? firstArtifact : secondArtifact;\n\tif (singleAgent.scenario.scenarioContentHash !== cavecrew.scenario.scenarioContentHash)\n\t\tthrow new Error(\"paired execution scenario hashes differ\");\n\tconst correctnessDelta = correctness(cavecrew) - correctness(singleAgent);\n\tconst safetyDelta = safety(cavecrew) - safety(singleAgent);\n\tconst wallTimeDeltaMs = metric(cavecrew, [\"wall_time_ms\"]) - metric(singleAgent, [\"wall_time_ms\"]);\n\tconst modelCallDelta = metric(cavecrew, [\"model_calls\"]) - metric(singleAgent, [\"model_calls\"]);\n\tconst tokenDelta = metric(cavecrew, [\"tokens\", \"total_tokens\"]) - metric(singleAgent, [\"tokens\", \"total_tokens\"]);\n\tconst toolCallDelta = metric(cavecrew, [\"tool_calls\"]) - metric(singleAgent, [\"tool_calls\"]);\n\tconst retrievalDelta =\n\t\tmetric(cavecrew, [\"retrieval_relevance\", \"retrieval_usage\"]) -\n\t\tmetric(singleAgent, [\"retrieval_relevance\", \"retrieval_usage\"]);\n\tconst rollbackDelta = metric(cavecrew, [\"rollback_count\"]) - metric(singleAgent, [\"rollback_count\"]);\n\tconst deterministicWinner =\n\t\tsafetyDelta < 0 || correctnessDelta < 0\n\t\t\t? \"single_agent\"\n\t\t\t: safetyDelta > 0 || correctnessDelta > 0\n\t\t\t\t? \"cavecrew\"\n\t\t\t\t: \"tie\";\n\treturn {\n\t\tcomparison: {\n\t\t\tcomparisonId: `comparison-${randomUUID()}`,\n\t\t\tscenarioId: input.scenario.scenarioId,\n\t\t\tsingleAgentRunId: singleAgent.run.evaluationRunId,\n\t\t\tcavecrewRunId: cavecrew.run.evaluationRunId,\n\t\t\tcorrectnessDelta,\n\t\t\tsafetyDelta,\n\t\t\twallTimeDeltaMs,\n\t\t\tmodelCallDelta,\n\t\t\ttokenDelta,\n\t\t\ttoolCallDelta,\n\t\t\tretrievalDelta,\n\t\t\trollbackDelta,\n\t\t\tdeterministicWinner,\n\t\t},\n\t\tsingleAgent,\n\t\tcavecrew,\n\t};\n}\n"]}