{"version":3,"file":"comparison.d.ts","sourceRoot":"","sources":["../../../src/core/evaluation/comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAEzG,wBAAgB,gBAAgB,CAC/B,QAAQ,EAAE,kBAAkB,EAC5B,SAAS,EAAE,kBAAkB,GAC3B,wBAAwB,CAuB1B;AAED,wBAAgB,sBAAsB,CACrC,QAAQ,EAAE,kBAAkB,EAC5B,SAAS,EAAE,kBAAkB,EAC7B,KAAK,EAAE,wBAAwB,EAAE,GAC/B,MAAM,EAAE,CAsBV;AAED,wBAAgB,iBAAiB,CAAC,WAAW,EAAE,wBAAwB,EAAE,8CAExE","sourcesContent":["import { comparePairedResults } from \"./statistics.js\";\nimport type { EvaluationArtifact, EvaluationRegressionRule, PairwiseEvaluationResult } from \"./types.js\";\n\nexport function compareArtifacts(\n\tbaseline: EvaluationArtifact,\n\tcandidate: EvaluationArtifact,\n): PairwiseEvaluationResult {\n\tif (\n\t\tbaseline.scenario.scenarioId !== candidate.scenario.scenarioId ||\n\t\tbaseline.scenario.scenarioVersion !== candidate.scenario.scenarioVersion ||\n\t\tbaseline.scenario.scenarioContentHash !== candidate.scenario.scenarioContentHash\n\t)\n\t\tthrow new Error(\"paired comparison requires matching scenario identity\");\n\tconst baselinePassed = baseline.verdict === \"pass\" && !hasSafetyFailure(baseline);\n\tconst candidatePassed = candidate.verdict === \"pass\" && !hasSafetyFailure(candidate);\n\tconst baselineMetrics = new Map(baseline.metrics.map((metric) => [metric.metricId, metric.value]));\n\tconst metricDeltas: Record<string, number | undefined> = {};\n\tfor (const metric of candidate.metrics) {\n\t\tconst baselineValue = baselineMetrics.get(metric.metricId);\n\t\tmetricDeltas[metric.metricId] =\n\t\t\tbaselineValue === undefined || metric.value === undefined ? undefined : metric.value - baselineValue;\n\t}\n\treturn {\n\t\tscenarioId: candidate.scenario.scenarioId,\n\t\tbaselineRunId: baseline.run.evaluationRunId,\n\t\tcandidateRunId: candidate.run.evaluationRunId,\n\t\tdeterministicComparison: { baselinePassed, candidatePassed },\n\t\tmetricDeltas,\n\t};\n}\n\nexport function compareRegressionRules(\n\tbaseline: EvaluationArtifact,\n\tcandidate: EvaluationArtifact,\n\trules: EvaluationRegressionRule[],\n): string[] {\n\tconst baselineMetrics = new Map(baseline.metrics.map((metric) => [metric.metricId, metric.value]));\n\tconst candidateMetrics = new Map(candidate.metrics.map((metric) => [metric.metricId, metric.value]));\n\tconst regressions: string[] = [];\n\tfor (const rule of rules) {\n\t\tconst before = baselineMetrics.get(rule.metricId);\n\t\tconst after = candidateMetrics.get(rule.metricId);\n\t\tif (before === undefined || after === undefined) continue;\n\t\tif (\n\t\t\trule.maximumRelativeIncrease !== undefined &&\n\t\t\tbefore !== 0 &&\n\t\t\t(after - before) / Math.abs(before) > rule.maximumRelativeIncrease\n\t\t)\n\t\t\tregressions.push(`${rule.metricId}: relative increase ${(after - before) / Math.abs(before)}`);\n\t\tif (rule.minimumAbsoluteDecrease !== undefined && before - after > rule.minimumAbsoluteDecrease)\n\t\t\tregressions.push(`${rule.metricId}: decrease ${before - after}`);\n\t}\n\tif (baselinePassed(baseline) && !candidatePassed(candidate))\n\t\tregressions.push(\"correctness: candidate no longer passes\");\n\tif (!hasSafetyFailure(baseline) && hasSafetyFailure(candidate))\n\t\tregressions.push(\"safety: new critical or high failure\");\n\treturn regressions;\n}\n\nexport function aggregatePairwise(comparisons: PairwiseEvaluationResult[]) {\n\treturn comparePairedResults(comparisons);\n}\n\nfunction baselinePassed(artifact: EvaluationArtifact): boolean {\n\treturn artifact.verdict === \"pass\" && !hasSafetyFailure(artifact);\n}\n\nfunction candidatePassed(artifact: EvaluationArtifact): boolean {\n\treturn artifact.verdict === \"pass\" && !hasSafetyFailure(artifact);\n}\n\nfunction hasSafetyFailure(artifact: EvaluationArtifact): boolean {\n\treturn artifact.assertions.some(\n\t\t(assertion) =>\n\t\t\t(assertion.severity === \"critical\" || assertion.severity === \"high\") && assertion.status !== \"pass\",\n\t);\n}\n"]}