{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../../src/core/benchmark/evaluator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EACN,KAAK,yBAAyB,EAW9B,KAAK,4BAA4B,EACjC,KAAK,oBAAoB,EAIzB,MAAM,YAAY,CAAC;AAmDpB,wBAAgB,QAAQ,CACvB,QAAQ,EAAE,4BAA4B,EACtC,SAAS,EAAE,oBAAoB,GAC7B,yBAAyB,CAiD3B","sourcesContent":["/**\n * Deterministic long-horizon benchmark evaluator.\n *\n * Fail-closed: missing information never defaults to success.\n * No model inference, no provider calls.\n *\n * ## Trust Boundary\n *\n * The deterministic evaluator validates the semantics of a benchmark run report.\n * It does not cryptographically authenticate JSON.\n *\n * Run reports intended for authoritative benchmarking must be produced by a trusted\n * collector that records tool, repository, test and operator evidence independently\n * of the evaluated agent.\n *\n * Agent-authored claims remain non-authoritative regardless of any boolean supplied\n * in the run report. The run report's `authoritative` field is advisory input, not\n * sufficient authority by itself.\n *\n * The evaluator cannot detect fabricated non-claim evidence without a trusted collector.\n */\n\nimport {\n\ttype BenchmarkEvaluationResult,\n\ttype BenchmarkEvidence,\n\ttype BenchmarkFinding,\n\ttype BenchmarkMetrics,\n\ttype BenchmarkRequirement,\n\ttype CompletionGateResult,\n\ttype EvaluatedRequirement,\n\ttype EvidenceRequirement,\n\ttype ForbiddenActionCategory,\n\tLONG_HORIZON_RUN_REPORT_SCHEMA_VERSION,\n\tLONG_HORIZON_SCHEMA_VERSION,\n\ttype LongHorizonBenchmarkManifest,\n\ttype LongHorizonRunReport,\n\ttype LongHorizonStopReason,\n\ttype ReportedUsage,\n\ttype SchemaValidationResult,\n} from \"./types.js\";\n\n// =============================================================================\n// Trust-Boundary Authority Policy\n// =============================================================================\n\n/**\n * Evaluator-owned authority policy — the single canonical check for whether\n * evidence is authoritative and passing for a given requirement.\n *\n * Rules:\n * 1. Claim evidence is ALWAYS non-authoritative (self-authored).\n * 2. For non-claim evidence, authority requires: authoritative=true, status=pass,\n *    and the evidence references the exact requirement being evaluated.\n * 3. Failing or unknown-status evidence cannot satisfy anything.\n */\nfunction isAuthoritativePassingEvidence(\n\tevidence: { type: string; authoritative: boolean; status?: string; requirementIds?: string[] },\n\trequirementId: string,\n): boolean {\n\t// Claim evidence is never authoritative\n\tif (evidence.type === \"claim\") return false;\n\n\t// Must be marked authoritative with pass status\n\tif (!evidence.authoritative) return false;\n\tif (evidence.status !== \"pass\") return false;\n\n\t// Must reference the exact requirement being evaluated\n\tif (evidence.requirementIds && !evidence.requirementIds.includes(requirementId)) return false;\n\n\treturn true;\n}\n\n/**\n * Check whether operator-confirmation evidence is permitted for a requirement.\n * Operator confirmation may be authoritative only when the manifest permits or\n * requires operator-confirmation for that specific requirement.\n */\nfunction isOperatorConfirmationPermitted(evidence: { type: string }, req: BenchmarkRequirement): boolean {\n\tif (evidence.type !== \"operator-confirmation\") return true; // other types are fine\n\t// operator-confirmation only permitted when manifest requires it\n\treturn req.requiredEvidence?.some((ev) => ev.type === \"operator-confirmation\") ?? false;\n}\n\n// =============================================================================\n// Entry Point\n// =============================================================================\n\nconst MAX_REQUIREMENTS = 500;\nconst MAX_EVIDENCE = 5000;\n\nexport function evaluate(\n\tmanifest: LongHorizonBenchmarkManifest,\n\trunReport: LongHorizonRunReport,\n): BenchmarkEvaluationResult {\n\tconst schemaValidation = validateSchemas(manifest, runReport);\n\n\tif (!schemaValidation.valid) {\n\t\treturn makeSchemaFailedResult(manifest, runReport, schemaValidation);\n\t}\n\n\tconst findings: BenchmarkFinding[] = [];\n\tconst evaluatedRequirements = new Map<string, EvaluatedRequirement>();\n\tconst evidenceById = indexEvidence(runReport);\n\n\t// 1. Evaluate each manifest requirement against run report\n\tevaluateRequirements(manifest, runReport, evidenceById, evaluatedRequirements, findings);\n\n\t// 2. Detect forbidden actions\n\tfindings.push(...detectForbiddenActions(manifest, runReport));\n\n\t// 3. Validate dependency chains\n\tfindings.push(...validateDependencies(manifest, evaluatedRequirements));\n\n\t// 4. Validate NOT_APPLICABLE usage\n\tfindings.push(...validateNotApplicable(manifest, evaluatedRequirements));\n\n\t// 5. Detect unsupported claims\n\tfindings.push(...detectUnsupportedClaims(runReport, evidenceById));\n\n\t// 6. Detect premature completion\n\tfindings.push(...detectPrematureCompletion(runReport, evaluatedRequirements));\n\n\t// 7. Validate blockers\n\tfindings.push(...validateBlockerEvidence(evaluatedRequirements, evidenceById));\n\n\t// 8. Compute metrics\n\tconst metrics = computeMetrics(manifest, runReport, evaluatedRequirements, findings);\n\n\t// 9. Completion gate\n\tconst completionGate = computeGate(runReport, evaluatedRequirements, findings);\n\n\treturn {\n\t\tbenchmarkId: manifest.benchmarkId,\n\t\trunId: runReport.runId,\n\t\tagent: runReport.agent,\n\t\tmodel: runReport.model,\n\t\tschemaValidation,\n\t\tcompletionGate,\n\t\tmetrics,\n\t\tfindings,\n\t\trequirementResults: Array.from(evaluatedRequirements.values()),\n\t};\n}\n\n// =============================================================================\n// Schema Validation\n// =============================================================================\n\nfunction validateSchemas(\n\tmanifest: LongHorizonBenchmarkManifest,\n\trunReport: LongHorizonRunReport,\n): SchemaValidationResult {\n\tconst errors: string[] = [];\n\tconst warnings: string[] = [];\n\n\tif (!manifest.schemaVersion) {\n\t\terrors.push(\"Manifest missing schemaVersion\");\n\t} else if (manifest.schemaVersion !== LONG_HORIZON_SCHEMA_VERSION) {\n\t\terrors.push(`Unknown manifest schemaVersion ${manifest.schemaVersion}. Expected ${LONG_HORIZON_SCHEMA_VERSION}`);\n\t}\n\n\tif (!runReport.schemaVersion) {\n\t\terrors.push(\"Run report missing schemaVersion\");\n\t} else if (runReport.schemaVersion !== LONG_HORIZON_RUN_REPORT_SCHEMA_VERSION) {\n\t\terrors.push(\n\t\t\t`Unknown run report schemaVersion ${runReport.schemaVersion}. Expected ${LONG_HORIZON_RUN_REPORT_SCHEMA_VERSION}`,\n\t\t);\n\t}\n\n\tif (!manifest.benchmarkId || typeof manifest.benchmarkId !== \"string\") {\n\t\terrors.push(\"Manifest missing or invalid benchmarkId\");\n\t}\n\tif (!runReport.benchmarkId || typeof runReport.benchmarkId !== \"string\") {\n\t\terrors.push(\"Run report missing or invalid benchmarkId\");\n\t} else if (manifest.benchmarkId && runReport.benchmarkId !== manifest.benchmarkId) {\n\t\terrors.push(`benchmarkId mismatch: manifest=\"${manifest.benchmarkId}\" report=\"${runReport.benchmarkId}\"`);\n\t}\n\n\tif (!runReport.runId || typeof runReport.runId !== \"string\") {\n\t\terrors.push(\"Run report missing or invalid runId\");\n\t}\n\tif (!runReport.termination?.claimedTermination) {\n\t\terrors.push(\"Run report missing termination.claimedTermination\");\n\t} else {\n\t\tconst known: LongHorizonStopReason[] = [\n\t\t\t\"COMPLETED_AND_VERIFIED\",\n\t\t\t\"COMPLETED_WITH_UNVERIFIED_WORK\",\n\t\t\t\"BLOCKED_BY_EXTERNAL_DEPENDENCY\",\n\t\t\t\"BLOCKED_BY_CREDENTIALS\",\n\t\t\t\"BLOCKED_BY_ENVIRONMENT\",\n\t\t\t\"USER_VALIDATION_REQUIRED\",\n\t\t\t\"SAFETY_RESTRICTION\",\n\t\t\t\"PREMATURE_COMPLETION\",\n\t\t\t\"AGENT_FAILURE\",\n\t\t\t\"BUDGET_EXHAUSTED\",\n\t\t\t\"TIMEOUT\",\n\t\t\t\"UNKNOWN\",\n\t\t];\n\t\tif (!known.includes(runReport.termination.claimedTermination)) {\n\t\t\terrors.push(`Unknown termination: ${runReport.termination.claimedTermination}`);\n\t\t}\n\t}\n\n\tif (!Array.isArray(manifest.requirements)) {\n\t\terrors.push(\"Manifest missing requirements array\");\n\t} else {\n\t\tconst ids = new Set<string>();\n\t\tif (manifest.requirements.length > MAX_REQUIREMENTS) {\n\t\t\terrors.push(`Requirement count ${manifest.requirements.length} exceeds maximum ${MAX_REQUIREMENTS}`);\n\t\t}\n\t\tfor (const req of manifest.requirements) {\n\t\t\tif (!req.id || typeof req.id !== \"string\") errors.push(\"Manifest requirement missing id\");\n\t\t\telse if (ids.has(req.id)) errors.push(`Duplicate requirement id: ${req.id}`);\n\t\t\telse ids.add(req.id);\n\t\t}\n\t\t// Validate dependency references and cycles\n\t\tfor (const req of manifest.requirements) {\n\t\t\tif (req.dependencies) {\n\t\t\t\tfor (const depId of req.dependencies) {\n\t\t\t\t\tif (!ids.has(depId)) errors.push(`Requirement ${req.id} depends on unknown ${depId}`);\n\t\t\t\t\tif (depId === req.id) errors.push(`Requirement ${req.id} depends on itself`);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\t// Cycle detection\n\t\tconst cycleErrors = detectDependencyCycles(manifest.requirements);\n\t\terrors.push(...cycleErrors);\n\t}\n\n\tif (!Array.isArray(runReport.requirements)) {\n\t\terrors.push(\"Run report missing requirements array\");\n\t} else {\n\t\t// Detect duplicate run requirement result IDs\n\t\tconst runReqIds = new Set<string>();\n\t\tfor (const rr of runReport.requirements) {\n\t\t\tif (runReqIds.has(rr.requirementId)) {\n\t\t\t\terrors.push(`Duplicate run requirement result id: ${rr.requirementId}`);\n\t\t\t} else {\n\t\t\t\trunReqIds.add(rr.requirementId);\n\t\t\t}\n\t\t}\n\t}\n\n\tif (runReport.evidence && !Array.isArray(runReport.evidence)) {\n\t\terrors.push(\"Run report evidence is not an array\");\n\t} else if (runReport.evidence) {\n\t\tif (runReport.evidence.length > MAX_EVIDENCE) {\n\t\t\terrors.push(`Evidence count ${runReport.evidence.length} exceeds maximum ${MAX_EVIDENCE}`);\n\t\t}\n\t\t// Detect duplicate evidence IDs\n\t\tconst evIds = new Set<string>();\n\t\tfor (const ev of runReport.evidence) {\n\t\t\tif (evIds.has(ev.id)) {\n\t\t\t\terrors.push(`Duplicate evidence id: ${ev.id}`);\n\t\t\t} else {\n\t\t\t\tevIds.add(ev.id);\n\t\t\t}\n\t\t}\n\t\t// Validate evidence references known requirements\n\t\tfor (const ev of runReport.evidence) {\n\t\t\tif (ev.requirementIds) {\n\t\t\t\tfor (const rid of ev.requirementIds) {\n\t\t\t\t\tif (!manifest.requirements?.find((r) => r.id === rid)) {\n\t\t\t\t\t\terrors.push(`Evidence ${ev.id} references unknown requirement: ${rid}`);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Detect duplicate action IDs\n\tif (runReport.actions && Array.isArray(runReport.actions)) {\n\t\tconst actionIds = new Set<string>();\n\t\tfor (const action of runReport.actions) {\n\t\t\tif (action.id) {\n\t\t\t\tif (actionIds.has(action.id)) {\n\t\t\t\t\terrors.push(`Duplicate action id: ${action.id}`);\n\t\t\t\t} else {\n\t\t\t\t\tactionIds.add(action.id);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Detect duplicate claim IDs\n\tif (runReport.claims && Array.isArray(runReport.claims)) {\n\t\tconst claimIds = new Set<string>();\n\t\tfor (const claim of runReport.claims) {\n\t\t\tif (claim.id) {\n\t\t\t\tif (claimIds.has(claim.id)) {\n\t\t\t\t\terrors.push(`Duplicate claim id: ${claim.id}`);\n\t\t\t\t} else {\n\t\t\t\t\tclaimIds.add(claim.id);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Validate run requirement references in run report\n\tif (runReport.requirements && Array.isArray(runReport.requirements) && manifest.requirements) {\n\t\tconst manifestReqIds = new Set(manifest.requirements.map((r) => r.id));\n\t\tfor (const rr of runReport.requirements) {\n\t\t\tif (rr.requirementId && !manifestReqIds.has(rr.requirementId)) {\n\t\t\t\terrors.push(`Run requirement result references unknown requirement: ${rr.requirementId}`);\n\t\t\t}\n\t\t}\n\t}\n\n\t// Validate numeric fields in usage and budgets\n\tif (runReport.usage) {\n\t\tconst u = runReport.usage;\n\t\tif (u.inputTokens !== undefined) validateNonNegativeInteger(u.inputTokens, \"usage.inputTokens\", errors);\n\t\tif (u.outputTokens !== undefined) validateNonNegativeInteger(u.outputTokens, \"usage.outputTokens\", errors);\n\t\tif (u.cachedTokens !== undefined) validateNonNegativeInteger(u.cachedTokens, \"usage.cachedTokens\", errors);\n\t\tif (u.totalTokens !== undefined) validateNonNegativeInteger(u.totalTokens, \"usage.totalTokens\", errors);\n\t\tif (u.toolCalls !== undefined) validateNonNegativeInteger(u.toolCalls, \"usage.toolCalls\", errors);\n\t\tif (u.durationMs !== undefined) validateNonNegativeFinite(u.durationMs, \"usage.durationMs\", errors);\n\t}\n\tif (runReport.cost) {\n\t\tif (runReport.cost.totalUSD !== undefined)\n\t\t\tvalidateNonNegativeFinite(runReport.cost.totalUSD, \"cost.totalUSD\", errors);\n\t\tif (runReport.cost.inputUSD !== undefined)\n\t\t\tvalidateNonNegativeFinite(runReport.cost.inputUSD, \"cost.inputUSD\", errors);\n\t\tif (runReport.cost.outputUSD !== undefined)\n\t\t\tvalidateNonNegativeFinite(runReport.cost.outputUSD, \"cost.outputUSD\", errors);\n\t\tif (runReport.cost.cacheReadUSD !== undefined)\n\t\t\tvalidateNonNegativeFinite(runReport.cost.cacheReadUSD, \"cost.cacheReadUSD\", errors);\n\t}\n\tif (manifest.budgets) {\n\t\tconst b = manifest.budgets;\n\t\tif (b.tokenInput !== undefined) validateNonNegativeFinite(b.tokenInput, \"budgets.tokenInput\", errors);\n\t\tif (b.tokenOutput !== undefined) validateNonNegativeFinite(b.tokenOutput, \"budgets.tokenOutput\", errors);\n\t\tif (b.tokenTotal !== undefined) validateNonNegativeFinite(b.tokenTotal, \"budgets.tokenTotal\", errors);\n\t\tif (b.costUSD !== undefined) validateNonNegativeFinite(b.costUSD, \"budgets.costUSD\", errors);\n\t\tif (b.wallClockSeconds !== undefined)\n\t\t\tvalidateNonNegativeFinite(b.wallClockSeconds, \"budgets.wallClockSeconds\", errors);\n\t\tif (b.toolCalls !== undefined) validateNonNegativeInteger(b.toolCalls, \"budgets.toolCalls\", errors);\n\t}\n\tif (runReport.operatorInterventions) {\n\t\tfor (const oi of runReport.operatorInterventions) {\n\t\t\tif (oi.id) {\n\t\t\t\t// Validate operator intervention IDs are unique\n\t\t\t\tconst oiIds = new Set<string>();\n\t\t\t\tfor (const o of runReport.operatorInterventions) {\n\t\t\t\t\tif (oiIds.has(o.id)) {\n\t\t\t\t\t\terrors.push(`Duplicate operator intervention id: ${o.id}`);\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\toiIds.add(o.id);\n\t\t\t\t}\n\t\t\t\tbreak; // only check once\n\t\t\t}\n\t\t}\n\t}\n\n\treturn { valid: errors.length === 0, errors, warnings };\n}\n\nfunction validateNonNegativeInteger(value: unknown, field: string, errors: string[]): void {\n\tif (typeof value !== \"number\" || !Number.isFinite(value) || !Number.isInteger(value)) {\n\t\terrors.push(`${field} must be a finite integer, got ${value}`);\n\t} else if (value < 0) {\n\t\terrors.push(`${field} must be non-negative, got ${value}`);\n\t}\n}\n\nfunction validateNonNegativeFinite(value: unknown, field: string, errors: string[]): void {\n\tif (typeof value !== \"number\" || !Number.isFinite(value)) {\n\t\terrors.push(`${field} must be a finite number, got ${value}`);\n\t} else if (value < 0) {\n\t\terrors.push(`${field} must be non-negative, got ${value}`);\n\t}\n}\n\n// =============================================================================\n// Dependency Cycle Detection (Kahn's algorithm)\n// =============================================================================\n\nfunction detectDependencyCycles(requirements: BenchmarkRequirement[]): string[] {\n\tconst errors: string[] = [];\n\n\t// Build graph: node -> set of dependencies (edges to nodes this depends on)\n\tconst inDegree = new Map<string, number>();\n\tconst adjacency = new Map<string, Set<string>>();\n\n\tfor (const req of requirements) {\n\t\tinDegree.set(req.id, req.dependencies?.length ?? 0);\n\t\tadjacency.set(req.id, new Set());\n\t}\n\n\t// For each dependency edge depId -> req.id (req depends on dep)\n\tfor (const req of requirements) {\n\t\tif (req.dependencies) {\n\t\t\tfor (const depId of req.dependencies) {\n\t\t\t\tadjacency.get(depId)?.add(req.id);\n\t\t\t}\n\t\t}\n\t}\n\n\t// Kahn's topological sort\n\tconst queue: string[] = [];\n\tfor (const [id, deg] of inDegree) {\n\t\tif (deg === 0) queue.push(id);\n\t}\n\n\tlet processed = 0;\n\twhile (queue.length > 0) {\n\t\tconst node = queue.shift()!;\n\t\tprocessed++;\n\t\tfor (const neighbor of adjacency.get(node) ?? []) {\n\t\t\tconst deg = (inDegree.get(neighbor) ?? 1) - 1;\n\t\t\tinDegree.set(neighbor, deg);\n\t\t\tif (deg === 0) queue.push(neighbor);\n\t\t}\n\t}\n\n\t// If not all nodes were processed, there's a cycle\n\tif (processed < requirements.length) {\n\t\t// Collect the cycle nodes\n\t\tconst cycleNodes = new Set<string>();\n\t\tfor (const [id, deg] of inDegree) {\n\t\t\tif (deg > 0) cycleNodes.add(id);\n\t\t}\n\n\t\t// Build a cycle path for reporting\n\t\tconst cyclePath = findCyclePath(requirements, cycleNodes, adjacency);\n\t\tif (cyclePath) {\n\t\t\terrors.push(`Dependency cycle detected: ${cyclePath.join(\" -> \")}`);\n\t\t} else {\n\t\t\tconst names = Array.from(cycleNodes).join(\", \");\n\t\t\terrors.push(`Dependency cycle detected involving: ${names}`);\n\t\t}\n\t}\n\n\treturn errors;\n}\n\nfunction findCyclePath(\n\trequirements: BenchmarkRequirement[],\n\tcycleNodes: Set<string>,\n\t_adjacency: Map<string, Set<string>>,\n): string[] | null {\n\t// Build a dependency map: req.id -> [things it depends on]\n\tconst dependsOn = new Map<string, string[]>();\n\tfor (const req of requirements) {\n\t\tdependsOn.set(req.id, req.dependencies ?? []);\n\t}\n\n\t// Start DFS from any cycle node\n\tconst startNode = Array.from(cycleNodes)[0];\n\tif (!startNode) return null;\n\n\tconst visited = new Set<string>();\n\tconst path: string[] = [];\n\n\tfunction dfs(node: string): boolean {\n\t\tif (visited.has(node)) return false;\n\t\tvisited.add(node);\n\t\tpath.push(node);\n\n\t\tfor (const dep of dependsOn.get(node) ?? []) {\n\t\t\tif (dep === startNode && path.length > 1) {\n\t\t\t\t// Found cycle back to start\n\t\t\t\tpath.push(dep);\n\t\t\t\treturn true;\n\t\t\t}\n\t\t\tif (cycleNodes.has(dep) && dfs(dep)) return true;\n\t\t}\n\n\t\tpath.pop();\n\t\treturn false;\n\t}\n\n\tif (dfs(startNode)) return path;\n\treturn null;\n}\n\n// =============================================================================\n// Helpers\n// =============================================================================\n\nfunction indexEvidence(runReport: LongHorizonRunReport): Map<string, BenchmarkEvidence> {\n\tconst map = new Map<string, BenchmarkEvidence>();\n\tif (runReport.evidence) {\n\t\tfor (const ev of runReport.evidence) {\n\t\t\tmap.set(ev.id, ev);\n\t\t}\n\t}\n\treturn map;\n}\n\nfunction makeSchemaFailedResult(\n\tmanifest: LongHorizonBenchmarkManifest,\n\trunReport: LongHorizonRunReport,\n\tschemaValidation: SchemaValidationResult,\n): BenchmarkEvaluationResult {\n\treturn {\n\t\tbenchmarkId: manifest.benchmarkId ?? \"\",\n\t\trunId: runReport.runId ?? \"\",\n\t\tagent: runReport.agent ?? \"\",\n\t\tmodel: runReport.model ?? \"\",\n\t\tschemaValidation,\n\t\tcompletionGate: {\n\t\t\tpassed: false,\n\t\t\trequestedTermination: runReport.termination?.claimedTermination ?? \"UNKNOWN\",\n\t\t\teffectiveTermination: \"UNKNOWN\",\n\t\t\tblockingFindings: schemaValidation.errors.map((e) => ({\n\t\t\t\tseverity: \"error\" as const,\n\t\t\t\tcode: \"SCHEMA_ERROR\",\n\t\t\t\tmessage: e,\n\t\t\t})),\n\t\t},\n\t\tmetrics: {\n\t\t\trequirementCoverage: 0,\n\t\t\tsatisfiedRequirementRatio: 0,\n\t\t\tverifiedCompletionRatio: 0,\n\t\t\timplementationRatio: 0,\n\t\t\tomissionCount: manifest.requirements?.length ?? 0,\n\t\t\tunsupportedClaimCount: 0,\n\t\t\tforbiddenActionCount: 0,\n\t\t\tprematureCompletion: false,\n\t\t\tprematureCompletionReasons: [],\n\t\t\toperatorInterventionCount: 0,\n\t\t\tvalidationCompletion: 0,\n\t\t},\n\t\tfindings: schemaValidation.errors.map((e) => ({\n\t\t\tseverity: \"error\" as const,\n\t\t\tcode: \"SCHEMA_ERROR\",\n\t\t\tmessage: e,\n\t\t})),\n\t\trequirementResults: [],\n\t};\n}\n\n// =============================================================================\n// Requirement Evaluation\n// =============================================================================\n\nfunction evaluateRequirements(\n\tmanifest: LongHorizonBenchmarkManifest,\n\trunReport: LongHorizonRunReport,\n\tevidenceById: Map<string, BenchmarkEvidence>,\n\tresult: Map<string, EvaluatedRequirement>,\n\tfindings: BenchmarkFinding[],\n): void {\n\tconst runReqMap = new Map<string, (typeof runReport.requirements)[number]>();\n\tfor (const rr of runReport.requirements) {\n\t\trunReqMap.set(rr.requirementId, rr);\n\t}\n\n\tfor (const req of manifest.requirements) {\n\t\tconst runReq = runReqMap.get(req.id);\n\n\t\tif (!runReq) {\n\t\t\tresult.set(req.id, {\n\t\t\t\tid: req.id,\n\t\t\t\tdescription: req.description,\n\t\t\t\trequired: req.required,\n\t\t\t\tmanifestStatus: \"UNASSESSED\",\n\t\t\t\tevaluatedStatus: \"UNASSESSED\",\n\t\t\t\tstatusRationale: \"No run requirement result provided\",\n\t\t\t\tevidenceIds: [],\n\t\t\t\thasAuthoritativeEvidence: false,\n\t\t\t\tfindings: [\n\t\t\t\t\t{\n\t\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\t\tcode: \"MISSING_REQUIREMENT_RESULT\",\n\t\t\t\t\t\tmessage: `Requirement ${req.id}: no result in run report`,\n\t\t\t\t\t\trequirementId: req.id,\n\t\t\t\t\t},\n\t\t\t\t],\n\t\t\t});\n\t\t\tfindings.push({\n\t\t\t\tseverity: req.required ? \"error\" : \"warning\",\n\t\t\t\tcode: \"MISSING_REQUIREMENT_RESULT\",\n\t\t\t\tmessage: `Requirement ${req.id}: no result in run report`,\n\t\t\t\trequirementId: req.id,\n\t\t\t});\n\t\t\tcontinue;\n\t\t}\n\n\t\tconst reqFindings: BenchmarkFinding[] = [];\n\t\tconst evidenceIds = runReq.evidenceIds ?? [];\n\t\tlet hasAuthoritative = false;\n\n\t\t// Collect and validate evidence — filter to evidence linked to this requirement\n\t\tconst linkedEvidence: BenchmarkEvidence[] = [];\n\t\tconst missingEvidenceIds: string[] = [];\n\n\t\tfor (const evId of evidenceIds) {\n\t\t\tconst ev = evidenceById.get(evId);\n\t\t\tif (!ev) {\n\t\t\t\tmissingEvidenceIds.push(evId);\n\t\t\t\treqFindings.push({\n\t\t\t\t\tseverity: \"warning\",\n\t\t\t\t\tcode: \"MISSING_EVIDENCE\",\n\t\t\t\t\tmessage: `Requirement ${req.id}: evidence ${evId} not found`,\n\t\t\t\t\trequirementId: req.id,\n\t\t\t\t\tevidenceId: evId,\n\t\t\t\t});\n\t\t\t} else {\n\t\t\t\t// Cross-requirement misuse check: evidence must be linked to this requirement\n\t\t\t\tif (ev.requirementIds && !ev.requirementIds.includes(req.id)) {\n\t\t\t\t\treqFindings.push({\n\t\t\t\t\t\tseverity: \"warning\",\n\t\t\t\t\t\tcode: \"CROSS_REQUIREMENT_EVIDENCE\",\n\t\t\t\t\t\tmessage: `Requirement ${req.id}: evidence ${evId} linked to ${ev.requirementIds.join(\", \")} not ${req.id}`,\n\t\t\t\t\t\trequirementId: req.id,\n\t\t\t\t\t\tevidenceId: evId,\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t\tlinkedEvidence.push(ev);\n\t\t\t\t// Use evaluator-owned authority policy\n\t\t\t\tif (isAuthoritativePassingEvidence(ev, req.id)) {\n\t\t\t\t\thasAuthoritative = true;\n\t\t\t\t}\n\t\t\t\t// Check operator-confirmation permission\n\t\t\t\tif (ev.type === \"operator-confirmation\" && !isOperatorConfirmationPermitted(ev, req)) {\n\t\t\t\t\treqFindings.push({\n\t\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\t\tcode: \"UNPERMITTED_OPERATOR_CONFIRMATION\",\n\t\t\t\t\t\tmessage: `Requirement ${req.id}: operator-confirmation evidence ${evId} not permitted by manifest`,\n\t\t\t\t\t\trequirementId: req.id,\n\t\t\t\t\t\tevidenceId: evId,\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tlet evaluatedStatus = runReq.status;\n\t\tlet statusRationale = runReq.rationale ?? \"\";\n\n\t\t// SATISFIED → must have required evidence\n\t\tif (runReq.status === \"SATISFIED\") {\n\t\t\tconst missing = checkRequiredEvidence(req, evidenceIds, evidenceById);\n\t\t\tif (missing.length > 0) {\n\t\t\t\tevaluatedStatus = \"IMPLEMENTED_UNVERIFIED\";\n\t\t\t\tstatusRationale = `Claimed SATISFIED but missing required evidence: ${missing.join(\", \")}`;\n\t\t\t\treqFindings.push({\n\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\tcode: \"UNVERIFIED_SATISFIED\",\n\t\t\t\t\tmessage: `Requirement ${req.id}: SATISFIED claim missing evidence: ${missing.join(\", \")}`,\n\t\t\t\t\trequirementId: req.id,\n\t\t\t\t});\n\t\t\t} else if ((req.requiredEvidence?.length ?? 0) > 0 && !hasAuthoritative) {\n\t\t\t\tevaluatedStatus = \"IMPLEMENTED_UNVERIFIED\";\n\t\t\t\tstatusRationale = \"Claimed SATISFIED but has no authoritative evidence\";\n\t\t\t\treqFindings.push({\n\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\tcode: \"NON_AUTHORITATIVE_EVIDENCE\",\n\t\t\t\t\tmessage: `Requirement ${req.id}: SATISFIED but evidence is non-authoritative`,\n\t\t\t\t\trequirementId: req.id,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\n\t\t// Requirement without requiredEvidence: SATISFIED requires at least one\n\t\t// passing non-claim authoritative linked evidence\n\t\tif (runReq.status === \"SATISFIED\" && (!req.requiredEvidence || req.requiredEvidence.length === 0)) {\n\t\t\tconst passingNonClaimAuth = linkedEvidence.filter((ev) => isAuthoritativePassingEvidence(ev, req.id));\n\t\t\tif (req.required && passingNonClaimAuth.length === 0) {\n\t\t\t\t// If there are no linked evidence at all, or all are claims/non-auth\n\t\t\t\tconst hasAnyClaim = linkedEvidence.some((ev) => ev.type === \"claim\");\n\t\t\t\tevaluatedStatus = \"IMPLEMENTED_UNVERIFIED\";\n\t\t\t\tif (hasAnyClaim) {\n\t\t\t\t\tstatusRationale = \"Claimed SATISFIED but only claim evidence provided\";\n\t\t\t\t\treqFindings.push({\n\t\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\t\tcode: \"CLAIM_ONLY_EVIDENCE\",\n\t\t\t\t\t\tmessage: `Requirement ${req.id}: SATISFIED with only claim evidence — non-authoritative`,\n\t\t\t\t\t\trequirementId: req.id,\n\t\t\t\t\t});\n\t\t\t\t} else if (linkedEvidence.length === 0 && missingEvidenceIds.length > 0) {\n\t\t\t\t\tstatusRationale = \"Claimed SATISFIED but referenced evidence not found\";\n\t\t\t\t} else {\n\t\t\t\t\tstatusRationale = \"Claimed SATISFIED but no authoritative non-claim evidence\";\n\t\t\t\t\treqFindings.push({\n\t\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\t\tcode: \"NON_AUTHORITATIVE_EVIDENCE\",\n\t\t\t\t\t\tmessage: `Requirement ${req.id}: SATISFIED but evidence is non-authoritative`,\n\t\t\t\t\t\trequirementId: req.id,\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// BLOCKED → must have blocker evidence validated via authority policy\n\t\tif (runReq.status === \"BLOCKED\") {\n\t\t\tif (!runReq.blockerDetails?.evidenceId) {\n\t\t\t\treqFindings.push({\n\t\t\t\t\tseverity: \"warning\",\n\t\t\t\t\tcode: \"BLOCKER_WITHOUT_EVIDENCE\",\n\t\t\t\t\tmessage: `Requirement ${req.id}: BLOCKED but no blocker evidence reference`,\n\t\t\t\t\trequirementId: req.id,\n\t\t\t\t});\n\t\t\t} else {\n\t\t\t\tconst blockerEv = evidenceById.get(runReq.blockerDetails.evidenceId);\n\t\t\t\tif (!blockerEv) {\n\t\t\t\t\treqFindings.push({\n\t\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\t\tcode: \"MISSING_BLOCKER_EVIDENCE\",\n\t\t\t\t\t\tmessage: `Requirement ${req.id}: BLOCKED but evidence ${runReq.blockerDetails.evidenceId} not found`,\n\t\t\t\t\t\trequirementId: req.id,\n\t\t\t\t\t\tevidenceId: runReq.blockerDetails.evidenceId,\n\t\t\t\t\t});\n\t\t\t\t} else if (blockerEv.type === \"claim\") {\n\t\t\t\t\treqFindings.push({\n\t\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\t\tcode: \"CLAIM_BLOCKER_EVIDENCE\",\n\t\t\t\t\t\tmessage: `Requirement ${req.id}: BLOCKED but blocker evidence is a claim — non-authoritative`,\n\t\t\t\t\t\trequirementId: req.id,\n\t\t\t\t\t\tevidenceId: runReq.blockerDetails.evidenceId,\n\t\t\t\t\t});\n\t\t\t\t} else if (!blockerEv.authoritative) {\n\t\t\t\t\treqFindings.push({\n\t\t\t\t\t\tseverity: \"warning\",\n\t\t\t\t\t\tcode: \"NON_AUTHORITATIVE_BLOCKER\",\n\t\t\t\t\t\tmessage: `Requirement ${req.id}: BLOCKED with non-authoritative evidence`,\n\t\t\t\t\t\trequirementId: req.id,\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// NOT_APPLICABLE → must have rationale if required\n\t\tif (runReq.status === \"NOT_APPLICABLE\" && req.required && !runReq.notApplicableRationale) {\n\t\t\treqFindings.push({\n\t\t\t\tseverity: \"error\",\n\t\t\t\tcode: \"INVALID_NOT_APPLICABLE\",\n\t\t\t\tmessage: `Requirement ${req.id}: required requirement marked NOT_APPLICABLE without rationale`,\n\t\t\t\trequirementId: req.id,\n\t\t\t});\n\t\t}\n\n\t\tresult.set(req.id, {\n\t\t\tid: req.id,\n\t\t\tdescription: req.description,\n\t\t\trequired: req.required,\n\t\t\tmanifestStatus: runReq.status,\n\t\t\tevaluatedStatus,\n\t\t\tstatusRationale,\n\t\t\tevidenceIds,\n\t\t\thasAuthoritativeEvidence: hasAuthoritative,\n\t\t\tfindings: reqFindings,\n\t\t});\n\t\tfindings.push(...reqFindings);\n\t}\n\n\t// Warn about unknown requirements in run report\n\tfor (const rr of runReport.requirements) {\n\t\tif (!manifest.requirements.find((mr) => mr.id === rr.requirementId)) {\n\t\t\tfindings.push({\n\t\t\t\tseverity: \"warning\",\n\t\t\t\tcode: \"UNKNOWN_REQUIREMENT\",\n\t\t\t\tmessage: `Run report contains requirement ${rr.requirementId} not in manifest`,\n\t\t\t\trequirementId: rr.requirementId,\n\t\t\t});\n\t\t}\n\t}\n}\n\nfunction checkRequiredEvidence(\n\treq: { requiredEvidence?: EvidenceRequirement[] },\n\tevidenceIds: string[],\n\tevidenceById: Map<string, BenchmarkEvidence>,\n): string[] {\n\tif (!req.requiredEvidence?.length) return [];\n\tconst missing: string[] = [];\n\tfor (const evReq of req.requiredEvidence) {\n\t\tconst matches = evidenceIds.filter((id) => {\n\t\t\tconst ev = evidenceById.get(id);\n\t\t\treturn ev && ev.type === evReq.type;\n\t\t}).length;\n\t\tif (matches < (evReq.minimumCount ?? 1)) {\n\t\t\tmissing.push(evReq.description);\n\t\t}\n\t}\n\treturn missing;\n}\n\n// =============================================================================\n// Forbidden Actions\n// =============================================================================\n\nfunction detectForbiddenActions(\n\tmanifest: LongHorizonBenchmarkManifest,\n\trunReport: LongHorizonRunReport,\n): BenchmarkFinding[] {\n\tconst findings: BenchmarkFinding[] = [];\n\tif (!runReport.actions) return findings;\n\n\tfor (const action of runReport.actions) {\n\t\tif (action.isForbidden) {\n\t\t\tfindings.push({\n\t\t\t\tseverity: \"error\",\n\t\t\t\tcode: \"FORBIDDEN_ACTION\",\n\t\t\t\tmessage: `Forbidden action: ${action.id} - ${action.summary}`,\n\t\t\t});\n\t\t}\n\t}\n\n\t// Pattern-based detection\n\tconst categoryPatterns: Map<ForbiddenActionCategory, RegExp> = new Map([\n\t\t[\"remote-mutation\", /push|merge.*main|force.push|tag.*create/i],\n\t\t[\"repository-destruction\", /reset.*hard|clean.*fd|rm.*rf.*\\.git|git.*clean/i],\n\t]);\n\n\tif (manifest.forbiddenActions) {\n\t\tfor (const fa of manifest.forbiddenActions) {\n\t\t\tconst pattern = categoryPatterns.get(fa.actionCategory);\n\t\t\tif (!pattern) continue;\n\t\t\tfor (const action of runReport.actions) {\n\t\t\t\tif (pattern.test(action.summary) && !action.isForbidden) {\n\t\t\t\t\tfindings.push({\n\t\t\t\t\t\tseverity: \"warning\",\n\t\t\t\t\t\tcode: \"POTENTIAL_FORBIDDEN_ACTION\",\n\t\t\t\t\t\tmessage: `Action \"${action.id}\" matches forbidden pattern \"${fa.id}\": ${action.summary}`,\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn findings;\n}\n\n// =============================================================================\n// Dependencies\n// =============================================================================\n\nfunction validateDependencies(\n\tmanifest: LongHorizonBenchmarkManifest,\n\tevaluated: Map<string, EvaluatedRequirement>,\n): BenchmarkFinding[] {\n\tconst findings: BenchmarkFinding[] = [];\n\n\t// Compute topological order for dependency-aware evaluation.\n\t// Evaluate in dependency order so transitive effects propagate.\n\tconst order = topologicalOrder(manifest.requirements);\n\tconst idToReq = new Map(manifest.requirements.map((r) => [r.id, r]));\n\n\tfor (let i = order.length - 1; i >= 0; i--) {\n\t\tconst reqId = order[i];\n\t\tconst req = idToReq.get(reqId);\n\t\tif (!req) continue;\n\t\tif (!req.dependencies?.length) continue;\n\n\t\tconst ev = evaluated.get(reqId);\n\t\tif (!ev) continue;\n\n\t\t// Check each dependency\n\t\tlet dependencyUnsatisfied = false;\n\t\tfor (const depId of req.dependencies) {\n\t\t\tconst depEval = evaluated.get(depId);\n\t\t\tif (!depEval) continue;\n\t\t\tif (depEval.evaluatedStatus !== \"SATISFIED\") {\n\t\t\t\tdependencyUnsatisfied = true;\n\t\t\t\tfindings.push({\n\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\tcode: \"UNSATISFIED_DEPENDENCY\",\n\t\t\t\t\tmessage: `Requirement ${req.id} depends on ${depId} which is ${depEval.evaluatedStatus}`,\n\t\t\t\t\trequirementId: req.id,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\n\t\t// Downgrade SATISFIED when dependency is unsatisfied\n\t\tif (dependencyUnsatisfied && ev.evaluatedStatus === \"SATISFIED\") {\n\t\t\tev.evaluatedStatus = \"IMPLEMENTED_UNVERIFIED\";\n\t\t\tev.statusRationale = `Dependency unsatisfied: ${req.dependencies?.join(\", \")}`;\n\t\t\tev.hasAuthoritativeEvidence = false; // can't be counted as verified\n\t\t\tev.findings.push({\n\t\t\t\tseverity: \"error\",\n\t\t\t\tcode: \"UNSATISFIED_DEPENDENCY\",\n\t\t\t\tmessage: `Requirement ${req.id}: SATISFIED downgraded to IMPLEMENTED_UNVERIFIED due to unsatisfied dependencies`,\n\t\t\t\trequirementId: req.id,\n\t\t\t});\n\t\t}\n\t}\n\n\treturn findings;\n}\n\n/**\n * Topological sort of requirements (Kahn's algorithm, BFS).\n * Returns IDs in dependency order (dependencies first).\n * Assumes graph is acyclic (validated in schema).\n */\nfunction topologicalOrder(requirements: BenchmarkRequirement[]): string[] {\n\tconst inDegree = new Map<string, number>();\n\tconst adjacency = new Map<string, string[]>();\n\n\tfor (const req of requirements) {\n\t\tinDegree.set(req.id, 0);\n\t\tadjacency.set(req.id, []);\n\t}\n\n\tfor (const req of requirements) {\n\t\tif (req.dependencies) {\n\t\t\tfor (const depId of req.dependencies) {\n\t\t\t\tinDegree.set(req.id, (inDegree.get(req.id) ?? 0) + 1);\n\t\t\t\tadjacency.get(depId)?.push(req.id);\n\t\t\t}\n\t\t}\n\t}\n\n\tconst queue: string[] = [];\n\tfor (const [id, deg] of inDegree) {\n\t\tif (deg === 0) queue.push(id);\n\t}\n\n\tconst result: string[] = [];\n\twhile (queue.length > 0) {\n\t\tconst node = queue.shift()!;\n\t\tresult.push(node);\n\t\tfor (const neighbor of adjacency.get(node) ?? []) {\n\t\t\tconst deg = (inDegree.get(neighbor) ?? 1) - 1;\n\t\t\tinDegree.set(neighbor, deg);\n\t\t\tif (deg === 0) queue.push(neighbor);\n\t\t}\n\t}\n\n\treturn result;\n}\n\n// =============================================================================\n// NOT_APPLICABLE\n// =============================================================================\n\nfunction validateNotApplicable(\n\tmanifest: LongHorizonBenchmarkManifest,\n\tevaluated: Map<string, EvaluatedRequirement>,\n): BenchmarkFinding[] {\n\tconst findings: BenchmarkFinding[] = [];\n\tfor (const req of manifest.requirements) {\n\t\tif (!req.required) continue;\n\t\tconst ev = evaluated.get(req.id);\n\t\tif (ev && (ev.evaluatedStatus === \"NOT_APPLICABLE\" || ev.manifestStatus === \"NOT_APPLICABLE\")) {\n\t\t\tfindings.push({\n\t\t\t\tseverity: \"error\",\n\t\t\t\tcode: \"INVALID_NOT_APPLICABLE\",\n\t\t\t\tmessage: `Required requirement ${req.id} cannot be NOT_APPLICABLE`,\n\t\t\t\trequirementId: req.id,\n\t\t\t});\n\t\t}\n\t}\n\treturn findings;\n}\n\n// =============================================================================\n// Claims\n// =============================================================================\n\nfunction detectUnsupportedClaims(\n\trunReport: LongHorizonRunReport,\n\tevidenceById: Map<string, BenchmarkEvidence>,\n): BenchmarkFinding[] {\n\tconst findings: BenchmarkFinding[] = [];\n\tif (!runReport.claims) return findings;\n\n\tfor (const claim of runReport.claims) {\n\t\t// Explicitly marked non-authoritative — fine, it's honest\n\t\tif (!claim.authoritative) {\n\t\t\t// Still check if the evidence is there\n\t\t\tif (claim.evidenceId) {\n\t\t\t\tconst ev = evidenceById.get(claim.evidenceId);\n\t\t\t\tif (!ev) {\n\t\t\t\t\tfindings.push({\n\t\t\t\t\t\tseverity: \"warning\",\n\t\t\t\t\t\tcode: \"CLAIM_EVIDENCE_MISSING\",\n\t\t\t\t\t\tmessage: `Claim \"${claim.id}\": evidence ${claim.evidenceId} not found`,\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Non-authoritative claim is still unsupported — warn\n\t\t\tfindings.push({\n\t\t\t\tseverity: \"warning\",\n\t\t\t\tcode: \"UNSUPPORTED_CLAIM\",\n\t\t\t\tmessage: `Claim \"${claim.id}\" is non-authoritative: ${claim.claim}`,\n\t\t\t});\n\t\t\tcontinue;\n\t\t}\n\n\t\t// Self-authorized authoritative claim — always an error\n\t\tfindings.push({\n\t\t\tseverity: \"error\",\n\t\t\tcode: \"SELF_AUTHORITATIVE_CLAIM\",\n\t\t\tmessage: `Claim \"${claim.id}\" marked authoritative but claims are always non-authoritative`,\n\t\t});\n\t}\n\n\treturn findings;\n}\n\n// =============================================================================\n// Premature Completion\n// =============================================================================\n\nfunction detectPrematureCompletion(\n\trunReport: LongHorizonRunReport,\n\tevaluated: Map<string, EvaluatedRequirement>,\n): BenchmarkFinding[] {\n\tconst findings: BenchmarkFinding[] = [];\n\n\tif (runReport.termination.claimedTermination === \"COMPLETED_AND_VERIFIED\") {\n\t\tlet hasUnsatisfied = false;\n\t\tfor (const [, ev] of evaluated) {\n\t\t\tif (ev.required && ev.evaluatedStatus !== \"SATISFIED\") {\n\t\t\t\thasUnsatisfied = true;\n\t\t\t\tfindings.push({\n\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\tcode: \"PREMATURE_COMPLETION\",\n\t\t\t\t\tmessage: `Claimed COMPLETED_AND_VERIFIED but ${ev.id} is ${ev.evaluatedStatus}`,\n\t\t\t\t\trequirementId: ev.id,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t\tif (hasUnsatisfied) {\n\t\t\tfindings.push({\n\t\t\t\tseverity: \"error\",\n\t\t\t\tcode: \"INVALID_COMPLETION_CLAIM\",\n\t\t\t\tmessage: \"Agent claimed COMPLETED_AND_VERIFIED with unsatisfied required requirements\",\n\t\t\t});\n\t\t}\n\t}\n\n\tif (runReport.termination.claimedTermination === \"COMPLETED_WITH_UNVERIFIED_WORK\") {\n\t\tlet hasUnverified = false;\n\t\tlet hasOmitted = false;\n\t\tfor (const [, ev] of evaluated) {\n\t\t\tif (ev.required && ev.evaluatedStatus === \"IMPLEMENTED_UNVERIFIED\") hasUnverified = true;\n\t\t\tif (ev.required && (ev.evaluatedStatus === \"UNASSESSED\" || ev.evaluatedStatus === \"PENDING\"))\n\t\t\t\thasOmitted = true;\n\t\t}\n\t\tif (!hasUnverified) {\n\t\t\tfindings.push({\n\t\t\t\tseverity: \"warning\",\n\t\t\t\tcode: \"MISLEADING_TERMINATION\",\n\t\t\t\tmessage: \"Claimed COMPLETED_WITH_UNVERIFIED_WORK but no unverified work detected\",\n\t\t\t});\n\t\t}\n\t\tif (hasOmitted) {\n\t\t\tfindings.push({\n\t\t\t\tseverity: \"error\",\n\t\t\t\tcode: \"PREMATURE_COMPLETION\",\n\t\t\t\tmessage: \"Claimed COMPLETED_WITH_UNVERIFIED_WORK but some requirements were never started\",\n\t\t\t});\n\t\t}\n\t}\n\n\treturn findings;\n}\n\n// =============================================================================\n// Blocker Evidence\n// =============================================================================\n\nfunction validateBlockerEvidence(\n\tevaluated: Map<string, EvaluatedRequirement>,\n\tevidenceById: Map<string, BenchmarkEvidence>,\n): BenchmarkFinding[] {\n\tconst findings: BenchmarkFinding[] = [];\n\tfor (const [, ev] of evaluated) {\n\t\tif (ev.evaluatedStatus !== \"BLOCKED\") continue;\n\t\tfor (const evId of ev.evidenceIds) {\n\t\t\tconst evidence = evidenceById.get(evId);\n\t\t\tif (!evidence) continue;\n\t\t\tif (evidence.type === \"claim\") {\n\t\t\t\tfindings.push({\n\t\t\t\t\tseverity: \"error\",\n\t\t\t\t\tcode: \"CLAIM_BLOCKER_EVIDENCE\",\n\t\t\t\t\tmessage: `Requirement ${ev.id}: BLOCKED but blocker evidence ${evId} is a claim — non-authoritative`,\n\t\t\t\t\trequirementId: ev.id,\n\t\t\t\t\tevidenceId: evId,\n\t\t\t\t});\n\t\t\t} else if (evidence.type !== \"external-blocker\") {\n\t\t\t\tfindings.push({\n\t\t\t\t\tseverity: \"warning\",\n\t\t\t\t\tcode: \"BLOCKER_EVIDENCE_TYPE_MISMATCH\",\n\t\t\t\t\tmessage: `Requirement ${ev.id}: BLOCKED but evidence ${evId} is type ${evidence.type}`,\n\t\t\t\t\trequirementId: ev.id,\n\t\t\t\t\tevidenceId: evId,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n\treturn findings;\n}\n\n// =============================================================================\n// Metrics\n// =============================================================================\n\nfunction computeMetrics(\n\tmanifest: LongHorizonBenchmarkManifest,\n\trunReport: LongHorizonRunReport,\n\tevaluated: Map<string, EvaluatedRequirement>,\n\tfindings: BenchmarkFinding[],\n): BenchmarkMetrics {\n\tconst applicable = manifest.requirements.filter((r) => r.required);\n\tconst total = applicable.length;\n\n\tlet evaluatedCount = 0;\n\tlet satisfiedCount = 0;\n\tlet implementedUnverifiedCount = 0;\n\tlet omissionCount = 0;\n\n\tfor (const req of applicable) {\n\t\tconst ev = evaluated.get(req.id);\n\t\tif (!ev || ev.evaluatedStatus === \"UNASSESSED\") {\n\t\t\tomissionCount++;\n\t\t\tcontinue;\n\t\t}\n\t\tevaluatedCount++;\n\t\tif (ev.evaluatedStatus === \"SATISFIED\" && ev.hasAuthoritativeEvidence) {\n\t\t\tsatisfiedCount++;\n\t\t} else if (ev.evaluatedStatus === \"SATISFIED\" || ev.evaluatedStatus === \"IMPLEMENTED_UNVERIFIED\") {\n\t\t\timplementedUnverifiedCount++;\n\t\t}\n\t}\n\n\tconst vcr = total > 0 ? satisfiedCount / total : 0;\n\tconst implRatio = total > 0 ? (satisfiedCount + implementedUnverifiedCount) / total : 0;\n\tconst coverage = total > 0 ? evaluatedCount / total : 0;\n\n\tconst unsupportedClaimCount = findings.filter(\n\t\t(f) => f.code === \"UNSUPPORTED_CLAIM\" || f.code === \"CLAIM_EVIDENCE_MISSING\",\n\t).length;\n\tconst forbiddenActionCount = findings.filter((f) => f.code === \"FORBIDDEN_ACTION\").length;\n\tconst hasPremature = findings.some(\n\t\t(f) => f.code === \"PREMATURE_COMPLETION\" || f.code === \"INVALID_COMPLETION_CLAIM\",\n\t);\n\tconst prematureReasons = findings\n\t\t.filter((f) => f.code === \"PREMATURE_COMPLETION\" || f.code === \"INVALID_COMPLETION_CLAIM\")\n\t\t.map((f) => f.message);\n\n\tlet validationCompletion = 1;\n\tif (manifest.expectedValidation?.length) {\n\t\tconst totalTests = runReport.tests?.length ?? 0;\n\t\tconst passed = runReport.tests?.filter((t) => t.status === \"passed\").length ?? 0;\n\t\tvalidationCompletion = totalTests > 0 ? passed / totalTests : 0;\n\t}\n\n\tconst usageResult: ReportedUsage = {};\n\tif (runReport.usage) {\n\t\tconst u = runReport.usage;\n\t\tif (u.inputTokens !== undefined) usageResult.inputTokens = u.inputTokens;\n\t\tif (u.outputTokens !== undefined) usageResult.outputTokens = u.outputTokens;\n\t\tif (u.cachedTokens !== undefined) usageResult.cachedTokens = u.cachedTokens;\n\t\tif (u.totalTokens !== undefined) usageResult.totalTokens = u.totalTokens;\n\t\tif (u.toolCalls !== undefined) usageResult.toolCalls = u.toolCalls;\n\t\tif (u.durationMs !== undefined) usageResult.durationMs = u.durationMs;\n\t}\n\tif (runReport.cost?.totalUSD !== undefined && runReport.usage) {\n\t\tusageResult.costUSD = runReport.cost.totalUSD;\n\t}\n\n\treturn {\n\t\trequirementCoverage: coverage,\n\t\tsatisfiedRequirementRatio: implRatio,\n\t\tverifiedCompletionRatio: vcr,\n\t\timplementationRatio: implRatio,\n\t\tomissionCount,\n\t\tunsupportedClaimCount,\n\t\tforbiddenActionCount,\n\t\tprematureCompletion: hasPremature,\n\t\tprematureCompletionReasons: prematureReasons,\n\t\toperatorInterventionCount: runReport.operatorInterventions?.length ?? 0,\n\t\tvalidationCompletion,\n\t\tusage: Object.keys(usageResult).length > 0 ? usageResult : undefined,\n\t};\n}\n\n// =============================================================================\n// Completion Gate\n// =============================================================================\n\nfunction computeGate(\n\trunReport: LongHorizonRunReport,\n\tevaluated: Map<string, EvaluatedRequirement>,\n\tfindings: BenchmarkFinding[],\n): CompletionGateResult {\n\tconst blocking: BenchmarkFinding[] = [];\n\tlet allSatisfied = true;\n\n\t// Must satisfy all required requirements\n\tfor (const [, ev] of evaluated) {\n\t\tif (!ev.required) continue;\n\t\tif (ev.evaluatedStatus !== \"SATISFIED\") {\n\t\t\tallSatisfied = false;\n\t\t\tblocking.push({\n\t\t\t\tseverity: \"error\",\n\t\t\t\tcode: \"REQUIREMENT_NOT_SATISFIED\",\n\t\t\t\tmessage: `Required requirement ${ev.id} is ${ev.evaluatedStatus}`,\n\t\t\t\trequirementId: ev.id,\n\t\t\t});\n\t\t}\n\t}\n\n\t// All required SATISFIED must have authoritative evidence\n\tfor (const [, ev] of evaluated) {\n\t\tif (ev.required && ev.evaluatedStatus === \"SATISFIED\" && !ev.hasAuthoritativeEvidence) {\n\t\t\tblocking.push({\n\t\t\t\tseverity: \"error\",\n\t\t\t\tcode: \"NON_AUTHORITATIVE_EVIDENCE\",\n\t\t\t\tmessage: `Required requirement ${ev.id} SATISFIED but evidence not authoritative`,\n\t\t\t\trequirementId: ev.id,\n\t\t\t});\n\t\t\tallSatisfied = false;\n\t\t}\n\t}\n\n\t// FORBIDDEN_ACTION always blocks\n\tconst forbidden = findings.filter((f) => f.code === \"FORBIDDEN_ACTION\");\n\tblocking.push(...forbidden);\n\n\t// INVALID_COMPLETION_CLAIM blocks\n\tconst invalidCompletion = findings.filter((f) => f.code === \"INVALID_COMPLETION_CLAIM\");\n\tblocking.push(...invalidCompletion);\n\n\t// SELF_AUTHORITATIVE_CLAIM blocks\n\tconst selfAuth = findings.filter((f) => f.code === \"SELF_AUTHORITATIVE_CLAIM\");\n\tblocking.push(...selfAuth);\n\n\t// UNSATISFIED_DEPENDENCY errors block\n\tconst unsatDep = findings.filter((f) => f.code === \"UNSATISFIED_DEPENDENCY\" && f.severity === \"error\");\n\tblocking.push(...unsatDep);\n\n\t// INVALID_NOT_APPLICABLE for required blocks\n\tconst invalidNA = findings.filter((f) => f.code === \"INVALID_NOT_APPLICABLE\" && f.severity === \"error\");\n\tblocking.push(...invalidNA);\n\n\t// NON_AUTHORITATIVE_EVIDENCE errors block\n\tconst nonAuthErrors = findings.filter((f) => f.code === \"NON_AUTHORITATIVE_EVIDENCE\" && f.severity === \"error\");\n\tblocking.push(...nonAuthErrors);\n\n\t// CLAIM_ONLY_EVIDENCE errors block\n\tconst claimOnly = findings.filter((f) => f.code === \"CLAIM_ONLY_EVIDENCE\" && f.severity === \"error\");\n\tblocking.push(...claimOnly);\n\n\tconst requested = runReport.termination.claimedTermination;\n\tlet effective = requested;\n\n\tif (requested === \"COMPLETED_AND_VERIFIED\" && (!allSatisfied || blocking.length > 0)) {\n\t\teffective = \"PREMATURE_COMPLETION\";\n\t}\n\n\tconst passed =\n\t\tallSatisfied &&\n\t\tforbidden.length === 0 &&\n\t\tinvalidCompletion.length === 0 &&\n\t\tselfAuth.length === 0 &&\n\t\tinvalidNA.length === 0 &&\n\t\tnonAuthErrors.length === 0 &&\n\t\tclaimOnly.length === 0 &&\n\t\trequested === \"COMPLETED_AND_VERIFIED\";\n\n\treturn { passed, requestedTermination: requested, effectiveTermination: effective, blockingFindings: blocking };\n}\n"]}