{"version":3,"file":"scoring.d.ts","sourceRoot":"","sources":["../../../src/core/routing/scoring.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EACX,iBAAiB,EACjB,sBAAsB,EACtB,2BAA2B,EAC3B,iBAAiB,EACjB,eAAe,EACf,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C,MAAM,WAAW,gBAAgB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CAChB;AAED,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,eAAe,EAAE,gBAAgB,CAOvE,CAAC;AAiBF,2EAA2E;AAC3E,wBAAgB,cAAc,CAC7B,SAAS,EAAE,sBAAsB,GAAG;IAAE,WAAW,EAAE,MAAM,CAAA;CAAE,EAC3D,QAAQ,EAAE,iBAAiB,GAAG,SAAS,EACvC,YAAY,EAAE,MAAM,GAClB,2BAA2B,CAiC7B;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC7B,KAAK,EAAE,2BAA2B,EAClC,OAAO,EAAE,gBAAgB,EACzB,OAAO,GAAE;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACpC,MAAM,CAqBR;AAED,MAAM,WAAW,gBAAgB;IAChC,MAAM,EAAE,eAAe,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC/B,QAAQ,EAAE,2BAA2B,GAAG,SAAS,CAAC;IAClD,SAAS,EAAE,2BAA2B,EAAE,CAAC;IACzC,8DAA8D;IAC9D,MAAM,EAAE;QAAE,KAAK,EAAE,2BAA2B,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,kBAAkB,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAChG,WAAW,EAAE,MAAM,EAAE,CAAC;CACtB;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,2BAA2B,EAAE,EAAE,OAAO,EAAE,gBAAgB,GAAG,eAAe,CAwC5G;AAED,iFAAiF;AACjF,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;CACtB,GAAG,iBAAiB,CAOpB","sourcesContent":["/**\n * Evaluation-informed scoring, uncertainty, and multi-objective selection.\n *\n * - Missing evidence is explicitly \"no evidence\", never zero (zero would imply\n *   a confident failure).\n * - Safety failures remain separate hard constraints and are never averaged away.\n * - Aggregate weights are versioned and explicit.\n * - Uncertainty penalizes selection.\n * - Selection supports a Pareto frontier and explicit objectives.\n */\n\nimport { buildCandidateId } from \"./baseline.js\";\nimport type {\n\tCandidateEvidence,\n\tOrchestrationCandidate,\n\tOrchestrationCandidateScore,\n\tRetrievalStrategy,\n\tSelectionPolicy,\n} from \"./types.js\";\n\nexport const AGGREGATE_WEIGHTS_VERSION = 1;\n\nexport interface AggregateWeights {\n\tcorrectness: number;\n\tsafety: number;\n\treliability: number;\n\tcost: number;\n\tlatency: number;\n}\n\nexport const WEIGHTS_BY_POLICY: Record<SelectionPolicy, AggregateWeights> = {\n\tquality_first: { correctness: 0.4, safety: 0.3, reliability: 0.2, cost: 0.05, latency: 0.05 },\n\tbalanced: { correctness: 0.3, safety: 0.25, reliability: 0.2, cost: 0.15, latency: 0.1 },\n\tcost_constrained: { correctness: 0.2, safety: 0.25, reliability: 0.1, cost: 0.4, latency: 0.05 },\n\tlatency_constrained: { correctness: 0.2, safety: 0.25, reliability: 0.1, cost: 0.05, latency: 0.4 },\n\tlocal_only: { correctness: 0.25, safety: 0.4, reliability: 0.15, cost: 0.05, latency: 0.15 },\n\thigh_assurance: { correctness: 0.35, safety: 0.4, reliability: 0.15, cost: 0.05, latency: 0.05 },\n};\n\n/** Normalize a 0..1 rate or a raw cost/latency into a 0..1 higher-is-better score. */\nfunction normalized(value: number | undefined, kind: \"rate\" | \"cost\" | \"latency\"): number | undefined {\n\tif (value === undefined) return undefined;\n\tif (kind === \"rate\") return Math.max(0, Math.min(1, value));\n\tif (kind === \"cost\") {\n\t\t// FrugalGPT-style: cost score in reverse, bounded to $2 to keep scale.\n\t\treturn Math.max(0, 1 - value / 2);\n\t}\n\tif (kind === \"latency\") {\n\t\t// Latency score in reverse, bounded to 60s.\n\t\treturn Math.max(0, 1 - value / 60_000);\n\t}\n\treturn undefined;\n}\n\n/** Score one candidate from evidence. Missing evidence stays undefined. */\nexport function scoreCandidate(\n\tcandidate: OrchestrationCandidate | { candidateId: string },\n\tevidence: CandidateEvidence | undefined,\n\t_sampleCount: number,\n): OrchestrationCandidateScore {\n\tconst id = \"candidateId\" in candidate ? candidate.candidateId : buildCandidateId(candidate);\n\tif (!evidence) {\n\t\treturn {\n\t\t\tcandidateId: id,\n\t\t\tuncertainty: 1,\n\t\t\tsampleCount: 0,\n\t\t\treasonCodes: [\"no_evidence\"],\n\t\t\tevidenceIds: [],\n\t\t};\n\t}\n\tconst correctness = normalized(evidence.correctnessRate, \"rate\");\n\tconst safety = normalized(evidence.safetyRate, \"rate\");\n\tconst reliability = normalized(evidence.reliabilityRate, \"rate\");\n\tconst cost = normalized(evidence.avgCostUsd, \"cost\");\n\tconst latency = normalized(evidence.medianLatencyMs, \"latency\");\n\n\t// Uncertainty from low sample count and high flakiness.\n\tlet uncertainty = Math.max(0, 1 - Math.min(1, evidence.sampleCount / 30));\n\tuncertainty = Math.max(0, Math.min(1, uncertainty + (evidence.flakyRate ?? 0) * 0.5));\n\n\treturn {\n\t\tcandidateId: id,\n\t\tcorrectnessScore: correctness,\n\t\tsafetyScore: safety,\n\t\treliabilityScore: reliability,\n\t\tcostScore: cost,\n\t\tlatencyScore: latency,\n\t\tuncertainty,\n\t\tsampleCount: evidence.sampleCount,\n\t\treasonCodes: [\"scored_from_evidence\"],\n\t\tevidenceIds: [evidence.evidenceHash],\n\t};\n}\n\n/**\n * Compute the aggregate score under a policy's explicit weights.\n * Safety is treated as a hard gate: if the safety score is present and below a\n * safety floor, aggregate collapses (safety must never be averaged away).\n */\nexport function aggregateScore(\n\tscore: OrchestrationCandidateScore,\n\tweights: AggregateWeights,\n\toptions: { safetyFloor?: number } = {},\n): number {\n\tconst safetyFloor = options.safetyFloor ?? 0.5;\n\tlet totalWeight = 0;\n\tlet acc = 0;\n\tacc += (score.correctnessScore ?? 0) * weights.correctness;\n\ttotalWeight += weights.correctness;\n\tacc += (score.reliabilityScore ?? 0) * weights.reliability;\n\ttotalWeight += weights.reliability;\n\tacc += (score.costScore ?? 0) * weights.cost;\n\ttotalWeight += weights.cost;\n\tacc += (score.latencyScore ?? 0) * weights.latency;\n\ttotalWeight += weights.latency;\n\n\t// Safety is a hard gate and always weighted (never averaged away).\n\tif (score.safetyScore !== undefined && score.safetyScore < safetyFloor) {\n\t\treturn -Infinity;\n\t}\n\tacc += (score.safetyScore ?? 0) * weights.safety;\n\ttotalWeight += weights.safety;\n\tif (totalWeight === 0) return 0;\n\treturn acc / totalWeight;\n}\n\nexport interface SelectionOptions {\n\tpolicy: SelectionPolicy;\n\tsafetyFloor?: number;\n}\n\nexport interface SelectionResult {\n\tselected: OrchestrationCandidateScore | undefined;\n\trunnersUp: OrchestrationCandidateScore[];\n\t/** Highest aggregate per candidate (pre-actual selection). */\n\tranked: { score: OrchestrationCandidateScore; aggregate: number; uncertaintyPenalty: number }[];\n\treasonCodes: string[];\n}\n\n/**\n * Multi-objective selection over scored candidates.\n * - Explicit operator objective via weights.\n * - Uncertainty penalizes: subtract uncertainty * uncertaintyWeight.\n * - Pareto-aware: if equal aggregates, keep deterministic tie-break.\n * - Missing evidence (aggregate would be 0 from all-undefined) is not preferred\n *   over a candidate with real (even partial) evidence.\n */\nexport function selectBest(scored: OrchestrationCandidateScore[], options: SelectionOptions): SelectionResult {\n\tconst weights = WEIGHTS_BY_POLICY[options.policy];\n\tconst ranked = scored.map((score) => {\n\t\tconst raw = aggregateScore(score, weights, { safetyFloor: options.safetyFloor });\n\t\tlet aggregate = raw;\n\t\tif (raw === -Infinity) {\n\t\t\taggregate = -Infinity;\n\t\t} else {\n\t\t\t// Uncertainty penalty.\n\t\t\tconst hasAnyEvidence =\n\t\t\t\t(score.correctnessScore ??\n\t\t\t\t\tscore.safetyScore ??\n\t\t\t\t\tscore.reliabilityScore ??\n\t\t\t\t\tscore.costScore ??\n\t\t\t\t\tscore.latencyScore) !== undefined;\n\t\t\tif (!hasAnyEvidence) {\n\t\t\t\t// Missing evidence: never prefer over a scored candidate; heavily penalized.\n\t\t\t\taggregate = Math.max(0, aggregate) * 0.1;\n\t\t\t} else {\n\t\t\t\taggregate = aggregate - score.uncertainty * 0.15;\n\t\t\t}\n\t\t}\n\t\treturn { score, aggregate, uncertaintyPenalty: score.uncertainty * 0.15 };\n\t});\n\n\t// Deterministic tie-break by candidateId after aggregation.\n\tranked.sort((a, b) => {\n\t\tif (a.aggregate === b.aggregate) return a.score.candidateId.localeCompare(b.score.candidateId);\n\t\treturn b.aggregate - a.aggregate;\n\t});\n\n\tconst reasonCodes: string[] = [`policy:${options.policy}`, `weights_v${AGGREGATE_WEIGHTS_VERSION}`];\n\tif (ranked.length === 0) {\n\t\treturn { selected: undefined, runnersUp: [], ranked, reasonCodes: [...reasonCodes, \"no_candidates\"] };\n\t}\n\tconst selected = ranked[0].score;\n\tif (selected.uncertainty >= 0.999) {\n\t\treasonCodes.push(\"insufficient_evidence\");\n\t}\n\treturn { selected, runnersUp: ranked.slice(1).map((r) => r.score), ranked, reasonCodes };\n}\n\n/** Infer a sensible default retrieval policy from features deterministically. */\nexport function inferRetrievalStrategy(features: {\n\tambiguity: number;\n\trequiresMutation: boolean;\n\ttaskCategory: string;\n\tlanguageIds: string[];\n}): RetrievalStrategy {\n\t// Exact identifiers don't need embeddings.\n\tif (features.languageIds.length === 0) return \"lexical\";\n\tif (features.ambiguity > 0.6) return \"hybrid\";\n\tif (features.requiresMutation) return \"hybrid\";\n\tif (features.taskCategory === \"analysis\") return \"hybrid\";\n\treturn \"lexical\";\n}\n"]}