import { BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT, MAX_DYNAMIC_ROUTES, MODEL_AGGREGATE_MAX_ROWS, MODEL_RANKING_EFFORT_COST_MULTIPLIER_HIGH, MODEL_RANKING_EFFORT_COST_MULTIPLIER_LOW, MODEL_RANKING_EFFORT_COST_MULTIPLIER_MEDIUM, MODEL_RANKING_UPLIFT_MIN_CONFIDENCE, MODEL_RANKING_UPLIFT_MIN_UTILITY_DELTA, } from "../../constants.ts"; import { type ModelMetricAggregate, type ModelTaskDomain, type ModelTaskEffort, type ModelTaskType, TASK_DOMAINS, TASK_EFFORTS, TASK_TYPES, } from "../../observability/model-observation.ts"; import { type BenchmarkObservation, normalizeModelIdentity } from "./benchmark.ts"; export type ScopeAuthority = "exact-session" | "available-models"; export type UtilityComponentName = "quality" | "cost" | "latency" | "context" | "reliability"; export interface ModelCandidate { provider: string; model: string; thinking: string; } export interface UtilityWeights { quality: number; cost: number; latency: number; context: number; reliability: number; } export interface ModelRankingInput { candidates: ModelCandidate[]; scopeAuthority: ScopeAuthority; domain: ModelTaskDomain; type: ModelTaskType; /** Predicts how much reasoning/engineering effort this turn needs; shifts the effective cost weight (see MODEL_RANKING_EFFORT_COST_MULTIPLIER_*) the same way budgetPressure already does. */ effort: ModelTaskEffort; /** The model actually in use right now -- the uplift gate's baseline. A recommendation requires a real, evidence-backed improvement over this, never just "ranked #1". Null when there is no active model to compare against (pure ranking still proceeds; there is simply nothing to gate an uplift over, so recommendation stays null). */ currentCandidate: ModelCandidate | null; budgetPressure: number; weights: UtilityWeights; externalEvidence: BenchmarkObservation[]; localEvidence: ModelMetricAggregate[]; now: number; } export interface ModelRankingRecommendation { candidate: ModelCandidate; utilityDelta: number; confidence: number; } export interface UtilityComponent { name: UtilityComponentName; score: number | null; confidence: number; weight: number; evidenceCount: number; reason: string; } export interface RankingProvenance { sourceId: string; publisher: string; url: string; revision: string; freshness: "fresh" | "stale"; } export interface RankedModel { candidate: ModelCandidate; identity: string; utility: number | null; confidence: number; components: UtilityComponent[]; provenance: RankingProvenance[]; trace: string[]; } export interface ModelRankingResult { scopeAuthority: ScopeAuthority; scopeWarning: string | null; domain: ModelTaskDomain; type: ModelTaskType; completeness: "complete" | "partial" | "insufficient-evidence"; ranked: RankedModel[]; /** A candidate that cleared the uplift gate (real utility margin + confidence over currentCandidate), regardless of scope authority -- the evidence question, independent of whether automation is currently allowed. Null when the current model is already the best choice, or no candidate clears the gate. */ recommendation: ModelRankingRecommendation | null; /** recommendation's candidate, but only once scopeAuthority is "exact-session" -- the governance question layered on top of the evidence question. */ automaticSelection: ModelCandidate | null; } interface RawComponent { value: number | null; confidence: number; evidenceCount: number; reason: string; lowerIsBetter: boolean; } const COMPONENTS: UtilityComponentName[] = ["quality", "cost", "latency", "context", "reliability"]; function finiteBound(value: number, name: string, minimum: number, maximum: number): number { if (!Number.isFinite(value) || value < minimum || value > maximum) throw new Error(`${name} is outside its supported range`); return value; } function candidateIdentity(candidate: ModelCandidate): string { const identity = normalizeModelIdentity(candidate.provider, candidate.model); if (typeof candidate.thinking !== "string" || candidate.thinking.length === 0 || candidate.thinking.length > 160) throw new Error("candidate thinking level is invalid"); return `${identity.canonical}:${candidate.thinking}`; } function average(values: number[]): number { return values.reduce((sum, value) => sum + value, 0) / values.length; } function externalValues( candidate: ModelCandidate, evidence: BenchmarkObservation[], dimensions: string[], now: number, ): { values: number[]; confidences: number[]; provenance: RankingProvenance[] } { const identity = normalizeModelIdentity(candidate.provider, candidate.model); const matching = evidence.filter( (item) => (item.model.canonical === identity.canonical || item.model.aliases.includes(identity.canonical)) && dimensions.includes(item.dimension), ); return { values: matching.map((item) => item.value), confidences: matching.map((item) => item.provenance.confidence * (now <= item.provenance.freshUntil ? 1 : 0.25)), provenance: matching.map((item) => ({ sourceId: item.provenance.sourceId, publisher: item.provenance.publisher, url: item.provenance.url, revision: item.provenance.revision, freshness: now <= item.provenance.freshUntil ? "fresh" : "stale", })), }; } function localValues( candidate: ModelCandidate, domain: ModelTaskDomain, type: ModelTaskType, evidence: ModelMetricAggregate[], dimension: string, ): ModelMetricAggregate[] { const identity = normalizeModelIdentity(candidate.provider, candidate.model); return evidence.filter( (item) => item.provider === identity.provider && item.model === identity.model && item.thinking === candidate.thinking && item.domain === domain && item.type === type && item.dimension === dimension, ); } /** * quality-{domain} is a subject-matter signal (e.g. quality-coding from a coding benchmark); * quality-type-{type} is an activity signal (e.g. quality-type-planning from an agentic/tool-use * benchmark). Both are optional and additive on top of the universal quality-general fallback -- * a candidate with no domain- or type-specific evidence still gets ranked on general quality * rather than being treated as having zero evidence. */ function qualityDimensions(domain: ModelTaskDomain, type: ModelTaskType): string[] { const dimensions: string[] = []; if (domain !== "general") dimensions.push(`quality-${domain}`); if (type !== "general") dimensions.push(`quality-type-${type}`); dimensions.push("quality-general"); return dimensions; } function rawComponents( candidate: ModelCandidate, input: ModelRankingInput, ): { components: Record; provenance: RankingProvenance[] } { const quality = externalValues(candidate, input.externalEvidence, qualityDimensions(input.domain, input.type), input.now); const priceInput = externalValues(candidate, input.externalEvidence, ["price-input"], input.now); const priceOutput = externalValues(candidate, input.externalEvidence, ["price-output"], input.now); const measuredLatency = externalValues(candidate, input.externalEvidence, ["latency"], input.now); const rankedLatency = externalValues(candidate, input.externalEvidence, ["latency-rank", "throughput-rank"], input.now); const latency = measuredLatency.values.length > 0 ? measuredLatency : rankedLatency; const context = externalValues(candidate, input.externalEvidence, ["context-window"], input.now); const localLatency = localValues(candidate, input.domain, input.type, input.localEvidence, "wall-latency"); const failures = localValues(candidate, input.domain, input.type, input.localEvidence, "failure"); const outcomes = localValues(candidate, input.domain, input.type, input.localEvidence, "outcome-accepted"); const qualityValues = quality.values; const prices = [...priceInput.values, ...priceOutput.values]; const latencyValues = localLatency.length > 0 ? localLatency.map((item) => item.median) : latency.values; const latencyConfidences = localLatency.length > 0 ? localLatency.map((item) => item.confidence) : latency.confidences; const reliabilityValues = [...failures.map((item) => 1 - item.median), ...outcomes.map((item) => item.median)]; const withEvidence = (values: number[], confidences: number[], lowerIsBetter: boolean, label: string): RawComponent => values.length === 0 ? { value: null, confidence: 0, evidenceCount: 0, reason: `${label} evidence is missing`, lowerIsBetter } : { value: average(values), confidence: average(confidences) / (1 + (Math.max(...values) - Math.min(...values)) / Math.max(Math.abs(average(values)), Number.EPSILON)), evidenceCount: values.length, reason: `${values.length} ${label} observation${values.length === 1 ? "" : "s"}`, lowerIsBetter, }; const components: Record = { quality: withEvidence(qualityValues, quality.confidences, false, "task quality"), cost: withEvidence(prices, [...priceInput.confidences, ...priceOutput.confidences], true, "price"), latency: withEvidence(latencyValues, latencyConfidences, true, "latency"), context: withEvidence(context.values, context.confidences, false, "context window"), reliability: withEvidence( reliabilityValues, [...failures, ...outcomes].map((item) => item.confidence), false, "local reliability", ), }; return { components, provenance: [...quality.provenance, ...priceInput.provenance, ...priceOutput.provenance, ...latency.provenance, ...context.provenance], }; } function normalizedScore(value: number, values: number[], lowerIsBetter: boolean): number { const minimum = Math.min(...values); const maximum = Math.max(...values); if (maximum === minimum) return 0.5; const score = (value - minimum) / (maximum - minimum); return lowerIsBetter ? 1 - score : score; } export function rankModelCandidates(value: ModelRankingInput): ModelRankingResult { if (!Array.isArray(value.candidates) || value.candidates.length === 0 || value.candidates.length > MAX_DYNAMIC_ROUTES) throw new Error("candidate count is outside its supported range"); if (!Array.isArray(value.externalEvidence) || value.externalEvidence.length > BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT * 4) throw new Error("external evidence exceeds the supported bound"); if (!Array.isArray(value.localEvidence) || value.localEvidence.length > MODEL_AGGREGATE_MAX_ROWS) throw new Error("local evidence exceeds the supported bound"); if (value.scopeAuthority !== "exact-session" && value.scopeAuthority !== "available-models") throw new Error("scope authority is invalid"); if (!TASK_DOMAINS.includes(value.domain)) throw new Error("task domain is invalid"); if (!TASK_TYPES.includes(value.type)) throw new Error("task type is invalid"); if (!TASK_EFFORTS.includes(value.effort)) throw new Error("task effort is invalid"); const currentIdentity = value.currentCandidate === null ? null : candidateIdentity(value.currentCandidate); if (!Number.isSafeInteger(value.now) || value.now <= 0) throw new Error("ranking time is invalid"); const budgetPressure = finiteBound(value.budgetPressure, "budget pressure", 0, 2); const weights = Object.fromEntries( COMPONENTS.map((name) => [name, finiteBound(value.weights[name], `${name} weight`, 0, 10)]), ) as unknown as UtilityWeights; const effortCostMultiplier = value.effort === "low" ? MODEL_RANKING_EFFORT_COST_MULTIPLIER_LOW : value.effort === "high" ? MODEL_RANKING_EFFORT_COST_MULTIPLIER_HIGH : MODEL_RANKING_EFFORT_COST_MULTIPLIER_MEDIUM; const seen = new Set(); const candidates = value.candidates .map((candidate) => ({ ...candidate })) .filter((candidate) => { const identity = candidateIdentity(candidate); if (seen.has(identity)) return false; seen.add(identity); return true; }); const raw = candidates.map((candidate) => rawComponents(candidate, value)); const effectiveWeights: UtilityWeights = { ...weights, cost: weights.cost * (1 + budgetPressure) * effortCostMultiplier }; const ranked = candidates .map((candidate, index): RankedModel => { const source = raw[index]!; const components = COMPONENTS.map((name): UtilityComponent => { const component = source.components[name]; const comparable = raw.map((item) => item.components[name].value).filter((item): item is number => item !== null); return { name, score: component.value === null ? null : normalizedScore(component.value, comparable, component.lowerIsBetter), confidence: component.confidence, weight: effectiveWeights[name], evidenceCount: component.evidenceCount, reason: component.reason, }; }); const known = components.filter( (component): component is UtilityComponent & { score: number } => component.score !== null && component.weight > 0, ); const knownWeight = known.reduce((sum, component) => sum + component.weight, 0); const totalWeight = components.reduce((sum, component) => sum + component.weight, 0); const utility = knownWeight === 0 ? null : known.reduce((sum, component) => sum + component.score * component.weight, 0) / knownWeight; const confidence = totalWeight === 0 ? 0 : known.reduce((sum, component) => sum + component.confidence * component.weight, 0) / totalWeight; const provenance = [ ...new Map(source.provenance.map((item) => [`${item.sourceId}:${item.revision}:${item.url}`, item])).values(), ].sort((left, right) => left.sourceId.localeCompare(right.sourceId) || left.revision.localeCompare(right.revision)); return { candidate, identity: candidateIdentity(candidate), utility, confidence, components, provenance, trace: [ `domain ${value.domain}, type ${value.type}, effort ${value.effort}`, `budget pressure ${budgetPressure.toFixed(3)} and effort ${value.effort} make cost weight ${effectiveWeights.cost.toFixed(3)}`, `${known.length}/${components.length} utility components have evidence`, `scope authority ${value.scopeAuthority}`, ], }; }) .sort( (left, right) => (right.utility ?? -1) - (left.utility ?? -1) || right.confidence - left.confidence || left.identity.localeCompare(right.identity), ); const knownComponents = ranked.reduce((sum, item) => sum + item.components.filter((component) => component.score !== null).length, 0); const possibleComponents = ranked.length * COMPONENTS.length; const completeness = knownComponents === 0 ? "insufficient-evidence" : knownComponents === possibleComponents ? "complete" : "partial"; const exact = value.scopeAuthority === "exact-session"; const top = ranked[0]; const current = currentIdentity === null ? undefined : ranked.find((item) => item.identity === currentIdentity); // Uplift gate (Cursor-style): a recommendation requires a real, evidence-backed improvement // over the current model -- never just "ranked #1". Without a located current baseline (it // wasn't among the input candidates) there is nothing to uplift over, so no recommendation. const recommendation: ModelRankingRecommendation | null = top && current && top.identity !== current.identity && top.utility !== null && current.utility !== null ? (() => { const utilityDelta = top.utility! - current.utility!; return utilityDelta >= MODEL_RANKING_UPLIFT_MIN_UTILITY_DELTA && top.confidence >= MODEL_RANKING_UPLIFT_MIN_CONFIDENCE ? { candidate: top.candidate, utilityDelta, confidence: top.confidence } : null; })() : null; return { scopeAuthority: value.scopeAuthority, scopeWarning: exact ? null : "Pi available models are not the exact session scope; automatic selection is disabled", domain: value.domain, type: value.type, completeness, ranked, recommendation, automaticSelection: exact && recommendation ? recommendation.candidate : null, }; }