/** * Selection: turn a classification into a concrete model decision. * * This is the economic core of the router: profile clamping, hysteresis, * candidate widening, the cache-aware stay/switch arithmetic, the budget * guard, fallbacks, cache breakpoints, probe planning, and capability clamps. */ import type { CatalogSnapshot } from "../catalog/types.ts"; import type { ProfileConfig, RouterConfig } from "../config/types.ts"; import { forecast, priceAt } from "../cost/forecast.ts"; import type { LedgerSignals, ModelCacheReliability } from "../cost/types.ts"; import { explorationDraw } from "./explore.ts"; import type { CompactionEdit, NormRequest, ReasoningLevel } from "../wire/types.ts"; import { compactedBytes, planCompaction, validatePlan, type CompactionResult } from "./compaction.ts"; import { planCacheBreakpoints } from "./cache-control.ts"; import { buildCandidates } from "./candidates.ts"; import { TIER_ORDER, type Candidate, type Classification, type ConversationState, type Decision, type Exploration, type Features, type ProbePlan, type Rejection, type Tier, } from "./types.ts"; export interface SelectArgs { req: NormRequest; features: Features; classification: Classification; profile: ProfileConfig; state: ConversationState; snapshot: CatalogSnapshot; cfg: RouterConfig; nowMs: number; /** * Slugs that already failed on this turn. Passed straight to * `buildCandidates` so a failover retry lands on a different model. */ excludeSlugs?: readonly string[]; /** * Session override (/router pin): route to this slug. Admitted into the * tier as if pinned there, then chosen over ranking, hysteresis and the * stay/switch comparison. Ignored when the catalog has no such model. */ forceSlug?: string; /** Ledger reads already performed for this turn; see TurnReads. */ reads?: TurnReads; } /** * Every ledger read a turn needs, fetched BEFORE selection. * * `select` is synchronous on purpose — it is a pure ranking function, and * `explain` depends on being able to run it without side effects. A ledger on * Postgres cannot be read synchronously, so the reads move up to `route`, * which is already async, and arrive here as data. Absent ⇒ read through the * absent reads degrade to no signals rather than reaching for the store. */ export interface TurnReads { /** Trust and latency per candidate slug; one batch query per signal kind. */ signals?: Map; /** Observed warm-cache hit rate per slug. */ cacheReliability?: Map; /** What an escalated retry bills per prompt token; null ⇒ the term is inert. */ escalationUsdPerPromptToken?: number | null; /** Spend since the start of the UTC month, scoped as the filters say. */ monthSpendUsd?: number; /** Spend over the rolling 24h, scoped as the filters say. */ daySpendUsd?: number; } /** * Thrown when the budget guard rejects a turn. `status`/`code` mirror the * WireError shape so the server can render it as a 402-class response. */ export class BudgetExceededError extends Error { readonly status = 402; readonly code = "budget_exceeded"; constructor(message: string) { super(message); this.name = "BudgetExceededError"; } } // Mid-range completion assumption for forecasts. Long generations amortize // into prompt-dominated cost anyway; precision here does not move rankings. const EXPECTED_COMPLETION_TOKENS = 1024; /** * Authors known to accept replayed assistant reasoning over chat completions: * Anthropic requires thinking-block replay for tool-use continuity, and Google * requires thought signatures. Everything else gets reasoning stripped — a * rejected replay costs a whole turn. */ const REASONING_REPLAY_AUTHORS: Record = { anthropic: true, google: true, }; function tierIdx(t: Tier): number { return TIER_ORDER.indexOf(t); } function tierAt(i: number): Tier | null { const t = TIER_ORDER[i]; return t === undefined ? null : t; } /** Tier search order: the tier itself, then one up, one down, two up, ... within [minTier, maxTier]. */ function wideningOrder(tier: Tier, minTier: Tier, maxTier: Tier): Tier[] { const lo = tierIdx(minTier); const hi = tierIdx(maxTier); const c = tierIdx(tier); const out: Tier[] = []; for (let d = 0; d <= Math.max(hi - c, c - lo); d++) { const up = tierAt(c + d); const down = tierAt(c - d); if (d === 0) { if (up !== null) out.push(up); continue; } if (up !== null && c + d <= hi) out.push(up); if (down !== null && c - d >= lo) out.push(down); } return out; } /** Evidence that an upgrade will stick: the conversation is failing or asked for deep reasoning. */ function hardSignal(f: Features): boolean { const reasoning: string = f.requestedReasoning ?? ""; return f.lastToolFailed || f.circularToolCall || f.repeatedToolCall || reasoning === "high" || reasoning === "xhigh" || reasoning === "max"; } /** Start of the UTC calendar month containing `nowMs`. */ export function monthStartMs(nowMs: number): number { const d = new Date(nowMs); return Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), 1); } /** The daily ceiling that spends what remains of a monthly target evenly over the days left (today included). */ export function monthPace(nowMs: number, perMonthUsd: number, spentUsd: number): { spentUsd: number; daysLeft: number; dailyCapUsd: number } { const d = new Date(nowMs); const daysInMonth = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth() + 1, 0)).getUTCDate(); const daysLeft = Math.max(1, daysInMonth - d.getUTCDate() + 1); return { spentUsd, daysLeft, dailyCapUsd: Math.max(0, (perMonthUsd - spentUsd) / daysLeft) }; } export function select(args: SelectArgs): Decision { const { req, features, classification, profile, state, snapshot, cfg, nowMs } = args; const reasons: string[] = []; const minI = tierIdx(profile.minTier); const maxI = tierIdx(profile.maxTier); const clampTier = (t: Tier): Tier => tierAt(Math.min(Math.max(tierIdx(t), minI), maxI)) ?? t; // 1. Clamp the classified tier to the requesting profile's envelope. let effective = clampTier(classification.tier); if (effective !== classification.tier) { reasons.push(`classified ${classification.tier}, clamped to profile ${profile.id} [${profile.minTier}..${profile.maxTier}] → ${effective}`); } // 2. Hysteresis: while the sticky window is open, never route below the // held tier — per-turn flapping would repeatedly cold-start prompt caches. // // Exception, when `breakHoldOnMechanical` is on: a hold is a bet that the // NEXT turn resembles the one that armed it. A tool-result continuation // whose own score lands below the held tier is direct evidence against // that bet — the classifier already docks it for being a mechanical next // step — so paying the held tier for it buys nothing. Measured on 24h of // live traffic: 37 of 44 sticky `hard` dispatches were exactly this shape, // one scoring 0.154 (trivial) yet served by claude-opus-5; $2.66 billed // against $0.05 for the identical tokens on the moderate pick. let cls = classification; const mechanicalOverride = cfg.hysteresis.breakHoldOnMechanical && features.isToolResultContinuation && tierIdx(effective) < tierIdx(clampTier(state.currentTier ?? effective)); if (state.stickyUntilTurn > state.turn && state.currentTier !== null && tierIdx(state.currentTier) >= tierIdx(effective)) { const held = clampTier(state.currentTier); if (mechanicalOverride) { reasons.push(`hysteresis hold ${held} broken: mechanical tool-result continuation classified ${effective}`); } else if (held !== effective) { reasons.push(`hysteresis: holding ${held} until turn ${state.stickyUntilTurn} (classified ${effective})`); cls = { ...classification, tier: held, source: "sticky", reasons: [`hysteresis hold ${held} until turn ${state.stickyUntilTurn}`, ...classification.reasons], }; effective = held; } } // Never downgrade more than maxDowngradePerTurn tiers in one turn, so // quality never falls off a cliff on a single odd classification. if (state.currentTier !== null) { const floor = Math.max(tierIdx(state.currentTier) - cfg.hysteresis.maxDowngradePerTurn, minI); if (tierIdx(effective) < floor) { const clamped = tierAt(floor); if (clamped !== null) { reasons.push(`downgrade limited to ${cfg.hysteresis.maxDowngradePerTurn} tier(s)/turn: ${effective} → ${clamped}`); effective = clamped; } } } // Whether a usable warm prompt cache exists right now. Shared by // exploration (2c) and candidate building (3) so both agree on the term. const cacheWarm = state.cacheWarmSlug !== null && nowMs - state.cacheWarmAtMs <= cfg.hysteresis.cacheWarmTtlMs; // 2a. Cache-aware upgrade confirmation. A low-confidence heuristic upgrade // from a warm model waits one turn; the next turn's classification // confirms or forgets it. Measured on 7 days of live traffic: 65 of 67 // moderate→hard upgrades bounced back within 3 turns, 50 of them below // 0.6 confidence, and each paid a cold hard-tier read of a ~120k prompt // ($17.90 in total against $0.23 for staying warm). Step 4's stay/switch // comparison never sees these — the warm cheap model is below the new // tier's floor, so it is not a candidate there. Escalations, explicit // high reasoning and failing tool loops bypass the wait: those are the // upgrades that stick. let upgradeDeferred: Tier | null = null; const confirmBelow = cfg.hysteresis.confirmUpgradesBelowConfidence; if ( confirmBelow > 0 && cls.source === "heuristic" && classification.confidence < confirmBelow && state.currentTier !== null && state.currentSlug !== null && tierIdx(effective) > tierIdx(clampTier(state.currentTier)) && cacheWarm && state.cacheWarmSlug === state.currentSlug && (args.excludeSlugs === undefined || args.excludeSlugs.length === 0) && !hardSignal(features) ) { const held = clampTier(state.currentTier); if (state.upgradeDeferredTier !== undefined && state.upgradeDeferredTier !== null) { reasons.push(`upgrade ${held} → ${effective} confirmed: classified above ${held} on consecutive turns`); } else { reasons.push( `upgrade ${held} → ${effective} deferred one turn: heuristic confidence ${classification.confidence.toFixed(2)} < ${confirmBelow} with ${state.currentSlug} warm`, ); upgradeDeferred = effective; effective = held; } } // 2b. Context compaction: shrink stale tool output before dispatch when the // prompt exceeds the token budget (or would overflow the profile window). // Deterministic and content-only (never removes a message), so downstream // forecasting, the context_too_small filter, cache breakpoints, and the // agentdox block append all operate on the compacted size / stay valid. // // Two properties make this cache-safe, and both are load-bearing: // // 1. The plan is PERSISTED per conversation and re-applied verbatim. omp // re-sends the original bytes every turn, so a re-applied edit yields // byte-identical output; a plan re-derived from scratch could differ // (a looser target, a re-tuned knob) and rewrite already-cached bytes. // 2. Compaction is triggered on the COMPACTED size and then overshoots // to `floorRatio` of the budget. Comparing the RAW prompt against the // budget re-planned on every single turn, so the plan gained one more // edit per turn — and each plan change rewrites cached prompt bytes. // Measured on live ledger data (7 long conversations, 894 compacted // dispatches): a turn whose plan changed ran 15.4% cold vs 8.9% when // the plan held, and a cold prompt costs 4.34x a warm one per token. // Overshooting buys several byte-stable turns per plan change. let compactionPlan: CompactionEdit[] = []; let promptTokensSaved = 0; let compactionSavedBytes = 0; let compactionPlanTokens = state.compactionPlanTokens ?? 0; let effFeatures = features; if (cfg.compaction.enabled && req.promptBytes > 0 && features.promptTokens > 0) { const bytesPerToken = req.promptBytes / features.promptTokens; const carried = validatePlan(state.compactionPlan ?? [], req.messages); const carriedSavedBytes = carried.reduce((sum, e) => sum + (e.bytes - compactedBytes(e.bytes, e)), 0); const tokensOf = (savedBytes: number): number => Math.min(features.promptTokens - 1, Math.round(features.promptTokens * (savedBytes / req.promptBytes))); // What the upstream would actually receive if nothing new were planned. const compactedTokens = features.promptTokens - tokensOf(carriedSavedBytes); const headroom = cfg.filters.contextHeadroom; const overBudget = compactedTokens > cfg.compaction.budgetTokens; const overWindow = cfg.compaction.fitToWindow && compactedTokens * headroom + EXPECTED_COMPLETION_TOKENS > profile.contextWindow; // Rationing: when the budget is unreachable (the observed case — compacted // prompts of 100-160k against a 40k budget) `overBudget` is true on every // turn, and a plan gains an edit the moment a tool result ages out of the // protected window, so `floorRatio` never gets to hold a plan. Instead, // extend an existing plan only once the compacted prompt has grown by // `replanGrowthRatio` since it was made. Never rations the window fit. const rationed = cfg.compaction.replanGrowthRatio > 1 && carried.length > 0 && compactionPlanTokens > 0 && compactedTokens < compactionPlanTokens * cfg.compaction.replanGrowthRatio; let plan: CompactionResult = { edits: carried, savedBytes: carriedSavedBytes }; if (overWindow || (overBudget && !rationed)) { const targets: number[] = []; // Overshoot the budget so the next re-plan is several turns away. if (overBudget) targets.push(Math.max(1, Math.floor(cfg.compaction.budgetTokens * cfg.compaction.floorRatio))); if (overWindow) targets.push(Math.max(1, Math.floor((profile.contextWindow - EXPECTED_COMPLETION_TOKENS) / headroom))); const targetBytes = Math.min(...targets) * bytesPerToken; plan = planCompaction(req.messages, cfg.compaction, targetBytes, req.promptBytes, carried); } if (plan.edits.length > 0) { compactionPlan = [...plan.edits]; compactionSavedBytes = plan.savedBytes; promptTokensSaved = tokensOf(plan.savedBytes); effFeatures = { ...features, promptTokens: features.promptTokens - promptTokensSaved }; const added = plan.edits.length - carried.length; // A plan that gained edits was made at THIS compacted size; a carried // plan keeps the size it was made at, so growth accrues against it. if (added > 0 || compactionPlanTokens === 0) compactionPlanTokens = effFeatures.promptTokens; reasons.push( `compaction: ${plan.edits.length} tool result(s) shrunk (${carried.length} carried, ${added} new), ~${promptTokensSaved} tokens saved (prompt ${features.promptTokens}→${effFeatures.promptTokens})${rationed ? " [re-plan rationed]" : ""}`, ); } else { compactionPlanTokens = 0; } } // 2c. Epsilon-greedy exploration: on a small deterministic fraction of // turns, route one tier BELOW the tier we would otherwise use, so the // ledger witnesses whether the cheaper tier would have sufficed. // // Escalation is what makes this safe rather than reckless: if the // cheaper tier flounders, the probe rejects the attempt and the turn // escalates, so a bad draw costs one wasted cheap attempt rather than // a failed turn. // // Sticky turns are explored only once their cache has gone cold. The // hold exists to protect a warm cache, so exploring while one is live // would destroy precisely what the hold is for; once it has expired // the objection lapses. This matters more than it sounds: most // expensive turns reach their tier by hold rather than by // classification, so excluding held turns confines exploration to the // cheapest boundary in the system. // // Still skipped on forced escalations (the probe already proved the // cheaper tier failed) and on failover retries, where a second // confound is not wanted. // // This deliberately bypasses maxDowngradePerTurn by one tier: that // limiter guards against a noisy classification, not against a probe // that is sampled on purpose and escalates when it is wrong. let explored: Exploration | null = null; const ex = cfg.exploration; const stickyAllows = cls.source !== "sticky" || ex.stickyPolicy === "always" || (ex.stickyPolicy === "cold-cache" && !cacheWarm); const tierRate = ex.rates[effective] ?? 0; if ( ex.enabled && tierRate > 0 && stickyAllows && classification.source !== "escalation" && (args.excludeSlugs === undefined || args.excludeSlugs.length === 0) ) { const target = tierAt(Math.max(tierIdx(effective) - 1, minI)); if (target !== null && target !== effective && explorationDraw(`explore:${req.conversationKey}:${state.turn}`) < tierRate) { const held = cls.source === "sticky" ? `, held tier (${cacheWarm ? "warm" : "cold"} cache)` : ""; reasons.push(`exploration: deliberately routing ${effective} → ${target} (rate ${tierRate}${held})`); explored = { from: effective, to: target }; effective = target; } } // 3. Candidates for the effective tier; widen one tier upward, then // downward, and only fail when the whole profile envelope is exhausted. // The task type selects the quality axis and capability filters; the tier // still bounds cost (task selects, tier budgets). const warmSlug = cacheWarm ? state.cacheWarmSlug : null; // Expected cache hit for a model: its observed rate once enough warm-expected // samples exist (filters.cacheReliabilityMinSamples), else a reliable 1. const reads = args.reads; const cacheHitExpectation = (slug: string): { rate: number; measured: boolean; samples: number } => { const min = cfg.filters.cacheReliabilityMinSamples; const rel = min > 0 ? (reads?.cacheReliability?.get(slug) ?? null) : null; if (rel === null || rel.samples < min) return { rate: 1, measured: false, samples: rel?.samples ?? 0 }; return { rate: rel.hitRate, measured: true, samples: rel.samples }; }; // Trust and latency for every candidate, fetched by `route` before this ran. // There is no fallback read here on purpose: selection is synchronous and the // store may be a shared database, so a missing prefetch must degrade to // "no signals" rather than silently reach for a handle it cannot await. const candidateSignals = reads?.signals; // What an escalated retry has actually been billing per prompt token, for // the escalation-cost term in candidate scoring. Read once per turn; null // (term inert) when the weight is 0 or the ledger has too few samples. const escalationUsdPerPromptToken = cfg.filters.escalationCostWeight > 0 ? (reads?.escalationUsdPerPromptToken ?? null) : null; // A pinned slug is admitted the way a config pin is: into the tier's pin // list for this call only, so the quality floor cannot keep it out. const pinSlug = args.forceSlug !== undefined && snapshot.models.some((m) => m.slug === args.forceSlug) ? args.forceSlug : undefined; const buildCfg = pinSlug === undefined ? cfg : { ...cfg, tiers: { ...cfg.tiers, [effective]: { ...cfg.tiers[effective], pin: [...cfg.tiers[effective].pin, pinSlug] } } }; if (args.forceSlug !== undefined && pinSlug === undefined) reasons.push(`pin ${args.forceSlug} ignored: not in the catalog`); const build = (t: Tier, relaxLevel = 0): { candidates: Candidate[]; rejected: Rejection[] } => buildCandidates({ req, features: effFeatures, tier: t, task: classification.task, snapshot, cfg: buildCfg, expectedCompletionTokens: EXPECTED_COMPLETION_TOKENS, warmSlug, relaxLevel, ...(args.excludeSlugs === undefined ? {} : { excludeSlugs: args.excludeSlugs }), ...(candidateSignals === undefined ? {} : { signals: candidateSignals }), ...(escalationUsdPerPromptToken === null ? {} : { escalationUsdPerPromptToken }), }); let chosenTier = effective; let built: { candidates: Candidate[]; rejected: Rejection[] } | null = null; // Relax level the tier rescue used (0 = no rescue). The budget downgrade // search must rebuild at the same level, or it re-applies the strict config let rescuedRelax = 0; for (const t of wideningOrder(effective, profile.minTier, profile.maxTier)) { // A session pin is absolute: price ceiling, quality floor and trust all // relax so the pinned model is admitted; hard exclusions still apply. const b = build(t, pinSlug === undefined ? 0 : 3); if (b.candidates.length > 0) { built = b; chosenTier = t; break; } reasons.push(`no candidates in ${t} (${b.rejected.length} rejected)`); built ??= b; } // Tier rescue: the configured envelopes (price ceilings, quality floors, // trust bar) were tuned against the full catalog, and a guardrail can shrink // availability so no configured tier admits anything. Rather than 500, relax // the tier's economic constraints — in order, price → quality → trust — until // some AVAILABLE model qualifies. Hard capability filters (tools/images/ // context) and the key-scoped allowlist are never lifted. if (built === null || built.candidates.length === 0) { const envelope = wideningOrder(effective, profile.minTier, profile.maxTier); let rescued = false; for (let relax = 1; relax <= 3 && !rescued; relax++) { for (const t of envelope) { const b = build(t, relax); if (b.candidates.length > 0) { built = b; chosenTier = t; rescued = true; rescuedRelax = relax; break; } } } if (rescued) { const first = built!.candidates[0]; const label = rescuedRelax === 1 ? "price ceilings" : rescuedRelax === 2 ? "price ceilings + quality floors" : "price ceilings + quality floors + trust bar"; reasons.push( `tier rescue: strict config excluded all available models; relaxed ${label} to pick ${first!.model.slug} (${chosenTier})`, ); } else if (built === null || built.candidates.length === 0) { throw new Error(`no viable model: catalog exhausted across profile ${profile.id}`); } } if (chosenTier !== effective) reasons.push(`widened ${effective} → ${chosenTier}`); // After the rescue block, `built` is guaranteed non-null: the branch either // rescued a non-empty candidate set, or threw the `catalog exhausted` error. const resolved = built as { candidates: Candidate[]; rejected: Rejection[] }; let candidates = resolved.candidates; const first = candidates[0]; if (first === undefined) throw new Error(`no viable model: catalog exhausted across profile ${profile.id}`); let chosen = first; const pinnedCandidate = pinSlug === undefined ? undefined : candidates.find((c) => c.model.slug === pinSlug); if (pinnedCandidate !== undefined) { chosen = pinnedCandidate; reasons.push(`pinned to ${pinSlug} by session override (/router pin)`); } // 4. Cache-aware switch decision. Staying prices the previous turn's prompt // at the warm model's cache-read rate; switching prices the full current // prompt at the new model's cold rate plus its cache-write premium (we // assume the whole prompt is written). Switch only when the saving // clears switchMargin. let sticky = false; if (pinnedCandidate === undefined && warmSlug !== null && chosen.model.slug !== warmSlug) { const warm = candidates.find((c) => c.model.slug === warmSlug); if (warm !== undefined) { const warmPrice = priceAt(warm.model, Math.max(1, state.lastPromptTokens)); const newPrice = priceAt(chosen.model, Math.max(1, effFeatures.promptTokens)); // A warm read is only as cheap as the cache is reliable: price the // expected mix of hits and provider-side misses, per model. const warmHit = cacheHitExpectation(warm.model.slug); const newHit = cacheHitExpectation(chosen.model.slug); const stayWarm = state.lastPromptTokens * (warmHit.rate * (warmPrice.cacheRead ?? warmPrice.prompt) + (1 - warmHit.rate) * warmPrice.prompt); const switchCold = effFeatures.promptTokens * (newPrice.prompt + (newPrice.cacheWrite ?? 0)); const newWarm = effFeatures.promptTokens * (newHit.rate * (newPrice.cacheRead ?? newPrice.prompt) + (1 - newHit.rate) * newPrice.prompt); const hitNote = warmHit.measured ? `, warm hit ${(warmHit.rate * 100).toFixed(0)}% over ${warmHit.samples}` : ""; // Amortise over the horizon: H turns of staying warm against one cold // switch plus H−1 turns warm on the new model. H = 1 is the one-turn // comparison, which kept a 25x-priced model warm for a 33-dispatch run // because no single turn could recoup the cold write on its own. const horizon = Math.max(1, cfg.hysteresis.switchHorizonTurns); const stayCost = horizon * stayWarm; const switchCost = switchCold + (horizon - 1) * newWarm; const over = horizon > 1 ? ` over ${horizon} turns` : ""; if (stayCost > switchCost * cfg.hysteresis.switchMargin) { reasons.push( `cache: switch ${warmSlug} → ${chosen.model.slug} (stay $${stayCost.toFixed(4)} > switch $${switchCost.toFixed(4)} × ${cfg.hysteresis.switchMargin}${over}${hitNote})`, ); } else { chosen = warm; sticky = true; reasons.push( `cache: keeping warm ${warmSlug} (stay $${stayCost.toFixed(4)} ≤ switch $${switchCost.toFixed(4)} × ${cfg.hysteresis.switchMargin}${over}${hitNote})`, ); } } } // 5. Budget guard, against the COLD forecast: a budget must survive a cache miss. const budget = { perTurnUsd: profile.budget?.perTurnUsd ?? cfg.budget.perTurnUsd, perConversationUsd: profile.budget?.perConversationUsd ?? cfg.budget.perConversationUsd, perDayUsd: profile.budget?.perDayUsd ?? cfg.budget.perDayUsd, perMonthUsd: profile.budget?.perMonthUsd ?? cfg.budget.perMonthUsd, onExceeded: profile.budget?.onExceeded ?? cfg.budget.onExceeded, }; // Month pacing: what is left of the month's target, spread over the days // left, becomes a daily ceiling that tightens as the month runs ahead. let paceNote = ""; if (budget.perMonthUsd !== undefined) { const monthSpend = reads?.monthSpendUsd ?? 0; const pace = monthPace(nowMs, budget.perMonthUsd, monthSpend); if (budget.perDayUsd === undefined || pace.dailyCapUsd < budget.perDayUsd) { budget.perDayUsd = pace.dailyCapUsd; paceNote = ` (month pacing: $${pace.spentUsd.toFixed(2)} of $${budget.perMonthUsd} spent, $${pace.dailyCapUsd.toFixed(2)}/day for ${pace.daysLeft} more days)`; } } const daySpend = budget.perDayUsd !== undefined ? (reads?.daySpendUsd ?? 0) : 0; const breach = (c: Candidate): string | null => { if (budget.perTurnUsd !== undefined && c.forecast.coldUsd > budget.perTurnUsd) { return `cold forecast $${c.forecast.coldUsd.toFixed(4)} > per-turn budget $${budget.perTurnUsd}`; } if (budget.perConversationUsd !== undefined && state.spentUsd + c.forecast.coldUsd > budget.perConversationUsd) { return `conversation spend $${state.spentUsd.toFixed(4)} + cold forecast > per-conversation budget $${budget.perConversationUsd}`; } if (budget.perDayUsd !== undefined) { // Scope the rolling 24h ceiling to the requesting harness when it // identifies itself, so multiple harnesses sharing one router each get // their own daily budget instead of one exhausting it for the others. if (daySpend + c.forecast.coldUsd > budget.perDayUsd) { return `24h spend $${daySpend.toFixed(4)} + cold forecast > per-day budget $${budget.perDayUsd.toFixed(2)}${paceNote}`; } } return null; }; let budgetDowngraded = false; const why = breach(chosen); if (why !== null) { if (budget.onExceeded === "reject") throw new BudgetExceededError(why); // Downgrade: the cheapest candidate in the cheapest tier that fits. let rescue: { tier: Tier; candidate: Candidate; candidates: Candidate[] } | null = null; for (const t of wideningOrder(profile.minTier, profile.minTier, profile.maxTier)) { const b = build(t, rescuedRelax); let cheapest: Candidate | null = null; for (const c of b.candidates) { if (cheapest === null || c.forecast.coldUsd < cheapest.forecast.coldUsd) cheapest = c; } if (cheapest !== null && breach(cheapest) === null) { rescue = { tier: t, candidate: cheapest, candidates: b.candidates }; break; } } if (rescue === null) throw new BudgetExceededError(`${why}; no cheaper candidate fits the budget`); reasons.push(`budget: ${why}; downgraded ${chosenTier} → ${rescue.tier} (${rescue.candidate.model.slug})`); chosen = rescue.candidate; chosenTier = rescue.tier; candidates = rescue.candidates; budgetDowngraded = true; sticky = false; } // 6. Same-tier fallbacks for OpenRouter's transient-error `models[]` cascade. const fallbacks: string[] = []; for (const c of candidates) { if (c.model.slug === chosen.model.slug) continue; // The cascade is served by ONE upstream: an OpenRouter `models[]` array // cannot name an Ollama model and vice versa. if (c.model.provider !== chosen.model.provider) continue; fallbacks.push(c.model.slug); if (fallbacks.length >= 2) break; } // 7. Cache breakpoints, measured over post-compaction sizes so the // boundaries match the bytes that actually get dispatched. const cacheBreakpointMessageIndices = planCacheBreakpoints(req, chosen.model, cfg, compactionPlan); // 8. Guarded probe: only tiers configured for probing, and only when a // strictly higher tier exists inside the profile envelope to escalate into. const nextTier = tierAt(tierIdx(chosenTier) + 1); const escalateTo = nextTier !== null && tierIdx(nextTier) <= maxI ? nextTier : null; const probeEnabled = cfg.escalation.enabled && escalateTo !== null && cfg.escalation.probeTiers.includes(chosenTier); const probe: ProbePlan = { enabled: probeEnabled, maxTokens: cfg.escalation.probeTokens, maxHoldMs: cfg.escalation.maxHoldMs, escalateTo: probeEnabled ? escalateTo : null, }; // 9. Clamp reasoning and output budget to what the target supports. let reasoning: ReasoningLevel | undefined = req.reasoning; if (reasoning !== undefined && !chosen.model.supportsReasoning) { if (reasoning !== "off") reasons.push(`dropped reasoning=${reasoning}: ${chosen.model.slug} does not support it`); reasoning = undefined; } if (chosen.model.reasoningMandatory && (reasoning === undefined || reasoning === "off")) { reasoning = "minimal"; reasons.push(`reasoning forced to minimal: ${chosen.model.slug} has mandatory reasoning`); } let maxTokens: number | undefined = req.maxTokens; const ceiling = chosen.model.maxCompletionTokens; if (ceiling !== undefined) { // The ceiling is a hard limit anyway; passing it explicitly also caps runaway completions. maxTokens = maxTokens === undefined ? ceiling : Math.min(maxTokens, ceiling); } // A reasoning model spends the budget thinking before it answers, so a caller's // tight cap returns nothing and the turn fails over having paid for the dispatch. // Raise it to the floor for those models only; the ceiling still wins. const floor = cfg.filters.reasoningCompletionFloor; const thinksBeforeAnswering = chosen.model.reasoningMandatory || (chosen.model.supportsReasoning && reasoning !== "off"); if (floor > 0 && thinksBeforeAnswering && maxTokens !== undefined && maxTokens < floor) { const raised = ceiling === undefined ? floor : Math.min(floor, ceiling); if (raised > maxTokens) { reasons.push(`completion budget raised ${maxTokens} → ${raised}: ${chosen.model.slug} reasons before it answers`); maxTokens = raised; } } const stripAssistantReasoning = !(chosen.model.supportsReasoning && REASONING_REPLAY_AUTHORS[chosen.model.author] === true); // The recorded forecast is the EXPECTED price of this dispatch, not the // cold worst case candidates are ranked on. When the chosen model's cache // is warm, the previous prompt's tokens are priced as cache reads at the // model's measured hit rate; coldUsd stays the cold figure the budget // guards used. Before this every recorded forecast was cold while nine // turns in ten were warm: 89% over-predicted, median error 220%. let expectedForecast = chosen.forecast; if (warmSlug !== null && chosen.model.slug === warmSlug && effFeatures.promptTokens > 0 && state.lastPromptTokens > 0) { const cachedShare = Math.min(1, state.lastPromptTokens / effFeatures.promptTokens); let images = 0; if (req.hasImages) for (const m of req.messages) images += m.images; const warm = forecast(chosen.model, { promptTokens: effFeatures.promptTokens, completionTokens: EXPECTED_COMPLETION_TOKENS, cacheHitRate: cacheHitExpectation(chosen.model.slug).rate * cachedShare, images, }); expectedForecast = { ...warm, coldUsd: chosen.forecast.coldUsd }; } return { slug: chosen.model.slug, fallbacks, tier: chosenTier, classification: cls, features, forecast: expectedForecast, sessionId: state.sessionId, sticky, cacheBreakpointMessageIndices, compactionPlan, promptTokensSaved, compactionSavedBytes, compactionPlanTokens, reasoning, maxTokens, stripAssistantReasoning, probe, considered: candidates, rejected: resolved.rejected, reasons, explored, budgetDowngraded, upgradeDeferred, }; }