import { describe, expect, test } from "bun:test"; import { fakeLedger } from "./fakes.ts"; import type { ModelLatency } from "../src/cost/types.ts"; import { prefetchTurnReads } from "../src/router/index.ts"; import type { AsyncLedger } from "../src/cost/types.ts"; import { normalizeCatalogModel } from "../src/catalog/openrouter-catalog.ts"; import type { CatalogModel, CatalogSnapshot } from "../src/catalog/types.ts"; import { DEFAULT_CONFIG } from "../src/config/defaults.ts"; import { loadConfig } from "../src/config/load.ts"; import type { ProfileConfig, RouterConfig } from "../src/config/types.ts"; import { extractFeatures } from "../src/router/features.ts"; import { scoreHeuristic } from "../src/router/classify.ts"; import { latencyWeightFor } from "../src/router/candidates.ts"; import { BudgetExceededError, monthPace, monthStartMs, select } from "../src/router/select.ts"; import type { ConversationState, Tier } from "../src/router/types.ts"; import { parseChatRequest } from "../src/wire/openai/request.ts"; import type { NormRequest } from "../src/wire/types.ts"; const FIXTURE = (await Bun.file("test/fixtures/openrouter-models.json").json()) as { data: unknown[] }; const MODELS: CatalogModel[] = FIXTURE.data.map(normalizeCatalogModel).filter((m): m is CatalogModel => m !== null); const SNAPSHOT: CatalogSnapshot = { models: MODELS, fetchedAtMs: Date.now() }; const BASE = loadConfig({}); const PROFILE: ProfileConfig = { id: "auto", name: "Auto", minTier: "trivial", maxTier: "hard", contextWindow: 400_000, maxTokens: 32_000, }; const TOOLS = [ { type: "function", function: { name: "read", description: "Read a file", parameters: { type: "object", properties: { path: { type: "string" } } }, }, }, ]; function request(userText: string): NormRequest { return parseChatRequest( { model: "auto", tools: TOOLS, messages: [ { role: "system", content: "You are a coding agent." }, { role: "user", content: userText }, ], }, new Headers(), ); } function state(over: Partial = {}): ConversationState { return { key: "abc123", sessionId: "omp-abc123", turn: 1, currentSlug: null, currentTier: null, stickyUntilTurn: 0, escalations: 0, spentUsd: 0, lastPromptTokens: 0, cacheWarmSlug: null, cacheWarmAtMs: 0, contextVersion: null, contextFetchedAtMs: 0, compactionPlan: null, updatedAtMs: Date.now(), ...over, }; } async function run(opts: { userText?: string; promptTokens?: number; cfg?: RouterConfig; st?: ConversationState; tier?: Tier; ledger?: AsyncLedger | null; harnessId?: string; maxTokens?: number; }) { const cfg = opts.cfg ?? BASE; const base = request(opts.userText ?? "tidy the retry helper"); const req = opts.maxTokens === undefined ? base : { ...base, maxTokens: opts.maxTokens }; const features = extractFeatures(req, opts.promptTokens ?? 4000); const heuristic = scoreHeuristic(features, cfg); const classification = opts.tier === undefined ? heuristic : { ...heuristic, tier: opts.tier }; const finalReq = opts.harnessId === undefined ? req : { ...req, harnessId: opts.harnessId }; // `select` takes the ledger's answers as data; the fakes below are read // through the same prefetch the router uses, so the tests exercise the real // path rather than a second one. const reads = await prefetchTurnReads(opts.ledger ?? null, finalReq, PROFILE, cfg, SNAPSHOT, classification.task); return select({ req: finalReq, features, classification, profile: PROFILE, state: opts.st ?? state(), snapshot: SNAPSHOT, reads, cfg, nowMs: Date.now(), }); } describe("hard exclusions", () => { test("never selects a meta-router, floating alias, batch endpoint, or cloaked model", async () => { for (const tier of ["trivial", "simple", "moderate", "hard"] as Tier[]) { const d = await run({ tier }); expect(d.slug.startsWith("openrouter/")).toBe(false); expect(d.slug.startsWith("~")).toBe(false); expect(d.slug.endsWith(":batch")).toBe(false); expect(d.slug.startsWith("stealth/")).toBe(false); for (const f of d.fallbacks) { expect(f.startsWith("openrouter/")).toBe(false); expect(f.endsWith(":batch")).toBe(false); expect(f.startsWith("~")).toBe(false); } } }); test("only offers tool-capable models when the request offers tools", async () => { for (const tier of ["trivial", "simple", "moderate", "hard"] as Tier[]) { const d = await run({ tier }); for (const c of d.considered) expect(c.model.supportsTools).toBe(true); } }); test("excludes free models by default", async () => { const d = await run({ tier: "trivial" }); for (const c of d.considered) expect(c.model.isFree).toBe(false); }); }); describe("quality floor", () => { test("an unscored model never satisfies a tier with a floor above zero", async () => { for (const tier of ["simple", "moderate", "hard"] as Tier[]) { const d = await run({ tier }); for (const c of d.considered) { const q = c.model.quality; const unscored = q.coding === undefined && q.agentic === undefined && q.intelligence === undefined; expect(unscored).toBe(false); } } }); test("unscored models are eligible in the trivial tier, whose floor is zero", async () => { const d = await run({ tier: "trivial" }); expect(BASE.tiers.trivial.minQuality).toBe(0); expect(d.considered.length).toBeGreaterThan(0); }); test("a higher tier selects a higher-quality model than a lower tier", async () => { const cheap = await run({ tier: "trivial" }); const dear = await run({ tier: "hard" }); const cheapModel = MODELS.find((m) => m.slug === cheap.slug); const dearModel = MODELS.find((m) => m.slug === dear.slug); expect(cheapModel).toBeDefined(); expect(dearModel).toBeDefined(); expect(dear.forecast.expectedUsd).toBeGreaterThan(cheap.forecast.expectedUsd); }); }); describe("context window", () => { test("rejects models whose context cannot hold the prompt", async () => { // Far larger than the small-context models in the catalog can take. const d = await run({ tier: "trivial", promptTokens: 300_000 }); expect(d.rejected.some((r) => r.reason === "context_too_small")).toBe(true); const chosen = MODELS.find((m) => m.slug === d.slug); expect(chosen).toBeDefined(); expect(chosen?.contextLength ?? 0).toBeGreaterThan(300_000); }); test("applies headroom so a token-estimate error cannot overflow the window", async () => { const d = await run({ tier: "trivial", promptTokens: 100_000 }); const chosen = MODELS.find((m) => m.slug === d.slug); expect(chosen?.contextLength ?? 0).toBeGreaterThanOrEqual(100_000 * BASE.filters.contextHeadroom); }); }); describe("cache-aware switching", () => { // Deliberately NOT the top-ranked hard candidate: staying must be a real // choice against a better option, or the switch logic is never exercised. const warmSlug = "x-ai/grok-4.6"; test("keeps the warm model when switching does not clear the margin", async () => { const cfg: RouterConfig = { ...BASE, hysteresis: { ...BASE.hysteresis, switchMargin: 1e6 } }; const d = await run({ tier: "hard", promptTokens: 80_000, cfg, st: state({ currentSlug: warmSlug, currentTier: "hard", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 80_000, }), }); expect(d.slug).toBe(warmSlug); expect(d.sticky).toBe(true); }); test("abandons a warm cache whose TTL has expired", async () => { const d = await run({ tier: "hard", promptTokens: 80_000, st: state({ currentSlug: warmSlug, currentTier: "hard", cacheWarmSlug: warmSlug, // Long past the sticky-session window, so there is no cache left to keep. cacheWarmAtMs: Date.now() - BASE.hysteresis.cacheWarmTtlMs * 10, lastPromptTokens: 80_000, }), }); expect(d.sticky).toBe(false); }); }); describe("budget guard", () => { test("downgrades when the cold forecast breaches the per-turn cap", async () => { // A hard-tier turn at this size forecasts ~$0.02 cold, while cheaper // tiers land well under a cent, so a $0.005 cap is breachable AND // satisfiable further down. const cfg: RouterConfig = { ...BASE, budget: { ...BASE.budget, perTurnUsd: 0.005, onExceeded: "downgrade" }, }; const d = await run({ tier: "hard", promptTokens: 50_000, cfg }); expect(d.budgetDowngraded).toBe(true); expect(d.forecast.coldUsd).toBeLessThanOrEqual(0.005); }); test("throws in downgrade mode when no candidate at any tier fits", async () => { // Failing loudly beats silently spending past an impossible cap. const cfg: RouterConfig = { ...BASE, budget: { ...BASE.budget, perTurnUsd: 1e-9, onExceeded: "downgrade" }, }; await expect(run({ tier: "hard", promptTokens: 50_000, cfg })).rejects.toThrow(BudgetExceededError); }); test("rejects outright when configured to", async () => { const cfg: RouterConfig = { ...BASE, budget: { ...BASE.budget, perTurnUsd: 1e-9, onExceeded: "reject" }, }; await expect(run({ tier: "hard", promptTokens: 50_000, cfg })).rejects.toThrow(BudgetExceededError); }); test("a satisfiable budget does not downgrade", async () => { const cfg: RouterConfig = { ...BASE, budget: { ...BASE.budget, perTurnUsd: 100, onExceeded: "reject" } }; const d = await run({ tier: "moderate", promptTokens: 5000, cfg }); expect(d.budgetDowngraded).toBe(false); }); test("scopes the daily budget to the requesting harness", async () => { // Harness A has already spent the whole daily cap; harness B has spent // nothing. A request from B must NOT be budget-blocked by A's spend. const spendByHarness: Record = { "harness-a": 1.0 }; const ledger: AsyncLedger = fakeLedger({ record: async () => {}, conversationSpend: async () => 0, spendSince: async (_sinceMs, harnessId) => (harnessId === undefined ? 1.0 : spendByHarness[harnessId] ?? 0), blendedRate: async () => null, latency: async () => null, trust: async () => null, allTrust: async () => [], tokenRatio: async () => null, recentEntries: async () => [], }); const cfg: RouterConfig = { ...BASE, budget: { ...BASE.budget, perDayUsd: 0.5, onExceeded: "reject" } }; // Harness A is over its daily cap → rejected. await expect(run({ tier: "hard", promptTokens: 50_000, cfg, ledger, harnessId: "harness-a" })).rejects.toThrow( BudgetExceededError, ); // Harness B has spent nothing → not blocked by A's spend. const d = await run({ tier: "hard", promptTokens: 50_000, cfg, ledger, harnessId: "harness-b" }); expect(d.budgetDowngraded).toBe(false); }); }); describe("per-harness trust scoping", () => { // When filters.trustScopedByHarness is on, trust is read from the requesting // harness's own ledger rows, so one harness's flaky-model demotion does not // leak into another's routing. Off (default), trust is shared. test("scoped trust passes the harness id into the ledger trust query", async () => { // The feature's contract is that the router's trust lookup is scoped to // the requesting harness when enabled. The lookup is the prefetch, so // assert the harness id arrives there. let queriedWith: string | undefined; const ledger: AsyncLedger = fakeLedger({ signals: async (slugs, harnessId) => { queriedWith = harnessId; return new Map(slugs.map((slug) => [slug, { trust: null, latency: null }])); }, }); const cfg: RouterConfig = { ...BASE, filters: { ...BASE.filters, trustScopedByHarness: true }, }; await run({ tier: "simple", cfg, ledger, harnessId: "harness-a" }); expect(queriedWith).toBe("harness-a"); }); test("shared trust (default) reads the whole ledger, not per-harness", async () => { // With scoping off, the trust lookup must NOT carry the harness id, so // harness A's flaky history is visible globally (shared reliability). let queriedWith: string | undefined; const ledger: AsyncLedger = fakeLedger({ signals: async (slugs, harnessId) => { queriedWith = harnessId; return new Map(slugs.map((slug) => [slug, { trust: null, latency: null }])); }, }); const cfg: RouterConfig = { ...BASE, filters: { ...BASE.filters, trustScopedByHarness: false }, }; await run({ tier: "simple", cfg, ledger, harnessId: "harness-a" }); // The trust lookup must NOT carry the harness id when scoping is off. expect(queriedWith).toBeUndefined(); }); }); describe("decision shape", () => { test("clamps max tokens to the chosen model's published ceiling", async () => { const d = await run({ tier: "moderate" }); const chosen = MODELS.find((m) => m.slug === d.slug); const ceiling = chosen?.maxCompletionTokens; if (ceiling !== undefined && d.maxTokens !== undefined) { expect(d.maxTokens).toBeLessThanOrEqual(ceiling); } }); test("a reasoning model gets the completion floor; a direct one keeps the caller's cap", async () => { const usable = (m: (typeof MODELS)[number]): boolean => m.supportsTools && m.contextLength >= 32_000 && (m.maxCompletionTokens ?? 100_000) >= 4096; const thinker = MODELS.find((m) => usable(m) && m.supportsReasoning); const direct = MODELS.find((m) => usable(m) && !m.supportsReasoning && !m.reasoningMandatory); expect(thinker).toBeDefined(); expect(direct).toBeDefined(); const withFloor = (slug: string, floor: number): RouterConfig => ({ ...BASE, filters: { ...BASE.filters, allow: [slug], reasoningCompletionFloor: floor } }); // omp asks for a dozen tokens for a title; a reasoning model would spend them thinking // and return nothing, so the dispatch is raised. const raised = await run({ tier: "trivial", cfg: withFloor(thinker!.slug, 512), maxTokens: 12 }); expect(raised.slug).toBe(thinker!.slug); expect(raised.maxTokens).toBe(512); expect(raised.reasons.some((r) => r.includes("reasons before it answers"))).toBe(true); // A model that answers directly is untouched: its cap is the caller's. const kept = await run({ tier: "trivial", cfg: withFloor(direct!.slug, 512), maxTokens: 12 }); expect(kept.slug).toBe(direct!.slug); expect(kept.maxTokens).toBe(12); // The floor never raises past what the caller already asked for, and 0 disables it. expect((await run({ tier: "trivial", cfg: withFloor(thinker!.slug, 512), maxTokens: 4000 })).maxTokens).toBe(4000); expect((await run({ tier: "trivial", cfg: withFloor(thinker!.slug, 0), maxTokens: 12 })).maxTokens).toBe(12); }); test("plans a probe for cheap tiers and leaves the top tier unprobed", async () => { expect((await run({ tier: "trivial" })).probe.enabled).toBe(true); // Nothing above `hard` to escalate into, so probing it would only add latency. expect((await run({ tier: "hard" })).probe.enabled).toBe(false); }); test("carries the session id, features, and a reasoning trail", async () => { const d = await run({ tier: "simple" }); expect(d.sessionId.startsWith("omp-")).toBe(true); // `d.reasons` holds decision-level notes — a widening, a hysteresis hold — and is // legitimately empty when a tier serves the turn without incident. The trail that is // always present is the per-candidate one, so that is what a caller can rely on. expect(d.considered[0]!.reasons.length).toBeGreaterThan(0); expect(d.features.toolCount).toBe(1); expect(d.considered.length).toBeGreaterThan(0); }); test("respects a profile that caps the tier", async () => { const req = request("redesign the whole architecture and explain the race condition root cause"); const features = extractFeatures(req, 4000); const d = select({ req, features, classification: scoreHeuristic(features, BASE), profile: { ...PROFILE, id: "auto-cheap", maxTier: "simple" }, state: state(), snapshot: SNAPSHOT, cfg: BASE, nowMs: Date.now(), }); expect(["trivial", "simple"]).toContain(d.tier); }); }); describe("tier rescue under a guardrail-constrained catalog", () => { // A tiny catalog containing only models that all fail the strict `trivial` // tier config: they exceed its price ceiling or fail its trust/quality bar. // Under the full catalog the cheap alternatives masked this; a guardrail // can remove them entirely. const pick = (slug: string): CatalogModel => { const m = MODELS.find((x) => x.slug === slug); if (m === undefined) throw new Error(`fixture missing ${slug}`); return m; }; const constrained: CatalogSnapshot = { models: [pick("z-ai/glm-5.3"), pick("qwen/qwen3.8-2.4t-a95b"), pick("x-ai/grok-4.6")], fetchedAtMs: Date.now(), keyScoped: true, }; async function runConstrained(ledger: AsyncLedger | null = null) { const req = request("refactor the service layer and explain the cache coherence contract"); const features = extractFeatures(req, 4000); const heuristic = scoreHeuristic(features, BASE); const reads = await prefetchTurnReads(ledger, req, PROFILE, BASE, constrained, heuristic.task); return select({ req, features, classification: { ...heuristic, tier: "trivial" as Tier }, profile: PROFILE, state: state(), snapshot: constrained, reads, cfg: BASE, nowMs: Date.now(), }); } /** Every model is probed-and-failed: below the trust floor at every tier. */ function untrustedLedger(): AsyncLedger { const burned = (slug: string) => ({ slug, attempts: 40, escalations: 30, errors: 30, successRate: 0.1, meanCostError: 0.2 }); return fakeLedger({ trust: async (slug) => burned(slug), // Candidate scoring reads `signals`, which is what the prefetch fills. signals: async (slugs) => new Map(slugs.map((slug) => [slug, { trust: burned(slug), latency: null }])), }); } test("rescues a model instead of throwing when no strict tier admits the catalog", async () => { const d = await runConstrained(untrustedLedger()); // It must pick one of the available models, not throw `catalog exhausted`. expect(constrained.models.some((m) => m.slug === d.slug)).toBe(true); }); test("records the rescue in the reasoning trail", async () => { const d = await runConstrained(untrustedLedger()); expect(d.reasons.some((r) => r.startsWith("tier rescue:"))).toBe(true); }); test("the rescue chooses the cheapest available model when quality is secondary", async () => { const d = await runConstrained(untrustedLedger()); const chosen = MODELS.find((m) => m.slug === d.slug); expect(chosen).toBeDefined(); // Price ceilings are relaxed first; the cheapest surviving model wins. const cheapest = constrained.models.reduce((a, b) => (a.price.prompt <= b.price.prompt ? a : b)); expect(d.slug).toBe(cheapest.slug); }); test("a guardrail that leaves every model below the trust bar is rescued by relaxing it", async () => { // Reproduces the real failure: a tiny guardrail catalog whose models are // all marked untrusted (probed and failed). The trust floor (minTrust 0.7 // over minTrustSamples 12) excludes them at EVERY tier, so strict widening // finds nothing; the rescue relaxes trust and picks a model. const d = await runConstrained(untrustedLedger()); expect(constrained.models.some((m) => m.slug === d.slug)).toBe(true); expect(d.reasons.some((r) => r.startsWith("tier rescue:"))).toBe(true); }); test("still throws when the catalog is empty after relaxing all economic constraints", async () => { const empty: CatalogSnapshot = { models: [], fetchedAtMs: Date.now(), keyScoped: true }; const req = request("anything"); const features = extractFeatures(req, 4000); const heuristic = scoreHeuristic(features, BASE); expect(() => select({ req, features, classification: { ...heuristic, tier: "trivial" as Tier }, profile: PROFILE, state: state(), snapshot: empty, cfg: BASE, nowMs: Date.now(), }), ).toThrow(/catalog exhausted/); }); }); describe("task-type routing", () => { test("a vision task only considers image-capable models", async () => { // Force the vision task and a tier; every considered candidate must // support image input. const req = request("describe this image"); const features = extractFeatures(req, 4000); const heuristic = scoreHeuristic(features, BASE); const d = select({ req: { ...req, hasImages: true }, features: { ...features, hasImages: true }, classification: { ...heuristic, task: "vision" }, profile: PROFILE, state: state(), snapshot: SNAPSHOT, cfg: BASE, nowMs: Date.now(), }); expect(d.considered.length).toBeGreaterThan(0); for (const c of d.considered) expect(c.model.inputModalities.includes("image")).toBe(true); }); test("the task config's quality floor overrides the tier floor when higher", async () => { // A coding task with a high minQuality must not admit models below it, // even in a tier whose own floor is lower. const cfg: RouterConfig = { ...BASE, tasks: { ...BASE.tasks, coding: { axis: "coding", minQuality: 60 } }, }; const req = request("refactor the service layer"); const features = extractFeatures(req, 4000); const heuristic = scoreHeuristic(features, cfg); const d = select({ req, features, classification: { ...heuristic, task: "coding" }, profile: PROFILE, state: state(), snapshot: SNAPSHOT, cfg, nowMs: Date.now(), }); // The task floor (60) is higher than the trivial tier floor (0); every // considered candidate must clear it. (A floor so high nothing qualifies // would trip tier rescue, so 60 is the meaningful override test.) expect(d.considered.length).toBeGreaterThan(0); for (const c of d.considered) { const q = c.model.quality.coding ?? c.model.quality.intelligence ?? 0; expect(q).toBeGreaterThanOrEqual(60); } }); }); describe("latency scoring", () => { function ledgerWithLatency(bySlug: Record): AsyncLedger { const latencyOf = (slug: string): ModelLatency | null => { const v = bySlug[slug]; // Default throughput is fast, so these cases isolate the TTFT axis // unless a test sets tokensPerSec explicitly. return v === undefined ? null : { slug, samples: v.samples, ttftMs: v.ttftMs, tokensPerSec: v.tokensPerSec ?? 1000 }; }; return fakeLedger({ latency: async (slug) => latencyOf(slug), // Scoring reads latency out of the prefetched signals. signals: async (slugs) => new Map(slugs.map((slug) => [slug, { trust: null, latency: latencyOf(slug) }])), }); } const withWeight = (latencyWeight: number): RouterConfig => ({ ...BASE, filters: { ...BASE.filters, latencyWeight, latencyReferenceMs: 5000, latencyMinSamples: 20 }, }); test("penalises a chronically slow model out of the top slot", async () => { const slow = (await run({ tier: "simple" })).slug; const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } }); const d = await run({ tier: "simple", cfg: withWeight(2), ledger }); expect(d.slug).not.toBe(slow); }); test("latencyWeight 0 disables the penalty", async () => { const slow = (await run({ tier: "simple" })).slug; const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } }); expect((await run({ tier: "simple", cfg: withWeight(0), ledger })).slug).toBe(slow); }); test("a model with too few samples is not penalised", async () => { const slow = (await run({ tier: "simple" })).slug; const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 5 } }); expect((await run({ tier: "simple", cfg: withWeight(2), ledger })).slug).toBe(slow); }); test("penalises a model that starts fast but streams slowly", async () => { // The case TTFT-only scoring misses: quick first token, slow body. const slow = (await run({ tier: "simple" })).slug; const ledger = ledgerWithLatency({ [slow]: { ttftMs: 1500, samples: 50, tokensPerSec: 12 } }); const d = await run({ tier: "simple", cfg: withWeight(2), ledger }); expect(d.slug).not.toBe(slow); }); const withCeiling = (maxExpectedWaitMs: number, latencyWeight = 0): RouterConfig => ({ ...BASE, filters: { ...BASE.filters, latencyWeight, latencyReferenceMs: 5000, latencyMinSamples: 20, maxExpectedWaitMs }, }); test("ceiling hard-drops a proven-slow model the penalty cannot, even at weight 0", async () => { const slow = (await run({ tier: "simple" })).slug; const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } }); // latencyWeight 0 → the multiplier is inert; only the hard ceiling can act. const d = await run({ tier: "simple", cfg: withCeiling(20_000), ledger }); expect(d.slug).not.toBe(slow); }); test("ceiling spares an under-sampled slow model (cold-start grace)", async () => { const slow = (await run({ tier: "simple" })).slug; const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 5 } }); expect((await run({ tier: "simple", cfg: withCeiling(20_000), ledger })).slug).toBe(slow); }); test("ceiling unset ⇒ no latency gate (proven-slow model still wins on price)", async () => { const slow = (await run({ tier: "simple" })).slug; const ledger = ledgerWithLatency({ [slow]: { ttftMs: 60_000, samples: 50 } }); expect((await run({ tier: "simple", cfg: withWeight(0), ledger })).slug).toBe(slow); }); }); describe("context compaction", () => { const COMPACT_CFG: RouterConfig = { ...BASE, compaction: { enabled: true, budgetTokens: 1_000, floorRatio: 1, replanGrowthRatio: 1, fitToWindow: false, protectRecentTurns: 1, maxToolResultBytes: 100, keepHeadBytes: 20, keepTailBytes: 20, elideSupersededReads: true, collapseDuplicateResults: true, digestToolResults: false, digestMaxPerTurn: 2, }, }; function loopReq(): NormRequest { return parseChatRequest( { model: "auto", tools: TOOLS, messages: [ { role: "system", content: "You are a coding agent." }, { role: "user", content: "read the file" }, { role: "assistant", content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "read", arguments: '{"path":"big.ts"}' } }] }, { role: "tool", tool_call_id: "c1", content: "x".repeat(4000) }, { role: "user", content: "continue" }, ], }, new Headers(), ); } test("an over-budget turn produces a compaction plan and records savings", async () => { const req = loopReq(); const features = extractFeatures(req, 5_000); // over budgetTokens=1000 const d = select({ req, features, classification: scoreHeuristic(features, COMPACT_CFG), profile: PROFILE, state: state(), snapshot: SNAPSHOT, cfg: COMPACT_CFG, nowMs: Date.now(), }); expect(d.compactionPlan.length).toBeGreaterThan(0); expect(d.promptTokensSaved).toBeGreaterThan(0); expect(d.reasons.some((r) => r.startsWith("compaction:"))).toBe(true); }); test("a small turn is left untouched", async () => { const req = loopReq(); const features = extractFeatures(req, 500); // under budgetTokens=1000 const d = select({ req, features, classification: scoreHeuristic(features, COMPACT_CFG), profile: PROFILE, state: state(), snapshot: SNAPSHOT, cfg: COMPACT_CFG, nowMs: Date.now(), }); expect(d.compactionPlan).toEqual([]); expect(d.promptTokensSaved).toBe(0); }); test("a carried plan is re-applied even when the turn is now under budget", async () => { // The prompt cache is a byte-prefix cache: dropping an edit that was // already dispatched rewrites history the upstream had cached, and // re-sends the tokens the edit saved. So a carried plan survives a turn // that would not have triggered compaction on its own. const req = loopReq(); const over = extractFeatures(req, 5_000); const first = select({ req, features: over, classification: scoreHeuristic(over, COMPACT_CFG), profile: PROFILE, state: state(), snapshot: SNAPSHOT, cfg: COMPACT_CFG, nowMs: Date.now(), }); expect(first.compactionPlan.length).toBeGreaterThan(0); const under = extractFeatures(req, 500); // under budgetTokens=1000 const second = select({ req, features: under, classification: scoreHeuristic(under, COMPACT_CFG), profile: PROFILE, state: state({ compactionPlan: first.compactionPlan }), snapshot: SNAPSHOT, cfg: COMPACT_CFG, nowMs: Date.now(), }); expect(second.compactionPlan).toEqual(first.compactionPlan); expect(second.promptTokensSaved).toBeGreaterThan(0); }); test("floorRatio below 1 compacts strictly past the budget so the plan holds longer", async () => { // Each plan change rewrites cached prompt bytes, so compaction overshoots // deliberately: eliding more now buys byte-stable turns later. const req = parseChatRequest( { model: "auto", tools: TOOLS, messages: [ { role: "system", content: "You are a coding agent." }, { role: "user", content: "read the files" }, ...[1, 2, 3, 4, 5, 6].flatMap((n) => [ { role: "assistant", content: null, tool_calls: [{ id: `c${n}`, type: "function", function: { name: "read", arguments: `{"path":"f${n}.ts"}` } }] }, { role: "tool", tool_call_id: `c${n}`, content: `F${n}${"x".repeat(2000)}` }, ]), { role: "user", content: "continue" }, ], }, new Headers(), ); // Prompt is ~12k bytes; claim 4000 tokens against a 1000-token budget, so // floorRatio 1 targets 1000 and floorRatio 0.5 targets 500. const features = extractFeatures(req, 4_000); const run = (floorRatio: number) => select({ req, features, classification: scoreHeuristic(features, COMPACT_CFG), profile: PROFILE, state: state(), snapshot: SNAPSHOT, cfg: { ...COMPACT_CFG, compaction: { ...COMPACT_CFG.compaction, floorRatio } }, nowMs: Date.now(), }); const tight = run(0.5); const loose = run(1); expect(tight.compactionPlan.length).toBeGreaterThan(loose.compactionPlan.length); expect(tight.promptTokensSaved).toBeGreaterThan(loose.promptTokensSaved); }); }); describe("hysteresis.breakHoldOnMechanical", () => { // A hold bets the next turn resembles the one that armed it. A tool-result // continuation the classifier has already docked, scoring below the held // tier, is evidence against that bet. Measured on 24h of live traffic: 37 of // 44 sticky `hard` dispatches were exactly that shape — one scoring 0.154 // (trivial) yet served by claude-opus-5 — $2.66 billed against $0.05 for the // same tokens on the moderate pick. function continuation(): NormRequest { return parseChatRequest( { model: "auto", tools: TOOLS, messages: [ { role: "system", content: "You are a coding agent." }, { role: "user", content: "read the file" }, { role: "assistant", content: null, tool_calls: [{ id: "c1", type: "function", function: { name: "read", arguments: '{"path":"a.ts"}' } }] }, { role: "tool", tool_call_id: "c1", content: "export const x = 1;" }, ], }, new Headers(), ); } const held = state({ currentTier: "hard", currentSlug: "x-ai/grok-4.6", stickyUntilTurn: 9, turn: 1 }); function decide(req: NormRequest, breakHold: boolean) { const cfg: RouterConfig = { ...BASE, hysteresis: { ...BASE.hysteresis, breakHoldOnMechanical: breakHold } }; const features = extractFeatures(req, 4_000); return { d: select({ req, features, classification: scoreHeuristic(features, cfg), profile: PROFILE, state: held, snapshot: SNAPSHOT, cfg, nowMs: Date.now() }), features }; } test("off by default, so a hold still pins the tier", async () => { expect(DEFAULT_CONFIG.hysteresis.breakHoldOnMechanical).toBe(false); // SHIPPED default, not the live config.yml (machine-dependent) const { d, features } = decide(continuation(), false); expect(features.isToolResultContinuation).toBe(true); expect(d.tier).toBe("hard"); expect(d.classification.source).toBe("sticky"); }); test("on, a mechanical continuation escapes the hold", async () => { const { d } = decide(continuation(), true); expect(d.tier).not.toBe("hard"); expect(d.classification.source).not.toBe("sticky"); expect(d.reasons.some((r) => /hold hard broken/.test(r))).toBe(true); }); test("a NON-mechanical turn still gets the hold, so flap protection survives", async () => { // This is the case hysteresis exists for: fresh user work mid-conversation // must not bounce the model and cold-start its cache. const { d, features } = decide(request("now refactor the retry helper"), true); expect(features.isToolResultContinuation).toBe(false); expect(d.tier).toBe("hard"); expect(d.classification.source).toBe("sticky"); }); test("the downgrade clamp still applies, so quality steps rather than falls", async () => { const cfg: RouterConfig = { ...BASE, hysteresis: { ...BASE.hysteresis, breakHoldOnMechanical: true, maxDowngradePerTurn: 1 }, }; const req = continuation(); const features = extractFeatures(req, 4_000); // Force the fresh classification far below the hold to exercise the clamp. const d = select({ req, features, classification: { ...scoreHeuristic(features, cfg), tier: "trivial" }, profile: PROFILE, state: held, snapshot: SNAPSHOT, cfg, nowMs: Date.now(), }); expect(d.tier).toBe("moderate"); }); }); describe("hysteresis.switchHorizonTurns (review 2026-09-05 §4)", () => { // Two moderate-eligible models built from raw catalog records: a kimi-shaped // warm model (cheap to READ from cache, dear cold) and a gemini-shaped ranked // winner (dear cold, cheap warm). With a one-turn horizon the cold write on // the winner never pays for itself, so the dear model stays warm forever; // over a run of turns the switch is obviously right. function rawModel(id: string, coding: number, prompt: number, cacheRead: number, cacheWrite: number | null): Record { const pricing: Record = { prompt: String(prompt / 1e6), completion: String((prompt * 4) / 1e6), input_cache_read: String(cacheRead / 1e6), }; if (cacheWrite !== null) pricing.input_cache_write = String(cacheWrite / 1e6); return { id, canonical_slug: id, name: id, context_length: 1_000_000, pricing, supported_parameters: ["tools"], architecture: { input_modalities: ["text"], tokenizer: "GPT" }, benchmarks: { artificial_analysis: { coding_index: coding, intelligence_index: coding, agentic_index: coding } }, created: 1_700_000_000, }; } const warmDear = normalizeCatalogModel(rawModel("test/warm-dear", 76.2, 2.55, 0.256, null))!; const winner = normalizeCatalogModel(rawModel("test/winner", 76.0, 0.75, 0.075, 0.04))!; const snap: CatalogSnapshot = { models: [warmDear, winner], fetchedAtMs: Date.now() }; const promptTokens = 100_000; function decide(horizon: number) { // BASE is the live config.yml, which may have exploration on; a // deterministic exploration draw would route this turn down to `simple`, // where the dear model is over the price ceiling and never compared. const cfg: RouterConfig = { ...BASE, exploration: { ...BASE.exploration, enabled: false }, hysteresis: { ...BASE.hysteresis, switchMargin: 1.3, switchHorizonTurns: horizon }, }; const req = request("keep going"); const features = extractFeatures(req, promptTokens); return select({ req, features, classification: { ...scoreHeuristic(features, cfg), tier: "moderate" }, profile: PROFILE, state: state({ currentSlug: "test/warm-dear", currentTier: "moderate", cacheWarmSlug: "test/warm-dear", cacheWarmAtMs: Date.now(), lastPromptTokens: promptTokens }), snapshot: snap, cfg, nowMs: Date.now(), }); } test("the ranked winner is the cheaper cold model", async () => { const d = decide(1); expect(d.considered[0]!.model.slug).toBe("test/winner"); }); test("a one-turn horizon keeps the dear model warm (the shipped behaviour)", async () => { const d = decide(1); expect(d.slug).toBe("test/warm-dear"); expect(d.sticky).toBe(true); expect(d.reasons.some((r) => r.startsWith("cache: keeping warm test/warm-dear"))).toBe(true); }); test("amortised over a run of turns, the switch is taken", async () => { const d = decide(8); expect(d.slug).toBe("test/winner"); expect(d.sticky).toBe(false); expect(d.reasons.some((r) => r.startsWith("cache: switch test/warm-dear → test/winner") && r.includes("over 8 turns"))).toBe(true); }); }); describe("compaction.replanGrowthRatio (review 2026-09-05 §7)", () => { const TOOL_RESULT = "y".repeat(3000); function turn(n: number): NormRequest { // n completed read/result/"continue" rounds; every result but the newest // sits outside the protected window and is eligible for truncation. const messages: unknown[] = [ { role: "system", content: "You are a coding agent." }, { role: "user", content: "read the files" }, ]; for (let i = 0; i < n; i++) { messages.push({ role: "assistant", content: null, tool_calls: [{ id: `c${i}`, type: "function", function: { name: "read", arguments: `{"path":"f${i}.ts"}` } }] }); messages.push({ role: "tool", tool_call_id: `c${i}`, content: TOOL_RESULT }); messages.push({ role: "user", content: "continue" }); } return parseChatRequest({ model: "auto", tools: TOOLS, messages }, new Headers()); } function cfgWith(ratio: number): RouterConfig { return { ...BASE, compaction: { enabled: true, budgetTokens: 100, // unreachable: every turn is over budget, as observed live floorRatio: 1, replanGrowthRatio: ratio, fitToWindow: false, protectRecentTurns: 2, // the newest result and its "continue" stay protected; older rounds are eligible maxToolResultBytes: 100, keepHeadBytes: 20, keepTailBytes: 20, elideSupersededReads: false, collapseDuplicateResults: false, digestToolResults: false, digestMaxPerTurn: 2, }, }; } function decide(req: NormRequest, cfg: RouterConfig, st: ConversationState, promptTokens: number) { const features = extractFeatures(req, promptTokens); return select({ req, features, classification: scoreHeuristic(features, cfg), profile: PROFILE, state: st, snapshot: SNAPSHOT, cfg, nowMs: Date.now() }); } test("the plan records the compacted size it was made at", async () => { const first = decide(turn(2), cfgWith(1), state(), 4_000); expect(first.compactionPlan.length).toBe(1); expect(first.compactionPlanTokens).toBe(4_000 - first.promptTokensSaved); expect(first.compactionSavedBytes).toBeGreaterThan(0); }); test("at 1 (shipped) a newly eligible result is compacted on the very next turn", async () => { const first = decide(turn(2), cfgWith(1), state(), 4_000); const carried = state({ compactionPlan: first.compactionPlan, compactionPlanTokens: first.compactionPlanTokens }); // One more round: the prompt grew ~25%, one more result aged out. const second = decide(turn(3), cfgWith(1), carried, 5_000); expect(second.compactionPlan.length).toBe(2); expect(second.reasons.some((r) => r.includes("(1 carried, 1 new)"))).toBe(true); }); test("above 1, an existing plan holds until the compacted prompt has grown by the ratio", async () => { const first = decide(turn(2), cfgWith(2), state(), 4_000); const carried = state({ compactionPlan: first.compactionPlan, compactionPlanTokens: first.compactionPlanTokens }); // The raw prompt grew 25% and the COMPACTED prompt ~60% (the carried // edit saves a smaller share of a bigger prompt) — still under 2x, so // the carried plan is re-applied verbatim and nothing new is added. const held = decide(turn(3), cfgWith(2), carried, 5_000); expect(held.compactionPlan).toEqual(first.compactionPlan); expect(held.compactionPlanTokens).toBe(first.compactionPlanTokens); // growth still accrues against the original size expect(held.reasons.some((r) => r.includes("(1 carried, 0 new)") && r.includes("[re-plan rationed]"))).toBe(true); // Past 2x the plan is extended and the new size is recorded. const grown = decide(turn(3), cfgWith(2), carried, 8_000); expect(grown.compactionPlan.length).toBe(2); expect(grown.compactionPlanTokens).toBe(8_000 - grown.promptTokensSaved); expect(grown.reasons.some((r) => r.includes("[re-plan rationed]"))).toBe(false); }); }); describe("hysteresis.confirmUpgradesBelowConfidence", () => { // A low-confidence heuristic upgrade from a warm model waits one turn. // Measured: 65 of 67 moderate→hard upgrades in a week bounced back within // 3 turns, each paying a cold hard-tier read of a ~120k prompt. // Resolved per test: a describe body cannot await. const warmSlugOf = async (): Promise => (await run({ tier: "moderate" })).slug; async function upgrade(opts: { confidence?: number; source?: "heuristic" | "escalation"; st?: Partial; cfg?: RouterConfig; lastToolFailed?: boolean }) { const cfg = opts.cfg ?? BASE; const warmSlug = await warmSlugOf(); const req = request("now rework the whole scheduler"); const base = extractFeatures(req, 120_000); const features = opts.lastToolFailed === true ? { ...base, lastToolFailed: true } : base; const heuristic = scoreHeuristic(features, cfg); return select({ req, features, classification: { ...heuristic, tier: "hard", confidence: opts.confidence ?? 0.45, source: opts.source ?? "heuristic" }, profile: PROFILE, state: state({ turn: 4, currentTier: "moderate", currentSlug: warmSlug, cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 110_000, ...opts.st }), snapshot: SNAPSHOT, cfg, nowMs: Date.now(), }); } test("a low-confidence upgrade from a warm model is deferred to the held tier", async () => { const d = await upgrade({}); expect(d.tier).toBe("moderate"); expect(d.upgradeDeferred).toBe("hard"); expect(d.reasons.some((r) => r.includes("upgrade moderate → hard deferred one turn"))).toBe(true); }); test("a second consecutive upgrade classification confirms it", async () => { const d = await upgrade({ st: { upgradeDeferredTier: "hard" } }); expect(d.tier).toBe("hard"); expect(d.upgradeDeferred).toBeNull(); expect(d.reasons.some((r) => r.includes("upgrade moderate → hard confirmed"))).toBe(true); }); test("confident classifications, cold caches, escalations, failing tools and the off switch all upgrade at once", async () => { expect((await upgrade({ confidence: 0.9 })).tier).toBe("hard"); expect((await upgrade({ st: { cacheWarmAtMs: Date.now() - 3_600_000 } })).tier).toBe("hard"); expect((await upgrade({ source: "escalation" })).tier).toBe("hard"); expect((await upgrade({ lastToolFailed: true })).tier).toBe("hard"); const off: RouterConfig = { ...BASE, hysteresis: { ...BASE.hysteresis, confirmUpgradesBelowConfidence: 0 } }; expect((await upgrade({ cfg: off })).tier).toBe("hard"); for (const d of [await upgrade({ confidence: 0.9 }), await upgrade({ source: "escalation" })]) expect(d.upgradeDeferred).toBeNull(); }); test("a downgrade or a same-tier turn is never deferred", async () => { const warmSlug = await warmSlugOf(); const d = await run({ tier: "simple", st: state({ turn: 4, currentTier: "moderate", currentSlug: warmSlug, cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now() }) }); expect(d.upgradeDeferred).toBeNull(); }); }); describe("recorded forecast is the expected price, not the cold worst case", () => { const warmSlug = "x-ai/grok-4.6"; test("a warm stay prices the previous prompt as cache reads; coldUsd keeps the cold figure", async () => { const cfg: RouterConfig = { ...BASE, hysteresis: { ...BASE.hysteresis, switchMargin: 1e6 } }; const d = await run({ tier: "hard", promptTokens: 80_000, cfg, st: state({ currentSlug: warmSlug, currentTier: "hard", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 60_000 }), }); expect(d.slug).toBe(warmSlug); // 60k of the 80k prompt is the cached prefix; no reliability sample ⇒ assumed reliable. expect(d.forecast.assumedCacheHitRate).toBeCloseTo(0.75, 6); expect(d.forecast.expectedUsd).toBeLessThan(d.forecast.coldUsd); expect(d.forecast.breakdown.cacheRead).toBeGreaterThan(0); }); test("a cold turn records the cold price", async () => { const d = await run({ tier: "hard", promptTokens: 80_000 }); expect(d.forecast.assumedCacheHitRate).toBe(0); expect(d.forecast.expectedUsd).toBeLessThanOrEqual(d.forecast.coldUsd); }); }); describe("cache reliability in the stay/switch comparison", () => { const warmSlug = "x-ai/grok-4.6"; function ledgerWithReliability(rate: number | null, samples = 50): AsyncLedger { return fakeLedger({ // Only the warm model has a measured rate; everything else is unmeasured, // which is what the stay/switch comparison treats as "assume reliable". cacheReliability: async (slugs) => rate === null ? new Map() : new Map(slugs.filter((s) => s === warmSlug).map((slug) => [slug, { slug, samples, hitRate: rate }])), }); } const stayCostOf = async (ledger: AsyncLedger, cfg: RouterConfig = BASE): Promise => { const d = await run({ tier: "hard", promptTokens: 80_000, cfg, ledger, st: state({ currentSlug: warmSlug, currentTier: "hard", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 80_000 }), }); const m = /cache: (?:keeping warm|switch) .*?stay \$([0-9.]+)/.exec(d.reasons.join("\n")); if (m === null) throw new Error(`no stay/switch reason in ${d.reasons.join(" | ")}`); return Number(m[1]); }; test("an unreliable cache prices staying at the fresh rate, a reliable one at the cached rate", async () => { const reliable = await stayCostOf(ledgerWithReliability(1)); const flaky = await stayCostOf(ledgerWithReliability(0)); const unknown = await stayCostOf(ledgerWithReliability(null)); expect(flaky).toBeGreaterThan(reliable); expect(unknown).toBeCloseTo(reliable, 6); }); test("too few samples, or the feature off, assume a reliable cache", async () => { const reliable = await stayCostOf(ledgerWithReliability(1)); expect(await stayCostOf(ledgerWithReliability(0, 3))).toBeCloseTo(reliable, 6); const off: RouterConfig = { ...BASE, filters: { ...BASE.filters, cacheReliabilityMinSamples: 0 } }; expect(await stayCostOf(ledgerWithReliability(0), off)).toBeCloseTo(reliable, 6); }); test("the reason names the measured hit rate", async () => { const d = await run({ tier: "hard", promptTokens: 80_000, ledger: ledgerWithReliability(0.5, 40), st: state({ currentSlug: warmSlug, currentTier: "hard", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 80_000 }), }); expect(d.reasons.some((r) => r.includes("warm hit 50% over 40"))).toBe(true); }); }); describe("session pin (forceSlug)", () => { test("a pinned catalog model wins over ranking and the warm model; an unknown pin is ignored with a reason", async () => { const warmSlug = (await run({ tier: "moderate" })).slug; const pinSlug = (await run({ tier: "hard" })).slug; // a real, differently-ranked model const req = request("tidy the retry helper"); const features = extractFeatures(req, 50_000); const base = { req, features, classification: scoreHeuristic(features, BASE), profile: PROFILE, state: state({ currentSlug: warmSlug, currentTier: "moderate", cacheWarmSlug: warmSlug, cacheWarmAtMs: Date.now(), lastPromptTokens: 50_000 }), snapshot: SNAPSHOT, cfg: { ...BASE, hysteresis: { ...BASE.hysteresis, switchMargin: 1e6 } }, nowMs: Date.now(), }; const pinned = select({ ...base, forceSlug: pinSlug }); expect(pinned.slug).toBe(pinSlug); expect(pinned.sticky).toBe(false); expect(pinned.reasons.some((r) => r.includes(`pinned to ${pinSlug} by session override`))).toBe(true); const unknown = select({ ...base, forceSlug: "nope/model" }); expect(unknown.slug).toBe(warmSlug); // the huge switch margin keeps the warm model expect(unknown.reasons.some((r) => r.includes("pin nope/model ignored"))).toBe(true); }); }); describe("budget.perMonthUsd pacing", () => { test("monthPace spreads what is left over the days left, today included", async () => { const sep7 = Date.UTC(2026, 8, 7, 12); expect(monthStartMs(sep7)).toBe(Date.UTC(2026, 8, 1)); const p = monthPace(sep7, 60, 30); expect(p.daysLeft).toBe(24); // 7th..30th expect(p.dailyCapUsd).toBeCloseTo(30 / 24, 6); expect(monthPace(sep7, 60, 70).dailyCapUsd).toBe(0); expect(monthPace(Date.UTC(2026, 8, 30, 12), 60, 0).daysLeft).toBe(1); }); test("a month running ahead of pace tightens the daily cap and says so", async () => { const ledger: AsyncLedger = fakeLedger({ record: async () => {}, conversationSpend: async () => 0, spendSince: async (sinceMs) => (sinceMs <= monthStartMs(Date.now()) + 1 ? 59.99 : 0), // month-to-date $59.99, last 24h $0 blendedRate: async () => null, trust: async () => null, allTrust: async () => [], latency: async () => null, tokenRatio: async () => null, recentEntries: async () => [], }); const cfg: RouterConfig = { ...BASE, budget: { ...BASE.budget, perMonthUsd: 60, onExceeded: "reject" } }; await expect(run({ tier: "hard", promptTokens: 50_000, cfg, ledger })).rejects.toThrow(/month pacing: \$59\.99 of \$60 spent/); // Under pace: the cap is generous and nothing breaches. const easy: AsyncLedger = { ...ledger, spendSince: async () => 1 }; expect((await run({ tier: "hard", promptTokens: 50_000, cfg, ledger: easy })).budgetDowngraded).toBe(false); }); }); describe("filters.latencyWeightContinuation", () => { test("applies only to tool-result continuations, and only when set", async () => { const f = { ...BASE.filters, latencyWeight: 0.75 }; expect(latencyWeightFor(f, false)).toBe(0.75); expect(latencyWeightFor(f, true)).toBe(0.75); const g = { ...f, latencyWeightContinuation: 0.1 }; expect(latencyWeightFor(g, false)).toBe(0.75); expect(latencyWeightFor(g, true)).toBe(0.1); }); });