import { analyzeTask, type ShapeInput } from "../controller/task-shape.js"; import { routeTask } from "../controller/routing.js"; import { scoutModelForLens } from "../models/model-policy.js"; import { estimateCredits } from "../models/price-catalog.js"; import { scoutShards } from "../workflows/scout.js"; import { swarmShards } from "../workflows/swarm.js"; import type { PriceCatalog, UltraConfig } from "../types.js"; export interface RoutingScenario { id: string; input: ShapeInput; } export interface RoutingObservation { id: string; topology: string; model: string; fanout: number; scoutModels: string[]; estimatedCredits: number; shapeSources: string[]; } const SCOUT_INPUT_TOKENS = 8_000; export function observeRouting(scenarios: readonly RoutingScenario[], config: UltraConfig, catalog: PriceCatalog): RoutingObservation[] { return scenarios.map((scenario) => { const shape = analyzeTask(scenario.input); const route = routeTask(shape, config, scenario.input.mode ?? config.mode, scenario.input.contextRatio ?? 0); const shards = route.topology === "scout" ? scoutShards(shape) : route.topology === "swarm" ? swarmShards(shape.mentionedPaths.length ? shape.mentionedPaths : ["."]) : []; const scoutModels = shards.map((shard) => scoutModelForLens(config, shard.lens)); const scoutCredits = scoutModels.reduce((sum, model) => sum + estimateCredits(catalog, model, SCOUT_INPUT_TOKENS, 0, config.scout.maxOutputTokens), 0); const rootCredits = estimateCredits(catalog, route.model, SCOUT_INPUT_TOKENS, 0, config.boundedWriter.maxOutputTokens); return { id: scenario.id, topology: route.topology, model: route.model, fanout: route.fanout, scoutModels, estimatedCredits: Number((scoutCredits + rootCredits).toFixed(6)), shapeSources: shape.reasonCodes.filter((code) => code.startsWith("shape:")).sort(), }; }); } export interface RoutingRegression { id: string; field: string; baseline: unknown; observed: unknown; } export function compareRouting(baseline: readonly RoutingObservation[], observed: readonly RoutingObservation[], costTolerance = 0.05): { passed: boolean; regressions: RoutingRegression[] } { const regressions: RoutingRegression[] = []; const byId = new Map(observed.map((entry) => [entry.id, entry])); for (const expected of baseline) { const actual = byId.get(expected.id); if (!actual) { regressions.push({ id: expected.id, field: "scenario", baseline: "present", observed: "missing" }); continue; } for (const field of ["topology", "model", "fanout"] as const) { if (actual[field] !== expected[field]) regressions.push({ id: expected.id, field, baseline: expected[field], observed: actual[field] }); } for (const field of ["scoutModels", "shapeSources"] as const) { if (JSON.stringify(actual[field]) !== JSON.stringify(expected[field])) regressions.push({ id: expected.id, field, baseline: expected[field], observed: actual[field] }); } const ceiling = expected.estimatedCredits * (1 + costTolerance); if (actual.estimatedCredits > ceiling) regressions.push({ id: expected.id, field: "estimatedCredits", baseline: expected.estimatedCredits, observed: actual.estimatedCredits }); } for (const actual of observed) { if (!baseline.some((entry) => entry.id === actual.id)) regressions.push({ id: actual.id, field: "scenario", baseline: "missing", observed: "present" }); } return { passed: regressions.length === 0, regressions }; }