import type { BaseEvent } from "../types.js"; export interface RoutingRegretMetrics { overSpawnCandidates: number | null; underSpawnCandidates: number | null; modelOverkillCandidates: number | null; modelUnderpowerCandidates: number | null; misscopedRoleCandidates: number | null; } export const ROUTING_REGRET_POLICY = { version: 1, zeroImpactRate: 0.75, duplicateFindingRate: 0.5, coordinationToWorkerOutput: 1, rootToolCalls: 12, rootInputGrowth: 0.5, repeatedFailures: 2, repairAttempts: 2, shallowRootToolCalls: 3, comparableTasks: 8, comparablePairs: 5, roleBoundaryViolations: 2, cheaperRatio: 0.2, quickStrongModelResolutionMs: 300_000, earlyStrongModelSavingsRatio: 0.2, } as const; export type Verdict = boolean | null; function number(event: BaseEvent | undefined, key: string): number | undefined { const value = event?.[key]; return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; } function text(event: BaseEvent | undefined, key: string): string | undefined { const value = event?.[key]; return typeof value === "string" && value ? value : undefined; } function time(event: BaseEvent | undefined): number | undefined { const value = event && Date.parse(event.timestamp); return value !== undefined && Number.isFinite(value) ? value : undefined; } function finalResult(events: readonly BaseEvent[]): BaseEvent | undefined { return events.findLast((event) => event.eventType === "result.completed"); } function classified(events: readonly BaseEvent[]): BaseEvent | undefined { return events.findLast((event) => event.eventType === "request.classified"); } function candidateCount(verdicts: readonly Verdict[]): number | null { return verdicts.some((verdict) => verdict === null) ? null : verdicts.filter(Boolean).length; } export function overSpawn(events: readonly BaseEvent[]): Verdict { const shape = classified(events); const result = finalResult(events); if (!result || text(shape, "topology") === "direct") return false; const agents = events.filter((event) => event.eventType === "agent.spawned" && text(event, "role") !== "reviewer"); if (agents.length < 2 || result.success !== true || result.verified !== true || shape?.knownScope !== true || number(shape, "independentUnits") !== 1) return false; if (agents.some((event) => !text(event, "nodeId"))) return null; const nodeIds = [...new Set(agents.map((event) => text(event, "nodeId")!))]; if (nodeIds.length < 2) return false; const attributions = new Map(events.filter((event) => event.eventType === "attribution.created").map((event) => [text(event, "nodeId"), event])); const selected = nodeIds.map((nodeId) => attributions.get(nodeId)); if (selected.some((event) => !event)) return null; const required = ["factsTotal", "factsUsed", "duplicateFacts", "decisionCount"]; if (selected.some((event) => required.some((key) => number(event, key) === undefined))) return null; const facts = selected.reduce((sum, event) => sum + number(event, "factsTotal")!, 0); if (!facts) return false; const zeroImpactRate = selected.filter((event) => number(event, "factsUsed") === 0).length / selected.length; const duplicateRate = selected.reduce((sum, event) => sum + number(event, "duplicateFacts")!, 0) / facts; const decisions = selected.reduce((sum, event) => sum + number(event, "decisionCount")!, 0); let workerOutput = 0; for (const nodeId of nodeIds) { const calls = events.filter((event) => ["model.call.completed", "model.call.failed"].includes(event.eventType) && text(event, "nodeId") === nodeId); if (!calls.length || calls.some((event) => number(event, "outputTokens") === undefined)) return null; workerOutput += calls.reduce((sum, event) => sum + number(event, "outputTokens")!, 0); } let coordination = 0; const handoffHashes = new Set(events.filter((event) => event.eventType === "handoff.sent").map((event) => text(event, "contentHash")).filter(Boolean)); for (const event of events.filter((entry) => entry.eventType === "handoff.sent" || (entry.eventType === "blackboard.event" && !handoffHashes.has(text(entry, "contentHash"))))) { const size = number(event, "payloadEstimatedTokens") ?? (number(event, "payloadChars") === undefined ? undefined : Math.ceil(number(event, "payloadChars")! / 4)); if (size === undefined) return null; coordination += size; } return zeroImpactRate >= ROUTING_REGRET_POLICY.zeroImpactRate && duplicateRate >= ROUTING_REGRET_POLICY.duplicateFindingRate && decisions === 0 && coordination > 0 && coordination / Math.max(1, workerOutput) >= ROUTING_REGRET_POLICY.coordinationToWorkerOutput; } export function underSpawn(events: readonly BaseEvent[]): Verdict { const shape = classified(events); const result = finalResult(events); const topology = text(shape, "topology"); if (!result || !["change", "fix"].includes(text(shape, "intent") ?? "") || !["direct", "scout"].includes(topology ?? "")) return false; if (result.success !== true || result.verified !== true) return false; const rootTools = events.filter((event) => event.eventType === "tool.started" && (!text(event, "role") || text(event, "role") === "root")).length; const failures = events.filter((event) => event.eventType === "verification.completed" && event.passed === false).length; const repairs = events.filter((event) => event.eventType === "repair.attempted" || event.eventType === "repair.started").length; if (rootTools < ROUTING_REGRET_POLICY.rootToolCalls || failures < ROUTING_REGRET_POLICY.repeatedFailures || repairs < ROUTING_REGRET_POLICY.repairAttempts) return false; const rootCalls = events.filter((event) => ["model.call.completed", "model.call.failed"].includes(event.eventType) && text(event, "role") === "root"); if (rootCalls.length < 2 || rootCalls.some((event) => number(event, "inputTokens") === undefined)) return null; const firstInput = number(rootCalls[0], "inputTokens")! + (number(rootCalls[0], "cachedInputTokens") ?? 0); const lastInput = number(rootCalls.at(-1), "inputTokens")! + (number(rootCalls.at(-1), "cachedInputTokens") ?? 0); if ((lastInput - firstInput) / Math.max(1, firstInput) < ROUTING_REGRET_POLICY.rootInputGrowth) return false; const initialUnits = number(shape, "independentUnits"); const affected = events.filter((event) => event.eventType === "attribution.created").map((event) => number(event, "affectedDirectories")).filter((value): value is number => value !== undefined); if (initialUnits === undefined || !affected.length) return null; return Math.max(...affected) > initialUnits; } interface ComparableTask { taskId: string; key: string; model: string; credits: number; } function comparableTask(events: readonly BaseEvent[]): ComparableTask | undefined { const request = events.find((event) => event.eventType === "request.received"); const shape = classified(events); const result = finalResult(events); if (!request || !shape || result?.success !== true || result.verified !== true || text(shape, "risk") !== "low" || text(shape, "topology") !== "direct") return undefined; if (events.some((event) => event.eventType === "repair.attempted" || event.eventType === "repair.started" || event.eventType === "escalation.triggered")) return undefined; if (events.filter((event) => event.eventType === "tool.started" && (!text(event, "role") || text(event, "role") === "root")).length > ROUTING_REGRET_POLICY.shallowRootToolCalls) return undefined; const rootAttribution = events.findLast((event) => event.eventType === "attribution.created" && text(event, "nodeId") === "root"); if (number(rootAttribution, "decisionCount") !== 0) return undefined; const calls = events.filter((event) => ["model.call.completed", "model.call.failed"].includes(event.eventType) && text(event, "role") === "root"); const models = [...new Set(calls.map((event) => text(event, "model")).filter((value): value is string => Boolean(value)))]; const providers = [...new Set(calls.map((event) => text(event, "provider")).filter((value): value is string => Boolean(value)))]; if (!calls.length || models.length !== 1 || providers.length !== 1 || calls.some((event) => number(event, "creditsEstimated") === undefined)) return undefined; const requestHash = text(request, "requestHash"); const commandHash = text(events.find((event) => event.eventType === "verification.started" || event.eventType === "verification.completed"), "commandHash"); const catalog = text(request, "priceCatalogVersion"); const policy = text(shape, "policy"); if (!requestHash || !commandHash || !catalog || !policy) return undefined; const reworked = events.some((event) => (event.eventType === "feedback.explicit" || event.eventType === "feedback.implicit" || event.eventType === "user.feedback") && (event.kind === "fixed" || event.explicit === false)); const comparisonVersion = request.experimentId ? `experiment:${request.experimentId}` : `config:${request.configVersion}`; const key = [requestHash, commandHash, request.profile, providers[0], text(shape, "topology"), policy, catalog, comparisonVersion, "verified", reworked ? "reworked" : "clean"].join("|"); return { taskId: request.taskId, key, model: models[0]!, credits: calls.reduce((sum, event) => sum + number(event, "creditsEstimated")!, 0) }; } function modelOverkill(tasks: readonly (readonly BaseEvent[])[]): number | null { const records = tasks.map(comparableTask).filter((value): value is ComparableTask => Boolean(value)); const groups = new Map(); for (const record of records) groups.set(record.key, [...(groups.get(record.key) ?? []), record]); const pairs: Array<[ComparableTask, ComparableTask]> = []; for (const group of groups.values()) for (let left = 0; left < group.length; left += 1) for (let right = left + 1; right < group.length; right += 1) if (group[left]!.model !== group[right]!.model) pairs.push([group[left]!, group[right]!]); const comparableTasks = new Set(pairs.flatMap(([left, right]) => [left.taskId, right.taskId])); if (comparableTasks.size < ROUTING_REGRET_POLICY.comparableTasks || pairs.length < ROUTING_REGRET_POLICY.comparablePairs) return null; const candidates = new Set(); for (const [left, right] of pairs) { if (right.credits <= left.credits * (1 - ROUTING_REGRET_POLICY.cheaperRatio)) candidates.add(left.taskId); if (left.credits <= right.credits * (1 - ROUTING_REGRET_POLICY.cheaperRatio)) candidates.add(right.taskId); } return candidates.size; } export function modelUnderpower(events: readonly BaseEvent[]): Verdict { const escalation = events.find((event) => event.eventType === "escalation.triggered" && event.reason === "same-fingerprint-twice"); if (!escalation) return false; const escalationAt = time(escalation); if (escalationAt === undefined) return null; const failures = events.filter((event) => event.eventType === "verification.completed" && event.passed === false && (time(event) ?? Infinity) < escalationAt).length; const repairs = events.filter((event) => (event.eventType === "repair.attempted" || event.eventType === "repair.started") && (time(event) ?? Infinity) < escalationAt).length; if (failures < ROUTING_REGRET_POLICY.repeatedFailures || repairs < ROUTING_REGRET_POLICY.repairAttempts) return false; const green = events.find((event) => event.eventType === "verification.completed" && event.passed === true && (time(event) ?? -Infinity) > escalationAt); const greenAt = time(green); if (greenAt === undefined || greenAt - escalationAt > ROUTING_REGRET_POLICY.quickStrongModelResolutionMs) return false; const calls = events.filter((event) => ["model.call.completed", "model.call.failed"].includes(event.eventType) && (time(event) ?? Infinity) <= greenAt); if (!calls.length || calls.some((event) => number(event, "creditsEstimated") === undefined || time(event) === undefined)) return null; const before = calls.filter((event) => time(event)! < escalationAt).reduce((sum, event) => sum + number(event, "creditsEstimated")!, 0); const after = calls.filter((event) => time(event)! >= escalationAt).reduce((sum, event) => sum + number(event, "creditsEstimated")!, 0); if (after <= 0) return null; return before / (before + after) >= ROUTING_REGRET_POLICY.earlyStrongModelSavingsRatio; } export function misscopedRole(task: readonly BaseEvent[]): Verdict { const perRole = new Map(); for (const event of task) { if (event.eventType !== "governance.violation") continue; const role = typeof event.role === "string" && event.role ? event.role : typeof event.nodeId === "string" ? event.nodeId : "unknown"; perRole.set(role, (perRole.get(role) ?? 0) + 1); } return [...perRole.values()].some((count) => count >= ROUTING_REGRET_POLICY.roleBoundaryViolations); } export function computeRoutingRegret(taskGroups: Iterable): { metrics: RoutingRegretMetrics; limitations: string[] } { const tasks = [...taskGroups]; const metrics: RoutingRegretMetrics = { overSpawnCandidates: candidateCount(tasks.map(overSpawn)), underSpawnCandidates: candidateCount(tasks.map(underSpawn)), modelOverkillCandidates: modelOverkill(tasks), modelUnderpowerCandidates: candidateCount(tasks.map(modelUnderpower)), misscopedRoleCandidates: candidateCount(tasks.map(misscopedRole)), }; const unavailable = Object.entries(metrics).filter(([, value]) => value === null).map(([key]) => key.replace(/Candidates$/, "")); return { metrics, limitations: unavailable.length ? [`Routing regret unavailable: incomplete evidence for ${unavailable.join(", ")}.`] : [] }; }