import { classify } from "../classifier/classifier"; import type { ModelKey, ProbeKind, ProbeOutcome } from "../episode/types"; import type { Clock } from "../platform/clock"; import type { ModelRegistryPort } from "../platform/ports"; export const PROBE_MESSAGE = { role: "user" as const, content: "ping" }; export interface ProbeRequest { model: Model; provider: string; modelId: string; generation: number; kind: ProbeKind; /** R7/OD-4: the canonical `provider/modelId` key this probe is aimed at. The prober treats it as * opaque — it never chooses, defaults or rewrites it — and echoes it onto the outcome so a verdict * about one entered candidate can never be read as a verdict about another. */ target: ModelKey; } export interface ProbeEvent { type: "probe_outcome"; outcome: ProbeOutcome; target: ModelKey; generation: number; probeKind: ProbeKind; scheduled: boolean; resetsAt?: number; } export interface ProbeResult { outcome: ProbeOutcome; target: ModelKey; generation: number; kind: ProbeKind; resetsAt?: number; } export interface ProviderResponseHint { status: number; } export interface NormalizedProbeError { message: string; code?: string; status?: number; diagnostics?: unknown; } export interface ProbeExecutorOptions { modelRegistry: ModelRegistryPort; onOutcome?: (event: ProbeEvent) => void; isGenerationCurrent?: (generation: number) => boolean; clock?: Pick; now?: () => number; hintWindowMs?: number; } export class ProbeBusyError extends Error { constructor() { super("a recovery probe is already in flight"); this.name = "ProbeBusyError"; } } const record = (value: unknown): Record | undefined => value !== null && typeof value === "object" ? value as Record : undefined; const text = (value: unknown): string | undefined => typeof value === "string" && value.length > 0 ? value : undefined; const number = (value: unknown): number | undefined => typeof value === "number" && Number.isFinite(value) ? value : undefined; const json = (value: unknown): string => { try { return JSON.stringify(value) ?? String(value); } catch { return String(value); } }; export function normalizeProbeError(error: unknown): NormalizedProbeError { const value = record(error), nested = record(value?.error) ?? record(value?.cause), response = record(value?.response); const message = text(value?.errorMessage) ?? text(value?.message) ?? text(nested?.message) ?? (typeof error === "string" ? error : json(error)); const code = text(value?.code) ?? text(value?.type) ?? text(nested?.code); const status = number(value?.status) ?? number(value?.statusCode) ?? number(response?.status) ?? number(nested?.status); const diagnostics = value?.diagnostics ?? value?.details ?? value?.body ?? nested?.details; return { message, ...(code ? { code } : {}), ...(status === undefined ? {} : { status }), ...(diagnostics === undefined ? {} : { diagnostics }) }; } interface ProbeMapping { outcome: ProbeOutcome; resetsAt?: number; } const authCode = (code?: string): boolean => /^(?:auth|oauth)$/.test(code?.toLowerCase() ?? ""); export function classifyProbeError(error: unknown, request: Pick, httpHint?: ProviderResponseHint, now = Date.now()): ProbeMapping { const normalized = normalizeProbeError(error); const verdict = classify({ provider: request.provider, modelId: request.modelId, errorMessage: normalized.message, diagnostics: normalized.diagnostics ?? error, httpHint }, now); if (verdict.kind === "quota") return { outcome: "quota", ...(verdict.resetsAt === undefined ? {} : { resetsAt: verdict.resetsAt }) }; if (verdict.kind === "transient") return { outcome: "outage" }; if (authCode(normalized.code)) return { outcome: "unknown" }; if ((normalized.code === "stream" || normalized.code === "provider") && verdict.matchedRule !== "context-overflow-guard") return { outcome: "outage" }; return { outcome: "unknown" }; } const errorResult = (value: unknown): unknown => { const object = record(value); return object && (object.stopReason === "error" || object.ok === false || "errorMessage" in object || "error" in object || "code" in object) ? value : undefined; }; export class RecoveryProber { private readonly now: () => number; private readonly hintWindowMs: number; private inFlight?: { generation: number; token: number }; private hint?: { generation: number; status: number; expiresAt: number; token: number }; private token = 0; private readonly invalidatedTokens = new Set(); constructor(private readonly options: ProbeExecutorOptions) { this.now = options.now ?? (() => options.clock?.now() ?? Date.now()); this.hintWindowMs = Number.isFinite(options.hintWindowMs) && (options.hintWindowMs as number) >= 0 ? options.hintWindowMs as number : 1_000; } get busy(): boolean { return this.inFlight !== undefined; } afterProviderResponse(response: ProviderResponseHint, generation = this.inFlight?.generation): void { const flight = this.inFlight; if (!flight || generation === undefined || generation !== flight.generation || !this.current(generation)) return; if (!Number.isFinite(response.status)) return; this.hint = { generation, status: response.status, expiresAt: this.now() + this.hintWindowMs, token: flight.token }; } captureAfterProviderResponse(response: ProviderResponseHint, generation?: number): void { this.afterProviderResponse(response, generation); } cancel(generation?: number): void { const flight = this.inFlight; if (flight && (generation === undefined || flight.generation === generation)) { this.invalidatedTokens.add(flight.token); this.inFlight = undefined; } if (this.hint && (generation === undefined || this.hint.generation === generation)) this.hint = undefined; } invalidateGeneration(generation: number): void { this.cancel(generation); } probe(request: ProbeRequest): Promise { if (this.inFlight) return Promise.reject(new ProbeBusyError()); const flight = { generation: request.generation, token: ++this.token }; this.inFlight = flight; this.hint = undefined; return this.run(request, flight); } start(request: ProbeRequest): Promise { return this.probe(request); } private current(generation: number, token = this.inFlight?.token): boolean { return this.inFlight?.generation === generation && this.inFlight.token === token && !this.invalidatedTokens.has(token) && this.options.isGenerationCurrent?.(generation) !== false; } private activeHint(flight: { generation: number; token: number }): ProviderResponseHint | undefined { if (!this.hint || this.hint.generation !== flight.generation || this.hint.token !== flight.token || this.hint.expiresAt < this.now()) return undefined; return { status: this.hint.status }; } private async run(request: ProbeRequest, flight: { generation: number; token: number }): Promise { try { let value!: Result; let thrown: unknown; let failed = false; try { value = await this.options.modelRegistry.complete(request.model, { messages: [PROBE_MESSAGE] }); } catch (error) { failed = true; thrown = error; } if (!this.current(flight.generation, flight.token)) return undefined; const error = failed ? thrown : errorResult(value); const mapped = failed || error !== undefined ? classifyProbeError(error, request, this.activeHint(flight), this.now()) : { outcome: "usable" as const }; if (!this.current(flight.generation, flight.token)) return undefined; const result: ProbeResult = { ...mapped, target: request.target, generation: request.generation, kind: request.kind }; this.options.onOutcome?.({ type: "probe_outcome", outcome: result.outcome, target: result.target, generation: result.generation, probeKind: result.kind, scheduled: result.kind === "scheduled", ...(result.resetsAt === undefined ? {} : { resetsAt: result.resetsAt }) }); return result; } finally { if (this.inFlight?.token === flight.token) { this.inFlight = undefined; this.hint = undefined; } } } } export { RecoveryProber as ProbeExecutor };