/** * Compatibility Probe engine — stages, budget, stop conditions (ticket 1 #43), * target doctor precheck gate (ticket 3 #45). * * Pure orchestration: inject transport + optional precheck (faux in tests). * Never calls setModel; never carries session history. */ import { buildContractRequest } from "./contracts.ts"; import { classifyStageFailure, evaluateContract } from "./evidence.ts"; import { DEFAULT_PROBE_CONTRACTS, PROBE_MAX_REQUESTS, PROBE_TIMEOUT_MS, probeMaxTokensFor, type ProbeContractId, type ProbeEngineOptions, type ProbeRunPrecheckSnapshot, type ProbeRunResult, type ProbeStageResult, type ProbeStoppedReason, type ProbeTarget, type ProbeVerifier, } from "./types.ts"; function planContracts( target: ProbeTarget, contracts?: ProbeContractId[], ): { /** Contracts to execute (order preserved). */ order: ProbeContractId[]; /** Pre-built skip stages keyed by contract. */ skips: Map; } { const order = contracts?.length ? [...contracts] : [...DEFAULT_PROBE_CONTRACTS]; const skips = new Map(); for (const c of order) { if (c === "reasoning" && !target.reasoning) { skips.set(c, { contract: "reasoning", status: "skip", summary: "skipped: target does not claim reasoning support", requestCount: 0, }); } } return { order, skips }; } async function resolvePrecheck( input: ProbeEngineOptions["precheck"], ): Promise { if (input === undefined) return undefined; if (typeof input === "function") return await input(); return input; } function budgetSnapshot( maxRequests: number, used: number, maxTokens: number, timeoutMs: number, ) { return { maxRequests, used, maxTokens, timeoutMs }; } /** * Run a Compatibility Probe against a Probe Target. * Read-only: does not change configuration or Session Model. */ export async function runProbe(opts: ProbeEngineOptions): Promise { const maxRequests = opts.maxRequests ?? PROBE_MAX_REQUESTS; const timeoutMs = opts.timeoutMs ?? PROBE_TIMEOUT_MS; const maxTokens = probeMaxTokensFor(opts.target, opts.maxTokens); const createSignal = opts.createSignal ?? ((ms: number) => AbortSignal.timeout(ms)); const now = opts.now ?? (() => Date.now()); const { order, skips } = planContracts(opts.target, opts.contracts); const stages: ProbeStageResult[] = []; // Target doctor precheck (ticket 3): FAIL blocks all network; WARN continues. const precheck = await resolvePrecheck(opts.precheck); if (precheck && !precheck.allowProbe) { const haltSummary = `stopped: doctor precheck failed (${precheck.summary})`; for (const contract of order) { const skip = skips.get(contract); if (skip) { stages.push(skip); continue; } stages.push({ contract, status: "stopped", summary: haltSummary, requestCount: 0, }); } return { target: { ...opts.target }, stages, ok: false, stoppedReason: "precheck", requestCount: 0, budget: budgetSnapshot(maxRequests, 0, maxTokens, timeoutMs), precheck, }; } let requestCount = 0; let stoppedReason: ProbeStoppedReason | undefined; let halt = false; let haltSummary = ""; for (const contract of order) { const skip = skips.get(contract); if (skip) { stages.push(skip); continue; } if (halt) { stages.push({ contract, status: "stopped", summary: haltSummary, requestCount: 0, }); continue; } if (requestCount >= maxRequests) { halt = true; stoppedReason = "budget"; haltSummary = "stopped: request budget exhausted"; stages.push({ contract, status: "stopped", summary: haltSummary, requestCount: 0, }); continue; } const built = buildContractRequest(contract, now()); const signal = createSignal(timeoutMs); requestCount += 1; const transportResult = await opts.transport({ contract, // Forward a copy so a transport cannot mutate the caller's target object // (repair verification sends the in-memory candidate target). target: { ...opts.target }, model: opts.model, context: built.context, options: { maxTokens, signal, ...built.optionExtras, }, }); const evaluation = evaluateContract(contract, transportResult); if (evaluation.ok) { stages.push({ contract, status: "pass", summary: evaluation.summary, requestCount: 1, httpStatus: transportResult.httpStatus, }); continue; } const failure = classifyStageFailure( evaluation, transportResult.httpStatus, transportResult.message.errorMessage, ); stages.push({ contract, status: "fail", category: failure.category, unrepairable: failure.unrepairable || undefined, httpStatus: transportResult.httpStatus, summary: failure.summary, requestCount: 1, }); halt = true; if (failure.hardStop) { stoppedReason = "unrepairable"; haltSummary = `stopped: unrepairable (${failure.category})`; } else { stoppedReason = "failure"; haltSummary = "stopped: previous stage failed"; } } const hasFailOrStopped = stages.some( (s) => s.status === "fail" || s.status === "stopped", ); const ok = !hasFailOrStopped; return { target: { ...opts.target }, stages, ok, stoppedReason, requestCount, budget: budgetSnapshot(maxRequests, requestCount, maxTokens, timeoutMs), ...(precheck ? { precheck } : {}), }; } /** Bind one resolved model/transport/precheck context for later verification. */ export function createProbeVerifier( context: Pick, ): ProbeVerifier { return (options) => runProbe({ ...options, ...context }); }