import type { AttemptBeginFailure, AttemptCompletionFailure, AttemptHandle, AttemptLifecyclePort, AttemptOutcome, CompletedAttempt, ProviderTargetIdentity, TransitionResult } from "./kernel/contracts.js"; import { type PingRecord } from "./ping/metrics.js"; import { type QuotaObservation } from "./quota-observation.js"; /** A provider/model deployment, deliberately without a credential cell selector. */ export interface ProviderDeploymentIdentity { readonly provider: string; readonly model: string | null; } export interface CircuitState { /** The exact credential/model cell this in-memory state belongs to. */ readonly target: ProviderTargetIdentity; consecutiveFailures: number; lastFailureTime: number; cooldownUntil: number; cooldownSource: CooldownSource | null; unexplained429s: number; lastStatus?: number | undefined; pings: PingRecord[]; quotaObservations: QuotaObservation[]; credentialFailures: number; lastCredentialStatus?: number | undefined; credentialFaultUntil: number; } /** * One started attempt: when, and the relay's own chars/4 estimate of its input size — an entry in * the per-cell attempt-start log `pacing.ts` counts its trailing window over (2026-09-15). * * ⚠ The log is NOT part of `CircuitState`, deliberately. It is recorded in `beginAttempt` — the * one choke point every egress on both fronts passes through — and creating HEALTH state there * would surface a deployment in `/candidates`, telemetry and the breaker export before any * outcome existed, and would break the pinned rule that a relay-local fault creates no provider * health state (`test/closed-vocabulary-routing.test.ts`). So it lives in its own map keyed by the * same cell, in memory only and NOT carried by `breaker-persistence.ts`: a restart forgets at most * one window of pacing memory, and the failure direction is LESS pacing (the provider's own 429 * then teaches the cell, as before), never more. Bounded by `ATTEMPT_START_WINDOW_MS` and * `MAX_ATTEMPT_STARTS`. */ export interface AttemptStart { readonly at: number; /** * `estimateRequestTokens` for the request — INPUT only, an estimate, and a LOWER bound of what a * provider meters (it counts output too). Null when the caller had no estimate; a null never * contributes to a token count, so a window holding one is reported as partial, never as 0. */ readonly estimatedInputTokens: number | null; } /** What `CircuitBreaker.attemptsInWindow` answers about one cell's trailing window. */ export interface AttemptWindow { /** Attempts started inside the window. A LOWER bound while `saturated` is true. */ readonly requests: number; /** * Sum of the estimated INPUT tokens of those attempts, or null when any attempt in the window * carried no estimate — an unknown must not read as 0 (the provenance invariant). */ readonly estimatedInputTokens: number | null; /** * True when the log was capped at `MAX_ATTEMPT_STARTS` with every retained entry inside the * window, so the true count is unknowable from here and `requests` is only a floor. */ readonly saturated: boolean; } /** The narrowest handle that names one breaker cell: exactly what `getKey` reads. */ export type BreakerCellSelector = Pick; /** One cell cooling on a relay-invented rung that the ping loop may re-test. */ export interface RateLimitCoolingCell { readonly provider: string; readonly model: string | null; readonly credentialId: string; readonly cooldownUntil: number; readonly source: CooldownSource; } /** * One whole circuit-breaker cell, in a shape that can be written to disk and read back. * * Declared here rather than in `breaker-persistence.ts` so the dependency runs one way only * (persistence imports the breaker, never the reverse) and `exportState`/`restoreState` * can stay IO-free. * * All fields are OPTIONAL on the wire; absent means the value a fresh cell has. */ export interface BreakerCellRow { readonly provider: string; readonly model: string | null; readonly kind: string; readonly credentialId: string; readonly base?: string | undefined; /** Absolute epoch ms. A row whose value is not in the future restores as lapsed. */ readonly cooldownUntil: number; readonly cooldownSource: CooldownSource | null; /** Consecutive unexplained 429s — the ladder index that makes the next 429 escalate correctly. */ readonly unexplained429s: number; readonly lastStatus?: number | undefined; /** Failures since the last success on this cell; `MAX_FAILURES_BEFORE_TRIP` reads it. */ readonly consecutiveFailures?: number; /** Epoch ms of the last failure; the dashboard's Cooldowns panel shows it as `observedAt`. */ readonly lastFailureTime?: number; /** 401/403 count on this credential×model cell — the credential axis, never health. */ readonly credentialFailures?: number; readonly lastCredentialStatus?: number | undefined; /** Epoch ms; the fault is active only while this is in the future (`CREDENTIAL_FAULT_TTL_MS`). */ readonly credentialFaultUntil?: number; /** The served-request window (`MAX_PING_HISTORY` newest); `telemetry.ts` scores stability from it. */ readonly pings?: PingRecord[]; /** Provider-stated quota headers; `availability.ts` discards a stale one at read time. */ readonly quotaObservations?: QuotaObservation[]; } /** Provisional upstream metadata, committed only with a terminal attempt outcome. */ export interface HeaderObservation { readonly target: ProviderTargetIdentity; readonly status: number; readonly elapsedMs: number; readonly observedAt: number; readonly quotaObservations?: readonly QuotaObservation[] | undefined; readonly retryAfterMs?: number | undefined; } export type HeaderObservationFailure = AttemptCompletionFailure | { readonly kind: "duplicate-observation"; }; /** Deployment-level observations merged from its credential cells. */ export interface DeploymentMeasurement { readonly pings: readonly PingRecord[]; readonly stabilityScore: number | null; /** Least-observed contributing cell: use this, never merged sample count, for confidence. */ readonly minSamples: number; } /** * Every reason a cell may be cooling — the ONE definition. * * ⚠ The list is the source of truth and the type is DERIVED from it, deliberately. * `breaker-persistence.ts` used to re-state all five members by hand in `isCooldownSource`, which * is the "runtime list hand-copied from the type" defect this repo records against nine * `dashboard-contract.ts` unions and against `UNTIL_BASES`: the compiler cannot connect the two, so * a new member silently fails to load from disk while type-checking clean. It now imports this. */ export declare const COOLDOWN_SOURCES: readonly ["default", "escalation", "retry-after", "loopback", "quota", "elapsed", "failure-escalation"]; export type CooldownSource = (typeof COOLDOWN_SOURCES)[number]; export interface CooldownClearSelector { readonly provider: string; readonly model?: string; readonly credentialId?: string; } export interface ClearedCircuitCell { readonly provider: string; readonly model: string | null; readonly credentialId: string; } export interface CircuitCooldownClearResult { readonly breakerCells: ClearedCircuitCell[]; readonly credentialFaults: ClearedCircuitCell[]; } /** * The widest trailing window `pacing.ts` may ask for — one day, the longest period a stated * rate limit names (`rpd`/`tpd`). An older start can never be inside any window, so it is dropped * at the next append. */ export declare const ATTEMPT_START_WINDOW_MS = 86400000; /** * Hard cap on retained starts per cell. Ten thousand covers every daily free-tier ceiling this * relay has met (the largest measured was 1,500 RPD) with room; above it the count is a FLOOR * (`AttemptWindow.saturated`) and pacing declines rather than guess. */ export declare const MAX_ATTEMPT_STARTS = 10000; /** Ordering-only middle band retained for telemetry consumers during migration. */ export declare const UNMEASURED_STABILITY = 50; /** * How long a generic (non-429, non-402, no `Retry-After`) failure cools a cell. * * ⚠ **A cooldown must outlast the failure that caused it.** Measured 2026-08-30: * `nim/deepseek-ai/deepseek-v4-flash-0731` hung on **43 consecutive attempts**, each costing the * full 120000 ms provider timeout, and its breaker read `closed` every time anyone looked — so the * relay walked into the same 120-second hole on every request. Nothing was broken in the charging * path: a `deadline` provenance reaches the health path and a 504 passes the 4xx filter, so the * trip fired exactly as written. The CONSTANT was simply smaller than the failure it punished. A * 120 s waste bought a 60 s cooldown, and requests arrived 78-139 s apart, so the cell was always * closed again by the next walk. * * So `DEFAULT_COOLDOWN_MS` becomes a FLOOR, and a slow failure cools for the time it actually * wasted. That figure is MEASURED (`elapsedMs`), never invented, which is what permits setting a * duration at all under this relay's rule against inventing one — the same standing that lets a * provider-stated `Retry-After` set one. It takes the same `MAX_RETRY_AFTER_MS` ceiling, so a * 30-minute `timeoutMs` cannot buy a 30-minute cooldown off a single sample. * * ⚠ **Fast failures are unaffected by construction.** A 300 ms error keeps the 60 s default, * because the floor wins. Only a failure slow enough to hurt moves the number — which is why this * needs no failure-kind plumbing, no new configuration, and no change to any other branch of the * ladder: a `Retry-After`, a 429 escalation and a 402 all still win where they applied before. * * Pure, so it is pinned directly rather than through the breaker's state machine. * Evidence: `docs/history/latency-demotion-regression-2026-08-30.md` §3. */ export declare function failureCooldown(elapsedMs: number): { ms: number; source: CooldownSource; }; export declare class CircuitBreaker implements AttemptLifecyclePort { #private; private states; /** Credential-domain leases are deliberately wider than a deployment cell. */ private credentialInFlight; /** Per-cell attempt-start log, same key as `states` — pacing's dataset, not health (see `AttemptStart`). */ private attemptStarts; /** Cell key. This intentionally has no provider/model-string compatibility path. */ private getKey; private getOrCreate; /** * Begin an attempt. `start` is optional so the `AttemptLifecyclePort` contract is unchanged; * the request path passes its egress instant and the request's own input estimate so the * cell's attempt-start log (`pacing.ts`'s dataset) records the same clock the walk runs on. */ beginAttempt(target: ProviderTargetIdentity, start?: { at?: number; estimatedInputTokens?: number | null; }): TransitionResult; /** * Append one start to the cell's log and prune it: entries older than the widest window * `pacing.ts` reads (`ATTEMPT_START_WINDOW_MS`) fall off, then the newest `MAX_ATTEMPT_STARTS` * are kept. Pruning at both bounds keeps a hot cell at a fixed cost and a quiet one empty. */ private recordAttemptStart; /** * The attempts this relay started against ONE cell inside the trailing `windowMs` — the * sliding-window count `pacing.ts` holds against a stated ceiling. Exact-cell only, like every * other reader here: a sibling credential's starts are that credential's own rate. * * ⚠ A sliding window is the conservative reading on purpose. A count that never exceeds L in * ANY trailing window cannot exceed L in a provider's fixed window either (a fixed window is * one position of the sliding one), so this bound holds whichever window shape the provider * runs — which the relay is never told. */ attemptsInWindow(cell: BreakerCellSelector, windowMs: number, now: number): AttemptWindow; /** * A PROBE answered 200 for this exact cell: end a relay-invented recovery cooldown it disproves. * * Two gates, both required: `PROBE_SUCCESS_ENDS_COOLDOWN` names which SOURCES a probe may end, * then `probeDisprovesCooldown` checks the status that earned it. The legacy guessed 429 sources * still require `lastStatus === 429`; `failure-escalation` accepts only 402 or 5xx. A stated * quota, credential fault, operator hard cap, ordinary generic-failure floor, or slow measured * failure is left exactly as it was. * * ⚠ `unexplained429s` — the escalation ladder's index — is deliberately NOT reset. A probe is a * one-token completion; the ladder counts what REAL traffic saw, and only a real success (the * ordinary `applyHealthOutcome` path) resets it. So a cell that keeps 429ing real requests while * passing probes still escalates its nominal rung, and the probe cadence sets the effective * floor — that is the stated cost of polling for recovery. */ endRateLimitCooldown(cell: BreakerCellSelector, at?: number): boolean; /** * Every cell on a relay-invented recovery rung worth spending a probe on: guessed 429 escalation * plus repeated-failure escalation for 402/5xx. A deployment that recovered should not stay * parked for the rest of a window the relay itself invented. * * A stated `Retry-After` is still honoured rather than second-guessed; loopback is only 5 s; * quota/elapsed are not proactive re-probe targets. `probeDisprovesCooldown` keeps the status * constraint aligned with `endRateLimitCooldown`. Sorted by soonest lift so the bounded probe * budget reaches the cells closest to recovery first. */ rateLimitCoolingCells(now?: number): RateLimitCoolingCell[]; observeHeaders(handle: AttemptHandle, observation: HeaderObservation): TransitionResult; completeAttempt(handle: AttemptHandle, outcome: AttemptOutcome): TransitionResult; private getAttemptRecord; private applyTerminalOutcome; /** Exact-cell health only. A sibling credential never participates. */ isHealthy(target: ProviderTargetIdentity, now?: number): boolean; /** Typed non-request writer; it is still exact-cell only. */ recordOutcome(target: ProviderTargetIdentity, outcome: { ok: boolean; elapsedMs: number; status?: number; quotaObservations?: readonly QuotaObservation[] | undefined; at?: number; retryAfterMs?: number | undefined; }): void; /** * A cancelled attempt: teach the cell only what the cancellation actually proves. * * ⚠ This is the ONE place the old flat `if (outcome.terminal === "cancelled") return;` used to * be, and its two guards are why that line could not simply be deleted. Deleting it would have * charged provider health for every ordinary client disconnect — `PROVENANCE_REACHES_HEALTH_PATH` * already answers `true` for `client-cancellation` — and for every hedge loser, silently * repealing a documented hedging invariant. Two routing changes nobody asked for, from one line. * * ⚠ No quota observations are recorded here. The `HeaderObservation` a cancelled attempt carries * is not passed on: a walk that was abandoned may hold a partially-read response, and the * ordinary path records observations only alongside an outcome it also charges. */ private applyCancelledOutcome; private applyHealthOutcome; /** Retain a fresh axis/period tuple without discarding another axis from an earlier response. */ private recordQuotaObservations; /** Record a credential fault on the explicit credential/model cell only. */ recordCredentialFault(target: ProviderTargetIdentity, status: number, at?: number): void; private applyCredentialFault; /** Clear faults only for the stated credential across its own model cells. */ clearCredentialFaults(credentialId: string): number; /** Clear only credential-fault fields inside an operator-addressed selection. */ clearCredentialFaultState(selector: CooldownClearSelector): ClearedCircuitCell[]; /** * Clear operator-addressed cooling state without manufacturing a successful observation. * Failure/stability history and quota measurements remain evidence; only the fields that * currently demote a cell, plus the unexplained-429 ladder, are reset. */ clearCooldownState(selector: CooldownClearSelector): CircuitCooldownClearResult; /** Exact-cell credential fault only. */ hasCredentialFault(target: ProviderTargetIdentity, now?: number): boolean; /** * A quota demotion is a cooldown with a KNOWN end: the `resetsAt` the evidence stated, or the * period boundary derived from it (availability's `derived-boundary` rung). It never touches the * failure counters — a spent allowance is not a sick backend — and it is cleared by any success * through the same path as every other cooldown (`applyHealthOutcome`), which also means it can * never trip the breaker or drop the candidate; `orderByUsability` only ever reads * `cooldownUntil`. * * `until <= now` is declined outright rather than clamped to some minimum: a reset already in * the past means the evidence is stale, and cooling a healthy cell on stale evidence is worse * than doing nothing. */ recordQuotaCooldown(target: ProviderTargetIdentity, until: number, at?: number): void; /** Active backend attempts for one credential slot across all of its deployments. */ inFlightCredential(credentialId: string): number; /** Exact-cell state only. */ getState(target: ProviderTargetIdentity): CircuitState | undefined; /** Process-local cell states; their identity is stored rather than decoded from keys. */ getAllStates(): ReadonlyMap; /** * Every cell — cooling or not — with every persisted field, for `breaker-persistence.ts`. * * Pure and IO-free on purpose: this module holds no file handles, so persistence stays a * separate concern that a bare programmatic proxy can simply not install. An optional field is * omitted only when it is `undefined` (`lastStatus`, `lastCredentialStatus`, `base`); `pings` * is written as held, already capped at `MAX_PING_HISTORY`. */ exportState(): BreakerCellRow[]; /** * Re-apply persisted rows, returning how many were applied. * * ⚠ NEVER touches a cell this process has already created. A restore runs at startup, where that * is every row; making it defensive costs nothing and means a late or repeated call cannot * shorten, extend or overwrite live state — the `recordQuotaCooldown` rule ("a quota demotion * never SHORTENS someone else's cooldown"), generalised to the whole cell. * * ⚠ FAITHFUL, not future-only. Every field is copied as it was, `cooldownUntil` included even when * it is already in the past: a lapsed cooldown restores as lapsed and the cell reads ready, * exactly as in memory. The old rule — restore only while the cooldown is still in the future — * reset the escalation ladder on every restart, a behaviour the running process does not have. * In memory a lapsed cooldown keeps its `unexplained429s`; the counter alone demotes nothing, * only a FRESH 429 applies it, and that 429 is a fresh measurement of the same condition. A * restart is not a success. Absent optional fields take the fresh-cell defaults; `pings` is * trimmed to the newest `MAX_PING_HISTORY` defensively. Nothing here notifies persistence — * rewriting the file with what was just read from it would be a pointless write. */ restoreState(rows: readonly BreakerCellRow[]): number; /** * Register the persistence listener. At most one: a second install would double every write, * and there is exactly one file. */ onStateChanged(listener: () => void): void; private notifyStateChanged; /** Deployment measurement merges timestamp-sorted pings without inflating confidence. */ getDeploymentMeasurement(deployment: ProviderDeploymentIdentity): DeploymentMeasurement; /** Demote unhealthy cells without deleting any candidate; preserve within-band order. */ orderByUsability(targets: readonly T[], now?: number): T[]; reset(): void; } export declare const globalCircuitBreaker: CircuitBreaker;