/** * ProviderModelStatusTracker — centralized, shared status tracking for every * provider/model combination used across the agent, subagents, fallback * chains, and the one-shot LLM helper. * * ## Design * * - **Single shared instance**: created once per process, injected into every * component that calls providers or resolves fallback chains. The same * in-memory state is visible to the fallback extension, the one-shot * orchestrator, the dispatcher, and fleet-spawn — so a provider blocked * by a rate-limit spike is filtered EVERYWHERE, not just in one path. * * - **State machine** per (providerId, model) pair: * ``` * healthy ──(failure)──▶ degraded ──(more failures)──▶ blocked * ▲ │ * └──────────────(success streak / timeout)────────────┘ * ``` * * - **Thresholds** are configurable via constructor opts. Defaults: * | Metric | Threshold | Action | * |--------|-----------|--------| * | consecutive failures ≥ 2 | → `degraded` for 30 s | * | rate-limit hits ≥ 3 | → `blocked` for 5 min | * | consecutive failures ≥ 5 | → `blocked` for 5 min | * | success streak ≥ 3 | → back to `healthy` | * | `blocked` timeout elapses | → back to `healthy` | * * - **Error history**: keeps the last N errors per pair, with session id, * agent id, kind, and message so the WebUI and `/provider-status` can * render a meaningful timeline. * * - **Events**: emits `provider.status_changed` on every state transition. * * - **Thread-safety**: not needed — Node.js event-loop concurrency means * all access is single-threaded (async gaps don't race on the Map). * * @module coordination/provider-status-tracker */ import type { EventBus } from '../kernel/events.js'; import type { ProviderErrorKind } from '../types/provider.js'; export type ProviderModelState = 'healthy' | 'degraded' | 'blocked'; export interface ErrorHistoryEntry { readonly timestamp: number; readonly kind: ProviderErrorKind; readonly status: number; readonly message: string; readonly sessionId?: string | undefined; readonly agentId?: string | undefined; readonly retryAfterMs?: number | undefined; } export interface ProviderModelStatus { readonly providerId: string; readonly model: string; /** Current state in the state machine. */ readonly state: ProviderModelState; readonly consecutiveFailures: number; readonly totalFailures: number; readonly rateLimitHits: number; readonly overloadedHits: number; readonly serverErrors: number; readonly otherErrors: number; readonly consecutiveSuccesses: number; readonly totalSuccesses: number; readonly lastSuccessAt: number | null; readonly firstFailureAt: number | null; readonly lastFailureAt: number | null; /** When the current degraded or blocked state expires (ms epoch), or null. */ readonly stateExpiresAt: number | null; readonly lastErrorKind: ProviderErrorKind | null; readonly lastErrorMessage: string | null; readonly lastErrorStatus: number | null; readonly lastSessionId: string | null; readonly lastAgentId: string | null; /** Recent error history (newest-first, capped to `maxErrorHistory`). */ readonly recentErrors: readonly ErrorHistoryEntry[]; } export interface ProviderStatusTrackerConfig { /** * Consecutive failures before entering `degraded`. Default: 2. */ degradedAfterFailures?: number; /** * Duration (ms) the provider stays in `degraded` before reverting to healthy. * Default: 30_000 (30 s). */ degradedDurationMs?: number; /** * Rate-limit hits (error kind = 'rate_limit') before entering `blocked`. * Default: 3. */ blockAfterRateLimitHits?: number; /** * Consecutive failures before entering `blocked` directly. * Default: 5. */ blockAfterFailures?: number; /** * Duration (ms) the provider stays `blocked`. Default: 300_000 (5 min). */ blockDurationMs?: number; /** * Cooldown for an explicit exhausted-credit/quota response when the * provider does not publish a reset hint. These failures enter the waiting * room immediately instead of consuming the ordinary rate-limit threshold. * Default: 900_000 (15 min). */ quotaBlockDurationMs?: number; /** * Consecutive successes needed to leave `degraded` or `blocked` and return * to `healthy` (if the timeout hasn't already cleared it). Default: 3. */ recoverAfterSuccesses?: number; /** * Maximum error history entries kept per (providerId, model) pair. * Default: 50. */ maxErrorHistory?: number; } export declare class ProviderModelStatusTracker { private readonly cfg; private readonly map; private readonly events; constructor(opts?: { config?: ProviderStatusTrackerConfig | undefined; events?: EventBus | undefined; }); /** Resolve the logical health/display identity without changing wire routing. */ logicalIdentity(providerId: string, model: string): { providerId: string; model: string; }; /** * Record a successful provider call. Resets consecutive failure counters * and may transition out of degraded/blocked. */ recordSuccess(providerId: string, model: string, _meta?: { sessionId?: string | undefined; agentId?: string | undefined; }): void; /** * Record a failed provider call. Transitions state based on thresholds. * * @returns The new state after recording this failure. */ recordFailure(providerId: string, model: string, kind: ProviderErrorKind, status: number, message: string, meta?: { sessionId?: string | undefined; agentId?: string | undefined; retryAfterMs?: number | undefined; }): ProviderModelState; /** * Check if a (providerId, model) pair is currently usable. * Returns `true` when healthy or degraded (degraded is still usable, * just known flaky). Returns `false` when blocked or when the error * kind is `auth` (no point retrying ever). */ isAvailable(providerId: string, model: string): boolean; /** * Check if a (providerId, model) pair is currently rate-limited. * This is a stronger signal than just `!isAvailable()` — it tells * callers that requests should be re-tried after a delay rather * than skipped permanently. */ isRateLimited(providerId: string, model: string): boolean; /** * Get the full status for a (providerId, model) pair, or `undefined` * if no failures have been recorded. */ getStatus(providerId: string, model: string): ProviderModelStatus | undefined; /** * Returns a snapshot of ALL tracked provider/model statuses. * Useful for the `/provider-status` command and WebUI. */ getAllStatuses(): ProviderModelStatus[]; /** * Move every expired waiting-room entry back to healthy. Runtime routing * already performs this check lazily; this explicit sweep lets a UI/status * timer refresh the room even while no model calls are being made. * * @returns Number of entries released by this sweep. */ sweepExpired(): number; /** * Release one entry for an immediate half-open probe on its next real use. * History and totals are retained so operators do not lose diagnostics. */ retryNow(providerId: string, model: string): boolean; /** * Returns only currently blocked entries. */ getBlocked(): ProviderModelStatus[]; /** * Returns only currently degraded entries. */ getDegraded(): ProviderModelStatus[]; /** * Filter an array of objects that have `providerId` and `model` fields, * removing entries whose (providerId, model) is blocked. */ filterAvailable(entries: readonly T[]): T[]; /** * Check if a (providerId, model) pair would be blocked, WITHOUT the * side-effect of auto-recovering stale entries. Used in hot paths * where we don't want to mutate state during a quick peek. */ isBlocked(providerId: string, model: string): boolean; /** * Reset tracking for a specific (providerId, model) pair, or for ALL * pairs when both arguments are omitted. */ clear(providerId?: string, model?: string): void; /** * Get a JSON-safe snapshot suitable for WebUI rendering. * Includes summary stats + per-pair details. */ getSnapshot(): ProviderStatusSnapshot; /** Restore non-expired waiting-room entries from a previous process. */ restoreSnapshot(value: unknown): number; private getOrCreate; private freezeStatus; private emitStatusChanged; } export interface ProviderStatusSnapshot { readonly totalPairs: number; readonly healthy: number; readonly degraded: number; readonly blocked: number; readonly totalFailures: number; readonly totalRateLimits: number; readonly statuses: readonly ProviderModelStatus[]; } //# sourceMappingURL=provider-status-tracker.d.ts.map