import type { ProviderConfig } from "./config.js"; /** * How long one provider is held off after a model-missing signal forced its catalog to refresh. * * The TTL above answers "how old may a catalog be before we re-read it on a clock". This answers a * different question: "how often may EVIDENCE make us re-read it". A 404 stating that a listed model * does not exist is real evidence the roster moved, but it is also the shape a broken or * half-migrated deployment produces on EVERY request — so without a floor, one member of a pool * would drive the provider's `/models` endpoint as fast as traffic arrives, which is the stampede * the TTL exists to prevent in the first place. One refresh per provider per window, and the * evidence inside the window is not lost: the refresh it caused is the response to all of it. * * 60 s, not the 10-minute TTL: the point of the signal is to act sooner than the clock would, and a * roster change is discovered by the very refresh this bounds. It is a floor on FREQUENCY, never a * claim about how stale the roster is. */ export declare const MODEL_MISSING_REFRESH_COOLDOWN_MS = 60000; /** * Where the catalog caches `/models` when no explicit `cachePath` is given. * * ⚠ Under vitest, never touch the developer's real catalog cache — a test run would otherwise * overwrite the live roster the router ranks on, the same way the suite was once found writing * `openai_mock` entries into the real probe cache. Tests needing persistence pass an explicit * `cachePath`. Exported so `test/persistent-paths-vitest.test.ts` can assert the redirect * DIRECTLY rather than inferring it from behaviour. */ export declare function defaultCatalogCachePath(): string; /** * Limits a provider publishes about its OWN deployment of a model. * * Deliberately per-(provider, model): the same model id served by two providers is two different * deployments with different ceilings, so one provider's numbers must never be presented as * another's. Null means "this provider does not publish it" — NIM's /models returns only * id/object/created/owned_by, while Groq and Mistral publish real limits. */ /** * Rate limits a provider publishes IN ITS `/models` RECORD about its own deployment. * * This is spec §4 rung 2 (PUBLISHED) and is expected to stay nearly empty — free providers publish * even less here than they publish context windows. It is deliberately separate from * `QuotaObservation` (point-in-time header state) and from configured limits (operator-asserted): * a published ceiling is durable knowledge about the deployment, and its basis is * provider-stated by construction. An omitted axis is unpublished, i.e. null, never guessed from * a bare "limit"-shaped field — a number without a stated period bounds nothing. */ export interface ModelRateLimits { /** Requests per minute. */ rpm: number | null; /** Requests per day. */ rpd: number | null; /** Tokens per minute. */ tpm: number | null; /** Tokens per day. */ tpd: number | null; } export interface ModelLimits { contextLength: number | null; maxOutputTokens: number | null; /** Per-TOKEN price, as published. Per-provider for the same reason limits are. */ pricePromptPerToken: number | null; priceCompletionPerToken: number | null; /** null when the record published no rate-limit figure at all. Absent on older cache files. */ rateLimits: ModelRateLimits | null; } /** Read limits + pricing out of one `/models` record, including a nested `top_provider` (OpenRouter). */ export declare function limitsFromRecord(rec: Record): ModelLimits; /** * Live per-provider model catalog — model ids are DISCOVERED from each provider's * OpenAI-compatible `/models` endpoint, never hand-maintained. In-memory TTL cache * backed by a small on-disk cache so restarts start warm. Fail-open everywhere: a * fetch failure serves the last-known list (or an empty one), never blocks routing. */ export declare class ModelCatalog { private mem; private readonly ttlMs; private readonly cachePath; private loaded; /** Providers with a background refresh in flight — dedups stampeding probes. */ private refreshing; /** In-flight blocking fetches (cold start / forced) — dedups concurrent requests. */ private pending; /** provider → the last time EVIDENCE forced a refresh; see `noteProviderStale`. */ private readonly forcedAt; private revision; private readonly writeBehind; private readonly flushTimer; constructor(opts?: { ttlMs?: number; cachePath?: string | null; writeBehind?: boolean; }); private loadDisk; private saveDisk; private persistSoon; /** Monotonic in-memory catalog revision used to invalidate routing snapshots cheaply. */ getRevision(): number; /** Force any write-behind catalog update to disk (graceful shutdown / explicit durability). */ flushPersistence(): void; /** Cached model ids for a provider; empty array if fetch fails and no prior cache. */ private cached; /** Already-cached model ids for synchronous routing decisions. Never performs network I/O. */ cachedModels(name: string): string[]; /** Whether a provider has a successfully loaded catalog, including a legitimately empty one. */ hasCachedCatalog(name: string): boolean; /** * Live model ids for a provider, cached with TTL. `force` bypasses the TTL. * * STALE-WHILE-REVALIDATE: a fresh cache (within TTL) is served directly; a STALE * cache is ALSO served immediately while a background refresh updates it — so a * discovery/liveness probe (`GET /registry`) NEVER blocks on an upstream refetch. * Blocking `await` happens only on a genuine cold start (no prior at all, in memory * or on disk). Serves a stale cache if a blocking refresh fails; returns [] only * when there is no cache AND the fetch fails. `force` still awaits (explicit refresh). */ list(name: string, cfg: ProviderConfig, opts?: { force?: boolean; now?: number; fetchFn?: typeof fetch; }): Promise; /** * Fire-and-forget catalog refresh for a stale provider, deduped so repeated probes * during one refresh window spawn at most one upstream fetch. A failed refresh * leaves the stale entry in place (fail-open); it is retried on the next `list`. */ private refreshInBackground; /** * Evidence that a provider's roster moved: refresh it now, rather than waiting for the TTL. * * The backlog property this exists for — "if we get a hint that our model catalog might be stale, * we update it" — is deliberately narrower than "something went wrong". Exactly one signal * qualifies: a refusal that STATES, on a model this catalog currently lists, that the model does * not exist. That is the provider contradicting our own roster, which is the one thing a TTL * cannot know about. Rate limits, auth walls, quota exhaustion and generic 5xx all say something * about the ACCOUNT or the moment, never about which models exist, and none of them re-fetch a * roster the provider has not contradicted. * * ⚠ **This TRIGGERS a re-fetch; it never edits the catalog.** The refreshed list is whatever the * provider's own endpoint answers next — nothing here removes the 404'd model by hand, because * "the model is gone" is a guess about the roster while the endpoint is the measurement. A model * that 404s and is then re-listed by the refresh stays exactly where it was. * * ⚠ **Fire-and-forget and FAIL-OPEN, like every other fetch here.** It never throws, never * blocks a request, and a failed re-fetch leaves the previous list in place — the caller is a * failing request path, and a catalog refresh must not be able to turn one failure into two. * * The cooldown is per provider and holds off CONSECUTIVE signals, so a burst — a pool walking * into the same dead deployment several times in a minute — costs one upstream fetch. The * in-flight dedup in `refreshInBackground` covers the overlapping case the cooldown cannot. * * Returns whether this call actually started a refresh, which is what the tests read. */ noteProviderStale(name: string, cfg: ProviderConfig, opts?: { now?: number; fetchFn?: typeof fetch; cooldownMs?: number; }): boolean; /** * Whether a provider serves a model. Returns null when the catalog is * unavailable (no cache and fetch failed) — callers treat null as "unknown, * proceed" so routing never hard-fails on a catalog miss. */ has(name: string, cfg: ProviderConfig, model: string, opts?: { force?: boolean; now?: number; fetchFn?: typeof fetch; }): Promise; /** * Already-cached limits for a model — synchronous, never fetches. * * For the request hot path, where a blocking upstream fetch to learn a context window would be a * worse outcome than simply not enforcing a guardrail on the first request. Returns null until * the catalog has been warmed (startup does that), which callers must treat as "unknown". */ cachedLimits(name: string, model: string): ModelLimits | null; /** * Already-cached PUBLISHED rate limits for a model — synchronous, never fetches. * * Spec §4 rung 2's read side. Deliberately NOT folded into `resolveMetadata()`: rate limits are * per-(provider, model) facts with no meaningful `reference` rung — another provider's request * allowance says nothing about this deployment's — so there is nothing for the per-field * provenance ladder to resolve and callers read this directly, basis provider-stated. */ publishedRateLimits(name: string, model: string): ModelRateLimits | null; /** * Limits this provider publishes for one of its own models, or null when it publishes none. * * Null is meaningful and must not be papered over with another provider's number — see * `resolveMetadata()` in metadata.ts, which decides what to fall back to and labels it. */ limits(name: string, cfg: ProviderConfig, model: string, opts?: { force?: boolean; now?: number; fetchFn?: typeof fetch; }): Promise; /** Returned together so concurrent fetches for different providers can never cross-assign * one provider's limits to another's cache entry (which shared mutable state used to allow). */ private fetch; private fetchForSlot; }