/** * The plan-allowance guard: a second economic control beside `dailySpendUsd`. * * `dailySpendUsd` meters money, which is the only thing an API-billed account * can run out of. A fixed-price subscription cannot be expressed that way at * all — the fleet this package was written for burns a *weekly token * allowance* whose marginal dollar cost is zero and whose exhaustion stops * every session on the host. Pricing that allowance to fit the dollar meter * would mean inventing a number, and a ceiling built on an invented number is * worse than no ceiling (#110). * * So the guard reads the provider's own figures — `omp usage --json`, which is * the structured form of the harness `/usage` view — and normalises them into * windows the admission gate compares. Three properties of the real payload * shape every decision in this file, and each of them is a way an obvious * implementation gets it wrong: * * - **`limits` is a list, not a scalar.** Anthropic alone reports * `anthropic:5h`, `anthropic:7d` and the tier-scoped `anthropic:7d:fable` in * one reply, and other providers add their own. A gate that read "the first * limit" would bind a config that meant the weekly window to whichever entry * the provider happened to serialise first, and would rebind itself silently * when that order changed. The cap therefore names its window. * - **`unit` is not comparable across providers.** Anthropic reports * `unit: "percent"` with `used: 5`; `xai-oauth` reports `unit: "unknown"` * with `used: 4709` against a limit of 20000. A threshold compared against * `used` reads the second account as 4709% spent. Nothing here ever compares * a raw count: the only number the gate looks at is `usedFraction`. * - **A source can legitimately report nothing.** A provider with no readable * allowance simply contributes no limits. That is the honest origin of * "unmetered/unavailable", and it is why no code path here manufactures a * zero to fill the gap: a fabricated `0% used` renders as a guard that is * working when it is in fact blind. * * Availability policy — which way each failure falls — is decided in * {@link planUsageStatus} and documented beside it. */ import type { PlanUsageCap } from "./types.ts"; /** Bounded so a wedged provider call cannot hold a dispatch tick open. A warm * `omp usage --json` answers in well under a second; this leaves room for a * cold fetch across several provider endpoints without becoming a stall. */ export const USAGE_PROBE_TIMEOUT_MS = 15_000; /** How long one reading serves every caller. One admission pass asks the gate * once, but `status` and `board` re-render far faster than any allowance * moves, so the provider is asked at most once a minute per process. */ export const USAGE_TTL_MS = 60_000; /** A failed read is retried sooner than a good one is refreshed. The guard is * open while the source is unreadable (see below), so sitting on a stale * failure keeps the fleet unguarded for longer than the failure lasted. */ export const USAGE_ERROR_TTL_MS = 15_000; /** * How long an unreadable source is tolerated before the guard stops claiming. * * This is the whole of the "which way does it fail" question. A short outage — * a token refresh, a provider 502, `omp` briefly absent mid-upgrade — must not * stall a fleet: the other ceilings (spend, turns, wall clock, concurrency) * are all still enforced, so admitting through a blip costs at most one extra * worker. A source that has been unreadable for half an hour is not an outage, * it is a broken meter, and running a plan-capped fleet on a broken meter is * exactly how the allowance gets spent to zero with nobody watching. */ export const USAGE_BLIND_GRACE_MS = 30 * 60_000; /** Past this age a rendered reading says how old it is. Anything an operator * reads as current has to actually be current — a stale figure presented as * live is the dishonesty this guard was specified to avoid. */ export const USAGE_STALE_MS = 10 * 60_000; /** Bound on how many allowance ids an error message lists back at an operator. */ const NAMED_WINDOW_LIMIT = 12; /** * One allowance window, normalised away from any single provider's shape. * * `id` is the primary key and `windowId` deliberately is not: `anthropic:7d` * and `anthropic:7d:fable` share the window key `7d` and are different * allowances with different numbers. */ export interface UsageWindow { /** Fully-qualified allowance id, unique within a reading (`anthropic:7d`). */ id: string; provider: string; /** Provider-scoped window key (`7d`, `1mo`). Not unique — see above. */ windowId?: string; /** The provider's own name for the window, for display only. */ label?: string; used: number; limit?: number; remaining?: number; /** * Consumed share of the allowance. The only figure the gate compares, and * absent when the provider gave nothing a fraction can be derived from — * which is a state the guard reports rather than papers over. */ usedFraction?: number; remainingFraction?: number; /** The provider's word for the counts (`percent`, `unknown`). Display only: * treating it as a scale is the bug this field exists to warn about. */ unit: string; /** Derived from the provider's reset instant and window length when both are * reported; the payload carries no explicit start. */ windowStart?: number; /** The provider's own reset instant. Never a locally assumed calendar week: * a weekly plan resets on the account's clock, not on Monday. */ resetsAt?: number; /** Per-limit health the provider attached. Carried and displayed, never * turned into arithmetic — the vocabulary is the provider's, not ours. */ status?: string; /** When the provider fetched this figure (its `fetchedAt`, not our clock). */ observedAt: number; } /** * One reading of the whole source. * * `ok` carries at least one window by construction: a payload that parsed but * yielded no allowances is `unavailable`, because "nothing is metered" and * "the meter is fine and reads zero" are different facts and only one of them * may be displayed as a percentage. */ export type UsageReading = | { kind: "ok"; observedAt: number; windows: UsageWindow[] } | { kind: "unavailable"; observedAt: number; /** Operator-facing cause, already bounded for a status line. */ reason: string; /** * Start of the current unbroken run of unreadable results. A single * failure and a fortnight of failures are the same object otherwise, and * the grace period in {@link USAGE_BLIND_GRACE_MS} needs to tell them * apart. */ since: number; }; /** * A source of readings. The one contract that matters: `read` never throws. An * unreadable provider is a *reading* the caller applies policy to, not an * exception that aborts a dispatch tick. */ export interface UsageProvider { read(now: number): Promise; } /** A provider with a cache in front of it. */ export interface UsageSource { read(now?: number): Promise; /** Drop the cached reading so the next `read` asks the provider. The * operator-facing half of the same idea is `omp usage invalidate`. */ invalidate(): void; } // ------------------------------------------------------------------ normalising function asObject(v: unknown): Record | undefined { return typeof v === "object" && v !== null && !Array.isArray(v) ? (v as Record) : undefined; } function asNumber(v: unknown): number | undefined { return typeof v === "number" && Number.isFinite(v) ? v : undefined; } function asString(v: unknown): string | undefined { return typeof v === "string" && v.length > 0 ? v : undefined; } /** * The single place a threshold-comparable number is produced. * * Preference order is deliberate. `usedFraction` is what the provider itself * computed. `remainingFraction` is the same statement inverted. `used/limit` * is our own arithmetic and only valid when the limit is a positive count — a * limit of `0` divides to `Infinity`/`NaN`, which would render as a plausible * percentage and hold a fleet on nonsense. */ function usedFractionOf(amount: Record): number | undefined { const direct = asNumber(amount["usedFraction"]); if (direct !== undefined) return direct; const remaining = asNumber(amount["remainingFraction"]); if (remaining !== undefined) return 1 - remaining; const used = asNumber(amount["used"]); const limit = asNumber(amount["limit"]); if (used !== undefined && limit !== undefined && limit > 0) return used / limit; return undefined; } /** * `omp usage --json` → normalised windows. * * Pure and exported so the traps in the real payload can be pinned without a * provider on the machine running the tests. Unrecognisable input is never * partially believed: it becomes `unavailable` with a reason an operator can * act on. */ export function normalizeUsagePayload(payload: unknown, observedAt: number): UsageReading { const root = asObject(payload); const reports = root?.["reports"]; if (root === undefined || !Array.isArray(reports)) { return { kind: "unavailable", observedAt, reason: "usage payload has no reports array", since: observedAt, }; } const windows: UsageWindow[] = []; reports.forEach((entry: unknown, reportIndex) => { const report = asObject(entry); if (report === undefined) return; const provider = asString(report["provider"]) ?? "unknown"; const fetchedAt = asNumber(report["fetchedAt"]) ?? observedAt; const limits = report["limits"]; // A provider that contributes no limits is not an error. It is the // ordinary shape of "this account exposes no readable allowance", and it // must reach the caller as absence rather than as a zero. if (!Array.isArray(limits)) return; limits.forEach((limitEntry: unknown, limitIndex) => { const limit = asObject(limitEntry); const amount = asObject(limit?.["amount"]); if (limit === undefined || amount === undefined) return; const scope = asObject(limit["scope"]); const window = asObject(limit["window"]); const windowId = asString(scope?.["windowId"]) ?? asString(window?.["id"]); const resetsAt = asNumber(window?.["resetsAt"]); const durationMs = asNumber(window?.["durationMs"]); const normalized: UsageWindow = { // Synthesised only as a last resort, and deterministically: a window // nothing can name is a window a config can never select. id: asString(limit["id"]) ?? `${provider}:${windowId ?? String(reportIndex)}.${String(limitIndex)}`, provider, unit: asString(amount["unit"]) ?? "unknown", used: asNumber(amount["used"]) ?? 0, observedAt: fetchedAt, }; if (windowId !== undefined) normalized.windowId = windowId; const label = asString(limit["label"]) ?? asString(window?.["label"]); if (label !== undefined) normalized.label = label; const limitAmount = asNumber(amount["limit"]); if (limitAmount !== undefined) normalized.limit = limitAmount; const remaining = asNumber(amount["remaining"]); if (remaining !== undefined) normalized.remaining = remaining; const usedFraction = usedFractionOf(amount); if (usedFraction !== undefined) normalized.usedFraction = usedFraction; const remainingFraction = asNumber(amount["remainingFraction"]); if (remainingFraction !== undefined) normalized.remainingFraction = remainingFraction; if (resetsAt !== undefined) normalized.resetsAt = resetsAt; if (resetsAt !== undefined && durationMs !== undefined) { normalized.windowStart = resetsAt - durationMs; } const status = asString(limit["status"]); if (status !== undefined) normalized.status = status; windows.push(normalized); }); }); if (windows.length === 0) { return { kind: "unavailable", observedAt, reason: "no provider reported a readable allowance", since: observedAt, }; } return { kind: "ok", observedAt, windows }; } // -------------------------------------------------------------------- adapters export type UsageCommandResult = | { kind: "completed"; exitCode: number; stdout: string; stderr: string } | { kind: "timeout" } | { kind: "unavailable"; reason: string }; async function runOmpUsage(timeoutMs: number): Promise { try { const proc = Bun.spawn(["omp", "usage", "--json"], { // Closed stdin: a command that never reads it sees EOF instead of an // open pipe nobody ends. stdin: "ignore", stdout: "pipe", stderr: "pipe", }); let timedOut = false; const timer = setTimeout(() => { timedOut = true; proc.kill("SIGKILL"); }, timeoutMs); try { const [exitCode, stdout, stderr] = await Promise.all([ proc.exited, new Response(proc.stdout).text(), new Response(proc.stderr).text(), ]); if (timedOut) return { kind: "timeout" }; return { kind: "completed", exitCode, stdout, stderr }; } finally { clearTimeout(timer); } } catch (err) { // `omp` missing from PATH lands here, and it is a perfectly ordinary state // for a conductor CLI invoked from a bare shell. return { kind: "unavailable", reason: err instanceof Error ? err.message : String(err) }; } } /** Bounded so a provider that dumps a stack trace cannot own the status line. */ function firstLine(text: string, max = 120): string { const line = text.replace(/\s+/g, " ").trim(); return line.length > max ? `${line.slice(0, max - 1)}…` : line; } /** * The first adapter: `omp usage --json`. * * The command is injectable because every interesting case — a non-zero exit, * a truncated stdout, a provider that reports three windows for one account — * has to be pinned without the machine running the tests owning a * subscription. */ export function ompUsageProvider( run: (timeoutMs: number) => Promise = runOmpUsage, timeoutMs = USAGE_PROBE_TIMEOUT_MS, ): UsageProvider { return { async read(now: number): Promise { const result = await run(timeoutMs); if (result.kind === "timeout") { return { kind: "unavailable", observedAt: now, reason: `omp usage --json did not answer within ${String(Math.round(timeoutMs / 1000))}s`, since: now, }; } if (result.kind === "unavailable") { return { kind: "unavailable", observedAt: now, reason: `omp usage --json could not run: ${firstLine(result.reason)}`, since: now, }; } if (result.exitCode !== 0) { const detail = firstLine(result.stderr); return { kind: "unavailable", observedAt: now, reason: `omp usage --json exited ${String(result.exitCode)}${detail === "" ? "" : `: ${detail}`}`, since: now, }; } let payload: unknown; try { payload = JSON.parse(result.stdout); } catch (err) { return { kind: "unavailable", observedAt: now, reason: `omp usage --json returned unparseable output: ${firstLine(err instanceof Error ? err.message : String(err))}`, since: now, }; } return normalizeUsagePayload(payload, now); }, }; } // ----------------------------------------------------------------------- cache /** * TTL cache with explicit invalidation, plus one rule the TTL alone cannot * express: a reading whose window has already reset is stale regardless of its * age. That is what makes "admission resumes once the allowance rolls over" * true at the rollover rather than up to a TTL after it. * * Concurrent callers share one in-flight request. An admission pass and a * board repaint landing in the same second must cost one provider call, not * two — "do not make every candidate trigger a provider request" is an * acceptance criterion of #110, not an optimisation. */ export function cacheUsage( provider: UsageProvider, opts: { ttlMs?: number; errorTtlMs?: number } = {}, ): UsageSource { const ttlMs = opts.ttlMs ?? USAGE_TTL_MS; const errorTtlMs = opts.errorTtlMs ?? USAGE_ERROR_TTL_MS; let cached: UsageReading | undefined; let inFlight: Promise | undefined; let unreadableSince: number | undefined; const usable = (reading: UsageReading, now: number): boolean => { const ttl = reading.kind === "ok" ? ttlMs : errorTtlMs; if (now - reading.observedAt >= ttl) return false; if (reading.kind === "ok") { return !reading.windows.some((w) => w.resetsAt !== undefined && w.resetsAt <= now); } return true; }; return { async read(now = Date.now()): Promise { if (cached !== undefined && usable(cached, now)) return cached; inFlight ??= provider .read(now) .catch( (err: unknown): UsageReading => ({ kind: "unavailable", observedAt: now, // The provider contract says it never throws. If one ever does, // the tick still has to survive it as data. reason: `usage provider threw: ${firstLine(err instanceof Error ? err.message : String(err))}`, since: now, }), ) .then((reading) => { if (reading.kind === "unavailable") { unreadableSince ??= reading.since; cached = { ...reading, since: unreadableSince }; } else { unreadableSince = undefined; cached = reading; } return cached; }) .finally(() => { inFlight = undefined; }); return inFlight; }, invalidate(): void { cached = undefined; // The unreadable streak deliberately survives. An operator asking for a // re-read has not repaired anything, and forgetting how long the meter // has been broken is how a permanently blind guard stays open forever. }, }; } let shared: UsageSource | undefined; /** One cache per process. `status`, `board` and the dispatch tick all want the * same reading, and three caches would mean three provider calls. */ export function sharedUsageSource(): UsageSource { shared ??= cacheUsage(ompUsageProvider()); return shared; } // -------------------------------------------------------------------- decision /** * Every distinguishable answer the guard can give. They exist separately * because each one sends an operator somewhere different: `at-cap` is normal * flow control, `window-missing` is a config edit, `blind` is a broken host. * Collapsing them into a boolean is how "the fleet is not claiming" becomes * unexplainable. */ export type PlanUsageState = | "unmetered" | "ok" | "at-cap" | "window-missing" | "window-ambiguous" | "window-uncomparable" | "unreadable" | "blind"; export interface PlanUsageStatus { state: PlanUsageState; /** * Whether new claims are held. Stated once, here, so no renderer and no gate * re-derives the availability policy and disagrees with the other. */ blocking: boolean; cap?: PlanUsageCap; window?: UsageWindow; /** Present only when a real figure was read. Never a filled-in zero. */ usedFraction?: number; resetsAt?: number; /** When the underlying reading was taken. */ observedAt?: number; /** The operator-facing sentence `status` and `board` render verbatim. */ detail: string; } /** Where a configured window id landed in a reading. */ export type WindowSelection = | { kind: "found"; window: UsageWindow } | { kind: "missing"; available: string[] } | { kind: "ambiguous"; candidates: string[] }; /** * Resolve a configured window id against a reading. * * Exact `id` first, because that is the only unique key the payload has. A * bare window key (`7d`) is accepted as a convenience, but *only* when it * resolves to exactly one allowance: on an Anthropic account `7d` matches both * `anthropic:7d` and the tier-scoped `anthropic:7d:fable`, and picking either * is the silent binding this guard was specified to refuse. */ export function selectUsageWindow( windows: readonly UsageWindow[], windowId: string, ): WindowSelection { const exact = windows.filter((w) => w.id === windowId); if (exact.length === 1) return { kind: "found", window: exact[0]! }; if (exact.length > 1) return { kind: "ambiguous", candidates: exact.map((w) => w.id) }; const byWindow = windows.filter((w) => w.windowId === windowId); if (byWindow.length === 1) return { kind: "found", window: byWindow[0]! }; if (byWindow.length > 1) return { kind: "ambiguous", candidates: byWindow.map((w) => w.id) }; return { kind: "missing", available: windows.slice(0, NAMED_WINDOW_LIMIT).map((w) => w.id) }; } function percent(fraction: number): string { const whole = Math.round(fraction * 100); // A live reading of 0.4% must not render as `0%`: that is character-for- // character the fabricated zero this guard refuses to display (#110). if (whole === 0 && fraction > 0) return "<1%"; return `${String(whole)}%`; } function duration(ms: number): string { const minutes = Math.max(0, Math.floor(ms / 60_000)); if (minutes < 1) return "<1m"; if (minutes < 60) return `${String(minutes)}m`; const hours = Math.floor(minutes / 60); if (hours < 24) return `${String(hours)}h ${String(minutes % 60)}m`; return `${String(Math.floor(hours / 24))}d ${String(hours % 24)}h`; } function resetsText(resetsAt: number | undefined, now: number): string { if (resetsAt === undefined) return "no reset time reported"; return resetsAt <= now ? "resetting now" : `resets in ${duration(resetsAt - now)}`; } /** Appended only when the figure is old enough that calling it current would * be a lie. Silence below the threshold keeps the common line short. */ function ageSuffix(observedAt: number, now: number): string { const age = now - observedAt; return age >= USAGE_STALE_MS ? ` · stale, read ${duration(age)} ago` : ""; } function unmeteredStatus(): PlanUsageStatus { return { state: "unmetered", blocking: false, detail: "unmetered — no plan allowance cap configured", }; } /** * The whole availability policy, in one pure function. * * | state | holds new claims | why | * | --- | --- | --- | * | `unmetered` | no | the operator did not ask for this guard | * | `ok` | no | below the threshold | * | `at-cap` | **yes** | the guard doing its job | * | `window-missing` | **yes** | fail closed on config, like the rest of this package | * | `window-ambiguous` | **yes** | as above; never guess which allowance was meant | * | `window-uncomparable` | **yes** | the window exists and cannot be compared | * | `unreadable` | no | a transient read error must not stall a fleet | * | `blind` | **yes** | unreadable for longer than {@link USAGE_BLIND_GRACE_MS} | * * The split that matters is between the last two rows and the config rows. A * *read error* is transient by nature, so it is tolerated for a bounded time * and then stops being tolerated. A *successful read that does not contain the * configured window* is not a read error: the source answered, and it says the * operator named something that is not there. That is a config fault, and this * package fails closed on config faults everywhere else (see * `normalizeAuthority` in `config.ts`). It recovers by itself the moment a * reading contains the window again, so one bad tick costs one tick. */ export function planUsageStatus( cap: PlanUsageCap | null, reading: UsageReading, now: number, graceMs = USAGE_BLIND_GRACE_MS, ): PlanUsageStatus { if (cap === null) return unmeteredStatus(); if (reading.kind === "unavailable") { const blindFor = now - reading.since; if (blindFor >= graceMs) { return { state: "blind", blocking: true, cap, observedAt: reading.observedAt, detail: `unavailable for ${duration(blindFor)} — ${reading.reason}; holding new claims`, }; } return { state: "unreadable", blocking: false, cap, observedAt: reading.observedAt, detail: `unavailable — ${reading.reason}; retrying, guard open for up to ${duration(graceMs - blindFor)}`, }; } const selection = selectUsageWindow(reading.windows, cap.windowId); if (selection.kind === "missing") { return { state: "window-missing", blocking: true, cap, observedAt: reading.observedAt, detail: `window "${cap.windowId}" is not in this reading — holding new claims. ` + `Reported: ${selection.available.join(", ")}`, }; } if (selection.kind === "ambiguous") { return { state: "window-ambiguous", blocking: true, cap, observedAt: reading.observedAt, detail: `window "${cap.windowId}" matches ${selection.candidates.join(", ")} — ` + "holding new claims until the cap names exactly one", }; } const window = selection.window; const threshold = percent(cap.maxUsedFraction); if (window.usedFraction === undefined) { return { state: "window-uncomparable", blocking: true, cap, window, observedAt: window.observedAt, detail: `window "${window.id}" reports no comparable fraction ` + `(unit ${window.unit}, used ${String(window.used)}, limit ${window.limit === undefined ? "unreported" : String(window.limit)}) — ` + "holding new claims", }; } const used = percent(window.usedFraction); const statusNote = window.status === undefined || window.status === "ok" ? "" : ` · ${window.status}`; const base: PlanUsageStatus = { state: window.usedFraction >= cap.maxUsedFraction ? "at-cap" : "ok", blocking: window.usedFraction >= cap.maxUsedFraction, cap, window, usedFraction: window.usedFraction, observedAt: window.observedAt, detail: "", }; if (window.resetsAt !== undefined) base.resetsAt = window.resetsAt; base.detail = base.state === "at-cap" ? `${used} / ${threshold} of ${window.id} used — holding new claims · ${resetsText(window.resetsAt, now)}${statusNote}${ageSuffix(window.observedAt, now)}` : `${used} / ${threshold} of ${window.id} used · ${resetsText(window.resetsAt, now)}${statusNote}${ageSuffix(window.observedAt, now)}`; return base; } /** * Read and decide in one call. An unmetered project never spawns the provider: * a fleet that does not use this guard must not pay a subprocess for it every * time somebody runs `status`. */ export async function readPlanUsage( cap: PlanUsageCap | null, source: UsageSource, now = Date.now(), ): Promise { if (cap === null) return unmeteredStatus(); return planUsageStatus(cap, await source.read(now), now); } /** * The compact form for the board's admission line, where the whole fleet has * to fit on one row. Derived from the same status object as the long form so * the two cannot disagree about whether the guard is active. */ /** * The allowance window nearest its ceiling, named for a reader (#984). * * A subscription has an allowance where an API key has a price, so this is what * "the constraint that binds" means on a subscription-billed fleet — and the * one nearest exhaustion is the one that will stop the fleet first, whatever * the others say. Read from the provider's own reply, never derived: a fleet * whose provider reports no window gets `undefined`, which is the honest answer * and keeps a genuine telemetry fault a fault. * * Only windows that reported a comparable fraction are eligible, for the reason * this module already documents at length: `used` is not comparable across * providers, so a raw count can never stand in for one. */ export function bindingAllowanceWindow(reading: UsageReading): string | undefined { if (reading.kind !== "ok") return undefined; const comparable = reading.windows.filter((w) => w.usedFraction !== undefined); if (comparable.length === 0) return undefined; let worst = comparable[0]!; for (const window of comparable.slice(1)) { if ((window.usedFraction ?? 0) > (worst.usedFraction ?? 0)) worst = window; } return worst.label === undefined ? worst.id : `${worst.id} (${worst.label})`; } export function planUsageBadge(status: PlanUsageStatus | undefined): string { if (status === undefined) return "plan not read"; let core: string; switch (status.state) { case "unmetered": core = "plan unmetered"; break; case "ok": case "at-cap": // No `?? 0` fallback anywhere on this path. Both fields are always set // in these two states, and a default would put the one number this // guard must never invent — a zero — on an operator's screen (#110). core = status.usedFraction === undefined || status.cap === undefined ? "plan read incomplete" : `plan ${percent(status.usedFraction)}/${percent(status.cap.maxUsedFraction)}`; break; case "unreadable": case "blind": core = "plan unavailable"; break; default: // Every remaining state is "the configured window did not resolve", and // naming it is the point: an operator who sees `plan unmetered` when // they configured a cap will not go looking for a typo. core = `plan window "${status.cap?.windowId ?? "?"}" ${status.state.replace("window-", "")}`; break; } return status.blocking ? `${core} · HELD` : core; } /** The long form for `status`. Absent means nobody read the source on this * path, which is its own honest answer and not "0% used". */ export function planUsageLine(status: PlanUsageStatus | undefined): string { return status?.detail ?? "not read"; }