/** * model-roundrobin.ts — virtual roundrobin provider with multi-candidate failover. * * Each preset under ~/.pi/agent/roundrobin/ (config.json = "default", presets/* * = named groups) registers as one virtual model; model id = preset file stem * (the .json suffix is stripped). `/model roundrobin/` routes to that * group's candidate pool. * * Strategies (config.strategy): * sticky — pin to the last successful candidate; only move on failure. * primary — always try the preferred (index 0) candidate first; fall to * others only while it is cooling down, then snap back to 0. * round-robin — rotate the start index by one after each successful request. * * Mid-stream errors after real content are terminal (no replay) to avoid * duplicating already-emitted content or tool calls. A start-only stream that * errors before any content is treated as a candidate failure and fails over * to the next — nothing was emitted, so there's nothing to replay. A stream * that starts then goes silent trips the idle timeout (timeoutMs) on every * chunk — the fix this fork exists for. * * A relay that surfaces `stopReason:"aborted"` (socket drop / upstream cancel) * without a user abort is treated as a REAL failure, not a cancel: the candidate * is cooled down and the pool rotates. Only `options.signal.aborted` short- * circuits the turn. * * Cooldown tracking is process-local in-memory; no cross-CLI health file. */ import { appendFileSync, mkdirSync, statSync, truncateSync } from "node:fs"; import { dirname, join } from "node:path"; // `streamSimple` lives on pi-ai's compat surface — the top-level package entry // does not re-export it (since pi-ai 0.83), so importing it from the top-level // silently yields `undefined` and the virtual provider never intercepts the // stream. Pull it from `/compat` explicitly. import { createAssistantMessageEventStream, type Api, type AssistantMessage, type AssistantMessageEventStream, type Context, type Model, type SimpleStreamOptions, } from "@earendil-works/pi-ai"; import { streamSimple } from "@earendil-works/pi-ai/compat"; import { getAgentDir } from "@earendil-works/pi-coding-agent"; import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { loadAllGroupsConfig, type RrCandidate, type RrConfig } from "./rr-config.js"; // Types type ResolvedCandidate = Model & { label: string }; type Strategy = "sticky" | "round-robin" | "primary"; /** One preset/config.json → one virtual model + its candidate pool. */ type Group = { name: string; enabled: boolean; virtualModel: Model; candidates: ResolvedCandidate[]; timeoutMs: number; cooldownMs: number; strategy: Strategy; log: boolean; currentIndex: number; // sticky/round-robin cursor; mutated only on success /** Per-candidate last-failure ts (epoch ms). Process-local cooldown. */ health: Map; }; // Module state const LOG_PATH = join(getAgentDir(), "roundrobin", "roundrobin.log"); const LOG_MAX_BYTES = 2 * 1024 * 1024; const PROVIDER = "roundrobin"; const API = "model-roundrobin-api" as Api; /** Bound the worst-case all-fail loop so a permanent outage can't hang a turn * forever (each round still waits for the earliest cooldown before retrying). */ const MAX_ROUNDS = 3; /** Hard cap on a single turn's total wall time (attempts + inter-round waits) * so a full outage can't hang the agent indefinitely. */ const TURN_DEADLINE_MS = 180000; const ZERO_COST = Object.freeze({ input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }); const EMPTY_USAGE = Object.freeze({ input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: Object.freeze({ input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }), }); const groups = new Map(); let stateModelRegistry: ExtensionContext["modelRegistry"] | undefined; // Logging — best-effort appendFileSync audit trail, rotate-truncated at 2MB. mkdirSync(dirname(LOG_PATH), { recursive: true }); let logBytes = (() => { try { return statSync(LOG_PATH).size; } catch { return 0; } })(); function logLine(group: Group | null, message: string): void { if (group && !group.log) return; const line = `${new Date().toISOString()} ${group ? `[${group.name}] ` : ""}${message}\n`; try { if (logBytes > LOG_MAX_BYTES) { truncateSync(LOG_PATH, 0); logBytes = 0; } appendFileSync(LOG_PATH, line); logBytes += line.length; } catch { /* best-effort */ } } // Config → Group function resolveVirtualModel( name: string, raw: RrConfig["virtualModel"] | undefined, candidates: ResolvedCandidate[], ): Model { const v = raw ?? {}; // Explicit virtualModel field > built-in default. `compat` is the one field // presets commonly omit, so it inherits from the first candidate when absent; // every other field is always set in real configs. const src = candidates[0]; const vm: Model = { id: name, name: v.name || name, api: API, provider: PROVIDER, baseUrl: "model-roundrobin://local", reasoning: v.reasoning ?? true, input: (v.input ?? ["text", "image"]) as Model["input"], cost: { ...ZERO_COST }, contextWindow: v.contextWindow ?? 200000, maxTokens: v.maxTokens ?? 16384, }; if (v.thinkingLevelMap) vm.thinkingLevelMap = v.thinkingLevelMap; const compat = v.compat ?? src?.compat; if (compat) vm.compat = compat as Model["compat"]; return vm; } function resolveCandidates( candidates: RrCandidate[], registry: ExtensionContext["modelRegistry"] | undefined, ): ResolvedCandidate[] { const resolved: ResolvedCandidate[] = []; for (const c of candidates) { const m = registry?.find(c.provider, c.model); if (registry && !m) { logLine(null, `skip unknown model ${c.provider}/${c.model}`); continue; } // Pre session_start: no registry yet. Build a placeholder so the virtual // model still appears in /model and in static model lists (pi-web). // session_start rebuilds with the real registry and overwrites. const model = m ?? ({ id: c.model, name: c.model, provider: c.provider, api: "openai-responses" as Api, cost: { ...ZERO_COST }, } as Model); resolved.push({ ...model, label: `${c.provider}/${c.model}` }); } return resolved; } function buildGroups(allConfigs: { name: string; config: RrConfig }[]): void { // Mutate existing Group objects in place so an in-flight turn keeps the same // reference (its currentIndex writes + health Map persist across a hot // reload). Removed presets are deleted; new presets are inserted. const seen = new Set(); for (const { name, config } of allConfigs) { seen.add(name); const candidates = resolveCandidates(config.candidates ?? [], stateModelRegistry); const virtualModel = resolveVirtualModel(name, config.virtualModel, candidates); // Reuse the existing Group object when present (in-flight turns keep the // same reference + currentIndex + health Map); otherwise insert a fresh one. let target = groups.get(name); if (!target) { target = { name, enabled: false, virtualModel, candidates, timeoutMs: 0, cooldownMs: 0, strategy: "sticky", log: false, currentIndex: 0, health: new Map(), }; } target.candidates = candidates; target.virtualModel = virtualModel; target.timeoutMs = config.timeoutMs!; target.cooldownMs = config.cooldownMs!; target.strategy = config.strategy!; target.log = config.log!; target.enabled = candidates.length > 0; // Prune stale cooldown keys for candidates no longer in the pool. In-place // on the preserved Map — never replace the ref, or in-flight writes vanish. // No-op for fresh groups whose health Map is empty. const valid = new Set(candidates.map((c) => c.label)); for (const label of target.health.keys()) if (!valid.has(label)) target.health.delete(label); groups.set(name, target); } for (const name of [...groups.keys()]) if (!seen.has(name)) groups.delete(name); } function registerAllVirtualProviders(pi: ExtensionAPI): void { const allModels = [...groups.values()].filter((g) => g.enabled).map((g) => g.virtualModel); pi.registerProvider(PROVIDER, { baseUrl: "model-roundrobin://local", apiKey: "model-roundrobin", api: API, models: allModels, streamSimple: streamRoundRobin, }); } // Cooldown (in-memory, process-local) function isCoolingDown(group: Group, c: ResolvedCandidate, now: number): boolean { const at = group.health.get(c.label); return at !== undefined && now - at < group.cooldownMs; } function recordFailure(group: Group, c: ResolvedCandidate): void { group.health.set(c.label, Date.now()); } /** Soonest we can retry any cooling candidate; full cooldownMs if none failed. */ function nextRetryWaitMs(group: Group): number { const now = Date.now(); let soonest = Infinity; for (const c of group.candidates) { const at = group.health.get(c.label); if (at !== undefined) soonest = Math.min(soonest, at + group.cooldownMs - now); } const wait = soonest === Infinity ? group.cooldownMs : Math.max(0, soonest); return Math.max(25, wait); } // Helpers /** True only when the *user* aborted — not when an upstream relay surfaces a * spurious `stopReason:"aborted"`. The latter is a real failure (cooldown + * rotate); only a real user cancel short-circuits the turn. */ function isUserAbort(options: SimpleStreamOptions | undefined, error?: unknown): boolean { if (options?.signal?.aborted) return true; if (error instanceof Error && error.message === "Request was aborted") return true; return false; } function makeErrorMessage(model: Model, message: string, stopReason: "error" | "aborted" = "error"): AssistantMessage { return { role: "assistant", content: [], api: model.api, provider: model.provider ?? PROVIDER, model: model.id, usage: EMPTY_USAGE, stopReason, errorMessage: message, timestamp: Date.now(), } as AssistantMessage; } type StreamEvent = { type: string; [k: string]: unknown }; /** Overwrite identity fields so the virtual model (not the upstream candidate) is persisted. */ function withVirtualMsg(msg: AssistantMessage, model: Model): AssistantMessage { const { responseModel: _, ...rest } = msg; return { ...rest, provider: PROVIDER, model: model.id, api: model.api } as AssistantMessage; } /** * Per-attempt idle guard. Bounds auth + every stream chunk against timeoutMs. * One long-lived promise (allocated once) rejects on timeout / abort / dispose; * win() just clears the timer and leaves the promise pending for the next * chunk, so the hot path allocates nothing per chunk. Race-free: each arm() * clears any prior timer, and the single rejector can only fire once (guarded * by `done`) — a stale timer can never reject a future chunk's race. */ const ABORTED_ERROR = new Error("Request was aborted"); class IdleGuard { private timer: NodeJS.Timeout | undefined; private rejector: (e: Error) => void = () => {}; private readonly promise: Promise; private done = false; constructor( private readonly timeoutMs: number, private readonly signal: AbortSignal, ) { this.promise = new Promise((_, rej) => { this.rejector = rej; }); this.promise.catch(() => {}); // never unhandled — the race may resolve via iter.next this.signal.addEventListener("abort", this.onAbort, { once: true }); } private onAbort = (): void => { this.done = true; this.rejector(ABORTED_ERROR); }; private fire = (): void => { this.done = true; this.rejector(new Error(`timeout after ${this.timeoutMs}ms`)); }; /** Arm the idle timer for the next chunk. Returns the shared promise that * rejects on timeout or abort (never resolves). Reused across chunks. */ arm(): Promise { if (this.done) return this.promise; if (this.timer) clearTimeout(this.timer); this.timer = setTimeout(this.fire, this.timeoutMs); return this.promise; } /** iter.next() won this chunk — clear the timer. The promise stays pending * and is reused by the next arm(); settled only on timeout/abort/dispose. */ win(): void { if (this.timer) { clearTimeout(this.timer); this.timer = undefined; } } dispose(): void { this.done = true; if (this.timer) { clearTimeout(this.timer); this.timer = undefined; } this.signal.removeEventListener("abort", this.onAbort); this.rejector(new Error("disposed")); } } function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { if (signal?.aborted) return Promise.resolve(false); return new Promise((resolve) => { let settled = false; const finish = (v: boolean): void => { if (settled) return; settled = true; clearTimeout(timer); signal?.removeEventListener("abort", onAbort); resolve(v); }; const onAbort = (): void => finish(false); const timer = setTimeout(() => finish(true), delayMs); signal?.addEventListener("abort", onAbort, { once: true }); if (signal?.aborted) onAbort(); }); } /** Where to start scanning this round, per strategy. */ function startIndexFor(group: Group, total: number, round: number): number { if (round > 0) return 0; // every retry pass starts from the preferred candidate if (group.strategy === "primary") return 0; return ((group.currentIndex % total) + total) % total; } /** Promote the cursor after a success, per strategy. */ function advanceOnSuccess(group: Group, index: number, total: number): void { switch (group.strategy) { case "round-robin": group.currentIndex = (index + 1) % total; // rotate to next for the next request break; case "primary": group.currentIndex = 0; // always snap back to preferred break; case "sticky": default: group.currentIndex = index; // pin to the one that worked break; } } /** * Failover loop. Returns null on success (outer already ended), or the terminal * AssistantMessage (error/aborted) for the caller to push + end. */ async function tryCandidates( group: Group, model: Model, context: Context, options: SimpleStreamOptions | undefined, outer: AssistantMessageEventStream, ): Promise { // Snapshot the candidate array: buildGroups mutates the Group in place and // may swap `candidates` to a new array mid-flight; this turn keeps its own. const candidates = group.candidates; const total = candidates.length; if (total === 0) return makeErrorMessage(model, `roundrobin/${group.name}: no valid candidates`); // Registry gate: pre-session placeholders must not be invoked (their failure // would poison cooldown keys the post-session real candidates inherit). if (!stateModelRegistry) return makeErrorMessage(model, `roundrobin/${group.name}: model registry not ready (session not started)`); let lastFailure: AssistantMessage | null = null; let round = 0; const turnStart = Date.now(); let startForwarded = false; // outer received a `start` this turn (one per turn; dupes suppressed across failovers) let contentForwarded = false; // outer received real content this turn → later errors are terminal (no replay) while (true) { if (options?.signal?.aborted) return makeErrorMessage(model, "Request was aborted", "aborted"); const startIndex = startIndexFor(group, total, round); let attempted = 0; const now = Date.now(); candidate: for (let offset = 0; offset < total; offset += 1) { if (options?.signal?.aborted) return makeErrorMessage(model, "Request was aborted", "aborted"); const index = (startIndex + offset) % total; const candidate = candidates[index]; if (isCoolingDown(group, candidate, now)) { logLine(group, `skip cooling-down ${candidate.label}`); continue; } attempted += 1; logLine(group, `use ${candidate.label}`); const startTime = Date.now(); let iteratorEnded = false; let iter: AsyncIterator | undefined; const attemptController = new AbortController(); const abortAttempt = (): void => attemptController.abort(); options?.signal?.addEventListener("abort", abortAttempt, { once: true }); if (options?.signal?.aborted) abortAttempt(); const attemptOptions: SimpleStreamOptions = { ...options, signal: attemptController.signal }; const guard = new IdleGuard(group.timeoutMs, attemptController.signal); try { // Resolve auth; bound by the same idle timeout + attempt abort as stream // chunks so a hung key/secret lookup can't stall the turn past timeoutMs. const auth = stateModelRegistry ? await Promise.race([Promise.resolve(stateModelRegistry.getApiKeyAndHeaders(candidate)), guard.arm()]) : undefined; guard.win(); if (auth && !auth.ok) throw new Error(auth.error || `no API key for ${candidate.label}`); const streamOpts = auth ? { ...attemptOptions, apiKey: auth.apiKey, headers: auth.headers } : attemptOptions; const stream = streamSimple(candidate, context, streamOpts); iter = stream[Symbol.asyncIterator]() as AsyncIterator; // First byte guard: the stream must produce a start event (or error) // within timeoutMs, else we failover to the next candidate. const first = await Promise.race([iter.next(), guard.arm()]); if (first.done) { iteratorEnded = true; throw new Error("stream ended before start event"); } guard.win(); const firstEvent = first.value; if (firstEvent.type === "error") { const upstreamError = (firstEvent as { error?: AssistantMessage }).error; const reason = upstreamError?.errorMessage || upstreamError?.stopReason || "unknown upstream error"; // Only a real user abort short-circuits; an upstream "aborted" // (socket drop / proxy cancel) is a failure → cool down + rotate. if (isUserAbort(options)) { iteratorEnded = true; return makeErrorMessage(model, reason, "aborted"); } recordFailure(group, candidate); logLine(group, `fail ${candidate.label}: ${reason}`); lastFailure = upstreamError ? withVirtualMsg(upstreamError, model) : makeErrorMessage(model, reason); continue; } if (firstEvent.type === "done") { // Some relays surface a completed error as a first `done` event. const msg = (firstEvent as { message?: AssistantMessage }).message; const stop = msg?.stopReason; const reason = msg?.errorMessage || stop || "stream ended before start event"; if (isUserAbort(options)) { iteratorEnded = true; return makeErrorMessage(model, reason, "aborted"); } recordFailure(group, candidate); logLine(group, `fail ${candidate.label}: done before start (${stop || "unknown"})`); lastFailure = msg ? withVirtualMsg(msg, model) : makeErrorMessage(model, reason); continue; } if (firstEvent.type !== "start") throw new Error(`unexpected first event: ${firstEvent.type}`); // Transient — the consumer persists only the final `done` message. // Suppress on failover: a prior candidate already opened the assistant // message; its empty start partial is replaced by the next candidate's // content/done, so only one `start` is ever forwarded per turn. if (!startForwarded) { outer.push(firstEvent as never); startForwarded = true; } // Mid-stream loop: every chunk races against the same idle timeout so a // stream that starts then goes silent fails fast instead of hanging. while (true) { const r = await Promise.race([iter.next(), guard.arm()]); guard.win(); if (r.done) { iteratorEnded = true; break; } const ev = r.value; if (ev.type === "error") { const e = (ev as unknown as { error: AssistantMessage }).error; const reason = e.errorMessage || e.stopReason || "unknown error"; iteratorEnded = true; if (isUserAbort(options)) { logLine(group, `aborted ${candidate.label}: ${reason}`); return withVirtualMsg(e, model); } recordFailure(group, candidate); logLine(group, `fail ${candidate.label}: ${reason}`); lastFailure = withVirtualMsg(e, model); if (contentForwarded) return lastFailure; // content already shown → no replay continue candidate; // start-only → fail over (nothing emitted to replay) } if (ev.type === "done") { const msg = (ev as unknown as { message: AssistantMessage }).message; const stop = msg.stopReason; iteratorEnded = true; if (stop === "error") { recordFailure(group, candidate); logLine(group, `fail ${candidate.label}: done with error`); lastFailure = withVirtualMsg(msg, model); if (contentForwarded) return lastFailure; continue candidate; } if (stop === "aborted") { if (!isUserAbort(options)) { recordFailure(group, candidate); logLine(group, `fail ${candidate.label}: done aborted`); lastFailure = withVirtualMsg(msg, model); if (contentForwarded) return lastFailure; continue candidate; } logLine(group, `aborted ${candidate.label}`); return withVirtualMsg(msg, model); // user cancel → terminal } advanceOnSuccess(group, index, total); logLine(group, `success ${candidate.label} (${Date.now() - startTime}ms)`); outer.push({ ...ev, message: withVirtualMsg((ev as { message: AssistantMessage }).message, model) } as never); outer.end(); return null; } outer.push(ev as never); contentForwarded = true; } // Iterator exhausted without a done/error event (r.done) — abnormal. const reason = "stream ended without done or error event"; recordFailure(group, candidate); logLine(group, `fail ${candidate.label}: ${reason}`); lastFailure = makeErrorMessage(model, reason); if (contentForwarded) return lastFailure; continue candidate; // no content emitted → fail over } catch (error) { if (isUserAbort(options, error)) return makeErrorMessage(model, "Request was aborted", "aborted"); const reason = error instanceof Error ? error.message : String(error); recordFailure(group, candidate); logLine(group, `fail ${candidate.label}: ${reason}`); lastFailure = makeErrorMessage(model, reason); if (contentForwarded) { // Content already shown; replaying risks duplicating it. The iterator // threw, so it's finished — don't let finally abort/return it again. iteratorEnded = true; return lastFailure; } // else: no content emitted → fail over (finally aborts this attempt) } finally { options?.signal?.removeEventListener("abort", abortAttempt); guard.dispose(); // Only abort the attempt stream if it didn't end cleanly — aborting a // completed stream is unnecessary and could disturb SDK cleanup. if (!iteratorEnded) { attemptController.abort(); if (iter) void Promise.resolve(iter.return?.()).catch(() => {}); } } } round += 1; if (round >= MAX_ROUNDS) { logLine(group, `giving up after ${round} rounds: ${lastFailure?.errorMessage || "unknown error"}`); return lastFailure ?? makeErrorMessage(model, "all candidates failed"); } const waitMs = nextRetryWaitMs(group); const reason = lastFailure?.errorMessage || "unknown error"; logLine(group, `round ${round} exhausted (${attempted}/${total} attempted); retry from preferred in ${waitMs}ms; last error: ${reason}`); if (Date.now() - turnStart + waitMs > TURN_DEADLINE_MS) { logLine(group, `giving up: turn deadline ${TURN_DEADLINE_MS}ms exceeded (round ${round})`); return lastFailure ?? makeErrorMessage(model, "all candidates failed"); } if (!(await waitForRetry(waitMs, options?.signal))) return makeErrorMessage(model, "Request was aborted", "aborted"); } } // streamSimple — failover entry function streamRoundRobin(model: Model, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream { const outer = createAssistantMessageEventStream(); (async () => { const group = groups.get(model.id); if (!group || !group.enabled) { outer.push({ type: "error", reason: "error", error: makeErrorMessage(model, `roundrobin preset "${model.id}" not found or disabled`) }); outer.end(); return; } const lastFailure = await tryCandidates(group, model, context, options, outer); if (lastFailure) { const reason = lastFailure.stopReason === "aborted" ? "aborted" : "error"; outer.push({ type: "error", reason, error: lastFailure }); outer.end(); } })().catch((error) => { outer.push({ type: "error", reason: "error", error: makeErrorMessage(model, error instanceof Error ? error.message : String(error)) }); outer.end(); }); return outer; } // Extension entry export default function (pi: ExtensionAPI): void { // Build groups immediately with placeholder candidates (no registry yet) so // the virtual models appear in /model and in clients that only read the // model list without starting a session (pi-web's static /api/models). // session_start rebuilds with the real model registry and overwrites them. try { buildGroups(loadAllGroupsConfig()); } catch (e) { logLine(null, `load failed: ${e instanceof Error ? e.message : String(e)}`); } registerAllVirtualProviders(pi); pi.on("session_start", async (_event, ctx) => { stateModelRegistry = ctx.modelRegistry; try { buildGroups(loadAllGroupsConfig()); registerAllVirtualProviders(pi); const enabled = [...groups.values()].filter((g) => g.enabled).map((g) => g.name); logLine(null, `session_start groups=${groups.size} enabled=${enabled.join(",") || "(none)"}`); } catch (e) { logLine(null, `session_start failed: ${e instanceof Error ? e.message : String(e)}`); } }); }