// #1676 — orchestrator registration retry. // // The second half of the crash loop. When the orchestrator restarts into a relay it // cannot reach, the old retry was a fixed 5s `for(;;)` in index.ts: no backoff, no // ceiling on how long it stayed silent, and no state anyone could observe. A host stuck // there looks identical to a healthy one from the outside — spawn requests are routed to // it and burn their full 300s timeout instead of failing fast against a known-degraded // host. // // This driver keeps the "retry forever" semantics (an orchestrator that gives up is // worse than one that keeps trying) but adds exponential backoff with jitter, and after // N consecutive failures raises a DEGRADED state that is logged once and exposed on the // orchestrator's own /api/health. // // Backoff comes from the SDK's ReconnectionManager — the same primitive the relay client // uses for its reconnects — so there is one backoff implementation, not two. import { ReconnectionManager } from "agent-relay-sdk"; import { describeFailure } from "./async-guard"; /** 5s first retry — unchanged from the pre-#1676 fixed interval, so a relay that is * merely starting up alongside us still reconnects just as fast. */ export const REGISTER_INITIAL_MS = 5_000; /** Ceiling. Long enough to stop hammering an unreachable relay, short enough that * recovery is noticed within a couple of minutes. */ export const REGISTER_MAX_MS = 120_000; export const REGISTER_JITTER_MS = 1_000; /** Consecutive failures before the host declares itself degraded. Five attempts spans * roughly 5+10+20+40s ≈ 75s of unreachability — past any ordinary relay restart. */ export const REGISTER_DEGRADED_AFTER_ATTEMPTS = 5; export interface RegistrationHealth { /** True once a registration attempt has succeeded and none has failed since. */ registered: boolean; /** True after REGISTER_DEGRADED_AFTER_ATTEMPTS consecutive failures, until one succeeds. */ degraded: boolean; /** Consecutive failed attempts since the last success. */ consecutiveFailures: number; lastError?: string; degradedSince?: number; lastAttemptAt?: number; /** Backoff applied before the next attempt — the observable proof this is not a fixed spin. */ nextRetryInMs?: number; } export interface RegistrationDriverOptions { register: () => Promise; log?: (message: string) => void; sleep?: (ms: number) => Promise; now?: () => number; initialMs?: number; maxMs?: number; jitterMs?: number; degradedAfterAttempts?: number; } export interface RegistrationDriver { /** Retries until registration succeeds. NEVER throws and never terminates the process: * a registration failure — including an AbortError against an unreachable peer — is an * expected operating condition, not a fatal one. */ registerUntilConnected(): Promise; getHealth(): RegistrationHealth; } export function createRegistrationDriver({ register, log = console.error, sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)), now = Date.now, initialMs = REGISTER_INITIAL_MS, maxMs = REGISTER_MAX_MS, jitterMs = REGISTER_JITTER_MS, degradedAfterAttempts = REGISTER_DEGRADED_AFTER_ATTEMPTS, }: RegistrationDriverOptions): RegistrationDriver { const backoff = new ReconnectionManager({ initialMs, maxMs, jitterMs }); let registered = false; let degraded = false; let consecutiveFailures = 0; let lastError: string | undefined; let degradedSince: number | undefined; let lastAttemptAt: number | undefined; let nextRetryInMs: number | undefined; function formatDelay(ms: number): string { return ms < 1000 ? `${ms}ms` : `${Math.round(ms / 1000)}s`; } async function registerUntilConnected(): Promise { for (;;) { lastAttemptAt = now(); try { await register(); registered = true; consecutiveFailures = 0; lastError = undefined; nextRetryInMs = undefined; backoff.reset(); if (degraded) { const forMs = degradedSince ? now() - degradedSince : 0; log(`[orchestrator] Registration recovered after ${Math.round(forMs / 1000)}s degraded`); degraded = false; degradedSince = undefined; } return; } catch (err) { registered = false; consecutiveFailures += 1; lastError = describeFailure(err); const delayMs = backoff.nextDelay(); nextRetryInMs = delayMs; log(`[orchestrator] Register failed (attempt ${consecutiveFailures}): ${lastError}`); // Surface the degraded state ONCE, at the threshold — not on every retry, which is // what made the old 5s spin unreadable, and not silently, which is what made a stuck // host indistinguishable from a healthy one. if (!degraded && consecutiveFailures >= degradedAfterAttempts) { degraded = true; degradedSince = now(); log( `[orchestrator] DEGRADED: cannot register with the relay after ${consecutiveFailures} attempts ` + `(last: ${lastError}). This host cannot accept spawns or commands until it re-registers; ` + `still retrying, now backing off up to ${formatDelay(maxMs)}.`, ); } log(`[orchestrator] Retrying registration in ${formatDelay(delayMs)}...`); await sleep(delayMs); } } } return { registerUntilConnected, getHealth(): RegistrationHealth { return { registered, degraded, consecutiveFailures, ...(lastError ? { lastError } : {}), ...(degradedSince ? { degradedSince } : {}), ...(lastAttemptAt ? { lastAttemptAt } : {}), ...(nextRetryInMs !== undefined ? { nextRetryInMs } : {}), }; }, }; }