/** * Orchestrator Restart Service * * Automatically restarts the orchestrator when it becomes unresponsive * (child process dies inside the PTY shell). Enforces a cooldown window * to prevent restart loops. * * @module orchestrator-restart.service */ import type { AgentRegistrationService } from '../agent/agent-registration.service.js'; import type { ISessionBackend } from '../session/session-backend.interface.js'; /** Options of {@link OrchestratorRestartService.attemptRestart}. */ export interface RestartOptions { /** * A relaunch Crewly does on purpose (the runtime fallback switching the * orchestrator to another runtime or back), not a recovery from a crash or * hang: no cooldown slot or attempt is counted, no "Orchestrator * Restarted … unresponsive" alarm is sent, and a failure does not count * toward giving up. */ planned?: boolean; /** Who asked for a planned relaunch (logged) */ reason?: string; } /** * Restart statistics for monitoring */ export interface RestartStats { /** Total number of restarts since service creation */ totalRestarts: number; /** Restarts within the current cooldown window */ restartsInWindow: number; /** Whether a restart is currently in progress */ isRestarting: boolean; /** Timestamp of the last successful restart */ lastRestartAt: string | null; /** Whether further restarts are allowed under cooldown */ restartAllowed: boolean; /** Failed restarts in a row since the orchestrator last ran */ consecutiveFailures: number; /** Set once auto-restart has stopped trying; null while it still tries */ gaveUp: RestartGiveUp | null; } /** Why auto-restart stopped trying, surfaced to the user. */ export interface RestartGiveUp { /** The last failure's user-facing reason */ reason: string; /** Failed attempts that led here */ attempts: number; /** ISO time auto-restart stopped */ at: string; /** True when the failure needs the user (e.g. runtime not installed / not signed in) */ blocked: boolean; } /** * OrchestratorRestartService manages automatic restart of the orchestrator * when the child process (Claude Code) dies inside the PTY shell. * * Features: * - Kills the old PTY session before creating a new one * - Initializes memory and starts chat monitoring * - Notifies via Slack (if configured) * - Broadcasts WebSocket event for UI updates * - Enforces max 3 restarts per hour cooldown (failed attempts count too) * - Stops after MAX_CONSECUTIVE_FAILURES failures in a row, or at once when * start-up is blocked on the user, and surfaces the reason (getGiveUp) * * @example * ```typescript * const service = OrchestratorRestartService.getInstance(); * service.setDependencies(agentRegistrationService, sessionBackend, io); * await service.attemptRestart(); * ``` */ export declare class OrchestratorRestartService { private static instance; private logger; /** Timestamps of recent restarts for cooldown tracking */ private restartTimestamps; /** Total restart count since service creation */ private totalRestarts; /** Flag to prevent concurrent restart attempts */ private isRestarting; /** Failed restarts in a row since the orchestrator last ran */ private consecutiveFailures; /** Set when auto-restart has stopped trying (see {@link markGaveUp}) */ private gaveUp; /** Retry time of the usage-limit state last logged as blocking a restart (log once) */ private loggedUsageLimitRetryAt; /** External dependencies (injected via setDependencies) */ private agentRegistrationService; private sessionBackend; private socketIO; private constructor(); /** * Get the singleton instance. * * @returns The OrchestratorRestartService instance */ static getInstance(): OrchestratorRestartService; /** * Reset the singleton instance (for testing). */ static resetInstance(): void; /** * Inject external dependencies. * * @param agentRegistrationService - Service for creating agent sessions * @param sessionBackend - Session backend for killing old sessions * @param socketIO - Socket.IO server for broadcasting events */ setDependencies(agentRegistrationService: AgentRegistrationService, sessionBackend: ISessionBackend, socketIO?: { emit: (event: string, data: unknown) => void; }): void; /** * Check if a restart is currently allowed under the cooldown window. * * @returns true if restart is allowed */ isRestartAllowed(): boolean; /** * Attempt to restart the orchestrator. * * This method: * 1. Checks cooldown limits * 2. Kills the old PTY session * 3. Creates a new agent session * 4. Initializes memory * 5. Starts chat monitoring * 6. Notifies Slack (if configured) * 7. Broadcasts WebSocket event * * A planned relaunch ({@link RestartOptions.planned}) skips the cooldown and * give-up checks, is not counted, and sends no alarm. While one is in its * window, unplanned callers (heartbeat / exit monitors) are refused: the * orchestrator is being restarted on purpose, not down. * * @param options - `planned` for a relaunch Crewly does on purpose * @returns true if restart succeeded, false otherwise */ attemptRestart(options?: RestartOptions): Promise; /** * Kill and recreate the orchestrator session. * * @param planned - A relaunch on purpose: not counted, no alarm * @param reason - Who asked for a planned relaunch * @returns true on success */ private runRestart; /** * Count a failed restart. Failed attempts also take a slot in the cooldown * window (before, only successes did, so failures were never limited). On a * start-up blocked on the user, or after MAX_CONSECUTIVE_FAILURES in a row, * auto-restart stops (see {@link markGaveUp}). * * @param reason - User-facing reason of this failure * @param blocked - True when retrying cannot help (RUNTIME_STARTUP_BLOCKED) */ private recordFailure; /** * Stop auto-restarting the orchestrator and surface why: logged once at * ERROR, broadcast as `orchestrator:restart_gave_up`, and returned by * {@link getGiveUp} for the orchestrator status endpoint. Also called by the * boot-time auto-start when it gives up, so the heartbeat monitor does not * start a second retry loop. Cleared by {@link clearGiveUp}. * * @param reason - User-facing reason * @param options - `blocked` when the user must act; `attempts` made */ markGaveUp(reason: string, options: { blocked: boolean; attempts: number; }): void; /** * Why auto-restart stopped, or null while it still tries. * * @returns The give-up record, or null */ getGiveUp(): RestartGiveUp | null; /** * Re-arm auto-restart once the orchestrator runs again (it was started by * hand, or the user fixed the runtime and it came back). */ clearGiveUp(): void; /** * Resolve orchestrator runtime type from persisted orchestrator status. * Falls back to Claude Code if status is unavailable/invalid. */ private resolveOrchestratorRuntimeType; /** * Get current restart statistics. * * @returns Restart statistics object */ getRestartStats(): RestartStats; /** * Send a Slack notification about the orchestrator restart. * Best-effort: silently ignores failures (Slack may not be configured). */ private notifySlack; } //# sourceMappingURL=orchestrator-restart.service.d.ts.map