import type { AgentRegistrationService } from './agent-registration.service.js'; import type { EventBusService } from '../event-bus/event-bus.service.js'; import { type RuntimeType } from '../../constants.js'; /** * Delivers a notice to the owner (Slack). Resolves true when it was sent, * false when there was no way to reach the owner. */ export type RuntimeExitOwnerNotifier = (notice: { title: string; message: string; urgent: boolean; }) => Promise; /** * Service that monitors PTY sessions for runtime exit patterns. * * When an agent CLI (Claude Code, Gemini CLI, Codex CLI, OpenCode) exits inside a PTY * session, the PTY shell itself stays alive. This service watches terminal * output for runtime-specific exit patterns and reacts by: * * 1. Updating the agent status to `inactive` * 2. Capturing session memory * 3. Broadcasting WebSocket events to the frontend * 4. Cancelling pending registration operations * * For Gemini CLI, the service also detects failure patterns (API errors, * quota exhaustion, network issues) that indicate the CLI is stuck and * needs recovery. These failure patterns bypass the shell prompt verification * since the CLI may still be running but non-functional. * * For Claude Code, the service detects fatal API errors (e.g. thinking * block corruption) that make the CLI permanently non-functional within * the current session. These trigger immediate recovery without retry. * * @example * ```typescript * const monitor = RuntimeExitMonitorService.getInstance(); * monitor.startMonitoring('agent-dev-001', 'claude-code', 'developer'); * // ... later ... * monitor.stopMonitoring('agent-dev-001'); * ``` */ export declare class RuntimeExitMonitorService { private static instance; private logger; private sessions; private onExitDetectedCallback?; private agentRegistrationService; private eventBusService; /** * Restart timestamps per session, persisted across monitoring cycles. * Unlike MonitoredSession (which is recreated on each startMonitoring call), * this map survives agent restarts to prevent infinite restart loops * when agents keep crashing (e.g. during network outages). */ private restartHistory; /** #989: recent `startup_exit` times per runtime (within STARTUP_EXIT_WINDOW_MS). */ private startupExitHistory; /** #989: when the owner was last told a runtime keeps dying at start. */ private startupExitNoticeAt; private ownerNotifier; private constructor(); /** * Get the singleton instance. */ static getInstance(): RuntimeExitMonitorService; /** * Reset the singleton (for testing). */ static resetInstance(): void; /** * Register a callback invoked when a runtime exit is detected. * Used by AgentRegistrationService to cancel pending registrations. * * @param callback - Called with the sessionName on exit detection */ setOnExitDetectedCallback(callback: (sessionName: string) => void): void; /** * Set the AgentRegistrationService dependency for agent restart. * * @param service - The AgentRegistrationService instance */ setAgentRegistrationService(service: AgentRegistrationService): void; /** * Backwards-compatible no-op. Kept so `index.ts` compiles during the * TaskTrackingService deletion window. In-progress task queries now * read directly from TaskPoolService. * * @deprecated Will be removed in a follow-up. Do not call from new code. */ setTaskTrackingService(_service: unknown): void; /** * Set the EventBusService dependency for publishing agent lifecycle events. * * @param service - The EventBusService instance */ setEventBusService(service: EventBusService): void; /** * Set how the owner is told that a runtime keeps dying at start (#989). * * @param notifier - Owner notifier (Slack), or null to stop notices */ setOwnerNotifier(notifier: RuntimeExitOwnerNotifier | null): void; /** * Start monitoring a PTY session for runtime exit patterns. * * @param sessionName - PTY session name * @param runtimeType - Agent runtime type (claude-code, gemini-cli, codex-cli) * @param role - Agent role name * @param teamId - Optional team identifier * @param memberId - Optional member identifier */ startMonitoring(sessionName: string, runtimeType: RuntimeType, role: string, teamId?: string, memberId?: string): void; /** * Stop monitoring a PTY session. * * Also prunes the restartHistory entry for this session if it has no * recent timestamps within the cooldown window, preventing slow memory * leak for terminated sessions. * * @param sessionName - PTY session name */ stopMonitoring(sessionName: string): void; /** * Check if a session is being monitored. */ isMonitoring(sessionName: string): boolean; /** * Destroy all monitoring subscriptions. */ destroy(): void; /** * Handle incoming PTY data for a monitored session. */ private handleData; /** * Confirm exit by checking for a shell prompt, then react. * * Runtime-specific failure checks are gated by runtimeType so that * patterns for one runtime cannot accidentally match another (e.g. a * Codex exit matching Claude or Gemini patterns). * * For Gemini CLI failure patterns (API errors, quota exhaustion, etc.), * the system retries with exponential backoff before declaring the agent * dead. Gemini CLI often recovers automatically from transient API errors * (RESOURCE_EXHAUSTED, UNAVAILABLE, Connection error). Only after * MAX_RETRIES does the system proceed with the exit/restart flow. * * For Claude Code fatal patterns (thinking block corruption, etc.), * the system skips retry and forces immediate recovery since these * errors are permanently unrecoverable within the same session. */ private confirmAndReact; /** * Check for runtime-specific failure patterns when shell prompt is not confirmed. * * Gated by runtimeType so patterns for one runtime cannot accidentally * match another. Returns true if a runtime-specific failure was detected, * or 'deferred' handling occurred (Gemini retry). * * @param sessionName - PTY session name * @param monitored - Monitored session state * @param helper - Session command helper * @returns True if a runtime-specific failure was confirmed */ private checkRuntimeSpecificFailure; /** * Fire the onExitDetected callback safely. * * @param sessionName - PTY session name */ private fireExitDetectedCallback; /** * Try to restart an agent that has in-progress tasks. * * @param sessionName - PTY session name * @param monitored - Monitored session state * @returns True if the agent was successfully restarted (caller should return) */ private tryAgentRestartWithTasks; /** * Try to restart the orchestrator after runtime exit. * * Sets status to inactive before restart to prevent queue processor * from delivering messages during postInitialize, captures session * memory, and broadcasts restart status. * * @param sessionName - PTY session name (should be ORCHESTRATOR_SESSION_NAME) * @param monitored - Monitored session state * @returns True if orchestrator was successfully restarted (caller should return) */ private tryOrchestratorRestart; /** * Transition agent to inactive state: update storage, capture memory, * publish events, and broadcast WebSocket status. * * @param sessionName - PTY session name * @param monitored - Monitored session state */ private transitionToInactive; /** * Handle a Gemini CLI failure with exponential backoff retry. * * Waits with increasing delays and checks if the CLI recovers (returns * to a ready prompt). Returns 'recovered' if the CLI is back, * 'deferred' if retries remain (skip exit flow, re-check on next * failure detection), or 'exhausted' if max retries have been reached. * * @param sessionName - PTY session name * @param monitored - Monitored session state (retry counter is mutated) * @param helper - Session command helper for terminal output capture * @returns 'recovered' | 'deferred' | 'exhausted' */ private handleGeminiFailureWithRetry; /** * Restart an agent that exited while it had in-progress tasks. * * Mirrors the restart pattern from AgentHeartbeatMonitorService: * 1. Save Claude session ID for resume * 2. Kill old PTY session + clear PTY activity tracker * 3. Pre-set session ID for resume * 4. Recreate agent session via AgentRegistrationService * 5. Broadcast restarted status via WebSocket * 6. Re-deliver tasks asynchronously * * @param sessionName - PTY session name * @param monitored - Monitored session state * @param activeTasks - In-progress tasks to re-deliver */ private restartAgentWithTasks; /** * Re-deliver in-progress tasks to a restarted agent. * * Waits for the agent to initialize, reads task files, and writes * summaries into the agent's PTY session. * * @param sessionName - PTY session name * @param activeTasks - Tasks to re-deliver */ private redeliverTasks; /** * Periodic check: is the runtime child process still alive? * * Uses the session backend's isChildProcessAlive() (pgrep -P ) * to determine if the CLI process has exited. This is a version-agnostic * fallback that doesn't depend on any text the CLI prints. * * @param sessionName - PTY session name * @param backend - Session backend for process checks * @param helper - Session command helper for shell prompt verification */ private checkProcessAlive; /** * Verify that a shell prompt is visible in the terminal, confirming * that the agent CLI has actually exited and returned to the shell. */ private verifyExitWithShellPrompt; /** * Check if an agent restart is allowed based on cooldown window. * Prevents infinite restart loops when agents keep crashing (e.g. during * network outages). Uses the same limits as AgentHeartbeatMonitorService. * * @param sessionName - PTY session name * @returns True if restart is allowed, false if cooldown is active */ private isAgentRestartAllowed; /** * Record a restart attempt for cooldown tracking. * * @param sessionName - PTY session name */ private recordAgentRestart; /** * Classify an error reason as transient or persistent. * * Transient errors (connectivity, timeout, quota) may resolve on retry. * Persistent errors (auth, config, crash loop) require manual intervention. * * @param reason - The failure reason string * @returns 'TRANSIENT' | 'PERSISTENT' | 'UNKNOWN' */ private classifyError; /** * Notify the orchestrator about an agent failure via the chat API (#129). * * Sends a structured failure notification with error classification so the * orchestrator can decide whether to restart the agent, reassign tasks, * or inform the user. This is fire-and-forget — failures are logged but * do not block the exit flow. * * @param sessionName - Failed agent's PTY session name * @param reason - Failure reason (runtime_exited, api_failure, etc.) * @param activeTasks - In-progress tasks the agent had, if any * @param restartSucceeded - Whether automatic restart succeeded */ private notifyOrchestratorOfFailure; /** * #235: Infer dropout reason from the monitored session's buffer. * * When nothing matches, the exit is `idle_exit` — unless it came within * EARLY_EXIT_WINDOW_MS of monitoring start, when it is `startup_exit` * (#791): a runtime that dies in its first minute did not idle out. * * @param monitored - Monitored session state * @returns Inferred dropout reason */ private inferDropoutReason; /** * #989: Count a `startup_exit` for this runtime. When one runtime dies at * start STARTUP_EXIT_ALERT_THRESHOLD times within STARTUP_EXIT_WINDOW_MS, * tell the owner once (then not again for STARTUP_EXIT_NOTICE_COOLDOWN_MS), * because auto-restart keeps hiding it: the agent looks alive between deaths. * * @param monitored - Monitored session that just exited at start */ private recordStartupExit; /** * #791: Diagnostics for an exit whose cause was not recognised — the * runtime, how long after start it went, and the cleaned tail of its * terminal output (where a CLI prints its own error or stderr). * * @param monitored - Monitored session state * @param reason - The fallback reason recorded * @returns Fields for the log line */ private exitDiagnostics; /** * Publish agent:status_changed and agent:inactive events to the EventBus. * * Looks up team/member names from StorageService to build the full * AgentEvent. This mirrors the pattern used in TeamsJsonWatcherService. * * @param sessionName - PTY session name * @param monitored - Monitored session state with teamId/memberId * @param storageService - StorageService instance for team/member lookup */ private publishInactiveEvent; } //# sourceMappingURL=runtime-exit-monitor.service.d.ts.map