import * as plugins from './plugins.js'; import { controllerMcpCallerCredentialEnvironmentVariable, type IControllerTerminal, type IControllerTerminalSnapshotFrame, type TControllerTerminalId, type TControllerTerminalAgentKind, type TControllerTerminalAgentLaunchMode, } from '../ts_interfaces/index.js'; import type { IControllerResourceDocument } from './interfaces.projects.js'; import { ControllerTerminalMirror } from './classes.terminalmirror.js'; import { createSanitizedRuntimeEnvironment } from './functions.runtimeenvironment.js'; import { assertControllerTerminalAgentSessionIsFree, newControllerTerminalAgentSessionId, resolveControllerTerminalAgentExecutable, resolveControllerTerminalAgentLaunch, } from './functions.terminalagents.js'; export interface ITerminalOutputPayload { terminalId: TControllerTerminalId; /** * Absolute byte offset of this chunk in the terminal's output stream, or the offset a snapshot * frame's reconstructed state is exact at. */ offset: number; dataBase64: string; /** Set on every frame of a reconstructed screen state. */ snapshot?: IControllerTerminalSnapshotFrame; ended?: boolean; } /** * Only involuntary loss is announced. An explicit detach, a hung-up connection and a peer * superseding its own attachment are all client-initiated, so they need no notification. */ export type TControllerTerminalDetachReason = 'delivery_failed'; export interface IControllerPtySpawner { ensurePtySupport(): Promise; execSpawnStreamingInteractiveControlPty( commandArg: string, argsArg?: string[], optionsArg?: plugins.smartshell.IPtyDirectSpawnOptions, ): Promise; } export interface IControllerTerminalShell { executable: string; args: string[]; environment: NodeJS.ProcessEnv; } export type TControllerTerminalStopReason = 'exited' | 'closed'; export interface IControllerTerminalStartResult { terminal: IControllerTerminal; /** Which upstream flag an agent root resolved to; absent for plain shells. */ launchMode?: TControllerTerminalAgentLaunchMode; } export interface IControllerTerminalManagerOptions { resolveProjectDirectory: (projectIdArg: string) => Promise; sendOutput: (peerIdArg: string, payloadArg: ITerminalOutputPayload) => Promise; onTerminalsChanged: (projectIdArg: string) => void; onTerminalStopped: ( projectIdArg: string, terminalIdArg: TControllerTerminalId, exitCodeArg: number | undefined, /** * `exited` is the root ending on its own, `closed` is the controller taking it down. Only the * former may record a stopped intent; the latter must leave an agent chat wanting to run. */ reasonArg: TControllerTerminalStopReason, ) => Promise; /** * An attachment ended without the peer asking for it. Purely informational: the manager has * already dropped the peer, and the notification travels over the same transport whose failure * usually caused it, so it is best effort. */ onPeerDetached?: ( peerIdArg: string, terminalIdArg: TControllerTerminalId, reasonArg: TControllerTerminalDetachReason, ) => void; ptySpawner?: IControllerPtySpawner; resolveShell?: () => Promise; cleanupTimeoutMs?: number; /** * Delay schedule between output delivery attempts. The last entry is the clamp used once the * schedule is shorter than the attempt count. Configurable so tests do not sleep the * production budget. */ outputRetryBackoffMs?: readonly number[]; /** * Mints the caller credential this terminal's descendants present to `agl mcp`, so the * controller authorizes the chat running in the terminal instead of a caller-supplied task id. * Minted per start, which makes every restart mint a fresh credential and strand the old one. */ mintCallerCredential?: (inputArg: { projectId: string; resourceId: string; agentSessionId?: string; }) => string | undefined; } interface IProjectAdmissionState { activeOperations: number; retiring: boolean; retired: boolean; drainPromise?: Promise; resolveDrain?: () => void; retirementPromise?: Promise; } interface IOutputSlab { buffer: Buffer; start: number; end: number; } interface ITerminalOutputStore { slabs: IOutputSlab[]; start: number; end: number; bytes: number; } interface ITerminalPeerState { peerId: string; generation: number; cursor: number; detaching: boolean; finalizing: boolean; deliveryPromise?: Promise; /** * A reconstructed screen state is owed before any live byte: set when the peer is admitted, and * again whenever the in-flight window moved past its cursor. Delivering the current state is * both cheaper and more accurate than replaying the bytes the peer missed. */ needsSnapshot: boolean; /** Hand-overs since the peer last reached live output; bounds repeated re-serialization. */ consecutiveSnapshots: number; } type TTerminalFinalState = 'running' | 'failed-unconfirmed' | 'exited'; interface IControllerOwnedTerminal extends Omit { id: TControllerTerminalId; } interface ITerminalEntry { projectId: string; terminal: IControllerOwnedTerminal; execution: plugins.smartshell.IExecResultPtyStreaming; peers: Map; sizesByPeer: Map; output: ITerminalOutputStore; /** Parses the whole output stream, so the current screen is always available as a snapshot. */ mirror: ControllerTerminalMirror; controlQueue: Promise; pendingControlOperations: number; sizeRevision: number; sizeReconciliationActive: boolean; sizeReconciliationPromise?: Promise; closing: boolean; /** Agent roots get SIGTERM first so they can flush their transcript before dying. */ graceful: boolean; finalState: TTerminalFinalState; exitCode?: number; finalError?: Error; signalError?: Error; rootExitPromise: Promise; resolveRootExit: () => void; exitSubscription?: plugins.smartshell.IPtySubscription; finalizePromise?: Promise; closeLifecyclePromise?: Promise; finalizationDeadlineAt?: number; } type TTerminalManagerState = 'new' | 'open' | 'closing' | 'closed'; const maxTerminals = 32; const maxTerminalsPerProject = 8; /** * Raw output is only retained for the delivery window between the slowest peer's cursor and the * live end — scrollback is the mirror's job, and it keeps rows rather than bytes. A peer that * falls behind this window receives a fresh snapshot, so the window has to be wide enough for * ordered frame-by-frame delivery not to trip over itself, and wider than the history bound of a * snapshot: a peer acknowledging the state it was handed must still be able to continue from the * offset that state is exact at. */ const maxOutputWindowBytes = 1024 * 1024; const maxPtyCaptureBytes = 4 * 1024 * 1024; const maxInputBytes = 8 * 1024; const maxOutputFrameBytes = 64 * 1024; const terminalSpawnCols = 120; const terminalSpawnRows = 30; const maxPeersPerTerminal = 32; const maxPendingControlOperations = 256; const defaultCleanupTimeoutMs = 10_000; /** * A single slow or failed output round trip is a transient transport condition, not the end of * an attachment. Frames are therefore retried against a bounded budget — re-sending is safe * because the peer cursor only advances after a delivery resolves, so a retry repeats the exact * same byte range at the exact same offset — and only an exhausted budget drops the peer. */ const maxFrameDeliveryAttempts = 5; /** * How often one attachment may be handed the state again before it is given up. Every hand-over * serializes the mirror on the loop that serves every session, so a peer that keeps losing the * window while it acknowledges a snapshot must not re-serialize forever. From the second * consecutive attempt the state is serialized without its reconstructed scrollback, which is what * lets a peer on a slow transport win the race; an exhausted budget detaches it exactly like an * exhausted frame budget does, and the client re-attaches. */ const maxConsecutivePeerSnapshots = 4; const maxFrameDeliveryWallMs = 30_000; const defaultOutputRetryBackoffMs: readonly number[] = [250, 500, 1_000, 2_000, 4_000]; /** * closeAll() shares one deadline across every PTY, peer finalization and removal cleanup, and a * blown deadline makes it throw — which leaves the manager set and fails the typedServer stop * gate, holding the port through an upgrade. The grace therefore never takes more than half of * whatever remains. */ const maxGracefulTerminationMs = 2_500; const normalizeError = (errorArg: unknown): Error => errorArg instanceof Error ? errorArg : new Error(String(errorArg)); const toTerminalDto = (terminalArg: IControllerOwnedTerminal): IControllerTerminal => ({ ...terminalArg, id: { ...terminalArg.id }, }); const appendBoundedChunk = ( slabsArg: IOutputSlab[], currentBytesArg: number, chunkArg: Buffer, ): number => { if (chunkArg.byteLength === 0) return currentBytesArg; let sourceOffset = 0; let currentBytes = currentBytesArg; if (chunkArg.byteLength >= maxOutputWindowBytes) { slabsArg.splice(0, slabsArg.length); sourceOffset = chunkArg.byteLength - maxOutputWindowBytes; currentBytes = 0; } while (sourceOffset < chunkArg.byteLength) { let slab = slabsArg.at(-1); if (!slab || slab.end >= slab.buffer.byteLength) { slab = { buffer: Buffer.allocUnsafe(maxOutputFrameBytes), start: 0, end: 0, }; slabsArg.push(slab); } const copyBytes = Math.min( chunkArg.byteLength - sourceOffset, slab.buffer.byteLength - slab.end, ); chunkArg.copy(slab.buffer, slab.end, sourceOffset, sourceOffset + copyBytes); slab.end += copyBytes; sourceOffset += copyBytes; currentBytes += copyBytes; } let excess = currentBytes - maxOutputWindowBytes; while (excess > 0) { const first = slabsArg[0]; const firstBytes = first.end - first.start; if (firstBytes <= excess) { slabsArg.shift(); currentBytes -= firstBytes; excess -= firstBytes; } else { first.start += excess; currentBytes -= excess; excess = 0; } } return currentBytes; }; const appendOutput = (outputArg: ITerminalOutputStore, chunkArg: Buffer): void => { if (chunkArg.byteLength === 0) return; outputArg.end += chunkArg.byteLength; outputArg.bytes = appendBoundedChunk(outputArg.slabs, outputArg.bytes, chunkArg); outputArg.start = outputArg.end - outputArg.bytes; }; const validateExecutable = async ( candidateArg: string, platformArg: NodeJS.Platform, ): Promise => { const pathApi = platformArg === 'win32' ? plugins.path.win32 : plugins.path.posix; if (!pathApi.isAbsolute(candidateArg)) { throw new Error('The terminal shell executable must be an absolute path.'); } const executable = await plugins.fs.promises.realpath(candidateArg); const stats = await plugins.fs.promises.stat(executable); if (!stats.isFile()) throw new Error('The terminal shell executable is not a file.'); if (platformArg !== 'win32') { await plugins.fs.promises.access(executable, plugins.fs.constants.X_OK); } return executable; }; export const resolveControllerTerminalShell = async ( environmentArg: NodeJS.ProcessEnv = process.env, platformArg: NodeJS.Platform = process.platform, validateExecutableArg: ( candidateArg: string, platformArg: NodeJS.Platform, ) => Promise = validateExecutable, ): Promise => { const environment = createSanitizedRuntimeEnvironment(environmentArg); if (platformArg === 'win32') { const candidates: string[] = []; if (environmentArg.COMSPEC && plugins.path.win32.isAbsolute(environmentArg.COMSPEC)) { candidates.push(environmentArg.COMSPEC); } const systemRoot = environmentArg.SystemRoot ?? environmentArg.SYSTEMROOT; if (systemRoot && plugins.path.win32.isAbsolute(systemRoot)) { candidates.push(plugins.path.win32.join(systemRoot, 'System32', 'cmd.exe')); } for (const candidate of [...new Set(candidates)]) { try { const executable = await validateExecutableArg(candidate, platformArg); delete environment.SHELL; environment.COMSPEC = executable; return { executable, args: [], environment }; } catch { // Try the trusted SystemRoot fallback before failing closed. } } throw new Error('No trusted Windows terminal shell executable is available.'); } const candidates = [ ...(environmentArg.SHELL && plugins.path.posix.isAbsolute(environmentArg.SHELL) ? [environmentArg.SHELL] : []), '/bin/sh', ]; for (const candidate of [...new Set(candidates)]) { try { const executable = await validateExecutableArg(candidate, platformArg); delete environment.COMSPEC; environment.SHELL = executable; return { executable, args: [], environment }; } catch { // Fall through to /bin/sh, then fail closed. } } throw new Error('No trusted POSIX terminal shell executable is available.'); }; /** * Retry backoff must never be the reason a process stays alive: an unreffed timer lets shutdown * proceed while a delivery is sleeping between attempts. */ const delayUnreffed = (delayMsArg: number): Promise => new Promise((resolve) => { setTimeout(resolve, delayMsArg).unref(); }); const waitUntil = async (promiseArg: Promise, deadlineAtArg: number): Promise => { const remainingMs = deadlineAtArg - Date.now(); if (remainingMs <= 0) throw new Error('Terminal cleanup exceeded its deadline.'); let timeout: NodeJS.Timeout | undefined; try { return await Promise.race([ promiseArg, new Promise((_resolve, reject) => { timeout = setTimeout(() => reject(new Error('Terminal cleanup exceeded its deadline.')), remainingMs); }), ]); } finally { if (timeout) clearTimeout(timeout); } }; export class ControllerTerminalManager { private readonly ptySpawner: IControllerPtySpawner; private readonly resolveShell: () => Promise; private readonly cleanupTimeoutMs: number; private readonly outputRetryBackoffMs: readonly number[]; private readonly entries = new Map(); private readonly projectStates = new Map(); private readonly activeRetirements = new Set>(); private readonly pendingRemovalCleanups = new Map, string>(); private state: TTerminalManagerState = 'new'; private shell?: IControllerTerminalShell; private initPromise?: Promise; private closePromise?: Promise; private pendingCreates = 0; private readonly pendingCreatesByProject = new Map(); private peerGeneration = 0; constructor(private readonly options: IControllerTerminalManagerOptions) { this.ptySpawner = options.ptySpawner ?? new plugins.smartshell.Smartshell({ executor: 'sh', sourceFilePaths: [], }); this.resolveShell = options.resolveShell ?? (() => resolveControllerTerminalShell()); this.cleanupTimeoutMs = options.cleanupTimeoutMs ?? defaultCleanupTimeoutMs; if ( !Number.isSafeInteger(this.cleanupTimeoutMs) || this.cleanupTimeoutMs < 1 || this.cleanupTimeoutMs > 60_000 ) { throw new Error('cleanupTimeoutMs must be an integer between 1 and 60000.'); } this.outputRetryBackoffMs = options.outputRetryBackoffMs ?? defaultOutputRetryBackoffMs; if ( this.outputRetryBackoffMs.length === 0 || this.outputRetryBackoffMs.some((delayMs) => ( !Number.isSafeInteger(delayMs) || delayMs < 0 || delayMs > maxFrameDeliveryWallMs )) ) { throw new Error( `outputRetryBackoffMs must be a non-empty list of integers between 0 and ${maxFrameDeliveryWallMs}.`, ); } } public async init(): Promise { if (this.state === 'open') return; if (this.initPromise) return this.initPromise; if (this.state !== 'new') throw new Error('The terminal manager cannot be initialized now.'); const initPromise = (async () => { const [shell] = await Promise.all([ this.resolveShell(), this.ptySpawner.ensurePtySupport(), ]); if (this.state !== 'new') throw new Error('Terminal manager initialization was cancelled.'); this.shell = shell; this.state = 'open'; })(); this.initPromise = initPromise; try { await initPromise; } finally { if (this.initPromise === initPromise) this.initPromise = undefined; } } public async listTerminals(projectIdArg: string): Promise { return this.runProjectOperation(projectIdArg, async () => { await this.options.resolveProjectDirectory(projectIdArg); return [...this.entries.values()] .filter((entry) => entry.projectId === projectIdArg && entry.finalState !== 'exited') .map((entry) => toTerminalDto(entry.terminal)); }); } public async createTerminal(projectIdArg: string, titleArg?: string): Promise { return this.runProjectOperation(projectIdArg, async () => { this.reserveCreate(projectIdArg); try { const directory = await this.options.resolveProjectDirectory(projectIdArg); const shell = this.requireShell(); const terminalNativeId = this.createTerminalId(); return this.spawnTerminal({ projectId: projectIdArg, terminalNativeId, title: titleArg ?? plugins.path.basename(shell.executable), command: shell.executable, args: [...shell.args], cwd: directory, environment: { ...shell.environment }, graceful: false, }); } finally { this.releaseCreate(projectIdArg); } }); } public async resolveTerminalResourceMetadata( projectIdArg: string, titleArg?: string, agentKindArg?: TControllerTerminalAgentKind, ): Promise<{ title: string; terminal: NonNullable }> { const directory = await this.options.resolveProjectDirectory(projectIdArg); const shell = this.requireShell(); if (agentKindArg) { // The argv is resolved per start, because the launch flag depends on whether the // conversation already exists. Only the launcher path is pinned here. const command = await resolveControllerTerminalAgentExecutable(agentKindArg, shell.environment); return { title: titleArg ?? 'Claude', terminal: { command, args: [], cwd: directory, stoppedAt: new Date(), agent: { kind: agentKindArg, sessionId: newControllerTerminalAgentSessionId(), desiredState: 'running', consecutiveFailures: 0, }, }, }; } return { title: titleArg ?? plugins.path.basename(shell.executable), terminal: { command: shell.executable, args: [...shell.args], cwd: directory, stoppedAt: new Date(), }, }; } public async startTerminalResource( resourceArg: IControllerResourceDocument, ): Promise { if (resourceArg.kind !== 'terminal' || !resourceArg.terminal) { throw new Error('The resource is not a terminal.'); } const terminal = resourceArg.terminal; return this.runProjectOperation(resourceArg.projectId, async () => { this.reserveCreate(resourceArg.projectId); try { if (this.entries.has(resourceArg.id)) { throw new Error('The terminal resource is already running.'); } const directory = await this.options.resolveProjectDirectory(resourceArg.projectId); if (directory !== terminal.cwd) { throw new Error('The terminal resource project directory binding changed.'); } const shell = this.requireShell(); const environment = this.spawnEnvironmentFor(resourceArg, shell); if (!terminal.agent) { const spawned = await this.spawnTerminal({ projectId: resourceArg.projectId, terminalNativeId: resourceArg.id, title: resourceArg.title, command: terminal.command, args: [...terminal.args], cwd: directory, environment, graceful: false, }); return { terminal: spawned }; } // The executable is resolved once and used for both the fence and the spawn. Fencing with // the persisted path instead would park a chat permanently whenever that launcher moved, // even though a working binary is on PATH. const launch = await resolveControllerTerminalAgentLaunch( terminal.agent.kind, terminal.agent.sessionId, directory, shell.environment, ); // Concurrent resumes of one conversation both succeed and branch its transcript into a // DAG, silently discarding a branch. Prove the conversation is free before resuming it. // A brand-new conversation needs no listing: `--session-id` refuses an id already in use, // so Claude Code enforces that case itself and creation stays subprocess-free. if (launch.mode === 'resume') { await assertControllerTerminalAgentSessionIsFree( launch.command, terminal.agent.sessionId, shell.environment, ); } const spawned = await this.spawnTerminal({ projectId: resourceArg.projectId, terminalNativeId: resourceArg.id, title: resourceArg.title, command: launch.command, args: launch.args, cwd: directory, environment, graceful: true, }); return { terminal: spawned, launchMode: launch.mode }; } finally { this.releaseCreate(resourceArg.projectId); } }); } public async stopTerminalResource(projectIdArg: string, resourceIdArg: string): Promise { const entry = this.entries.get(resourceIdArg); if (!entry || entry.projectId !== projectIdArg) return; await this.runProjectOperation(projectIdArg, async () => { await this.closeEntry(entry, Date.now() + this.cleanupTimeoutMs); }); } public isTerminalRunning(projectIdArg: string, resourceIdArg: string): boolean { const entry = this.entries.get(resourceIdArg); return entry?.projectId === projectIdArg && entry.finalState === 'running'; } /** * The sanitized shell environment plus this start's caller credential. The credential is written * explicitly here and is never inheritable, so a value present in the controller's own * environment can never reach a child and impersonate a terminal. */ private spawnEnvironmentFor( resourceArg: IControllerResourceDocument, shellArg: IControllerTerminalShell, ): NodeJS.ProcessEnv { const environment: NodeJS.ProcessEnv = { ...shellArg.environment }; delete environment[controllerMcpCallerCredentialEnvironmentVariable]; const credential = this.options.mintCallerCredential?.({ projectId: resourceArg.projectId, resourceId: resourceArg.id, ...(resourceArg.terminal?.agent === undefined ? {} : { agentSessionId: resourceArg.terminal.agent.sessionId }), }); if (credential !== undefined) { environment[controllerMcpCallerCredentialEnvironmentVariable] = credential; } return environment; } private async spawnTerminal(inputArg: { projectId: string; terminalNativeId: string; title: string; command: string; args: string[]; cwd: string; environment: NodeJS.ProcessEnv; graceful: boolean; }): Promise { let entry: ITerminalEntry | undefined; const output: ITerminalOutputStore = { slabs: [], start: 0, end: 0, bytes: 0, }; // The mirror exists before the PTY does, because a root can write before the spawn resolves // and that output is part of the state a first attachment must see. Flow control therefore // only becomes effective once the PTY is there, and is reconciled right after. let ptyProcess: plugins.smartshell.IPtyProcess | undefined; const mirror = new ControllerTerminalMirror({ cols: terminalSpawnCols, rows: terminalSpawnRows, setPtyPaused: (pausedArg) => { if (!ptyProcess) return; if (pausedArg) ptyProcess.pause(); else ptyProcess.resume(); }, }); let execution: plugins.smartshell.IExecResultPtyStreaming; try { execution = await this.ptySpawner.execSpawnStreamingInteractiveControlPty( inputArg.command, [...inputArg.args], { cwd: inputArg.cwd, env: { ...inputArg.environment }, maxBuffer: maxPtyCaptureBytes, ptyCols: terminalSpawnCols, ptyRows: terminalSpawnRows, ptyTerm: 'xterm-256color', onData: (chunkArg) => { const chunk = Buffer.from(chunkArg, 'utf8'); mirror.write(chunk); if (entry) this.appendOutput(entry, chunk); else appendOutput(output, chunk); }, }, ); } catch (errorArg) { mirror.dispose(); throw errorArg; } ptyProcess = execution.ptyProcess; if (mirror.ptyPaused) ptyProcess.pause(); let resolveRootExit!: () => void; const rootExitPromise = new Promise((resolve) => { resolveRootExit = resolve; }); const createdEntry: ITerminalEntry = { projectId: inputArg.projectId, terminal: { id: { harnessId: 'controller', nativeId: inputArg.terminalNativeId }, title: inputArg.title, command: inputArg.command, cwd: inputArg.cwd, }, execution, peers: new Map(), sizesByPeer: new Map(), output, mirror, controlQueue: Promise.resolve(), pendingControlOperations: 0, sizeRevision: 0, sizeReconciliationActive: false, closing: false, graceful: inputArg.graceful, finalState: 'running', rootExitPromise, resolveRootExit, }; entry = createdEntry; this.entries.set(inputArg.terminalNativeId, createdEntry); createdEntry.exitSubscription = execution.ptyProcess.onExit( (eventArg) => this.confirmRootExit(createdEntry, eventArg.exitCode), ); if (createdEntry.finalState === 'exited') createdEntry.exitSubscription.dispose(); void execution.finalPromise.then( (resultArg) => this.confirmRootExit(createdEntry, resultArg.exitCode), (errorArg) => { if (createdEntry.finalState === 'running') createdEntry.finalState = 'failed-unconfirmed'; createdEntry.finalError = normalizeError(errorArg); }, ); if (createdEntry.finalState === 'exited' || this.state !== 'open') { await this.closeEntry(createdEntry, Date.now() + this.cleanupTimeoutMs); throw new Error('The terminal exited while it was being created.'); } const projectState = this.projectState(inputArg.projectId); if (projectState.retiring || projectState.retired) { await this.closeEntry(createdEntry, Date.now() + this.cleanupTimeoutMs); throw new Error('The project stopped accepting terminal operations.'); } this.options.onTerminalsChanged(inputArg.projectId); return toTerminalDto(createdEntry.terminal); } public async renameTerminal( projectIdArg: string, terminalIdArg: string, titleArg: string, ): Promise { const entry = this.requireEntry(projectIdArg, terminalIdArg); return this.runProjectOperation(projectIdArg, () => ( this.runEntryControl(entry, async () => { this.assertEntryRunning(entry); entry.terminal.title = titleArg; this.options.onTerminalsChanged(projectIdArg); return toTerminalDto(entry.terminal); }) )); } public async renameTerminalResource( projectIdArg: string, terminalIdArg: string, titleArg: string, ): Promise { const entry = this.entries.get(terminalIdArg); if (!entry || entry.projectId !== projectIdArg || entry.finalState === 'exited') return; await this.runProjectOperation(projectIdArg, () => ( this.runEntryControl(entry, async () => { this.assertEntryRunning(entry); entry.terminal.title = titleArg; this.options.onTerminalsChanged(projectIdArg); }) )); } public async removeTerminal(projectIdArg: string, terminalIdArg: string): Promise { const entry = this.entries.get(terminalIdArg); return this.runProjectOperation(projectIdArg, async () => { if (!entry || entry.projectId !== projectIdArg) return false; await this.closeEntry(entry, Date.now() + this.cleanupTimeoutMs); return true; }); } public async attach( peerIdArg: string, projectIdArg: string, terminalIdArg: string, isPeerConnectedArg: () => boolean = () => true, ): Promise { const entry = this.requireEntry(projectIdArg, terminalIdArg); return this.runProjectOperation(projectIdArg, () => ( this.runEntryControl(entry, async () => { this.assertEntryRunning(entry); if (!isPeerConnectedArg()) throw new Error('The terminal peer disconnected before attach.'); const previous = entry.peers.get(peerIdArg); if (previous) { // Not announced: this peer is the one superseding its own attachment. this.detachPeerState(entry, previous); if (previous.deliveryPromise) { await waitUntil( previous.deliveryPromise.catch(() => undefined), Date.now() + this.cleanupTimeoutMs, ).catch(() => undefined); } await this.applyMinimumSizeUntilStable(entry); } if (entry.peers.size >= maxPeersPerTerminal) { throw new Error(`At most ${maxPeersPerTerminal} peers may attach to one terminal.`); } const mirrorFailure = entry.mirror.failure; if (mirrorFailure) { throw new AggregateError( [mirrorFailure], 'The terminal state is no longer being tracked; restart the terminal to view it.', ); } const peer: ITerminalPeerState = { peerId: peerIdArg, generation: ++this.peerGeneration, cursor: entry.output.end, detaching: false, finalizing: false, needsSnapshot: true, consecutiveSnapshots: 0, }; entry.peers.set(peerIdArg, peer); // Admission is complete here. The snapshot and the live stream that follows it run outside // the entry control queue, so input and resize from the freshly opened terminal are not // queued behind the delivery. Ordering is not weakened: delivery order is owned by the // per-peer single-flight deliveryPromise and the monotonic cursor, never by the control // queue — every steady-state delivery from appendOutput already runs outside it. void this.schedulePeerDelivery(entry, peer); }) )); } public async detach(peerIdArg: string, projectIdArg: string, terminalIdArg: string): Promise { const entry = this.entries.get(terminalIdArg); return this.runProjectOperation(projectIdArg, () => { if (!entry || entry.projectId !== projectIdArg) return Promise.resolve(); return this.runEntryControl(entry, async () => { const peer = entry.peers.get(peerIdArg); if (!peer) return; this.detachPeerState(entry, peer); if (peer.deliveryPromise) { await waitUntil( peer.deliveryPromise.catch(() => undefined), Date.now() + this.cleanupTimeoutMs, ).catch(() => undefined); } await this.applyMinimumSizeUntilStable(entry); }); }); } public async detachPeer(peerIdArg: string): Promise { const reconciliations: Promise[] = []; for (const entry of this.entries.values()) { const peer = entry.peers.get(peerIdArg); if (!peer) continue; this.detachPeerState(entry, peer); reconciliations.push(this.requestSizeReconciliation(entry)); } await Promise.allSettled(reconciliations); } public async input( peerIdArg: string, projectIdArg: string, terminalIdArg: string, dataArg: Buffer, ): Promise { if (dataArg.byteLength > maxInputBytes) { throw new Error('Terminal input exceeds the per-message limit.'); } const entry = this.requireEntry(projectIdArg, terminalIdArg); return this.runProjectOperation(projectIdArg, () => ( this.runEntryControl(entry, async () => { this.assertAttached(entry, peerIdArg); await entry.execution.sendInput(dataArg.toString('utf8')); }) )); } public async resize( peerIdArg: string, projectIdArg: string, terminalIdArg: string, rowsArg: number, colsArg: number, ): Promise { const entry = this.requireEntry(projectIdArg, terminalIdArg); return this.runProjectOperation(projectIdArg, () => ( this.runEntryControl(entry, async () => { this.assertAttached(entry, peerIdArg); entry.sizesByPeer.set(peerIdArg, { rows: rowsArg, cols: colsArg }); entry.sizeRevision += 1; await this.applyMinimumSizeUntilStable(entry); }) )); } public async retireProject( projectIdArg: string, retireArg: () => Promise, ): Promise { if (this.state !== 'open') throw new Error('The terminal manager is not accepting operations.'); const projectState = this.projectState(projectIdArg); if (projectState.retiring || projectState.retired) { throw new Error('The project is already leaving terminal admission.'); } projectState.retiring = true; const deadlineAt = Date.now() + this.cleanupTimeoutMs; const retirementPromise = (async () => { await this.closeProjectEntries(projectIdArg, deadlineAt); await waitUntil(this.waitForProjectDrain(projectState), deadlineAt); await this.closeProjectEntries(projectIdArg, deadlineAt); if ([...this.pendingRemovalCleanups.values()].includes(projectIdArg)) { await waitUntil(this.waitForRemovalCleanups(projectIdArg), deadlineAt); } const retired = await retireArg(); projectState.retired = retired; return retired; })(); projectState.retirementPromise = retirementPromise; this.activeRetirements.add(retirementPromise); try { return await retirementPromise; } finally { this.activeRetirements.delete(retirementPromise); if (projectState.retirementPromise === retirementPromise) { projectState.retirementPromise = undefined; } projectState.retiring = false; this.pruneProjectState(projectIdArg, projectState); } } public async closeAll(): Promise { if (this.state === 'closed') return; if (this.closePromise) return this.closePromise; this.state = 'closing'; const deadlineAt = Date.now() + this.cleanupTimeoutMs; const closePromise = (async () => { const initialClosures = [...this.entries.values()] .map((entry) => this.closeEntry(entry, deadlineAt)); const coordination = Promise.all([ Promise.allSettled(initialClosures).then(() => undefined), this.waitForAllProjectDrains(), Promise.allSettled([...this.activeRetirements]).then(() => undefined), ]).then(() => this.waitForRemovalCleanups()); let coordinationError: unknown; try { await waitUntil(coordination, deadlineAt); } catch (errorArg) { coordinationError = errorArg; } const finalOutcomes = await Promise.allSettled( [...this.entries.values()].map((entry) => this.closeEntry(entry, deadlineAt)), ); let cleanupError: unknown; if (this.pendingRemovalCleanups.size > 0) { try { await waitUntil(this.waitForRemovalCleanups(), deadlineAt); } catch (errorArg) { cleanupError = errorArg; if (this.entries.size === 0) this.pendingRemovalCleanups.clear(); } } const activeOperations = [...this.projectStates.values()] .reduce((sum, projectState) => sum + projectState.activeOperations, 0); const errors: unknown[] = []; if (this.entries.size > 0) { for (const entry of this.entries.values()) this.abandonEntry(entry); errors.push(new Error('One or more terminal roots remain unconfirmed.')); } if (activeOperations > 0) { errors.push(new Error('One or more terminal operations remain active.')); } if (this.activeRetirements.size > 0) { errors.push(new Error('One or more project retirements remain active.')); } if (this.pendingRemovalCleanups.size > 0) { errors.push(new Error('One or more terminal layout cleanups remain active.')); } if (errors.length > 0 && coordinationError) errors.push(coordinationError); if (errors.length > 0 && cleanupError) errors.push(cleanupError); if (this.entries.size > 0) { errors.push(...finalOutcomes.flatMap((outcome) => ( outcome.status === 'rejected' ? [outcome.reason] : [] ))); } if (errors.length > 0) { throw new AggregateError(errors, 'The terminal manager did not close every PTY.'); } this.state = 'closed'; })(); this.closePromise = closePromise; try { await closePromise; } finally { if (this.closePromise === closePromise) this.closePromise = undefined; } } private projectState(projectIdArg: string): IProjectAdmissionState { let state = this.projectStates.get(projectIdArg); if (!state) { const newState: IProjectAdmissionState = { activeOperations: 0, retiring: false, retired: false, }; this.projectStates.set(projectIdArg, newState); state = newState; } return state; } private pruneProjectState(projectIdArg: string, projectStateArg: IProjectAdmissionState): void { if ( this.projectStates.get(projectIdArg) === projectStateArg && projectStateArg.activeOperations === 0 && !projectStateArg.retiring && !projectStateArg.retirementPromise ) { this.projectStates.delete(projectIdArg); } } private async runProjectOperation( projectIdArg: string, operationArg: () => Promise, ): Promise { if (this.state !== 'open') throw new Error('The terminal manager is not accepting operations.'); const projectState = this.projectState(projectIdArg); if (projectState.retiring || projectState.retired) { throw new Error('The project is not accepting terminal operations.'); } if (projectState.activeOperations === 0) { let resolveDrain!: () => void; projectState.drainPromise = new Promise((resolve) => { resolveDrain = resolve; }); projectState.resolveDrain = resolveDrain; } projectState.activeOperations += 1; try { return await operationArg(); } finally { projectState.activeOperations -= 1; if (projectState.activeOperations === 0) { projectState.resolveDrain?.(); projectState.resolveDrain = undefined; projectState.drainPromise = undefined; this.pruneProjectState(projectIdArg, projectState); } } } private waitForProjectDrain(projectStateArg: IProjectAdmissionState): Promise { return projectStateArg.drainPromise ?? Promise.resolve(); } private async waitForAllProjectDrains(): Promise { await Promise.all([...this.projectStates.values()].map((state) => this.waitForProjectDrain(state))); } private async waitForRemovalCleanups(projectIdArg?: string): Promise { while (true) { const pending = [...this.pendingRemovalCleanups.entries()] .filter(([, projectId]) => projectIdArg === undefined || projectId === projectIdArg) .map(([cleanup]) => cleanup); if (pending.length === 0) return; await Promise.all(pending); } } private reserveCreate(projectIdArg: string): void { const activeForProject = [...this.entries.values()] .filter((entry) => entry.projectId === projectIdArg && entry.finalState !== 'exited').length; const pendingForProject = this.pendingCreatesByProject.get(projectIdArg) ?? 0; if (this.entries.size + this.pendingCreates >= maxTerminals) { throw new Error(`At most ${maxTerminals} controller terminals may run concurrently.`); } if (activeForProject + pendingForProject >= maxTerminalsPerProject) { throw new Error(`At most ${maxTerminalsPerProject} terminals may run in one project.`); } this.pendingCreates += 1; this.pendingCreatesByProject.set(projectIdArg, pendingForProject + 1); } private releaseCreate(projectIdArg: string): void { this.pendingCreates -= 1; const remaining = (this.pendingCreatesByProject.get(projectIdArg) ?? 1) - 1; if (remaining <= 0) this.pendingCreatesByProject.delete(projectIdArg); else this.pendingCreatesByProject.set(projectIdArg, remaining); } private createTerminalId(): string { for (let attempt = 0; attempt < 8; attempt += 1) { const id = `terminal_${plugins.crypto.randomBytes(16).toString('base64url')}`; if (!this.entries.has(id)) return id; } throw new Error('Unable to allocate a unique terminal identifier.'); } private requireShell(): IControllerTerminalShell { if (!this.shell || this.state !== 'open') throw new Error('The terminal manager is not initialized.'); return this.shell; } private requireEntry(projectIdArg: string, terminalIdArg: string): ITerminalEntry { const entry = this.entries.get(terminalIdArg); if (!entry || entry.projectId !== projectIdArg || entry.finalState === 'exited') { throw new Error('The requested terminal does not exist in this project.'); } return entry; } private assertEntryRunning(entryArg: ITerminalEntry): void { if (entryArg.closing || entryArg.finalState !== 'running') { throw new Error('The terminal is not accepting operations.'); } } private assertAttached(entryArg: ITerminalEntry, peerIdArg: string): ITerminalPeerState { this.assertEntryRunning(entryArg); const peer = entryArg.peers.get(peerIdArg); if (!peer || peer.detaching) throw new Error('The terminal is not attached by this peer.'); return peer; } private runEntryControl( entryArg: ITerminalEntry, operationArg: () => Promise, internalArg = false, ): Promise { if (!internalArg && entryArg.pendingControlOperations >= maxPendingControlOperations) { return Promise.reject(new Error('The terminal control queue is full.')); } if (!internalArg) entryArg.pendingControlOperations += 1; const operation = entryArg.controlQueue.then(async () => { if (entryArg.closing || entryArg.finalState !== 'running') { throw new Error('The terminal is not accepting operations.'); } return operationArg(); }); entryArg.controlQueue = operation.then(() => undefined, () => undefined); if (internalArg) return operation; return operation.finally(() => { entryArg.pendingControlOperations -= 1; }); } private appendOutput(entryArg: ITerminalEntry, chunkArg: Buffer): void { if (chunkArg.byteLength === 0) return; appendOutput(entryArg.output, chunkArg); if (entryArg.closing) return; for (const peer of entryArg.peers.values()) { if (!peer.deliveryPromise) void this.schedulePeerDelivery(entryArg, peer); } } private schedulePeerDelivery( entryArg: ITerminalEntry, peerArg: ITerminalPeerState, ): Promise { if (peerArg.detaching || peerArg.finalizing || entryArg.peers.get(peerArg.peerId) !== peerArg) { return Promise.resolve(); } if (peerArg.deliveryPromise) return peerArg.deliveryPromise; const delivery = this.deliverToPeer(entryArg, peerArg, entryArg.output.end); peerArg.deliveryPromise = delivery; // Only an exhausted delivery budget reaches this catch: deliverPeerFrame retries transient // failures, and the exit-finalization deadline path never routes through here. void delivery.catch(() => { if (entryArg.peers.get(peerArg.peerId) === peerArg) { this.detachPeerState(entryArg, peerArg); this.announcePeerDetached(entryArg, peerArg, 'delivery_failed'); void this.requestSizeReconciliation(entryArg); } }).finally(() => { if (peerArg.deliveryPromise === delivery) peerArg.deliveryPromise = undefined; if ( !peerArg.detaching && !peerArg.finalizing && entryArg.peers.get(peerArg.peerId) === peerArg && (peerArg.needsSnapshot || peerArg.cursor < entryArg.output.end) ) { void this.schedulePeerDelivery(entryArg, peerArg); } }); return delivery; } /** * One delivery pass: the reconstructed screen first when the peer owes one, then raw output up * to the end this pass was scheduled for. A snapshot already carries the stream up to at least * that end, so the range after it is usually empty and whatever arrived meanwhile is picked up * by the next pass. A pass that discovers the window moved past the cursor owes a snapshot * again and resolves it here rather than leaving it to the rescheduler: exit finalization calls * this directly and has no rescheduler, so a peer would otherwise be detached without its final * state and without `ended`. The repetition is bounded by the per-peer snapshot budget. */ private async deliverToPeer( entryArg: ITerminalEntry, peerArg: ITerminalPeerState, endArg: number, deadlineAtArg?: number, ): Promise { for (;;) { if (peerArg.needsSnapshot) { if (!await this.deliverSnapshot(entryArg, peerArg, deadlineAtArg)) return; } await this.deliverOutputRange(entryArg, peerArg, endArg, deadlineAtArg); if ( !peerArg.needsSnapshot || peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg ) return; } } /** * Serializes the mirror at the stream position it has parsed up to and sends it as the frames * that open the peer's stream. The peer's cursor moves to that position, so live output picks * up exactly where the reconstructed state ends — no gap, no repeated byte. * * Returns false when the peer went away, which leaves delivery to the caller's own teardown. A * mirror that is gone, or a peer that cannot be caught up within its snapshot budget, throws: * the delivery budget's own catch detaches the peer and announces it, where returning would * leave the peer owing a snapshot nobody can deliver. */ private async deliverSnapshot( entryArg: ITerminalEntry, peerArg: ITerminalPeerState, deadlineAtArg?: number, ): Promise { if (entryArg.mirror.disposed) { throw new Error('The terminal state is no longer being tracked.'); } peerArg.consecutiveSnapshots += 1; if (peerArg.consecutiveSnapshots > maxConsecutivePeerSnapshots) { throw new Error('The terminal state could not be handed over before the output moved on.'); } const request = entryArg.mirror.requestSnapshot({ // The first hand-over carries the reconstructed history; a repeat has to be small enough to // reach the live stream before the window moves again. screenOnly: peerArg.consecutiveSnapshots > 1, }); const snapshot = deadlineAtArg === undefined ? await request : await waitUntil(request, deadlineAtArg); if (peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg) return false; peerArg.cursor = snapshot.offset; peerArg.needsSnapshot = false; let sentBytes = 0; for (let index = 0; ; index += 1) { const frame = snapshot.data.subarray( sentBytes, Math.min(sentBytes + maxOutputFrameBytes, snapshot.data.byteLength), ); const last = sentBytes + frame.byteLength >= snapshot.data.byteLength; await this.deliverPeerFrame({ entry: entryArg, peer: peerArg, frame, offset: snapshot.offset, snapshot: { index, last, cols: snapshot.cols, rows: snapshot.rows }, deadlineAt: deadlineAtArg, }); if (peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg) return false; sentBytes += frame.byteLength; if (last) return true; } } private async deliverOutputRange( entryArg: ITerminalEntry, peerArg: ITerminalPeerState, endArg: number, deadlineAtArg?: number, ): Promise { while (peerArg.cursor < endArg) { if (peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg) return; if (peerArg.cursor < entryArg.output.start) { // The delivery window moved past this peer. Replaying what it missed would be both larger // and less accurate than the state those bytes produced, so it owes a snapshot instead. peerArg.needsSnapshot = true; return; } const frame = this.readOutputFrame(entryArg, peerArg.cursor, endArg); if (!frame || frame.byteLength === 0) { // The window holds nothing at a cursor that is inside it. Owing a snapshot is what makes // the next pass deliver the current state instead of reading empty forever. peerArg.needsSnapshot = true; return; } await this.deliverPeerFrame({ entry: entryArg, peer: peerArg, frame, offset: peerArg.cursor, deadlineAt: deadlineAtArg, }); if (peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg) return; peerArg.cursor += frame.byteLength; // Live output reached the peer, so the hand-over is complete and its budget is whole again. peerArg.consecutiveSnapshots = 0; } } /** * Delivers one frame against a bounded retry budget. Re-sending is idempotent at the client * because a raw frame carries its absolute offset, a snapshot frame carries its position inside * the snapshot — both documented on the protocol as the rule a client deduplicates by — and the * peer cursor only advances after a delivery resolves. When a deadline is supplied the caller is * exit finalization, which owns the deadline outright — retrying there would fight it, so a * single attempt is made. */ private async deliverPeerFrame(inputArg: { entry: ITerminalEntry; peer: ITerminalPeerState; frame: Buffer; offset: number; snapshot?: IControllerTerminalSnapshotFrame; deadlineAt?: number; }): Promise { const { entry, peer, deadlineAt } = inputArg; if (deadlineAt !== undefined) { await waitUntil(this.sendPeerOutput(inputArg), deadlineAt); return; } const wallDeadlineAt = Date.now() + maxFrameDeliveryWallMs; let attempt = 0; for (;;) { if (peer.detaching || entry.peers.get(peer.peerId) !== peer) return; try { await this.sendPeerOutput(inputArg); return; } catch (errorArg) { attempt += 1; if (peer.detaching || entry.peers.get(peer.peerId) !== peer) return; const backoffMs = this.outputRetryBackoffMs[ Math.min(attempt - 1, this.outputRetryBackoffMs.length - 1) ]; if (attempt >= maxFrameDeliveryAttempts || Date.now() + backoffMs >= wallDeadlineAt) { throw normalizeError(errorArg); } await delayUnreffed(backoffMs); } } } private announcePeerDetached( entryArg: ITerminalEntry, peerArg: ITerminalPeerState, reasonArg: TControllerTerminalDetachReason, ): void { this.options.onPeerDetached?.(peerArg.peerId, { ...entryArg.terminal.id }, reasonArg); } private readOutputFrame( entryArg: ITerminalEntry, startArg: number, endArg: number, ): Buffer | undefined { if (startArg >= endArg) return undefined; let slabStart = entryArg.output.start; for (const slab of entryArg.output.slabs) { const slabBytes = slab.end - slab.start; const slabEnd = slabStart + slabBytes; if (slabEnd <= startArg) { slabStart = slabEnd; continue; } if (slabStart >= endArg) break; const offset = Math.max(0, startArg - slabStart); const frameBytes = Math.min( maxOutputFrameBytes, slabBytes - offset, endArg - startArg, ); return slab.buffer.subarray(slab.start + offset, slab.start + offset + frameBytes); } return undefined; } private async sendPeerOutput(inputArg: { entry: ITerminalEntry; peer: ITerminalPeerState; frame: Buffer; offset: number; snapshot?: IControllerTerminalSnapshotFrame; }): Promise { const { entry, peer } = inputArg; if (peer.detaching || entry.peers.get(peer.peerId) !== peer) return; await this.options.sendOutput(peer.peerId, { terminalId: { ...entry.terminal.id }, offset: inputArg.offset, dataBase64: inputArg.frame.toString('base64'), ...(inputArg.snapshot ? { snapshot: { ...inputArg.snapshot } } : {}), }); } private detachPeerState(entryArg: ITerminalEntry, peerArg: ITerminalPeerState): void { peerArg.detaching = true; if (entryArg.peers.get(peerArg.peerId) === peerArg) { entryArg.peers.delete(peerArg.peerId); if (entryArg.sizesByPeer.delete(peerArg.peerId)) entryArg.sizeRevision += 1; } } private async applyMinimumSize(entryArg: ITerminalEntry): Promise { if (entryArg.sizesByPeer.size === 0 || entryArg.closing) return; let rows = Number.POSITIVE_INFINITY; let cols = Number.POSITIVE_INFINITY; for (const size of entryArg.sizesByPeer.values()) { rows = Math.min(rows, size.rows); cols = Math.min(cols, size.cols); } if (Number.isFinite(rows) && Number.isFinite(cols)) { await entryArg.execution.resize(cols, rows); // The stream carries no resize marker, so the mirror is told to change grid at the position // the PTY changed it: everything produced before stays parsed at the previous grid. entryArg.mirror.resize(cols, rows); } } private async applyMinimumSizeUntilStable(entryArg: ITerminalEntry): Promise { if (entryArg.sizeReconciliationActive || entryArg.closing) return; entryArg.sizeReconciliationActive = true; try { let appliedRevision: number; do { appliedRevision = entryArg.sizeRevision; await this.applyMinimumSize(entryArg); } while (!entryArg.closing && appliedRevision !== entryArg.sizeRevision); } finally { entryArg.sizeReconciliationActive = false; } } private requestSizeReconciliation(entryArg: ITerminalEntry): Promise { if (entryArg.closing || entryArg.sizeReconciliationActive) return Promise.resolve(); if (entryArg.sizeReconciliationPromise) return entryArg.sizeReconciliationPromise; const reconciliation = this.runEntryControl( entryArg, () => this.applyMinimumSizeUntilStable(entryArg), true, ); entryArg.sizeReconciliationPromise = reconciliation; void reconciliation.catch(() => undefined).finally(() => { if (entryArg.sizeReconciliationPromise === reconciliation) { entryArg.sizeReconciliationPromise = undefined; } }); return reconciliation; } private confirmRootExit(entryArg: ITerminalEntry, exitCodeArg?: number): void { if (entryArg.finalState === 'exited') return; entryArg.finalState = 'exited'; if (exitCodeArg !== undefined) entryArg.exitCode = exitCodeArg; entryArg.exitSubscription?.dispose(); entryArg.resolveRootExit(); entryArg.finalizePromise = this.finalizeExitedEntry( entryArg, entryArg.finalizationDeadlineAt ?? Date.now() + this.cleanupTimeoutMs, ); } private async finalizeExitedPeer( entryArg: ITerminalEntry, peerArg: ITerminalPeerState, deadlineAtArg: number, ): Promise { if (peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg) return; peerArg.finalizing = true; try { if (peerArg.deliveryPromise) { await waitUntil(peerArg.deliveryPromise, deadlineAtArg); } if (peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg) return; // Read once, after the in-flight delivery settled: node-pty reports a root's exit only after // it has closed the PTY, so every byte the PTY delivered is in the stream by now, and a fixed // end is what keeps `ended` the last frame this peer ever sees. const finalOutputEnd = entryArg.output.end; await this.deliverToPeer(entryArg, peerArg, finalOutputEnd, deadlineAtArg); if ( peerArg.detaching || entryArg.peers.get(peerArg.peerId) !== peerArg || peerArg.cursor < finalOutputEnd ) return; await waitUntil(this.options.sendOutput(peerArg.peerId, { terminalId: { ...entryArg.terminal.id }, offset: peerArg.cursor, dataBase64: '', ended: true, }), deadlineAtArg); } catch { // A stalled peer is detached without an ended frame; output can never race ended. } finally { this.detachPeerState(entryArg, peerArg); } } private async finalizeExitedEntry( entryArg: ITerminalEntry, deadlineAtArg: number, ): Promise { if (this.entries.get(entryArg.terminal.id.nativeId) !== entryArg) return; await Promise.allSettled([...entryArg.peers.values()].map((peer) => ( this.finalizeExitedPeer(entryArg, peer, deadlineAtArg) ))); this.entries.delete(entryArg.terminal.id.nativeId); entryArg.peers.clear(); entryArg.sizesByPeer.clear(); // The emulator outlives the root exactly as long as the peers that still need its state. entryArg.mirror.dispose(); const terminalId = { ...entryArg.terminal.id }; // A natural exit can be observed microseconds before closeEntry sets `closing`, so the // manager's own lifecycle state is consulted too: misreading a shutdown as a self-exit would // record a stopped intent and silently fail to bring an agent chat back. const reason: TControllerTerminalStopReason = entryArg.closing || this.state === 'closing' || this.state === 'closed' ? 'closed' : 'exited'; const removalCleanup = Promise.resolve() .then(() => this.options.onTerminalStopped( entryArg.projectId, terminalId, entryArg.exitCode, reason, )) .catch(() => undefined); this.pendingRemovalCleanups.set(removalCleanup, entryArg.projectId); try { await waitUntil(removalCleanup, deadlineAtArg); } catch { // Resource ownership is already released; startup pruning recovers a timed-out cleanup. } finally { this.pendingRemovalCleanups.delete(removalCleanup); } try { this.options.onTerminalsChanged(entryArg.projectId); } catch { // Root ownership is already released; a notification callback cannot restore it. } } /** * Gives up on an entry whose root could not be confirmed stopped within its deadline. Such an * entry can no longer serve anyone — admission is refused from the moment a close is signalled — * so its emulator is released rather than left parsing a pty that outlived its kill for the rest * of the controller's life. Peers are left in place: a root that does exit later still finalizes * them, and one that owes a snapshot is detached through the delivery budget instead of being * served a state nobody tracks any more. */ private abandonEntry(entryArg: ITerminalEntry): void { entryArg.mirror.dispose(); } /** * SIGTERM lets an agent root flush its transcript and run its own shutdown hooks, which is what * makes the conversation resumable rather than truncated. It is bounded so that a hung flush * cannot consume the shutdown deadline shared by every other close. */ private async terminateGracefully( entryArg: ITerminalEntry, deadlineAtArg: number, ): Promise { const remainingMs = deadlineAtArg - Date.now(); const graceMs = Math.max(0, Math.min(maxGracefulTerminationMs, Math.floor(remainingMs / 2))); if (graceMs === 0) { await entryArg.execution.kill(); return; } await entryArg.execution.terminate(); // The timer must never outlive the race it arbitrates: a root that exits promptly would // otherwise hold the event loop open for the rest of its grace, during the very shutdown the // upgrade is waiting on. const graceAbort = new AbortController(); const grace = plugins.timersPromises .setTimeout(graceMs, false, { signal: graceAbort.signal, ref: false }) .catch(() => false as const); try { const exitedInTime = await Promise.race([ entryArg.rootExitPromise.then(() => true), grace, ]); if (!exitedInTime) await entryArg.execution.kill(); } finally { graceAbort.abort(); } } private async closeEntry(entryArg: ITerminalEntry, deadlineAtArg: number): Promise { if (this.entries.get(entryArg.terminal.id.nativeId) !== entryArg) return; if (entryArg.finalState === 'exited') { await waitUntil(entryArg.finalizePromise ?? Promise.resolve(), deadlineAtArg); return; } if (!entryArg.closeLifecyclePromise) { entryArg.closing = true; entryArg.finalizationDeadlineAt = deadlineAtArg; // The PTY is handed back before the signal, never after it: node-pty closes a PTY shortly // after its child exits, loses whatever was unread then, and reports the exit only once it // has — so a pause that is still live when the root dies cannot be undone in time by anything // downstream of the exit. This is the one moment at which the loss is preventable, because it // is the process' own decision to stop the root, and from here on everything the root writes // is the final flush a graceful stop exists to capture. entryArg.mirror.releaseFlowControl(); const lifecycle = (async () => { await entryArg.rootExitPromise; await entryArg.finalizePromise; })(); entryArg.closeLifecyclePromise = lifecycle; try { const signal = entryArg.graceful ? this.terminateGracefully(entryArg, deadlineAtArg) : entryArg.execution.kill(); void signal.catch((errorArg) => { entryArg.signalError = normalizeError(errorArg); }); } catch (errorArg) { entryArg.signalError = normalizeError(errorArg); } } try { await waitUntil(entryArg.closeLifecyclePromise, deadlineAtArg); } catch (errorArg) { if (this.entries.get(entryArg.terminal.id.nativeId) !== entryArg) return; this.abandonEntry(entryArg); throw entryArg.signalError ? new AggregateError( [entryArg.signalError, errorArg], 'The terminal root could not be confirmed stopped.', ) : errorArg; } } private async closeProjectEntries(projectIdArg: string, deadlineAtArg: number): Promise { const outcomes = await Promise.allSettled( [...this.entries.values()] .filter((entry) => entry.projectId === projectIdArg) .map((entry) => this.closeEntry(entry, deadlineAtArg)), ); const errors = outcomes.flatMap((outcome) => ( outcome.status === 'rejected' ? [outcome.reason] : [] )); const remaining = [...this.entries.values()] .filter((entry) => entry.projectId === projectIdArg); if (remaining.length > 0) { for (const entry of remaining) this.abandonEntry(entry); errors.push(new Error('One or more project terminal roots remain unconfirmed.')); } if (errors.length > 0) { throw new AggregateError(errors, 'The project terminal roots could not all be confirmed stopped.'); } } }