import * as plugins from './plugins.js'; import { prepareContainerTerminalAssets } from './classes.terminalexecution.js'; export type TContainerEnvironmentContainer = Pick< plugins.docker.DockerContainer, | 'Id' | 'Name' | 'Names' | 'Labels' | 'start' | 'stop' | 'inspectState' | 'inspectRunState' | 'attach' | 'exec' | 'execInteractive' | 'resizeExec' | 'remove' >; /** The public Docker client satisfies this boundary; tests can supply a fake engine. */ export interface IContainerEnvironmentDockerHost { info(): Promise; pullImage( descriptorArg: plugins.docker.IImagePullDescriptor, ): Promise>; listContainers( optionsArg: plugins.docker.IContainerListOptions, ): Promise; createContainer( descriptorArg: plugins.docker.IContainerCreationDescriptor, ): Promise; openEventMonitor( optionsArg: plugins.docker.IDockerEventMonitorOptions, ): Promise; } /** Implemented by AGL's durable SmartData environment record, never by a process-local cache. */ export interface IContainerEnvironmentSetupReadiness { getReadyContainerId(): Promise; markReady(containerIdArg: string): Promise; } export type TContainerEnvironmentLossReason = 'container_lost' | 'environment_oom' | 'unexplained_exit_137'; /** The caller owns and closes the returned host after all environments have stopped. */ export const createControllerRootlessDockerHost = (): plugins.docker.DockerHost => { if (typeof process.getuid !== 'function') { throw new Error('Container environments require a Linux rootless Docker user daemon.'); } return new plugins.docker.DockerHost({ socketPath: `unix:///run/user/${process.getuid()}/docker.sock`, enableImageStore: false, }); }; export interface IContainerEnvironmentSetupStep { id: string; command: string; timeoutMs: number; } export interface IContainerEnvironmentOptions { docker: IContainerEnvironmentDockerHost; setupReadiness: IContainerEnvironmentSetupReadiness; /** Opaque and unique within this controller. Account layout belongs to the caller. */ id: string; name: string; labels: Record; imageReference: string; expectedRepoDigest: string; projectDirectory: string; stateDirectory: string; assetsDirectory: string; /** Other mounts are caller-owned; the project, state and assets mounts are always added here. */ additionalBindMounts?: plugins.docker.IContainerBindMount[]; networkEndpoints?: plugins.docker.IContainerNetworkEndpoint[]; namedVolumeMounts?: plugins.docker.IContainerNamedVolumeMount[]; memoryBytes?: number; nanoCpus?: number; pidsLimit?: number; shmSize?: number; lifelineTimeoutSeconds?: number; lifelineHeartbeatSeconds?: number; /** Runs after the first heartbeat and before setup or chats. */ onStartProbe?: ( containerArg: TContainerEnvironmentContainer, signalArg: AbortSignal, ) => Promise; /** The caller marks affected chats lost before the environment is started again. */ onLost?: (reasonArg: TContainerEnvironmentLossReason, errorArg: Error) => Promise; onFailure?: (errorArg: Error) => void; } export type TContainerEnvironmentState = | 'stopped' | 'starting' | 'running' | 'stopping' | 'failed' | 'closed'; export interface IContainerEnvironmentSnapshot { id: string; state: TContainerEnvironmentState; containerId?: string; /** False until the initial setup list has completed successfully. */ setupComplete: boolean; } /** Object identity fences terminal operations to one exact Docker container start. */ export interface IContainerEnvironmentRunLease { readonly containerId: string; readonly startedAtUnixNano: string; } const defaultMemoryBytes = 4 * 1024 ** 3; const defaultNanoCpus = 8_000_000_000; const defaultPidsLimit = 8192; const defaultShmSize = 512 * 1024 ** 2; const defaultLifelineTimeoutSeconds = 30; const defaultLifelineHeartbeatSeconds = 10; const lifelineScript = '#!/bin/bash\nwhile IFS= read -r -t "${1:?lifeline timeout required}" _; do :; done\n'; const exactDigest = /^.+@sha256:[0-9a-f]{64}$/u; const requireNonempty = (valueArg: string, nameArg: string): string => { if (typeof valueArg !== 'string' || valueArg.trim().length === 0) { throw new Error(`${nameArg} must be a nonempty string.`); } return valueArg; }; const requirePositiveInteger = (valueArg: number, nameArg: string): number => { if (!Number.isSafeInteger(valueArg) || valueArg <= 0) { throw new Error(`${nameArg} must be a positive safe integer.`); } return valueArg; }; const requireAbsoluteDirectory = async (directoryArg: string, nameArg: string): Promise => { if ( typeof directoryArg !== 'string' || !plugins.path.isAbsolute(directoryArg) || plugins.path.normalize(directoryArg) !== directoryArg ) { throw new Error(`${nameArg} must be an absolute normalized directory.`); } const stats = await plugins.fs.promises.stat(directoryArg); if (!stats.isDirectory()) throw new Error(`${nameArg} is not a directory.`); }; const asError = (reasonArg: unknown): Error => reasonArg instanceof Error ? reasonArg : new Error(String(reasonArg)); const classifyStoppedRun = (physicalArg?: plugins.docker.IContainerRuntimeState): TContainerEnvironmentLossReason => physicalArg?.OOMKilled ? 'environment_oom' : physicalArg?.ExitCode === 137 ? 'unexplained_exit_137' : 'container_lost'; const lossError = (reasonArg: TContainerEnvironmentLossReason): Error => new Error( reasonArg === 'environment_oom' ? 'Environment was killed by its memory limit.' : reasonArg === 'unexplained_exit_137' ? 'Environment exited with code 137; the cause is unknown.' : 'Environment stopped outside AGL.', ); /** * One installable rootless environment. The caller supplies its identity, durable setup list and * bind paths; this class owns only Docker's running process, attach and timer lifecycle. */ export class ControllerContainerEnvironment { private container?: TContainerEnvironmentContainer; private currentRunLease?: IContainerEnvironmentRunLease; private attachment?: Awaited>; private attachmentToken?: object; private attachmentHandlers?: { attachment: Awaited>; onEnd: () => void; onClose: () => void; onError: (errorArg: Error) => void; }; private eventMonitor?: plugins.docker.IDockerEventMonitor; private eventMonitorToken?: object; private heartbeatTimer?: ReturnType; private heartbeatWrite?: { token: object; cancel: (errorArg: Error) => void }; private startAbortController?: AbortController; private operationQueue: Promise = Promise.resolve(); private desiredRunning = false; private lifecycleIntentToken: object = {}; private closeRequested = false; private closePromise?: Promise; private lossQueued = false; private setupSteps: readonly IContainerEnvironmentSetupStep[] = []; private state: TContainerEnvironmentState = 'stopped'; private setupComplete = false; private readonly memoryBytes: number; private readonly lifelineTimeoutSeconds: number; private readonly lifelineHeartbeatSeconds: number; constructor(private readonly options: IContainerEnvironmentOptions) { requireNonempty(options.id, 'Environment ID'); requireNonempty(options.name, 'Container name'); if (!exactDigest.test(options.imageReference) || !exactDigest.test(options.expectedRepoDigest)) { throw new Error('Container image must use an exact repository digest.'); } if (options.imageReference !== options.expectedRepoDigest) { throw new Error('Container image reference must match its expected repository digest.'); } if (Object.keys(options.labels).length === 0) { throw new Error('Container ownership labels are required.'); } for (const [key, value] of Object.entries(options.labels)) { requireNonempty(key, 'Container label key'); requireNonempty(value, 'Container label value'); } this.memoryBytes = requirePositiveInteger(options.memoryBytes ?? defaultMemoryBytes, 'memoryBytes'); requirePositiveInteger(options.nanoCpus ?? defaultNanoCpus, 'nanoCpus'); requirePositiveInteger(options.pidsLimit ?? defaultPidsLimit, 'pidsLimit'); requirePositiveInteger(options.shmSize ?? defaultShmSize, 'shmSize'); this.lifelineTimeoutSeconds = requirePositiveInteger( options.lifelineTimeoutSeconds ?? defaultLifelineTimeoutSeconds, 'lifelineTimeoutSeconds', ); this.lifelineHeartbeatSeconds = requirePositiveInteger( options.lifelineHeartbeatSeconds ?? defaultLifelineHeartbeatSeconds, 'lifelineHeartbeatSeconds', ); if ( this.lifelineHeartbeatSeconds < 2 || this.lifelineTimeoutSeconds > 300 || this.lifelineTimeoutSeconds < 3 * this.lifelineHeartbeatSeconds ) { throw new Error('Lifeline requires interval >= 2 s and 3 × interval <= timeout <= 300 s.'); } } public snapshot(): IContainerEnvironmentSnapshot { return { id: this.options.id, state: this.state, ...(this.container ? { containerId: this.container.Id } : {}), setupComplete: this.setupComplete, }; } /** Existing runs are always stopped before attach: stdinOnce cannot be adopted after a crash. */ public async start(setupStepsArg: readonly IContainerEnvironmentSetupStep[]): Promise { this.assertNotClosed(); this.setupSteps = [...setupStepsArg]; const intentToken = this.requestLifecycleIntent(true); this.startAbortController?.abort(new Error('Environment start superseded by a newer start request.')); await this.enqueue(async () => { this.assertNotClosed(); if (this.lifecycleIntentToken !== intentToken) { throw new Error('Environment start was superseded by a newer lifecycle request.'); } if (this.state === 'running') { let physical: plugins.docker.IContainerRuntimeState | undefined; try { physical = await this.container?.inspectState(); } catch (errorArg) { await this.handleLoss('container_lost', asError(errorArg), intentToken, true); this.assertRequestedRun(intentToken, 'start'); return; } if (physical?.Running && !physical.OOMKilled) { this.assertRequestedRun(intentToken, 'start'); return; } const reason = classifyStoppedRun(physical); await this.handleLoss(reason, lossError(reason), intentToken, true); this.assertRequestedRun(intentToken, 'start'); return; } await this.startOnce(setupStepsArg); this.assertRequestedRun(intentToken, 'start'); }); } public async stop(): Promise { if (this.state === 'closed') return; this.requestLifecycleIntent(false); this.startAbortController?.abort(new Error('Environment stop requested.')); await this.enqueue(async () => { await this.stopOnce(); }); } /** Rebuild discards the writable layer; the caller's setup definition is replayed in order. */ public async rebuild(setupStepsArg: readonly IContainerEnvironmentSetupStep[]): Promise { this.assertNotClosed(); this.setupSteps = [...setupStepsArg]; const intentToken = this.requestLifecycleIntent(true); this.startAbortController?.abort(new Error('Environment rebuild requested.')); await this.enqueue(async () => { this.assertNotClosed(); if (this.lifecycleIntentToken !== intentToken) { throw new Error('Environment rebuild was superseded by a newer lifecycle request.'); } await this.stopOnce(); const container = await this.findOwnedContainer(); if (container) await container.remove({ force: false, removeAnonymousVolumes: true }); this.container = undefined; this.setupComplete = false; await this.startOnce(setupStepsArg); this.assertRequestedRun(intentToken, 'rebuild'); }); } /** Removes only Docker's container. Account folders and transcripts belong to the caller. */ public async delete(): Promise { this.requestLifecycleIntent(false); this.startAbortController?.abort(new Error('Environment delete requested.')); await this.enqueue(async () => { await this.stopOnce(); const container = await this.findOwnedContainer(); if (container) await container.remove({ force: false, removeAnonymousVolumes: true }); this.container = undefined; this.setupComplete = false; }); } public async close(): Promise { if (this.state === 'closed') return; if (this.closePromise) return this.closePromise; this.closeRequested = true; this.requestLifecycleIntent(false); this.startAbortController?.abort(new Error('Environment close requested.')); const closing = this.enqueue(async () => { await this.stopOnce(); this.state = 'closed'; }); this.closePromise = closing; let completed = false; try { await closing; completed = true; } finally { this.closePromise = undefined; if (!completed) this.closeRequested = false; } } /** Read exact Docker state; a missing container never counts as a healthy running environment. */ public async reconcile(): Promise { const observedIntentToken = this.lifecycleIntentToken; return this.enqueue(async () => { const container = await this.findOwnedContainer(); if (!container) { if (this.state === 'running') { this.container = undefined; await this.handleLoss('container_lost', new Error('Environment container disappeared.'), observedIntentToken); } return this.snapshot(); } this.container = container; let physical: plugins.docker.IContainerRuntimeState; try { physical = await container.inspectState(); } catch (errorArg) { if (this.state !== 'running') { await container.stop({ timeoutSeconds: 10, signal: 'SIGTERM' }); this.state = 'failed'; throw errorArg; } await this.handleLoss('container_lost', asError(errorArg), observedIntentToken); return this.snapshot(); } if (this.state !== 'running' && physical.Running) { // A new controller cannot adopt the old process's single-use stdin lifeline. await container.stop({ timeoutSeconds: 10, signal: 'SIGTERM' }); const reason = physical.OOMKilled ? 'environment_oom' : 'container_lost'; this.state = reason === 'environment_oom' ? 'failed' : 'stopped'; await this.options.onLost?.(reason, new Error('Prior environment run was stopped during reconciliation.')); return this.snapshot(); } if ((physical.OOMKilled || physical.ExitCode === 137) && this.state !== 'running' && this.state !== 'failed') { const reason = classifyStoppedRun(physical); if (this.lifecycleIntentToken === observedIntentToken) this.desiredRunning = false; this.state = 'failed'; await this.options.onLost?.(reason, lossError(reason)); return this.snapshot(); } if (this.state === 'running' && (physical.OOMKilled || !physical.Running)) { const reason = classifyStoppedRun(physical); await this.handleLoss(reason, lossError(reason), observedIntentToken); } return this.snapshot(); }); } /** Bounded administration and setup commands cannot silently become interactive sessions. */ public async exec( argvArg: plugins.docker.TContainerCommand, optionsArg: plugins.docker.IContainerExecOptions, ): Promise { if (this.state !== 'running' || !this.container) { throw new Error('Container environment is not running.'); } return this.container.exec(argvArg, optionsArg); } /** Caller owns the returned stream and must close it on chat exit. */ public async execInteractive( argvArg: plugins.docker.TContainerCommand, optionsArg: plugins.docker.IContainerInteractiveExecOptions, ): Promise { if (this.state !== 'running' || !this.container) { throw new Error('Container environment is not running.'); } return this.container.execInteractive(argvArg, optionsArg); } /** Capture the exact physical run together with a new terminal exec inside the lifecycle queue. */ public async execInteractiveWithRunLease( argvArg: plugins.docker.TContainerCommand, optionsArg: plugins.docker.IContainerInteractiveExecOptions, ): Promise<{ session: plugins.docker.IContainerInteractiveExec; runLease: IContainerEnvironmentRunLease }> { return this.enqueue(async () => { const container = this.container; const runLease = this.currentRunLease; if (this.state !== 'running' || !container || !runLease) { throw new Error('Container environment is not running.'); } const session = await container.execInteractive(argvArg, optionsArg); if (this.state !== 'running' || this.currentRunLease !== runLease) { await session.close(); throw new Error('Container run ended while starting its terminal.'); } return { session, runLease }; }); } /** A stale terminal cannot signal a process in the next run of the same container ID. */ public async execIfCurrentRun( runLeaseArg: IContainerEnvironmentRunLease, argvArg: plugins.docker.TContainerCommand, optionsArg: plugins.docker.IContainerExecOptions, ): Promise { if (this.currentRunLease !== runLeaseArg) return undefined; return this.enqueue(async () => { if (this.state !== 'running' || this.currentRunLease !== runLeaseArg || !this.container) { return undefined; } return this.container.exec(argvArg, optionsArg); }); } public async resizeExecIfCurrentRun( runLeaseArg: IContainerEnvironmentRunLease, execIdArg: string, rowsArg: number, colsArg: number, ): Promise { if (this.currentRunLease !== runLeaseArg) return false; return this.enqueue(async () => { if (this.state !== 'running' || this.currentRunLease !== runLeaseArg || !this.container) { return false; } await this.container.resizeExec(execIdArg, rowsArg, colsArg); return true; }); } /** The lease comparison and stop occur in one serialized lifecycle operation. */ public async stopIfCurrentRun(runLeaseArg: IContainerEnvironmentRunLease): Promise { if (this.currentRunLease !== runLeaseArg) return false; const observedIntentToken = this.lifecycleIntentToken; return this.enqueue(async () => { if (this.state !== 'running' || this.currentRunLease !== runLeaseArg) return false; if (this.lifecycleIntentToken === observedIntentToken) this.desiredRunning = false; await this.stopOnce(); return true; }); } public async resizeExec(execIdArg: string, rowsArg: number, colsArg: number): Promise { if (this.state !== 'running' || !this.container) { throw new Error('Container environment is not running.'); } await this.container.resizeExec(execIdArg, rowsArg, colsArg); } private assertNotClosed(): void { if (this.closeRequested || this.state === 'closed') throw new Error('Container environment is closed.'); } private requestLifecycleIntent(desiredRunningArg: boolean): object { const token = {}; this.lifecycleIntentToken = token; this.desiredRunning = desiredRunningArg; return token; } private assertRequestedRun(intentTokenArg: object, actionArg: 'start' | 'rebuild'): void { if (this.lifecycleIntentToken !== intentTokenArg) { throw new Error(`Environment ${actionArg} was superseded by a newer lifecycle request.`); } if (this.state !== 'running' || !this.currentRunLease) { throw new Error(`Environment ${actionArg} did not establish a running run.`); } } private enqueue(actionArg: () => Promise): Promise { const result = this.operationQueue.then(actionArg); this.operationQueue = result.then(() => undefined, () => undefined); return result; } private async prepareAssets(): Promise { await requireAbsoluteDirectory(this.options.projectDirectory, 'Project directory'); await plugins.fs.promises.mkdir(this.options.stateDirectory, { recursive: true, mode: 0o700 }); await plugins.fs.promises.mkdir(this.options.assetsDirectory, { recursive: true, mode: 0o700 }); await requireAbsoluteDirectory(this.options.stateDirectory, 'Environment state directory'); await requireAbsoluteDirectory(this.options.assetsDirectory, 'Environment assets directory'); const lifelinePath = plugins.path.join(this.options.assetsDirectory, 'lifeline'); const temporaryPath = `${lifelinePath}.${process.pid}.${plugins.crypto.randomUUID()}.tmp`; try { await plugins.fs.promises.writeFile(temporaryPath, lifelineScript, { mode: 0o755, flag: 'wx' }); await plugins.fs.promises.rename(temporaryPath, lifelinePath); } finally { await plugins.fs.promises.rm(temporaryPath, { force: true }); } await prepareContainerTerminalAssets(this.options.assetsDirectory); } private async findOwnedContainer(): Promise { const candidates = await this.options.docker.listContainers({ all: true, filters: { name: [this.options.name] }, }); const exactName = `/${this.options.name}`; const exact = candidates.filter((candidate) => ( candidate.Name === exactName || candidate.Names?.includes(exactName) )); if (exact.length > 1) throw new Error('More than one Docker container has the environment name.'); const container = exact[0]; if (!container) return undefined; for (const [key, value] of Object.entries(this.options.labels)) { if (container.Labels?.[key] !== value) { throw new Error(`Container ${this.options.name} is not owned by this environment.`); } } return container; } private async resolveContainer(): Promise<{ container: TContainerEnvironmentContainer; created: boolean }> { const existing = await this.findOwnedContainer(); if (existing) return { container: existing, created: false }; const image = await this.options.docker.pullImage({ reference: this.options.imageReference, expectedRepoDigest: this.options.expectedRepoDigest, }); if (image.VerifiedRepoDigest !== this.options.expectedRepoDigest) { throw new Error('Pulled image lacks the requested immutable digest proof.'); } const container = await this.options.docker.createContainer({ name: this.options.name, imageId: image.Id, imageReference: this.options.expectedRepoDigest, user: 'root', allowRootOnRootless: true, entrypoint: ['/opt/agl/lifeline'], command: [String(this.lifelineTimeoutSeconds)], workingDirectory: this.options.projectDirectory, tty: false, openStdin: true, stdinOnce: true, init: true, stopSignal: 'SIGTERM', stopTimeout: 10, memoryBytes: this.memoryBytes, memorySwapBytes: this.memoryBytes, nanoCpus: this.options.nanoCpus ?? defaultNanoCpus, pidsLimit: this.options.pidsLimit ?? defaultPidsLimit, shmSize: this.options.shmSize ?? defaultShmSize, logDriver: 'none', labels: { ...this.options.labels }, bindMounts: [ { source: this.options.projectDirectory, target: this.options.projectDirectory }, { source: this.options.stateDirectory, target: '/opt/agl-state' }, { source: this.options.assetsDirectory, target: '/opt/agl', readOnly: true }, ...(this.options.additionalBindMounts ?? []), ], ...(this.options.networkEndpoints ? { networkEndpoints: this.options.networkEndpoints } : {}), ...(this.options.namedVolumeMounts ? { namedVolumeMounts: this.options.namedVolumeMounts } : {}), }); return { container, created: true }; } private async startOnce(setupStepsArg: readonly IContainerEnvironmentSetupStep[]): Promise { this.state = 'starting'; const abortController = new AbortController(); this.startAbortController = abortController; try { const info = await this.options.docker.info(); if (!info.SecurityOptions.includes('name=rootless')) { throw new Error('Docker daemon is not rootless.'); } await this.prepareAssets(); const resolved = await this.resolveContainer(); const container = resolved.container; this.container = container; if (!resolved.created) { // A previously attached stdinOnce stream cannot be taken over, even if still running. const physical = await container.inspectState(); if (physical.Running) await container.stop({ timeoutSeconds: 10, signal: 'SIGTERM' }); const readyContainerId = await this.options.setupReadiness.getReadyContainerId(); if (readyContainerId !== container.Id) { throw new Error('Environment setup is incomplete or unproven; rebuild explicitly before starting.'); } this.setupComplete = true; } if (!this.desiredRunning) return; const publishRunStart = await this.watchContainerEvents(container, abortController.signal); abortController.signal.throwIfAborted(); await container.start(); const physicalRun = await container.inspectRunState(); publishRunStart(physicalRun.startedAtUnixNano); abortController.signal.throwIfAborted(); if (!physicalRun.Running || physicalRun.OOMKilled) { throw new Error('Container stopped before its lifeline could be attached.'); } const attachment = await container.attach({ stdin: true, stdout: false, stderr: false, stream: true, timeoutMs: 30_000, }); this.attachment = attachment; const attachmentToken = {}; this.attachmentToken = attachmentToken; this.bindAttachment(attachment, attachmentToken); await this.writeHeartbeat(attachment, attachmentToken); this.heartbeatTimer = setInterval(() => { if (this.heartbeatWrite?.token === attachmentToken) { this.attachmentLost(attachment, attachmentToken, new Error('Container lifeline heartbeat write stalled.')); return; } void this.writeHeartbeat(attachment, attachmentToken).catch((errorArg) => { this.attachmentLost(attachment, attachmentToken, asError(errorArg)); }); }, this.lifelineHeartbeatSeconds * 1000); this.heartbeatTimer.unref(); abortController.signal.throwIfAborted(); await this.options.onStartProbe?.(container, abortController.signal); abortController.signal.throwIfAborted(); if (resolved.created) { await this.runSetupSteps(container, setupStepsArg, abortController.signal); abortController.signal.throwIfAborted(); await this.options.setupReadiness.markReady(container.Id); if (await this.options.setupReadiness.getReadyContainerId() !== container.Id) { throw new Error('Environment setup readiness was not durably recorded.'); } } abortController.signal.throwIfAborted(); this.setupComplete = true; this.currentRunLease = Object.freeze({ containerId: container.Id, startedAtUnixNano: physicalRun.startedAtUnixNano, }); this.state = 'running'; } catch (errorArg) { this.state = 'failed'; await this.stopContainerAfterFailure(); throw errorArg; } finally { if (this.startAbortController === abortController) this.startAbortController = undefined; } } private async runSetupSteps( containerArg: TContainerEnvironmentContainer, stepsArg: readonly IContainerEnvironmentSetupStep[], signalArg: AbortSignal, ): Promise { for (const step of stepsArg) { signalArg.throwIfAborted(); requireNonempty(step.id, 'Setup step ID'); requireNonempty(step.command, 'Setup command'); requirePositiveInteger(step.timeoutMs, 'Setup timeoutMs'); const result = await containerArg.exec(['/bin/bash', '-lc', step.command], { workingDirectory: this.options.projectDirectory, timeoutMs: step.timeoutMs, maxOutputBytes: 1024 * 1024, }); signalArg.throwIfAborted(); if (result.exitCode !== 0) { throw new Error(`Environment setup step ${step.id} failed with exit code ${result.exitCode}.`); } } } private bindAttachment( attachmentArg: Awaited>, tokenArg: object, ): void { const handlers = { attachment: attachmentArg, onEnd: () => this.attachmentLost(attachmentArg, tokenArg, new Error('Container lifeline stream ended.')), onClose: () => this.attachmentLost(attachmentArg, tokenArg, new Error('Container lifeline stream closed.')), onError: (errorArg: Error) => this.attachmentLost(attachmentArg, tokenArg, errorArg), }; this.attachmentHandlers = handlers; attachmentArg.stream.once('end', handlers.onEnd); attachmentArg.stream.once('close', handlers.onClose); attachmentArg.stream.once('error', handlers.onError); } private unbindAttachment( attachmentArg: Awaited>, ): void { const handlers = this.attachmentHandlers; if (!handlers || handlers.attachment !== attachmentArg) return; this.attachmentHandlers = undefined; attachmentArg.stream.off('end', handlers.onEnd); attachmentArg.stream.off('close', handlers.onClose); attachmentArg.stream.off('error', handlers.onError); } private attachmentLost( attachmentArg: Awaited>, tokenArg: object, errorArg: Error, ): void { if (this.attachment !== attachmentArg || this.attachmentToken !== tokenArg) return; this.startAbortController?.abort(errorArg); this.queueLoss('container_lost', errorArg); } private queueLoss(reasonArg: TContainerEnvironmentLossReason, errorArg: Error): void { if (!this.desiredRunning || this.state !== 'running' || this.lossQueued) return; const observedIntentToken = this.lifecycleIntentToken; this.lossQueued = true; this.state = 'failed'; void this.enqueue(async () => { try { await this.handleLoss(reasonArg, errorArg, observedIntentToken); } finally { this.lossQueued = false; } }).catch((failureArg) => { this.state = 'failed'; this.options.onFailure?.(asError(failureArg)); }); } private async handleLoss( reasonArg: TContainerEnvironmentLossReason, errorArg: Error, observedIntentTokenArg: object, explicitRestartArg = false, ): Promise { const requiresExplicitRestart = reasonArg === 'environment_oom' || reasonArg === 'unexplained_exit_137'; if (requiresExplicitRestart && !explicitRestartArg && this.lifecycleIntentToken === observedIntentTokenArg) { this.desiredRunning = false; } await this.stopOnce(); await this.options.onLost?.(reasonArg, errorArg); if (requiresExplicitRestart && !explicitRestartArg) { this.state = 'failed'; } else if (this.lifecycleIntentToken === observedIntentTokenArg && this.desiredRunning) { await this.startOnce(this.setupSteps); } } private async watchContainerEvents( containerArg: TContainerEnvironmentContainer, signalArg: AbortSignal, ): Promise<(startedAtUnixNanoArg: string) => void> { const token = {}; this.eventMonitorToken = token; let startedAtUnixNano: bigint | undefined; const pendingEvents: plugins.docker.IContainerTimedDockerEvent[] = []; const loseMonitor = (errorArg: Error): void => { if (this.eventMonitorToken !== token || signalArg.aborted) return; this.eventMonitor?.close(); this.startAbortController?.abort(errorArg); this.queueLoss('container_lost', errorArg); }; const handleEvent = (eventArg: plugins.docker.IContainerTimedDockerEvent): void => { if (this.eventMonitorToken !== token || signalArg.aborted) return; if (startedAtUnixNano === undefined) { if (pendingEvents.length >= 128) { loseMonitor(new Error('Docker event monitor exceeded its startup event buffer.')); } else { pendingEvents.push(eventArg); } return; } if (BigInt(eventArg.timeNanoExact) < startedAtUnixNano) return; if (eventArg.Action === 'oom') { const error = new Error('Environment or a child process exceeded its memory limit.'); this.startAbortController?.abort(error); this.queueLoss('environment_oom', error); return; } if (!['die', 'destroy', 'kill'].includes(eventArg.Action)) return; void containerArg.inspectState().then((physical) => { if (this.eventMonitorToken !== token || signalArg.aborted) return; if (!physical.Running || physical.OOMKilled) { // A prior Docker signal does not prove what later caused exit 137. const reason = classifyStoppedRun(physical); this.queueLoss(reason, lossError(reason)); } }).catch((errorArg) => { if (this.eventMonitorToken === token && !signalArg.aborted) { this.startAbortController?.abort(asError(errorArg)); this.queueLoss('container_lost', asError(errorArg)); } }); }; const monitor = await this.options.docker.openEventMonitor({ filters: { type: ['container'], event: ['die', 'destroy', 'kill', 'oom'], label: Object.entries(this.options.labels).map(([key, value]) => `${key}=${value}`), }, signal: signalArg, onEvent: (rawEventArg) => { if (this.eventMonitorToken !== token || signalArg.aborted) return; let event: plugins.docker.IContainerTimedDockerEvent | undefined; try { event = plugins.docker.parseDockerTimedContainerEvent(rawEventArg); } catch (errorArg) { loseMonitor(asError(errorArg)); return; } if (event?.Actor.ID === containerArg.Id) handleEvent(event); }, onLost: loseMonitor, }); this.eventMonitor = monitor; if (monitor.status !== 'connected') throw new Error('Docker event monitor was lost before startup.'); return (startedAtUnixNanoArg) => { startedAtUnixNano = BigInt(startedAtUnixNanoArg); for (const event of pendingEvents.splice(0)) handleEvent(event); }; } private async writeHeartbeat( attachmentArg: Awaited>, tokenArg: object, ): Promise { if ( attachmentArg !== this.attachment || tokenArg !== this.attachmentToken || attachmentArg.stream.destroyed ) { throw new Error('Container lifeline is no longer attached.'); } if (this.heartbeatWrite?.token === tokenArg) { throw new Error('Container lifeline heartbeat write is already pending.'); } await new Promise((resolve, reject) => { let settled = false; const finish = (errorArg?: Error): void => { if (settled) return; settled = true; clearTimeout(timeout); if (this.heartbeatWrite?.token === tokenArg) this.heartbeatWrite = undefined; if (errorArg) reject(errorArg); else resolve(); }; const timeout = setTimeout(() => { const error = new Error('Container lifeline heartbeat write timed out.'); if (this.attachment === attachmentArg && this.attachmentToken === tokenArg) { attachmentArg.stream.destroy(error); } finish(error); }, this.lifelineHeartbeatSeconds * 1000); timeout.unref(); this.heartbeatWrite = { token: tokenArg, cancel: (errorArg) => finish(errorArg) }; try { attachmentArg.stream.write('\n', (errorArg) => { if (this.attachment !== attachmentArg || this.attachmentToken !== tokenArg) { finish(new Error('Container lifeline was detached during heartbeat write.')); } else { finish(errorArg ?? undefined); } }); } catch (errorArg) { finish(asError(errorArg)); } }); } private async detachLifeline(): Promise { if (this.heartbeatTimer) clearInterval(this.heartbeatTimer); this.heartbeatTimer = undefined; this.currentRunLease = undefined; this.eventMonitorToken = undefined; this.eventMonitor?.close(); this.eventMonitor = undefined; this.startAbortController?.abort(new Error('Environment is stopping.')); const attachment = this.attachment; this.attachment = undefined; const attachmentToken = this.attachmentToken; this.attachmentToken = undefined; if (attachmentToken && this.heartbeatWrite?.token === attachmentToken) { this.heartbeatWrite.cancel(new Error('Container lifeline was detached during heartbeat write.')); } if (!attachment) return; this.unbindAttachment(attachment); await attachment.close(); } private async stopContainerAfterFailure(): Promise { const errors: unknown[] = []; try { await this.detachLifeline(); } catch (errorArg) { errors.push(errorArg); } if (this.container) { try { await this.container.stop({ timeoutSeconds: 10, signal: 'SIGTERM' }); } catch (errorArg) { errors.push(errorArg); } } if (errors.length > 0) throw new AggregateError(errors, 'Container failure cleanup was incomplete.'); } private async stopOnce(): Promise { if (this.state === 'closed') return; this.state = 'stopping'; const errors: unknown[] = []; try { await this.detachLifeline(); } catch (errorArg) { errors.push(errorArg); } if (this.container) { try { await this.container.stop({ timeoutSeconds: 10, signal: 'SIGTERM' }); } catch (errorArg) { errors.push(errorArg); } } this.state = errors.length === 0 ? 'stopped' : 'failed'; if (errors.length > 0) throw new AggregateError(errors, 'Container stop was incomplete.'); } }