import * as plugins from './plugins.js'; import { controllerPackageName, type IControllerRuntimeId, type IControllerUpgradeStatus, } from '../ts_interfaces/index.js'; import { processIdentityHasCliCommand, readControllerProcessIdentity, readProcessGroupMemberPids, } from './classes.processinspection.js'; import { resolveAGLHomePaths } from './classes.aglhome.js'; import { upgradeCoordinationRootEnvironmentVariable, upgradeTokenEnvironmentVariable, } from './constants.upgradeenvironment.js'; export const upgradeCoordinationVersion = 2 as const; const coordinationVersion = upgradeCoordinationVersion; export const upgradePackageTransitionTransactionVersion = 3 as const; export const upgradePackageTransitionSource = { packageName: '@modelprofile.com/harness-controller', version: '20.0.3', managementVersion: 1, cliName: 'hcon', cliRelativePath: './cli.js', } as const; export const upgradePackageTransitionTarget = { packageName: 'agl', version: '21.0.0', managementVersion: 2, cliName: 'agl', cliRelativePath: './cli.js', } as const; const currentCliName = String(controllerPackageName) === upgradePackageTransitionSource.packageName ? upgradePackageTransitionSource.cliName : upgradePackageTransitionTarget.cliName; const ownerFileName = 'owner.json'; const maximumMetadataBytes = 256 * 1024; const initializationGraceMs = 30_000; const startLeaseDrainTimeoutMs = 60_000; const upgradeTokenPattern = /^[A-Za-z0-9_-]{43}$/; const upgradeTokenHashPattern = /^[a-f0-9]{64}$/; const upgradeSemverPattern = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/; const leaseDirectoryPattern = /^[a-f0-9]{20}$/; const tokenMetadataPattern = /^(?:grant|ack|action-(?:prepare|finalize))-[a-f0-9]{20}\.json(?:\.consuming-[1-9][0-9]*-[a-f0-9]{8})?$/; const acknowledgementTemporaryPattern = /^ack-[a-f0-9]{20}\.json\.tmp-[1-9][0-9]*-[a-f0-9]{8}$/; const ownerTransferTemporaryPattern = /^owner-transfer-[1-9][0-9]*-[a-f0-9]{8}\.json\.tmp$/; const transactionMetadataPattern = /^transaction-[a-f0-9]{20}\.json$/; const transactionTemporaryMetadataPattern = /^transaction-[a-f0-9]{20}\.json\.tmp-[1-9][0-9]*-[a-f0-9]{8}$/; const transactionMutationDirectoryPattern = /^transaction-[a-f0-9]{20}\.json\.lock$/; const transactionAdoptionPattern = /^transaction-([a-f0-9]{20})\.json\.adopting-([a-f0-9]{20})$/; const transactionAdoptionAuditPattern = /^adoption-([a-f0-9]{20})-([a-f0-9]{20})\.json$/; const transactionAdoptionAuditTemporaryPattern = /^adoption-[a-f0-9]{20}-[a-f0-9]{20}\.json\.tmp-[1-9][0-9]*-[a-f0-9]{8}$/; const ownershipGuardDirectoryPattern = /^[a-f0-9]{64}\.guard$/; const ownershipGuardTemporaryPattern = /^[a-f0-9]{64}\.guard\.tmp-[1-9][0-9]*-[a-f0-9]{32}$/; const ownerRemovalPattern = /^\.owner-removing-([1-9][0-9]*)-([a-f0-9]{64})-([a-f0-9]{16})$/; const legacyOwnerRemovalPattern = /^\.owner-removing-([1-9][0-9]*)-([a-f0-9]{16})$/; const tokenMetadataRetentionMs = 10 * 60 * 1000; const transactionRetentionMs = 24 * 60 * 60 * 1000; const terminalStatusRetentionMs = 10 * 60 * 1000; const transactionMutationTimeoutMs = 10_000; const ownershipGuardTimeoutMs = 10_000; const legacyOwnerRemovalGraceMs = 30_000; const workerTerminationGraceMs = 5_000; type TUpgradeOwnerKind = 'upgrade' | 'start'; interface IUpgradeSemver { major: string; minor: string; patch: string; prerelease: string[]; } interface IUpgradeProcessOwner { version: typeof coordinationVersion; kind: TUpgradeOwnerKind; uid: number; tokenHash: string; pid: number; processGroupId: number; fingerprint: string; cliPath: string; command: '__serve' | '__upgrade-worker' | 'upgrade' | 'start' | 'foreground' | 'temp-password'; createdAt: number; } export interface IUpgradeExpectedController { pid: number; processGroupId: number; processFingerprint: string; } interface IUpgradeLaunchGrant { version: typeof coordinationVersion; tokenHash: string; expiresAt: number; port: number; packageName: string; packageVersion: string; cliPath: string; controller: IUpgradeExpectedController; } interface IUpgradeWorkerAcknowledgement { version: typeof coordinationVersion; tokenHash: string; pid: number; processGroupId: number; fingerprint: string; } export interface IUpgradeWorkerPayload { version: typeof coordinationVersion; token: string; port: number; gracePeriodMs: number; continueSessions: boolean; expectedController?: IUpgradeExpectedController; } export type TUpgradeSessionTransitionState = | 'pending' | 'submitting' | 'accepted' | 'paused' | 'outcomeUnknown' | 'failed'; export interface IUpgradeSessionState { projectId: string; sessionId: IControllerRuntimeId; pauseState: TUpgradeSessionTransitionState; pauseQueueId?: string; cleanupState?: TUpgradeSessionTransitionState; continueState?: TUpgradeSessionTransitionState; error?: string; } interface IUpgradeTransactionBase { tokenHash: string; revision: number; port: number; controllerWasRunning: boolean; continueSessions: boolean; gracePeriodMs: number; preparationAcceptedAt?: number; preparationDeadlineAt?: number; preparationCompletedAt?: number; phase: IControllerUpgradeStatus['phase'] | 'starting' | 'checking' | 'stopping'; message: string; createdAt: number; phaseStartedAt: number; updatedAt: number; retainedUntil?: number; worker?: { pid: number; processGroupId: number; fingerprint: string; cliPath: string; logFilePath?: string; }; controller?: { pid: number; processGroupId: number; fingerprint: string; cliPath: string; command: '__serve' | 'foreground'; }; sessions: IUpgradeSessionState[]; terminal?: { success: boolean; error?: string; }; } export interface IUpgradeTransactionV2 extends IUpgradeTransactionBase { version: typeof coordinationVersion; sourceVersion: string; targetVersion?: string; registryUrl?: string; targetStartupInvoked?: true; } export interface IUpgradeTransactionV3 extends IUpgradeTransactionBase { version: typeof upgradePackageTransitionTransactionVersion; sourcePackageName: typeof upgradePackageTransitionSource.packageName; sourceVersion: typeof upgradePackageTransitionSource.version; sourceManagementVersion: typeof upgradePackageTransitionSource.managementVersion; sourceCliName: typeof upgradePackageTransitionSource.cliName; sourceCliRelativePath: typeof upgradePackageTransitionSource.cliRelativePath; targetPackageName: typeof upgradePackageTransitionTarget.packageName; targetVersion: typeof upgradePackageTransitionTarget.version; targetManagementVersion: typeof upgradePackageTransitionTarget.managementVersion; targetCliName: typeof upgradePackageTransitionTarget.cliName; targetCliRelativePath: typeof upgradePackageTransitionTarget.cliRelativePath; recoveryTargetVersion?: string; packageTransitionStarted?: true; groupedTargetVerified?: true; targetPackageCommitStarted?: true; targetPackageCommitted?: true; targetStartupInvoked?: true; } export type TUpgradeTransaction = IUpgradeTransactionV2 | IUpgradeTransactionV3; export interface IUpgradeTransactionInventoryEntry { fileName: string; state: 'active' | 'adopting'; transaction: TUpgradeTransaction; } export interface IUpgradeTransactionInventory { transactions: IUpgradeTransactionInventoryEntry[]; tokenMetadataFileNames: string[]; mutationDirectoryNames: string[]; } export interface IAdoptOrphanedUpgradeTransactionOptions { lock: UpgradeInstallationLock; token: string; expectedTokenHash: string; expectedRevision: number; port: number; installedVersion: string; registryUrl?: string; } interface IUpgradeTransactionAdoptionAudit { version: 1; adoptedAt: number; adoptedTargetVersion: string; previousTransaction: TUpgradeTransaction; } export type TUpgradeControllerAction = 'prepare' | 'finalize'; interface IUpgradeControllerActionGrant { version: typeof coordinationVersion; tokenHash: string; action: TUpgradeControllerAction; mode?: 'continue' | 'compensate' | 'reopen'; expiresAt: number; packageVersion: string; controller: IUpgradeExpectedController; } interface IUpgradeTransactionMutationOwner { version: typeof coordinationVersion; uid: number; tokenHash: string; pid: number; fingerprint: string; nonce: string; createdAt: number; } interface IUpgradeOwnershipMutationGuardOwner { version: typeof coordinationVersion; uid: number; targetHash: string; pid: number; fingerprint: string; nonce: string; createdAt: number; } interface IUpgradeOwnershipMutationGuardState { state: 'absent' | 'empty' | 'live' | 'stale'; identity?: { dev: number; ino: number }; owner?: IUpgradeOwnershipMutationGuardOwner; ownerMetadata?: IOwnedDirectoryOwnerMetadata; } const assertUid = (): number => { if (typeof process.getuid !== 'function') { throw new Error(`${currentCliName} upgrade requires a POSIX effective user identity.`); } const uid = process.getuid(); if (!Number.isSafeInteger(uid) || uid < 0) { throw new Error('The effective user identity is invalid.'); } return uid; }; const assertToken = (tokenArg: unknown): string => { if (typeof tokenArg !== 'string' || !upgradeTokenPattern.test(tokenArg)) { throw new Error('The upgrade coordination token is invalid.'); } return tokenArg; }; const hashToken = (tokenArg: unknown): string => plugins.crypto .createHash('sha256') .update(assertToken(tokenArg), 'utf8') .digest('hex'); const tokensEqual = (leftArg: string, rightArg: string): boolean => { const left = Buffer.from(leftArg, 'utf8'); const right = Buffer.from(rightArg, 'utf8'); return left.byteLength === right.byteLength && plugins.crypto.timingSafeEqual(left, right); }; const assertPort = (portArg: unknown): number => { if (!Number.isSafeInteger(portArg) || (portArg as number) < 1 || (portArg as number) > 65_535) { throw new Error('The upgrade coordination port is invalid.'); } return portArg as number; }; const exactKeys = (valueArg: Record, expectedArg: string[]): boolean => { const actual = Object.keys(valueArg).sort(); const expected = [...expectedArg].sort(); return actual.length === expected.length && actual.every((key, index) => key === expected[index]); }; const assertPrivateDirectory = async (directoryArg: string): Promise => { const stats = await plugins.fs.promises.lstat(directoryArg); if (!stats.isDirectory() || stats.isSymbolicLink()) { throw new Error(`Upgrade coordination path must be a real directory: ${directoryArg}`); } if (typeof process.getuid === 'function' && stats.uid !== process.getuid()) { throw new Error(`Upgrade coordination path is not owned by the current user: ${directoryArg}`); } if ((stats.mode & 0o077) !== 0) { throw new Error(`Upgrade coordination path must not be group or world accessible: ${directoryArg}`); } }; const lstatIfPresent = async (filePathArg: string): Promise => { try { return await plugins.fs.promises.lstat(filePathArg); } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code === 'ENOENT') return undefined; throw errorArg; } }; const ensurePrivateDirectory = async (directoryArg: string): Promise => { try { await plugins.fs.promises.mkdir(directoryArg, { mode: 0o700 }); } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code !== 'EEXIST') throw errorArg; } await assertPrivateDirectory(directoryArg); }; const syncDirectory = async (directoryArg: string): Promise => { const handle = await plugins.fs.promises.open(directoryArg, plugins.fs.constants.O_RDONLY); try { await handle.sync(); } finally { await handle.close(); } }; const writePrivateJson = async ( filePathArg: string, valueArg: unknown, exclusiveArg = true, ): Promise => { const flags = plugins.fs.constants.O_WRONLY | plugins.fs.constants.O_CREAT | plugins.fs.constants.O_NOFOLLOW | (exclusiveArg ? plugins.fs.constants.O_EXCL : plugins.fs.constants.O_TRUNC); const handle = await plugins.fs.promises.open(filePathArg, flags, 0o600); try { await handle.writeFile(`${JSON.stringify(valueArg)}\n`, 'utf8'); await handle.sync(); } finally { await handle.close(); } await syncDirectory(plugins.path.dirname(filePathArg)); }; const readPrivateJson = async (filePathArg: string): Promise => { const handle = await plugins.fs.promises.open( filePathArg, plugins.fs.constants.O_RDONLY | plugins.fs.constants.O_NOFOLLOW, ); try { const stats = await handle.stat(); if (!stats.isFile() || stats.size < 2 || stats.size > maximumMetadataBytes) { throw new Error(`Upgrade coordination metadata is not a bounded regular file: ${filePathArg}`); } if (typeof process.getuid === 'function' && stats.uid !== process.getuid()) { throw new Error(`Upgrade coordination metadata is not owned by the current user: ${filePathArg}`); } if ((stats.mode & 0o077) !== 0) { throw new Error(`Upgrade coordination metadata is not private: ${filePathArg}`); } try { return JSON.parse(await handle.readFile('utf8')) as unknown; } catch (errorArg) { const error = new Error(`Upgrade coordination metadata is malformed: ${filePathArg}`, { cause: errorArg, }) as NodeJS.ErrnoException; error.code = 'HCON_INVALID_JSON'; throw error; } } finally { await handle.close(); } }; const parseOwner = (valueArg: unknown): IUpgradeProcessOwner => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new Error('Upgrade owner metadata is malformed.'); } const value = valueArg as Record; if (!exactKeys(value, [ 'version', 'kind', 'uid', 'tokenHash', 'pid', 'processGroupId', 'fingerprint', 'cliPath', 'command', 'createdAt', ])) { throw new Error('Upgrade owner metadata has unexpected fields.'); } if ( value.version !== coordinationVersion || (value.kind !== 'upgrade' && value.kind !== 'start') || !Number.isSafeInteger(value.uid) || (value.uid as number) < 0 || !Number.isSafeInteger(value.pid) || (value.pid as number) < 2 || !Number.isSafeInteger(value.processGroupId) || (value.processGroupId as number) < 2 || !Number.isSafeInteger(value.createdAt) || (value.createdAt as number) < 1 || typeof value.fingerprint !== 'string' || value.fingerprint.length < 8 || value.fingerprint.length > 256 || typeof value.cliPath !== 'string' || !plugins.path.isAbsolute(value.cliPath) || (value.command !== '__serve' && value.command !== '__upgrade-worker' && value.command !== 'start' && value.command !== 'upgrade' && value.command !== 'foreground' && value.command !== 'temp-password') ) { throw new Error('Upgrade owner metadata is invalid.'); } if (typeof value.tokenHash !== 'string' || !upgradeTokenHashPattern.test(value.tokenHash)) { throw new Error('Upgrade owner metadata token binding is invalid.'); } if ( (value.kind === 'upgrade' && value.command !== '__serve' && value.command !== '__upgrade-worker' && value.command !== 'upgrade') || (value.kind === 'start' && value.command === '__upgrade-worker') ) { throw new Error('Upgrade owner metadata kind and command do not match.'); } return value as unknown as IUpgradeProcessOwner; }; const parseExpectedController = (valueArg: unknown): IUpgradeExpectedController => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new Error('The expected controller identity is malformed.'); } const value = valueArg as Record; if ( !exactKeys(value, ['pid', 'processGroupId', 'processFingerprint']) || !Number.isSafeInteger(value.pid) || (value.pid as number) < 2 || !Number.isSafeInteger(value.processGroupId) || (value.processGroupId as number) < 2 || typeof value.processFingerprint !== 'string' || value.processFingerprint.length < 8 || value.processFingerprint.length > 256 ) { throw new Error('The expected controller identity is invalid.'); } return value as unknown as IUpgradeExpectedController; }; const assertBoundedText = (valueArg: unknown, nameArg: string, maximumBytesArg: number): string => { if ( typeof valueArg !== 'string' || valueArg.length === 0 || Buffer.byteLength(valueArg, 'utf8') > maximumBytesArg ) { throw new Error(`${nameArg} is invalid.`); } return valueArg; }; const parseUpgradeSemver = (valueArg: string): IUpgradeSemver => { const match = upgradeSemverPattern.exec(valueArg); if (!match) throw new Error(`Invalid semantic version: ${valueArg}`); const prerelease = match[4]?.split('.') ?? []; if (prerelease.some((identifierArg) => /^\d+$/.test(identifierArg) && identifierArg.length > 1 && identifierArg.startsWith('0'))) { throw new Error(`Invalid semantic version: ${valueArg}`); } return { major: match[1], minor: match[2], patch: match[3], prerelease, }; }; const compareUpgradeNumericStrings = (leftArg: string, rightArg: string): number => { if (leftArg.length !== rightArg.length) return leftArg.length < rightArg.length ? -1 : 1; return leftArg === rightArg ? 0 : leftArg < rightArg ? -1 : 1; }; export const compareUpgradeSemver = (leftArg: string, rightArg: string): number => { const left = parseUpgradeSemver(leftArg); const right = parseUpgradeSemver(rightArg); for (const key of ['major', 'minor', 'patch'] as const) { const comparison = compareUpgradeNumericStrings(left[key], right[key]); if (comparison !== 0) return comparison; } if (left.prerelease.length === 0 || right.prerelease.length === 0) { if (left.prerelease.length === right.prerelease.length) return 0; return left.prerelease.length === 0 ? 1 : -1; } const identifierCount = Math.max(left.prerelease.length, right.prerelease.length); for (let index = 0; index < identifierCount; index++) { const leftIdentifier = left.prerelease[index]; const rightIdentifier = right.prerelease[index]; if (leftIdentifier === undefined || rightIdentifier === undefined) { return leftIdentifier === rightIdentifier ? 0 : leftIdentifier === undefined ? -1 : 1; } if (leftIdentifier === rightIdentifier) continue; const leftNumeric = /^\d+$/.test(leftIdentifier); const rightNumeric = /^\d+$/.test(rightIdentifier); if (leftNumeric && rightNumeric) { return compareUpgradeNumericStrings(leftIdentifier, rightIdentifier); } if (leftNumeric !== rightNumeric) return leftNumeric ? -1 : 1; return leftIdentifier < rightIdentifier ? -1 : 1; } return 0; }; export const normalizeUpgradeRegistryUrl = (valueArg: unknown): string => { const value = assertBoundedText(valueArg, 'The upgrade registry URL', 2_048); let parsed: URL; try { parsed = new URL(value); } catch (errorArg) { throw new Error('The upgrade registry URL is invalid.', { cause: errorArg }); } const loopbackHttp = parsed.protocol === 'http:' && new Set([ 'localhost', '127.0.0.1', '[::1]', ]).has(parsed.hostname); if ( (parsed.protocol !== 'https:' && !loopbackHttp) || parsed.hostname.length === 0 || parsed.username.length > 0 || parsed.password.length > 0 || parsed.search.length > 0 || parsed.hash.length > 0 ) throw new Error('The upgrade registry URL is unsafe.'); return parsed.toString(); }; const assertRuntimeId = (valueArg: unknown): IControllerRuntimeId => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new Error('The upgrade session identity is malformed.'); } const value = valueArg as Record; if ( !exactKeys(value, ['harnessId', 'nativeId']) || (value.harnessId !== 'opencode' && value.harnessId !== 'flex' && value.harnessId !== 'codex') ) throw new Error('The upgrade session identity is invalid.'); return { harnessId: value.harnessId, nativeId: assertBoundedText(value.nativeId, 'The upgrade native session ID', 512), }; }; const sessionTransitionStates: ReadonlySet = new Set([ 'pending', 'submitting', 'accepted', 'paused', 'outcomeUnknown', 'failed', ]); const parseUpgradeSessionState = (valueArg: unknown): IUpgradeSessionState => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new Error('Upgrade session state is malformed.'); } const value = valueArg as Record; if (!exactKeys(value, [ 'projectId', 'sessionId', 'pauseState', ...(value.pauseQueueId === undefined ? [] : ['pauseQueueId']), ...(value.cleanupState === undefined ? [] : ['cleanupState']), ...(value.continueState === undefined ? [] : ['continueState']), ...(value.error === undefined ? [] : ['error']), ])) throw new Error('Upgrade session state has unexpected fields.'); if ( typeof value.pauseState !== 'string' || !sessionTransitionStates.has(value.pauseState) || (value.cleanupState !== undefined && ( typeof value.cleanupState !== 'string' || !sessionTransitionStates.has(value.cleanupState) )) || (value.continueState !== undefined && ( typeof value.continueState !== 'string' || !sessionTransitionStates.has(value.continueState) )) ) throw new Error('Upgrade session transition state is invalid.'); const sessionId = assertRuntimeId(value.sessionId); if (value.pauseQueueId !== undefined && sessionId.harnessId !== 'flex') { throw new Error('Only Flex upgrade sessions may retain a pause queue ID.'); } return { projectId: assertBoundedText(value.projectId, 'The upgrade project ID', 512), sessionId, pauseState: value.pauseState as TUpgradeSessionTransitionState, ...(value.pauseQueueId === undefined ? {} : { pauseQueueId: assertBoundedText(value.pauseQueueId, 'The Flex pause queue ID', 512) }), ...(value.cleanupState === undefined ? {} : { cleanupState: value.cleanupState as TUpgradeSessionTransitionState }), ...(value.continueState === undefined ? {} : { continueState: value.continueState as TUpgradeSessionTransitionState }), ...(value.error === undefined ? {} : { error: assertBoundedText(value.error, 'The upgrade session error', 2_048) }), }; }; const transactionPhases: ReadonlySet = new Set([ 'starting', 'checking', 'preparing', 'pausing', 'stopping', 'installing', 'restarting', 'continuing', 'completed', 'failed', ]); const transactionPhaseOrder: Record = { starting: 0, checking: 1, preparing: 2, pausing: 3, stopping: 4, installing: 5, restarting: 6, continuing: 7, completed: 8, failed: 8, }; const parseUpgradeTransactionV2 = (valueArg: unknown): IUpgradeTransactionV2 => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new Error('Upgrade transaction metadata is malformed.'); } const value = valueArg as Record; if (!exactKeys(value, [ 'version', 'tokenHash', 'revision', 'port', 'sourceVersion', 'controllerWasRunning', 'continueSessions', 'gracePeriodMs', 'phase', 'message', 'createdAt', 'updatedAt', 'phaseStartedAt', 'sessions', ...(value.targetVersion === undefined ? [] : ['targetVersion']), ...(value.registryUrl === undefined ? [] : ['registryUrl']), ...(value.targetStartupInvoked === undefined ? [] : ['targetStartupInvoked']), ...(value.preparationAcceptedAt === undefined ? [] : ['preparationAcceptedAt']), ...(value.preparationDeadlineAt === undefined ? [] : ['preparationDeadlineAt']), ...(value.preparationCompletedAt === undefined ? [] : ['preparationCompletedAt']), ...(value.retainedUntil === undefined ? [] : ['retainedUntil']), ...(value.worker === undefined ? [] : ['worker']), ...(value.controller === undefined ? [] : ['controller']), ...(value.terminal === undefined ? [] : ['terminal']), ])) throw new Error('Upgrade transaction metadata has unexpected fields.'); if ( value.version !== coordinationVersion || typeof value.tokenHash !== 'string' || !upgradeTokenHashPattern.test(value.tokenHash) || !Number.isSafeInteger(value.revision) || (value.revision as number) < 0 || typeof value.controllerWasRunning !== 'boolean' || (value.targetStartupInvoked !== undefined && value.targetStartupInvoked !== true) || typeof value.continueSessions !== 'boolean' || !Number.isSafeInteger(value.gracePeriodMs) || (value.gracePeriodMs as number) < 1_000 || (value.gracePeriodMs as number) > 60 * 60 * 1000 || ((value.preparationAcceptedAt === undefined) !== (value.preparationDeadlineAt === undefined)) || (value.preparationAcceptedAt !== undefined && ( !Number.isSafeInteger(value.preparationAcceptedAt) || (value.preparationAcceptedAt as number) < (value.createdAt as number) || (value.preparationAcceptedAt as number) > (value.updatedAt as number) || !Number.isSafeInteger(value.preparationDeadlineAt) || (value.preparationDeadlineAt as number) !== (value.preparationAcceptedAt as number) + (value.gracePeriodMs as number) )) || (value.preparationCompletedAt !== undefined && ( value.preparationAcceptedAt === undefined || !Number.isSafeInteger(value.preparationCompletedAt) || (value.preparationCompletedAt as number) < (value.preparationAcceptedAt as number) || (value.preparationCompletedAt as number) > (value.preparationDeadlineAt as number) || typeof value.phase !== 'string' || (transactionPhaseOrder[value.phase as TUpgradeTransaction['phase']] ?? -1) < transactionPhaseOrder.stopping )) || typeof value.phase !== 'string' || !transactionPhases.has(value.phase) || !Number.isSafeInteger(value.createdAt) || (value.createdAt as number) < 1 || !Number.isSafeInteger(value.updatedAt) || (value.updatedAt as number) < (value.createdAt as number) || !Number.isSafeInteger(value.phaseStartedAt) || (value.phaseStartedAt as number) < (value.createdAt as number) || (value.phaseStartedAt as number) > (value.updatedAt as number) || (value.retainedUntil !== undefined && ( !Number.isSafeInteger(value.retainedUntil) || (value.retainedUntil as number) < (value.updatedAt as number) )) || !Array.isArray(value.sessions) || value.sessions.length > 64 ) throw new Error('Upgrade transaction metadata is invalid.'); let worker: IUpgradeTransactionV2['worker']; if (value.worker !== undefined) { if (!value.worker || typeof value.worker !== 'object' || Array.isArray(value.worker)) { throw new Error('Upgrade worker metadata is malformed.'); } const rawWorker = value.worker as Record; if ( !exactKeys(rawWorker, [ 'pid', 'processGroupId', 'fingerprint', 'cliPath', ...(rawWorker.logFilePath === undefined ? [] : ['logFilePath']), ]) || !Number.isSafeInteger(rawWorker.pid) || (rawWorker.pid as number) < 2 || !Number.isSafeInteger(rawWorker.processGroupId) || (rawWorker.processGroupId as number) < 2 || typeof rawWorker.cliPath !== 'string' || !plugins.path.isAbsolute(rawWorker.cliPath) ) throw new Error('Upgrade worker metadata is invalid.'); worker = { pid: rawWorker.pid as number, processGroupId: rawWorker.processGroupId as number, fingerprint: assertBoundedText(rawWorker.fingerprint, 'The upgrade worker fingerprint', 256), cliPath: rawWorker.cliPath, ...(rawWorker.logFilePath === undefined ? {} : { logFilePath: assertBoundedText( rawWorker.logFilePath, 'The upgrade worker log path', 4_096, ), }), }; } let terminal: IUpgradeTransactionV2['terminal']; if (value.terminal !== undefined) { if (!value.terminal || typeof value.terminal !== 'object' || Array.isArray(value.terminal)) { throw new Error('Upgrade terminal metadata is malformed.'); } const rawTerminal = value.terminal as Record; if ( !exactKeys(rawTerminal, ['success', ...(rawTerminal.error === undefined ? [] : ['error'])]) || typeof rawTerminal.success !== 'boolean' ) throw new Error('Upgrade terminal metadata is invalid.'); terminal = { success: rawTerminal.success, ...(rawTerminal.error === undefined ? {} : { error: assertBoundedText(rawTerminal.error, 'The upgrade terminal error', 4_096) }), }; } let controller: IUpgradeTransactionV2['controller']; if (value.controller !== undefined) { if (!value.controller || typeof value.controller !== 'object' || Array.isArray(value.controller)) { throw new Error('Upgrade controller metadata is malformed.'); } const rawController = value.controller as Record; if ( !exactKeys(rawController, [ 'pid', 'processGroupId', 'fingerprint', 'cliPath', 'command', ]) || !Number.isSafeInteger(rawController.pid) || (rawController.pid as number) < 2 || !Number.isSafeInteger(rawController.processGroupId) || (rawController.processGroupId as number) < 2 || typeof rawController.cliPath !== 'string' || !plugins.path.isAbsolute(rawController.cliPath) || (rawController.command !== '__serve' && rawController.command !== 'foreground') ) throw new Error('Upgrade controller metadata is invalid.'); controller = { pid: rawController.pid as number, processGroupId: rawController.processGroupId as number, fingerprint: assertBoundedText( rawController.fingerprint, 'The upgrade controller fingerprint', 256, ), cliPath: rawController.cliPath, command: rawController.command, }; } return { version: coordinationVersion, tokenHash: value.tokenHash, revision: value.revision as number, port: assertPort(value.port), sourceVersion: assertBoundedText(value.sourceVersion, 'The upgrade source version', 128), ...(value.targetVersion === undefined ? {} : { targetVersion: assertBoundedText(value.targetVersion, 'The upgrade target version', 128) }), ...(value.registryUrl === undefined ? {} : { registryUrl: normalizeUpgradeRegistryUrl(value.registryUrl) }), ...(value.targetStartupInvoked === true ? { targetStartupInvoked: true } : {}), controllerWasRunning: value.controllerWasRunning, continueSessions: value.continueSessions, gracePeriodMs: value.gracePeriodMs as number, ...(value.preparationAcceptedAt === undefined ? {} : { preparationAcceptedAt: value.preparationAcceptedAt as number, preparationDeadlineAt: value.preparationDeadlineAt as number, }), ...(value.preparationCompletedAt === undefined ? {} : { preparationCompletedAt: value.preparationCompletedAt as number }), phase: value.phase as TUpgradeTransaction['phase'], message: assertBoundedText(value.message, 'The upgrade transaction message', 4_096), createdAt: value.createdAt as number, phaseStartedAt: value.phaseStartedAt as number, updatedAt: value.updatedAt as number, ...(value.retainedUntil === undefined ? {} : { retainedUntil: value.retainedUntil as number }), ...(worker ? { worker } : {}), ...(controller ? { controller } : {}), sessions: value.sessions.map(parseUpgradeSessionState), ...(terminal ? { terminal } : {}), }; }; const parseUpgradeTransactionV3 = (valueArg: unknown): IUpgradeTransactionV3 => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new Error('Upgrade transaction metadata is malformed.'); } const value = valueArg as Record; if (!exactKeys(value, [ 'version', 'tokenHash', 'revision', 'port', 'sourcePackageName', 'sourceVersion', 'sourceManagementVersion', 'sourceCliName', 'sourceCliRelativePath', 'targetPackageName', 'targetVersion', 'targetManagementVersion', 'targetCliName', 'targetCliRelativePath', 'controllerWasRunning', 'continueSessions', 'gracePeriodMs', 'phase', 'message', 'createdAt', 'updatedAt', 'phaseStartedAt', 'sessions', ...(value.recoveryTargetVersion === undefined ? [] : ['recoveryTargetVersion']), ...(value.packageTransitionStarted === undefined ? [] : ['packageTransitionStarted']), ...(value.groupedTargetVerified === undefined ? [] : ['groupedTargetVerified']), ...(value.targetPackageCommitStarted === undefined ? [] : ['targetPackageCommitStarted']), ...(value.targetPackageCommitted === undefined ? [] : ['targetPackageCommitted']), ...(value.targetStartupInvoked === undefined ? [] : ['targetStartupInvoked']), ...(value.preparationAcceptedAt === undefined ? [] : ['preparationAcceptedAt']), ...(value.preparationDeadlineAt === undefined ? [] : ['preparationDeadlineAt']), ...(value.preparationCompletedAt === undefined ? [] : ['preparationCompletedAt']), ...(value.retainedUntil === undefined ? [] : ['retainedUntil']), ...(value.worker === undefined ? [] : ['worker']), ...(value.controller === undefined ? [] : ['controller']), ...(value.terminal === undefined ? [] : ['terminal']), ])) throw new Error('Upgrade transaction metadata has unexpected fields.'); if ( value.version !== upgradePackageTransitionTransactionVersion || value.sourcePackageName !== upgradePackageTransitionSource.packageName || value.sourceVersion !== upgradePackageTransitionSource.version || value.sourceManagementVersion !== upgradePackageTransitionSource.managementVersion || value.sourceCliName !== upgradePackageTransitionSource.cliName || value.sourceCliRelativePath !== upgradePackageTransitionSource.cliRelativePath || value.targetPackageName !== upgradePackageTransitionTarget.packageName || value.targetVersion !== upgradePackageTransitionTarget.version || value.targetManagementVersion !== upgradePackageTransitionTarget.managementVersion || value.targetCliName !== upgradePackageTransitionTarget.cliName || value.targetCliRelativePath !== upgradePackageTransitionTarget.cliRelativePath || (value.recoveryTargetVersion !== undefined && ( typeof value.recoveryTargetVersion !== 'string' || compareUpgradeSemver( value.recoveryTargetVersion, upgradePackageTransitionTarget.version, ) <= 0 || value.targetPackageCommitStarted !== true )) || (value.packageTransitionStarted !== undefined && value.packageTransitionStarted !== true) || (value.groupedTargetVerified !== undefined && value.groupedTargetVerified !== true) || (value.targetPackageCommitStarted !== undefined && value.targetPackageCommitStarted !== true) || (value.targetPackageCommitted !== undefined && value.targetPackageCommitted !== true) || (value.targetStartupInvoked !== undefined && value.targetStartupInvoked !== true) ) throw new Error('Upgrade package-transition identity or checkpoint metadata is invalid.'); if ( (value.groupedTargetVerified === true && value.packageTransitionStarted !== true) || (value.targetPackageCommitStarted === true && value.groupedTargetVerified !== true) || (value.targetPackageCommitted === true && value.targetPackageCommitStarted !== true) || (value.targetStartupInvoked === true && value.targetPackageCommitted !== true) || (value.targetStartupInvoked === true && value.controllerWasRunning !== true) ) throw new Error('Upgrade package-transition checkpoints violate their implication chain.'); const compatible = parseUpgradeTransactionV2({ version: coordinationVersion, tokenHash: value.tokenHash, revision: value.revision, port: value.port, sourceVersion: value.sourceVersion, targetVersion: value.targetVersion, ...(value.targetStartupInvoked === true ? { targetStartupInvoked: true } : {}), controllerWasRunning: value.controllerWasRunning, continueSessions: value.continueSessions, gracePeriodMs: value.gracePeriodMs, ...(value.preparationAcceptedAt === undefined ? {} : { preparationAcceptedAt: value.preparationAcceptedAt, preparationDeadlineAt: value.preparationDeadlineAt, }), ...(value.preparationCompletedAt === undefined ? {} : { preparationCompletedAt: value.preparationCompletedAt }), phase: value.phase, message: value.message, createdAt: value.createdAt, phaseStartedAt: value.phaseStartedAt, updatedAt: value.updatedAt, ...(value.retainedUntil === undefined ? {} : { retainedUntil: value.retainedUntil }), ...(value.worker === undefined ? {} : { worker: value.worker }), ...(value.controller === undefined ? {} : { controller: value.controller }), sessions: value.sessions, ...(value.terminal === undefined ? {} : { terminal: value.terminal }), }); if ( compatible.terminal?.success === true && ( value.targetPackageCommitted !== true || (compatible.controllerWasRunning && value.targetStartupInvoked !== true) || (!compatible.controllerWasRunning && value.targetStartupInvoked === true) ) ) throw new Error('A successful package transition is missing its terminal checkpoint.'); return { ...compatible, version: upgradePackageTransitionTransactionVersion, sourcePackageName: upgradePackageTransitionSource.packageName, sourceVersion: upgradePackageTransitionSource.version, sourceManagementVersion: upgradePackageTransitionSource.managementVersion, sourceCliName: upgradePackageTransitionSource.cliName, sourceCliRelativePath: upgradePackageTransitionSource.cliRelativePath, targetPackageName: upgradePackageTransitionTarget.packageName, targetVersion: upgradePackageTransitionTarget.version, targetManagementVersion: upgradePackageTransitionTarget.managementVersion, targetCliName: upgradePackageTransitionTarget.cliName, targetCliRelativePath: upgradePackageTransitionTarget.cliRelativePath, ...(value.recoveryTargetVersion === undefined ? {} : { recoveryTargetVersion: assertBoundedText( value.recoveryTargetVersion, 'The package-transition recovery target version', 128, ), }), ...(value.packageTransitionStarted === true ? { packageTransitionStarted: true } : {}), ...(value.groupedTargetVerified === true ? { groupedTargetVerified: true } : {}), ...(value.targetPackageCommitStarted === true ? { targetPackageCommitStarted: true } : {}), ...(value.targetPackageCommitted === true ? { targetPackageCommitted: true } : {}), ...(value.targetStartupInvoked === true ? { targetStartupInvoked: true } : {}), }; }; const parseUpgradeTransaction = (valueArg: unknown): TUpgradeTransaction => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new Error('Upgrade transaction metadata is malformed.'); } return (valueArg as Record).version === coordinationVersion ? parseUpgradeTransactionV2(valueArg) : parseUpgradeTransactionV3(valueArg); }; const parseUpgradeTransactionAdoptionAudit = ( valueArg: unknown, ): IUpgradeTransactionAdoptionAudit => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new Error('Upgrade transaction adoption metadata is malformed.'); } const value = valueArg as Record; if (!exactKeys(value, [ 'version', 'adoptedAt', 'adoptedTargetVersion', 'previousTransaction', ])) throw new Error('Upgrade transaction adoption metadata has unexpected fields.'); const previousTransaction = parseUpgradeTransaction(value.previousTransaction); if ( value.version !== 1 || !Number.isSafeInteger(value.adoptedAt) || (value.adoptedAt as number) < previousTransaction.createdAt ) throw new Error('Upgrade transaction adoption metadata is invalid.'); return { version: 1, adoptedAt: value.adoptedAt as number, adoptedTargetVersion: assertBoundedText( value.adoptedTargetVersion, 'The adopted upgrade target version', 128, ), previousTransaction, }; }; const parseTransactionMutationOwner = (valueArg: unknown): IUpgradeTransactionMutationOwner => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new Error('The upgrade transaction mutation owner is malformed.'); } const value = valueArg as Record; if ( !exactKeys(value, [ 'version', 'uid', 'tokenHash', 'pid', 'fingerprint', 'nonce', 'createdAt', ]) || value.version !== coordinationVersion || !Number.isSafeInteger(value.uid) || (value.uid as number) < 0 || typeof value.tokenHash !== 'string' || !upgradeTokenHashPattern.test(value.tokenHash) || !Number.isSafeInteger(value.pid) || (value.pid as number) < 2 || typeof value.nonce !== 'string' || !/^[a-f0-9]{32}$/.test(value.nonce) || !Number.isSafeInteger(value.createdAt) || (value.createdAt as number) < 1 ) throw new Error('The upgrade transaction mutation owner is invalid.'); return { version: coordinationVersion, uid: value.uid as number, tokenHash: value.tokenHash, pid: value.pid as number, fingerprint: assertBoundedText( value.fingerprint, 'The upgrade transaction mutation fingerprint', 256, ), nonce: value.nonce, createdAt: value.createdAt as number, }; }; const parseOwnershipMutationGuardOwner = ( valueArg: unknown, ): IUpgradeOwnershipMutationGuardOwner => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new Error('The upgrade ownership mutation guard is malformed.'); } const value = valueArg as Record; if ( !exactKeys(value, [ 'version', 'uid', 'targetHash', 'pid', 'fingerprint', 'nonce', 'createdAt', ]) || value.version !== coordinationVersion || !Number.isSafeInteger(value.uid) || (value.uid as number) < 0 || typeof value.targetHash !== 'string' || !upgradeTokenHashPattern.test(value.targetHash) || !Number.isSafeInteger(value.pid) || (value.pid as number) < 2 || typeof value.nonce !== 'string' || !/^[a-f0-9]{32}$/.test(value.nonce) || !Number.isSafeInteger(value.createdAt) || (value.createdAt as number) < 1 ) throw new Error('The upgrade ownership mutation guard is invalid.'); return { version: coordinationVersion, uid: value.uid as number, targetHash: value.targetHash, pid: value.pid as number, fingerprint: assertBoundedText( value.fingerprint, 'The upgrade ownership mutation guard fingerprint', 256, ), nonce: value.nonce, createdAt: value.createdAt as number, }; }; export const upgradeTransactionIsStalled = ( transactionArg: TUpgradeTransaction, nowArg = Date.now(), ): boolean => { if (transactionArg.terminal) return false; if ( transactionArg.preparationDeadlineAt !== undefined && transactionArg.preparationCompletedAt === undefined && (transactionArg.phase === 'preparing' || transactionArg.phase === 'pausing') ) return nowArg > transactionArg.preparationDeadlineAt + 60_000; const maximumPhaseDurationMs: Record = { starting: 2 * 60 * 1000, checking: 3 * 60 * 1000, preparing: transactionArg.gracePeriodMs + 60_000, pausing: transactionArg.gracePeriodMs + 60_000, stopping: 90_000, installing: 22 * 60 * 1000, restarting: 3 * 60 * 1000, continuing: 2 * 60 * 1000, completed: Number.MAX_SAFE_INTEGER, failed: Number.MAX_SAFE_INTEGER, }; return nowArg - transactionArg.phaseStartedAt > maximumPhaseDurationMs[transactionArg.phase]; }; export const upgradeTransactionRequiresForwardRecovery = ( transactionArg: TUpgradeTransaction, ): boolean => transactionArg.version === coordinationVersion ? transactionArg.targetStartupInvoked === true : transactionArg.targetPackageCommitStarted === true; export const upgradeTransactionTargetVersion = ( transactionArg: TUpgradeTransaction, ): string | undefined => transactionArg.version === upgradePackageTransitionTransactionVersion ? transactionArg.recoveryTargetVersion ?? transactionArg.targetVersion : transactionArg.targetVersion; const ownerIsLive = async (ownerArg: IUpgradeProcessOwner): Promise => { if (ownerArg.uid !== assertUid()) return false; const identity = await readControllerProcessIdentity(ownerArg.pid); if (identity && identity.fingerprint === ownerArg.fingerprint) { return await processIdentityHasCliCommand( identity, ownerArg.cliPath, ownerArg.command, { allowMissingAbsolutePath: true }, ); } if (ownerArg.kind !== 'upgrade' || ownerArg.processGroupId !== ownerArg.pid) return false; return (await readProcessGroupMemberPids(ownerArg.processGroupId)) .some((processIdArg) => processIdArg !== ownerArg.pid); }; const ownerForCurrentProcess = async ( kindArg: TUpgradeOwnerKind, tokenArg: string, cliPathArg: string, commandArg: IUpgradeProcessOwner['command'], ): Promise => { const identity = await readControllerProcessIdentity(process.pid); if (!identity) throw new Error('Unable to establish upgrade process ownership.'); return { version: coordinationVersion, kind: kindArg, uid: assertUid(), tokenHash: hashToken(tokenArg), pid: process.pid, processGroupId: identity.processGroupId, fingerprint: identity.fingerprint, cliPath: cliPathArg, command: commandArg, createdAt: Date.now(), }; }; const coordinationPathHash = (valueArg: string): string => plugins.crypto .createHash('sha256') .update(valueArg) .digest('hex') .slice(0, 20); const ownershipTargetHash = (valueArg: string): string => plugins.crypto .createHash('sha256') .update(valueArg) .digest('hex'); interface IOwnedDirectoryRemovalExpectation { identity: { dev: number; ino: number }; assertOwner?: (valueArg: unknown) => void; assertRemovalAuthorized?: () => void; } interface IOwnedDirectoryOwnerMetadata { owner: TOwner; shape: 'owner' | 'removing' | 'restoring'; tombstoneName?: string; tombstoneIdentity?: { dev: number; ino: number }; tombstoneCtimeMs?: number; removalPid?: number; removalFingerprintHash?: string; } const sameOwnedDirectoryIdentity = ( statsArg: plugins.fs.Stats, identityArg: IOwnedDirectoryRemovalExpectation['identity'], ): boolean => statsArg.dev === identityArg.dev && statsArg.ino === identityArg.ino; const exactPrimitiveRecordMatches = (leftArg: object, rightArg: object): boolean => { const left = leftArg as Record; const right = rightArg as Record; const keys = Object.keys(left); return exactKeys(right, keys) && keys.every((keyArg) => left[keyArg] === right[keyArg]); }; const parseOwnerRemovalName = (nameArg: string): { pid: number; fingerprintHash?: string; } | undefined => { const current = ownerRemovalPattern.exec(nameArg); if (current) return { pid: Number(current[1]), fingerprintHash: current[2] }; const legacy = legacyOwnerRemovalPattern.exec(nameArg); return legacy ? { pid: Number(legacy[1]) } : undefined; }; const inspectOwnedDirectoryOwnerMetadata = async ( directoryArg: string, parseOwnerArg: (valueArg: unknown) => TOwner, ): Promise | undefined> => { const entries = await plugins.fs.promises.readdir(directoryArg, { withFileTypes: true }); if (entries.length === 0) return undefined; if (entries.length > 2) { throw new Error(`Upgrade coordination owner directory contains unsafe entries: ${directoryArg}`); } const ownerEntry = entries.find((entryArg) => entryArg.name === ownerFileName); const tombstoneEntry = entries.find((entryArg) => parseOwnerRemovalName(entryArg.name)); if ( entries.some((entryArg) => entryArg !== ownerEntry && entryArg !== tombstoneEntry) || !ownerEntry && !tombstoneEntry ) throw new Error(`Upgrade coordination owner directory contains unsafe entries: ${directoryArg}`); for (const entry of entries) { if (!entry.isFile() || entry.isSymbolicLink()) { throw new Error(`Upgrade coordination owner metadata is unsafe: ${directoryArg}`); } } const ownerPath = ownerEntry ? plugins.path.join(directoryArg, ownerFileName) : undefined; const tombstonePath = tombstoneEntry ? plugins.path.join(directoryArg, tombstoneEntry.name) : undefined; const [ownerStats, tombstoneStats] = await Promise.all([ ownerPath ? plugins.fs.promises.lstat(ownerPath) : undefined, tombstonePath ? plugins.fs.promises.lstat(tombstonePath) : undefined, ]); for (const metadata of [ownerStats, tombstoneStats]) { if (!metadata) continue; if ( !metadata.isFile() || metadata.isSymbolicLink() || (typeof process.getuid === 'function' && metadata.uid !== process.getuid()) || (metadata.mode & 0o077) !== 0 ) throw new Error(`Upgrade coordination owner metadata is unsafe: ${directoryArg}`); } let shape: IOwnedDirectoryOwnerMetadata['shape']; if (ownerStats && tombstoneStats) { if ( ownerStats.dev !== tombstoneStats.dev || ownerStats.ino !== tombstoneStats.ino || ownerStats.nlink !== 2 || tombstoneStats.nlink !== 2 ) throw new Error(`Upgrade coordination owner restoration is invalid: ${directoryArg}`); shape = 'restoring'; } else if (ownerStats) { if (ownerStats.nlink !== 1) { throw new Error(`Upgrade coordination owner link count is invalid: ${directoryArg}`); } shape = 'owner'; } else { if (tombstoneStats!.nlink !== 1) { throw new Error(`Upgrade coordination owner tombstone link count is invalid: ${directoryArg}`); } shape = 'removing'; } return { owner: parseOwnerArg(await readPrivateJson(ownerPath ?? tombstonePath!)), shape, ...(tombstoneEntry ? (() => { const removal = parseOwnerRemovalName(tombstoneEntry.name)!; return { tombstoneName: tombstoneEntry.name, tombstoneIdentity: { dev: tombstoneStats!.dev, ino: tombstoneStats!.ino }, tombstoneCtimeMs: tombstoneStats!.ctimeMs, removalPid: removal.pid, ...(removal.fingerprintHash ? { removalFingerprintHash: removal.fingerprintHash } : {}), }; })() : {}), }; }; const restoreOwnedDirectoryOwnerMetadata = async ( directoryArg: string, metadataArg: IOwnedDirectoryOwnerMetadata, ): Promise => { if (metadataArg.shape === 'owner') return; const tombstonePath = plugins.path.join(directoryArg, metadataArg.tombstoneName!); const tombstoneStats = await plugins.fs.promises.lstat(tombstonePath); if (!sameOwnedDirectoryIdentity(tombstoneStats, metadataArg.tombstoneIdentity!)) { throw new Error('Upgrade coordination owner tombstone changed during recovery.'); } if (metadataArg.shape === 'restoring') { await plugins.fs.promises.unlink(tombstonePath); } else { await plugins.fs.promises.rename( tombstonePath, plugins.path.join(directoryArg, ownerFileName), ); } await syncDirectory(directoryArg); }; const removeExactOwnedDirectory = async ( directoryArg: string, expectationArg: IOwnedDirectoryRemovalExpectation, ): Promise => { const initialStats = await plugins.fs.promises.lstat(directoryArg); if (!sameOwnedDirectoryIdentity(initialStats, expectationArg.identity)) { throw new Error('Upgrade coordination ownership changed before removal.'); } const entries = await plugins.fs.promises.readdir(directoryArg, { withFileTypes: true }); if ( entries.length > 1 || entries.some((entryArg) => ( entryArg.name !== ownerFileName || !entryArg.isFile() || entryArg.isSymbolicLink() )) ) throw new Error(`Upgrade coordination owner directory contains unsafe entries: ${directoryArg}`); if (Boolean(entries.length) !== Boolean(expectationArg.assertOwner)) { throw new Error('Upgrade coordination ownership changed before removal.'); } if (entries.length === 1) { const ownerPath = plugins.path.join(directoryArg, ownerFileName); const removerIdentity = await readControllerProcessIdentity(process.pid); if (!removerIdentity) { throw new Error('Unable to establish upgrade ownership-removal identity.'); } const removerFingerprintHash = plugins.crypto .createHash('sha256') .update(removerIdentity.fingerprint, 'utf8') .digest('hex'); const tombstonePath = plugins.path.join( directoryArg, `.owner-removing-${process.pid}-${removerFingerprintHash}-${plugins.crypto.randomBytes(8).toString('hex')}`, ); expectationArg.assertRemovalAuthorized?.(); await plugins.fs.promises.rename(ownerPath, tombstonePath); try { expectationArg.assertOwner!(await readPrivateJson(tombstonePath)); } catch (errorArg) { let restoreError: unknown; try { await plugins.fs.promises.link(tombstonePath, ownerPath); await plugins.fs.promises.unlink(tombstonePath); await syncDirectory(directoryArg); } catch (restoreErrorArg) { restoreError = restoreErrorArg; } if (restoreError !== undefined) { throw new AggregateError( [errorArg, restoreError], 'Upgrade coordination ownership changed and could not be restored safely.', ); } throw errorArg; } await plugins.fs.promises.unlink(tombstonePath); await syncDirectory(directoryArg); } const currentStats = await lstatIfPresent(directoryArg); if (!currentStats || !sameOwnedDirectoryIdentity(currentStats, expectationArg.identity)) return; try { await plugins.fs.promises.rmdir(directoryArg); } catch (errorArg) { const code = (errorArg as NodeJS.ErrnoException).code; if (code === 'ENOENT') return; if (code === 'ENOTEMPTY' || code === 'EEXIST') { throw new Error('Upgrade coordination ownership changed during removal.', { cause: errorArg, }); } throw errorArg; } await syncDirectory(plugins.path.dirname(directoryArg)); }; const removeExactFailedOwnerPublication = async ( directoryArg: string, identityArg: { dev: number; ino: number }, ): Promise => { const initialStats = await lstatIfPresent(directoryArg); if (!initialStats) return; if (!sameOwnedDirectoryIdentity(initialStats, identityArg)) { throw new Error('Upgrade coordination publication changed before cleanup.'); } const entries = await plugins.fs.promises.readdir(directoryArg, { withFileTypes: true }); if ( entries.length > 1 || entries.some((entryArg) => ( entryArg.name !== ownerFileName || !entryArg.isFile() || entryArg.isSymbolicLink() )) ) throw new Error(`Upgrade coordination publication contains unsafe entries: ${directoryArg}`); if (entries.length === 1) { const ownerPath = plugins.path.join(directoryArg, ownerFileName); const ownerStats = await plugins.fs.promises.lstat(ownerPath); if ( !ownerStats.isFile() || ownerStats.isSymbolicLink() || (typeof process.getuid === 'function' && ownerStats.uid !== process.getuid()) || (ownerStats.mode & 0o077) !== 0 || ownerStats.nlink !== 1 ) throw new Error(`Upgrade coordination publication metadata is unsafe: ${ownerPath}`); await plugins.fs.promises.unlink(ownerPath); await syncDirectory(directoryArg); } const currentStats = await lstatIfPresent(directoryArg); if (!currentStats) return; if (!sameOwnedDirectoryIdentity(currentStats, identityArg)) { throw new Error('Upgrade coordination publication changed during cleanup.'); } try { await plugins.fs.promises.rmdir(directoryArg); } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code === 'ENOENT') return; throw errorArg; } await syncDirectory(plugins.path.dirname(directoryArg)); }; class UpgradeOwnershipMutationGuard { private released = false; private releaseTask?: Promise; constructor( public readonly directory: string, public readonly owner: IUpgradeOwnershipMutationGuardOwner, private readonly identity: { dev: number; ino: number }, ) {} public release(): Promise { if (this.released) return Promise.resolve(); if (!this.releaseTask) { let task: Promise; task = (async () => { const stats = await lstatIfPresent(this.directory); if (!stats) { this.released = true; return; } if (!sameOwnedDirectoryIdentity(stats, this.identity)) { throw new Error('Upgrade ownership mutation guard changed unexpectedly.'); } const metadata = await inspectOwnedDirectoryOwnerMetadata( this.directory, parseOwnershipMutationGuardOwner, ); if (metadata && !exactPrimitiveRecordMatches(metadata.owner, this.owner)) { throw new Error('Upgrade ownership mutation guard changed unexpectedly.'); } if (metadata) await restoreOwnedDirectoryOwnerMetadata(this.directory, metadata); await removeExactOwnedDirectory(this.directory, { identity: this.identity, ...(metadata ? { assertOwner: (valueArg: unknown) => { const removedOwner = parseOwnershipMutationGuardOwner(valueArg); if (!exactPrimitiveRecordMatches(removedOwner, this.owner)) { throw new Error('Upgrade ownership mutation guard changed during release.'); } }, } : {}), }); this.released = true; })().finally(() => { if (this.releaseTask === task && !this.released) this.releaseTask = undefined; }); this.releaseTask = task; } return this.releaseTask; } } class UpgradeOwnedDirectory { private released = false; private releaseTask?: Promise; constructor( public readonly directory: string, public readonly owner: IUpgradeProcessOwner, private readonly releaseOwned: ( directoryArg: string, ownerArg: IUpgradeProcessOwner, shouldReleaseArg?: () => boolean, ) => Promise, ) {} public async assertOwned(): Promise { if (this.released) throw new Error('Upgrade coordination ownership has already been released.'); let owner: IUpgradeProcessOwner; try { owner = parseOwner(await readPrivateJson(plugins.path.join(this.directory, ownerFileName))); } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code === 'ENOENT') { throw new Error('Upgrade coordination ownership changed unexpectedly.', { cause: errorArg }); } throw errorArg; } if (!exactPrimitiveRecordMatches(owner, this.owner) || !await ownerIsLive(owner)) { throw new Error('Upgrade coordination ownership changed unexpectedly.'); } } public release(shouldReleaseArg?: () => boolean): Promise { if (this.released) return Promise.resolve(); if (!this.releaseTask) { let task: Promise; task = this.releaseOwned(this.directory, this.owner, shouldReleaseArg).then(() => { this.released = true; }).finally(() => { if (this.releaseTask === task && !this.released) this.releaseTask = undefined; }); this.releaseTask = task; } return this.releaseTask; } public relinquishAfterTransfer(): void { if (this.released) throw new Error('Upgrade coordination ownership has already been released.'); this.released = true; } } export class UpgradeInstallationLock extends UpgradeOwnedDirectory {} export class UpgradeStartLease extends UpgradeOwnedDirectory {} export interface IVerifiedUpgradeWorkerProcessGroup { pid: number; processGroupId: number; fingerprint: string; cliPath: string; } export const terminateVerifiedUpgradeWorkerProcessGroup = async ( worker: IVerifiedUpgradeWorkerProcessGroup, ): Promise => { const identity = await readControllerProcessIdentity(worker.pid); const existingMembers = worker.processGroupId === worker.pid ? await readProcessGroupMemberPids(worker.processGroupId) : []; if (!identity || identity.fingerprint !== worker.fingerprint) { if (existingMembers.length === 0) return; throw new Error( 'The stalled upgrade worker leader is unavailable; its remaining group cannot be signaled safely.', ); } if ( identity.processGroupId !== worker.processGroupId || !identity.processGroupLeader || !await processIdentityHasCliCommand( identity, worker.cliPath, '__upgrade-worker', { allowMissingAbsolutePath: true }, ) ) throw new Error('The stalled upgrade worker identity changed unexpectedly.'); const readStableMemberSnapshot = async (): Promise >>[]> => { for (let attempt = 0; attempt < 4; attempt++) { const processIdsBefore = await readProcessGroupMemberPids(worker.processGroupId); const identities = await Promise.all( processIdsBefore.map(async (processIdArg) => await readControllerProcessIdentity(processIdArg)), ); const processIdsAfter = await readProcessGroupMemberPids(worker.processGroupId); if ( processIdsBefore.length === processIdsAfter.length && processIdsBefore.every((processIdArg, indexArg) => processIdArg === processIdsAfter[indexArg]) && identities.every((candidateArg) => ( candidateArg !== null && candidateArg.processGroupId === worker.processGroupId )) && identities.some((candidateArg) => ( candidateArg!.pid === worker.pid && candidateArg!.fingerprint === worker.fingerprint )) ) return identities as NonNullable[]; } throw new Error('The stalled upgrade worker group could not be snapshotted safely.'); }; const members = await readStableMemberSnapshot(); const signalOwnedMembers = async (signalArg: NodeJS.Signals): Promise => { const orderedMembers = [...members].sort((leftArg, rightArg) => ( leftArg.pid === worker.pid ? 1 : rightArg.pid === worker.pid ? -1 : 0 )); for (const member of orderedMembers) { const current = await readControllerProcessIdentity(member.pid); if (!current || current.fingerprint !== member.fingerprint) continue; if (current.processGroupId !== worker.processGroupId) { throw new Error('A captured upgrade worker process escaped its owned process group.'); } try { process.kill(member.pid, signalArg); } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code !== 'ESRCH') throw errorArg; } } }; const waitForDrain = async (): Promise => { const deadline = Date.now() + workerTerminationGraceMs; while (Date.now() < deadline) { const [captured, group] = await Promise.all([ Promise.all(members.map(async (memberArg) => { const current = await readControllerProcessIdentity(memberArg.pid); return current?.fingerprint === memberArg.fingerprint; })), readProcessGroupMemberPids(worker.processGroupId), ]); if (!captured.some(Boolean) && group.length === 0) return true; await new Promise((resolve) => setTimeout(resolve, 50)); } const [captured, group] = await Promise.all([ Promise.all(members.map(async (memberArg) => { const current = await readControllerProcessIdentity(memberArg.pid); return current?.fingerprint === memberArg.fingerprint; })), readProcessGroupMemberPids(worker.processGroupId), ]); return !captured.some(Boolean) && group.length === 0; }; await signalOwnedMembers('SIGTERM'); if (await waitForDrain()) return; await signalOwnedMembers('SIGKILL'); if (!await waitForDrain()) throw new Error('The stalled upgrade worker group did not terminate.'); }; export class UpgradeCoordinator { public readonly uid: number; public readonly baseDirectory: string; public readonly upgradeLockDirectory: string; public readonly startLeasesDirectory: string; public readonly transactionsDirectory: string; public readonly ownershipGuardsDirectory: string; private readonly usesLegacyLocation: boolean; private readonly usesInheritedLocation: boolean; private initializationTask?: Promise; constructor( public readonly canonicalGlobalRoot: string, ) { if (!plugins.path.isAbsolute(canonicalGlobalRoot)) { throw new Error('The canonical pnpm global root must be absolute.'); } this.uid = assertUid(); const canonicalBaseDirectory = plugins.path.join( resolveAGLHomePaths().upgrade, `coordination-${this.uid}-${coordinationPathHash(canonicalGlobalRoot)}`, ); const inheritedBaseDirectory = process.env[upgradeCoordinationRootEnvironmentVariable]; if (inheritedBaseDirectory !== undefined) { if ( !plugins.path.isAbsolute(inheritedBaseDirectory) || plugins.path.normalize(inheritedBaseDirectory) !== inheritedBaseDirectory || plugins.path.parse(inheritedBaseDirectory).root === inheritedBaseDirectory || inheritedBaseDirectory.includes('\0') || Buffer.byteLength(inheritedBaseDirectory, 'utf8') > 4096 ) throw new Error('The inherited AGL upgrade coordination root is invalid.'); } const inheritedUpgradeToken = process.env[upgradeTokenEnvironmentVariable]; const usesLegacyLocation = inheritedUpgradeToken !== undefined && process.env[upgradeCoordinationRootEnvironmentVariable] === undefined; if ( usesLegacyLocation && (inheritedUpgradeToken === undefined || !upgradeTokenPattern.test(inheritedUpgradeToken)) ) { throw new Error('Legacy upgrade coordination requires the exact inherited upgrade token.'); } this.usesLegacyLocation = usesLegacyLocation; this.usesInheritedLocation = inheritedBaseDirectory !== undefined; this.baseDirectory = this.usesLegacyLocation ? plugins.path.join( '/tmp', `harness-controller-upgrade-${this.uid}-${coordinationPathHash(canonicalGlobalRoot)}`, ) : inheritedBaseDirectory ?? canonicalBaseDirectory; this.upgradeLockDirectory = plugins.path.join(this.baseDirectory, 'upgrade.lock'); this.startLeasesDirectory = plugins.path.join(this.baseDirectory, 'start-leases'); this.transactionsDirectory = plugins.path.join(this.baseDirectory, 'transactions'); this.ownershipGuardsDirectory = plugins.path.join(this.baseDirectory, 'ownership-guards'); } public async init(): Promise { if (!this.initializationTask) { let task: Promise; task = this.performInitialization().catch((errorArg) => { if (this.initializationTask === task) this.initializationTask = undefined; throw errorArg; }); this.initializationTask = task; } await this.initializationTask; } public async initializeExistingCanonicalState(): Promise { if (this.usesLegacyLocation || this.usesInheritedLocation) { throw new Error('Canonical upgrade initialization cannot use an inherited root.'); } if (!await lstatIfPresent(this.baseDirectory)) return; await this.init(); } private async performInitialization(): Promise { if (!this.usesLegacyLocation && !this.usesInheritedLocation) { const home = resolveAGLHomePaths(); await ensurePrivateDirectory(home.root); await ensurePrivateDirectory(home.upgrade); } await ensurePrivateDirectory(this.baseDirectory); await ensurePrivateDirectory(this.startLeasesDirectory); await ensurePrivateDirectory(this.transactionsDirectory); await ensurePrivateDirectory(this.ownershipGuardsDirectory); await this.cleanupOwnershipGuardTemporaryDirectories(); await this.cleanupOwnershipGuardDirectories(); const entries = await plugins.fs.promises.readdir(this.baseDirectory, { withFileTypes: true }); for (const entry of entries) { if (entry.isFile() && transactionAdoptionAuditTemporaryPattern.test(entry.name)) { const filePath = plugins.path.join(this.baseDirectory, entry.name); const stats = await lstatIfPresent(filePath); if (!stats) continue; if ( !stats.isFile() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 ) throw new Error(`Upgrade transaction adoption temporary is unsafe: ${filePath}`); if (Date.now() - stats.mtimeMs > initializationGraceMs) { await plugins.fs.promises.unlink(filePath).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); } continue; } if (entry.isFile() && transactionAdoptionAuditPattern.test(entry.name)) { const filePath = plugins.path.join(this.baseDirectory, entry.name); const stats = await lstatIfPresent(filePath); if (!stats) continue; if ( !stats.isFile() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 ) throw new Error(`Upgrade transaction adoption metadata is unsafe: ${filePath}`); parseUpgradeTransactionAdoptionAudit(await readPrivateJson(filePath)); if (Date.now() - stats.mtimeMs > transactionRetentionMs) { const lockState = await this.inspectOwnedDirectory(this.upgradeLockDirectory); if (lockState.state === 'absent' || lockState.state === 'stale') { await plugins.fs.promises.unlink(filePath).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); } } continue; } if (entry.isFile() && acknowledgementTemporaryPattern.test(entry.name)) { const filePath = plugins.path.join(this.baseDirectory, entry.name); const stats = await lstatIfPresent(filePath); if (!stats) continue; if ( !stats.isFile() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 ) throw new Error(`Upgrade acknowledgement temporary is unsafe: ${filePath}`); if (Date.now() - stats.mtimeMs > initializationGraceMs) { await plugins.fs.promises.unlink(filePath).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); } continue; } if (entry.isFile() && ownerTransferTemporaryPattern.test(entry.name)) { const filePath = plugins.path.join(this.baseDirectory, entry.name); const stats = await lstatIfPresent(filePath); if (!stats) continue; if ( !stats.isFile() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 || stats.nlink !== 1 ) throw new Error(`Upgrade owner-transfer temporary is unsafe: ${filePath}`); if (Date.now() - stats.mtimeMs > initializationGraceMs) { await plugins.fs.promises.unlink(filePath).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); await syncDirectory(this.baseDirectory); } continue; } if (entry.name.startsWith('owner-transfer-')) { throw new Error(`Upgrade owner-transfer temporary has an invalid name: ${entry.name}`); } if (entry.name.startsWith('adoption-')) { throw new Error(`Upgrade transaction adoption metadata has an invalid name: ${entry.name}`); } if (!entry.isFile() || !tokenMetadataPattern.test(entry.name)) continue; const filePath = plugins.path.join(this.baseDirectory, entry.name); const stats = await lstatIfPresent(filePath); if (!stats) continue; if ( !stats.isFile() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 ) { throw new Error(`Upgrade coordination metadata is unsafe: ${filePath}`); } if (Date.now() - stats.mtimeMs > tokenMetadataRetentionMs) { await plugins.fs.promises.unlink(filePath).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); } } for (const entry of await plugins.fs.promises.readdir(this.transactionsDirectory, { withFileTypes: true, })) { const filePath = plugins.path.join(this.transactionsDirectory, entry.name); if (entry.isFile() && transactionAdoptionPattern.test(entry.name)) { const stats = await lstatIfPresent(filePath); if (!stats) continue; if ( !stats.isFile() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 ) throw new Error(`Upgrade transaction adoption state is unsafe: ${filePath}`); parseUpgradeTransaction(await readPrivateJson(filePath)); continue; } if (entry.isFile() && transactionTemporaryMetadataPattern.test(entry.name)) { const stats = await lstatIfPresent(filePath); if (!stats) continue; if ( !stats.isFile() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 ) throw new Error(`Upgrade transaction temporary metadata is unsafe: ${filePath}`); if (Date.now() - stats.mtimeMs > initializationGraceMs) { await plugins.fs.promises.unlink(filePath).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); } continue; } if (entry.isDirectory() && transactionMutationDirectoryPattern.test(entry.name)) { const stats = await lstatIfPresent(filePath); if (!stats) continue; if ( !stats.isDirectory() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 ) throw new Error(`Upgrade transaction mutation metadata is unsafe: ${filePath}`); if (Date.now() - stats.mtimeMs > transactionRetentionMs) { await this.reclaimStaleTransactionMutationDirectory(filePath); } continue; } if (entry.isFile() && transactionMetadataPattern.test(entry.name)) { const stats = await lstatIfPresent(filePath); if (!stats) continue; if ( !stats.isFile() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 ) throw new Error(`Upgrade transaction metadata is unsafe: ${filePath}`); if (Date.now() - stats.mtimeMs > transactionRetentionMs) { const transaction = parseUpgradeTransaction(await readPrivateJson(filePath)); const workerIsLive = await this.transactionWorkerIsLive(transaction).catch(() => true); const controllerIsLive = transaction.controller ? await this.transactionControllerIsLive(transaction).catch(() => true) : false; if ( transaction.terminal || ( !upgradeTransactionRequiresForwardRecovery(transaction) && !workerIsLive && !controllerIsLive ) ) { await plugins.fs.promises.unlink(filePath).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); } } continue; } if (entry.name.startsWith('transaction-')) { throw new Error(`Upgrade transaction metadata has an invalid name: ${entry.name}`); } } } public async inspectCanonicalTransactionInventory(): Promise { if (this.usesLegacyLocation || this.usesInheritedLocation) { throw new Error('Canonical upgrade transaction inventory cannot use an inherited root.'); } const empty = (): IUpgradeTransactionInventory => ({ transactions: [], tokenMetadataFileNames: [], mutationDirectoryNames: [], }); const baseStats = await lstatIfPresent(this.baseDirectory); if (!baseStats) return empty(); await assertPrivateDirectory(this.baseDirectory); const tokenMetadataFileNames: string[] = []; for (const entry of await plugins.fs.promises.readdir(this.baseDirectory, { withFileTypes: true, })) { if (entry.isFile() && tokenMetadataPattern.test(entry.name)) { const filePath = plugins.path.join(this.baseDirectory, entry.name); await readPrivateJson(filePath); tokenMetadataFileNames.push(entry.name); continue; } if (entry.isFile() && acknowledgementTemporaryPattern.test(entry.name)) { await readPrivateJson(plugins.path.join(this.baseDirectory, entry.name)); tokenMetadataFileNames.push(entry.name); continue; } if (entry.isFile() && transactionAdoptionAuditPattern.test(entry.name)) { parseUpgradeTransactionAdoptionAudit( await readPrivateJson(plugins.path.join(this.baseDirectory, entry.name)), ); continue; } if ( entry.name.startsWith('grant-') || entry.name.startsWith('ack-') || entry.name.startsWith('action-') || entry.name.startsWith('adoption-') ) throw new Error(`Upgrade coordination metadata has an invalid name: ${entry.name}`); } const transactionsStats = await lstatIfPresent(this.transactionsDirectory); if (!transactionsStats) { return { transactions: [], tokenMetadataFileNames, mutationDirectoryNames: [] }; } await assertPrivateDirectory(this.transactionsDirectory); const transactions: IUpgradeTransactionInventoryEntry[] = []; const mutationDirectoryNames: string[] = []; for (const entry of await plugins.fs.promises.readdir(this.transactionsDirectory, { withFileTypes: true, })) { if (entry.isDirectory() && transactionMutationDirectoryPattern.test(entry.name)) { mutationDirectoryNames.push(entry.name); continue; } if (entry.isFile() && transactionTemporaryMetadataPattern.test(entry.name)) continue; const active = entry.isFile() && transactionMetadataPattern.test(entry.name); const adoptionMatch = entry.isFile() ? transactionAdoptionPattern.exec(entry.name) : null; if (!active && !adoptionMatch) { if (entry.name.startsWith('transaction-')) { throw new Error(`Upgrade transaction metadata has an invalid name: ${entry.name}`); } continue; } const filePath = plugins.path.join(this.transactionsDirectory, entry.name); const before = await lstatIfPresent(filePath); if (!before) continue; if ( !before.isFile() || before.isSymbolicLink() || before.uid !== this.uid || (before.mode & 0o077) !== 0 ) throw new Error(`Upgrade transaction metadata is unsafe: ${filePath}`); const transaction = parseUpgradeTransaction(await readPrivateJson(filePath)); const after = await lstatIfPresent(filePath); if ( !after || !sameOwnedDirectoryIdentity(after, { dev: before.dev, ino: before.ino }) || after.size !== before.size || after.mtimeMs !== before.mtimeMs ) throw new Error('Upgrade transaction metadata changed during inventory.'); const tokenPrefix = transaction.tokenHash.slice(0, 20); if (active && entry.name !== `transaction-${tokenPrefix}.json`) { throw new Error('Upgrade transaction filename and token binding do not match.'); } if (adoptionMatch && tokenPrefix !== adoptionMatch[1] && tokenPrefix !== adoptionMatch[2]) { throw new Error('Upgrade transaction adoption state has an invalid token binding.'); } transactions.push({ fileName: entry.name, state: active ? 'active' : 'adopting', transaction, }); } return { transactions: transactions.sort((leftArg, rightArg) => ( leftArg.transaction.createdAt - rightArg.transaction.createdAt || leftArg.fileName.localeCompare(rightArg.fileName) )), tokenMetadataFileNames: tokenMetadataFileNames.sort(), mutationDirectoryNames: mutationDirectoryNames.sort(), }; } private ownershipGuardDirectory(directoryArg: string): string { return plugins.path.join( this.ownershipGuardsDirectory, `${ownershipTargetHash(directoryArg)}.guard`, ); } private async inspectOwnershipGuardDirectory( directoryArg: string, expectedTargetHashArg: string, attemptArg = 0, ): Promise { const stats = await lstatIfPresent(directoryArg); if (!stats) return { state: 'absent' }; if ( !stats.isDirectory() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 ) throw new Error(`Upgrade ownership mutation guard is unsafe: ${directoryArg}`); let ownerMetadata: IOwnedDirectoryOwnerMetadata | undefined; try { ownerMetadata = await inspectOwnedDirectoryOwnerMetadata( directoryArg, parseOwnershipMutationGuardOwner, ); } catch (errorArg) { const currentStats = await lstatIfPresent(directoryArg); if (!currentStats) return { state: 'absent' }; if (!sameOwnedDirectoryIdentity(currentStats, { dev: stats.dev, ino: stats.ino })) { if (attemptArg < 2) { return await this.inspectOwnershipGuardDirectory( directoryArg, expectedTargetHashArg, attemptArg + 1, ); } throw new Error(`Upgrade ownership mutation guard changed repeatedly: ${directoryArg}`, { cause: errorArg, }); } if ((errorArg as NodeJS.ErrnoException).code === 'ENOENT') { return { state: Date.now() - currentStats.mtimeMs < initializationGraceMs ? 'live' : 'empty', identity: { dev: currentStats.dev, ino: currentStats.ino }, }; } throw errorArg; } if (!ownerMetadata) { return { state: Date.now() - stats.mtimeMs < initializationGraceMs ? 'live' : 'empty', identity: { dev: stats.dev, ino: stats.ino }, }; } const owner = ownerMetadata.owner; if (owner.uid !== this.uid || owner.targetHash !== expectedTargetHashArg) { throw new Error(`Upgrade ownership mutation guard binding is invalid: ${directoryArg}`); } let processIdentity: Awaited>; let removalProcessIdentity: Awaited>; try { processIdentity = await readControllerProcessIdentity(owner.pid); removalProcessIdentity = ownerMetadata.removalPid === undefined ? null : ownerMetadata.removalPid === owner.pid ? processIdentity : await readControllerProcessIdentity(ownerMetadata.removalPid); } catch (errorArg) { throw new Error(`The upgrade ownership mutation guard cannot be verified safely: ${directoryArg}`, { cause: errorArg, }); } const ownerIsStillLive = processIdentity?.fingerprint === owner.fingerprint; const removerIsStillLive = ownerMetadata.removalFingerprintHash ? removalProcessIdentity !== null && plugins.crypto.createHash('sha256') .update(removalProcessIdentity.fingerprint, 'utf8') .digest('hex') === ownerMetadata.removalFingerprintHash : ownerMetadata.removalPid !== undefined && removalProcessIdentity !== null && Date.now() - ownerMetadata.tombstoneCtimeMs! < legacyOwnerRemovalGraceMs; return { state: ownerIsStillLive || removerIsStillLive ? 'live' : 'stale', identity: { dev: stats.dev, ino: stats.ino }, owner, ownerMetadata, }; } private async reclaimOwnershipGuardDirectory( directoryArg: string, targetHashArg: string, ): Promise<'retry' | 'live'> { const inspected = await this.inspectOwnershipGuardDirectory(directoryArg, targetHashArg); if (inspected.state === 'absent') return 'retry'; if (inspected.state === 'live') return 'live'; const identity = inspected.identity!; if (inspected.state === 'empty') { const current = await lstatIfPresent(directoryArg); if (!current) return 'retry'; if (!sameOwnedDirectoryIdentity(current, identity)) { throw new Error('Upgrade ownership mutation guard changed before empty recovery.'); } if ((await plugins.fs.promises.readdir(directoryArg)).length !== 0) { throw new Error('Upgrade ownership mutation guard changed during empty recovery.'); } await plugins.fs.promises.rmdir(directoryArg).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); await syncDirectory(this.ownershipGuardsDirectory); return 'retry'; } await restoreOwnedDirectoryOwnerMetadata(directoryArg, inspected.ownerMetadata!); await removeExactOwnedDirectory(directoryArg, { identity, assertOwner: (valueArg) => { const removedOwner = parseOwnershipMutationGuardOwner(valueArg); if (!exactPrimitiveRecordMatches(removedOwner, inspected.owner!)) { throw new Error('Upgrade ownership mutation guard changed during stale recovery.'); } }, }); return 'retry'; } private async cleanupOwnershipGuardDirectories(): Promise { const entries = await plugins.fs.promises.readdir(this.ownershipGuardsDirectory, { withFileTypes: true, }); for (const entry of entries) { const path = plugins.path.join(this.ownershipGuardsDirectory, entry.name); if (ownershipGuardTemporaryPattern.test(entry.name)) continue; if (!entry.isDirectory() || !ownershipGuardDirectoryPattern.test(entry.name)) { throw new Error(`Upgrade ownership guard directory contains an unexpected entry: ${path}`); } await this.reclaimOwnershipGuardDirectory(path, entry.name.slice(0, 64)); } } private async cleanupOwnershipGuardTemporaryDirectories(): Promise { const entries = await plugins.fs.promises.readdir(this.ownershipGuardsDirectory, { withFileTypes: true, }); for (const entry of entries) { if (!ownershipGuardTemporaryPattern.test(entry.name)) continue; const match = /^([a-f0-9]{64})\.guard\.tmp-([1-9][0-9]*)-([a-f0-9]{32})$/.exec(entry.name); if (!match) throw new Error(`Upgrade ownership guard temporary name is invalid: ${entry.name}`); const pid = Number(match[2]); const path = plugins.path.join(this.ownershipGuardsDirectory, entry.name); const stats = await lstatIfPresent(path); if (!stats) continue; if ( !stats.isDirectory() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 ) throw new Error(`Upgrade ownership guard temporary is unsafe: ${path}`); const ownerPath = plugins.path.join(path, ownerFileName); const ownerStats = await lstatIfPresent(ownerPath); let owner: IUpgradeOwnershipMutationGuardOwner | undefined; if (ownerStats) { if ( !ownerStats.isFile() || ownerStats.isSymbolicLink() || ownerStats.uid !== this.uid || (ownerStats.mode & 0o077) !== 0 || ownerStats.nlink !== 1 ) throw new Error(`Upgrade ownership guard temporary owner is unsafe: ${ownerPath}`); try { owner = parseOwnershipMutationGuardOwner(await readPrivateJson(ownerPath)); } catch (errorArg) { if (await readControllerProcessIdentity(pid)) { throw new Error(`Upgrade ownership guard temporary is owned by live PID ${pid}.`, { cause: errorArg, }); } } } const live = await readControllerProcessIdentity(pid); if (live && (!owner || owner.pid !== pid || owner.fingerprint !== live.fingerprint)) { throw new Error(`Upgrade ownership guard temporary is owned by live PID ${pid}.`); } if (live) continue; if (owner && (owner.pid !== pid || owner.uid !== this.uid || owner.targetHash !== match[1])) { throw new Error(`Upgrade ownership guard temporary binding is invalid: ${path}`); } const stable = await lstatIfPresent(path); if (!stable || stable.dev !== stats.dev || stable.ino !== stats.ino) { throw new Error(`Upgrade ownership guard temporary changed before cleanup: ${path}`); } if (ownerStats) { const stableOwner = await lstatIfPresent(ownerPath); if (!stableOwner || stableOwner.dev !== ownerStats.dev || stableOwner.ino !== ownerStats.ino) { throw new Error(`Upgrade ownership guard temporary owner changed before cleanup: ${ownerPath}`); } await plugins.fs.promises.unlink(ownerPath); } if ((await plugins.fs.promises.readdir(path)).length !== 0) { throw new Error(`Upgrade ownership guard temporary contains unexpected entries: ${path}`); } await plugins.fs.promises.rmdir(path); await syncDirectory(this.ownershipGuardsDirectory); } } private async acquireOwnershipMutationGuard( directoryArg: string, ): Promise { const targetHash = ownershipTargetHash(directoryArg); const guardDirectory = this.ownershipGuardDirectory(directoryArg); const identity = await readControllerProcessIdentity(process.pid); if (!identity) throw new Error('Unable to establish upgrade ownership mutation guard identity.'); const owner: IUpgradeOwnershipMutationGuardOwner = { version: coordinationVersion, uid: this.uid, targetHash, pid: identity.pid, fingerprint: identity.fingerprint, nonce: plugins.crypto.randomBytes(16).toString('hex'), createdAt: Date.now(), }; const deadline = Date.now() + ownershipGuardTimeoutMs; const temporaryDirectory = `${guardDirectory}.tmp-${owner.pid}-${owner.nonce}`; while (true) { let temporaryCreated = false; let publishedIdentity: { dev: number; ino: number } | undefined; try { await plugins.fs.promises.mkdir(temporaryDirectory, { mode: 0o700 }); temporaryCreated = true; await writePrivateJson(plugins.path.join(temporaryDirectory, ownerFileName), owner); await syncDirectory(temporaryDirectory); const temporaryStats = await plugins.fs.promises.lstat(temporaryDirectory); const temporaryIdentity = { dev: temporaryStats.dev, ino: temporaryStats.ino }; await plugins.fs.promises.rename(temporaryDirectory, guardDirectory); temporaryCreated = false; publishedIdentity = temporaryIdentity; const publishedStats = await plugins.fs.promises.lstat(guardDirectory); if (!sameOwnedDirectoryIdentity(publishedStats, publishedIdentity)) { throw new Error('Published upgrade ownership guard identity changed unexpectedly.'); } await syncDirectory(this.ownershipGuardsDirectory); return new UpgradeOwnershipMutationGuard(guardDirectory, owner, publishedIdentity); } catch (errorArg) { const cleanupErrors: unknown[] = []; if (temporaryCreated) { try { await plugins.fs.promises.rm(temporaryDirectory, { recursive: true, force: true }); } catch (cleanupErrorArg) { cleanupErrors.push(cleanupErrorArg); } } if (publishedIdentity) { try { await removeExactOwnedDirectory(guardDirectory, { identity: publishedIdentity, assertOwner: (valueArg) => { const publishedOwner = parseOwnershipMutationGuardOwner(valueArg); if (!exactPrimitiveRecordMatches(publishedOwner, owner)) { throw new Error('Published upgrade ownership guard changed during cleanup.'); } }, }); } catch (cleanupErrorArg) { cleanupErrors.push(cleanupErrorArg); } } if (cleanupErrors.length > 0) { throw new AggregateError( [errorArg, ...cleanupErrors], 'Upgrade ownership guard publication failed and cleanup was incomplete.', { cause: errorArg }, ); } const code = (errorArg as NodeJS.ErrnoException).code; if (code !== 'EEXIST' && code !== 'ENOTEMPTY') throw errorArg; } if (await this.reclaimOwnershipGuardDirectory(guardDirectory, targetHash) === 'retry') { continue; } if (Date.now() >= deadline) { throw new Error('Timed out waiting for an upgrade ownership mutation guard.'); } await new Promise((resolve) => setTimeout(resolve, 25)); } } private async withOwnershipMutationGuard( directoryArg: string, operationArg: () => Promise, ): Promise { const guard = await this.acquireOwnershipMutationGuard(directoryArg); let result: T | undefined; let operationError: unknown; try { result = await operationArg(); } catch (errorArg) { operationError = errorArg; } let releaseError: unknown; try { await guard.release(); } catch (errorArg) { releaseError = errorArg; } if (operationError !== undefined && releaseError !== undefined) { throw new AggregateError( [operationError, releaseError], 'Upgrade ownership mutation failed and its guard could not be released.', ); } if (operationError !== undefined) throw operationError; if (releaseError !== undefined) throw releaseError; return result as T; } private async reclaimStaleOwnedDirectory(directoryArg: string): Promise { return await this.withOwnershipMutationGuard(directoryArg, async () => { const current = await this.inspectOwnedDirectory(directoryArg); if (current.state === 'absent') return false; if (current.state === 'indeterminate') { throw new Error(`The existing ${currentCliName} ownership cannot be verified safely.`); } if (current.state !== 'stale') return false; if (current.ownerMetadata) { await restoreOwnedDirectoryOwnerMetadata(directoryArg, current.ownerMetadata); } await removeExactOwnedDirectory(directoryArg, { identity: current.identity!, ...(current.owner ? { assertOwner: (valueArg: unknown) => { const removedOwner = parseOwner(valueArg); if (!exactPrimitiveRecordMatches(removedOwner, current.owner!)) { throw new Error('Upgrade coordination ownership changed during stale removal.'); } }, } : {}), }); return true; }); } private async releaseOwnedDirectory( directoryArg: string, expectedOwnerArg: IUpgradeProcessOwner, expectedIdentityArg: { dev: number; ino: number }, shouldReleaseArg?: () => boolean, ): Promise { await this.withOwnershipMutationGuard(directoryArg, async () => { const stats = await lstatIfPresent(directoryArg); if (!stats) return; if (!sameOwnedDirectoryIdentity(stats, expectedIdentityArg)) { throw new Error('Upgrade coordination ownership changed unexpectedly.'); } const metadata = await inspectOwnedDirectoryOwnerMetadata(directoryArg, parseOwner); if (metadata && !exactPrimitiveRecordMatches(metadata.owner, expectedOwnerArg)) { throw new Error('Upgrade coordination ownership changed unexpectedly.'); } if (shouldReleaseArg && !shouldReleaseArg()) { throw new Error('Upgrade coordination release is no longer authorized.'); } if (metadata) await restoreOwnedDirectoryOwnerMetadata(directoryArg, metadata); await removeExactOwnedDirectory(directoryArg, { identity: expectedIdentityArg, ...(metadata ? { assertOwner: (valueArg: unknown) => { const removedOwner = parseOwner(valueArg); if (!exactPrimitiveRecordMatches(removedOwner, expectedOwnerArg)) { throw new Error('Upgrade coordination ownership changed during release.'); } }, } : {}), }); }); } private async inspectTransactionMutationDirectory( directoryArg: string, expectedTokenHashArg?: string, attemptArg = 0, ): Promise<{ state: 'absent' | 'live' | 'recent-invalid' | 'stale' | 'indeterminate'; identity?: { dev: number; ino: number }; owner?: IUpgradeTransactionMutationOwner; ownerMetadata?: IOwnedDirectoryOwnerMetadata; inspectionError?: unknown; }> { const stats = await lstatIfPresent(directoryArg); if (!stats) return { state: 'absent' }; if ( !stats.isDirectory() || stats.isSymbolicLink() || stats.uid !== this.uid || (stats.mode & 0o077) !== 0 ) throw new Error('The upgrade transaction mutation lock is unsafe.'); let ownerMetadata: IOwnedDirectoryOwnerMetadata | undefined; try { ownerMetadata = await inspectOwnedDirectoryOwnerMetadata( directoryArg, parseTransactionMutationOwner, ); } catch (errorArg) { const currentStats = await lstatIfPresent(directoryArg); if (!currentStats) return { state: 'absent' }; if (!sameOwnedDirectoryIdentity(currentStats, { dev: stats.dev, ino: stats.ino })) { if (attemptArg < 2) { return await this.inspectTransactionMutationDirectory( directoryArg, expectedTokenHashArg, attemptArg + 1, ); } return { state: 'indeterminate', identity: { dev: currentStats.dev, ino: currentStats.ino }, inspectionError: errorArg, }; } const code = (errorArg as NodeJS.ErrnoException).code; if (code === 'ENOENT') { return { state: Date.now() - currentStats.mtimeMs < initializationGraceMs ? 'recent-invalid' : 'stale', identity: { dev: currentStats.dev, ino: currentStats.ino }, }; } if ( Date.now() - currentStats.mtimeMs < initializationGraceMs && (code === undefined || code === 'HCON_INVALID_JSON') ) { return { state: 'recent-invalid', identity: { dev: currentStats.dev, ino: currentStats.ino }, }; } return { state: 'indeterminate', identity: { dev: currentStats.dev, ino: currentStats.ino }, inspectionError: errorArg, }; } if (!ownerMetadata) { return { state: Date.now() - stats.mtimeMs < initializationGraceMs ? 'recent-invalid' : 'stale', identity: { dev: stats.dev, ino: stats.ino }, }; } const owner = ownerMetadata.owner; if ( owner.uid !== this.uid || (expectedTokenHashArg !== undefined && !tokensEqual(owner.tokenHash, expectedTokenHashArg)) ) throw new Error('The upgrade transaction mutation lock binding is invalid.'); try { const ownerIdentity = await readControllerProcessIdentity(owner.pid); return { state: ownerIdentity?.fingerprint === owner.fingerprint ? 'live' : 'stale', identity: { dev: stats.dev, ino: stats.ino }, owner, ownerMetadata, }; } catch (errorArg) { return { state: 'indeterminate', identity: { dev: stats.dev, ino: stats.ino }, inspectionError: errorArg, }; } } private async reclaimStaleTransactionMutationDirectory( directoryArg: string, expectedTokenHashArg?: string, ): Promise { return await this.withOwnershipMutationGuard(directoryArg, async () => { const current = await this.inspectTransactionMutationDirectory( directoryArg, expectedTokenHashArg, ); if (current.state === 'absent') return false; if (current.state === 'indeterminate') { throw new Error('The upgrade transaction mutation owner cannot be verified safely.', { cause: current.inspectionError, }); } if (current.state !== 'stale') return false; if (current.ownerMetadata) { await restoreOwnedDirectoryOwnerMetadata(directoryArg, current.ownerMetadata); } await removeExactOwnedDirectory(directoryArg, { identity: current.identity!, ...(current.owner ? { assertOwner: (valueArg: unknown) => { const removedOwner = parseTransactionMutationOwner(valueArg); if (!exactPrimitiveRecordMatches(removedOwner, current.owner!)) { throw new Error( 'The upgrade transaction mutation owner changed during stale removal.', ); } }, } : {}), }); return true; }); } private async inspectOwnedDirectory( directoryArg: string, ): Promise<{ state: 'absent' | 'live' | 'recent-invalid' | 'stale' | 'indeterminate'; identity?: { dev: number; ino: number }; owner?: IUpgradeProcessOwner; ownerMetadata?: IOwnedDirectoryOwnerMetadata; }> { let stats: plugins.fs.Stats; try { stats = await plugins.fs.promises.lstat(directoryArg); } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code === 'ENOENT') return { state: 'absent' }; throw errorArg; } if (!stats.isDirectory() || stats.isSymbolicLink()) { throw new Error(`Upgrade coordination owner path is unsafe: ${directoryArg}`); } if (stats.uid !== this.uid || (stats.mode & 0o077) !== 0) { throw new Error(`Upgrade coordination owner path is not private: ${directoryArg}`); } let ownerMetadata: IOwnedDirectoryOwnerMetadata | undefined; try { ownerMetadata = await inspectOwnedDirectoryOwnerMetadata(directoryArg, parseOwner); } catch { return { state: 'indeterminate', identity: { dev: stats.dev, ino: stats.ino } }; } if (!ownerMetadata) { return { state: Date.now() - stats.mtimeMs < initializationGraceMs ? 'recent-invalid' : 'stale', identity: { dev: stats.dev, ino: stats.ino }, }; } const owner = ownerMetadata.owner; try { return { state: await ownerIsLive(owner) ? 'live' : 'stale', identity: { dev: stats.dev, ino: stats.ino }, owner, ownerMetadata, }; } catch { return { state: 'indeterminate', identity: { dev: stats.dev, ino: stats.ino } }; } } private async createOwnedDirectory( directoryArg: string, ownerArg: IUpgradeProcessOwner, ): Promise<{ directory: string; owner: IUpgradeProcessOwner; identity: { dev: number; ino: number }; } | undefined> { return await this.withOwnershipMutationGuard(directoryArg, async () => { try { await plugins.fs.promises.mkdir(directoryArg, { mode: 0o700 }); } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code === 'EEXIST') return undefined; throw errorArg; } const stats = await plugins.fs.promises.lstat(directoryArg); const identity = { dev: stats.dev, ino: stats.ino }; try { await writePrivateJson(plugins.path.join(directoryArg, ownerFileName), ownerArg); await syncDirectory(plugins.path.dirname(directoryArg)); return { directory: directoryArg, owner: ownerArg, identity, }; } catch (errorArg) { try { await removeExactFailedOwnerPublication(directoryArg, identity); } catch (cleanupErrorArg) { throw new AggregateError( [errorArg, cleanupErrorArg], 'Upgrade coordination owner publication failed and cleanup was incomplete.', { cause: errorArg }, ); } throw errorArg; } }); } public async acquireUpgradeLock(optionsArg: { token: string; cliPath: string; command?: '__serve' | '__upgrade-worker' | 'upgrade' | 'foreground'; }): Promise { await this.init(); const owner = await ownerForCurrentProcess( 'upgrade', optionsArg.token, optionsArg.cliPath, optionsArg.command ?? '__upgrade-worker', ); for (let attempt = 0; attempt < 4; attempt++) { const created = await this.createOwnedDirectory(this.upgradeLockDirectory, owner); if (created) return new UpgradeInstallationLock( created.directory, created.owner, (directoryArg, ownerArg, shouldReleaseArg) => this.releaseOwnedDirectory( directoryArg, ownerArg, created.identity, shouldReleaseArg, ), ); const state = await this.inspectOwnedDirectory(this.upgradeLockDirectory); if (state.state === 'live') { const currentOwner = state.owner!; if ( currentOwner.pid === owner.pid && currentOwner.fingerprint === owner.fingerprint && tokensEqual(currentOwner.tokenHash, owner.tokenHash) ) return new UpgradeInstallationLock( this.upgradeLockDirectory, currentOwner, (directoryArg, ownerArg, shouldReleaseArg) => this.releaseOwnedDirectory( directoryArg, ownerArg, state.identity!, shouldReleaseArg, ), ); throw new Error(`Another ${currentCliName} upgrade is already running.`); } if (state.state === 'recent-invalid') { throw new Error(`Another ${currentCliName} upgrade is initializing; retry shortly.`); } if (state.state === 'stale') { await this.reclaimStaleOwnedDirectory(this.upgradeLockDirectory); } if (state.state === 'indeterminate') { throw new Error(`The existing ${currentCliName} upgrade owner cannot be verified safely.`); } } throw new Error(`Unable to acquire exclusive ${currentCliName} upgrade ownership.`); } public async transferUpgradeLockToWorker(optionsArg: { lock: UpgradeInstallationLock; token: string; worker: NonNullable; }): Promise { const identity = await readControllerProcessIdentity(optionsArg.worker.pid); if ( !identity || identity.fingerprint !== optionsArg.worker.fingerprint || identity.processGroupId !== optionsArg.worker.processGroupId || !identity.processGroupLeader || !await processIdentityHasCliCommand( identity, optionsArg.worker.cliPath, '__upgrade-worker', { allowMissingAbsolutePath: true }, ) ) throw new Error('The upgrade lock transfer worker identity is invalid.'); const owner: IUpgradeProcessOwner = { version: coordinationVersion, kind: 'upgrade', uid: this.uid, tokenHash: hashToken(optionsArg.token), pid: identity.pid, processGroupId: identity.processGroupId, fingerprint: identity.fingerprint, cliPath: optionsArg.worker.cliPath, command: '__upgrade-worker', createdAt: Date.now(), }; await this.withOwnershipMutationGuard(this.upgradeLockDirectory, async () => { await optionsArg.lock.assertOwned(); const ownerPath = plugins.path.join(this.upgradeLockDirectory, ownerFileName); const temporaryPath = plugins.path.join( this.baseDirectory, `owner-transfer-${process.pid}-${plugins.crypto.randomBytes(4).toString('hex')}.json.tmp`, ); try { await writePrivateJson(temporaryPath, owner); await plugins.fs.promises.rename(temporaryPath, ownerPath); await syncDirectory(this.upgradeLockDirectory); await syncDirectory(this.baseDirectory); optionsArg.lock.relinquishAfterTransfer(); } finally { let removed = false; await plugins.fs.promises.unlink(temporaryPath).then(() => { removed = true; }).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); if (removed) await syncDirectory(this.baseDirectory); } }); } private async readLiveUpgradeOwner(): Promise { for (let attempt = 0; attempt < 4; attempt++) { const state = await this.inspectOwnedDirectory(this.upgradeLockDirectory); if (state.state === 'absent') return undefined; if (state.state === 'recent-invalid') { throw new Error(`An ${currentCliName} upgrade is initializing; retry shortly.`); } if (state.state === 'indeterminate') { throw new Error(`The existing ${currentCliName} upgrade owner cannot be verified safely.`); } if (state.state === 'stale') { await this.reclaimStaleOwnedDirectory(this.upgradeLockDirectory); continue; } return state.owner!; } throw new Error(`The ${currentCliName} upgrade owner changed repeatedly during admission.`); } public async acquireStartLease(optionsArg: { token: string; cliPath: string; command: 'start' | 'foreground' | 'temp-password'; upgradeToken?: string; }): Promise { await this.init(); const owner = await ownerForCurrentProcess( 'start', optionsArg.token, optionsArg.cliPath, optionsArg.command, ); const upgradeOwnerBefore = await this.readLiveUpgradeOwner(); const authorized = upgradeOwnerBefore !== undefined && optionsArg.upgradeToken !== undefined && tokensEqual(upgradeOwnerBefore.tokenHash, hashToken(optionsArg.upgradeToken)); if (upgradeOwnerBefore && !authorized) { throw new Error(`${currentCliName} cannot start while a global upgrade is in progress.`); } const leaseDirectory = plugins.path.join( this.startLeasesDirectory, coordinationPathHash(assertToken(optionsArg.token)), ); const created = await this.createOwnedDirectory(leaseDirectory, owner); if (!created) throw new Error(`The ${currentCliName} start admission token is already in use.`); const lease = new UpgradeStartLease( created.directory, created.owner, (directoryArg, ownerArg, shouldReleaseArg) => this.releaseOwnedDirectory( directoryArg, ownerArg, created.identity, shouldReleaseArg, ), ); try { const upgradeOwnerAfter = await this.readLiveUpgradeOwner(); if (upgradeOwnerAfter) { const remainsAuthorized = optionsArg.upgradeToken !== undefined && tokensEqual(upgradeOwnerAfter.tokenHash, hashToken(optionsArg.upgradeToken)); if (!remainsAuthorized) { throw new Error(`${currentCliName} upgrade began while startup was entering admission.`); } } return lease; } catch (errorArg) { await lease.release(); throw errorArg; } } public async waitForStartLeasesToDrain(lockArg: UpgradeInstallationLock): Promise { const deadline = Date.now() + startLeaseDrainTimeoutMs; while (Date.now() < deadline) { await lockArg.assertOwned(); const entries = await plugins.fs.promises.readdir(this.startLeasesDirectory, { withFileTypes: true, }); let blocked = false; for (const entry of entries) { if (!entry.isDirectory() || !leaseDirectoryPattern.test(entry.name)) { throw new Error(`The ${currentCliName} start admission directory contains an unexpected entry.`); } const leaseDirectory = plugins.path.join(this.startLeasesDirectory, entry.name); const state = await this.inspectOwnedDirectory(leaseDirectory); if (state.state === 'live' || state.state === 'recent-invalid') { blocked = true; continue; } if (state.state === 'indeterminate') { throw new Error(`An ${currentCliName} start admission owner cannot be verified safely.`); } if (state.state === 'stale') { if (!await this.reclaimStaleOwnedDirectory(leaseDirectory)) blocked = true; } } if (!blocked) return; await new Promise((resolve) => setTimeout(resolve, 100)); } throw new Error(`Timed out waiting for an in-flight ${currentCliName} start to finish.`); } private tokenFilePath(kindArg: 'grant' | 'ack', tokenArg: string): string { return plugins.path.join( this.baseDirectory, `${kindArg}-${coordinationPathHash(assertToken(tokenArg))}.json`, ); } private transactionFilePath(tokenArg: string): string { return plugins.path.join( this.transactionsDirectory, `transaction-${coordinationPathHash(assertToken(tokenArg))}.json`, ); } private transactionMutationDirectory(tokenArg: string): string { return `${this.transactionFilePath(tokenArg)}.lock`; } private async withTransactionMutation( tokenArg: string, operationArg: () => Promise, ): Promise { await this.init(); const lockDirectory = this.transactionMutationDirectory(tokenArg); const deadline = Date.now() + transactionMutationTimeoutMs; const identity = await readControllerProcessIdentity(process.pid); if (!identity) throw new Error('Unable to establish transaction mutation ownership.'); const owner: IUpgradeTransactionMutationOwner = { version: coordinationVersion, uid: this.uid, tokenHash: hashToken(tokenArg), pid: identity.pid, fingerprint: identity.fingerprint, nonce: plugins.crypto.randomBytes(16).toString('hex'), createdAt: Date.now(), }; while (true) { const acquisition = await this.withOwnershipMutationGuard(lockDirectory, async () => { try { await plugins.fs.promises.mkdir(lockDirectory, { mode: 0o700 }); } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code !== 'EEXIST') throw errorArg; const current = await this.inspectTransactionMutationDirectory( lockDirectory, owner.tokenHash, ); return { state: 'contended' as const, current }; } const lockStats = await plugins.fs.promises.lstat(lockDirectory); const lockIdentity = { dev: lockStats.dev, ino: lockStats.ino }; try { await writePrivateJson(plugins.path.join(lockDirectory, ownerFileName), owner); return { state: 'acquired' as const }; } catch (errorArg) { try { await removeExactFailedOwnerPublication(lockDirectory, lockIdentity); } catch (cleanupErrorArg) { throw new AggregateError( [errorArg, cleanupErrorArg], 'Upgrade transaction mutation owner publication failed and cleanup was incomplete.', { cause: errorArg }, ); } throw errorArg; } }); if (acquisition.state === 'acquired') break; const current = acquisition.current; if (current.state === 'absent') continue; if (current.state === 'stale') { await this.reclaimStaleTransactionMutationDirectory( lockDirectory, owner.tokenHash, ); continue; } if (current.state === 'indeterminate') { throw new Error('The upgrade transaction mutation owner cannot be verified safely.', { cause: current.inspectionError, }); } if (Date.now() >= deadline) { throw new Error('Timed out waiting to update the upgrade transaction.'); } await new Promise((resolve) => setTimeout(resolve, 25)); } let result: T | undefined; let operationError: unknown; try { result = await operationArg(); } catch (errorArg) { operationError = errorArg; } let releaseError: unknown; try { await this.withOwnershipMutationGuard(lockDirectory, async () => { const stats = await plugins.fs.promises.lstat(lockDirectory); const metadata = await inspectOwnedDirectoryOwnerMetadata( lockDirectory, parseTransactionMutationOwner, ); if (!metadata) { throw new Error('The upgrade transaction mutation lock ownership changed unexpectedly.'); } const currentOwner = metadata.owner; if (!exactPrimitiveRecordMatches(currentOwner, owner)) { throw new Error('The upgrade transaction mutation lock ownership changed unexpectedly.'); } await restoreOwnedDirectoryOwnerMetadata(lockDirectory, metadata); await removeExactOwnedDirectory(lockDirectory, { identity: { dev: stats.dev, ino: stats.ino }, assertOwner: (valueArg) => { const removedOwner = parseTransactionMutationOwner(valueArg); if (!exactPrimitiveRecordMatches(removedOwner, owner)) { throw new Error( 'The upgrade transaction mutation lock ownership changed during release.', ); } }, }); }); } catch (errorArg) { releaseError = errorArg; } if (operationError !== undefined && releaseError !== undefined) { throw new AggregateError( [operationError, releaseError], 'The upgrade transaction mutation failed and its ownership could not be released.', ); } if (operationError !== undefined) throw operationError; if (releaseError !== undefined) throw releaseError; return result as T; } private async cleanupUncommittedAdoptionAuditsUnderLock( lockArg: UpgradeInstallationLock, ): Promise { await lockArg.assertOwned(); for (const entry of await plugins.fs.promises.readdir(this.baseDirectory, { withFileTypes: true, })) { const match = entry.isFile() ? transactionAdoptionAuditPattern.exec(entry.name) : null; if (!match) continue; const auditPath = plugins.path.join(this.baseDirectory, entry.name); const audit = parseUpgradeTransactionAdoptionAudit(await readPrivateJson(auditPath)); const sourcePath = plugins.path.join( this.transactionsDirectory, `transaction-${match[1]}.json`, ); const intermediatePath = `${sourcePath}.adopting-${match[2]}`; const targetPath = plugins.path.join( this.transactionsDirectory, `transaction-${match[2]}.json`, ); await this.withOwnershipMutationGuard(sourcePath, async () => { await lockArg.assertOwned(); const [sourceStats, intermediateStats, targetStats] = await Promise.all([ lstatIfPresent(sourcePath), lstatIfPresent(intermediatePath), lstatIfPresent(targetPath), ]); if (!sourceStats || intermediateStats || targetStats) return; const source = parseUpgradeTransaction(await readPrivateJson(sourcePath)); if (!plugins.util.isDeepStrictEqual(source, audit.previousTransaction)) return; await plugins.fs.promises.unlink(auditPath); await syncDirectory(this.baseDirectory); }); } await lockArg.assertOwned(); } public async adoptOrphanedTransaction( optionsArg: IAdoptOrphanedUpgradeTransactionOptions, ): Promise { await optionsArg.lock.assertOwned(); await this.waitForStartLeasesToDrain(optionsArg.lock); await this.cleanupUncommittedAdoptionAuditsUnderLock(optionsArg.lock); const inventory = await this.inspectCanonicalTransactionInventory(); const nonterminal = inventory.transactions.filter((entryArg) => !entryArg.transaction.terminal); if (nonterminal.length !== 1) { throw new Error('Exactly one orphaned upgrade transaction is required for adoption.'); } const entry = nonterminal[0]; const transaction = entry.transaction; const eligibleV2 = transaction.version === coordinationVersion && transaction.controllerWasRunning === true && transaction.targetStartupInvoked === true && transaction.targetVersion !== undefined && transaction.preparationCompletedAt !== undefined && (transaction.phase === 'restarting' || transaction.phase === 'continuing'); const eligibleV3 = transaction.version === upgradePackageTransitionTransactionVersion && transaction.targetPackageCommitStarted === true && (!transaction.controllerWasRunning || transaction.preparationCompletedAt !== undefined) && ( transaction.phase === 'installing' || transaction.phase === 'restarting' || transaction.phase === 'continuing' ); if ( transaction.port !== optionsArg.port || transaction.tokenHash !== optionsArg.expectedTokenHash || transaction.revision !== optionsArg.expectedRevision || (!eligibleV2 && !eligibleV3) || !upgradeTransactionIsStalled(transaction) ) throw new Error('The retained upgrade transaction is not eligible for adoption.'); if (transaction.version === upgradePackageTransitionTransactionVersion && optionsArg.registryUrl) { throw new Error( 'Legacy package-transition recovery uses pnpm registry configuration and cannot retain --registry.', ); } if (await this.transactionWorkerIsLive(transaction)) { throw new Error('The retained upgrade worker is still running.'); } if (transaction.controller && await this.transactionControllerIsLive(transaction)) { throw new Error('The retained upgrade controller is still running.'); } const adoptionMatch = transactionAdoptionPattern.exec(entry.fileName); const tokenPrefixes = new Set([ transaction.tokenHash.slice(0, 20), ...(adoptionMatch ? [adoptionMatch[1], adoptionMatch[2]] : []), ]); for (const fileName of inventory.tokenMetadataFileNames) { if ([...tokenPrefixes].some((prefixArg) => fileName.includes(`-${prefixArg}.json`))) { throw new Error('The orphaned upgrade retains token-bound coordination metadata.'); } } for (const directoryName of inventory.mutationDirectoryNames) { const prefix = [...tokenPrefixes].find((candidateArg) => ( directoryName === `transaction-${candidateArg}.json.lock` )); if (!prefix) continue; const directory = plugins.path.join(this.transactionsDirectory, directoryName); const state = await this.inspectTransactionMutationDirectory( directory, transaction.tokenHash, ); if (state.state === 'stale') { await this.reclaimStaleTransactionMutationDirectory(directory, transaction.tokenHash); continue; } if (state.state !== 'absent') { throw new Error('The orphaned upgrade transaction still has mutation ownership.'); } } const installedVersion = assertBoundedText( optionsArg.installedVersion, 'The installed upgrade version', 128, ); compareUpgradeSemver(installedVersion, installedVersion); if ( transaction.version === upgradePackageTransitionTransactionVersion && compareUpgradeSemver( installedVersion, upgradeTransactionTargetVersion(transaction)!, ) < 0 ) throw new Error('The installed package-transition recovery target would downgrade AGL.'); const registryUrl = transaction.version === coordinationVersion ? transaction.registryUrl ?? optionsArg.registryUrl : undefined; if ( transaction.version === coordinationVersion && transaction.registryUrl !== undefined && optionsArg.registryUrl !== undefined && transaction.registryUrl !== optionsArg.registryUrl ) throw new Error('The orphaned upgrade registry cannot change during adoption.'); const newTokenHash = hashToken(optionsArg.token); const sourcePath = plugins.path.join(this.transactionsDirectory, entry.fileName); const sourcePrefix = transaction.tokenHash.slice(0, 20); const targetPrefix = newTokenHash.slice(0, 20); if (sourcePrefix === targetPrefix) { throw new Error('The adopted upgrade token collided with the retained token.'); } const intermediatePath = plugins.path.join( this.transactionsDirectory, `transaction-${sourcePrefix}.json.adopting-${targetPrefix}`, ); const targetPath = plugins.path.join( this.transactionsDirectory, `transaction-${targetPrefix}.json`, ); const auditPath = plugins.path.join( this.baseDirectory, `adoption-${sourcePrefix}-${targetPrefix}.json`, ); const auditTemporaryPath = `${auditPath}.tmp-${process.pid}-${plugins.crypto.randomBytes(4).toString('hex')}`; for (const path of [intermediatePath, targetPath, auditPath, auditTemporaryPath]) { if (await lstatIfPresent(path)) { throw new Error(`Upgrade transaction adoption destination already exists: ${path}`); } } const now = Date.now(); const { worker: _worker, controller: _controller, retainedUntil: _retainedUntil, ...retained } = transaction; const adopted = transaction.version === coordinationVersion ? parseUpgradeTransactionV2({ ...retained, tokenHash: newTokenHash, revision: transaction.revision + 1, targetVersion: installedVersion, ...(registryUrl === undefined ? {} : { registryUrl }), phaseStartedAt: now, updatedAt: now, message: `Recovering orphaned upgrade ${transaction.sourceVersion} -> ${transaction.targetVersion} with installed ${installedVersion}.`, }) : parseUpgradeTransactionV3({ ...retained, tokenHash: newTokenHash, revision: transaction.revision + 1, ...(installedVersion === transaction.targetVersion ? {} : { recoveryTargetVersion: installedVersion }), phaseStartedAt: now, updatedAt: now, message: `Recovering orphaned package transition to installed ${transaction.targetPackageName}@${installedVersion}.`, }); const audit = parseUpgradeTransactionAdoptionAudit({ version: 1, adoptedAt: now, adoptedTargetVersion: installedVersion, previousTransaction: transaction, }); const temporaryPath = `${targetPath}.tmp-${process.pid}-${plugins.crypto.randomBytes(4).toString('hex')}`; await optionsArg.lock.assertOwned(); try { await this.withOwnershipMutationGuard(sourcePath, async () => { await this.withOwnershipMutationGuard(intermediatePath, async () => { await this.withOwnershipMutationGuard(targetPath, async () => { const sourceStats = await lstatIfPresent(sourcePath); if (!sourceStats) throw new Error('The orphaned upgrade transaction disappeared.'); const current = parseUpgradeTransaction(await readPrivateJson(sourcePath)); if ( current.tokenHash !== transaction.tokenHash || current.revision !== transaction.revision ) throw new Error('The orphaned upgrade transaction changed before adoption.'); try { await writePrivateJson(auditTemporaryPath, audit); await plugins.fs.promises.rename(auditTemporaryPath, auditPath); await syncDirectory(this.baseDirectory); await plugins.fs.promises.rename(sourcePath, intermediatePath); await syncDirectory(this.transactionsDirectory); await writePrivateJson(temporaryPath, adopted); const intermediateStats = await lstatIfPresent(intermediatePath); if ( !intermediateStats || !sameOwnedDirectoryIdentity(intermediateStats, { dev: sourceStats.dev, ino: sourceStats.ino, }) ) throw new Error('The upgrade adoption source identity changed unexpectedly.'); await plugins.fs.promises.rename(temporaryPath, intermediatePath); await syncDirectory(this.transactionsDirectory); await plugins.fs.promises.rename(intermediatePath, targetPath); await syncDirectory(this.transactionsDirectory); } finally { await plugins.fs.promises.unlink(auditTemporaryPath).catch(() => undefined); await plugins.fs.promises.unlink(temporaryPath).catch(() => undefined); } }); }); }); } catch (errorArg) { const [sourceStats, intermediateStats, targetStats] = await Promise.all([ lstatIfPresent(sourcePath), lstatIfPresent(intermediatePath), lstatIfPresent(targetPath), ]); if (sourceStats && !intermediateStats && !targetStats) { await plugins.fs.promises.unlink(auditPath).catch((cleanupErrorArg) => { if ((cleanupErrorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw cleanupErrorArg; }); await syncDirectory(this.baseDirectory); } throw errorArg; } await optionsArg.lock.assertOwned(); return parseUpgradeTransaction(await readPrivateJson(targetPath)); } public async createTransaction(optionsArg: { token: string; port: number; sourceVersion: string; registryUrl?: string; controllerWasRunning: boolean; continueSessions: boolean; gracePeriodMs: number; }): Promise { await this.init(); const now = Date.now(); const transaction: IUpgradeTransactionV2 = { version: coordinationVersion, tokenHash: hashToken(optionsArg.token), revision: 0, port: assertPort(optionsArg.port), sourceVersion: assertBoundedText(optionsArg.sourceVersion, 'The upgrade source version', 128), ...(optionsArg.registryUrl === undefined ? {} : { registryUrl: normalizeUpgradeRegistryUrl(optionsArg.registryUrl) }), controllerWasRunning: optionsArg.controllerWasRunning, continueSessions: optionsArg.continueSessions, gracePeriodMs: optionsArg.gracePeriodMs, phase: 'starting', message: 'Upgrade worker is starting.', createdAt: now, phaseStartedAt: now, updatedAt: now, sessions: [], }; const validated = parseUpgradeTransactionV2(transaction); await writePrivateJson(this.transactionFilePath(optionsArg.token), validated); return validated; } public async createPackageTransitionTransaction(optionsArg: { token: string; port: number; controllerWasRunning: boolean; continueSessions: boolean; gracePeriodMs: number; }): Promise { await this.init(); const now = Date.now(); const transaction: IUpgradeTransactionV3 = { version: upgradePackageTransitionTransactionVersion, tokenHash: hashToken(optionsArg.token), revision: 0, port: assertPort(optionsArg.port), sourcePackageName: upgradePackageTransitionSource.packageName, sourceVersion: upgradePackageTransitionSource.version, sourceManagementVersion: upgradePackageTransitionSource.managementVersion, sourceCliName: upgradePackageTransitionSource.cliName, sourceCliRelativePath: upgradePackageTransitionSource.cliRelativePath, targetPackageName: upgradePackageTransitionTarget.packageName, targetVersion: upgradePackageTransitionTarget.version, targetManagementVersion: upgradePackageTransitionTarget.managementVersion, targetCliName: upgradePackageTransitionTarget.cliName, targetCliRelativePath: upgradePackageTransitionTarget.cliRelativePath, controllerWasRunning: optionsArg.controllerWasRunning, continueSessions: optionsArg.continueSessions, gracePeriodMs: optionsArg.gracePeriodMs, phase: 'starting', message: 'Upgrade worker is starting.', createdAt: now, phaseStartedAt: now, updatedAt: now, sessions: [], }; const validated = parseUpgradeTransactionV3(transaction); await writePrivateJson(this.transactionFilePath(optionsArg.token), validated); return validated; } public async readTransaction(tokenArg: string): Promise { await this.init(); const transaction = parseUpgradeTransaction(await readPrivateJson( this.transactionFilePath(tokenArg), )); if (!tokensEqual(transaction.tokenHash, hashToken(tokenArg))) { throw new Error('The upgrade transaction token binding is invalid.'); } return transaction; } public async updateTransaction( tokenArg: string, expectedRevisionArg: number, updateArg: (currentArg: TUpgradeTransaction) => TUpgradeTransaction, ): Promise { return await this.withTransactionMutation(tokenArg, async () => { const current = await this.readTransaction(tokenArg); if (current.revision !== expectedRevisionArg) { throw new Error('The upgrade transaction changed concurrently.'); } return await this.writeTransactionUpdate(tokenArg, current, updateArg(current)); }); } public async mutateTransaction( tokenArg: string, updateArg: (currentArg: TUpgradeTransaction) => TUpgradeTransaction, ): Promise { return await this.withTransactionMutation(tokenArg, async () => { const current = await this.readTransaction(tokenArg); return await this.writeTransactionUpdate(tokenArg, current, updateArg(current)); }); } private async writeTransactionUpdate( tokenArg: string, currentArg: TUpgradeTransaction, updatedArg: TUpgradeTransaction, ): Promise { const updatedAt = Date.now(); const workerIdentityChanged = ( currentArg.worker?.pid !== updatedArg.worker?.pid || currentArg.worker?.processGroupId !== updatedArg.worker?.processGroupId || currentArg.worker?.fingerprint !== updatedArg.worker?.fingerprint || currentArg.worker?.cliPath !== updatedArg.worker?.cliPath ); const next = parseUpgradeTransaction({ ...updatedArg, version: currentArg.version, tokenHash: currentArg.tokenHash, revision: currentArg.revision + 1, createdAt: currentArg.createdAt, phaseStartedAt: updatedArg.phase === currentArg.phase && !workerIdentityChanged ? currentArg.phaseStartedAt : updatedAt, updatedAt, }); if (transactionPhaseOrder[next.phase] < transactionPhaseOrder[currentArg.phase]) { throw new Error('The upgrade transaction phase cannot regress.'); } if (currentArg.version === upgradePackageTransitionTransactionVersion) { if (next.version !== upgradePackageTransitionTransactionVersion) { throw new Error('The upgrade transaction version cannot change.'); } for (const checkpoint of [ 'packageTransitionStarted', 'groupedTargetVerified', 'targetPackageCommitStarted', 'targetPackageCommitted', 'targetStartupInvoked', ] as const) { if (currentArg[checkpoint] === true && next[checkpoint] !== true) { throw new Error(`The upgrade package-transition checkpoint ${checkpoint} cannot regress.`); } } if (currentArg.recoveryTargetVersion !== next.recoveryTargetVersion) { throw new Error('The package-transition recovery target cannot change after adoption.'); } } if ( currentArg.version === coordinationVersion && next.version === coordinationVersion && currentArg.registryUrl !== next.registryUrl ) throw new Error('The upgrade registry cannot change after the transaction is created.'); if ( (currentArg.preparationAcceptedAt !== undefined && next.preparationAcceptedAt !== currentArg.preparationAcceptedAt) || (currentArg.preparationDeadlineAt !== undefined && next.preparationDeadlineAt !== currentArg.preparationDeadlineAt) || (currentArg.preparationCompletedAt !== undefined && next.preparationCompletedAt !== currentArg.preparationCompletedAt) || (currentArg.preparationAcceptedAt === undefined && currentArg.preparationDeadlineAt === undefined && next.preparationCompletedAt !== undefined) ) throw new Error('The upgrade preparation timeline cannot change after it is committed.'); const targetPath = this.transactionFilePath(tokenArg); const temporaryPath = `${targetPath}.tmp-${process.pid}-${plugins.crypto.randomBytes(4).toString('hex')}`; try { await writePrivateJson(temporaryPath, next); await plugins.fs.promises.rename(temporaryPath, targetPath); await syncDirectory(this.transactionsDirectory); } finally { await plugins.fs.promises.unlink(temporaryPath).catch(() => undefined); } return next; } public async registerWorker(tokenArg: string, cliPathArg: string): Promise { const identity = await readControllerProcessIdentity(process.pid); if (!identity) throw new Error('Unable to establish upgrade worker ownership.'); return await this.mutateTransaction(tokenArg, (current) => ({ ...current, worker: { pid: identity.pid, processGroupId: identity.processGroupId, fingerprint: identity.fingerprint, cliPath: cliPathArg, ...(current.worker?.logFilePath ? { logFilePath: current.worker.logFilePath } : {}), }, phase: 'checking', message: 'Checking the registry latest version.', })); } public async recordWorkerLaunch( tokenArg: string, pidArg: number, cliPathArg: string, logFilePathArg: string, ): Promise { const identity = await readControllerProcessIdentity(pidArg); if ( !identity || !identity.processGroupLeader || !await processIdentityHasCliCommand( identity, cliPathArg, '__upgrade-worker', { allowMissingAbsolutePath: true }, ) ) throw new Error('Unable to verify the launched upgrade worker identity.'); const logFilePath = assertBoundedText(logFilePathArg, 'The upgrade worker log path', 4_096); if (!plugins.path.isAbsolute(logFilePath)) { throw new Error('The upgrade worker log path must be absolute.'); } return await this.mutateTransaction(tokenArg, (current) => ({ ...current, worker: { pid: identity.pid, processGroupId: identity.processGroupId, fingerprint: identity.fingerprint, cliPath: cliPathArg, logFilePath, }, })); } public async recordController( tokenArg: string, controllerArg: TUpgradeTransaction['controller'], ): Promise { if (!controllerArg) throw new Error('The upgrade controller identity is required.'); const identity = await readControllerProcessIdentity(controllerArg.pid); if ( !identity || identity.fingerprint !== controllerArg.fingerprint || identity.processGroupId !== controllerArg.processGroupId || !await processIdentityHasCliCommand( identity, controllerArg.cliPath, controllerArg.command, { allowMissingAbsolutePath: true }, ) ) throw new Error('Unable to verify the upgrade controller identity.'); return await this.mutateTransaction(tokenArg, (current) => ({ ...current, controller: { ...controllerArg }, })); } public async transactionControllerIsLive(transactionArg: TUpgradeTransaction): Promise { const controller = transactionArg.controller; if (!controller) return false; const identity = await readControllerProcessIdentity(controller.pid); return Boolean( identity?.fingerprint === controller.fingerprint && identity.processGroupId === controller.processGroupId && await processIdentityHasCliCommand( identity, controller.cliPath, controller.command, { allowMissingAbsolutePath: true }, ) ); } public async transactionWorkerIsLive(transactionArg: TUpgradeTransaction): Promise { const worker = transactionArg.worker; if (!worker) return false; const identity = await readControllerProcessIdentity(worker.pid); if (Boolean( identity?.fingerprint === worker.fingerprint && await processIdentityHasCliCommand( identity, worker.cliPath, '__upgrade-worker', { allowMissingAbsolutePath: true }, ) )) return true; if (worker.processGroupId !== worker.pid) return false; return (await readProcessGroupMemberPids(worker.processGroupId)) .some((processIdArg) => processIdArg !== worker.pid); } public async terminateTransactionWorker( transactionArg: TUpgradeTransaction, ): Promise { const worker = transactionArg.worker; if (!worker) return; await this.terminateUpgradeWorkerCandidate(worker); } public async terminateUpgradeWorkerCandidate( worker: NonNullable, ): Promise { await terminateVerifiedUpgradeWorkerProcessGroup(worker); } public async readSanitizedUpgradeStatus( tokenArg: string, ): Promise { const transaction = await this.readTransaction(tokenArg); if (!transaction.targetVersion) return undefined; if (transaction.retainedUntil !== undefined && transaction.retainedUntil < Date.now()) return undefined; const phase = transaction.phase === 'starting' || transaction.phase === 'checking' || transaction.phase === 'stopping' ? 'preparing' : transaction.phase; return { fromVersion: transaction.sourceVersion, toVersion: upgradeTransactionTargetVersion(transaction)!, phase, }; } public async finishTransaction( tokenArg: string, successArg: boolean, messageArg: string, errorArg?: string, ): Promise { return await this.mutateTransaction(tokenArg, (current) => { if (current.terminal) return current; return { ...current, phase: successArg ? 'completed' : 'failed', message: assertBoundedText(messageArg, 'The upgrade terminal message', 4_096), retainedUntil: Date.now() + terminalStatusRetentionMs, terminal: { success: successArg, ...(errorArg === undefined ? {} : { error: assertBoundedText(errorArg, 'The upgrade terminal error', 4_096) }), }, }; }); } public async createLaunchGrant(optionsArg: { token: string; port: number; packageName: string; packageVersion: string; cliPath: string; controller: IUpgradeExpectedController; }): Promise { await this.init(); if ( typeof optionsArg.packageName !== 'string' || optionsArg.packageName.length < 1 || optionsArg.packageName.length > 256 || typeof optionsArg.packageVersion !== 'string' || optionsArg.packageVersion.length < 1 || optionsArg.packageVersion.length > 128 || !plugins.path.isAbsolute(optionsArg.cliPath) ) { throw new Error(`The ${currentCliName} upgrade launch grant package binding is invalid.`); } const grant: IUpgradeLaunchGrant = { version: coordinationVersion, tokenHash: hashToken(optionsArg.token), expiresAt: Date.now() + 30_000, port: assertPort(optionsArg.port), packageName: optionsArg.packageName, packageVersion: optionsArg.packageVersion, cliPath: optionsArg.cliPath, controller: parseExpectedController(optionsArg.controller), }; await writePrivateJson(this.tokenFilePath('grant', optionsArg.token), grant); } public async removeLaunchGrant(tokenArg: string): Promise { await plugins.fs.promises.unlink(this.tokenFilePath('grant', tokenArg)).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); await syncDirectory(this.baseDirectory); } public async consumeLaunchGrant(optionsArg: { token: string; port: number; packageName: string; packageVersion: string; cliPath: string; controller: IUpgradeExpectedController; }): Promise { await this.init(); const token = assertToken(optionsArg.token); const grantPath = this.tokenFilePath('grant', token); const consumedPath = `${grantPath}.consuming-${process.pid}-${plugins.crypto.randomBytes(4).toString('hex')}`; await plugins.fs.promises.rename(grantPath, consumedPath).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code === 'ENOENT') { throw new Error(`The ${currentCliName} upgrade launch grant is missing or already consumed.`); } throw errorArg; }); try { const value = await readPrivateJson(consumedPath); if (!value || typeof value !== 'object' || Array.isArray(value)) { throw new Error(`The ${currentCliName} upgrade launch grant is malformed.`); } const grant = value as Record; if (!exactKeys(grant, [ 'version', 'tokenHash', 'expiresAt', 'port', 'packageName', 'packageVersion', 'cliPath', 'controller', ])) { throw new Error(`The ${currentCliName} upgrade launch grant has unexpected fields.`); } const controller = parseExpectedController(grant.controller); const expected = parseExpectedController(optionsArg.controller); if ( grant.version !== coordinationVersion || typeof grant.expiresAt !== 'number' || !Number.isSafeInteger(grant.expiresAt) || grant.expiresAt < Date.now() || grant.port !== assertPort(optionsArg.port) || grant.packageName !== optionsArg.packageName || grant.packageVersion !== optionsArg.packageVersion || grant.cliPath !== optionsArg.cliPath || controller.pid !== expected.pid || controller.processGroupId !== expected.processGroupId || controller.processFingerprint !== expected.processFingerprint || typeof grant.tokenHash !== 'string' || !upgradeTokenHashPattern.test(grant.tokenHash) || !tokensEqual(grant.tokenHash, hashToken(token)) ) { throw new Error(`The ${currentCliName} upgrade launch grant binding is invalid or expired.`); } } finally { await plugins.fs.promises.unlink(consumedPath).catch(() => undefined); await syncDirectory(this.baseDirectory); } } private actionGrantFilePath(actionArg: TUpgradeControllerAction, tokenArg: string): string { return plugins.path.join( this.baseDirectory, `action-${actionArg}-${coordinationPathHash(assertToken(tokenArg))}.json`, ); } public async removeControllerActionGrant( actionArg: TUpgradeControllerAction, tokenArg: string, ): Promise { await plugins.fs.promises.unlink(this.actionGrantFilePath(actionArg, tokenArg)).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); await syncDirectory(this.baseDirectory); } public async createControllerActionGrant(optionsArg: { token: string; action: TUpgradeControllerAction; mode?: 'continue' | 'compensate' | 'reopen'; packageVersion: string; controller: IUpgradeExpectedController; }): Promise { await this.init(); if ( (optionsArg.action === 'prepare' && optionsArg.mode !== undefined) || (optionsArg.action === 'finalize' && optionsArg.mode === undefined) ) throw new Error('The upgrade controller action mode is invalid.'); const grant: IUpgradeControllerActionGrant = { version: coordinationVersion, tokenHash: hashToken(optionsArg.token), action: optionsArg.action, ...(optionsArg.mode === undefined ? {} : { mode: optionsArg.mode }), expiresAt: Date.now() + 30_000, packageVersion: assertBoundedText( optionsArg.packageVersion, 'The upgrade action package version', 128, ), controller: parseExpectedController(optionsArg.controller), }; await writePrivateJson( this.actionGrantFilePath(optionsArg.action, optionsArg.token), grant, ); } public async consumeControllerActionGrant(optionsArg: { token: string; action: TUpgradeControllerAction; mode?: 'continue' | 'compensate' | 'reopen'; packageVersion: string; controller: IUpgradeExpectedController; }): Promise { await this.init(); const grantPath = this.actionGrantFilePath(optionsArg.action, optionsArg.token); const consumedPath = `${grantPath}.consuming-${process.pid}-${plugins.crypto.randomBytes(4).toString('hex')}`; await plugins.fs.promises.rename(grantPath, consumedPath).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code === 'ENOENT') { throw new Error('The upgrade controller action grant is missing or already consumed.'); } throw errorArg; }); try { const raw = await readPrivateJson(consumedPath); if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { throw new Error('The upgrade controller action grant is malformed.'); } const grant = raw as Record; const expectedController = parseExpectedController(optionsArg.controller); const controller = parseExpectedController(grant.controller); if ( !exactKeys(grant, [ 'version', 'tokenHash', 'action', 'expiresAt', 'packageVersion', 'controller', ...(grant.mode === undefined ? [] : ['mode']), ]) || grant.version !== coordinationVersion || grant.action !== optionsArg.action || grant.mode !== optionsArg.mode || !Number.isSafeInteger(grant.expiresAt) || (grant.expiresAt as number) < Date.now() || grant.packageVersion !== optionsArg.packageVersion || controller.pid !== expectedController.pid || controller.processGroupId !== expectedController.processGroupId || controller.processFingerprint !== expectedController.processFingerprint || typeof grant.tokenHash !== 'string' || !tokensEqual(grant.tokenHash, hashToken(optionsArg.token)) ) throw new Error('The upgrade controller action grant binding is invalid or expired.'); } finally { await plugins.fs.promises.unlink(consumedPath).catch(() => undefined); await syncDirectory(this.baseDirectory); } } public async removeWorkerAcknowledgement(tokenArg: string): Promise { await this.init(); let removed = false; await plugins.fs.promises.unlink(this.tokenFilePath('ack', tokenArg)).then(() => { removed = true; }).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; }); if (removed) await syncDirectory(this.baseDirectory); } public async acknowledgeWorker( tokenArg: string, workerArg: Pick, ): Promise { await this.init(); const identity = await readControllerProcessIdentity(workerArg.pid); if ( !identity || identity.fingerprint !== workerArg.fingerprint || identity.processGroupId !== workerArg.processGroupId ) throw new Error('The upgrade acknowledgement worker identity is no longer current.'); const acknowledgement: IUpgradeWorkerAcknowledgement = { version: coordinationVersion, tokenHash: hashToken(tokenArg), pid: identity.pid, processGroupId: identity.processGroupId, fingerprint: identity.fingerprint, }; const acknowledgementPath = this.tokenFilePath('ack', tokenArg); const temporaryPath = `${acknowledgementPath}.tmp-${process.pid}-${plugins.crypto.randomBytes(4).toString('hex')}`; let published = false; try { await writePrivateJson(temporaryPath, acknowledgement); try { await plugins.fs.promises.link(temporaryPath, acknowledgementPath); published = true; await syncDirectory(this.baseDirectory); } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code !== 'EEXIST') { if (published) return; throw errorArg; } let existing: unknown; try { existing = await readPrivateJson(acknowledgementPath); } catch (readErrorArg) { if ((readErrorArg as NodeJS.ErrnoException).code === 'ENOENT') { published = true; return; } throw readErrorArg; } if ( !existing || typeof existing !== 'object' || Array.isArray(existing) || !exactKeys(existing as Record, [ 'version', 'tokenHash', 'pid', 'processGroupId', 'fingerprint', ]) || (existing as Record).version !== coordinationVersion || (existing as Record).tokenHash !== acknowledgement.tokenHash || (existing as Record).pid !== acknowledgement.pid || (existing as Record).processGroupId !== acknowledgement.processGroupId || (existing as Record).fingerprint !== acknowledgement.fingerprint ) throw new Error('The existing upgrade acknowledgement is invalid.'); published = true; } } catch (errorArg) { if (published) return; throw errorArg; } finally { await plugins.fs.promises.unlink(temporaryPath).catch((errorArg) => { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT' && !published) throw errorArg; }); } } public async waitForWorkerAcknowledgement( tokenArg: string, timeoutMsArg = 10_000, ): Promise { const token = assertToken(tokenArg); const identity = await readControllerProcessIdentity(process.pid); if (!identity) throw new Error('The upgrade worker acknowledgement identity is unavailable.'); const ackPath = this.tokenFilePath('ack', token); const deadline = Date.now() + timeoutMsArg; while (Date.now() < deadline) { try { const value = await readPrivateJson(ackPath); if (!value || typeof value !== 'object' || Array.isArray(value)) { throw new Error(`The ${currentCliName} upgrade acknowledgement is malformed.`); } const ack = value as Record; if ( !exactKeys(ack, ['version', 'tokenHash', 'pid', 'processGroupId', 'fingerprint']) || ack.version !== coordinationVersion || typeof ack.tokenHash !== 'string' || !upgradeTokenHashPattern.test(ack.tokenHash) || !tokensEqual(ack.tokenHash, hashToken(token)) || ack.pid !== identity.pid || ack.processGroupId !== identity.processGroupId || ack.fingerprint !== identity.fingerprint ) { throw new Error(`The ${currentCliName} upgrade acknowledgement binding is invalid.`); } await plugins.fs.promises.unlink(ackPath); await syncDirectory(this.baseDirectory); return; } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code !== 'ENOENT') throw errorArg; } await new Promise((resolve) => setTimeout(resolve, 50)); } throw new Error(`The ${currentCliName} upgrade caller did not acknowledge the detached worker.`); } } export const createUpgradeToken = (): string => plugins.crypto .randomBytes(32) .toString('base64url'); export const serializeUpgradeWorkerPayload = (payloadArg: IUpgradeWorkerPayload): string => { assertToken(payloadArg.token); const payload = { version: coordinationVersion, port: assertPort(payloadArg.port), gracePeriodMs: payloadArg.gracePeriodMs, continueSessions: payloadArg.continueSessions, ...(payloadArg.expectedController ? { expectedController: parseExpectedController(payloadArg.expectedController) } : {}), }; return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url'); }; export const parseUpgradeWorkerPayload = ( payloadArg: unknown, tokenArg: unknown, ): IUpgradeWorkerPayload => { if (typeof payloadArg !== 'string' || payloadArg.length < 8 || payloadArg.length > 4096) { throw new Error(`The ${currentCliName} upgrade worker payload is invalid.`); } let value: unknown; try { value = JSON.parse(Buffer.from(payloadArg, 'base64url').toString('utf8')) as unknown; } catch { throw new Error(`The ${currentCliName} upgrade worker payload is malformed.`); } if (!value || typeof value !== 'object' || Array.isArray(value)) { throw new Error(`The ${currentCliName} upgrade worker payload is malformed.`); } const payload = value as Record; if (!exactKeys(payload, [ 'version', 'port', 'gracePeriodMs', 'continueSessions', ...(payload.expectedController === undefined ? [] : ['expectedController']), ])) { throw new Error(`The ${currentCliName} upgrade worker payload has unexpected fields.`); } if (payload.version !== coordinationVersion) { throw new Error(`The ${currentCliName} upgrade worker payload version is unsupported.`); } return { version: coordinationVersion, token: assertToken(tokenArg), port: assertPort(payload.port), gracePeriodMs: (() => { if ( !Number.isSafeInteger(payload.gracePeriodMs) || (payload.gracePeriodMs as number) < 1_000 || (payload.gracePeriodMs as number) > 60 * 60 * 1000 ) throw new Error(`The ${currentCliName} upgrade grace period is invalid.`); return payload.gracePeriodMs as number; })(), continueSessions: (() => { if (typeof payload.continueSessions !== 'boolean') { throw new Error(`The ${currentCliName} upgrade continuation option is invalid.`); } return payload.continueSessions; })(), ...(payload.expectedController ? { expectedController: parseExpectedController(payload.expectedController) } : {}), }; };