import * as plugins from '../ts/plugins.js'; import type { IControllerDatabaseConfig } from '../ts/interfaces.config.js'; import { readControllerProcessIdentity, type IControllerProcessIdentity, } from '../ts/classes.processinspection.js'; import { embeddedDatabaseSocketPath, isSocketListening as defaultIsSocketListening, } from '../ts/functions.embeddeddb.js'; import { EmbeddedDatabaseLocationMigration, rebaseCompletedEmbeddedDatabaseLocationMigration, } from './v1_embeddeddatabaselocation.js'; import { flexProviderCredentialStoreId } from './v16_flexprovidercredentials.js'; const migrationVersion = 2 as const; const journalFileName = '.hcon-controller-data-root-migration.json'; const lockFileName = '.hcon-controller-data-root-migration.lock'; const markerFileName = '.hcon-controller-data-root-marker.json'; const embeddedDatabaseJournalFileName = '.smartdb-location-migration.json'; const maximumJournalBytes = 512 * 1024; const maximumLockBytes = 16 * 1024; const maximumMarkerBytes = 4 * 1024; const maximumSmartDbRelocationReceiptBytes = 16 * 1024; const maximumRootEntries = 2_048; const maximumCredentialStores = 512; const maximumMigrationTemporaryFiles = 32; const maximumControllerLogs = 256; const maximumUpgradeLogs = 64; const maximumLockArtifacts = 64; const maximumLockAttempts = 200; const lockWaitMilliseconds = 50; const lockGuardInitializationGraceMs = 30_000; const credentialHashPattern = /^[a-f0-9]{64}$/; const digestPattern = /^[a-f0-9]{64}$/; const noncePattern = /^[a-f0-9]{64}$/; const decimalPattern = /^(0|[1-9][0-9]*)$/; const controllerLogPattern = /^controller-([1-9][0-9]{0,4})\.log(?:\.old)?$/; const upgradeLogPattern = /^upgrade-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}\.[0-9]{3}Z-[A-Za-z0-9_-]{10}\.log$/; const embeddedDatabaseJournalTemporaryPattern = /^\.smartdb-location-migration\.json\.[1-9][0-9]*-[a-f0-9]{16}\.tmp$/; const lockTemporarySuffixPattern = /^\.tmp-([0-9]+)-([1-9][0-9]*)-([a-f0-9]{64})$/; export type TControllerDataRootMigrationPhase = | 'legacy-authoritative' | 'target-staged' | 'target-committed'; export type TControllerDataWriterKind = 'controller' | 'temp-password' | 'flex-child'; export interface IControllerDataWriterProcessRecord { kind: TControllerDataWriterKind; identity: Pick; } export interface IControllerCredentialRelocationIntent { controllerHash: string; service: string; storeId: string; sourceDirectory: string; destinationDirectory: string; } export interface IControllerCredentialMigrationTuple extends IControllerCredentialRelocationIntent { complete: boolean; } export type TControllerDatabaseMigrationMode = 'external' | 'override' | 'default'; export type TControllerDatabaseMigrationStrategy = | 'unchanged' | 'fresh' | 'historical-direct' | 'old-relocation'; export type TControllerDatabaseMigrationPhase = | 'pending' | 'prepared' | 'moved' | 'verified'; export interface IControllerDatabaseRootMigrationJournal { mode: TControllerDatabaseMigrationMode; strategy: TControllerDatabaseMigrationStrategy; phase: TControllerDatabaseMigrationPhase; configurationDigestSha256: string; oldDirectory: string | null; targetDirectory: string | null; historicalDirectory: string | null; sourcePath: string | null; sourceDevice: string | null; sourceInode: string | null; destinationDevice: string | null; destinationInode: string | null; contentDigestSha256: string | null; v1JournalLocation: 'none' | 'legacy' | 'target'; } export interface IControllerDataRootMigrationJournal { version: typeof migrationVersion; phase: TControllerDataRootMigrationPhase; sourcePath: string; sourceDevice: string | null; sourceInode: string | null; destinationPath: string; destinationDevice: string | null; destinationInode: string | null; targetCreationNonce: string; database: IControllerDatabaseRootMigrationJournal; credentials: IControllerCredentialMigrationTuple[]; } export interface IControllerDataRootMigrationOptions { oldRoot: string; newRoot: string; databaseConfig: IControllerDatabaseConfig; oldEmbeddedSocketPath: string; newEmbeddedSocketPath: string; invokerPid: number; listDataWriterProcesses: () => Promise; preserveEmbeddedDataDirectory?: boolean; isSocketListening?: (socketPathArg: string) => Promise; relocateCredentialStore?: ( intentArg: Readonly, signalArg?: AbortSignal, ) => Promise; relocateStoppedDatabaseRoot?: ( inputArg: Readonly, signalArg?: AbortSignal, ) => Promise; signal?: AbortSignal; } export interface IControllerDataRootStartupPreparation { directoryPath: string; createDirectory: boolean; } export interface IControllerDataRootMigrationResult { directoryPath: string; databaseConfig: IControllerDatabaseConfig; } interface IFilesystemIdentity { device: string; inode: string; } interface ILockOwner { version: 1; uid: number; pid: number; processFingerprint: string; nonce: string; } interface ILockSnapshot { owner: ILockOwner; identity: IFilesystemIdentity; } interface ILockLease { owner: ILockOwner; identity: IFilesystemIdentity; } interface IDatabaseConfiguration { mode: TControllerDatabaseMigrationMode; configurationDigestSha256: string; oldDirectory: string | null; targetDirectory: string | null; historicalDirectory: string | null; effectiveConfig: IControllerDatabaseConfig; } interface IMarker { version: typeof migrationVersion; targetCreationNonce: string; } const currentUid = (): number => { if (typeof process.getuid !== 'function') { throw new Error('Controller data-root migration requires a POSIX user identity.'); } return process.getuid(); }; const isMissingError = (errorArg: unknown): boolean => (errorArg as NodeJS.ErrnoException).code === 'ENOENT'; const pathIdentity = (statsArg: plugins.fs.BigIntStats): IFilesystemIdentity => ({ device: statsArg.dev.toString(10), inode: statsArg.ino.toString(10), }); const identitiesEqual = ( leftArg: IFilesystemIdentity, rightArg: IFilesystemIdentity, ): boolean => leftArg.device === rightArg.device && leftArg.inode === rightArg.inode; const assertAbsoluteNormalizedPath = (pathArg: string, labelArg: string): string => { if ( typeof pathArg !== 'string' || !plugins.path.isAbsolute(pathArg) || plugins.path.resolve(pathArg) !== pathArg || pathArg === plugins.path.parse(pathArg).root ) { throw new Error(`${labelArg} must be a normalized absolute non-root path.`); } return pathArg; }; const assertPlainRecord = (valueArg: unknown, labelArg: string): Record => { if (!valueArg || typeof valueArg !== 'object' || Array.isArray(valueArg)) { throw new Error(`${labelArg} is malformed.`); } const prototype = Object.getPrototypeOf(valueArg); if (prototype !== Object.prototype && prototype !== null) { throw new Error(`${labelArg} is malformed.`); } return valueArg as Record; }; const assertExactKeys = ( valueArg: Record, expectedArg: readonly string[], labelArg: string, ): void => { const actual = Object.keys(valueArg).sort(); const expected = [...expectedArg].sort(); if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { throw new Error(`${labelArg} has unexpected fields.`); } }; const requireString = ( valueArg: unknown, labelArg: string, maximumBytesArg = 16 * 1024, ): string => { if ( typeof valueArg !== 'string' || valueArg.length === 0 || Buffer.byteLength(valueArg, 'utf8') > maximumBytesArg ) throw new Error(`${labelArg} is invalid.`); return valueArg; }; const requireNullableString = (valueArg: unknown, labelArg: string): string | null => valueArg === null ? null : requireString(valueArg, labelArg); const requireNullableDecimal = (valueArg: unknown, labelArg: string): string | null => { const value = requireNullableString(valueArg, labelArg); if (value !== null && !decimalPattern.test(value)) throw new Error(`${labelArg} is invalid.`); return value; }; const assertNullableIdentityPair = ( deviceArg: string | null, inodeArg: string | null, labelArg: string, ): void => { if ((deviceArg === null) !== (inodeArg === null)) { throw new Error(`${labelArg} identity is incomplete.`); } }; const parseCredentialTuple = (valueArg: unknown): IControllerCredentialMigrationTuple => { const value = assertPlainRecord(valueArg, 'Controller credential migration tuple'); assertExactKeys(value, [ 'controllerHash', 'service', 'storeId', 'sourceDirectory', 'destinationDirectory', 'complete', ], 'Controller credential migration tuple'); const controllerHash = requireString(value.controllerHash, 'Controller credential hash', 64); if (!credentialHashPattern.test(controllerHash)) { throw new Error('Controller credential hash is invalid.'); } if (typeof value.complete !== 'boolean') { throw new Error('Controller credential completion state is invalid.'); } return { controllerHash, service: requireString(value.service, 'Controller credential service', 200), storeId: requireString(value.storeId, 'Controller credential store ID', 200), sourceDirectory: requireString(value.sourceDirectory, 'Controller credential source path'), destinationDirectory: requireString(value.destinationDirectory, 'Controller credential destination path'), complete: value.complete, }; }; const parseDatabaseJournal = (valueArg: unknown): IControllerDatabaseRootMigrationJournal => { const value = assertPlainRecord(valueArg, 'Controller database root migration journal'); assertExactKeys(value, [ 'mode', 'strategy', 'phase', 'configurationDigestSha256', 'oldDirectory', 'targetDirectory', 'historicalDirectory', 'sourcePath', 'sourceDevice', 'sourceInode', 'destinationDevice', 'destinationInode', 'contentDigestSha256', 'v1JournalLocation', ], 'Controller database root migration journal'); if (value.mode !== 'external' && value.mode !== 'override' && value.mode !== 'default') { throw new Error('Controller database migration mode is invalid.'); } if ( value.strategy !== 'unchanged' && value.strategy !== 'fresh' && value.strategy !== 'historical-direct' && value.strategy !== 'old-relocation' ) throw new Error('Controller database migration strategy is invalid.'); if ( value.phase !== 'pending' && value.phase !== 'prepared' && value.phase !== 'moved' && value.phase !== 'verified' ) throw new Error('Controller database migration phase is invalid.'); if ( value.v1JournalLocation !== 'none' && value.v1JournalLocation !== 'legacy' && value.v1JournalLocation !== 'target' ) throw new Error('Controller database v1 journal location is invalid.'); const configurationDigestSha256 = requireString( value.configurationDigestSha256, 'Controller database configuration digest', 64, ); if (!digestPattern.test(configurationDigestSha256)) { throw new Error('Controller database configuration digest is invalid.'); } const sourceDevice = requireNullableDecimal(value.sourceDevice, 'Controller database source device'); const sourceInode = requireNullableDecimal(value.sourceInode, 'Controller database source inode'); const destinationDevice = requireNullableDecimal( value.destinationDevice, 'Controller database destination device', ); const destinationInode = requireNullableDecimal( value.destinationInode, 'Controller database destination inode', ); assertNullableIdentityPair(sourceDevice, sourceInode, 'Controller database source'); assertNullableIdentityPair(destinationDevice, destinationInode, 'Controller database destination'); const contentDigestSha256 = requireNullableString( value.contentDigestSha256, 'Controller database content digest', ); if (contentDigestSha256 !== null && !digestPattern.test(contentDigestSha256)) { throw new Error('Controller database content digest is invalid.'); } return { mode: value.mode, strategy: value.strategy, phase: value.phase, configurationDigestSha256, oldDirectory: requireNullableString(value.oldDirectory, 'Controller old database path'), targetDirectory: requireNullableString(value.targetDirectory, 'Controller target database path'), historicalDirectory: requireNullableString( value.historicalDirectory, 'Controller historical database path', ), sourcePath: requireNullableString(value.sourcePath, 'Controller database source path'), sourceDevice, sourceInode, destinationDevice, destinationInode, contentDigestSha256, v1JournalLocation: value.v1JournalLocation, }; }; const parseJournal = (valueArg: unknown): IControllerDataRootMigrationJournal => { const value = assertPlainRecord(valueArg, 'Controller data-root migration journal'); assertExactKeys(value, [ 'version', 'phase', 'sourcePath', 'sourceDevice', 'sourceInode', 'destinationPath', 'destinationDevice', 'destinationInode', 'targetCreationNonce', 'database', 'credentials', ], 'Controller data-root migration journal'); if (value.version !== migrationVersion) { throw new Error('Controller data-root migration journal version is unsupported.'); } if ( value.phase !== 'legacy-authoritative' && value.phase !== 'target-staged' && value.phase !== 'target-committed' ) throw new Error('Controller data-root migration journal phase is invalid.'); const sourceDevice = requireNullableDecimal(value.sourceDevice, 'Controller source root device'); const sourceInode = requireNullableDecimal(value.sourceInode, 'Controller source root inode'); const destinationDevice = requireNullableDecimal( value.destinationDevice, 'Controller destination root device', ); const destinationInode = requireNullableDecimal( value.destinationInode, 'Controller destination root inode', ); assertNullableIdentityPair(sourceDevice, sourceInode, 'Controller source root'); assertNullableIdentityPair(destinationDevice, destinationInode, 'Controller destination root'); const targetCreationNonce = requireString( value.targetCreationNonce, 'Controller target creation nonce', 64, ); if (!noncePattern.test(targetCreationNonce)) { throw new Error('Controller target creation nonce is invalid.'); } if (!Array.isArray(value.credentials) || value.credentials.length > maximumCredentialStores) { throw new Error('Controller credential migration tuple list is invalid.'); } const credentials = value.credentials.map(parseCredentialTuple); return { version: migrationVersion, phase: value.phase, sourcePath: requireString(value.sourcePath, 'Controller source root path'), sourceDevice, sourceInode, destinationPath: requireString(value.destinationPath, 'Controller destination root path'), destinationDevice, destinationInode, targetCreationNonce, database: parseDatabaseJournal(value.database), credentials, }; }; const parseMarker = (valueArg: unknown): IMarker => { const value = assertPlainRecord(valueArg, 'Controller target marker'); assertExactKeys(value, ['version', 'targetCreationNonce'], 'Controller target marker'); if (value.version !== migrationVersion) throw new Error('Controller target marker version is invalid.'); const targetCreationNonce = requireString(value.targetCreationNonce, 'Controller target marker nonce', 64); if (!noncePattern.test(targetCreationNonce)) throw new Error('Controller target marker nonce is invalid.'); return { version: migrationVersion, targetCreationNonce }; }; const parseLockOwner = (valueArg: unknown): ILockOwner => { const value = assertPlainRecord(valueArg, 'Controller data-root migration lock owner'); assertExactKeys(value, [ 'version', 'uid', 'pid', 'processFingerprint', 'nonce', ], 'Controller data-root migration lock owner'); if (value.version !== 1) throw new Error('Controller data-root migration lock version is invalid.'); if (!Number.isSafeInteger(value.uid) || (value.uid as number) < 0) { throw new Error('Controller data-root migration lock UID is invalid.'); } if (!Number.isSafeInteger(value.pid) || (value.pid as number) < 2) { throw new Error('Controller data-root migration lock PID is invalid.'); } const processFingerprint = requireString( value.processFingerprint, 'Controller data-root migration lock process fingerprint', 4 * 1024, ); const nonce = requireString(value.nonce, 'Controller data-root migration lock nonce', 64); if (!noncePattern.test(nonce)) throw new Error('Controller data-root migration lock nonce is invalid.'); return { version: 1, uid: value.uid as number, pid: value.pid as number, processFingerprint, nonce, }; }; const syncDirectory = async (directoryArg: string): Promise => { const handle = await plugins.fs.promises.open( directoryArg, plugins.fs.constants.O_RDONLY | plugins.fs.constants.O_DIRECTORY, ); try { await handle.sync(); } finally { await handle.close(); } }; const readDirectoryIdentity = async ( directoryArg: string, labelArg: string, exactModeArg?: number, ): Promise => { let handle: plugins.fs.promises.FileHandle; try { handle = await plugins.fs.promises.open( directoryArg, plugins.fs.constants.O_RDONLY | plugins.fs.constants.O_DIRECTORY | plugins.fs.constants.O_NOFOLLOW, ); } catch (errorArg) { if (isMissingError(errorArg)) return undefined; throw new Error(`${labelArg} must be a real private directory: ${directoryArg}`, { cause: errorArg, }); } try { const stats = await handle.stat({ bigint: true }); const mode = Number(stats.mode & 0o777n); if ( !stats.isDirectory() || stats.uid !== BigInt(currentUid()) || (exactModeArg === undefined ? (mode & 0o077) !== 0 : mode !== exactModeArg) ) { throw new Error(`${labelArg} must be a real private directory: ${directoryArg}`); } return pathIdentity(stats); } finally { await handle.close(); } }; const inspectPrivateNode = async ( pathArg: string, kindArg: 'file' | 'directory', labelArg: string, exactModeArg?: number, ): Promise => { let stats: plugins.fs.BigIntStats; try { stats = await plugins.fs.promises.lstat(pathArg, { bigint: true }); } catch (errorArg) { if (isMissingError(errorArg)) return undefined; throw errorArg; } const mode = Number(stats.mode & 0o777n); const expectedKind = kindArg === 'file' ? stats.isFile() : stats.isDirectory(); if ( !expectedKind || stats.isSymbolicLink() || stats.uid !== BigInt(currentUid()) || (exactModeArg === undefined ? (mode & 0o077) !== 0 : mode !== exactModeArg) ) throw new Error(`${labelArg} is unsafe: ${pathArg}`); return pathIdentity(stats); }; const readPrivateJson = async ( pathArg: string, maximumBytesArg: number, parserArg: (valueArg: unknown) => T, labelArg: string, exactModeArg = 0o600, ): Promise<{ value: T; identity: IFilesystemIdentity } | undefined> => { let handle: plugins.fs.promises.FileHandle; try { handle = await plugins.fs.promises.open( pathArg, plugins.fs.constants.O_RDONLY | plugins.fs.constants.O_NOFOLLOW, ); } catch (errorArg) { if (isMissingError(errorArg)) return undefined; throw new Error(`${labelArg} is not a safe regular file.`, { cause: errorArg }); } try { const stats = await handle.stat({ bigint: true }); if ( !stats.isFile() || stats.uid !== BigInt(currentUid()) || Number(stats.mode & 0o777n) !== exactModeArg || stats.size < 2n || stats.size > BigInt(maximumBytesArg) ) throw new Error(`${labelArg} is not a bounded mode-${exactModeArg.toString(8)} regular file.`); let parsed: unknown; try { parsed = JSON.parse(await handle.readFile('utf8')) as unknown; } catch (errorArg) { throw new Error(`${labelArg} contains malformed JSON.`, { cause: errorArg }); } return { value: parserArg(parsed), identity: pathIdentity(stats) }; } finally { await handle.close(); } }; const unlinkExactPath = async ( pathArg: string, identityArg: IFilesystemIdentity, ): Promise => { let current: plugins.fs.BigIntStats; try { current = await plugins.fs.promises.lstat(pathArg, { bigint: true }); } catch (errorArg) { if (isMissingError(errorArg)) return; throw errorArg; } if (!identitiesEqual(pathIdentity(current), identityArg)) { throw new Error(`Refusing to remove a replaced migration artifact: ${pathArg}`); } await plugins.fs.promises.unlink(pathArg); }; const writePrivateJsonAtomic = async ( targetPathArg: string, valueArg: unknown, maximumBytesArg: number, ): Promise => { const serialized = `${JSON.stringify(valueArg)}\n`; if (Buffer.byteLength(serialized, 'utf8') > maximumBytesArg) { throw new Error('Controller migration record exceeds its size limit.'); } const parent = plugins.path.dirname(targetPathArg); const temporaryPath = `${targetPathArg}.tmp-${process.pid}-${plugins.crypto.randomBytes(8).toString('hex')}`; let handle: plugins.fs.promises.FileHandle | undefined; let temporaryIdentity: IFilesystemIdentity | undefined; try { handle = await plugins.fs.promises.open( temporaryPath, plugins.fs.constants.O_WRONLY | plugins.fs.constants.O_CREAT | plugins.fs.constants.O_EXCL | plugins.fs.constants.O_NOFOLLOW, 0o600, ); await handle.chmod(0o600); await handle.writeFile(serialized, 'utf8'); await handle.sync(); temporaryIdentity = pathIdentity(await handle.stat({ bigint: true })); await handle.close(); handle = undefined; await plugins.fs.promises.rename(temporaryPath, targetPathArg); temporaryIdentity = undefined; await syncDirectory(parent); } catch (errorArg) { if (handle) await handle.close().catch(() => undefined); if (temporaryIdentity) await unlinkExactPath(temporaryPath, temporaryIdentity).catch(() => undefined); throw errorArg; } }; const delay = async (millisecondsArg: number, signalArg?: AbortSignal): Promise => { signalArg?.throwIfAborted(); const signal = signalArg; await new Promise((resolve, reject) => { const timer = setTimeout(() => { signal?.removeEventListener('abort', onAbort); resolve(); }, millisecondsArg); const onAbort = (): void => { clearTimeout(timer); signal?.removeEventListener('abort', onAbort); reject(signal?.reason ?? new Error('Controller data-root migration was aborted.')); }; signal?.addEventListener('abort', onAbort, { once: true }); }); }; const databaseConfigurationDigest = (valueArg: unknown): string => plugins.crypto .createHash('sha256') .update(JSON.stringify(valueArg), 'utf8') .digest('hex'); const aggregateDatabaseDigests = ( digestsArg: Map, ): string => { const hash = plugins.crypto.createHash('sha256'); for (const [databaseName, digest] of [...digestsArg.entries()].sort(([left], [right]) => ( left < right ? -1 : left > right ? 1 : 0 ))) { hash.update(databaseName, 'utf8'); hash.update('\0'); hash.update(digest.sha256, 'ascii'); hash.update('\0'); } return hash.digest('hex'); }; const inspectDatabaseContentDigest = async (directoryArg: string): Promise => { const database = new plugins.smartdb.LocalSmartDb({ folderPath: directoryArg }); retainedMigratedLocalSmartDbs.add(database); let operationError: unknown; let result: string | undefined; try { await database.start(); const server = database.getServer(); const names = [...new Set((await server.getCollections()).map((entry) => entry.db))].sort(); const digests = new Map(); for (const name of names) { digests.set(name, await server.getDatabaseContentDigest({ databaseName: name })); } result = aggregateDatabaseDigests(digests); } catch (errorArg) { operationError = errorArg; } try { await stopLocalSmartDbConfirmed(database, 'Controller database inspection SmartDB'); retainedMigratedLocalSmartDbs.delete(database); } catch (errorArg) { operationError = operationError ? new AggregateError( [operationError, errorArg], 'Controller database inspection and cleanup both failed.', { cause: operationError }, ) : errorArg; } if (operationError) throw operationError; return result!; }; const closeKernelStoreConfirmed = async ( kernelStoreArg: plugins.smartsecret.SmartSecretKernelStore, ): Promise => { const errors: unknown[] = []; for (let attempt = 0; attempt < 2; attempt += 1) { try { await kernelStoreArg.close(); return; } catch (errorArg) { errors.push(errorArg); } } throw new AggregateError(errors, 'SmartSecret kernel-store closure could not be confirmed.'); }; const retainedEmbeddedDatabaseMigrations = new Set(); const retainedMigratedLocalSmartDbs = new Set(); const retainedCredentialKernelStores = new Set(); const drainRetainedCredentialKernelStores = async (): Promise => { for (const kernelStore of retainedCredentialKernelStores) { await closeKernelStoreConfirmed(kernelStore); retainedCredentialKernelStores.delete(kernelStore); } }; const stopLocalSmartDbConfirmed = async ( localDbArg: plugins.smartdb.LocalSmartDb, labelArg: string, ): Promise => { const errors: unknown[] = []; for (let attempt = 0; attempt < 2; attempt += 1) { try { await localDbArg.stop(); return; } catch (errorArg) { errors.push(errorArg); } } throw new AggregateError(errors, `${labelArg} could not be stopped cleanly.`); }; const drainRetainedEmbeddedDatabaseMigrationResources = async (): Promise => { for (const retainedLocalDb of retainedMigratedLocalSmartDbs) { await stopLocalSmartDbConfirmed(retainedLocalDb, 'Retained controller migration SmartDB'); retainedMigratedLocalSmartDbs.delete(retainedLocalDb); } for (const retainedMigration of retainedEmbeddedDatabaseMigrations) { const cleanupErrors: unknown[] = []; for (let attempt = 0; attempt < 2 && retainedMigration.hasOwnedResources; attempt += 1) { try { await retainedMigration.stop(); } catch (errorArg) { cleanupErrors.push(errorArg); } } if (retainedMigration.hasOwnedResources) { throw new AggregateError( cleanupErrors, 'A retained embedded database migration still owns resources.', ); } retainedEmbeddedDatabaseMigrations.delete(retainedMigration); } }; const runEmbeddedDatabaseLocationMigration = async ( migrationArg: EmbeddedDatabaseLocationMigration, ): Promise => { await drainRetainedEmbeddedDatabaseMigrationResources(); retainedEmbeddedDatabaseMigrations.add(migrationArg); let operationError: unknown; let result: Awaited> = undefined; try { result = await migrationArg.run(); } catch (errorArg) { operationError = errorArg; } if (result) { retainedMigratedLocalSmartDbs.add(result.localDb); try { await stopLocalSmartDbConfirmed(result.localDb, 'Migrated controller SmartDB'); retainedMigratedLocalSmartDbs.delete(result.localDb); } catch (errorArg) { operationError = operationError ? new AggregateError( [operationError, errorArg], 'Embedded database migration and transferred-engine cleanup both failed.', { cause: operationError }, ) : errorArg; } } if (migrationArg.hasOwnedResources) { const cleanupErrors: unknown[] = []; for (let attempt = 0; attempt < 2 && migrationArg.hasOwnedResources; attempt += 1) { try { await migrationArg.stop(); } catch (errorArg) { cleanupErrors.push(errorArg); } } if (migrationArg.hasOwnedResources) { const cleanupError = new AggregateError( cleanupErrors, 'Embedded database migration retained resources after cleanup retries.', ); operationError = operationError ? new AggregateError( [operationError, cleanupError], 'Embedded database migration and owned-resource cleanup both failed.', { cause: operationError }, ) : cleanupError; } } if (!migrationArg.hasOwnedResources) retainedEmbeddedDatabaseMigrations.delete(migrationArg); if (operationError) throw operationError; }; const defaultRelocateCredentialStore = async ( intentArg: Readonly, signalArg?: AbortSignal, ): Promise => { signalArg?.throwIfAborted(); await drainRetainedCredentialKernelStores(); let kernelStore: plugins.smartsecret.SmartSecretKernelStore | undefined; let sealedStore: plugins.smartsecret.SmartSecretSealedFileStore | undefined; let operationError: unknown; try { kernelStore = await plugins.smartsecret.SmartSecretKernelStore.create({ service: intentArg.service, }); retainedCredentialKernelStores.add(kernelStore); signalArg?.throwIfAborted(); sealedStore = await plugins.smartsecret.SmartSecretSealedFileStore.relocate({ kernelStore, storeId: intentArg.storeId, sourceDirectoryPath: intentArg.sourceDirectory, destinationDirectoryPath: intentArg.destinationDirectory, }); await sealedStore.close(); sealedStore = undefined; } catch (errorArg) { operationError = errorArg; } if (sealedStore) { try { await sealedStore.close(); } catch (errorArg) { operationError = operationError ? new AggregateError( [operationError, errorArg], 'SmartSecret relocation and sealed-store cleanup both failed.', { cause: operationError }, ) : errorArg; } } if (kernelStore) { try { await closeKernelStoreConfirmed(kernelStore); retainedCredentialKernelStores.delete(kernelStore); } catch (errorArg) { operationError = operationError ? new AggregateError( [operationError, errorArg], 'SmartSecret relocation and kernel-store cleanup both failed.', { cause: operationError }, ) : errorArg; } } if (operationError) throw operationError; }; export class ControllerDataRootMigrationRunner { private readonly oldRoot: string; private readonly newRoot: string; private readonly commonParent: string; private readonly journalPath: string; private readonly lockPath: string; private readonly markerPath: string; private readonly databaseConfiguration: IDatabaseConfiguration; private readonly isSocketListening: (socketPathArg: string) => Promise; private readonly relocateCredentialStore: ( intentArg: Readonly, signalArg?: AbortSignal, ) => Promise; private readonly relocateStoppedDatabaseRoot: ( inputArg: Readonly, signalArg?: AbortSignal, ) => Promise; constructor(private readonly options: IControllerDataRootMigrationOptions) { this.oldRoot = assertAbsoluteNormalizedPath(options.oldRoot, 'Old controller root'); this.newRoot = assertAbsoluteNormalizedPath(options.newRoot, 'New controller root'); if (this.oldRoot === this.newRoot) throw new Error('Controller roots must be different.'); if (plugins.path.dirname(this.oldRoot) !== plugins.path.dirname(this.newRoot)) { throw new Error('Controller roots must be siblings under one common parent.'); } this.commonParent = plugins.path.dirname(this.oldRoot); this.journalPath = plugins.path.join(this.commonParent, journalFileName); this.lockPath = plugins.path.join(this.commonParent, lockFileName); this.markerPath = plugins.path.join(this.newRoot, markerFileName); assertAbsoluteNormalizedPath(options.oldEmbeddedSocketPath, 'Old embedded database socket path'); assertAbsoluteNormalizedPath(options.newEmbeddedSocketPath, 'New embedded database socket path'); if (!Number.isSafeInteger(options.invokerPid) || options.invokerPid < 2) { throw new Error('Controller data-root migration invoker PID is invalid.'); } if (typeof options.listDataWriterProcesses !== 'function') { throw new Error('Controller data-root migration requires a data-writer process callback.'); } this.databaseConfiguration = this.normalizeDatabaseConfiguration(options.databaseConfig); this.isSocketListening = options.isSocketListening ?? defaultIsSocketListening; this.relocateCredentialStore = options.relocateCredentialStore ?? defaultRelocateCredentialStore; this.relocateStoppedDatabaseRoot = options.relocateStoppedDatabaseRoot ?? (async (inputArg, signalArg) => await plugins.smartdb.LocalSmartDb.relocateStoppedStorageRoot( { ...inputArg }, signalArg ? { signal: signalArg } : undefined, )); } public async prepareStartup(): Promise { const parentExists = await this.validateCommonParent(false); if (!parentExists) { return { directoryPath: this.newRoot, createDirectory: true }; } const journal = await this.readJournal(); if (journal) { if (journal.phase === 'target-committed') { this.assertJournalRootBinding(journal); await this.assertTargetIdentity(journal); await this.assertTargetMarker(journal.targetCreationNonce); return { directoryPath: journal.destinationPath, createDirectory: false }; } this.assertJournalMatches(journal); await this.assertSourceIdentity(journal); return { directoryPath: journal.sourcePath, createDirectory: false }; } const sourceIdentity = await readDirectoryIdentity( this.oldRoot, 'Legacy controller root', ); if (sourceIdentity) return { directoryPath: this.oldRoot, createDirectory: false }; await readDirectoryIdentity(this.newRoot, 'Target controller root', 0o700); return { directoryPath: this.newRoot, createDirectory: true }; } /** Validates all authoritative v2 state without creating roots, locks, or journals. */ public async preflight(): Promise { this.options.signal?.throwIfAborted(); if (this.options.invokerPid !== process.pid) { throw new Error('Controller data-root migration invoker PID is not the current process.'); } if (!await this.validateCommonParent(false)) return; const journal = await this.readJournal(); if (journal) { if (journal.phase === 'target-committed') { this.assertJournalRootBinding(journal); await this.assertTargetIdentity(journal); await this.assertTargetMarker(journal.targetCreationNonce); return; } this.assertJournalMatches(journal); await this.assertSourceIdentity(journal); await this.inspectLegacyRootInventory(journal); await this.validateCredentialLocations(journal, false); if (journal.phase !== 'legacy-authoritative') { await this.inspectTargetStagingInventory(journal); } return; } const sourceIdentity = await readDirectoryIdentity(this.oldRoot, 'Legacy controller root'); const targetIdentity = await readDirectoryIdentity( this.newRoot, 'Target controller root', 0o700, ); const targetContainsPreservedDatabase = targetIdentity ? await this.targetContainsOnlyPreservedDatabase() : false; if (sourceIdentity && targetIdentity && !targetContainsPreservedDatabase) { throw new Error('Both unjournaled controller product roots exist; refusing to choose authority.'); } if (sourceIdentity) { await this.inspectLegacyRootInventory(); await this.initialCredentialTuples(true); await this.initialDatabaseJournal(); return; } if (targetIdentity) { const entries = await this.readBoundedDirectory(this.newRoot, 'Target controller root'); if (entries.length !== 0 && !targetContainsPreservedDatabase) { throw new Error('An unjournaled target controller root is not empty.'); } } } public async run(): Promise { this.options.signal?.throwIfAborted(); if (this.options.invokerPid !== process.pid) { throw new Error('Controller data-root migration invoker PID is not the current process.'); } const invokerIdentity = await readControllerProcessIdentity(this.options.invokerPid); if (!invokerIdentity || invokerIdentity.pid !== this.options.invokerPid) { throw new Error('Controller data-root migration invoker identity cannot be verified.'); } await this.ensureCommonParent(); const lease = await this.acquireLock(invokerIdentity); let operationError: unknown; let result: IControllerDataRootMigrationResult | undefined; try { result = await this.runLocked(invokerIdentity); } catch (errorArg) { operationError = errorArg; } try { await this.releaseLock(lease); } catch (errorArg) { operationError = operationError ? new AggregateError( [operationError, errorArg], 'Controller data-root migration and lock release both failed.', { cause: operationError }, ) : errorArg; } if (operationError) throw operationError; return result!; } private normalizeDatabaseConfiguration( configArg: IControllerDatabaseConfig, ): IDatabaseConfiguration { const value = assertPlainRecord(configArg, 'Controller database configuration'); const allowedKeys = new Set([ 'mongoDbUrl', 'mongoDbName', 'embeddedDataDirectory', 'legacyEmbeddedDataDirectory', ]); if (Object.keys(value).some((key) => !allowedKeys.has(key))) { throw new Error('Controller database configuration has unexpected fields.'); } const mongoDbName = requireString(value.mongoDbName, 'Controller MongoDB database name', 256); const oldDirectory = plugins.path.join(this.oldRoot, 'smartdb'); const targetDirectory = plugins.path.join(this.newRoot, 'smartdb'); if (value.mongoDbUrl !== undefined) { const mongoDbUrl = requireString(value.mongoDbUrl, 'Controller MongoDB URL'); if (value.embeddedDataDirectory !== undefined || value.legacyEmbeddedDataDirectory !== undefined) { throw new Error('External MongoDB configuration must not include embedded database paths.'); } return { mode: 'external', configurationDigestSha256: databaseConfigurationDigest({ mode: 'external', mongoDbUrl, mongoDbName, }), oldDirectory: null, targetDirectory: null, historicalDirectory: null, effectiveConfig: { mongoDbUrl, mongoDbName }, }; } const embeddedDataDirectory = assertAbsoluteNormalizedPath( requireString(value.embeddedDataDirectory, 'Controller embedded database path'), 'Controller embedded database path', ); const historicalDirectory = value.legacyEmbeddedDataDirectory === undefined ? null : assertAbsoluteNormalizedPath( requireString(value.legacyEmbeddedDataDirectory, 'Historical embedded database path'), 'Historical embedded database path', ); if (this.options.preserveEmbeddedDataDirectory === true) { if (historicalDirectory !== null) { throw new Error('An explicit embedded database override must not include a historical path.'); } return { mode: 'override', configurationDigestSha256: databaseConfigurationDigest({ mode: 'override', mongoDbName, embeddedDataDirectory, }), oldDirectory: null, targetDirectory: null, historicalDirectory: null, effectiveConfig: { mongoDbName, embeddedDataDirectory }, }; } if (embeddedDataDirectory !== oldDirectory && embeddedDataDirectory !== targetDirectory) { if (historicalDirectory !== null) { throw new Error('An explicit embedded database override must not include a historical path.'); } return { mode: 'override', configurationDigestSha256: databaseConfigurationDigest({ mode: 'override', mongoDbName, embeddedDataDirectory, }), oldDirectory: null, targetDirectory: null, historicalDirectory: null, effectiveConfig: { mongoDbName, embeddedDataDirectory }, }; } if (historicalDirectory === oldDirectory || historicalDirectory === targetDirectory) { throw new Error('Historical embedded database path must be outside both product roots.'); } return { mode: 'default', configurationDigestSha256: databaseConfigurationDigest({ mode: 'default', mongoDbName, historicalDirectory, }), oldDirectory, targetDirectory, historicalDirectory, effectiveConfig: { mongoDbName, embeddedDataDirectory: targetDirectory }, }; } private async validateCommonParent(requiredArg: boolean): Promise { let stats: plugins.fs.BigIntStats; try { stats = await plugins.fs.promises.lstat(this.commonParent, { bigint: true }); } catch (errorArg) { if (!requiredArg && isMissingError(errorArg)) return false; throw errorArg; } if ( !stats.isDirectory() || stats.isSymbolicLink() || stats.uid !== BigInt(currentUid()) || (Number(stats.mode & 0o777n) & 0o022) !== 0 ) throw new Error(`Controller roots require a private real common parent: ${this.commonParent}`); if (await plugins.fs.promises.realpath(this.commonParent) !== this.commonParent) { throw new Error('Controller root common parent must not traverse symbolic links.'); } return true; } private async ensureCommonParent(): Promise { await plugins.fs.promises.mkdir(this.commonParent, { recursive: true, mode: 0o700 }); if (!await this.validateCommonParent(true)) { throw new Error('Controller root common parent could not be created.'); } } private async readJournal(): Promise { return (await readPrivateJson( this.journalPath, maximumJournalBytes, parseJournal, 'Controller data-root migration journal', ))?.value; } private async writeJournal(journalArg: IControllerDataRootMigrationJournal): Promise { await this.cleanupJournalTemporaryFiles(); await writePrivateJsonAtomic(this.journalPath, journalArg, maximumJournalBytes); const persisted = await this.readJournal(); if (!persisted || JSON.stringify(persisted) !== JSON.stringify(journalArg)) { throw new Error('Controller data-root migration journal was not persisted exactly.'); } } private async cleanupJournalTemporaryFiles(): Promise { const prefix = `${plugins.path.basename(this.journalPath)}.tmp-`; const entries = await plugins.fs.promises.readdir(this.commonParent, { withFileTypes: true }); const temporaryNames = entries .map((entry) => entry.name) .filter((name) => name.startsWith(prefix)); if (temporaryNames.length > maximumMigrationTemporaryFiles) { throw new Error('Too many controller migration journal artifacts exist.'); } for (const name of temporaryNames) { if (!new RegExp(`^${prefix.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}[1-9][0-9]*-[a-f0-9]{16}$`).test(name)) { throw new Error(`Unknown controller migration journal artifact: ${name}`); } const path = plugins.path.join(this.commonParent, name); const identity = await inspectPrivateNode(path, 'file', 'Controller migration journal artifact', 0o600); if (identity) await unlinkExactPath(path, identity); } if (temporaryNames.length > 0) await syncDirectory(this.commonParent); } private assertJournalMatches(journalArg: IControllerDataRootMigrationJournal): void { this.assertJournalRootBinding(journalArg); if (journalArg.database.configurationDigestSha256 !== this.databaseConfiguration.configurationDigestSha256) { throw new Error('Controller database configuration changed during root migration.'); } const database = journalArg.database; if ( database.mode !== this.databaseConfiguration.mode || database.oldDirectory !== this.databaseConfiguration.oldDirectory || database.targetDirectory !== this.databaseConfiguration.targetDirectory || database.historicalDirectory !== this.databaseConfiguration.historicalDirectory ) throw new Error('Controller database migration journal paths do not match current inputs.'); if (database.mode !== 'default') { if ( database.strategy !== 'unchanged' || database.phase !== 'verified' || database.v1JournalLocation !== 'none' || database.sourceDevice !== null || database.destinationDevice !== null || database.contentDigestSha256 !== null ) throw new Error('Unchanged controller database journal state is invalid.'); } else if (database.strategy === 'fresh') { if ( database.phase !== 'verified' || database.sourcePath !== null || database.sourceDevice !== null || database.destinationDevice !== null || database.contentDigestSha256 !== null || database.v1JournalLocation !== 'none' ) throw new Error('Fresh controller database journal state is invalid.'); } else if (database.strategy === 'historical-direct') { if ( database.sourcePath !== database.historicalDirectory || database.v1JournalLocation !== 'target' || database.phase === 'prepared' || database.phase === 'moved' ) throw new Error('Historical controller database journal state is invalid.'); } else if (database.strategy === 'old-relocation') { if ( database.sourcePath !== database.oldDirectory || database.v1JournalLocation === 'target' ) throw new Error('Old controller database relocation journal state is invalid.'); } else { throw new Error('Default controller database migration strategy is invalid.'); } if (database.phase === 'pending' && database.strategy === 'old-relocation') { if ( database.sourceDevice !== null || database.destinationDevice !== null || database.contentDigestSha256 !== null ) throw new Error('Pending controller database relocation state is invalid.'); } if (database.phase === 'prepared') { if ( database.sourceDevice === null || database.destinationDevice !== null || database.contentDigestSha256 === null ) throw new Error('Prepared controller database relocation state is invalid.'); } if (database.phase === 'moved' || ( database.phase === 'verified' && (database.strategy === 'historical-direct' || database.strategy === 'old-relocation') )) { if (database.destinationDevice === null || database.contentDigestSha256 === null) { throw new Error('Moved controller database relocation state is invalid.'); } } const seenHashes = new Set(); for (let index = 0; index < journalArg.credentials.length; index += 1) { const tuple = journalArg.credentials[index]; if (seenHashes.has(tuple.controllerHash)) { throw new Error('Controller credential migration journal contains duplicate hashes.'); } seenHashes.add(tuple.controllerHash); if ( (index > 0 && journalArg.credentials[index - 1].controllerHash > tuple.controllerHash) || tuple.service !== `modelprofile.flexharness.${tuple.controllerHash.slice(0, 32)}` || tuple.storeId !== flexProviderCredentialStoreId || tuple.sourceDirectory !== plugins.path.join( this.oldRoot, 'flex-provider-credentials', tuple.controllerHash, ) || tuple.destinationDirectory !== plugins.path.join( this.newRoot, 'flex-provider-credentials', tuple.controllerHash, ) ) throw new Error('Controller credential migration tuple does not match its roots.'); } } private assertJournalRootBinding(journalArg: IControllerDataRootMigrationJournal): void { if (journalArg.sourcePath !== this.oldRoot || journalArg.destinationPath !== this.newRoot) { throw new Error('Controller data-root migration journal belongs to different roots.'); } const destinationIdentityPresent = journalArg.destinationDevice !== null; if ((journalArg.phase === 'legacy-authoritative') === destinationIdentityPresent) { throw new Error('Controller data-root migration destination identity conflicts with its phase.'); } } private async assertSourceIdentity(journalArg: IControllerDataRootMigrationJournal): Promise { const identity = await readDirectoryIdentity(this.oldRoot, 'Legacy controller root'); if (journalArg.sourceDevice === null) { if (identity) throw new Error('Legacy controller root appeared after migration preparation.'); return; } if (!identity || !identitiesEqual(identity, { device: journalArg.sourceDevice, inode: journalArg.sourceInode!, })) throw new Error('Legacy controller root identity changed during migration.'); } private async assertTargetIdentity(journalArg: IControllerDataRootMigrationJournal): Promise { if (journalArg.destinationDevice === null) { throw new Error('Controller target root identity was not recorded.'); } const identity = await readDirectoryIdentity(this.newRoot, 'Target controller root', 0o700); if (!identity || !identitiesEqual(identity, { device: journalArg.destinationDevice, inode: journalArg.destinationInode!, })) throw new Error('Controller target root identity changed during migration.'); } private async assertTargetMarker(nonceArg: string): Promise { const marker = await readPrivateJson( this.markerPath, maximumMarkerBytes, parseMarker, 'Controller target marker', ); if (!marker || marker.value.targetCreationNonce !== nonceArg) { throw new Error('Controller target marker is missing or belongs to another migration.'); } } private async runLocked( invokerIdentityArg: IControllerProcessIdentity, ): Promise { let journal = await this.readJournal(); if (journal?.phase === 'target-committed') { this.assertJournalRootBinding(journal); await this.assertTargetIdentity(journal); await this.assertTargetMarker(journal.targetCreationNonce); return this.result(); } if (journal) this.assertJournalMatches(journal); await this.assertNoConflictingWriters(invokerIdentityArg); await drainRetainedEmbeddedDatabaseMigrationResources(); await this.cleanupJournalTemporaryFiles(); journal = await this.readJournal(); if (journal) this.assertJournalMatches(journal); if (!journal) journal = await this.createInitialJournal(); await this.assertSourceIdentity(journal); await this.inspectLegacyRootInventory(journal); await this.validateCredentialLocations(journal, false); if (journal.database.mode === 'default') await this.assertDefaultDatabaseSocketsIdle(); if ( journal.database.strategy === 'old-relocation' && journal.database.phase === 'pending' ) journal = await this.prepareOldDatabaseRelocation(journal); journal = await this.ensureTargetStaged(journal); journal = await this.performDatabaseMigration(journal); journal = await this.performCredentialMigrations(journal); if (journal.database.phase !== 'verified') { throw new Error('Controller database migration is not verified.'); } if (journal.credentials.some((tuple) => !tuple.complete)) { throw new Error('Controller credential migration is not complete.'); } await this.validateCredentialLocations(journal, true); await this.inspectTargetStagingInventory(journal); await this.assertTargetIdentity(journal); const committed: IControllerDataRootMigrationJournal = { ...journal, phase: 'target-committed', }; await this.writeJournal(committed); return this.result(); } private result(): IControllerDataRootMigrationResult { return { directoryPath: this.newRoot, databaseConfig: { ...this.databaseConfiguration.effectiveConfig }, }; } private preservedTargetDatabasePath(): string | undefined { if ( this.databaseConfiguration.mode !== 'override' || this.databaseConfiguration.effectiveConfig.embeddedDataDirectory !== plugins.path.join(this.newRoot, 'smartdb') ) return undefined; return this.databaseConfiguration.effectiveConfig.embeddedDataDirectory; } private async targetContainsOnlyPreservedDatabase(): Promise { const databasePath = this.preservedTargetDatabasePath(); if (!databasePath) return false; const names = await this.readBoundedDirectory(this.newRoot, 'Target controller root'); if (names.length !== 1 || names[0] !== 'smartdb') return false; await this.requirePrivateNode(databasePath, 'directory', 'Explicit target database override'); return true; } private async assertNoConflictingWriters( invokerIdentityArg: IControllerProcessIdentity, ): Promise { const records = await this.options.listDataWriterProcesses(); if (!Array.isArray(records) || records.length > 1_024) { throw new Error('Controller data-writer process inventory is invalid.'); } for (const record of records) { if ( !record || (record.kind !== 'controller' && record.kind !== 'temp-password' && record.kind !== 'flex-child') || !Number.isSafeInteger(record.identity?.pid) || record.identity.pid < 2 || typeof record.identity.fingerprint !== 'string' || record.identity.fingerprint.length === 0 ) throw new Error('Controller data-writer process record is invalid.'); if ( record.identity.pid === invokerIdentityArg.pid && record.identity.fingerprint === invokerIdentityArg.fingerprint ) continue; throw new Error( `Controller data-root migration is blocked by ${record.kind} writer PID ${record.identity.pid}.`, ); } } private async createInitialJournal(): Promise { const sourceIdentity = await readDirectoryIdentity(this.oldRoot, 'Legacy controller root'); const targetIdentity = await readDirectoryIdentity(this.newRoot, 'Target controller root', 0o700); const targetContainsPreservedDatabase = targetIdentity ? await this.targetContainsOnlyPreservedDatabase() : false; if (sourceIdentity && targetIdentity && !targetContainsPreservedDatabase) { throw new Error('Both unjournaled controller product roots exist; refusing to choose authority.'); } if (sourceIdentity) await this.inspectLegacyRootInventory(); if (targetIdentity) { const entries = await this.readBoundedDirectory(this.newRoot, 'Target controller root'); if (entries.length !== 0 && !targetContainsPreservedDatabase) { throw new Error('An unjournaled target controller root is not empty.'); } } const credentials = await this.initialCredentialTuples(sourceIdentity !== undefined); const database = await this.initialDatabaseJournal(); const journal: IControllerDataRootMigrationJournal = { version: migrationVersion, phase: 'legacy-authoritative', sourcePath: this.oldRoot, sourceDevice: sourceIdentity?.device ?? null, sourceInode: sourceIdentity?.inode ?? null, destinationPath: this.newRoot, destinationDevice: null, destinationInode: null, targetCreationNonce: plugins.crypto.randomBytes(32).toString('hex'), database, credentials, }; this.assertJournalMatches(journal); await this.writeJournal(journal); return journal; } private async readBoundedDirectory(directoryArg: string, labelArg: string): Promise { const entries = await plugins.fs.promises.readdir(directoryArg, { withFileTypes: true }); if (entries.length > maximumRootEntries) throw new Error(`${labelArg} has too many entries.`); return entries.map((entry) => entry.name).sort(); } private async inspectLegacyRootInventory( journalArg?: IControllerDataRootMigrationJournal, ): Promise { const identity = await readDirectoryIdentity(this.oldRoot, 'Legacy controller root'); if (!identity) return; const names = await this.readBoundedDirectory(this.oldRoot, 'Legacy controller root'); let migrationTemporaryFiles = 0; let controllerLogs = 0; let upgradeLogs = 0; for (const name of names) { const path = plugins.path.join(this.oldRoot, name); if (name === 'smartdb') { const databaseMayBeRelocated = journalArg?.database.strategy === 'old-relocation' && journalArg.database.phase !== 'pending'; if (databaseMayBeRelocated) { const stats = await plugins.fs.promises.lstat(path); if (stats.isFile()) { await this.requirePrivateNode(path, 'file', 'Legacy controller smartdb receipt', 0o600); } else { await this.requirePrivateNode(path, 'directory', 'Legacy controller smartdb'); } } else { await this.requirePrivateNode(path, 'directory', 'Legacy controller smartdb'); } continue; } if (name === 'browser-runtime') { await this.requirePrivateNode(path, 'directory', `Legacy controller ${name}`); continue; } if (name === 'flex-provider-credentials') { await this.requirePrivateNode(path, 'directory', 'Legacy Flex credential root', 0o700); continue; } if (name === embeddedDatabaseJournalFileName) { await this.requirePrivateNode(path, 'file', 'Legacy embedded database journal'); continue; } if (embeddedDatabaseJournalTemporaryPattern.test(name)) { migrationTemporaryFiles += 1; await this.requirePrivateNode(path, 'file', 'Legacy embedded database journal artifact'); continue; } const controllerLogMatch = controllerLogPattern.exec(name); if (controllerLogMatch && Number(controllerLogMatch[1]) <= 65_535) { controllerLogs += 1; await this.requirePrivateNode(path, 'file', 'Legacy controller log'); continue; } if (upgradeLogPattern.test(name)) { upgradeLogs += 1; await this.requirePrivateNode(path, 'file', 'Legacy controller upgrade log'); continue; } throw new Error(`Unknown authoritative entry in legacy controller root: ${name}`); } if ( migrationTemporaryFiles > maximumMigrationTemporaryFiles || controllerLogs > maximumControllerLogs || upgradeLogs > maximumUpgradeLogs ) throw new Error('Legacy controller root contains too many bounded migration assets.'); if (journalArg) await this.assertSourceIdentity(journalArg); } private async requirePrivateNode( pathArg: string, kindArg: 'file' | 'directory', labelArg: string, exactModeArg?: number, ): Promise { const identity = await inspectPrivateNode(pathArg, kindArg, labelArg, exactModeArg); if (!identity) throw new Error(`${labelArg} disappeared during inspection.`); return identity; } private async initialCredentialTuples( sourceRootExistsArg: boolean, ): Promise { if (!sourceRootExistsArg) return []; const credentialRoot = plugins.path.join(this.oldRoot, 'flex-provider-credentials'); const identity = await readDirectoryIdentity(credentialRoot, 'Legacy Flex credential root', 0o700); if (!identity) return []; const hashes = await this.readCredentialHashes(credentialRoot, 'Legacy Flex credential root'); return hashes.map((controllerHash) => ({ controllerHash, service: `modelprofile.flexharness.${controllerHash.slice(0, 32)}`, storeId: flexProviderCredentialStoreId, sourceDirectory: plugins.path.join(credentialRoot, controllerHash), destinationDirectory: plugins.path.join( this.newRoot, 'flex-provider-credentials', controllerHash, ), complete: false, })); } private async readCredentialHashes(directoryArg: string, labelArg: string): Promise { const entries = await plugins.fs.promises.readdir(directoryArg, { withFileTypes: true }); if (entries.length > maximumCredentialStores) { throw new Error(`${labelArg} contains too many credential stores.`); } const hashes = entries.map((entry) => entry.name).sort(); for (const hash of hashes) { if (!credentialHashPattern.test(hash)) { throw new Error(`${labelArg} contains an unexpected entry: ${hash}`); } await this.requirePrivateNode( plugins.path.join(directoryArg, hash), 'directory', 'Flex credential store directory', 0o700, ); } return hashes; } private async initialDatabaseJournal(): Promise { const configuration = this.databaseConfiguration; if (configuration.mode !== 'default') { return { mode: configuration.mode, strategy: 'unchanged', phase: 'verified', configurationDigestSha256: configuration.configurationDigestSha256, oldDirectory: null, targetDirectory: null, historicalDirectory: null, sourcePath: configuration.mode === 'override' ? configuration.effectiveConfig.embeddedDataDirectory! : null, sourceDevice: null, sourceInode: null, destinationDevice: null, destinationInode: null, contentDigestSha256: null, v1JournalLocation: 'none', }; } const oldDirectory = configuration.oldDirectory!; const targetDirectory = configuration.targetDirectory!; const historicalDirectory = configuration.historicalDirectory; const oldIdentity = await readDirectoryIdentity(oldDirectory, 'Legacy config-root database'); const historicalIdentity = historicalDirectory ? await readDirectoryIdentity(historicalDirectory, 'Historical state-root database') : undefined; const oldV1JournalPath = plugins.path.join(this.oldRoot, embeddedDatabaseJournalFileName); const oldV1Journal = await inspectPrivateNode( oldV1JournalPath, 'file', 'Legacy embedded database journal', ); if (oldV1Journal && !historicalDirectory) { throw new Error('Legacy embedded database journal has no configured historical source path.'); } if (historicalIdentity && oldIdentity && !oldV1Journal) { throw new Error( 'Historical and config-root controller databases collide without a matching v1 journal.', ); } if (oldV1Journal || oldIdentity) { return { mode: 'default', strategy: 'old-relocation', phase: 'pending', configurationDigestSha256: configuration.configurationDigestSha256, oldDirectory, targetDirectory, historicalDirectory, sourcePath: oldDirectory, sourceDevice: null, sourceInode: null, destinationDevice: null, destinationInode: null, contentDigestSha256: null, v1JournalLocation: oldV1Journal ? 'legacy' : 'none', }; } if (historicalIdentity) { return { mode: 'default', strategy: 'historical-direct', phase: 'pending', configurationDigestSha256: configuration.configurationDigestSha256, oldDirectory, targetDirectory, historicalDirectory, sourcePath: historicalDirectory, sourceDevice: historicalIdentity.device, sourceInode: historicalIdentity.inode, destinationDevice: null, destinationInode: null, contentDigestSha256: null, v1JournalLocation: 'target', }; } return { mode: 'default', strategy: 'fresh', phase: 'verified', configurationDigestSha256: configuration.configurationDigestSha256, oldDirectory, targetDirectory, historicalDirectory, sourcePath: null, sourceDevice: null, sourceInode: null, destinationDevice: null, destinationInode: null, contentDigestSha256: null, v1JournalLocation: 'none', }; } private async assertDefaultDatabaseSocketsIdle(): Promise { for (const socketPath of [ this.options.oldEmbeddedSocketPath, this.options.newEmbeddedSocketPath, ]) { this.options.signal?.throwIfAborted(); if (await this.isSocketListening(socketPath)) { throw new Error(`Controller embedded database socket is still active: ${socketPath}`); } } } private async prepareOldDatabaseRelocation( journalArg: IControllerDataRootMigrationJournal, ): Promise { const database = journalArg.database; if (database.strategy !== 'old-relocation' || database.phase !== 'pending') return journalArg; if (database.v1JournalLocation === 'legacy') { if (!database.historicalDirectory || !database.oldDirectory) { throw new Error('Legacy database migration journal paths are incomplete.'); } const migration = new EmbeddedDatabaseLocationMigration({ sourceDirectory: database.historicalDirectory, destinationDirectory: database.oldDirectory, sourceSocketPath: embeddedDatabaseSocketPath(database.historicalDirectory), destinationSocketPath: this.options.oldEmbeddedSocketPath, }); await runEmbeddedDatabaseLocationMigration(migration); } const sourceIdentity = await readDirectoryIdentity( database.oldDirectory!, 'Legacy config-root database', ); if (!sourceIdentity) throw new Error('Legacy config-root database is missing before relocation.'); this.options.signal?.throwIfAborted(); const contentDigestSha256 = await inspectDatabaseContentDigest(database.oldDirectory!); const prepared: IControllerDataRootMigrationJournal = { ...journalArg, database: { ...database, phase: 'prepared', sourceDevice: sourceIdentity.device, sourceInode: sourceIdentity.inode, contentDigestSha256, }, }; await this.writeJournal(prepared); return prepared; } private async ensureTargetStaged( journalArg: IControllerDataRootMigrationJournal, ): Promise { if (journalArg.phase !== 'legacy-authoritative') { await this.assertTargetIdentity(journalArg); await this.assertTargetMarker(journalArg.targetCreationNonce); await this.inspectTargetStagingInventory(journalArg); return journalArg; } let targetIdentity = await readDirectoryIdentity(this.newRoot, 'Target controller root', 0o700); if (!targetIdentity) { await plugins.fs.promises.mkdir(this.newRoot, { mode: 0o700 }); await syncDirectory(this.commonParent); targetIdentity = await readDirectoryIdentity(this.newRoot, 'Target controller root', 0o700); if (!targetIdentity) throw new Error('Target controller root was not created.'); } const entries = await this.readBoundedDirectory(this.newRoot, 'Target controller root'); const allowedEntries = new Set([markerFileName]); if (this.preservedTargetDatabasePath()) allowedEntries.add('smartdb'); if (entries.some((entry) => !allowedEntries.has(entry))) { throw new Error('Legacy-authoritative retry found an unexpected staged target entry.'); } await this.ensureTargetMarker(journalArg.targetCreationNonce); await syncDirectory(this.newRoot); const stagedIdentity = await readDirectoryIdentity(this.newRoot, 'Target controller root', 0o700); if (!stagedIdentity || !identitiesEqual(stagedIdentity, targetIdentity)) { throw new Error('Target controller root identity changed while it was staged.'); } const staged: IControllerDataRootMigrationJournal = { ...journalArg, phase: 'target-staged', destinationDevice: stagedIdentity.device, destinationInode: stagedIdentity.inode, }; await this.writeJournal(staged); return staged; } private async ensureTargetMarker(nonceArg: string): Promise { const existing = await readPrivateJson( this.markerPath, maximumMarkerBytes, parseMarker, 'Controller target marker', ); if (existing) { if (existing.value.targetCreationNonce !== nonceArg) { throw new Error('Controller target marker belongs to another migration.'); } return; } const temporaryPath = plugins.path.join( this.commonParent, `${markerFileName}.tmp-${nonceArg}`, ); let temporary = await readPrivateJson( temporaryPath, maximumMarkerBytes, parseMarker, 'Controller target marker artifact', ); if (!temporary) { const marker: IMarker = { version: migrationVersion, targetCreationNonce: nonceArg }; const serialized = `${JSON.stringify(marker)}\n`; let handle: plugins.fs.promises.FileHandle | undefined; let temporaryIdentity: IFilesystemIdentity | undefined; try { handle = await plugins.fs.promises.open( temporaryPath, plugins.fs.constants.O_WRONLY | plugins.fs.constants.O_CREAT | plugins.fs.constants.O_EXCL | plugins.fs.constants.O_NOFOLLOW, 0o600, ); temporaryIdentity = pathIdentity(await handle.stat({ bigint: true })); await handle.chmod(0o600); await handle.writeFile(serialized, 'utf8'); await handle.sync(); await handle.close(); handle = undefined; await syncDirectory(this.commonParent); temporary = await readPrivateJson( temporaryPath, maximumMarkerBytes, parseMarker, 'Controller target marker artifact', ); temporaryIdentity = undefined; } catch (errorArg) { const cleanupErrors: unknown[] = []; if (handle) { try { await handle.close(); } catch (cleanupErrorArg) { cleanupErrors.push(cleanupErrorArg); } } if (temporaryIdentity) { try { await unlinkExactPath(temporaryPath, temporaryIdentity); await syncDirectory(this.commonParent); } catch (cleanupErrorArg) { cleanupErrors.push(cleanupErrorArg); } } if (cleanupErrors.length > 0) { throw new AggregateError( [errorArg, ...cleanupErrors], 'Controller target marker publication failed and cleanup was incomplete.', { cause: errorArg }, ); } throw errorArg; } } if (!temporary || temporary.value.targetCreationNonce !== nonceArg) { throw new Error('Controller target marker artifact belongs to another migration.'); } await plugins.fs.promises.rename(temporaryPath, this.markerPath); await syncDirectory(this.newRoot); await syncDirectory(this.commonParent); await this.assertTargetMarker(nonceArg); } private async inspectTargetStagingInventory( journalArg: IControllerDataRootMigrationJournal, ): Promise { await this.assertTargetIdentity(journalArg); await this.assertTargetMarker(journalArg.targetCreationNonce); const names = await this.readBoundedDirectory(this.newRoot, 'Staged target controller root'); let migrationTemporaryFiles = 0; for (const name of names) { const path = plugins.path.join(this.newRoot, name); if (name === markerFileName) continue; if (name === 'smartdb') { await this.requirePrivateNode(path, 'directory', 'Staged target controller database'); continue; } if (name === 'flex-provider-credentials') { await this.requirePrivateNode(path, 'directory', 'Staged target Flex credential root', 0o700); continue; } if (name === embeddedDatabaseJournalFileName) { await this.requirePrivateNode(path, 'file', 'Staged target embedded database journal'); continue; } if (embeddedDatabaseJournalTemporaryPattern.test(name)) { migrationTemporaryFiles += 1; await this.requirePrivateNode(path, 'file', 'Staged embedded database journal artifact'); continue; } throw new Error(`Unexpected entry in staged target controller root: ${name}`); } if (migrationTemporaryFiles > maximumMigrationTemporaryFiles) { throw new Error('Staged target has too many embedded database journal artifacts.'); } } private async performDatabaseMigration( journalArg: IControllerDataRootMigrationJournal, ): Promise { if (journalArg.database.phase === 'verified') { if (journalArg.database.strategy === 'old-relocation') { await this.confirmOldDatabaseRelocation(journalArg); await this.assertRelocatedDatabaseDigest(journalArg); } return journalArg; } if (journalArg.database.strategy === 'historical-direct') { return await this.performDirectHistoricalDatabaseMigration(journalArg); } if (journalArg.database.strategy !== 'old-relocation') { throw new Error('Controller database migration journal is not actionable.'); } let journal = journalArg; if (journal.database.phase === 'prepared') journal = await this.moveOldDatabase(journal); if (journal.database.phase !== 'moved') { throw new Error('Controller database relocation did not reach its moved phase.'); } await this.confirmOldDatabaseRelocation(journal); const database = journal.database; await this.assertRelocatedDatabaseDigest(journal); if (database.v1JournalLocation === 'legacy') { await rebaseCompletedEmbeddedDatabaseLocationMigration({ sourceDirectory: database.historicalDirectory!, previousDestinationDirectory: database.oldDirectory!, destinationDirectory: database.targetDirectory!, }); } const verified: IControllerDataRootMigrationJournal = { ...journal, database: { ...database, phase: 'verified' }, }; await this.writeJournal(verified); return verified; } private async moveOldDatabase( journalArg: IControllerDataRootMigrationJournal, ): Promise { const database = journalArg.database; const destinationIdentity = await this.confirmOldDatabaseRelocation(journalArg); const moved: IControllerDataRootMigrationJournal = { ...journalArg, database: { ...database, phase: 'moved', destinationDevice: destinationIdentity.device, destinationInode: destinationIdentity.inode, }, }; await this.writeJournal(moved); return moved; } private async confirmOldDatabaseRelocation( journalArg: IControllerDataRootMigrationJournal, ): Promise { const database = journalArg.database; if ( database.strategy !== 'old-relocation' || database.phase === 'pending' || !database.oldDirectory || !database.targetDirectory || !database.sourceDevice || !database.sourceInode ) throw new Error('Controller database relocation is not prepared for replay.'); const input: plugins.smartdb.ILocalSmartDbStoppedStorageRootRelocationInput = { sourceFolderPath: database.oldDirectory, destinationFolderPath: database.targetDirectory, relocationId: `hcon-controller-data-root-v2:${journalArg.targetCreationNonce}:smartdb`, }; this.options.signal?.throwIfAborted(); const receipt = await this.relocateStoppedDatabaseRoot(input, this.options.signal); const receiptRecord = assertPlainRecord(receipt, 'Controller SmartDB relocation receipt'); const receiptKeys = [ 'format', 'version', 'relocationId', 'sourceFolderPath', 'destinationFolderPath', 'providerRootId', 'storageRootDevice', 'storageRootInode', 'receiptSha256', 'sourceReceiptRetained', ] as const; assertExactKeys(receiptRecord, receiptKeys, 'Controller SmartDB relocation receipt'); if ( receipt.format !== 'smartdb.storage-root-relocation.receipt.v1' || receipt.version !== 1 || receipt.relocationId !== input.relocationId || receipt.sourceFolderPath !== input.sourceFolderPath || receipt.destinationFolderPath !== input.destinationFolderPath || !digestPattern.test(receipt.providerRootId) || !digestPattern.test(receipt.receiptSha256) || receipt.storageRootDevice !== database.sourceDevice || receipt.storageRootInode !== database.sourceInode || (database.phase !== 'prepared' && ( receipt.storageRootDevice !== database.destinationDevice || receipt.storageRootInode !== database.destinationInode )) || receipt.sourceReceiptRetained !== true ) throw new Error('Controller SmartDB relocation receipt does not match its prepared source.'); const retainedReceipt = await readPrivateJson( database.oldDirectory, maximumSmartDbRelocationReceiptBytes, (valueArg) => { const value = assertPlainRecord(valueArg, 'Retained controller SmartDB relocation receipt'); assertExactKeys(value, receiptKeys, 'Retained controller SmartDB relocation receipt'); return value; }, 'Retained controller SmartDB relocation receipt', ); if ( !retainedReceipt || receiptKeys.some((key) => retainedReceipt.value[key] !== receiptRecord[key]) ) throw new Error('Retained controller SmartDB relocation receipt does not match the API receipt.'); const destinationIdentity = await readDirectoryIdentity( database.targetDirectory, 'Target controller database', ); const receiptIdentity: IFilesystemIdentity = { device: receipt.storageRootDevice, inode: receipt.storageRootInode, }; if (!destinationIdentity || !identitiesEqual(destinationIdentity, receiptIdentity)) { throw new Error('Controller database relocation identity could not be confirmed.'); } return destinationIdentity; } private async assertRelocatedDatabaseDigest( journalArg: IControllerDataRootMigrationJournal, ): Promise { const database = journalArg.database; const digest = await inspectDatabaseContentDigest(database.targetDirectory!); if (digest !== database.contentDigestSha256) { throw new Error('Relocated controller database content digest does not match its source.'); } } private async performDirectHistoricalDatabaseMigration( journalArg: IControllerDataRootMigrationJournal, ): Promise { const database = journalArg.database; if (database.phase !== 'pending' || !database.historicalDirectory) { throw new Error('Direct historical database migration state is invalid.'); } const sourceIdentity = await readDirectoryIdentity( database.historicalDirectory, 'Historical state-root database', ); if (!sourceIdentity || !identitiesEqual(sourceIdentity, { device: database.sourceDevice!, inode: database.sourceInode!, })) throw new Error('Historical controller database identity changed before migration.'); const migration = new EmbeddedDatabaseLocationMigration({ sourceDirectory: database.historicalDirectory, destinationDirectory: database.targetDirectory!, sourceSocketPath: embeddedDatabaseSocketPath(database.historicalDirectory), destinationSocketPath: this.options.newEmbeddedSocketPath, }); await runEmbeddedDatabaseLocationMigration(migration); const destinationIdentity = await readDirectoryIdentity( database.targetDirectory!, 'Target controller database', ); if (!destinationIdentity) throw new Error('Directly migrated controller database is missing.'); const contentDigestSha256 = await inspectDatabaseContentDigest(database.targetDirectory!); const verified: IControllerDataRootMigrationJournal = { ...journalArg, database: { ...database, phase: 'verified', destinationDevice: destinationIdentity.device, destinationInode: destinationIdentity.inode, contentDigestSha256, }, }; await this.writeJournal(verified); return verified; } private async performCredentialMigrations( journalArg: IControllerDataRootMigrationJournal, ): Promise { let journal = journalArg; if (journal.credentials.length === 0) return journal; const destinationRoot = plugins.path.join(this.newRoot, 'flex-provider-credentials'); const existingDestinationRoot = await readDirectoryIdentity( destinationRoot, 'Target Flex credential root', 0o700, ); if (!existingDestinationRoot) { await plugins.fs.promises.mkdir(destinationRoot, { mode: 0o700 }); await syncDirectory(this.newRoot); } await readDirectoryIdentity(destinationRoot, 'Target Flex credential root', 0o700); for (let index = 0; index < journal.credentials.length; index += 1) { const tuple = journal.credentials[index]; if (tuple.complete) continue; this.options.signal?.throwIfAborted(); await this.relocateCredentialStore({ controllerHash: tuple.controllerHash, service: tuple.service, storeId: tuple.storeId, sourceDirectory: tuple.sourceDirectory, destinationDirectory: tuple.destinationDirectory, }, this.options.signal); const sourceIdentity = await readDirectoryIdentity( tuple.sourceDirectory, 'Relocated Flex credential source', 0o700, ); const destinationIdentity = await readDirectoryIdentity( tuple.destinationDirectory, 'Relocated Flex credential destination', 0o700, ); if (sourceIdentity || !destinationIdentity) { throw new Error(`Flex credential relocation was not confirmed: ${tuple.controllerHash}`); } const credentials = journal.credentials.map((entry, entryIndex) => entryIndex === index ? { ...entry, complete: true } : entry); journal = { ...journal, credentials }; await this.writeJournal(journal); } return journal; } private async validateCredentialLocations( journalArg: IControllerDataRootMigrationJournal, requireCompleteArg: boolean, ): Promise { if (journalArg.credentials.length === 0) return; const sourceRoot = plugins.path.join(this.oldRoot, 'flex-provider-credentials'); const destinationRoot = plugins.path.join(this.newRoot, 'flex-provider-credentials'); const sourceRootIdentity = await readDirectoryIdentity( sourceRoot, 'Legacy Flex credential root', 0o700, ); if (!sourceRootIdentity) throw new Error('Legacy Flex credential root disappeared during migration.'); const destinationRootIdentity = await readDirectoryIdentity( destinationRoot, 'Target Flex credential root', 0o700, ); if (journalArg.phase !== 'legacy-authoritative' && !destinationRootIdentity) { const hasDestinationStore = journalArg.credentials.some((tuple) => tuple.complete); if (hasDestinationStore) throw new Error('Target Flex credential root is missing.'); } const sourceHashes = new Set(await this.readCredentialHashes(sourceRoot, 'Legacy Flex credential root')); const destinationHashes = destinationRootIdentity ? new Set(await this.readCredentialHashes(destinationRoot, 'Target Flex credential root')) : new Set(); const expectedHashes = new Set(journalArg.credentials.map((tuple) => tuple.controllerHash)); for (const hash of [...sourceHashes, ...destinationHashes]) { if (!expectedHashes.has(hash)) { throw new Error(`Unexpected Flex credential store appeared during migration: ${hash}`); } } for (const tuple of journalArg.credentials) { const sourcePresent = sourceHashes.has(tuple.controllerHash); const destinationPresent = destinationHashes.has(tuple.controllerHash); if (sourcePresent === destinationPresent) { throw new Error(`Flex credential store has ambiguous location: ${tuple.controllerHash}`); } if ((tuple.complete || requireCompleteArg) && !destinationPresent) { throw new Error(`Completed Flex credential store is not at its target: ${tuple.controllerHash}`); } } } private async acquireLock( invokerIdentityArg: IControllerProcessIdentity, ): Promise { await this.cleanupStaleLockTemporaries(); const owner: ILockOwner = { version: 1, uid: currentUid(), pid: invokerIdentityArg.pid, processFingerprint: invokerIdentityArg.fingerprint, nonce: plugins.crypto.randomBytes(32).toString('hex'), }; const temporaryPath = `${this.lockPath}.tmp-${owner.uid}-${owner.pid}-${owner.nonce}`; const serialized = `${JSON.stringify(owner)}\n`; let temporaryHandle: plugins.fs.promises.FileHandle | undefined; let temporaryIdentity: IFilesystemIdentity | undefined; let publishedLockIdentity: IFilesystemIdentity | undefined; try { temporaryHandle = await plugins.fs.promises.open( temporaryPath, plugins.fs.constants.O_WRONLY | plugins.fs.constants.O_CREAT | plugins.fs.constants.O_EXCL | plugins.fs.constants.O_NOFOLLOW, 0o600, ); temporaryIdentity = pathIdentity(await temporaryHandle.stat({ bigint: true })); await temporaryHandle.chmod(0o600); await temporaryHandle.writeFile(serialized, 'utf8'); await temporaryHandle.sync(); await temporaryHandle.close(); temporaryHandle = undefined; const publishedTemporaryIdentity = temporaryIdentity; for (let attempt = 0; attempt < maximumLockAttempts; attempt += 1) { this.options.signal?.throwIfAborted(); if ((await this.listLockArtifacts()).guards.length > 0) { await this.cleanupStaleLockTemporaries(); await delay(lockWaitMilliseconds, this.options.signal); continue; } try { await plugins.fs.promises.link(temporaryPath, this.lockPath); publishedLockIdentity = publishedTemporaryIdentity; await syncDirectory(this.commonParent); const published = await this.readLockSnapshot(this.lockPath); if ( !published || !identitiesEqual(published.identity, publishedTemporaryIdentity) || JSON.stringify(published.owner) !== JSON.stringify(owner) ) throw new Error('Published controller migration lock does not match its owner record.'); await unlinkExactPath(temporaryPath, publishedTemporaryIdentity); temporaryIdentity = undefined; await syncDirectory(this.commonParent); publishedLockIdentity = undefined; return { owner, identity: published.identity }; } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code !== 'EEXIST') throw errorArg; } const existing = await this.readLockSnapshot(this.lockPath); if (!existing) continue; if (existing.owner.uid !== currentUid()) { throw new Error('Controller data-root migration lock belongs to another user.'); } if (await this.lockOwnerIsLive(existing.owner)) { await delay(lockWaitMilliseconds, this.options.signal); continue; } await this.reclaimStaleLock(existing); } throw new Error('Timed out waiting for the controller data-root migration lock.'); } catch (errorArg) { const cleanupErrors: unknown[] = []; let directoryChanged = false; if (temporaryHandle) { try { await temporaryHandle.close(); temporaryHandle = undefined; } catch (cleanupErrorArg) { cleanupErrors.push(cleanupErrorArg); } } if (publishedLockIdentity) { try { await unlinkExactPath(this.lockPath, publishedLockIdentity); directoryChanged = true; } catch (cleanupErrorArg) { cleanupErrors.push(cleanupErrorArg); } } if (temporaryIdentity) { try { await unlinkExactPath(temporaryPath, temporaryIdentity); directoryChanged = true; } catch (cleanupErrorArg) { cleanupErrors.push(cleanupErrorArg); } } if (directoryChanged) { try { await syncDirectory(this.commonParent); } catch (cleanupErrorArg) { cleanupErrors.push(cleanupErrorArg); } } if (cleanupErrors.length > 0) { throw new AggregateError( [errorArg, ...cleanupErrors], 'Controller migration lock publication failed and cleanup was incomplete.', { cause: errorArg }, ); } throw errorArg; } } private async readLockSnapshot(pathArg: string): Promise { const snapshot = await readPrivateJson( pathArg, maximumLockBytes, parseLockOwner, 'Controller data-root migration lock', ); return snapshot ? { owner: snapshot.value, identity: snapshot.identity } : undefined; } private async lockOwnerIsLive(ownerArg: ILockOwner): Promise { const identity = await readControllerProcessIdentity(ownerArg.pid); return identity !== null && identity.fingerprint === ownerArg.processFingerprint; } private async reclaimStaleLock(snapshotArg: ILockSnapshot): Promise { const guardPath = `${this.lockPath}.guard-${snapshotArg.owner.nonce}`; let guardCreated = false; let guardIdentity: IFilesystemIdentity | undefined; let operationError: unknown; try { try { await plugins.fs.promises.link(this.lockPath, guardPath); guardCreated = true; guardIdentity = snapshotArg.identity; } catch (errorArg) { if ((errorArg as NodeJS.ErrnoException).code === 'EEXIST') return; throw errorArg; } await syncDirectory(this.commonParent); const guard = await this.readLockSnapshot(guardPath); const stable = await this.readLockSnapshot(this.lockPath); if ( !guard || !stable || !identitiesEqual(guard.identity, snapshotArg.identity) || !identitiesEqual(stable.identity, snapshotArg.identity) || JSON.stringify(guard.owner) !== JSON.stringify(snapshotArg.owner) ) throw new Error('Controller migration stale-lock guard does not match the guarded owner.'); guardIdentity = guard.identity; if (await this.lockOwnerIsLive(guard.owner)) { throw new Error('Controller migration lock owner became live during stale reclaim.'); } const stableBeforeUnlink = await this.readLockSnapshot(this.lockPath); if (!stableBeforeUnlink || !identitiesEqual(stableBeforeUnlink.identity, guard.identity)) { throw new Error('Controller migration lock changed during stale reclaim.'); } await plugins.fs.promises.unlink(this.lockPath); await syncDirectory(this.commonParent); await unlinkExactPath(guardPath, guard.identity); guardCreated = false; guardIdentity = undefined; await syncDirectory(this.commonParent); } catch (errorArg) { operationError = errorArg; } const cleanupErrors: unknown[] = []; if (guardCreated && guardIdentity) { try { await unlinkExactPath(guardPath, guardIdentity); await syncDirectory(this.commonParent); } catch (cleanupErrorArg) { cleanupErrors.push(cleanupErrorArg); } } if (operationError) { if (cleanupErrors.length > 0) { throw new AggregateError( [operationError, ...cleanupErrors], 'Controller stale-lock reclaim failed and guard cleanup was incomplete.', { cause: operationError }, ); } throw operationError; } if (cleanupErrors.length > 0) { throw new AggregateError( cleanupErrors, 'Controller stale-lock guard cleanup was incomplete.', ); } } private async releaseLock(leaseArg: ILockLease): Promise { const current = await this.readLockSnapshot(this.lockPath); if ( !current || !identitiesEqual(current.identity, leaseArg.identity) || current.owner.nonce !== leaseArg.owner.nonce || JSON.stringify(current.owner) !== JSON.stringify(leaseArg.owner) ) throw new Error('Controller data-root migration lock release is ambiguous.'); await plugins.fs.promises.unlink(this.lockPath); await syncDirectory(this.commonParent); } private async listLockArtifacts(): Promise<{ temporaries: string[]; guards: string[] }> { const base = plugins.path.basename(this.lockPath); const entries = await plugins.fs.promises.readdir(this.commonParent, { withFileTypes: true }); const related = entries.map((entry) => entry.name).filter((name) => name.startsWith(`${base}.`)); if (related.length > maximumLockArtifacts) { throw new Error('Too many controller migration lock artifacts exist.'); } const temporaryPattern = new RegExp(`^${base.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.tmp-[0-9]+-[1-9][0-9]*-[a-f0-9]{64}$`); const guardPattern = new RegExp(`^${base.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\.guard-[a-f0-9]{64}$`); const temporaries: string[] = []; const guards: string[] = []; for (const name of related) { if (temporaryPattern.test(name)) temporaries.push(name); else if (guardPattern.test(name)) guards.push(name); else throw new Error(`Unknown controller migration lock artifact: ${name}`); } return { temporaries: temporaries.sort(), guards: guards.sort() }; } private async cleanupStaleLockTemporaries(): Promise { const artifacts = await this.listLockArtifacts(); let directoryChanged = false; for (const name of artifacts.temporaries) { const path = plugins.path.join(this.commonParent, name); const match = lockTemporarySuffixPattern.exec(name.slice(plugins.path.basename(this.lockPath).length)); if (!match) throw new Error(`Controller migration lock temporary name is invalid: ${name}`); const uid = Number(match[1]); const pid = Number(match[2]); if (!Number.isSafeInteger(uid) || uid !== currentUid() || !Number.isSafeInteger(pid)) { throw new Error(`Controller migration lock temporary binding is invalid: ${name}`); } let stats: plugins.fs.BigIntStats; try { stats = await plugins.fs.promises.lstat(path, { bigint: true }); } catch (errorArg) { if (isMissingError(errorArg)) continue; throw errorArg; } if ( !stats.isFile() || stats.isSymbolicLink() || stats.uid !== BigInt(currentUid()) || (stats.nlink !== 1n && stats.nlink !== 2n) || Number(stats.mode & 0o777n) !== 0o600 || stats.size > BigInt(maximumLockBytes) ) throw new Error(`Controller migration lock temporary artifact is unsafe: ${path}`); const identity = pathIdentity(stats); let snapshot: ILockSnapshot | undefined; try { snapshot = await this.readLockSnapshot(path); } catch (errorArg) { if (await readControllerProcessIdentity(pid)) continue; await unlinkExactPath(path, identity); directoryChanged = true; continue; } if (!snapshot) continue; if (!identitiesEqual(snapshot.identity, identity)) { throw new Error(`Controller migration lock temporary changed during inspection: ${path}`); } if ( snapshot.owner.uid !== uid || snapshot.owner.pid !== pid || snapshot.owner.nonce !== match[3] ) { throw new Error(`Controller migration lock temporary owner binding is invalid: ${path}`); } if (await this.lockOwnerIsLive(snapshot.owner)) continue; await unlinkExactPath(path, snapshot.identity); directoryChanged = true; } for (const name of artifacts.guards) { const path = plugins.path.join(this.commonParent, name); let stats: plugins.fs.BigIntStats; try { stats = await plugins.fs.promises.lstat(path, { bigint: true }); } catch (errorArg) { if (isMissingError(errorArg)) continue; throw errorArg; } if ( !stats.isFile() || stats.isSymbolicLink() || stats.uid !== BigInt(currentUid()) || (stats.nlink !== 1n && stats.nlink !== 2n) || Number(stats.mode & 0o777n) !== 0o600 || stats.size < 2n || stats.size > BigInt(maximumLockBytes) ) throw new Error(`Controller migration lock guard is unsafe: ${path}`); const guard = await this.readLockSnapshot(path); if (!guard) continue; if (!identitiesEqual(guard.identity, pathIdentity(stats))) { throw new Error('Controller migration lock guard changed during inspection.'); } if (guard.owner.uid !== currentUid()) { throw new Error('Controller migration lock guard belongs to another user.'); } if (await this.lockOwnerIsLive(guard.owner)) continue; if (Date.now() - Number(stats.ctimeMs) < lockGuardInitializationGraceMs) continue; await unlinkExactPath(path, guard.identity); directoryChanged = true; } if (directoryChanged) { await syncDirectory(this.commonParent); } } }