import * as plugins from './plugins.js'; import { controllerPackageName } from '../ts_interfaces/index.js'; import { defaultControllerPort } from './classes.config.js'; export interface IControllerProcessIdentity { pid: number; processGroupId: number; processGroupLeader: boolean; /** Linux /proc start-time ticks, used to reject PID reuse. */ startTicks?: string; arguments: string[]; argumentsTruncated?: boolean; fingerprint: string; } export interface IControllerProcessCandidate { identity: IControllerProcessIdentity; command: '__serve' | 'foreground'; port: number; } export interface IControllerDataWriterProcessCandidate { kind: 'controller' | 'temp-password' | 'flex-child'; identity: IControllerProcessIdentity; } const maximumProcessIdentities = 16_384; const processIdentityBatchSize = 128; const maximumProcessArguments = 512; const maximumProcessArgumentBytes = 256 * 1024; const maximumProcessCommandBytes = maximumProcessArgumentBytes + maximumProcessArguments; const maximumReconstructedPathArguments = 32; const maximumDataWriterCliPaths = 8; const maximumDataWriterCandidates = 1_024; const currentRuntimeName = String(controllerPackageName) === 'agl' ? 'AGL' : 'hcon'; const installedControllerPackagePaths = [ plugins.path.join('node_modules', '@modelprofile.com', 'harness-controller'), plugins.path.join('node_modules', 'agl'), ] as const; const boundedProcessArgumentsPrefix = (argumentsArg: readonly string[]): string[] => { const bounded: string[] = []; let totalBytes = 0; for (const argument of argumentsArg) { const argumentBytes = Buffer.byteLength(argument, 'utf8'); if ( bounded.length >= maximumProcessArguments || totalBytes + argumentBytes > maximumProcessArgumentBytes ) break; bounded.push(argument); totalBytes += argumentBytes; } return bounded; }; const assertPid = (pidArg: number): number => { if (!Number.isSafeInteger(pidArg) || pidArg < 2) { throw new Error('Controller PID is invalid.'); } return pidArg; }; const execFile = async ( fileArg: string, argsArg: string[], optionsArg: { detached?: boolean; timeoutMs?: number; maxBufferBytes?: number; emptyExitCode?: number } = {}, ): Promise => { return await new Promise((resolve, reject) => { plugins.childProcess.execFile( fileArg, argsArg, { encoding: 'utf8', maxBuffer: optionsArg.maxBufferBytes ?? 128 * 1024, timeout: optionsArg.timeoutMs ?? 2_000, killSignal: 'SIGKILL', ...(optionsArg.detached === undefined ? {} : { detached: optionsArg.detached }), }, (errorArg, stdoutArg, stderrArg) => { const expectedEmptyResult = optionsArg.emptyExitCode !== undefined && errorArg?.code === optionsArg.emptyExitCode && stdoutArg.trim() === '' && stderrArg.trim() === ''; if (errorArg && !expectedEmptyResult) { reject(errorArg); return; } resolve(stdoutArg); }, ); }); }; const readBoundedLinuxCommandArguments = async ( pidArg: number, ): Promise<{ arguments: string[]; truncated: boolean }> => { // Read only the accepted argv budget plus one byte so oversized processes // cannot make a system-wide inventory retain unbounded command data. const handle = await plugins.fs.promises.open( `/proc/${pidArg}/cmdline`, plugins.fs.constants.O_RDONLY, ); try { const buffer = Buffer.allocUnsafe(maximumProcessCommandBytes + 1); let bytesRead = 0; while (bytesRead < buffer.length) { const read = await handle.read(buffer, bytesRead, buffer.length - bytesRead, null); if (read.bytesRead === 0) break; bytesRead += read.bytesRead; } const commandLength = Math.min(bytesRead, maximumProcessCommandBytes); const argumentsResult: string[] = []; let argumentStart = 0; let argumentCountExceeded = false; for (let index = 0; index < commandLength; index++) { if (buffer[index] !== 0) continue; if (argumentsResult.length >= maximumProcessArguments) { argumentCountExceeded = true; break; } if (index > argumentStart) { argumentsResult.push(buffer.subarray(argumentStart, index).toString('utf8')); } argumentStart = index + 1; } if ( bytesRead <= maximumProcessCommandBytes && !argumentCountExceeded && argumentStart < commandLength && argumentsResult.length < maximumProcessArguments ) { argumentsResult.push(buffer.subarray(argumentStart, commandLength).toString('utf8')); } return { arguments: argumentsResult, truncated: bytesRead > maximumProcessCommandBytes || argumentCountExceeded, }; } finally { await handle.close(); } }; const readLinuxProcessMetadata = async (pidArg: number) => { const stat = await plugins.fs.promises.readFile(`/proc/${pidArg}/stat`, 'utf8'); const statSuffixIndex = stat.lastIndexOf(') '); if (statSuffixIndex < 0) throw new Error('Unable to parse controller process metadata.'); const statFields = stat.slice(statSuffixIndex + 2).trim().split(/\s+/); const processGroupId = Number(statFields[2]); const startTime = statFields[19]; const state = statFields[0]; if (!Number.isSafeInteger(processGroupId) || !startTime || !state) { throw new Error('Unable to parse controller process identity.'); } return { processGroupId, startTime, state, fingerprint: `linux:${pidArg}:${startTime}` }; }; const readLinuxIdentity = async (pidArg: number): Promise => { const [metadata, command] = await Promise.all([ readLinuxProcessMetadata(pidArg), readBoundedLinuxCommandArguments(pidArg), ]); return { pid: pidArg, processGroupId: metadata.processGroupId, processGroupLeader: metadata.processGroupId === pidArg, startTicks: metadata.startTime, arguments: command.arguments, ...(command.truncated ? { argumentsTruncated: true } : {}), fingerprint: metadata.fingerprint, }; }; const readLinuxProcessGroupId = async (pidArg: number): Promise => { try { const stat = await plugins.fs.promises.readFile(`/proc/${pidArg}/stat`, 'utf8'); const statSuffixIndex = stat.lastIndexOf(') '); if (statSuffixIndex < 0) return undefined; const statFields = stat.slice(statSuffixIndex + 2).trim().split(/\s+/); const processGroupId = Number(statFields[2]); return Number.isSafeInteger(processGroupId) ? processGroupId : undefined; } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code === 'ENOENT' || code === 'ESRCH' || code === 'EACCES') return undefined; throw error; } }; const readPosixIdentity = async (pidArg: number): Promise => { const output = await execFile('ps', [ '-ww', '-p', String(pidArg), '-o', 'pgid=', '-o', 'lstart=', '-o', 'command=', ], { maxBufferBytes: 8 * 1024 * 1024 }); const line = output.trim(); const match = /^\s*(\d+)\s+(.{24})\s+(.+)$/.exec(line); if (!match) throw new Error('Unable to parse controller process metadata.'); const processGroupId = Number(match[1]); return { pid: pidArg, processGroupId, processGroupLeader: processGroupId === pidArg, arguments: parsePosixCommandArguments(match[3]), fingerprint: `posix:${pidArg}:${match[2].trim()}`, }; }; const parsePosixCommandArguments = (commandArg: string): string[] => { return (commandArg.match(/"[^"]*"|'[^']*'|\S+/g) ?? []).map((argumentArg) => { const quoted = ( (argumentArg.startsWith('"') && argumentArg.endsWith('"')) || (argumentArg.startsWith("'") && argumentArg.endsWith("'")) ); return quoted ? argumentArg.slice(1, -1) : argumentArg; }); }; export const readControllerProcessIdentity = async ( pidArg: number, ): Promise => { const pid = assertPid(pidArg); try { if (process.platform === 'linux') return await readLinuxIdentity(pid); if (process.platform !== 'win32') return await readPosixIdentity(pid); return null; } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code === 'ENOENT' || code === 'ESRCH') return null; throw error; } }; /** Track a previously verified process until its kernel resources are released. */ export const controllerProcessIdentityIsLive = async ( pidArg: number, fingerprintArg: string, ): Promise => { const pid = assertPid(pidArg); if (process.platform === 'linux') { try { const metadata = await readLinuxProcessMetadata(pid); // argv disappears during exit_mm, before exit_files closes listening // sockets. Zombie/dead state is reached only after that file teardown. return metadata.fingerprint === fingerprintArg && !['Z', 'X', 'x'].includes(metadata.state); } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code === 'ENOENT' || code === 'ESRCH') return false; throw error; } } if (process.platform === 'win32') throw new Error('Controller process lifetime inspection requires POSIX.'); // ps exits 1 with no output when the process has gone. Other execution and // parsing failures must propagate instead of falsely declaring it exited. const output = await execFile('ps', ['-p', String(pid), '-o', 'lstart=', '-o', 'stat='], { maxBufferBytes: 1024, emptyExitCode: 1, }); if (!output.trim()) return false; const match = /^\s*(.{24})\s+(\S+)\s*$/.exec(output); if (!match) throw new Error('Unable to parse controller process lifetime.'); return `posix:${pid}:${match[1].trim()}` === fingerprintArg && match[2][0] !== 'Z'; }; export const readProcessGroupMemberPids = async ( processGroupIdArg: number, ): Promise => { const processGroupId = assertPid(processGroupIdArg); if (process.platform === 'linux') { const entries = await plugins.fs.promises.readdir('/proc'); const processIds = entries .filter((entryArg) => /^[1-9][0-9]*$/.test(entryArg)) .map((entryArg) => Number(entryArg)) .filter((processIdArg) => processIdArg >= 2); const groupIds = await Promise.all( processIds.map(async (processIdArg) => ({ processId: processIdArg, processGroupId: await readLinuxProcessGroupId(processIdArg), })), ); return groupIds .filter((entryArg) => entryArg.processGroupId === processGroupId) .map((entryArg) => entryArg.processId) .sort((leftArg, rightArg) => leftArg - rightArg); } if (process.platform !== 'win32') { // A detached helper gets its own process group, so macOS `ps -ax` cannot // report the probe itself as a member of the controller group being read. const output = await execFile( 'ps', ['-ax', '-o', 'pid=', '-o', 'pgid='], { detached: true }, ); return output .split(/\r?\n/) .map((lineArg) => /^\s*(\d+)\s+(\d+)\s*$/.exec(lineArg)) .filter((matchArg): matchArg is RegExpExecArray => matchArg !== null) .filter((matchArg) => Number(matchArg[2]) === processGroupId) .map((matchArg) => Number(matchArg[1])) .sort((leftArg, rightArg) => leftArg - rightArg); } return []; }; const findCliArgumentIndex = async ( pidArg: number, argumentsArg: string[], cliPathArg: string, allowMissingAbsolutePathArg = false, ): Promise => { const boundedArguments = boundedProcessArgumentsPrefix(argumentsArg); const directIndex = boundedArguments.indexOf(cliPathArg); if (directIndex >= 0) return directIndex; // A controller launched with a relative path (`node cli.js foreground`) is // still the same file; resolve candidate tokens against the process's own // working directory before declaring the identity unverifiable. let processCwd: string | undefined; if (process.platform === 'linux') { try { processCwd = await plugins.fs.promises.readlink(`/proc/${pidArg}/cwd`); } catch { processCwd = undefined; } } const cliBasename = plugins.path.basename(cliPathArg); const normalizedCliPath = plugins.path.normalize(cliPathArg); for (let endIndex = 1; endIndex <= boundedArguments.length; endIndex++) { const minimumStartIndex = Math.max(0, endIndex - maximumReconstructedPathArguments); for (let startIndex = endIndex - 1; startIndex >= minimumStartIndex; startIndex--) { const candidate = boundedArguments.slice(startIndex, endIndex).join(' '); if (plugins.path.basename(candidate) !== cliBasename) continue; try { const resolved = await plugins.fs.promises.realpath( plugins.path.isAbsolute(candidate) ? candidate : plugins.path.resolve(processCwd ?? '', candidate), ); if (resolved === cliPathArg) return endIndex - 1; } catch { // pnpm may remove A's package while its worker still owns the upgrade // lock. Only that trusted caller may use an exact normalized old path. if ( allowMissingAbsolutePathArg && plugins.path.isAbsolute(candidate) && plugins.path.normalize(candidate) === normalizedCliPath ) return endIndex - 1; continue; } } } return -1; }; export const processIdentityHasFileArgument = async ( identityArg: IControllerProcessIdentity, filePathArg: string, optionsArg: { allowMissingAbsolutePath?: boolean } = {}, ): Promise => await findCliArgumentIndex( identityArg.pid, identityArg.arguments, filePathArg, optionsArg.allowMissingAbsolutePath === true, ) >= 0; const findInstalledControllerPackageFileArgumentIndex = ( argumentsArg: readonly string[], packageRelativePathArg: string, ): number => { const boundedArguments = boundedProcessArgumentsPrefix(argumentsArg); const expectedSuffixes = installedControllerPackagePaths.map((packagePathArg) => ( `${plugins.path.sep}${plugins.path.join(packagePathArg, packageRelativePathArg)}` )); const expectedBasename = plugins.path.basename(packageRelativePathArg); for (let endIndex = 1; endIndex <= boundedArguments.length; endIndex++) { const minimumStartIndex = Math.max(0, endIndex - maximumReconstructedPathArguments); for (let startIndex = endIndex - 1; startIndex >= minimumStartIndex; startIndex--) { const candidate = boundedArguments.slice(startIndex, endIndex).join(' '); if ( plugins.path.basename(candidate) === expectedBasename && plugins.path.isAbsolute(candidate) && expectedSuffixes.some((suffixArg) => ( plugins.path.normalize(candidate).endsWith(suffixArg) )) ) return endIndex - 1; } } return -1; }; const processIdentityHasInstalledControllerCommand = ( identityArg: IControllerProcessIdentity, commandArg: '__serve' | 'foreground' | 'temp-password', ): boolean => { const cliIndex = findInstalledControllerPackageFileArgumentIndex( identityArg.arguments, 'cli.js', ); return cliIndex >= 0 && identityArg.arguments[cliIndex + 1] === commandArg; }; const readEffectiveArgvPort = (argumentsArg: string[]): number => { let effectivePort = defaultControllerPort; const boundedArguments = boundedProcessArgumentsPrefix(argumentsArg); for (let index = 0; index < boundedArguments.length; index++) { if (boundedArguments[index] === '--port') { effectivePort = Number(boundedArguments[index + 1]); } else if (boundedArguments[index].startsWith('--port=')) { effectivePort = Number(boundedArguments[index].slice('--port='.length)); } } return effectivePort; }; export const processIdentityHasCliCommand = async ( identityArg: IControllerProcessIdentity, cliPathArg: string, commandArg: '__serve' | '__upgrade-worker' | 'upgrade' | 'start' | 'foreground' | 'temp-password', optionsArg: { allowMissingAbsolutePath?: boolean } = {}, ): Promise => { const cliIndex = await findCliArgumentIndex( identityArg.pid, identityArg.arguments, cliPathArg, optionsArg.allowMissingAbsolutePath === true, ); return cliIndex >= 0 && identityArg.arguments[cliIndex + 1] === commandArg; }; const hasExpectedArguments = async ( identityArg: IControllerProcessIdentity, cliPathArg: string, portArg: number, commandArg: '__serve' | 'foreground', ): Promise => { return await processIdentityHasCliCommand(identityArg, cliPathArg, commandArg) && readEffectiveArgvPort(identityArg.arguments) === portArg; }; const visitProcessIdentities = async ( visitorArg: (identityArg: IControllerProcessIdentity) => Promise, ): Promise => { if (process.platform === 'linux') { const entries = await plugins.fs.promises.readdir('/proc'); const processIds = entries .filter((entryArg) => /^[1-9][0-9]*$/.test(entryArg)) .map((entryArg) => Number(entryArg)) .filter((processIdArg) => processIdArg >= 2) .sort((leftArg, rightArg) => leftArg - rightArg); if (processIds.length > maximumProcessIdentities) { throw new Error('Controller data-writer process inventory exceeds its scan bound.'); } for (let index = 0; index < processIds.length; index += processIdentityBatchSize) { const batch = await Promise.all( processIds.slice(index, index + processIdentityBatchSize).map(async (processIdArg) => { try { return await readControllerProcessIdentity(processIdArg); } catch (errorArg) { const code = (errorArg as NodeJS.ErrnoException).code; if (code === 'EACCES' || code === 'EPERM') return null; throw errorArg; } }), ); for (const identity of batch) { if (identity) await visitorArg(identity); } } return; } if (process.platform === 'win32') return; const output = await execFile('ps', [ '-ax', '-ww', '-o', 'pid=', '-o', 'pgid=', '-o', 'lstart=', '-o', 'command=', ], { maxBufferBytes: 8 * 1024 * 1024 }); let identityCount = 0; for (const lineArg of output.split(/\r?\n/)) { const match = /^\s*(\d+)\s+(\d+)\s+(.{24})\s+(.+)$/.exec(lineArg); if (!match) continue; const pid = Number(match[1]); const processGroupId = Number(match[2]); if (!Number.isSafeInteger(pid) || !Number.isSafeInteger(processGroupId)) continue; identityCount += 1; if (identityCount > maximumProcessIdentities) { throw new Error('Controller data-writer process inventory exceeds its scan bound.'); } await visitorArg({ pid, processGroupId, processGroupLeader: processGroupId === pid, arguments: parsePosixCommandArguments(match[4]), fingerprint: `posix:${pid}:${match[3].trim()}`, }); } }; const compactProcessIdentity = ( identityArg: IControllerProcessIdentity, ): IControllerProcessIdentity => ({ ...identityArg, arguments: [], }); export const listControllerProcessesForCli = async ( cliPathArg: string, ): Promise => { const candidates: IControllerProcessCandidate[] = []; await visitProcessIdentities(async (identity) => { for (const command of ['__serve', 'foreground'] as const) { if (!await processIdentityHasCliCommand(identity, cliPathArg, command)) continue; const port = readEffectiveArgvPort(identity.arguments); if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) { throw new Error(`An ${currentRuntimeName} process has an invalid port argument (PID ${identity.pid}).`); } candidates.push({ identity: compactProcessIdentity(identity), command, port }); } }); return candidates.sort((leftArg, rightArg) => leftArg.identity.pid - rightArg.identity.pid); }; export const listControllerDataWriterProcessesForCli = async ( cliPathArg: string, optionsArg: { includeInstalledPackagePaths?: boolean } = {}, ): Promise => await listControllerDataWriterProcessesForCliPaths([cliPathArg], optionsArg); export const listControllerDataWriterProcessesForCliPaths = async ( cliPathsArg: readonly string[], optionsArg: { includeInstalledPackagePaths?: boolean } = {}, ): Promise => { const cliPaths = [...new Set(cliPathsArg)]; if ( cliPaths.length === 0 || cliPaths.length > maximumDataWriterCliPaths || cliPaths.some((cliPathArg) => !plugins.path.isAbsolute(cliPathArg)) ) throw new Error('Controller data-writer CLI path inventory is invalid.'); const flexChildPaths = cliPaths.map((cliPathArg) => plugins.path.join( plugins.path.dirname(cliPathArg), 'dist_ts', 'flexharness.child.js', )); const candidates: IControllerDataWriterProcessCandidate[] = []; await visitProcessIdentities(async (identity) => { let matched = false; for (const cliPath of cliPaths) { if ( await processIdentityHasCliCommand(identity, cliPath, '__serve') || await processIdentityHasCliCommand(identity, cliPath, 'foreground') ) { matched = true; break; } } if ( !matched && optionsArg.includeInstalledPackagePaths === true && (processIdentityHasInstalledControllerCommand(identity, '__serve') || processIdentityHasInstalledControllerCommand(identity, 'foreground')) ) matched = true; if (matched) { if (candidates.length >= maximumDataWriterCandidates) { throw new Error('Controller data-writer candidate inventory exceeds its scan bound.'); } candidates.push({ kind: 'controller', identity: compactProcessIdentity(identity) }); return; } for (const cliPath of cliPaths) { if (await processIdentityHasCliCommand(identity, cliPath, 'temp-password')) { matched = true; break; } } if ( !matched && optionsArg.includeInstalledPackagePaths === true && processIdentityHasInstalledControllerCommand(identity, 'temp-password') ) matched = true; if (matched) { if (candidates.length >= maximumDataWriterCandidates) { throw new Error('Controller data-writer candidate inventory exceeds its scan bound.'); } candidates.push({ kind: 'temp-password', identity: compactProcessIdentity(identity) }); return; } for (const flexChildPath of flexChildPaths) { if (await processIdentityHasFileArgument(identity, flexChildPath)) { matched = true; break; } } if ( !matched && optionsArg.includeInstalledPackagePaths === true && findInstalledControllerPackageFileArgumentIndex( identity.arguments, plugins.path.join('dist_ts', 'flexharness.child.js'), ) >= 0 ) matched = true; if (matched) { if (candidates.length >= maximumDataWriterCandidates) { throw new Error('Controller data-writer candidate inventory exceeds its scan bound.'); } candidates.push({ kind: 'flex-child', identity: compactProcessIdentity(identity) }); } }); return candidates.sort((leftArg, rightArg) => leftArg.identity.pid - rightArg.identity.pid); }; export const inspectControllerProcess = async (optionsArg: { pid: number; cliPath: string; port: number; expectedProcessGroupId?: number; expectedFingerprint?: string; command: '__serve' | 'foreground'; }): Promise => { const identity = await readControllerProcessIdentity(optionsArg.pid); if (!identity) return null; if (!await hasExpectedArguments(identity, optionsArg.cliPath, optionsArg.port, optionsArg.command)) { return null; } if ( optionsArg.expectedProcessGroupId !== undefined && identity.processGroupId !== optionsArg.expectedProcessGroupId ) return null; if ( optionsArg.expectedFingerprint !== undefined && identity.fingerprint !== optionsArg.expectedFingerprint ) return null; return identity; }; export const signalVerifiedControllerProcess = async (optionsArg: { pid: number; cliPath: string; port: number; expectedProcessGroupId: number; expectedFingerprint: string; signal: NodeJS.Signals; signalProcessGroup: boolean; command: '__serve' | 'foreground'; }): Promise => { const identity = await inspectControllerProcess(optionsArg); if (!identity) { throw new Error('Refusing to signal a process whose controller identity cannot be verified.'); } if (optionsArg.signalProcessGroup) { if (!identity.processGroupLeader) { throw new Error('Refusing to signal a process group whose controller is not its leader.'); } process.kill(-identity.processGroupId, optionsArg.signal); return; } process.kill(identity.pid, optionsArg.signal); };