import { spawnSync } from "node:child_process"; import { closeSync, existsSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, writeFileSync, } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { basename, dirname, join, resolve } from "node:path"; import { inspectSlot, lockPath, beginPublishing, recordOwnedPgid, releaseReusable, releaseUncertain, tryAcquireSlot, type OwnedSlot, } from "./sim-pool-lock"; import { buildAttemptCommand, deviceState, findMatchingPoolDevice, parseDevicesJson, parseRuntimesJson, poolDeviceName, preferredAcquireSlots, selectIosRuntime, type SimulatorDevice, } from "./sim-pool-simctl"; import { CommandSession, waitExitStatus, type StopResult, type Supervised, } from "./sim-pool-supervise"; export class PoolError extends Error { constructor(message: string) { super(message); this.name = "PoolError"; } } export type VideoPolicy = "off" | "on-failure" | "always"; export type RuntimePolicy = "latest" | "latest-stable"; export type PoolConfig = { count: number; slotStart: number; deviceType: string; runtime: string; runtimePolicy: RuntimePolicy; waitSeconds: number; bootTimeout: number; bootRetries: number; silenceTimeout: number; heartbeatInterval: number; hangRetries: number; keepBooted: boolean; forceCleanBoot: boolean; slim: boolean; indexStore: boolean; lockDir: string; videoPolicy: VideoPolicy; videoReadyTimeout: number; videoDir: string; videoName: string; allowSlowUnitTestScheme: boolean; progressPollSeconds: number; oppiRoot: string; oppiRootFromEnv: boolean; appleDir: string; buildBase: string; slimScript: string; homeDir: string; }; export function die(message: string): never { throw new PoolError(message); } function parsePositiveInt(raw: string, name: string, minimum = 1): number { if (!/^[0-9]+$/.test(raw)) { die(`invalid ${name} '${raw}' (expected ${minimum === 0 ? "non-negative" : "positive"} integer)`); } const value = Number(raw); if (value < minimum) { die(`invalid ${name} '${raw}' (expected ${minimum === 0 ? "non-negative" : "positive"} integer)`); } return value; } export function normalizeVideoPolicy(raw: string): VideoPolicy { switch (raw) { case "1": case "true": case "TRUE": case "yes": case "YES": case "always": return "always"; case "on-failure": case "on_failure": case "failure": case "fail": case "failed": return "on-failure"; case "0": case "false": case "FALSE": case "no": case "NO": case "off": case "": return "off"; default: die(`invalid simulator video policy '${raw}' (expected off, on-failure, or always)`); } } export function normalizeRuntimePolicy(raw: string): RuntimePolicy { switch (raw) { case "latest-stable": case "stable": case "latest-non-beta": case "non-beta": return "latest-stable"; case "latest": case "latest-any": case "any": case "": return "latest"; default: die(`invalid OPPI_SIM_RUNTIME_POLICY '${raw}' (expected latest-stable or latest)`); } } function gitTopLevel(cwd: string): string | null { const result = spawnSync("git", ["rev-parse", "--show-toplevel"], { cwd, encoding: "utf8", }); if (result.status !== 0) { return null; } return result.stdout.trim(); } export function extractRootFlag(args: string[]): { root?: string; rest: string[] } { const rest: string[] = []; let root: string | undefined; for (let i = 0; i < args.length; i += 1) { const arg = args[i]; if (arg === "--") { rest.push(...args.slice(i)); return { root, rest }; } if (arg === "--root") { const value = args[i + 1]; if (!value || value.startsWith("-")) { die("--root requires a path"); } root = value; i += 1; continue; } if (arg.startsWith("--root=")) { const value = arg.slice("--root=".length); if (!value) { die("--root requires a path"); } root = value; continue; } rest.push(arg); } return { root, rest }; } export function normalizeOppiRoot(raw: string, cwd = process.cwd()): string { const resolved = resolve(cwd, raw); if (existsSync(join(resolved, "clients", "apple"))) { return resolved; } if (basename(resolved) === "apple" && existsSync(join(resolved, "Oppi.xcodeproj"))) { const clients = dirname(resolved); if (basename(clients) === "clients") { return dirname(clients); } } return resolved; } const OPPI_TESTS_INFO_PLIST = ` CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleExecutable $(EXECUTABLE_NAME) CFBundleIdentifier $(PRODUCT_BUNDLE_IDENTIFIER) CFBundleInfoDictionaryVersion 6.0 CFBundleName $(PRODUCT_NAME) CFBundlePackageType BNDL CFBundleShortVersionString 1.0 CFBundleVersion 1 `; export function ensureOppiTestsInfoPlist(appleDir: string): string { const buildDir = join(appleDir, ".build"); const plist = join(buildDir, "OppiTestsInfo.plist"); if (!existsSync(plist)) { mkdirSync(buildDir, { recursive: true }); writeFileSync(plist, OPPI_TESTS_INFO_PLIST); } return plist; } export function applyRunCheckout(config: PoolConfig): string { const pbx = join(config.appleDir, "Oppi.xcodeproj", "project.pbxproj"); if (!existsSync(pbx)) { die(`Apple checkout is missing Oppi.xcodeproj: ${config.appleDir}`); } let current = process.cwd(); let target = config.appleDir; try { current = canonical(current); target = canonical(config.appleDir); } catch { current = resolve(current); target = resolve(config.appleDir); } if (current !== target) { process.chdir(config.appleDir); } log(`[sim-pool] Apple checkout ${process.cwd()}`); ensureOppiTestsInfoPlist(config.appleDir); return process.cwd(); } export function loadConfig(env: NodeJS.ProcessEnv, cwd: string, scriptDir: string): PoolConfig { let oppiRootFromEnv = false; let oppiRoot = env.OPPI_ROOT ?? ""; if (oppiRoot) { oppiRootFromEnv = true; oppiRoot = normalizeOppiRoot(oppiRoot, cwd); } else { const gitRoot = gitTopLevel(cwd); if (gitRoot && existsSync(join(gitRoot, "clients", "apple"))) { oppiRoot = gitRoot; } else { oppiRoot = env.PIOS_ROOT ?? join(homedir(), "workspace", "oppi"); } } const appleDir = join(oppiRoot, "clients", "apple"); const buildBase = join(appleDir, ".build"); const count = parsePositiveInt(env.OPPI_SIM_POOL_COUNT ?? "6", "OPPI_SIM_POOL_COUNT", 1); const slotStart = parsePositiveInt( env.OPPI_SIM_POOL_SLOT_START ?? env.OPPI_SIM_POOL_SLOT_OFFSET ?? "0", "OPPI_SIM_POOL_SLOT_START", 0, ); return { count, slotStart, deviceType: env.OPPI_SIM_DEVICE_TYPE ?? "com.apple.CoreSimulator.SimDeviceType.iPhone-16-Pro", runtime: env.OPPI_SIM_RUNTIME ?? "", runtimePolicy: normalizeRuntimePolicy(env.OPPI_SIM_RUNTIME_POLICY ?? ""), waitSeconds: parsePositiveInt(env.OPPI_SIM_POOL_WAIT ?? "60", "OPPI_SIM_POOL_WAIT", 0), bootTimeout: parsePositiveInt(env.OPPI_SIM_POOL_BOOT_TIMEOUT ?? "120", "OPPI_SIM_POOL_BOOT_TIMEOUT", 1), bootRetries: parsePositiveInt(env.OPPI_SIM_POOL_BOOT_RETRIES ?? "1", "OPPI_SIM_POOL_BOOT_RETRIES", 0), silenceTimeout: parsePositiveInt( env.OPPI_SIM_POOL_SILENCE_TIMEOUT ?? "180", "OPPI_SIM_POOL_SILENCE_TIMEOUT", 0, ), heartbeatInterval: parsePositiveInt( env.OPPI_SIM_POOL_HEARTBEAT_INTERVAL ?? "60", "OPPI_SIM_POOL_HEARTBEAT_INTERVAL", 0, ), hangRetries: parsePositiveInt(env.OPPI_SIM_POOL_HANG_RETRIES ?? "1", "OPPI_SIM_POOL_HANG_RETRIES", 0), keepBooted: (env.OPPI_SIM_POOL_KEEP_BOOTED ?? "1") === "1", forceCleanBoot: env.OPPI_SIM_POOL_FORCE_CLEAN_BOOT === "1", slim: (env.OPPI_SIM_SLIM ?? env.OPPI_SIM_POOL_SLIM ?? "1") === "1", indexStore: env.OPPI_SIM_POOL_INDEX_STORE === "1", lockDir: env.OPPI_SIM_POOL_LOCK_DIR ?? "/tmp/oppi-sim-pool", videoPolicy: normalizeVideoPolicy(env.OPPI_SIM_POOL_VIDEO_POLICY ?? env.OPPI_SIM_POOL_RECORD_VIDEO ?? "off"), videoReadyTimeout: parsePositiveInt( env.OPPI_SIM_POOL_VIDEO_READY_TIMEOUT ?? "10", "OPPI_SIM_POOL_VIDEO_READY_TIMEOUT", 0, ), videoDir: env.OPPI_SIM_POOL_VIDEO_DIR ?? join(buildBase, "videos"), videoName: env.OPPI_SIM_POOL_VIDEO_NAME ?? "", allowSlowUnitTestScheme: env.OPPI_SIM_POOL_ALLOW_SLOW_UNIT_TEST_SCHEME === "1", progressPollSeconds: Number(env.OPPI_SIM_POOL_PROGRESS_POLL ?? "5"), oppiRoot, oppiRootFromEnv, appleDir, buildBase, slimScript: join(scriptDir, "sim-slim.sh"), homeDir: env.HOME ?? homedir(), }; } export function poolSlotEnd(config: PoolConfig): number { return config.slotStart + config.count - 1; } export function poolSlotRange(config: PoolConfig): string { const end = poolSlotEnd(config); return config.slotStart === end ? String(config.slotStart) : `${config.slotStart}-${end}`; } export function slotNumbers(config: PoolConfig): number[] { const slots: number[] = []; for (let slot = config.slotStart; slot <= poolSlotEnd(config); slot += 1) { slots.push(slot); } return slots; } export function splitRunCommand(args: string[]): { command: string; argv: string[] } { if (args.length === 0) { die("run requires a command after --"); } return { command: args[0], argv: args.slice(1) }; } export function extractFlagValue(flag: string, args: string[]): string | undefined { for (let i = 0; i < args.length; i += 1) { if (args[i] === flag) { return args[i + 1]; } } return undefined; } export function commandHasBuildSetting(key: string, args: string[]): boolean { const prefix = `${key}=`; return args.some((arg) => arg.startsWith(prefix)); } export function applyPoolBuildSettings(config: PoolConfig, args: string[]): string[] { if (config.indexStore) { return []; } if (commandHasBuildSetting("COMPILER_INDEX_STORE_ENABLE", args)) { return []; } return ["COMPILER_INDEX_STORE_ENABLE=NO"]; } function isTestAction(args: string[]): boolean { return args.some((arg) => arg === "test" || arg === "build-for-testing" || arg === "test-without-building"); } export function hasOnlyTestingTarget(bundle: string, args: string[]): boolean { return args.some((arg) => arg === `-only-testing:${bundle}` || arg.startsWith(`-only-testing:${bundle}/`)); } export function onlyTestingTargetsAre(bundle: string, args: string[]): boolean { let saw = false; for (const arg of args) { if (arg === `-only-testing:${bundle}` || arg.startsWith(`-only-testing:${bundle}/`)) { saw = true; continue; } if (arg.startsWith("-only-testing:")) { return false; } } return saw; } export function rewriteSchemeInArgs(oldScheme: string, newScheme: string, args: string[]): string[] { const rewritten: string[] = []; let previous = ""; for (const arg of args) { rewritten.push(previous === "-scheme" && arg === oldScheme ? newScheme : arg); previous = arg; } return rewritten; } export function normalizeCommandArgs(config: PoolConfig, args: string[]): { args: string[]; log?: string } { const scheme = extractFlagValue("-scheme", args); if ( !config.allowSlowUnitTestScheme && isTestAction(args) && scheme === "Oppi" && onlyTestingTargetsAre("OppiTests", args) ) { return { args: rewriteSchemeInArgs("Oppi", "OppiUnitTests", args), log: "[sim-pool] Rewriting -scheme Oppi -> OppiUnitTests for OppiTests-only run", }; } return { args }; } export function validateCommandGuardrails(config: PoolConfig, args: string[]): void { for (const arg of args) { if (arg === "-destination" || arg === "-derivedDataPath") { die(`do not pass ${arg} — sim-pool.sh auto-injects it`); } } const scheme = extractFlagValue("-scheme", args); if ( !config.allowSlowUnitTestScheme && isTestAction(args) && scheme === "Oppi" && onlyTestingTargetsAre("OppiTests", args) ) { die( "slow unit-test invocation detected: '-scheme Oppi' still builds OppiPerfTests/OppiUITests/OppiE2ETests. Use '-scheme OppiUnitTests' for OppiTests (override with OPPI_SIM_POOL_ALLOW_SLOW_UNIT_TEST_SCHEME=1).", ); } } export function extractCompilerLinkerErrors(logText: string): string[] { const lines = logText.split("\n").filter((line) => /^([^:]+:[0-9]+:[0-9]+: (fatal )?error:|[ \t]*(error:|clang: error:|swiftc: error:|ld: ))/.test(line), ); return [...new Set(lines)]; } export function extractBuildTimingSummary(logText: string): string[] { const lines = logText.split("\n"); const output: string[] = []; let capturing = false; let sawRow = false; for (const line of lines) { if (/^Build Timing Summary$/.test(line)) { capturing = true; output.push(line); continue; } if (capturing && line.trim() !== "") { output.push(line); sawRow = true; continue; } if (capturing && sawRow) { break; } } return output; } export function progressMtime(logFile: string, derivedData?: string): number { const paths = [logFile]; if (derivedData) { paths.push(join(derivedData, "Build"), join(derivedData, "Index.noindex"), join(derivedData, "ModuleCache.noindex")); } let latest = 0; for (const path of paths) { if (!existsSync(path)) { continue; } const mtime = Math.floor(statSync(path).mtimeMs / 1000); if (mtime > latest) { latest = mtime; } } return latest; } function log(message: string): void { process.stderr.write(`${message}\n`); } function nowEpoch(): number { return Math.floor(Date.now() / 1000); } function iso8601(epoch: number): string { return new Date(epoch * 1000).toISOString().replace(/\.\d{3}Z$/, "Z"); } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } function fileSize(path: string): number { try { return statSync(path).size; } catch { return 0; } } function fileMtime(path: string): number { try { return Math.floor(statSync(path).mtimeMs / 1000); } catch { return 0; } } function requireQuiescent(result: T, action: string): T { if (!result.stop.quiescent) { die(`${action}: ${result.stop.note ?? "owned process group still running"}`); } return result; } export async function runXcrun( session: CommandSession, args: string[], options: { cwd?: string; env?: NodeJS.ProcessEnv } = {}, ): Promise<{ code: number; stdout: string; stderr: string; stop: StopResult }> { const completed = await session.run("xcrun", args, { cwd: options.cwd, env: options.env, }); return { code: completed.code, stdout: completed.stdout, stderr: completed.stderr, stop: completed.stop, }; } async function listDevices(session: CommandSession, config: PoolConfig): Promise { const result = requireQuiescent(await runXcrun(session, ["simctl", "list", "devices", "-j"]), "simctl list"); if (session.canceled) { die("canceled while listing simulators"); } if (result.code !== 0) { die("failed to list simulators"); } return parseDevicesJson(result.stdout); } async function resolveRuntime(session: CommandSession, config: PoolConfig): Promise { if (config.runtime) { return config.runtime; } const result = requireQuiescent(await runXcrun(session, ["simctl", "list", "runtimes", "-j"]), "simctl list runtimes"); if (session.canceled) { die("canceled while listing simulator runtimes"); } if (result.code !== 0) { die("failed to list simulator runtimes"); } const selected = selectIosRuntime(parseRuntimesJson(result.stdout), config.runtimePolicy); if (typeof selected !== "string") { die(selected.error); } return selected; } export async function ensureSim(session: CommandSession, config: PoolConfig, slot: number): Promise { const runtime = await resolveRuntime(session, config); const devices = await listDevices(session, config); const found = findMatchingPoolDevice(devices, slot, runtime, config.deviceType); if (found.match) { return found.match.udid; } for (const device of found.mismatches) { log( `[sim-pool] Replacing ${poolDeviceName(slot)} ${device.udid} (runtime=${device.runtime} device=${device.deviceTypeIdentifier}) to match runtime=${runtime} device=${config.deviceType}`, ); await runSimctl(session, ["shutdown", device.udid], "simctl shutdown"); await runSimctl(session, ["delete", device.udid], "simctl delete"); } log(`[sim-pool] Creating simulator: ${poolDeviceName(slot)}`); const created = requireQuiescent( await runXcrun(session, ["simctl", "create", poolDeviceName(slot), config.deviceType, runtime]), "simctl create", ); if (session.canceled) { die("canceled while creating simulator"); } if (created.code !== 0) { die(`failed to create simulator: ${created.stderr.trim()}`); } return created.stdout.trim(); } async function waitForBootReady( session: CommandSession, config: PoolConfig, udid: string, ): Promise<{ status: number; stop: StopResult }> { const spawned = await session.spawn("xcrun", ["simctl", "bootstatus", udid, "-b"]); if (!spawned.ok) { return { status: 127, stop: spawned.stop ?? { quiescent: true, note: spawned.reason }, }; } const completed = await session.complete(spawned.owned, { timeoutMs: config.bootTimeout * 1000 }); if (!completed.stop.quiescent) { return { status: 1, stop: completed.stop }; } if (completed.timedOut) { return { status: 124, stop: completed.stop }; } return { status: waitExitStatus(completed.wait), stop: completed.stop }; } async function waitForBootReadyWithRetries(session: CommandSession, config: PoolConfig, udid: string): Promise { let attempt = 0; while (true) { if (session.canceled) { return false; } const result = await waitForBootReady(session, config, udid); if (!result.stop.quiescent) { die(result.stop.note ?? "bootstatus did not prove process-group quiescence"); } if (result.status === 0) { return true; } if (attempt >= config.bootRetries) { return false; } attempt += 1; log( `[sim-pool] Simulator is still starting after ${config.bootTimeout}s; continuing readiness wait ${attempt + 1}/${config.bootRetries + 1}`, ); } } async function slimSimulator(session: CommandSession, config: PoolConfig, udid: string): Promise { if (!config.slim) { return { quiescent: true }; } const spawned = await session.spawn("bash", [ config.slimScript, "apply", udid, ], { env: { ...process.env, OPPI_SIM_POOL_BOOT_TIMEOUT: String(config.bootTimeout), OPPI_SIM_POOL_BOOT_RETRIES: String(config.bootRetries), }, }); if (!spawned.ok) { if (spawned.stop && !spawned.stop.quiescent) { return spawned.stop; } die(spawned.reason); } const completed = await session.complete(spawned.owned); if (completed.stderr) { process.stderr.write(completed.stderr); } if (!completed.stop.quiescent) { return completed.stop; } if (waitExitStatus(completed.wait) !== 0) { die(`sim-slim apply failed for ${udid}`); } return completed.stop; } function simctlAlreadyInRequestedState(args: string[], stderr: string): boolean { const verb = args[0]; if (verb === "shutdown" && /current state:\s*Shutdown/i.test(stderr)) { return true; } if (verb === "boot" && /current state:\s*Booted/i.test(stderr)) { return true; } return false; } async function runSimctl(session: CommandSession, args: string[], action: string): Promise { if (session.canceled) { die(`canceled before ${action}`); } const result = requireQuiescent(await runXcrun(session, ["simctl", ...args]), action); if (session.canceled) { die(`canceled during ${action}`); } if (result.code === 0 || simctlAlreadyInRequestedState(args, result.stderr)) { return; } die(`${action} failed: ${result.stderr.trim() || result.stdout.trim() || `exit ${result.code}`}`); } export async function prepareSimulator( session: CommandSession, config: PoolConfig, udid: string, mode: "normal" | "recovery", ): Promise { if (mode === "recovery") { log(`[sim-pool] Recovery: shutting down + erasing simulator ${udid}`); await runSimctl(session, ["shutdown", udid], "simctl shutdown"); await runSimctl(session, ["erase", udid], "simctl erase"); } else if (config.forceCleanBoot) { log(`[sim-pool] Preparing clean simulator boot for ${udid}`); await runSimctl(session, ["shutdown", udid], "simctl shutdown"); } else { const devices = await listDevices(session, config); if (deviceState(devices, udid) === "Booted") { log(`[sim-pool] Reusing already-booted simulator ${udid}`); if (await waitForBootReadyWithRetries(session, config, udid)) { const slim = await slimSimulator(session, config, udid); if (!slim.quiescent) { die(slim.note ?? "slim did not reach quiescence"); } return; } log(`[sim-pool] Booted simulator was not ready; recycling ${udid}`); await runSimctl(session, ["shutdown", udid], "simctl shutdown"); } else { log(`[sim-pool] Preparing simulator boot for ${udid}`); await runSimctl(session, ["shutdown", udid], "simctl shutdown"); } } await runSimctl(session, ["boot", udid], "simctl boot"); if (!(await waitForBootReadyWithRetries(session, config, udid))) { die( session.canceled ? "canceled while waiting for boot-ready" : `simulator ${udid} failed to reach boot-ready state after ${config.bootRetries + 1} readiness waits of ${config.bootTimeout}s`, ); } const slim = await slimSimulator(session, config, udid); if (!slim.quiescent) { die(slim.note ?? "slim did not reach quiescence"); } } async function acquireRunSlot( config: PoolConfig, argv: string[], session: CommandSession, ): Promise { const deadline = Date.now() + config.waitSeconds * 1000; let announced = false; const runtime = await resolveRuntime(session, config); while (true) { if (session.canceled) { die("canceled while waiting for a simulator slot"); } const devices = await listDevices(session, config); for (const slot of preferredAcquireSlots( slotNumbers(config), devices, runtime, config.deviceType, )) { if (session.canceled) { die("canceled while waiting for a simulator slot"); } const result = tryAcquireSlot({ lockDir: config.lockDir, slot, argv }); if (result.ok) { return result.owned; } } if (Date.now() >= deadline) { die( `all ${config.count} simulator slots (slots ${poolSlotRange(config)}) are busy or quarantined (waited ${config.waitSeconds}s)`, ); } if (!announced) { log( `[sim-pool] All ${config.count} slots (slots ${poolSlotRange(config)}) busy, waiting up to ${config.waitSeconds}s...`, ); announced = true; } await session.waitWhileIdle(3000); } } type ArtifactInput = { artifactPath: string; scope: "attempt" | "final"; startedAt: number; endedAt: number; attemptNumber: number; attemptCount: number; hangDetected: boolean; slot: number; udid: string; derivedData: string; logFile: string; exitCode: number; extra?: Record; }; function writeJsonArtifact(input: ArtifactInput): void { const payload: Record = { scope: input.scope, started_at: iso8601(input.startedAt), ended_at: iso8601(input.endedAt), started_at_epoch: input.startedAt, ended_at_epoch: input.endedAt, elapsed_seconds: input.endedAt - input.startedAt, attempt_number: input.attemptNumber, attempt_count: input.attemptCount, hang_detected: input.hangDetected, simulator_slot: input.slot, simulator_udid: input.udid, derived_data_path: input.derivedData, log_path: input.logFile, last_log_update_age_seconds: input.endedAt - fileMtime(input.logFile), exit_code: input.exitCode, }; if (input.extra) { Object.assign(payload, input.extra); } writeFileSync(input.artifactPath, `${JSON.stringify(payload, null, 2)}\n`); } function logTestFailureLines(logText: string): string[] { return [ ...new Set( logText.split("\n").filter((line) => /failed\b.*-\[|Test Case .* failed|error:.*assert|#expect.* failed|Issue recorded|Expectation failed/.test( line, ), ), ), ]; } function extractTestFailures(logText: string, derivedData?: string): { lines: string[]; uncertain: boolean } { const fallback = logTestFailureLines(logText); if (derivedData) { const testLogs = join(derivedData, "Logs", "Test"); if (existsSync(testLogs)) { const bundles = readdirSync(testLogs) .filter((name) => name.endsWith(".xcresult")) .map((name) => join(testLogs, name)) .sort(); const latest = bundles.at(-1); if (latest) { const tool = spawnSync( "xcrun", ["xcresulttool", "get", "test-results", "failures", "--path", latest], { encoding: "utf8", timeout: 15_000, killSignal: "SIGKILL" }, ); if (tool.error || tool.signal != null || tool.status == null) { return { lines: fallback, uncertain: true }; } if (tool.status === 0 && tool.stdout.trim() && tool.stdout.trim() !== "[]" && !tool.stdout.includes("No failures")) { return { lines: [tool.stdout.trim()], uncertain: false }; } } } } return { lines: fallback, uncertain: false }; } function printSummary(input: { logFile: string; exitCode: number; startedAt: number; endedAt: number; totalStartedAt: number; totalEndedAt: number; attemptCount: number; hangDetected: boolean; slotWait: number; simPrep: number; artifactPath: string; videoPath?: string; derivedData?: string; }): boolean { const duration = input.endedAt - input.startedAt; const totalDuration = input.totalEndedAt - input.totalStartedAt; const lines: string[] = ["", `Started: ${new Date(input.totalStartedAt * 1000).toLocaleString()}`, `Ended: ${new Date(input.totalEndedAt * 1000).toLocaleString()}`]; if (totalDuration !== duration) { lines.push(`Total elapsed: ${totalDuration}s`); lines.push(`xcodebuild elapsed: ${duration}s`); lines.push( `Breakdown: slot wait ${input.slotWait}s + simulator prep ${input.simPrep}s + xcodebuild ${duration}s`, ); } else { lines.push(`Elapsed: ${duration}s`); } lines.push(`Attempts: ${input.attemptCount}`); lines.push(`Log size: ${fileSize(input.logFile)} bytes`); lines.push(`Last log update age: ${input.endedAt - fileMtime(input.logFile)}s`); const logText = existsSync(input.logFile) ? readFileSync(input.logFile, "utf8") : ""; const timing = extractBuildTimingSummary(logText); if (timing.length > 0) { lines.push("", ...timing); } if (input.hangDetected) { lines.push(`Hang detection: triggered (no log or DerivedData growth for timeout)`); } lines.push(""); let extractUncertain = false; if (input.exitCode === 0) { lines.push("========== BUILD SUCCEEDED =========="); const swift = logText .split("\n") .filter((line) => /^\*\* TEST SUCCEEDED \*\*|Test run with [0-9]+ test|Suite .* passed after/.test(line), ) .slice(-5); const xctest = logText.split("\n").filter((line) => /Executed [1-9][0-9]* test/.test(line)).slice(-5); const testSummary = swift.length > 0 ? swift : xctest; if (testSummary.length > 0) { lines.push("", ...testSummary); } } else { lines.push("========== BUILD FAILED ==========", ""); const errors = extractCompilerLinkerErrors(logText); if (errors.length > 0) { lines.push(`Compiler/linker errors (${errors.length}):`, ...errors, ""); } const failures = extractTestFailures(logText, input.derivedData); extractUncertain = failures.uncertain; if (failures.lines.length > 0) { lines.push("Test failures:", ...failures.lines, ""); } if (errors.length === 0 && failures.lines.length === 0 && !input.hangDetected) { lines.push("(no specific errors extracted — check full log)", ""); } } lines.push(`Full log: ${input.logFile}`); lines.push(`Artifact: ${input.artifactPath}`); if (input.videoPath && existsSync(input.videoPath)) { lines.push(`Video: ${input.videoPath}`); } lines.push("======================================"); process.stdout.write(`${lines.join("\n")}\n`); return extractUncertain; } async function runXcodebuildAttempt(input: { session: CommandSession; config: PoolConfig; command: string; argv: string[]; logFile: string; derivedData: string; udid: string; extraSettings: string[]; }): Promise<{ code: number; hung: boolean; stop: StopResult }> { writeFileSync(input.logFile, ""); const logFd = openSync(input.logFile, "a"); const spawned = await input.session.spawn( input.command, [ ...input.argv, ...input.extraSettings, "-destination", `platform=iOS Simulator,id=${input.udid}`, "-derivedDataPath", input.derivedData, ], { stdio: ["ignore", logFd, logFd], }, ); closeSync(logFd); if (!spawned.ok) { return { code: 127, hung: false, stop: spawned.stop ?? { quiescent: true, note: spawned.reason } }; } const start = nowEpoch(); let lastProgress = start; let lastHeartbeat = start; let lastMtime = 0; let hung = false; const exitPromise = spawned.owned.exit; const pollMs = Math.max(50, Math.floor(input.config.progressPollSeconds * 1000)); while (true) { if (input.session.canceled) { hung = false; break; } const winner = await Promise.race([ exitPromise.then(() => "exit" as const), sleep(pollMs).then(() => "tick" as const), ]); if (winner === "exit") { break; } const now = nowEpoch(); const current = progressMtime(input.logFile, input.derivedData); if (current > lastMtime) { lastMtime = current; lastProgress = now; } if ( input.config.heartbeatInterval > 0 && now - lastHeartbeat >= input.config.heartbeatInterval ) { log( `[sim-pool] heartbeat: elapsed=${now - start}s idle=${now - lastProgress}s log=${fileSize(input.logFile)}B pid=${spawned.owned.pid}`, ); lastHeartbeat = now; } if (input.config.silenceTimeout > 0 && now - lastProgress >= input.config.silenceTimeout) { log( `[sim-pool] hang detected: no log or DerivedData growth for ${input.config.silenceTimeout}s (pid ${spawned.owned.pid})`, ); hung = true; break; } } const completed = await input.session.complete(spawned.owned, { stopFirst: hung || input.session.canceled, }); if (hung && !completed.stop.quiescent) { return { code: 1, hung: true, stop: completed.stop }; } return { code: waitExitStatus(completed.wait), hung, stop: completed.stop }; } function pruneOldLogs(logDir: string): void { if (!existsSync(logDir)) { return; } const cutoff = Date.now() - 3 * 24 * 60 * 60 * 1000; for (const name of readdirSync(logDir)) { if (!/^pool-.*\.(log|json|summary\.json)$/.test(name)) { continue; } const path = join(logDir, name); try { if (statSync(path).mtimeMs < cutoff) { rmSync(path, { force: true }); } } catch { // keep going } } } type VideoSession = { supervised: Supervised; path: string; logPath: string; ready: boolean; }; async function startVideoRecording( session: CommandSession, config: PoolConfig, slot: number, udid: string, stamp: string, ): Promise { if (config.videoPolicy === "off") { return null; } mkdirSync(config.videoDir, { recursive: true }); const baseName = config.videoName || `pool-${slot}-${stamp}`; const path = join(config.videoDir, `${baseName}.mp4`); const logPath = join(config.videoDir, `${baseName}.recordVideo.log`); writeFileSync(logPath, ""); log(`[sim-pool] Recording simulator video (${config.videoPolicy}): ${path}`); const errFd = openSync(logPath, "a"); const spawned = await session.spawn( "xcrun", ["simctl", "io", udid, "recordVideo", "--codec=h264", path], { stdio: ["ignore", "ignore", errFd] }, ); closeSync(errFd); if (!spawned.ok) { log(`[sim-pool] WARNING: simulator video recorder failed to start`); return null; } const deadline = Date.now() + config.videoReadyTimeout * 1000; let ready = false; while (Date.now() < deadline) { if (session.canceled) { break; } const text = existsSync(logPath) ? readFileSync(logPath, "utf8") : ""; if (text.includes("Recording started")) { ready = true; log("[sim-pool] Simulator video recording is ready"); break; } if (spawned.owned.child.exitCode != null) { log(`[sim-pool] WARNING: simulator video recorder exited before readiness; log: ${logPath}`); break; } await sleep(200); } if (!ready && spawned.owned.child.exitCode == null) { log( `[sim-pool] WARNING: simulator video recorder did not report readiness within ${config.videoReadyTimeout}s; continuing`, ); } return { supervised: spawned.owned, path, logPath, ready }; } function commandRunExitCode(input: { canceled: boolean; cancelExitCode: number; uncertain: boolean; exitCode: number; }): number { if (input.canceled) { return input.cancelExitCode; } if (input.uncertain && input.exitCode === 0) { return 1; } return input.exitCode; } export async function commandRun(config: PoolConfig, rawArgs: string[]): Promise { const session = new CommandSession(); session.installHandlers(); if (rawArgs[0] !== "--" || rawArgs.length < 2) { usage(); } let args = rawArgs.slice(1); applyRunCheckout(config); const normalized = normalizeCommandArgs(config, args); args = normalized.args; if (normalized.log) { log(normalized.log); } validateCommandGuardrails(config, args); const split = splitRunCommand(args); const extraSettings = applyPoolBuildSettings(config, split.argv); const runStart = nowEpoch(); let owned: OwnedSlot | undefined; const slotWaitEndHolder = { value: runStart }; let uncertain = false; let videoSession: VideoSession | null = null; let simUdId = ""; let finalArtifact = ""; let logFile = ""; let derivedData = ""; let exitCode = 1; let hangDetected = false; let attemptsUsed = 0; let prepStart = runStart; let prepEnd = runStart; let simPrep = 0; let xcodeStart = runStart; let xcodeEnd = runStart; let videoPath: string | undefined; let videoRecording: Record | undefined; const attemptArtifacts: string[] = []; try { owned = await acquireRunSlot(config, ["run", ...args], session); slotWaitEndHolder.value = nowEpoch(); const slotOwner = owned; session.onBeforeSpawn = () => { beginPublishing(slotOwner); }; session.onSpawned = (child) => { recordOwnedPgid(slotOwner, child.pgid); }; if (session.canceled) { uncertain = true; return session.cancelExitCode(); } log(`[sim-pool] Acquired slot ${owned.slot}`); simUdId = await ensureSim(session, config, owned.slot); derivedData = join(config.buildBase, `pool-${owned.slot}`); mkdirSync(derivedData, { recursive: true }); log(`[sim-pool] Simulator: ${poolDeviceName(owned.slot)} (${simUdId})`); log(`[sim-pool] DerivedData: ${derivedData}`); prepStart = nowEpoch(); await prepareSimulator(session, config, simUdId, "normal"); prepEnd = nowEpoch(); const logDir = join(config.buildBase, "logs"); mkdirSync(logDir, { recursive: true }); pruneOldLogs(logDir); const stamp = new Date().toISOString().replace(/[-:]/g, "").replace("T", "-").slice(0, 15); const baseLog = join(logDir, `pool-${owned.slot}-${stamp}.log`); const baseArtifact = baseLog.replace(/\.log$/, ".json"); finalArtifact = baseLog.replace(/\.log$/, ".summary.json"); log(`[sim-pool] Log: ${baseLog}`); videoSession = await startVideoRecording(session, config, owned.slot, simUdId, stamp); logFile = baseLog; xcodeStart = nowEpoch(); let attempt = 0; exitCode = 0; simPrep = prepEnd - prepStart; while (true) { if (session.canceled) { uncertain = true; exitCode = session.cancelExitCode(); break; } attemptsUsed = attempt + 1; let artifactFile = baseArtifact; const attemptStart = nowEpoch(); if (attempt > 0) { logFile = baseLog.replace(/\.log$/, `-retry${attempt}.log`); artifactFile = baseArtifact.replace(/\.json$/, `-retry${attempt}.json`); log(`[sim-pool] Retry ${attemptsUsed}/${config.hangRetries + 1} — log: ${logFile}`); const recoveryStart = nowEpoch(); await prepareSimulator(session, config, simUdId, "recovery"); simPrep += nowEpoch() - recoveryStart; } const attemptArgs = buildAttemptCommand(attempt, split.argv); if (attempt > 0) { const bundle = extractFlagValue("-resultBundlePath", attemptArgs); if (bundle) { log(`[sim-pool] Retry result bundle: ${bundle}`); } } const result = await runXcodebuildAttempt({ session, config, command: split.command, argv: attemptArgs, logFile, derivedData, udid: simUdId, extraSettings, }); exitCode = result.code; hangDetected = hangDetected || result.hung; if (!result.stop.quiescent) { uncertain = true; exitCode = exitCode || 1; break; } const attemptEnd = nowEpoch(); writeJsonArtifact({ artifactPath: artifactFile, scope: "attempt", startedAt: attemptStart, endedAt: attemptEnd, attemptNumber: attemptsUsed, attemptCount: attemptsUsed, hangDetected: result.hung, slot: owned.slot, udid: simUdId, derivedData, logFile, exitCode, }); attemptArtifacts.push(artifactFile); if (result.hung && attempt < config.hangRetries) { log("[sim-pool] Retrying after simulator hang recovery..."); attempt += 1; continue; } break; } xcodeEnd = nowEpoch(); if (videoSession) { const completed = await session.complete(videoSession.supervised, { stopFirst: true, firstSignal: "SIGINT", }); if (!completed.stop.quiescent) { uncertain = true; } const keep = config.videoPolicy === "always" || (config.videoPolicy === "on-failure" && (exitCode !== 0 || hangDetected)); const size = existsSync(videoSession.path) ? fileSize(videoSession.path) : 0; let deletedReason: string | undefined; if (keep && existsSync(videoSession.path)) { videoPath = videoSession.path; log(`[sim-pool] Video saved: ${videoSession.path}`); } else if (existsSync(videoSession.path)) { rmSync(videoSession.path, { force: true }); deletedReason = config.videoPolicy === "on-failure" && exitCode === 0 && !hangDetected ? "passed" : "missing"; if (deletedReason === "passed") { log("[sim-pool] Video discarded after passing run (policy: on-failure)"); } } videoRecording = { policy: config.videoPolicy, path: videoSession.path, retained: Boolean(videoPath), ready: videoSession.ready, log_path: videoSession.logPath, stop_status: completed.stop.quiescent ? 0 : 1, size_bytes: size, ...(deletedReason ? { deleted_reason: deletedReason } : {}), }; videoSession = null; } if (!config.keepBooted && simUdId) { log(`[sim-pool] Shutting down pool simulator ${simUdId}`); const shutdown = await runXcrun(session, ["simctl", "shutdown", simUdId]); if ( !shutdown.stop.quiescent || (shutdown.code !== 0 && !simctlAlreadyInRequestedState(["shutdown"], shutdown.stderr)) ) { uncertain = true; } } if (session.canceled) { uncertain = true; exitCode = session.cancelExitCode(); } } catch (error) { uncertain = true; if (session.canceled) { exitCode = session.cancelExitCode(); } else { throw error; } } finally { if (videoSession && !videoSession.supervised.retired) { const completed = await session.complete(videoSession.supervised, { stopFirst: true, firstSignal: "SIGINT", }); if (!completed.stop.quiescent) { uncertain = true; } } const stopped = await session.dispose(); if (!stopped.quiescent) { uncertain = true; } const slotWaitEnd = slotWaitEndHolder.value; const runEnd = nowEpoch(); if (owned && finalArtifact && !uncertain) { try { writeJsonArtifact({ artifactPath: finalArtifact, scope: "final", startedAt: xcodeStart, endedAt: xcodeEnd, attemptNumber: attemptsUsed, attemptCount: attemptsUsed, hangDetected, slot: owned.slot, udid: simUdId, derivedData, logFile, exitCode, extra: { attempt_artifacts: attemptArtifacts, ...(videoPath ? { video_path: videoPath } : {}), ...(videoRecording ? { video_recording: videoRecording } : {}), timing_breakdown_seconds: { slot_wait: slotWaitEnd - runStart, simulator_prepare: simPrep, xcodebuild: xcodeEnd - xcodeStart, total_wall: runEnd - runStart, }, timing_phase_epochs: { run_start: runStart, run_end: runEnd, slot_wait_start: runStart, slot_wait_end: slotWaitEnd, simulator_prepare_start: prepStart, simulator_prepare_end: prepEnd, xcodebuild_start: xcodeStart, xcodebuild_end: xcodeEnd, }, }, }); log(`[sim-pool] Artifact: ${finalArtifact}`); printSummary({ logFile, exitCode, startedAt: xcodeStart, endedAt: xcodeEnd, totalStartedAt: runStart, totalEndedAt: runEnd, attemptCount: attemptsUsed, hangDetected, slotWait: slotWaitEnd - runStart, simPrep, artifactPath: finalArtifact, videoPath, derivedData, }); } catch { uncertain = true; } } else if (owned && finalArtifact && uncertain) { try { writeJsonArtifact({ artifactPath: finalArtifact, scope: "final", startedAt: runStart, endedAt: runEnd, attemptNumber: attemptsUsed, attemptCount: attemptsUsed, hangDetected, slot: owned.slot, udid: simUdId, derivedData, logFile: logFile || finalArtifact, exitCode: session.canceled ? session.cancelExitCode() : exitCode || 1, extra: { incomplete: true }, }); } catch { // keep original error } } if (owned) { if (!stopped.quiescent) { releaseUncertain(owned, stopped.note ?? "run ended without proven quiescence"); } else { releaseReusable(owned); } } } return commandRunExitCode({ canceled: session.canceled, cancelExitCode: session.cancelExitCode(), uncertain, exitCode, }); } export type PruneKeep = { start: number; end: number } | null; export function parsePruneKeepSlots(spec: string): { start: number; end: number } { const match = /^([0-9]+)-([0-9]+)$/.exec(spec); if (!match) { die(`invalid --keep-slots '${spec}' (expected START-END)`); } const start = Number(match[1]); const end = Number(match[2]); if (end < start) { die(`invalid --keep-slots '${spec}' (END must be >= START)`); } return { start, end }; } export function pruneBuildKind(base: string): "pool" | "derived" | "mac-stale" | null { switch (base) { case "logs": case "videos": case "mac-tests": case "mac-debug": case "pre-push-mac": case "ci": case "privacy": case "oppi-dev": return null; default: break; } if (/^pool-[0-9]+$/.test(base)) { return "pool"; } if (base.startsWith("derived-data-")) { return "derived"; } if (base.startsWith("mac-")) { return "mac-stale"; } return null; } function canonical(path: string): string { return realpathSync.native(path); } export function commandPruneCache(config: PoolConfig, args: string[]): number { let apply = false; let keep: PruneKeep = null; for (let i = 0; i < args.length; i += 1) { const arg = args[i]; if (arg === "--apply") { apply = true; continue; } if (arg === "--keep-slots") { const spec = args[i + 1]; if (!spec) { die("usage: sim-pool.sh prune-cache [--apply] [--keep-slots START-END]"); } keep = parsePruneKeepSlots(spec); i += 1; continue; } if (arg.startsWith("--keep-slots=")) { keep = parsePruneKeepSlots(arg.slice("--keep-slots=".length)); continue; } die("usage: sim-pool.sh prune-cache [--apply] [--keep-slots START-END]"); } const gitRoot = gitTopLevel(process.cwd()); if (!gitRoot) { die("prune-cache requires a Git checkout"); } if (!existsSync(join(gitRoot, "clients", "apple"))) { die("prune-cache: checkout is missing clients/apple"); } if (config.oppiRootFromEnv) { try { if (canonical(config.oppiRoot) !== canonical(gitRoot)) { die(`prune-cache: OPPI_ROOT '${config.oppiRoot}' does not match Git checkout '${gitRoot}'`); } } catch { die("prune-cache: OPPI_ROOT is not resolvable"); } } const appleDir = join(gitRoot, "clients", "apple"); const buildBase = join(appleDir, ".build"); for (const path of [gitRoot, appleDir, buildBase]) { try { if (statSync(path).isSymbolicLink?.() || (existsSync(path) && statSync(path).isSymbolicLink())) { die("prune-cache: refusing symlinked cleanup root"); } } catch { // missing handled below } } if (existsSync(gitRoot) && (lstatIsSymlink(gitRoot) || lstatIsSymlink(appleDir) || (existsSync(buildBase) && lstatIsSymlink(buildBase)))) { die("prune-cache: refusing symlinked cleanup root"); } if (!existsSync(buildBase)) { log(`[sim-pool] prune-cache: no ${buildBase}`); return 0; } if (!statSync(buildBase).isDirectory()) { die(`prune-cache: cleanup root is not a directory: ${buildBase}`); } if (!apply) { log( "[sim-pool] prune-cache: dry-run (pass --apply to delete idle pool dirs with a slot lease; derived-data-* and mac-* need an exclusive lease and are skipped on apply)", ); } let status = 0; for (const name of readdirSync(buildBase)) { const path = join(buildBase, name); const kind = pruneBuildKind(name); if (!kind) { continue; } if (lstatIsSymlink(path)) { log(`[sim-pool] prune-cache: skipping symlinked ${path}`); continue; } if (!statSync(path).isDirectory()) { log(`[sim-pool] prune-cache: skipping non-directory ${path}`); continue; } if (kind === "pool") { const slot = Number(name.slice("pool-".length)); if (keep && slot >= keep.start && slot <= keep.end) { log(`[sim-pool] prune-cache: keeping ${path} (--keep-slots ${keep.start}-${keep.end})`); continue; } if (!apply) { log(`[sim-pool] prune-cache: would delete ${path}`); continue; } const acquired = tryAcquireSlot({ lockDir: config.lockDir, slot, argv: ["prune-cache"] }); if (!acquired.ok) { log(`[sim-pool] prune-cache: skipping busy or quarantined slot ${slot} (${acquired.reason})`); continue; } log(`[sim-pool] prune-cache: deleting ${path}`); try { rmSync(path, { recursive: true, force: false }); if (existsSync(path)) { throw new Error("path remains"); } releaseReusable(acquired.owned); } catch { log(`[sim-pool] prune-cache: failed to delete ${path}`); // In-process rm has no child group to record. A proven-empty group must // not leave the slot permanently uncertain; CLI still fails. releaseReusable(acquired.owned); status = 1; } continue; } if (!apply) { log(`[sim-pool] prune-cache: would delete if exclusive ${path}`); continue; } log(`[sim-pool] prune-cache: skipping ${path} (no exclusive lease)`); } return status; } function lstatIsSymlink(path: string): boolean { try { return lstatSync(path).isSymbolicLink(); } catch { return false; } } export async function commandShutdownIdle(config: PoolConfig): Promise { const session = new CommandSession(); session.installHandlers(); let status = 0; try { if (session.canceled) { return session.cancelExitCode(); } const list = await runXcrun(session, ["simctl", "list", "devices", "-j"]); if (session.canceled) { return session.cancelExitCode(); } if (!list.stop.quiescent) { log(`[sim-pool] shutdown-idle: list did not prove quiescence${list.stop.note ? ` (${list.stop.note})` : ""}`); return 1; } if (list.code !== 0) { log("[sim-pool] shutdown-idle: failed to list simulators"); return 1; } let devices: SimulatorDevice[]; try { devices = parseDevicesJson(list.stdout); } catch { log("[sim-pool] shutdown-idle: failed to parse simulator list"); return 1; } const candidates = devices.filter((device) => { const match = /^Oppi-Pool-(\d+)$/.exec(device.name); return Boolean(match) && device.state === "Booted"; }); for (const device of candidates) { if (session.canceled) { status = session.cancelExitCode(); break; } const slot = Number(/^Oppi-Pool-(\d+)$/.exec(device.name)?.[1]); const acquired = tryAcquireSlot({ lockDir: config.lockDir, slot, argv: ["shutdown-idle"] }); if (!acquired.ok) { if ( acquired.reason.includes("legacy") || acquired.reason.includes("busy") || acquired.reason.includes("in-flight") || acquired.reason.includes("uncertain") ) { log(`[sim-pool] shutdown-idle: skipping existing slot lock ${slot}`); } else { log(`[sim-pool] shutdown-idle: failed to acquire slot ${slot}`); status = 1; } continue; } const slotOwner = acquired.owned; session.onBeforeSpawn = () => { beginPublishing(slotOwner); }; session.onSpawned = (child) => { recordOwnedPgid(slotOwner, child.pgid); }; let released = false; const finishSlot = (kind: "reusable" | "uncertain", note?: string): void => { if (released) { return; } released = true; session.onBeforeSpawn = undefined; session.onSpawned = undefined; if (kind === "uncertain") { releaseUncertain(slotOwner, note ?? "shutdown-idle did not prove quiescence"); } else { releaseReusable(slotOwner); } }; try { if (session.canceled) { finishSlot("reusable"); status = session.cancelExitCode(); break; } const recheck = await runXcrun(session, ["simctl", "list", "devices", "-j"]); if (!recheck.stop.quiescent) { log(`[sim-pool] shutdown-idle: recheck did not prove quiescence for ${device.udid}`); finishSlot("uncertain", recheck.stop.note ?? "recheck did not prove quiescence"); status = 1; if (session.canceled) { status = session.cancelExitCode(); break; } continue; } if (session.canceled) { finishSlot("reusable"); status = session.cancelExitCode(); break; } if (recheck.code !== 0) { log(`[sim-pool] shutdown-idle: failed to recheck ${device.udid}`); finishSlot("reusable"); status = 1; continue; } let state = ""; try { state = deviceState(parseDevicesJson(recheck.stdout), device.udid) ?? ""; } catch { state = ""; } if (!state) { log(`[sim-pool] shutdown-idle: failed to recheck ${device.udid}`); finishSlot("reusable"); status = 1; continue; } if (state !== "Booted") { log(`[sim-pool] shutdown-idle: ${device.udid} state is ${state} (not Booted)`); finishSlot("reusable"); continue; } if (session.canceled) { finishSlot("reusable"); status = session.cancelExitCode(); break; } log(`[sim-pool] Shutting down idle simulator ${device.udid}`); const shutdown = await runXcrun(session, ["simctl", "shutdown", device.udid]); if (!shutdown.stop.quiescent) { log(`[sim-pool] shutdown-idle: shutdown did not prove quiescence for ${device.udid}`); finishSlot("uncertain", shutdown.stop.note ?? "shutdown did not prove quiescence"); status = 1; if (session.canceled) { status = session.cancelExitCode(); break; } continue; } if (shutdown.code !== 0) { log(`[sim-pool] shutdown-idle: failed to shut down ${device.udid}`); finishSlot("reusable"); status = 1; continue; } if (session.canceled) { finishSlot("reusable"); status = session.cancelExitCode(); break; } finishSlot("reusable"); } catch (error) { if (slotOwner.pgids.length > 0) { finishSlot("uncertain", error instanceof Error ? error.message : String(error)); } else { finishSlot("reusable"); } throw error; } finally { if (!released) { if (slotOwner.pgids.length > 0) { finishSlot("uncertain", "shutdown-idle released without completion"); } else { finishSlot("reusable"); } } } } if (session.canceled) { status = session.cancelExitCode(); } } finally { const stopped = await session.dispose(); if (!stopped.quiescent && status === 0 && !session.canceled) { status = 1; } } return status; } export function commandStatus(config: PoolConfig): number { process.stdout.write(`Pool count: ${config.count}\n`); process.stdout.write(`Pool slot start: ${config.slotStart}\n`); process.stdout.write(`Pool slot range: ${poolSlotRange(config)}\n`); process.stdout.write(`Lock dir: ${config.lockDir}\n`); process.stdout.write(`Build base: ${config.buildBase}\n\nLocks:\n`); mkdirSync(config.lockDir, { recursive: true }); let found = false; const names = existsSync(config.lockDir) ? readdirSync(config.lockDir) : []; for (const name of names) { const match = /^slot-([0-9]+)(\.lock)?$/.exec(name); if (!match) { continue; } if (name.endsWith(".state.json")) { continue; } } const slots = new Set(); for (const name of names) { const lockMatch = /^slot-([0-9]+)\.lock$/.exec(name); const dirMatch = /^slot-([0-9]+)$/.exec(name); if (lockMatch) { slots.add(Number(lockMatch[1])); } if (dirMatch) { slots.add(Number(dirMatch[1])); } } if (slots.size === 0) { process.stdout.write(" none\n"); } else { found = true; for (const slot of [...slots].sort((a, b) => a - b)) { const info = inspectSlot(config.lockDir, slot); const label = info.legacy ? "legacy" : info.flockHeld ? "live" : info.state && info.state !== "unreadable" ? info.state.status : "idle"; process.stdout.write(` slot-${slot} ${label}\n`); } } void found; process.stdout.write("\nPool and booted simulators:\n"); const listed = spawnSync("xcrun", ["simctl", "list", "devices", "-j"], { encoding: "utf8" }); if (listed.status === 0) { const devices = parseDevicesJson(listed.stdout); for (const device of devices) { const pool = /^Oppi-Pool-(\d+)$/.exec(device.name); if (!pool && device.state !== "Booted") { continue; } const slot = pool ? Number(pool[1]) : null; const info = slot != null ? inspectSlot(config.lockDir, slot) : null; const marker = info?.flockHeld || info?.legacy || (info?.state && info.state !== "unreadable" && info.state.status !== "reusable") ? "locked" : "idle"; process.stdout.write( ` ${device.name.padEnd(18)} ${device.state.padEnd(8)} ${marker.padEnd(6)} ${device.udid}\n`, ); } } return 0; } export function usage(): never { process.stderr.write(`Usage: sim-pool.sh run [--root ] -- sim-pool.sh self-test sim-pool.sh status sim-pool.sh shutdown-idle sim-pool.sh prune-cache [--apply] [--keep-slots START-END] --root PATH is the Oppi checkout or worktree to test (also clients/apple). It overrides OPPI_ROOT. run always executes xcodebuild in that checkout's clients/apple, even if the script was launched from another tree. run acquires a simulator pool slot, injects -destination and -derivedDataPath, runs xcodebuild, and releases the slot on exit. An already-booted pool simulator is reused unless OPPI_SIM_POOL_FORCE_CLEAN_BOOT=1. Pool simulators stay booted after a run unless OPPI_SIM_POOL_KEEP_BOOTED=0. Unused simulator daemons are disabled unless OPPI_SIM_SLIM=0. Compiler index store is disabled unless OPPI_SIM_POOL_INDEX_STORE=1 or the command already sets COMPILER_INDEX_STORE_ENABLE. Ordinary run does not delete unavailable simulators or CoreSimulator device caches. shutdown-idle acquires each slot with flock and records child process groups. Existing live, legacy, and flock-v1 in-flight slots are skipped. gated-v1 in-flight/uncertain reclaim when recorded groups are idle or the ledger is proven empty. Booted is rechecked as device state, not idleness. Killing xcrun does not mean CoreSimulator finished. prune-cache dry-runs this checkout's numeric pool-* dirs, derived-data-*, and one-off mac-* experiment dirs. --apply deletes pool-* only after acquiring the slot lease. derived-data-* and mac-* are skipped on apply (no exclusive lease). `); process.exit(1); }