/** * `runGoverned`: admit a heavy command through the machine-wide governor, * run it, and supervise it under pressure. The engine behind `cleo run`, * reusable by the supervisor (T12982). * * Lifecycle: * * 1. **Admission.** First, orphans of dead runners are reaped. Test, build * and full-build jobs are admitted by the admission ledger (T13133): one * byte budget and one FIFO queue shared with every evidence run on the * machine. Nested in an admitted run's process tree (a `cleo run` under a * `cleo run` or a `cleo verify`), a job rides that admission. Admission is * non-blocking by default: a refused job returns a `deferred` result, with * nothing started. With `wait`, it waits in the queue until admitted or * `timeoutMs`; while the memory gate refuses heavy work (T13127) it warns * "waiting: memory pressure" with the readings (at most once a minute), * after a minute it names the holders and any suspected wait cycle, and it * warns again when pressure falls. Other classes (db-heavy) keep their * governor slot. * 2. **Run.** The command is spawned as its own process group (so a pause or * a cancel reaches its workers too), niced, with the caller's env. The job * is recorded in the registry with its start times and a heartbeat. Its * stdout goes to stderr, or with `passthrough` to the caller's stdout. A * `foreground` child (a terminal's job) and a nested one stay in their * group instead, are signalled by pid and are never paused. * 3. **Supervise.** One serialized poll loop (never overlapping) samples * pressure and applies {@link decidePause}: SIGSTOP/SIGCONT to the group. * Every await is followed by an exit check, so nothing is signalled or * written after the child exits. * 4. **Exit.** If the job was ever paused, the group gets a SIGCONT (workers * of a killed leader must not stay stopped); the record is removed and the * slot released. SIGINT, SIGTERM and SIGHUP to the runner are forwarded to * the group (after a SIGCONT). A runner killed outright is recovered by the * next {@link reapOrphans}, which never acts on an unreadable `ps`. * * @module resources/run-governed * @task T12979 * @task T12980 * @task T12981 * @task T13127 * @task T13133 * @epic T12978 */ import type { EventEmitter } from 'node:events'; import type { AdmissionResult, ResourceClass } from '@cleocode/contracts'; import { type AdmissionOutcome, type AdmissionRequest, type AdmissionScope, type AdmitOptions, type LedgerEntry } from './admission-ledger.js'; import type { ResourceSample } from './backend.js'; import { type GovernorIoError } from './governor.js'; import { type ProcessTreeUsage, type RunAlternative, type RunDeferralDetails } from './run-admission.js'; /** Niceness applied to the command (children inherit it). */ export declare const RUN_NICENESS = 10; /** The minimal child-process surface the loop needs (tests fake it). */ export interface GovernedChild extends EventEmitter { readonly pid?: number | undefined; } /** * How out of the ordinary a notice is. `warn`: the run is ungoverned, was * paused or resumed under pressure, was admitted after waiting in the queue, * or the runner was signalled (`--passthrough` still prints these). `info`: * progress such as a nested run (left out by `--passthrough`). */ export type RunNoticeLevel = 'info' | 'warn'; /** Injectable effects. Defaults are the real process, governor and clock. */ export interface RunGovernedDeps { readonly sample: () => Promise; /** Admission for a ledger class (test, build, full-build): the admission ledger. */ readonly admit: (req: AdmissionRequest, opts: AdmitOptions) => Promise; /** Admission for any other class (db-heavy): a governor slot. */ readonly tryAcquire: (cls: ResourceClass, sample: ResourceSample) => Promise; /** * Start the child. `passthrough`: it gets the runner's stdin, stdout and * stderr; otherwise its stdout goes to stderr. `detached`: its own session * and process group. */ readonly spawn: (file: string, args: readonly string[], opts: { cwd: string; env: NodeJS.ProcessEnv; detached: boolean; passthrough: boolean; }) => GovernedChild; readonly signal: (pid: number, signal: NodeJS.Signals) => boolean; /** Signal a single process (a nested job's child shares its parent's group). */ readonly signalPid: (pid: number, signal: NodeJS.Signals) => boolean; readonly start: (pid: number) => string | null; readonly renice: (pid: number) => void; readonly now: () => number; /** * Wait `ms`. The timer keeps the process alive (a `--wait` queue has * nothing else to); `signal` cancels it early so a finished run exits at once. */ readonly sleep: (ms: number, signal?: AbortSignal) => Promise; /** Install runner signal forwarding; returns an uninstall function. */ readonly onRunnerSignal: (handler: (signal: NodeJS.Signals) => void) => () => void; readonly jobsDir: string; /** The runner's parent pid, recorded with a signal the runner received (T13680). */ readonly ppid: number; /** * The runner's parent pid now (T14080). When it no longer equals * {@link RunGovernedDeps.ppid}, the parent died (a wrapper shell SIGKILLed * by a harness) and the runner stops its child. @defaultValue process.ppid */ readonly currentPpid: () => number; /** The admission ledger's entries (for the `running[]` of a deferral). */ readonly ledger: () => readonly LedgerEntry[]; readonly pid: number; /** Process group of a pid (`ps -o pgid=`), or null when unknown. */ readonly groupOf: (pid: number) => number | null; /** Ancestors of a pid, nearest first (one `ps` of the process table), or null when unknown. */ readonly ancestorsOf: (pid: number) => readonly number[] | null; /** CPU time and resident memory of a pid's process tree (one `ps`), or null when unknown (T13563). */ readonly treeUsage: (pid: number, withCommands?: boolean) => ProcessTreeUsage | null; /** Where hook measurements are kept (T13921). @defaultValue the admission directory */ readonly hookFootprintsDir: string; /** * Whether a git hook is installed for the run's checkout (T14093). * @defaultValue probes `opts.cwd` with `gitHookInstalled` */ readonly hookInstalled?: (hook: string, globals: readonly string[]) => boolean; } /** How often the watchdog measures the child's process tree (T13563). */ export declare const TREE_SAMPLE_MS = 30000; /** How often a run that measures its hook's peak samples its tree (T13921). */ export declare const HOOK_SAMPLE_MS = 5000; /** * How recently an enclosing runner must have forwarded a signal to its child's * group for a nested runner to treat the same signal as that delivery (T13002). */ export declare const GROUP_SIGNAL_WINDOW_MS = 2000; /** How long a run whose parent died gets after SIGTERM before SIGKILL (T14080). */ export declare const PARENT_GONE_KILL_MS = 10000; /** Options for {@link runGoverned}. */ export interface RunGovernedOptions { readonly argv: readonly string[]; readonly cls: ResourceClass; readonly cwd: string; /** Child environment (caller applies heavyToolEnv). */ readonly env: NodeJS.ProcessEnv; readonly sessionId: string | null; readonly wait?: boolean; /** With `wait`: give up after this long. @defaultValue 30 min */ readonly timeoutMs?: number; /** Supervision cadence. @defaultValue 5000 */ readonly pollMs?: number; /** Queue cadence under `wait`. @defaultValue 1000 */ readonly queuePollMs?: number; /** * The child gets the runner's stdin, stdout and stderr (`cleo run * --passthrough`), so its stdout reaches the caller byte for byte. * @defaultValue false (stdout goes to stderr; the caller's stdout is reserved) */ readonly passthrough?: boolean; /** * Keep the child in the runner's process group instead of a detached one: * a terminal's foreground job, so reading or configuring the terminal never * stops it (SIGTTIN/SIGTTOU) and Ctrl-C reaches it. Such a child is never * paused. SIGTERM and SIGHUP to the runner are forwarded to it by pid; * SIGINT is not, since the terminal already sent it to the whole group. * @defaultValue false */ readonly foreground?: boolean; /** One-line notices (stderr in the CLI); see {@link RunNoticeLevel}. */ readonly notice?: (line: string, level: RunNoticeLevel) => void; /** * Bytes to ask the ledger for: what the planned env lets the child start * (`planFootprintBytes`, T13132). @defaultValue the class's default footprint */ readonly footprintBytes?: number; /** Why the run is charged `footprintBytes`, for the ledger and status (T13367). */ readonly footprintReason?: string; /** How much of the project the run covers, for status (T13132). */ readonly scope?: AdmissionScope; /** Queue ahead of routine waiters, still within the budget (T13921). @defaultValue false */ readonly urgent?: boolean; /** * Never SIGSTOP this run (`cleo run --no-pause`, T14075): a test with * wall-clock deadlines (an external database, a child timeout) fails if it * is frozen mid-run. It is admitted and charged like any run of its class, * and since pressure cannot pause it later, at `backoff` the ledger starts * it only by the idle rule (T14098): one at a time, in queue order, while no * CLEO run is admitted and its charge fits the reclaimable memory. Otherwise * it waits (with `wait`) or is deferred. * @defaultValue false */ readonly noPause?: boolean; /** * The run is meant to outlive the process that started it (`nohup`, a * launcher script that exits): it is exempt from the parent-death stop * (T14080) and keeps running when its parent exits. @defaultValue false */ readonly detach?: boolean; /** * The git hook or heavy `node` script the command carries * (`planRunFootprint`'s `hookKey`): its tree is sampled every * {@link HOOK_SAMPLE_MS} and the peak is recorded under this key, so the * next run of that hook (T13921) or script (T14050) is charged by it. */ readonly hookKey?: string; readonly deps?: Partial; } /** A signal the runner itself received, and forwarded to its child (T13680). */ export interface RunnerSignal { readonly signal: NodeJS.Signals; /** When it arrived (epoch ms). */ readonly atMs: number; /** The runner's parent pid then: the likeliest sender, or its process. */ readonly ppid: number; } /** Result of {@link runGoverned}. */ export type RunGovernedResult = { /** A signal reached the runner while it waited in the queue; nothing started (T13681). */ readonly kind: 'interrupted'; readonly signal: NodeJS.Signals; readonly waitedMs: number; readonly class: ResourceClass; readonly command: string; } | { readonly kind: 'deferred'; readonly reason: string; readonly details: RunDeferralDetails; readonly alternatives: RunAlternative[]; readonly fix: string; } | { readonly kind: 'exited'; readonly class: ResourceClass; readonly command: string; readonly exitCode: number | null; readonly signal: NodeJS.Signals | null; /** Set when the command could not be started at all. */ readonly spawnError: string | null; readonly durationMs: number; readonly waitedMs: number; readonly pausedMs: number; readonly pauses: number; readonly slot: number; /** * Set when the governor's state could not be written (a sandboxed or read-only CLEO home, a * full disk): the command ran ungoverned rather than not at all. */ readonly ungoverned: GovernorIoError | null; /** * The first SIGINT/SIGTERM/SIGHUP the runner itself received while the * child ran, which it forwarded; null when the runner was not signalled, * so a `signal` here with no `runnerSignal` was aimed at the child (T13680). */ readonly runnerSignal: RunnerSignal | null; /** True exactly when the run waited in the queue and said "admitted after" (T13681). */ readonly queued: boolean; /** * Why the run was never paused under pressure, when it may not be * (T14093): `--no-pause`, or it runs tests on wall-clock deadlines. */ readonly notPausable?: string; }; export { governorIoError } from './governor.js'; /** * The real child spawner {@link runGoverned} uses. Unless passed through, * stdout is reserved for the caller's envelope: output goes to stderr. * * Exported as the explicit opt-in for a test that means to start a process: * inside a test runner `runGoverned` refuses to spawn unless `deps.spawn` is * injected (T13236), and passing this one says "really spawn". * * @param file - Executable. * @param args - Arguments. * @param opts - Working directory, environment, process-group and stdio mode. * @returns The started child. */ export declare const spawnGovernedChild: RunGovernedDeps['spawn']; /** * Admit, run and supervise one heavy command. Never throws for admission or * child failures; those are results. */ export declare function runGoverned(opts: RunGovernedOptions): Promise; //# sourceMappingURL=run-governed.d.ts.map