import { execFile } from "node:child_process"; /** * The glob that matches every scope this agent creates, and NOTHING else. * * Exported because ctl/commands/uninstall.ts has to find the leftover scopes * before it deletes the jobs roots, and a second copy of this string living * there is exactly the kind of duplicated constant that drifts: rename the * prefix here and the uninstall would silently stop finding anything, leaving * live root jobs whose output.log was just deleted underneath them. */ export declare const JOB_SCOPE_UNIT_GLOB = "aic-job-*.scope"; /** * The unit name for a job, e.g. `aic-job-3f2a…c1.scope`. * * The id is re-validated here rather than trusted, even though every caller * already generated or validated it: a unit name is a string systemd parses, * and it is the one place where a job id would leave our own file-path handling * and become part of a command line. JOB_ID_PATTERN is 16 hex characters, so a * value that passes it cannot carry a separator, a glob character or a * template's `@`. Throwing is right — a caller with a malformed id has a bug, * and the alternative (a sanitised fallback name) would let two jobs share one * unit. */ export declare function jobScopeUnitName(jobId: string): string; /** * Whether a unit name is one of OURS, exactly — the same shape jobScopeUnitName * builds, job id and all. * * The uninstall path stops units by name, and the names it stops come out of * `systemctl list-units`' text output. Re-checking each one here is what keeps a * mis-parsed line (a column shift, a localised header a `--no-legend` did not * suppress, a bullet glyph) from turning into a `systemctl stop` aimed at some * other unit on the machine. */ export declare function isJobScopeUnitName(unit: string): boolean; /** * The absolute paths systemd-run is installed at, in the order they are probed. * * Deliberately NOT a PATH lookup. This process runs as root, and resolving a * bare program name through PATH means the binary we execute is chosen by an * environment variable — a hijack surface a root service has no business * offering for a convenience feature. Both entries are root-owned system * directories on every distribution that ships systemd (usr-merged systems make * the second a symlink to the first). */ export declare const SYSTEMD_RUN_PATHS: readonly ["/usr/bin/systemd-run", "/bin/systemd-run"]; /** * The first systemd-run that exists, or null. `exists` is injectable so the * decision can be unit-tested on a box that has neither path. */ export declare function resolveSystemdRun(exists?: (p: string) => boolean): string | null; /** What the decision function is told about the machine. All four are required. */ export interface JobScopeEnv { platform: NodeJS.Platform; /** `process.getuid?.()` — undefined on Windows, where there is no such call. */ uid: number | undefined; /** The result of resolveSystemdRun(), or null. */ systemdRun: string | null; /** Whether /run/systemd/system exists, i.e. whether systemd is the init here. */ hasSystemd: boolean; } export type JobScopeDecision = { supported: true; systemdRun: string; } | { supported: false; reason: string; }; /** * Whether this machine can launch jobs into scopes — pure, so the whole matrix * is testable, and so the reason an operator sees is the same string the logic * actually decided on. * * All four conditions are required, and the uid one is the one most likely to be * "fixed" by someone who has not thought it through: * * SYSTEM MODE ONLY, ON PURPOSE. A non-root agent cannot create a system scope at * all — measured on Ubuntu 24.04 / systemd 255, the attempt is refused with * `Failed to start transient scope unit: Interactive authentication required.` * — and `systemd-run --user --scope` is NOT the answer to that. It would be a * REGRESSION: a user-mode scope is created under `user@.service`, inside * the login session's cgroup, which systemd tears down when the session ends * (and, with `KillUserProcesses=yes` or a plain logout, sooner than that), while * today's plain detached child at least survives as an orphan of PID 1. It also * simply does not work where the agent actually runs: from a non-session context * it fails with `Failed to connect to bus: No medium found`. So a non-root agent * keeps the fallback rather than being moved into a cgroup with a NEW way to die. */ export declare function jobScopeDecision(env: JobScopeEnv): JobScopeDecision; /** What a job's spawn actually runs, once the scope wrapper is in front of it. */ export interface JobScopeArgv { file: string; args: string[]; } /** * The systemd-run command line, built rather than written out, so the ONE place * that decides the flag order is also the place the tests read. * * Every flag earns its place: * - `--scope`, for the exec-in-place semantics the whole design rests on; * - `--quiet`, because systemd-run otherwise prints "Running scope as unit …" * on stderr — which is the job's output.log — and that line would become the * first thing every job on Linux ever printed; * - `--collect`, so a scope that FAILED is reaped instead of lingering as a * failed unit that a later `--unit=` of the same name would refuse to reuse. * Be precise about what it does not cover: `--collect` reaps inactive and * failed units, never a LIVE one, so it is no defence against a name that is * still loaded — that case is `Failed to start transient scope unit: Unit * aic-job-….scope was already loaded or has a fragment file.` on stderr, * rc=1 (measured). It stays a non-issue for a different reason: a scope * disappears the instant its process tree ends (`LoadState=not-found`), and * job ids are 64 random bits; * - `--unit=`, so `systemctl list-units 'aic-job-*.scope'` finds our scopes and * only ours — that is what makes the uninstall able to stop them; * - `--description=`, which names the JOB ID and nothing else. The job's `name` * is caller payload and would end up in `systemctl status` output and the * journal; the id is validated, unique, and enough to find the record with. * - `--`, so that neither our own description nor anything the caller supplied * can be read back as an option by systemd-run's own parser. * * No quoting happens here, and none is needed: these are argv elements handed to * `execvp`, so `file`/`args` reach systemd-run byte for byte and it passes them * on the same way. */ export declare function buildJobScopeArgv(opts: { systemdRun: string; unit: string; description: string; file: string; args: readonly string[]; }): JobScopeArgv; /** The `--description=` a job's scope carries. No caller payload — see above. */ export declare function jobScopeDescription(jobId: string): string; /** What spawnJob needs to wrap a command, or null when this machine cannot. */ export interface JobScopeLauncher { systemdRun: string; } /** * The probe that turns "the binary is there" into "a scope actually gets * created here", bounded and run OFF the frame handler's thread. * * The difference is not academic: systemd-run needs to reach the system bus and * be allowed by polkit to create a transient unit, and `--collect` needs systemd * ≥ 236. A container with /run/systemd/system bind-mounted in, a locked-down * polkit, or an ancient distribution all pass every static check above and then * fail at spawn time — where the failure would land in a real job's output.log * instead of being noticed once at startup. * * `/bin/sh -c 'exit 0'` is the cheapest thing that exercises the whole path * (bus, polkit, unit creation, exec, teardown), and it returns 0 on the shape we * ship to (measured, Ubuntu 24.04.4 / systemd 255, where it costs milliseconds). * It is deliberately given no `--unit=`, so it cannot collide with anything and * systemd names it itself. * * ASYNC, AND THAT IS LOAD-BEARING. This used to be an `execFileSync` bounded at * five seconds, reached lazily from spawnJob — i.e. from inside what was then a * fully synchronous JobManager.start(), i.e. on the WS frame handler's own * thread. (start() is asynchronous today, for a related reason of its own — the * Windows job scripts — but it still never awaits anything from HERE.) On exactly the * machines the probe exists to detect (a wedged system bus, a polkit that never * answers) the first job start then stalled the agent's entire loop for up to * those five seconds: `do:ping` unanswered until the relay declares the machine * dead, every concurrent `do:exec` stuck behind it. It is now the same shape * login-shell-path.ts uses for the macOS PATH probe, wired the same way and for * the same reason: started on the agent's startup path (runConnectionLoop and * JobManager.recover()), AWAITED by connection.ts's do:job_start handler, and * read back SYNCHRONOUSLY by spawnJob out of the settled memo — which is also * what makes the documented "the agent logs which it got, once, at startup" * true, instead of a line that appeared on the first job start and never at all * on a machine that started none. */ export declare const JOB_SCOPE_PROBE_TIMEOUT_MS = 5000; /** * The floor between two probe attempts once one has FAILED. * * A probe failure is not permanent, and memoizing it as though it were was a * real bug. The STATIC half of the decision — platform, uid, * /run/systemd/system, the systemd-run binary — genuinely cannot change under a * running process, and is still memoized forever below. What the probe measures * cannot make that claim: bus reachability, polkit's answer and the machine's * load all change while we run. The agent's unit is `Restart=always` and these * boxes stay up for weeks, and the single likeliest moment for the probe to fail * is the one right after boot, when D-Bus may not be up yet and a * `Restart=always` agent is already running. Caching that answer silently * downgraded every job for the machine's whole uptime, with the one journal line * that explained it scrolled away days earlier. * * Five minutes, so that a genuinely broken box pays at most one bounded probe * per five minutes however many jobs are started against it (a failing probe * costs up to JOB_SCOPE_PROBE_TIMEOUT_MS, and do:job_start waits for it), while * a bus that comes up seconds after the agent is picked up long before the * operator's first long run. */ export declare const JOB_SCOPE_REPROBE_INTERVAL_MS: number; /** * Seams, and ONLY seams: production passes none of these. The scope path needs * root, systemd and a reachable system bus, none of which a test box has, so * without them a whole platform's behaviour would be code nothing ever executes. */ export interface JobScopeProbeDeps { /** What the machine looks like. Default: this process, read once. */ env?: () => JobScopeEnv; /** Runs the bounded probe. Default: probeScopeWorks below. */ runProbe?: (systemdRun: string) => Promise; /** The clock the re-probe floor is measured on. Default: Date.now. */ now?: () => number; } /** * One bounded attempt. Never throws and never rejects — every failure is `false`. * * `exec` is a seam, and ONLY a seam — production passes nothing, exactly as with * `resolveSystemdRun(exists)`. It is what makes the two things that decide * whether this measures the real path — the argv and the timeout — assertable * against the function the agent actually runs, on a box that is not Linux, not * root, or has no systemd, i.e. on every developer machine and every CI runner * we have. Without it the whole function is code nothing ever executes. */ export declare function probeScopeWorks(systemdRun: string, exec?: typeof execFile): Promise; /** * Start (or reuse) the capability probe for this machine. Called at process * start from the agent's two startup paths, and again from do:job_start, which * awaits it. * * SUCCESS is memoized for the life of the process — a machine that just created * a scope will create the next one, and the probe must not be paid per job. * FAILURE is not: it is retried, no more often than JOB_SCOPE_REPROBE_INTERVAL_MS. */ export declare function startJobScopeProbe(deps?: JobScopeProbeDeps): Promise; /** * The promise a caller must await before starting a job, or null when there is * nothing to wait for (this machine can never have scopes, or the answer is * already settled). Exactly pendingLoginShellPath()'s contract, and awaited in * exactly the same place for the same reason — see connection.ts's do:job_start. */ export declare function pendingJobScope(): Promise | null; /** * The launcher for this machine, read SYNCHRONOUSLY — this is what spawnJob * calls. JobManager.start() is asynchronous (it awaits the Windows job scripts; * see job-scripts.ts), but this call is not one of the things it awaits: it is a * plain read of an already-settled memo, and it must stay that way, because * every await on the start path is a window some other frame can arrive in. * * Null means "launch this job unscoped", and it covers two cases deliberately * folded together: the machine cannot have scopes, and the probe has not settled * yet. Falling back rather than blocking is the whole point of the async rework * above — a job started in the probe's first few milliseconds runs exactly as it * did before this feature existed, which is a far smaller cost than stalling the * frame handler. In practice the window is empty: the probe starts at process * startup and do:job_start awaits it. */ export declare function jobScopeLauncher(): JobScopeLauncher | null; /** * Test helper: pin the memoized systemctl path, so the sweep can be pointed at a * stand-in. Production never calls this — it would aim a root kill by hand. */ export declare function setSystemctlForTests(systemctl: string | null): void; /** Test helper: forget the memoized decision so another environment can be set up. */ export declare function resetJobScopeLauncherForTests(): void; /** * The absolute paths systemctl is installed at, probed in order — the same * no-PATH-lookup reasoning as SYSTEMD_RUN_PATHS, and for the stronger reason: * this one runs as root and kills things. * * DELIBERATELY NOT ctl/systemctl.ts. That module is the CLI layer — it drives * the agent's OWN unit for install/uninstall, prints to a human's terminal, and * may block for a stop timeout; core must not depend on it. The two modules * therefore both invoke systemctl, at different layers and for different verbs, * on purpose. */ export declare const SYSTEMCTL_PATHS: readonly ["/usr/bin/systemctl", "/bin/systemctl"]; /** The first systemctl that exists, or null. `exists` is injectable, as above. */ export declare function resolveSystemctl(exists?: (p: string) => boolean): string | null; /** A safety net only — the measured call returns in 0.01 s. */ export declare const JOB_SCOPE_KILL_TIMEOUT_MS = 5000; /** * The command that empties a job's scope cgroup, built rather than written out, * so the flags the tests read are the flags that run. * * `kill`, NOT `stop`, and this is the part someone will want to "simplify": * `systemctl stop` waits for the unit's stop job to finish, i.e. up to * DefaultTimeoutStopSec (90 s), which is precisely what a cancel must not do. * `kill` only sends signals and returns — measured at 0.01 s on Ubuntu 24.04.4 / * systemd 255, after which the scope was `inactive`/`not-found`, its cgroup was * empty and the runaway's log stopped growing. * * `--kill-who=all`, NOT `--kill-whom`: the newer spelling only exists on systemd * ≥ 252 (both work on 255, the older one works everywhere), and this feature * targets every systemd that can create a scope. `all` rather than the default * `main` is the entire point — the process we cannot reach is by definition not * the main one. * * The unit name is re-validated for the same reason jobScopeUnitName validates * the id: this is a root `systemctl kill` and the name decides what dies. */ export declare function buildJobScopeKillArgv(opts: { systemctl: string; unit: string; }): JobScopeArgv; /** * Tear down a job's scope cgroup. Fire-and-forget by construction: it NEVER * throws and never reports, so a failure here can never turn a successful cancel * into an error for the caller. * * WHY IT EXISTS. Scopes made one case worse, and it was measured. Before them a * descendant that called `setsid` escaped `kill(-pgid)` but stayed in the * agent's service cgroup, so the next `systemctl restart` reaped it. With a * scope it sits in the job's own unit and survives BOTH cancel and restarts, * while refresh() sees the wrapper pid gone, settles the job and releases its * GPU lock — putting the next job on a card a runaway is still using, the exact * OOM the lock exists to prevent. Measured on Ubuntu 24.04.4 / systemd 255: with * a scoped job whose child called `setsid`, `kill -TERM -` killed the * wrapper while the runaway kept running and kept writing to its log (25 → 27 * lines) with the scope still `active`; the command above then emptied it. * * A unit that is already gone answers `Failed to kill unit …: Unit … not * loaded.` with a non-zero exit. That is the NORMAL case — the job ended and * systemd collected its scope — so it is a no-op success, never an error worth * surfacing. */ export declare function killJobScope(unit: string, systemctl?: string | null): void; /** * A scope SURVIVES an agent restart by design — that is the whole point of the * feature — so a job that ended while the agent was down may still be holding a * `setsid`-escaped descendant in its unit. refresh() finds exactly those records * (terminal on disk, never seen by this process) and they are the ONE case the * process-group cancel can never cover, so they must still be reaped. * * THE FAILURE THIS FIXES, which has now been introduced and removed once — do * not reintroduce it a third time. Hanging a plain killJobScope off that branch * fires a root `systemctl kill` per retained record: recover() refreshes every * job directory inside the retention window, so an agent restarting on a machine * holding a week of finished scoped jobs spawned one systemctl per job, in a * synchronous loop, against units that ended days ago and provably no longer * exist. On the desktop path recover() is reached lazily from the first * `do:job_*` frame, so that burst landed on the frame handler's thread — the * exact invariant the async capability probe above was reworked to protect. * * So: EVIDENCE, NOT EXIT CODES, the same principle the uninstall teardown * follows. Ask systemd ONCE which of our scopes are actually alive, kill only * those, and skip the rest without spawning anything. Deferred and coalesced, so * the whole recovery pass costs one bounded child process — and zero when there * is nothing to ask about. */ export declare const JOB_SCOPE_LIST_TIMEOUT_MS = 5000; /** * The command that answers "which of our job scopes exist right now". * * `--type=scope` plus the unit glob narrows the answer to units this agent * created and could name itself; `--all` keeps loaded-but-inactive ones (a scope * whose last process died but which systemd has not collected) so the answer is * about existence rather than activity; `--no-legend` drops the header and the * "N loaded units listed" footer, and `--plain` drops the leading `●` status * column so the unit name really is the first field of every line. */ export declare function buildJobScopeListArgv(systemctl: string): JobScopeArgv; /** * The unit names in a `list-units` answer. Anything that is not one of OUR scope * names is dropped rather than trusted: the set this returns decides what a root * `systemctl kill` is aimed at, and an empty or garbled answer must mean "kill * nothing", never "kill something else". */ export declare function parseJobScopeUnits(stdout: string): Set; /** * How many units one `systemctl kill` may name. * * `kill` takes any number of units, and the sweep exists precisely so that N * leftovers cost one spawn — but an argv is not unbounded, and this one is built * from names read off disk. Past the bound the list is simply chunked: 65 live * leftovers cost two spawns rather than 65, and none is skipped. In practice the * intersection is empty or has one element; a machine with 64 simultaneously * live orphaned scopes has a much larger problem than this argv. */ export declare const JOB_SCOPE_KILL_BATCH_MAX = 64; /** * buildJobScopeKillArgv for several units at once — same flags, same reasons, * and EVERY name re-validated, not just the first: this is a root kill and the * names decide what dies. */ export declare function buildJobScopeKillManyArgv(opts: { systemctl: string; units: readonly string[]; }): JobScopeArgv; /** * Queue the scope of a record found ALREADY TERMINAL on disk for the next sweep. * * Costs nothing but a Set insert: no probing, no spawn, nothing synchronous. * recover() may call this once per retained job; they all coalesce into the * single deferred flush below, because recover() is synchronous and the timer * cannot fire until it has finished. * * Deliberately NOT used for the running→terminal TRANSITION, which keeps its * immediate killJobScope: that is the instant the GPU lock is released and the * card is handed to the next job, so the survivor has to be ended THEN, not one * turn of the event loop later. */ export declare function reapLeftoverJobScope(unit: string): void;