/** * Cross-process global concurrency limit for evidence-tool runs (T1534 / ADR-061). * * The cache layer ({@link runToolCached}) coalesces *identical* parallel runs * via a per-key file lock — when 10 sibling tasks against the same git HEAD * call `tool:test`, only one spawns. But orchestrator-spawned worktree * agents each operate on a *different* HEAD (one branch per worktree per * ADR-055), so their cache keys differ and the per-key lock does NOT * coalesce them. Without an additional bound, N worktree agents would each * spawn the full toolchain, multiplying CPU and resident memory by N. * * This module bounds *total* concurrent runs of a canonical tool across * the whole machine — independent of which project, worktree, or PID * issues the call. It uses a slot directory under * `~/.local/share/cleo/locks/tool-/` with `slot-0.lock` … * `slot-(N-1).lock` files; each slot is held by `proper-lockfile` so a * crashed process auto-releases via the standard stale-lock recovery. * * Defaults (configurable via env): * * | Tool | Default | Binding constraint | * |----------------|----------------------------------|---------------------------| * | test, build | min(RAM/24GiB, cpus/4), min 1 | MEMORY (each run forks) | * | lint, typecheck| max(2, cpus/2) | CPU (single-process) | * | audit | max(2, cpus/2) | network-bound, small RAM | * | security-scan | max(2, cpus/2) | network-bound, small RAM | * * T12091: `test`/`build` were `max(1, cpus/4)` — 6 slots on a 24-core box. Since * each `pnpm run test` is itself allowed 6 vitest forks × 4 GiB, the two bounds * composed to 144 GiB of permitted heap on 62 GiB of RAM. Neither layer was * individually violated, which is exactly why the machine froze without any * guard firing. Heavy tools are now bounded by the dimension that actually * limits them — see {@link defaultMaxConcurrent}. * * Override via `CLEO_TOOL_CONCURRENCY_` (e.g. * `CLEO_TOOL_CONCURRENCY_TEST=2`). Set to `0` or a negative number to * disable the limit for that tool — which also disables the RAM bound, so it is * the one setting that can still oversubscribe the machine. * * @task T1534 * @task T12091 * @adr ADR-061 */ import type { ResourceSample } from '../resources/backend.js'; import type { CanonicalTool } from './tool-resolver.js'; /** * Function returned by {@link acquireGlobalSlot} that must be called to * release the held slot. Always-callable; idempotent against re-entry. * * @task T1534 */ export type ReleaseSlotFn = () => Promise; /** * Options for {@link acquireGlobalSlot}. * * @task T1534 */ export interface AcquireSlotOptions { /** * Maximum wall-clock time to wait for a free slot before throwing. The * default — 60 minutes — covers a long-running monorepo test suite that * can leave the semaphore held for a while. * * @defaultValue `3_600_000` (60 min) */ timeoutMs?: number; /** * Sleep between acquire attempts when all slots are busy. Smaller values * burn more CPU on the polling loop; larger values delay the next worker. * * @defaultValue `100` */ pollMs?: number; /** * Stale-lock window passed to `proper-lockfile`. A slot held by a * process that exited without releasing is reaped after this many * milliseconds. * * @defaultValue `600_000` (10 min) */ staleMs?: number; /** * Override `os.availableParallelism()` for tests. * * @internal */ cpuCount?: number; /** * Override `os.totalmem()` (in GiB) for tests. * * T12091 made the heavy-tool budget RAM-derived, which would otherwise make * the slot count depend on whatever host the suite runs on — a 16 GiB CI * runner and a 62 GiB workstation resolve different budgets for identical * inputs. Injecting it keeps semaphore behaviour deterministic. * * @internal */ totalRamGib?: number; /** * Memory-pressure sample used to scale the effective slot count for the * pressure-sensitive `test`/`build` tools (T12001, Epic T11992). When * omitted, a best-effort live sample is taken (fail-open to the static slot * count on any error). Pass `null` to disable pressure scaling explicitly. * Tests inject a synthetic sample for determinism. * * @internal */ pressureSample?: ResourceSample | null; } /** * Worst-case resident footprint of ONE `tool:test` / `tool:build` invocation, * in GiB. * * This is not a guess. `vitest.memory-safe.ts` permits * `MEMORY_SAFE_MAX_WORKERS` forks (up to 6) each capped at `FORK_HEAP_MB` * (4096), so a single `pnpm run test` may legitimately hold ~24 GiB before any * guard fires. Keep this in step with that file if either bound changes. * * @task T12091 */ export declare const HEAVY_TOOL_FOOTPRINT_GIB = 24; /** * Compute the default max-concurrency for a canonical tool. * * ## Why this is RAM-derived, not core-derived (T12091) * * This returned `floor(cpus / 4)` for `test`/`build`, which on a 24-core box is * **6 concurrent full test suites**. Each of those is itself allowed 6 vitest * forks × a 4 GiB heap cap, so the composed permission was * * 6 runs × 6 forks × 4 GiB = 144 GiB * * on a 62 GiB machine. Both layers were individually "bounded" and their * composition was 2.3× the hardware — which is precisely how this box froze * repeatedly: no single guard was violated. The per-invocation cap (T12087) and * this cross-invocation cap were written independently and never multiplied out. * * The dimensional error is the root of it: what limits a test run is MEMORY, and * core count says nothing about memory. A 24-core/16 GiB VM got the same 6 slots * as a 24-core/256 GiB server. So heavy tools now divide TOTAL RAM by * {@link HEAVY_TOOL_FOOTPRINT_GIB} and are additionally capped by cores, never * exceeding what the machine can actually hold. * * Reactive pressure scaling ({@link pressureScaleSlots}) is not a substitute: * PSI `some avg10` is a ten-second average, and a fork fleet can exhaust RAM * faster than that window can report it. Admission has to be right up front. * * Light tools (lint, typecheck, audit, security-scan) are single-process and * short, and keep the core-derived half-of-cores budget. * * @param canonical - the canonical tool class. * @param cpuCount - logical cores available. * @param totalRamGib - total machine RAM in GiB; defaults to a live reading. * @returns the machine-wide slot count, always ≥ 1. * * @example * ```ts * // 24 cores, 62 GiB → floor(62/24) = 2 (was 6, permitting 144 GiB of heap) * defaultMaxConcurrent('test', 24, 62); // → 2 * // 24 cores, 16 GiB → 1: one suite is already more than this box can hold * defaultMaxConcurrent('test', 24, 16); // → 1 * ``` * * @task T1534 * @task T12091 */ export declare function defaultMaxConcurrent(canonical: CanonicalTool, cpuCount: number, totalRamGib?: number): number; /** * Resolve the active per-tool concurrency limit, honouring the * `CLEO_TOOL_CONCURRENCY_` env override when set. A value of * `0` (or any non-positive number) disables the bound and returns * `Number.POSITIVE_INFINITY`, in which case {@link acquireGlobalSlot} * returns a no-op release. * * @task T1534 */ export declare function resolveMaxConcurrent(canonical: CanonicalTool, cpuCount?: number, totalRamGib?: number): number; /** * Scale a static slot budget down under memory pressure (T12001 · choke-point * #6). Mirrors the governor's `test-run` budget: halve when `some avg10` exceeds * the hold threshold, floor to 1 when it exceeds the backoff/floor threshold. * Recovers automatically as pressure clears. `full-build` is not represented as * a canonical tool here; the dedicated `full-build` governor class (T11999) * pins that to one machine-wide slot. * * @task T12001 */ export declare function pressureScaleSlots(canonical: CanonicalTool, staticMax: number, sample: ResourceSample, thresholds?: { holdSomeAvg10?: number; floorSomeAvg10?: number; }): number; /** * Path to the global slot directory for a canonical tool. Sits under * `getCleoHome()/locks/tool-/` so all CLEO-driven processes * on a machine share the same semaphore — across projects, worktrees, * and PIDs. * * @task T1534 */ export declare function semaphoreDir(canonical: CanonicalTool): string; /** * Acquire one slot from the global semaphore for a canonical tool. Blocks * until a slot becomes free or `timeoutMs` elapses. * * Implementation detail: tries each slot file in turn with `retries: 0` * (proper-lockfile non-blocking acquire). When all are busy, sleeps for * `pollMs` and retries. This avoids the thundering-herd cost of having * many retriers wake at the exact same moment. * * @param canonical - Canonical tool name from the resolver. * @param opts - Acquisition options. * @returns A release function. Idempotent. * @throws When `timeoutMs` elapses without acquiring a slot. * * @example * ```ts * const release = await acquireGlobalSlot('test'); * try { * await runTheTool(); * } finally { * await release(); * } * ``` * * @task T1534 */ export declare function acquireGlobalSlot(canonical: CanonicalTool, opts?: AcquireSlotOptions): Promise; //# sourceMappingURL=tool-semaphore.d.ts.map