// Convention workflows carry real shell scripts inside YAML, and the two live in // different worlds: the runner's bare PATH outside `nix develop`, a single-quoted // word inside `bash -c '...'`. The guards over those hazards all need the same // question answered first — is this line inside a quoted block or not — so it is // asked in one place. /** 1-indexed line numbers inside a multi-line `bash -c '` block, which ends at * the first line whose leading character is the closing `'`. A one-liner * closing its own quote (`bash -c 'a; b'`) opens no block — it is already * balanced. * * The close is not always the whole line: smoke.yml ends its block with * `' 2>&1 | tee "$RUNNER_TEMP/smoke.log"`, piping the invocation into a log. * Requiring the line to be exactly `'` left the block open to end of file, so * every apostrophe below it was reported as unquoted shell — comments, and an * `if:` expression that is not shell at all. Inside a single-quoted word a * leading apostrophe can only be the close, so what follows it on that line is * outside the block and none of this guard's business. */ export function bashBlockLines(yaml: string): Set { const OPEN = "bash -c '"; const inside = new Set(); let open = false; yaml.split("\n").forEach((text, i) => { if (!open) { const at = text.indexOf(OPEN); if (at >= 0 && !text.slice(at + OPEN.length).includes("'")) open = true; return; } if (text.trim().startsWith("'")) { open = false; return; } inside.add(i + 1); }); return inside; } /** The `run: |` steps that pipe into `tee` without `pipefail` set on the * block's first non-comment line — each reported at its `run: |` line. The * default `bash -e {0}` has no pipefail, so such a step exits with TEE's * status and a dead suite reads green; `set -euo pipefail` inside a nested * `bash -c '...'` body cannot reach the pipeline one shell up. Read from the * raw text, not a YAML parse: what matters is the ORDER of lines in the script * the runner executes. */ export function teeWithoutPipefail(yaml: string): { line: number; text: string }[] { const out: { line: number; text: string }[] = []; const lines = yaml.split("\n"); for (let i = 0; i < lines.length; i++) { const head = lines[i] as string; if (!/^\s*run: \|\s*$/.test(head)) continue; const indent = (lines[i + 1] ?? "").match(/^\s*/)?.[0].length ?? 0; const body: string[] = []; for (let j = i + 1; j < lines.length; j++) { const line = lines[j] as string; if (line.trim() === "") continue; if ((line.match(/^\s*/)?.[0].length ?? 0) < indent) break; body.push(line.trim()); } if (!body.some((l) => /\|\s*tee\b/.test(l))) continue; const first = body.find((l) => !l.startsWith("#")); const pipefailFirst = first !== undefined && /^set\s+(-[a-zA-Z]*o\s+pipefail|-euo\s+pipefail|-o\s+pipefail)/.test(first); if (!pipefailFirst) out.push({ line: i + 1, text: head }); } return out; } /** The step every convention workflow runs before `actions/checkout`, named so * the guard below can find it. */ export const TEST_TEMP_STEP = "Route test-temp out of the workspace"; // What that step must do, and what breaks when it doesn't. All four were learned // the hard way over three months on the shared DooD runners: // // route -- THE fix. These runners are docker-outside-of-docker, so the daemon // a launch talks to is the HOST's, and a bind-mount source it has not // seen is created by dockerd as ROOT on the host. --container-user // does not help: it governs what the container WRITES, not what the // daemon CREATES. Leave the harness on its repo-local default and // every run seeds the checkout with root-owned dirs. The override // must therefore point outside $GITHUB_WORKSPACE. // reap -- rm cannot unlink a live mountpoint even as ROOT (EBUSY), and a // RESTARTING leaked container re-creates the path as root seconds // after any clean. The holders are untracked by construction (a // killed run labels nothing), so they are found by mount SOURCE. // base -- clearing only the CONTENTS leaves the base dir itself, and an // empty-but-root-owned base still EACCESes a later mkdtemp. // verify -- the old form swallowed every failure into `2>/dev/null || true`, // so a clean that removed nothing looked identical to one that // worked. The checkout then died with a bare "EACCES rmdir" and no // clue what held the path. It stayed that way for a week. // The exported base must TRACE to $RUNNER_TEMP. Checking only that the export // exists would pass a step that dutifully exports the workspace path it was // already defaulting to, which is the whole bug. function routesOutOfWorkspace(body: string): boolean { const exported = body.match(/NORSK_CTL_TEST_TMP=(\$\{?\w+\}?|[^\s"]+)[^\n]*>> "\$GITHUB_ENV"/); if (!exported) return false; const value = exported[1] ?? ""; if (value.startsWith("$RUNNER_TEMP")) return true; const name = value.replace(/^\$\{?/, "").replace(/\}$/, ""); return name.length > 0 && new RegExp(`\\b${name}="\\$RUNNER_TEMP/`).test(body); } const TEST_TEMP_REQUIREMENTS: { name: string; ok: (body: string) => boolean }[] = [ { name: "route", ok: (b) => routesOutOfWorkspace(b) }, { name: "reap", ok: (b) => /\.Mounts\b/.test(b) && /docker rm -f/.test(b) }, { name: "base", ok: (b) => /rm -rf [^\n]*\/test-temp(?![\w/*])/.test(b) }, { name: "verify", ok: (b) => /\[ -e "\$tt" \]/.test(b) && /\bexit 1\b/.test(b) }, ]; /** The `run: |` body of the step starting at `nameLine`, de-indented. */ function stepBody(lines: string[], nameLine: number): string { const runAt = lines.findIndex((l, i) => i > nameLine && /^\s*run: \|\s*$/.test(l)); if (runAt < 0) return ""; const indent = (lines[runAt + 1] ?? "").match(/^\s*/)?.[0].length ?? 0; const body: string[] = []; for (let j = runAt + 1; j < lines.length; j++) { const line = lines[j] as string; if (line.trim() === "") continue; if ((line.match(/^\s*/)?.[0].length ?? 0) < indent) break; body.push(line.trim()); } return body.join("\n"); } /** Pre-checkout test-temp steps that would leave the checkout exposed, each * reported at its `- name:` line with the requirements it does not meet. */ export function testTempStepProblems(yaml: string): { line: number; missing: string[] }[] { const lines = yaml.split("\n"); const out: { line: number; missing: string[] }[] = []; lines.forEach((text, i) => { if (!text.includes(`- name: ${TEST_TEMP_STEP}`)) return; const body = stepBody(lines, i); const missing = TEST_TEMP_REQUIREMENTS.filter((r) => !r.ok(body)).map((r) => r.name); if (missing.length > 0) out.push({ line: i + 1, missing }); }); return out; } /** Jobs that check out without a {@link TEST_TEMP_STEP} of their own. * * The per-step guard above can only inspect steps it finds BY NAME, so * deleting the step -- or renaming it back -- reads as "no problems". This is * the presence half. It is per JOB because both halves of the step are * per-job: `$GITHUB_ENV` does not cross jobs, and neither does the runner's * workspace state. Job ids are read off the two-space indent under `jobs:` * rather than a YAML parse, to stay consistent with the line-oriented guards * here and to keep reporting real line numbers. */ export function jobsMissingTestTempStep(yaml: string): string[] { const lines = yaml.split("\n"); const bad: string[] = []; let job: string | undefined; let checkouts = 0; let steps = 0; const settle = () => { if (job !== undefined && checkouts > steps) bad.push(job); }; for (const line of lines) { const head = line.match(/^ {2}([A-Za-z_][\w-]*):\s*$/); if (head) { settle(); job = head[1]; checkouts = 0; steps = 0; continue; } if (/^\s*-?\s*uses: actions\/checkout@/.test(line)) checkouts++; if (line.includes(`- name: ${TEST_TEMP_STEP}`)) steps++; } settle(); return bad; } /** The step every job that routes test-temp ends with, named so the guards * below can find it. */ export const JOB_REAP_STEP = "Reap the containers this job launched"; // Why a job needs a reap of its own at the END, when the pre-checkout step // already reaps. The pre-checkout reap only protects the NEXT job's checkout, // and only from containers mounting the legacy in-workspace test-temp. A // harness teardown that never finishes leaves the instance running on the // shared Docker daemon until something else removes it: bun times a test out, // kills the daemon with the test's other dangling processes, and the afterAll // that should delete the instance has nothing left to ask, then times out // itself. commentary's publish-docs left a composition guide's engine running // that way for 2.6 hours on 2026-10-03, where it grew to 28 GB and the host // OOM-killed the next morning's product CI. The job knows exactly what it // launched -- everything mounting its own $NORSK_CTL_TEST_TMP -- so the job // removes it, whatever happened before. const JOB_REAP_REQUIREMENTS: { name: string; ok: (body: string) => boolean }[] = [ { name: "scope", ok: (b) => /\$\{?NORSK_CTL_TEST_TMP/.test(b) && /\.Mounts\b/.test(b) }, { name: "remove", ok: (b) => /docker rm -f/.test(b) }, ]; /** {@link JOB_REAP_STEP}s that would not reap, each reported at its `- name:` * line with what it is missing: the `always()` guard (without it a failed test * skips the step, which is exactly when it is needed) or a requirement above. */ export function jobReapStepProblems(yaml: string): { line: number; missing: string[] }[] { const lines = yaml.split("\n"); const out: { line: number; missing: string[] }[] = []; lines.forEach((text, i) => { if (!text.includes(`- name: ${JOB_REAP_STEP}`)) return; const missing: string[] = []; const runAt = lines.findIndex((l, j) => j > i && /^\s*run: \|\s*$/.test(l)); const header = lines.slice(i + 1, runAt < 0 ? i + 4 : runAt); if (!header.some((l) => /^\s*if: always\(\)\s*$/.test(l))) missing.push("always"); const body = stepBody(lines, i); missing.push(...JOB_REAP_REQUIREMENTS.filter((r) => !r.ok(body)).map((r) => r.name)); if (missing.length > 0) out.push({ line: i + 1, missing }); }); return out; } /** Jobs that route test-temp (so may launch instances) but have no * {@link JOB_REAP_STEP}. The presence half, per job for the same reason as * {@link jobsMissingTestTempStep}. */ export function jobsMissingReapStep(yaml: string): string[] { const lines = yaml.split("\n"); const bad: string[] = []; let job: string | undefined; let routes = 0; let reaps = 0; const settle = () => { if (job !== undefined && routes > 0 && reaps === 0) bad.push(job); }; for (const line of lines) { const head = line.match(/^ {2}([A-Za-z_][\w-]*):\s*$/); if (head) { settle(); job = head[1]; routes = 0; reaps = 0; continue; } if (line.includes(`- name: ${TEST_TEMP_STEP}`)) routes++; if (line.includes(`- name: ${JOB_REAP_STEP}`)) reaps++; } settle(); return bad; } /** Steps that rewrite a lockfile -- an unfrozen `bun install`, or a * `bun update` -- yet are `continue-on-error`, by their `- name:` line. * * A gate may be allowed to fail: a red gate is information, and the sync * workflows turn it into a draft or reconciliation PR. A lockfile refresh may * not. One that dies part-way leaves bumped manifests beside the old * lockfiles, and a later step commits that pair: on 2026-10-03 the runner * OOM-killed sync-ctl-packages mid-install and its PR then failed every * frozen install. Keep the refresh in a step of its own that must succeed. */ export function lockfileRefreshCanFail(yaml: string): number[] { const lines = yaml.split("\n"); const starts = lines.flatMap((l, i) => (/^\s{6}- (name|uses|run)\b/.test(l) ? [i] : [])); const out: number[] = []; starts.forEach((a, k) => { const block = lines.slice(a, starts[k + 1] ?? lines.length); const tolerated = block.some((l) => /^\s+continue-on-error:\s*true\b/.test(l)); const rewrites = block.some((l) => { const t = l.trim(); if (t.startsWith("#") || t.startsWith('"')) return false; return /\bbun (install|update)\b/.test(t) && !/--frozen-lockfile/.test(t); }); if (tolerated && rewrites) out.push(a + 1); }); return out; } /** `- name:` lines of steps that rebase onto a fetched branch without first * unshallowing. * * The self-hosted runners keep each repo's .git between jobs, and * actions/checkout fetches at depth 1, so a commit an earlier job fetched is * left recorded as parentless. A plain `git fetch` will not re-fetch an object * it already holds, so when that commit is main's tip the rebase sees two * unrelated roots and replays our base commit over it: add/add conflicts on * every file, read as "a real conflict". It failed playout's sync-dev-kit and * studio's upgrade-latest that way on 2026-10-03. `--unshallow` re-fetches the * parents and clears the stale boundary. */ export function rebaseWithoutUnshallow(yaml: string): number[] { const lines = yaml.split("\n"); const out: number[] = []; lines.forEach((text, i) => { if (!/^\s+- name:/.test(text)) return; const body = stepBody(lines, i); if (/\bgit rebase\b/.test(body) && !/git fetch --unshallow\b/.test(body)) out.push(i + 1); }); return out; } /** A `ctl-candidate` validation that another candidate, or its own retry, can * cancel. * * release-collect scores anything other than a literal success as red — a * CANCELLED validation included — and answers a red with one re-dispatch. So a * concurrency group shared across candidates makes the gate eat itself: * candidate B's dispatch cancels A's in-flight run, A scores red, A retries, * and that retry cancels B's run. Observed 2026-09-05: commentary dispatched * four times in thirteen minutes, every run cancelled, and * `rc-gate/0.1.0-2026-09-05-98e09d1/FAILED` written for a candidate on which * nothing had actually failed. * * Both halves are needed. The group must SEPARATE candidates * (`client_payload.version`), so a different candidate never shares one; and a * dispatch run must not be cancellable, so a retry queues behind the run it is * asking for rather than killing it. A superseded PENDING run is dropped * before it starts, so it fires no `report-candidate` and scores nothing — * which is why queueing is safe where cancelling is not. */ export function candidateConcurrencyProblems(yaml: string): string[] { const block = yaml.match(/^concurrency:\n((?:[ \t]+.*\n)+)/m)?.[1] ?? ""; const group = block.match(/^\s*group:\s*(.*)$/m)?.[1] ?? ""; const cancel = (block.match(/^\s*cancel-in-progress:\s*(.*)$/m)?.[1] ?? "").trim(); const problems: string[] = []; if (!/client_payload\.version/.test(group)) { problems.push( "group does not separate candidates: a newer candidate cancels an older validation, which scores red", ); } if (cancel === "true") { problems.push("cancel-in-progress is unconditionally true: the re-dispatch cancels the run it is asking for"); } return problems; }