#!/usr/bin/env bun // Known-flaky tests, kept out of the gate but not out of the run (ci-fragility // proposal E). A test that fails one run in three on a loaded runner turns the // whole checks job red; a person re-runs it, and a real red hides among the // flaky ones. Quarantine takes it out of the gating run, runs it on its own // straight after, and reports the result without failing the job. // // The list is the repo's `.github/quarantine.json`, product-owned (not a // drift-checked copy), so a reader on GitHub, the dashboard and the job all see // the same thing. Every entry carries an owner, the date it went in and the // condition for taking it out, and an entry older than STALE_DAYS is flagged on // every run: quarantine is a holding pen, not a way to skip a test for ever. // // {"tests": [{"file": "tests/unit/x.test.ts", "name": "", // "owner": "", "since": "YYYY-MM-DD", // "reason": "", "exit": ""}]} // // Run by its node_modules path from the checks quality job (conventions/ // checks.yml), like zod-singleton.ts: // quarantine.ts check validate the list; exit 1 if it is malformed // quarantine.ts bun-options the BUN_OPTIONS that leave the listed tests out // quarantine.ts run run each listed test alone and report; exit 0 import { appendFileSync, existsSync, readFileSync } from "node:fs"; export const QUARANTINE_FILE = ".github/quarantine.json"; export const STALE_DAYS = 30; export interface QuarantineEntry { file: string; name: string; owner: string; since: string; reason: string; exit: string; issue?: string; } const REQUIRED = ["file", "name", "owner", "since", "reason", "exit"] as const; export interface ParsedQuarantine { entries: QuarantineEntry[]; errors: string[]; warnings: string[]; } const DAY_MS = 86_400_000; const isDate = (s: string) => /^\d{4}-\d{2}-\d{2}$/.test(s) && !Number.isNaN(Date.parse(`${s}T00:00:00Z`)); /** `text` is the file's contents, undefined when the repo has none. */ export function parseQuarantine( text: string | undefined, opts: { today: string; exists: (file: string) => boolean }, ): ParsedQuarantine { const out: ParsedQuarantine = { entries: [], errors: [], warnings: [] }; if (text === undefined) return out; let doc: unknown; try { doc = JSON.parse(text); } catch (e) { out.errors.push(`${QUARANTINE_FILE} is not JSON: ${e instanceof Error ? e.message : String(e)}`); return out; } const tests = (doc as { tests?: unknown } | null)?.tests; if (!Array.isArray(tests)) { out.errors.push(`${QUARANTINE_FILE} has no "tests" array`); return out; } const seen = new Set(); tests.forEach((raw, i) => { const e = (raw ?? {}) as Record; const at = `${QUARANTINE_FILE} tests[${i}]`; const missing = REQUIRED.filter((f) => typeof e[f] !== "string" || (e[f] as string).trim() === ""); if (missing.length > 0) { out.errors.push( `${at}: missing ${missing.join(", ")} (every entry needs an owner, a date and an exit condition)`, ); return; } const entry = e as unknown as QuarantineEntry; if (!isDate(entry.since)) { out.errors.push(`${at}: since "${entry.since}" is not a YYYY-MM-DD date`); return; } const age = Math.round((Date.parse(`${opts.today}T00:00:00Z`) - Date.parse(`${entry.since}T00:00:00Z`)) / DAY_MS); if (age < 0) { out.errors.push(`${at}: since ${entry.since} is in the future`); return; } if (!opts.exists(entry.file)) { out.errors.push(`${at}: ${entry.file} is not in the repo; drop the entry or fix the path`); return; } const key = `${entry.file}\0${entry.name}`; if (seen.has(key)) { out.errors.push(`${at}: "${entry.name}" is listed twice`); return; } seen.add(key); if (age > STALE_DAYS) { out.warnings.push( `"${entry.name}" has been quarantined since ${entry.since} (${age} days, owner ${entry.owner}): fix it and take it out, or delete it`, ); } out.entries.push(entry); }); return out; } const escapeRegex = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); /** A regex matching the one test, wherever its describe blocks put it. */ export function namePattern(name: string): string { return `${escapeRegex(name)}$`; } /** BUN_OPTIONS for the gating run: every test except the listed ones. Bun * splits BUN_OPTIONS on whitespace and has no exclude-by-name flag, so it is * one negative lookahead with each space written as \s. */ export function bunOptions(entries: readonly QuarantineEntry[]): string { if (entries.length === 0) return ""; const names = entries.map((e) => namePattern(e.name).replace(/\s/g, "\\s")).join("|"); return `--test-name-pattern=^(?!.*(?:${names})).*`; } export type RunResult = "passed" | "failed" | "missing"; /** A run that matched nothing means the test was renamed or deleted, and the * entry is stale; that is not a pass, whatever the exit code. */ export function classifyRun(exitCode: number, output: string): RunResult { if (/matched 0 tests/.test(output) || /Ran 0 tests/.test(output)) return "missing"; return exitCode === 0 ? "passed" : "failed"; } export function summary(results: readonly { entry: QuarantineEntry; result: RunResult }[]): string { const lines = [ `### Quarantined tests (run and reported, not gating)`, "", `From \`${QUARANTINE_FILE}\`. A failure here does not fail the job; it is the flake the entry is waiting on.`, "", "| Result | Test | File | Owner | Since | Exit condition |", "|---|---|---|---|---|---|", ]; for (const { entry, result } of results) { const cell = (s: string) => s.replace(/\|/g, "\\|"); lines.push( `| ${result} | ${cell(entry.name)} | \`${entry.file}\` | ${cell(entry.owner)} | ${entry.since} | ${cell(entry.exit)} |`, ); } return `${lines.join("\n")}\n`; } function load(): ParsedQuarantine { const text = existsSync(QUARANTINE_FILE) ? readFileSync(QUARANTINE_FILE, "utf8") : undefined; return parseQuarantine(text, { today: new Date().toISOString().slice(0, 10), exists: existsSync }); } function report(parsed: ParsedQuarantine): void { for (const w of parsed.warnings) console.log(`::warning title=quarantine::${w}`); for (const e of parsed.errors) console.log(`::error title=quarantine::${e}`); } async function main(cmd: string | undefined): Promise { const parsed = load(); if (cmd === "check") { report(parsed); console.log(`${parsed.entries.length} quarantined test(s) in ${QUARANTINE_FILE}`); return parsed.errors.length > 0 ? 1 : 0; } if (cmd === "bun-options") { if (parsed.errors.length > 0) return 1; process.stdout.write(bunOptions(parsed.entries)); return 0; } if (cmd === "run") { report(parsed); const results: { entry: QuarantineEntry; result: RunResult }[] = []; for (const entry of parsed.entries) { console.log(`== quarantined: ${entry.name} (${entry.file})`); const proc = Bun.spawn(["bun", "test", entry.file, `--test-name-pattern=${namePattern(entry.name)}`], { stdout: "pipe", stderr: "pipe", }); const [out, err] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text()]); const code = await proc.exited; process.stdout.write(out); process.stdout.write(err); const result = classifyRun(code, `${out}${err}`); results.push({ entry, result }); if (result !== "passed") { console.log(`::warning title=quarantined test ${result} (not gating)::${entry.name} (${entry.file})`); } } const failed = results.filter((r) => r.result !== "passed").length; if (process.env.GITHUB_STEP_SUMMARY && results.length > 0) { appendFileSync(process.env.GITHUB_STEP_SUMMARY, summary(results)); } if (process.env.GITHUB_OUTPUT) { appendFileSync(process.env.GITHUB_OUTPUT, `quarantined=${results.length}\nquarantine_failed=${failed}\n`); } console.log(`${results.length} quarantined test(s) run, ${failed} not passing (not gating)`); return 0; } console.error("usage: quarantine.ts check | bun-options | run"); return 2; } if (import.meta.main) process.exit(await main(process.argv[2]));