/** * Checkout provisioning for one run. * * Every issue gets its own repository so two workers can never see each other's * half-finished edits. Objects come from a per-repo bare mirror that is cloned * once and refreshed, rather than a fresh full clone per issue: a module repo's * history is fetched one time and every later run pays only for the delta. * * **A run gets its own repository, not a linked worktree of the mirror, and that * is load-bearing rather than a preference.** A linked worktree shares the * mirror's common dir — refs, objects, index locks — so every run would be * writing into one store: concurrent runs contend on the same index lock, a * `git gc` in one can pull objects from under another, and a run's refs are one * `git update-ref` away from a sibling's. Here the objects arrive read-only through git's alternates * mechanism and the run's refs, index and commits live in its own git dir. The * documented residual is that a run can still *read* another run's objects out * of the shared store: bounded, same source, no write path, no credential, and * the price of not cloning the repo per run. * * Publishing is therefore the daemon's job, never the worker's — see * `pushRunBranch` in `gitops.ts`. The run repo's `origin` deliberately * names the real clone URL rather than the mirror, so a worker's * `git push origin HEAD` attempts the network and fails on credentials instead * of quietly succeeding into the shared object store. */ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { credentialedEnv, runRepoSafeDirectoryExemption, scrubUserinfo } from "./gitops.ts"; import type { RepoTarget } from "./types.ts"; /** * Normal-clone fetch semantics. Deliberately *not* the mirror's own * `+refs/*:refs/*`: see `configureMirror`. */ const TRACKING_REFSPEC = "+refs/heads/*:refs/remotes/origin/*"; /** * The on-disk layout, in one place. Both are pure functions of config, so a * caller that has to clean up *before* provisioning — the dispatcher on a * retry, or reconciling orphaned trees at startup — can name a mirror or a * tree without a network hop, and cannot drift from what `ensureMirror` and * `addRunRepo` will actually create. */ export function mirrorPathFor(repo: RepoTarget, mirrorRoot: string): string { return join(mirrorRoot, `${repo.name}.git`); } /** One tree per issue: a retry reuses the number, never the contents. */ export function worktreePathFor(workspaceRoot: string, issue: number): string { return join(workspaceRoot, String(issue)); } /** One git invocation, decoded. Nothing here judges the exit code. */ async function runGit( args: string[], cwd?: string, ): Promise<{ code: number; stdout: string; stderr: string }> { const proc = Bun.spawn(["git", ...args], { cwd, stdin: "ignore", stdout: "pipe", stderr: "pipe", // Every git call in this module is the daemon's own — provisioning, // salvaging or publishing — so it goes through the one named construction // site of credential material. It also carries // GIT_TERMINAL_PROMPT=0, because an unattended dispatcher must fail loudly // rather than block forever on a prompt nobody is there to answer. env: credentialedEnv(), }); const [stdout, stderr, code] = await Promise.all([ new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited, ]); return { code, stdout, stderr }; } /** * The one way this module runs git *for effect*. Non-zero exit throws with the * argv, the exit code and stderr, because a bare "git failed" in a daemon log * is worth nothing at 3am. */ async function git(args: string[], cwd?: string): Promise { const { code, stdout, stderr } = await runGit(args, cwd); if (code !== 0) { const where = cwd === undefined ? "" : ` (cwd ${cwd})`; const detail = stderr.trim() || stdout.trim() || "no output"; // A clone URL can carry a token, and it lands in both our argv and git's // own error text; this string reaches logs and humans, so scrub it. throw new Error(scrubUserinfo(`git ${args.join(" ")}${where} exited ${code}: ${detail}`)); } return stdout.trim(); } /** * The way this module runs git *as a question*. For plumbing like `show-ref` * a non-zero exit is the answer "no", not a failure, so it must not throw — * otherwise every probe needs a try/catch that also swallows real breakage. */ async function gitSucceeds(args: string[], cwd?: string): Promise { const { code } = await runGit(args, cwd); return code === 0; } /** Fences the block below so it can be found, replaced, and never duplicated. */ const REF_LOCK_SIGNATURES = /cannot lock ref|unable to update local ref/i; /** * Refreshes a bare mirror's refs from its remote. Two dispatch loops can race * on git's *packed-refs* / ref locks for the same repo and one lands a transient * `cannot lock ref` / `unable to update local ref` — that is not corruption, but * an {@link ensureMirror} that throws on it aborts the whole dispatch at turn 0 * and chars a failed attempt (#168). Retry once after a short backoff; a second * failure (or any non-ref-lock failure) propagates unchanged, so a genuinely * broken mirror still surfaces loudly. */ async function refreshMirror(mirrorPath: string): Promise { try { return await git(["remote", "update", "--prune"], mirrorPath); } catch (err) { const message = err instanceof Error ? err.message : String(err); if (!REF_LOCK_SIGNATURES.test(message)) throw err; await new Promise((r) => setTimeout(r, 500)); return git(["remote", "update", "--prune"], mirrorPath); } } /** Fences the block below so it can be found, replaced, and never duplicated. */ const EXCLUDE_BEGIN = "# >>> omp-conductor (managed; edit outside this block)"; const EXCLUDE_END = "# <<< omp-conductor"; /** * Appended to every mirror's `info/exclude`, and so in force in every worktree * cut from it: a worker's own scaffolding is not the repo's business, and * keeping it out of `git status` keeps it out of the worker's own `git add -A` * as well as out of salvage. * * **Only the directory form of `.scratch*`; the slash is the design.** An ignore * here applies to every repo this fleet touches, and an ignored *new* file is * invisible to salvage — `git add -A` skips it, and a tree holding only such * files reports `nothing`. So a name that could plausibly be a deliverable must * never appear in this list. The first version also carried `.env.local` and * `*.local.sh`; its first correction still carried an unqualified `.scratch*`. * All three make the same mistake: repos can legitimately ship an `.env.local` * template, `bootstrap.local.sh`, `.scratchrc`, or `.scratchpad`, and a worker * asked to add one would watch it vanish. * * The directory-only pattern survives because its trailing slash limits it to a * directory whose name announces that its contents are disposable. That was * the 2026-08-07 incident's exact shape (`.scratch82/env.sh`). Broader * conventions belong in a repo's own `.gitignore`, where its operator chooses * them, rather than being imposed by whatever dispatcher happens to be driving. * * The advisor watchdog (`#542`) is the deliberate exception, and the exact-path * spelling is the whole point: with the advisor on, the dispatcher writes a * per-issue `/.omp/WATCHDOG.md` from the brief's acceptance criteria * so the mid-run reviewer checks the worker against its own issue — and that * file must never reach the diff a worker ships or the salvage commit. The * standing rule says an ignored *new* file is invisible to salvage, so a name * that could plausibly be a deliverable must never appear; `.omp/WATCHDOG.md` * is the narrowest name that can. Only this exact file is excluded, never the * `.omp/` directory — a directory-wide ignore would hide any future * deliverable an operator legitimately places under `.omp/` (the very trap the * `.scratch*` history describes), and the blind spot stays one path wide. */ const LOCAL_EXCLUDE = [".scratch*/", ".omp/WATCHDOG.md"]; /** * Adds the managed block to an `info/exclude`, preserving everything else. * * `info/exclude` is a *local* ignore file, which means it is exactly where an * operator or another tool puts patterns they could not put in the tracked * `.gitignore` — so overwriting it would silently destroy work that has no * other copy. The block is fenced and replaced in place, so re-running this on * every dispatch neither duplicates our lines nor disturbs theirs. */ export function mergeExclude(existing: string): string { const begin = existing.indexOf(EXCLUDE_BEGIN); const end = existing.indexOf(EXCLUDE_END); const theirs = begin === -1 || end === -1 || end < begin ? existing : existing.slice(0, begin) + existing.slice(end + EXCLUDE_END.length + 1); // Normalised before recomposing, so the result is byte-identical on every // call. Trimming the tail matters twice over: a hand-edited file often has no // trailing newline (the first managed line would glue onto their last // pattern and match nothing), and without it the blank separator below would // accumulate one more newline on each of the thousands of dispatches that // call this. const body = theirs.replace(/\n+$/, ""); const head = body === "" ? [] : [body, ""]; return [...head, EXCLUDE_BEGIN, ...LOCAL_EXCLUDE, EXCLUDE_END, ""].join("\n"); } /** * Rewrites the two `clone --mirror` defaults that are actively dangerous for a * cache we cut worktrees from. Applied on every `ensureMirror` so a mirror left * by an older build heals itself instead of behaving differently forever. * * - `remote.origin.mirror=true` makes a bare `git push` from *any* worktree of * this repository behave as `push --mirror`, force-updating the remote from * our local refs and deleting whatever we do not happen to have. Worktrees * share the mirror's config, so that footgun is pointed straight at the * worker. * - `fetch=+refs/*:refs/*` puts the upstream branches and our per-run branches * in the same namespace, so a `--prune` refresh would try to delete every run * branch that has not been pushed yet. * * With normal-clone semantics a refresh only ever moves `refs/remotes/origin/*` * and cannot touch a live run. */ async function configureMirror(mirrorPath: string): Promise { await git( ["config", "--replace-all", "remote.origin.mirror", "false"], mirrorPath, ); await git( ["config", "--replace-all", "remote.origin.fetch", TRACKING_REFSPEC], mirrorPath, ); // A mirror's `info/exclude` is the common git dir for every worktree cut from // it, so one write here keeps a worker's own scratch out of `git status` in // all of them — without touching the repo's tracked `.gitignore`, which is // the operator's file and not ours to edit. // // Belt and braces with the salvage excludes rather than a replacement for // them: this stops scratch reaching a *worker's* own `git add`, which salvage // never sees. Merged rather than written, because `info/exclude` is precisely // where an operator keeps patterns that cannot go in the tracked file — and // this runs on every dispatch, so overwriting would destroy them repeatedly. const exclude = join(mirrorPath, "info", "exclude"); mkdirSync(dirname(exclude), { recursive: true }); writeFileSync(exclude, mergeExclude(existsSync(exclude) ? readFileSync(exclude, "utf8") : "")); } /** * Rewrites this worktree's common `info/exclude` managed block to the package's * current list. Salvage must do this itself: it is the path that runs after a * package swap, when the mirror may still carry a previous release's patterns, * and `git add -A` would otherwise silently skip legitimate new files the old * list happened to name (dogfood 2026-08-07 / #44). * * Failures are swallowed — a missing common dir is "no exclude to heal", and * the salvage dirty check still runs. Never throws into the salvage outcome. */ function refreshManagedExclude(worktree: string): void { try { // `--git-common-dir` is relative for linked worktrees; resolve against the // tree so bare-mirror layouts and plain clones both land on info/exclude. // The tree is worker-owned by the time salvage runs (#798), so the // exact-path exemption rides on the argv (#816). const common = Bun.spawnSync(["git", ...runRepoSafeDirectoryExemption(worktree), "rev-parse", "--git-common-dir"], { cwd: worktree, stdin: "ignore", stdout: "pipe", stderr: "pipe", env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, }); if (common.exitCode !== 0) return; const raw = common.stdout.toString().trim(); if (raw === "") return; const commonDir = raw.startsWith("/") ? raw : join(worktree, raw); const exclude = join(commonDir, "info", "exclude"); mkdirSync(dirname(exclude), { recursive: true }); writeFileSync(exclude, mergeExclude(existsSync(exclude) ? readFileSync(exclude, "utf8") : "")); } catch { // ponytail: exclude heal is best-effort; salvage still prefers a commit of // whatever git can see over failing the whole orphan path. } } /** * Serializes concurrent {@link ensureMirror} calls per mirror path so two * dispatches cannot collide on git's ref locks for the same repo (#186). A * rejected (failed) call does not poison the chain: the next caller waits on a * settled promise. */ const mirrorLocks = new Map>(); /** * Returns the path of the bare mirror for `repo`, cloning it on first use and * refreshing it otherwise. * * ponytail: the lock is per process, not a cross-process lockfile in * `mirrorRoot`. Two *dispatch loops* can still collide on git's ref locks and * one will throw; the run is retried rather than corrupted. Upgrade path is a * lockfile keyed by repo name, for a future multi-daemon host. * * ponytail: if `repo.cloneUrl` embeds credentials, `git clone` persists them in * the mirror's config, exactly as it would for a hand-run clone. Prefer an SSH * URL or an https URL backed by a credential helper. */ export async function ensureMirror( repo: RepoTarget, mirrorRoot: string, ): Promise { const key = mirrorPathFor(repo, mirrorRoot); const prev = mirrorLocks.get(key) ?? Promise.resolve(); const next = prev.catch(() => {}).then(() => ensureMirrorUnlocked(repo, mirrorRoot)); mirrorLocks.set(key, next); void next.catch(() => {}).finally(() => { if (mirrorLocks.get(key) === next) mirrorLocks.delete(key); }); return next; } async function ensureMirrorUnlocked(repo: RepoTarget, mirrorRoot: string): Promise { mkdirSync(mirrorRoot, { recursive: true }); const mirrorPath = mirrorPathFor(repo, mirrorRoot); if (existsSync(mirrorPath)) { await configureMirror(mirrorPath); await refreshMirror(mirrorPath); return mirrorPath; } try { await git(["clone", "--mirror", repo.cloneUrl, mirrorPath]); } catch (err) { // A half-written mirror would be seen as "present" by the next call and // fail in a much more confusing place. Retry from scratch instead. rmSync(mirrorPath, { recursive: true, force: true }); throw err; } await configureMirror(mirrorPath); // `clone --mirror` writes upstream heads under `refs/heads/*`; the safe // normal-clone refspec above uses `refs/remotes/origin/*`. Populate that // namespace before returning so first-use callers see the same fresh mirror // shape as every later refresh. await refreshMirror(mirrorPath); return mirrorPath; } /** * Provisions `/` as a **repository of its own** for * `repo`/`branch`, and returns the path. On the first attempt the branch is cut * from the upstream tip of the default branch; on a retry the preserved branch * is reattached out of the mirror (see the comment on the fetch below). * * The objects come from the shared mirror through `objects/info/alternates` — * git's own read-only borrowing mechanism — so provisioning still costs one * fetch of the delta rather than a full clone, exactly as the linked worktree * it replaces did. What changed is who can write what: the run's refs, index * and commits are its own, and no ordinary git operation in the checkout reaches * a sibling run's refs or the mirror. * * `origin` is deliberately the real clone URL and not the mirror path. Pointing * it at the mirror would make a worker's ordinary `git push origin HEAD` * succeed *into the shared object store*, which is precisely the write access * this replacement exists to remove; pointing it upstream makes that push * attempt the network and fail on credentials it does not have, while the * daemon publishes the branch for it (`pushRunBranch`). * * ponytail: reattachment is the only retry mode, so attempt 2 always inherits * attempt 1's tip — including a half-finished or broken state it might rather * start clean from. There is no "start from upstream but keep the old work" * option because that needs somewhere safe to park the old tip first. Upgrade * path: snapshot the branch to `refs/conductor/attempt//` before * resetting the run branch to `base`, and surface both refs in the escalation. */ export async function addRunRepo( repo: RepoTarget, mirrorRoot: string, workspaceRoot: string, issue: number, branch: string, ): Promise<{ path: string; reattached: boolean }> { const mirrorPath = await ensureMirror(repo, mirrorRoot); // 0711: searchable but not listable, so no *other local account* can // enumerate the fleet's checkouts. It does not stop a session reaching a // sibling's tree: sessions run as the daemon's own uid, so the owner bits are // theirs. What confines a worker to its own checkout is the mechanical // worktree gate on its tools, not this mode. mkdirSync(workspaceRoot, { recursive: true, mode: 0o711 }); const runRepo = worktreePathFor(workspaceRoot, issue); if (existsSync(runRepo)) { // Reusing a tree is how one worker silently inherits another attempt's // uncommitted edits and pushes them under this issue's name. throw new Error( `worktree path already exists: ${runRepo}. Refusing to reuse it — ` + `it may hold a previous attempt's uncommitted work. Call ` + `removeWorktree(${mirrorPath}, ${runRepo}) first.`, ); } // The base ref has to be fetched by hand. The mirror's own copy of // `refs/heads/` is whatever the last refresh left behind (and // under normal-clone semantics is no longer refreshed at all), so branching // off it would silently start a run on a stale tip. const base = `refs/remotes/origin/${repo.defaultBranch}`; await git(["fetch", "--no-tags", "origin", `+refs/heads/${repo.defaultBranch}:${base}`], mirrorPath); // Branch names are a deterministic function of the issue number and the // mirror keeps the branch after a run ends, so its presence is what // distinguishes a continuation from a first attempt. The retry *reattaches* // rather than doing either of the two easier things: a fresh // `-attempt2` would break the fleet contract of one issue = one // branch = one PR, and a force-reset to upstream would be irreversible, // because that branch can hold the only copy of work attempt 1 committed but // never pushed. Attaching to it hands the new worker attempt 1's commits. const reattached = await gitSucceeds( ["show-ref", "--verify", "--quiet", `refs/heads/${branch}`], mirrorPath, ); const start = reattached ? `refs/heads/${branch}` : base; try { await git(["init", "--quiet", runRepo]); // Before the first fetch, or git does not know the mirror's objects are // reachable and the fetch below tries to copy the whole history. const info = join(runRepo, ".git", "objects", "info"); mkdirSync(info, { recursive: true }); writeFileSync(join(info, "alternates"), `${join(mirrorPath, "objects")}\n`); // HEAD is moved onto the run branch *after* the fetch, never before: git // refuses to fetch into the branch HEAD is pointing at, even an unborn one, // and the error ("refusing to fetch into branch … checked out at …") reads // like a stale worktree registration rather than an ordering mistake. A // fresh `git init` names its default branch, so the only way to collide is // for the run branch to be called that — cheap to rule out, and silent // breakage for exactly one issue number if it is not. const initialHead = await git(["symbolic-ref", "--quiet", "HEAD"], runRepo); if (initialHead === `refs/heads/${branch}`) { await git(["symbolic-ref", "HEAD", "refs/heads/omp-conductor-bootstrap"], runRepo); } // Fetched by path, from a remote that is never persisted: the run repo must // not keep a handle on the mirror it could push back through. No leading // `+` — the ref does not exist yet, so a fast-forward is trivially // satisfied, and there is no forcing refspec anywhere on a run's own branch. await git(["fetch", "--no-tags", mirrorPath, `${start}:refs/heads/${branch}`], runRepo); // The upstream tracking refs, so the worker can read `origin/main` and diff // against it. These are remote-tracking refs, which git force-updates by // definition; the run branch above is the one that must never be forced. await git( ["fetch", "--no-tags", mirrorPath, "+refs/remotes/origin/*:refs/remotes/origin/*"], runRepo, ); // Added after the fetches so it cannot capture them: `origin` is upstream, // and it exists so an ordinary `git push origin HEAD` fails on credentials // rather than succeeding into the mirror. No upstream is configured for the // run branch, for the reason the old `--no-track` existed: otherwise the // worker's push argues with `push.default` instead of publishing. await git(["remote", "add", "origin", repo.cloneUrl], runRepo); await git(["symbolic-ref", "HEAD", `refs/heads/${branch}`], runRepo); await git(["reset", "--hard"], runRepo); // The mirror's `info/exclude` is no longer this repo's common dir, so the // managed block has to be written here. Without it a worker's own scratch // reaches its `git add -A`, and salvage inherits it (#44). const exclude = join(runRepo, ".git", "info", "exclude"); mkdirSync(dirname(exclude), { recursive: true }); writeFileSync(exclude, mergeExclude(existsSync(exclude) ? readFileSync(exclude, "utf8") : "")); } catch (err) { // A half-provisioned repo would be seen as "present" by the next attempt // and refused as possibly holding work, stranding the issue. It holds // nothing yet — the branch's durable copy is in the mirror — so clear it. rmSync(runRepo, { recursive: true, force: true }); throw new Error( `failed to provision run repository ${runRepo} for issue ${issue} on branch ${branch} ` + `(${reattached ? "reattaching an existing branch" : "creating a new branch"}): ` + `${err instanceof Error ? err.message : String(err)}`, ); } return { path: runRepo, reattached }; } /** * Publishes one run branch: run repo → mirror → GitHub, fast-forward only. * * Declared here as a function type rather than imported as a concrete * implementation so this module stays free of the credential path: provisioning * and salvage decide *what* to publish, `gitops.ts` is the only place that * decides *how*, and the daemon is the only thing that holds both. */ export type RunPublisher = ( branch: string, ) => Promise<{ ok: true; sha: string } | { ok: false; stderr: string }>; /** * What one salvage attempt did. A failure is a *value* rather than a throw * because the only caller is a run that is already ending badly: it has to log * the outcome, word it into the escalation and still close its run record, and * none of that may be skipped by an exception from the last-ditch step. */ export type SalvageOutcome = | { kind: "salvaged"; sha: string; branch: string; pushed: boolean; pushError?: string; /** Paths in the salvage commit (count = length). */ files: string[]; /** Subset that were untracked before salvage (`A` in the commit). */ newPaths: string[]; } /** Nothing uncommitted was there to save — a clean tree, or no tree at all. */ | { kind: "nothing" } /** There was work and git would not commit it. This is the loud one. */ | { kind: "failed"; error: string }; /** * The salvage commit is the daemon's own, made unattended in a worktree whose * config it does not control. A machine with no global git identity, a global * `commit.gpgsign=true` whose key needs a passphrase nobody can type, or a repo * pre-commit hook that rejects half-finished code are all ordinary states — and * every one of them would turn "save the work" into "lose the work". So the * commit brings its own identity, signs nothing, and skips hooks: it is a * snapshot for a human to sort out, never something anyone merges. */ const SALVAGE_COMMIT_CONFIG = [ "-c", "user.name=conductor", "-c", "user.email=conductor@invalid", "-c", "commit.gpgsign=false", ]; /** * Subject stays the historical one-liner so status greps keep working; the * body is the manifest (#38). Cap the new-path list so a runaway tree cannot * push a multi-kilobyte commit message into every escalation. * * `ending` is the whole clause rather than a bare reason because not every * salvaged run was killed. A run that blocked for a decision stopped on * purpose (#118), and a commit message telling the operator reading it during * recovery that the attempt was "killed by a blocked run" is a lie about the * one artefact they are using to reconstruct what happened. */ export function salvageCommitMessage( issue: number, attempt: number, ending: string, files: string[], newPaths: string[], ): { subject: string; body: string } { const subject = `wip(#${issue}): attempt ${attempt} ${ending} — auto-salvaged`; const n = files.length; const count = `${n} file${n === 1 ? "" : "s"}`; if (newPaths.length === 0) { return { subject, body: `${count} (all modifications to tracked paths).` }; } const cap = 30; const shown = newPaths.slice(0, cap); const more = newPaths.length - shown.length; const list = shown.map((f) => ` ${f}`).join("\n"); const suffix = more > 0 ? `\n … and ${more} more` : ""; return { subject, body: `${count}. New (untracked before salvage):\n${list}${suffix}`, }; } function parseCachedNameStatus(raw: string): { files: string[]; newPaths: string[] } { const files: string[] = []; const newPaths: string[] = []; for (const line of raw.split("\n")) { if (line === "") continue; // name-status: "\t" or rename "R100\t\t" const parts = line.split("\t"); const status = parts[0] ?? ""; const path = parts.length >= 3 ? (parts[2] ?? parts[1] ?? "") : (parts[1] ?? ""); if (path === "") continue; files.push(path); if (status.startsWith("A")) newPaths.push(path); } return { files, newPaths }; } /** * Commits a run's uncommitted work to the run's own branch and pushes it, so * that the tree the next attempt destroys is no longer the only copy. * * This closes a deliberate asymmetry. `addRunRepo` preserves the run branch * precisely because "that branch can hold the only copy of work attempt 1 * committed but never pushed", while `removeWorktree` runs `worktree remove * --force` and `addRunRepo` refuses to reuse a tree that "may hold a previous * attempt's uncommitted work" — committed work is kept by design, uncommitted * work was discarded by design. * * That trade was first taken only for non-graceful ends, on the argument that * a worker which *stops* had turns left to commit for itself. #118 is what * that argument cost. veltro#349 blocked on a real decision — whether a * failing test was obsolete — after editing 34 files across a request/auth * boundary, and the daemon force-removed the tree seconds later; the branch * and `origin/main` still pointed at the same commit, and the session * transcript was the only surviving evidence the work had ever existed. A * worker asking permission is precisely a worker declining to commit a * half-migrated tree, so "had turns to commit" and "should have committed" * were never the same claim. Every end whose tree is about to be removed * salvages: killed, crashed, orphaned, and blocked alike. * * Never throws. Every outcome, including its own failure, comes back as a value * for the caller to log and to put in front of a human. * * Publishing is best-effort and deliberately last, after the sha exists: a * refused push (diverged branch, no credentials, no network) still leaves the * commit in the run's own repository, which the daemon can still read, and * which is strictly better than nothing. It is a plain fast-forward push — * never a force — and if the run already had a PR open, that PR gains the WIP * commit and re-runs its checks. That is the price of work outliving its host. * * `publish` is a **required** parameter rather than an optional one with a * no-op default, and that is the whole design. The salvage commit does not push * for itself: the network hop is the daemon's, through `pushRunBranch`, so the * sha that reaches the remote is the sha the dispatcher recorded. A defaulted parameter * is how a future call site silently stops publishing salvaged work and nobody * finds out until the host that held it is gone — which is #121's failure with * one extra step. Passing `undefined` is allowed, and is a visible decision at * the call site. */ export async function salvageWip( worktree: string, issue: number, attempt: number, ending: string, publish: RunPublisher | undefined, ): Promise { try { // A tree that is not there cannot be holding work. Checked before spawning // git, because a missing cwd fails at spawn time rather than as an exit // code, and "no tree" is not a salvage failure worth alarming anyone with. if (!existsSync(worktree)) return { kind: "nothing" }; // Heal the managed ignore *before* status/add. A mirror left by an older // build may still list patterns this release dropped; without this, an // untracked deliverable matching the stale list is invisible to salvage // and dies with the next `worktree remove --force` (#44). refreshManagedExclude(worktree); // The tree is worker-owned by the time salvage runs (#798): every daemon // git call against it carries the exact per-run ownership exemption — // never a wildcard, never a global config entry (#816). const exempt = runRepoSafeDirectoryExemption(worktree); if ((await git([...exempt, "status", "--porcelain"], worktree)) === "") return { kind: "nothing" }; // The tree's own branch, not one the caller believes it should be on: this // string ends up in an escalation as the place to go looking. const branch = await git([...exempt, "rev-parse", "--abbrev-ref", "HEAD"], worktree); // `-A` on purpose: the losses this exists for were mostly *new* files. // // Filtering scratch is git's job, not a pathspec's. The mirror's // `info/exclude` (see {@link mergeExclude}) is the common git dir for this // worktree, and git's own rules then give exactly the semantics needed: // untracked ignored files are skipped, while modifications to *tracked* // files are staged even when the name matches an ignore. That second half // is why an exclude pathspec here was wrong — it matched on filename alone, // so a repo legitimately versioning a `bootstrap.local.sh` would have lost // a worker's edits to it, salvage destroying the work it exists to save. // // A literal `:(exclude)` was also an outright bug: git counts it as // naming the path, so an already-ignored file made `add` exit 1 and every // cap-kill would have reported a salvage *failure*. await git([...exempt, "add", "-A"], worktree); // The dirty check above ran before git applied its ignores, so a tree whose // only changes were ignored scratch had work by that test and none by this. // Without this, `commit` exits non-zero on an empty index and a tree // holding nothing worth keeping gets reported as a salvage *failure*. const cached = await git([...exempt, "diff", "--cached", "--name-status"], worktree); if (cached === "") { return { kind: "nothing" }; } const { files, newPaths } = parseCachedNameStatus(cached); const msg = salvageCommitMessage(issue, attempt, ending, files, newPaths); await git( [ ...exempt, ...SALVAGE_COMMIT_CONFIG, "commit", "--no-verify", "-m", msg.subject, "-m", msg.body, ], worktree, ); const sha = await git([...exempt, "rev-parse", "HEAD"], worktree); const salvaged = { kind: "salvaged" as const, sha, branch, files, newPaths }; if (branch === "HEAD") { // Detached: the commit is real but reachable only by sha, and pushing // `HEAD` from here would publish a branch literally named HEAD. return { ...salvaged, pushed: false, pushError: "detached HEAD — no branch to push", }; } if (publish === undefined) { return { ...salvaged, pushed: false, pushError: "no publisher was supplied — the commit exists in the run's own repository and nowhere else", }; } const published = await publish(branch); if (published.ok) return { ...salvaged, pushed: true }; return { ...salvaged, pushed: false, pushError: scrubUserinfo(published.stderr) }; } catch (err) { return { kind: "failed", error: err instanceof Error ? err.message : String(err) }; } } /** * Removes a run's worktree and its registration in the mirror. Idempotent: a * path that is already gone resolves, so cleanup can be retried and can run on * a run that never got as far as a checkout. * * ponytail: the run's branch stays in the mirror's `refs/heads/*`, since it may * be the only copy of work that has not been pushed. Mirrors therefore grow one * ref per run; upgrade path is reaping branches that are merged or have a * closed PR. */ export async function removeWorktree( mirrorPath: string, worktreePath: string, ): Promise { if (!existsSync(mirrorPath)) { // No mirror means no registration left to clean up, and deleting a tree we // can no longer prove we created is not this function's call. return; } if (existsSync(worktreePath)) { // A run repo is a plain directory now, so removal is a plain removal — but // the `worktree remove` is still attempted first, because a fleet upgrading // from a release that used linked worktrees has live trees that ARE linked // worktrees of this mirror, and // deleting one of those without deregistering it leaves the mirror // believing the branch is still checked out. It refuses `worktree add` and, // worse, refuses to delete the branch at cleanup. The failure is expected // for a run repo and is ignored; what is not tolerated is a path that // survives, since the next attempt for this issue would trip over it. try { await git(["worktree", "remove", "--force", worktreePath], mirrorPath); } catch { // Not a registered worktree — the normal case from here on. } rmSync(worktreePath, { recursive: true, force: true }); if (existsSync(worktreePath)) { throw new Error(`failed to remove run repository ${worktreePath}: the path still exists`); } } await git(["worktree", "prune"], mirrorPath); } export type RetainedWorktreeCleanup = | { kind: "removed" } | { kind: "retained"; reason: "dirty" | "unpushed" | "unknown" | "quarantined"; detail: string; }; /** * Repairs a run repository whose `objects/info/alternates` names a path that no * longer exists, re-pointing the dangling entry at this project's current * mirror. Worktrees created before mirrors moved into per-project roots keep * the old flat path (`/mirrors/.git`); the store they borrow from * is gone, so any fetch against them dies on unresolved deltas (#728). The * repair belongs here — in the code that discovers the stale entry — rather * than in a one-off host edit, so the next layout migration heals itself too. * * Only missing entries are rewritten: a valid alternate is left alone and a * worktree with no alternates file is already sound. When the store cannot be * made sound — the git dir is unresolvable, the file cannot be read or * written, or a missing path survives the rewrite — the caller must not fetch * into the broken state; a quarantine detail is returned instead. */ async function repairAlternates( worktreePath: string, mirrorPath: string, ): Promise<{ kind: "ok" } | { kind: "quarantine"; detail: string }> { // The tree is worker-owned by cleanup time (#798); the exact-path exemption // makes this read the daemon's own (#816). const common = await runGit( [...runRepoSafeDirectoryExemption(worktreePath), "rev-parse", "--git-common-dir"], worktreePath, ); if (common.code !== 0) { return { kind: "quarantine", detail: `cannot resolve the git dir: ${common.stderr.trim() || common.stdout.trim() || "no output"}`, }; } const raw = common.stdout.trim(); if (raw === "") { return { kind: "quarantine", detail: "cannot resolve the git dir: git printed nothing" }; } // `--git-common-dir` is relative for linked worktrees; resolve against the // tree so bare-mirror layouts and plain clones both land on the alternates. const commonDir = raw.startsWith("/") ? raw : join(worktreePath, raw); const alternates = join(commonDir, "objects", "info", "alternates"); if (!existsSync(alternates)) return { kind: "ok" }; const current = join(mirrorPath, "objects"); let content: string; try { content = readFileSync(alternates, "utf8"); } catch (err) { return { kind: "quarantine", detail: `${alternates} cannot be read: ${err instanceof Error ? err.message : String(err)}` }; } let changed = false; const rewritten = content.split("\n").map((line) => { const candidate = line.trim(); if (candidate === "" || candidate === current || existsSync(candidate)) return line; changed = true; return current; }); if (changed) { try { writeFileSync(alternates, rewritten.join("\n")); } catch (err) { return { kind: "quarantine", detail: `${alternates} cannot be rewritten: ${err instanceof Error ? err.message : String(err)}` }; } } const dangling = rewritten.map((line) => line.trim()).filter((path) => path !== "" && !existsSync(path)); if (dangling.length > 0) { return { kind: "quarantine", detail: `${alternates} still names missing path(s): ${dangling.join(", ")}`, }; } return { kind: "ok" }; } /** * Reap a terminal run's tree and local mirror branch without deleting the only * copy of work. Tracker state is proved by the caller; this function proves the * local half after refreshing remote refs. Any ambiguity retains everything; * a remote read that failed is ambiguity, never a pushed/unpushed claim, and * the unpushed verdicts below are reachable only from reads that succeeded. */ export async function cleanupRetainedWorktree( mirrorPath: string, worktreePath: string, branch: string, ): Promise { if (!existsSync(mirrorPath)) { return existsSync(worktreePath) ? { kind: "retained", reason: "unknown", detail: "mirror is missing" } : { kind: "removed" }; } try { if (existsSync(worktreePath)) { // A run repo created before mirrors moved into per-project roots may // borrow objects from a path that no longer exists. Fetching into it is // a guaranteed failure, so repair the alternates first — and when the // store cannot be made sound, quarantine the tree instead of fetching // into the broken state. const repair = await repairAlternates(worktreePath, mirrorPath); if (repair.kind === "quarantine") { return { kind: "retained", reason: "quarantined", detail: `worktree quarantined: ${repair.detail}`, }; } // The tree is worker-owned by cleanup time (#798): every daemon git call // against it carries the exact per-run exemption (#816). const exempt = runRepoSafeDirectoryExemption(worktreePath); const dirty = await git([...exempt, "status", "--porcelain"], worktreePath); if (dirty !== "") { return { kind: "retained", reason: "dirty", detail: "worktree has uncommitted changes" }; } const actual = await git([...exempt, "rev-parse", "--abbrev-ref", "HEAD"], worktreePath); if (actual !== branch) { return { kind: "retained", reason: "unknown", detail: `worktree is on ${actual}, expected ${branch}`, }; } } // A deleted remote branch can make a pushed commit look local-only until // the default branch is fetched. A failed fetch is ambiguity, never // permission: without fresh remote refs the unpushed verdicts below are // unreachable and the run is retained on an explicit unknown. try { await git(["fetch", "--prune", "origin"], mirrorPath); } catch (err) { return { kind: "retained", reason: "unknown", detail: `remote refs could not be refreshed: ${err instanceof Error ? err.message : String(err)}`, }; } const ref = `refs/heads/${branch}`; // **Ask the run's own repository first.** Its commits live in its own // object store, not the mirror's, so a mirror-only check cannot even see // them — it would answer "nothing unpushed" about a branch whose only copy // is the directory this function is about to delete. That is the exact // data loss #121 exists to prevent, reintroduced by the move to per-run // repositories, so it is checked where the objects actually are. if (existsSync(worktreePath)) { try { await git( [...runRepoSafeDirectoryExemption(worktreePath), "fetch", "--no-tags", mirrorPath, "+refs/remotes/origin/*:refs/remotes/origin/*"], worktreePath, ); } catch (err) { return { kind: "retained", reason: "unknown", detail: `the run repository's remote refs could not be read: ${err instanceof Error ? err.message : String(err)}`, }; } const runUnique = await git([...runRepoSafeDirectoryExemption(worktreePath), "rev-list", ref, "--not", "--remotes"], worktreePath); if (runUnique !== "") { return { kind: "retained", reason: "unpushed", detail: `${branch} has commits in the run repository that are absent from every remote ref`, }; } } if (await gitSucceeds(["show-ref", "--verify", "--quiet", ref], mirrorPath)) { const unique = await git(["rev-list", ref, "--not", "--remotes"], mirrorPath); if (unique !== "") { return { kind: "retained", reason: "unpushed", detail: `${branch} has commits absent from every remote ref`, }; } } await removeWorktree(mirrorPath, worktreePath); if (await gitSucceeds(["show-ref", "--verify", "--quiet", ref], mirrorPath)) { await git(["branch", "-D", branch], mirrorPath); } return { kind: "removed" }; } catch (err) { return { kind: "retained", reason: "unknown", detail: err instanceof Error ? err.message : String(err), }; } }