/** * Worker panes: reconciling the terminal representation of a run against the * run rows, and answering who owns a workspace. * * A pane is a *representation* of a run, not the run itself — the session lives * or dies whether or not a pane ever opened. That is exactly why this is its own * module: pane reconciliation is a convergence loop with its own bounded * reattempts, and the dispatch path must be able to launch, settle and hand back * an issue without ever depending on it having converged. * * `workspaceOwnership` sits here because a pane and a run agree on a workspace * or they do not, and all three callers — `handleIssue`, this reconcile, and * `runDaemon`'s startup sweep — must ask the question exactly once, the same way. */ import { PANE_REATTEMPT_MAX, reconcileWorkerPanes, type WorkspaceOwnership } from "../fleet.ts"; import { log } from "../log.ts"; import type { RunRecord } from "../types.ts"; import type { Deps } from "./deps.ts"; /** * Pane re-establishment attempts spent per run, for this daemon process only * (#998). * * Deliberately in memory rather than on the run row: a restart is exactly the * event that makes another attempt worth making (Herdr came back), so a fresh * process legitimately gets a fresh budget, and giving up never marks a run * permanently unrepresentable. Pruned against the live set on every pass, so it * cannot outgrow the fleet. */ export const paneAttemptsSpent = new Map(); /** The store-backed {@link WorkspaceOwnership} adapter (#1035 review): the * conductor store is the durable leg of workspace discovery, surviving both * daemon and Herdr restarts. One shape for every caller so they cannot * disagree about what ownership reads. */ export function workspaceOwnership(store: Deps["store"]): WorkspaceOwnership { return { recordedWorkspaces: (project) => store.workerWorkspaceIds(project), rememberWorkspace: (project, workspaceId) => store.rememberWorkerWorkspace(project, workspaceId), forgetWorkspace: (project, workspaceId) => store.forgetWorkerWorkspace(project, workspaceId), }; } /** Exported for the lifecycle tests (#1035): the pass is pure over the store * and the injected Herdr runner, so a test drives it exactly as the tick does. */ /** * Make the worker workspaces agree with the live run set, once per pass * (#841, reworked by #1035). * * Runs on every dispatch pass rather than only at startup, because the thing it * repairs — a Herdr restart — is not a conductor event and announces itself * nowhere. It is idempotent by construction: a reconciled fleet reports `intact` * for every live worker and finds nothing left to close, so repeated passes * converge instead of accumulating panes or churning them. This pass owns this * project's worker workspace even when no run is live — that is how a settled-out * project's stale shells converge to zero and its empty workspace is removed. * * Nothing here can stop a worker: panes hold followers only; the authoritative * child lives in this daemon's process tree and is never signalled. */ export function reconcilePanes( d: Pick, project: string, log: (message: string) => void, ): void { const live = d.store.liveRuns(project); const liveKeys = new Set(live.map((run) => `${project}\u0000${run.id}`)); for (const key of [...paneAttemptsSpent.keys()]) { if (key.startsWith(`${project}\u0000`) && !liveKeys.has(key)) paneAttemptsSpent.delete(key); } const attempts = new Map( live.map((run) => [run.id, paneAttemptsSpent.get(`${project}\u0000${run.id}`) ?? 0]), ); const result = reconcileWorkerPanes( live.map((run) => ({ runId: run.id, issue: run.issue, attempt: run.attempt, project, ...(run.workerPid === undefined ? {} : { pid: run.workerPid }), ...(run.paneId === undefined ? {} : { paneId: run.paneId }), ...(run.paneLabel === undefined ? {} : { paneLabel: run.paneLabel }), ...(run.sessionFile === undefined ? {} : { sessionFile: run.sessionFile }), })), { ...(d.herdrRun === undefined ? {} : { run: d.herdrRun }), ownership: workspaceOwnership(d.store) }, attempts, PANE_REATTEMPT_MAX, [project], ); if (!result.ok) { // An unreadable workspace is not evidence that anything is stale, so nothing // is released and no run is relabelled — but a fleet that cannot see its own // panes says so, on every pass, rather than going quiet. log(`worker panes not reconciled: ${result.reason}`); return; } for (const outcome of result.outcomes) { switch (outcome.kind) { case "intact": break; case "visual-unreadable": // The pane keeps standing: an unreadable follower table is no evidence // of death, and a live worker's representation is never destroyed on a // guess (#1035 review). Said once per pass, not acted on. log( `#${runIssue(live, outcome.runId)} herdr pane ${outcome.paneId} visual unreadable, left standing: ${outcome.reason}`, ); break; case "reassociated": d.store.updateRun(outcome.runId, { paneId: outcome.paneId, paneLabel: outcome.label, paneUnavailable: null, }); log(`#${runIssue(live, outcome.runId)} herdr pane re-associated as ${outcome.paneId} after a restart`); break; case "untracked": { d.store.updateRun(outcome.runId, { paneId: null, paneLabel: null, paneUnavailable: outcome.reason }); if (outcome.attempted !== true) { // Cost no attempt (no recorded pid): say so every pass, as before. log(`#${runIssue(live, outcome.runId)} has no herdr pane: ${outcome.reason}`); break; } // #998: a terminal refusing to split will refuse the next one too, and // each retry made the pile worse — spend the whole budget at once. The // phase is structured now (#1035 review): the old substring sniff died // when the reason strings changed and never fired again. const key = `${project}\u0000${outcome.runId}`; const spent = outcome.phase === "split" ? PANE_REATTEMPT_MAX : (paneAttemptsSpent.get(key) ?? 0) + 1; paneAttemptsSpent.set(key, spent); // Once per run, at the moment the budget runs out — not once per pass. // Four identical lines a minute for #986 is what made the real cause // (a leaked pane per attempt) hard to see on 2026-08-23. log( spent >= PANE_REATTEMPT_MAX ? `#${runIssue(live, outcome.runId)} has no herdr pane after ${spent} attempt(s), not retrying: ${outcome.reason}` : `#${runIssue(live, outcome.runId)} has no herdr pane (attempt ${spent}/${PANE_REATTEMPT_MAX}): ${outcome.reason}`, ); break; } case "attempts-exhausted": // Already reported when the budget ran out; the run keeps working with // no pane, and a later restart gets a fresh budget. break; case "stale-released": { const what = outcome.cause === "duplicate" ? "closed duplicate herdr pane" : outcome.cause === "unidentified" ? "closed unidentified herdr pane" : "released settled herdr pane"; log(`${what} ${outcome.paneId} (run ${outcome.runId || "unidentified"}${outcome.cause === "settled" ? " is not live" : ""})`); break; } case "stale-release-failed": { const what = outcome.cause === "duplicate" ? "duplicate herdr pane" : outcome.cause === "unidentified" ? "unidentified herdr pane" : "settled herdr pane"; log(`${what} ${outcome.paneId} could not be closed: ${outcome.reason}`); break; } case "workspace-removed": log(`removed empty herdr worker workspace ${outcome.workspaceId} for ${outcome.project}`); break; case "workspace-remove-failed": log( `herdr worker workspace ${outcome.workspaceId} for ${outcome.project} could not be removed: ${outcome.reason}`, ); break; } } } /** The issue a run id belongs to, for a log line a human reads. */ export function runIssue(live: readonly RunRecord[], runId: string): string { return String(live.find((run) => run.id === runId)?.issue ?? "?"); }