import { chmodSync, existsSync, lstatSync, mkdirSync, readFileSync, readlinkSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync, type Stats } from "node:fs"; import { spawnSync } from "node:child_process"; import { homedir, userInfo } from "node:os"; import { dirname, join, relative, resolve, sep } from "node:path"; import { configPath, loadConfig, resolveCaps, stateDir } from "./config.ts"; import { isPaused, runDaemon, statusSnapshot, type StatusSnapshot } from "./daemon.ts"; import { AGENTS_BRIEF_NAME, CONSOLE_BRIEF_NAME, ORCHESTRATOR_BRIEF_NAME } from "./brief-upgrade.ts"; import { planConsole, writeConsole, type ConsolePlan } from "./setup.ts"; import { DEFAULT_FLEET_AGENT_NAME, DEFAULT_HERDR_SESSION, DEFAULT_HERDR_UNIT, resolveHerdrSession, } from "./fleet.ts"; import { WORKER_HARNESS_DIR, packageNodeModulesRoot, } from "./host.ts"; import { DEFAULT_PORT, healthCheck, livingDaemon, probeUnit, startDaemon, stopDaemon, SYSTEMD_UNIT, type DaemonRecord, type StopResult, type UnitOwnership, } from "./lifecycle.ts"; import { legacyArmedMarkerPath, readTickConfig, parseHerdrAgents, TICK_CONFIG_FILE, tickConfigMatchesProject, type HerdrAgentList, type TickConfig, } from "./orchestrator-tick.ts"; import { formatStep, type PrivilegedStep } from "./privileged.ts"; import type { Caps, ConductorConfig, ProjectConfig } from "./types.ts"; export const DEFAULT_TICK_INTERVAL_SECONDS = 900; export const STAGED_SERVICE_NAME = "omp-conductor.service"; export const SYSTEMD_UNIT_DIR = "/etc/systemd/system"; /** * The system directories every rendered service PATH ends with, in systemd's * own default order: git, gh and the other tools the fleet spawns by name * live here on every target host. The service PATH as a whole is canonical * (#879) — these directories plus the resolved fleet-binary directories — so * `setup host` stages and `doctor` compares one value, whichever process * renders it and whatever the invoking shell happened to carry. */ export const SERVICE_SYSTEM_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; /** * The one recovery unit every fleet unit's `OnFailure=` points at (#485). * * A plain (non-template) unit on purpose: both fleet units name the same * oneshot, and the playbook it runs determines which unit failed from live * systemd state rather than trusting an instance name it cannot verify. The * unit itself never carries `OnFailure=` — the one hard bound on * recovery-triggering-recovery. */ export const RECOVER_SERVICE_NAME = "omp-conductor-recover.service"; /** The playbook file shipped in the package's `systemd/` directory. */ export const RECOVER_SCRIPT_FILE = "omp-conductor-recover.sh"; /** * Where `setup host` installs the recovery playbook so the unit's ExecStart is * a stable absolute path that survives package upgrades. /usr/local/sbin is * root-executed operational tooling, on PATH for root on every target host. */ export const RECOVER_SCRIPT_INSTALL_PATH = "/usr/local/sbin/omp-conductor-recover"; /** * The mount unit releases 0.18.1-era installed to bind the operator's install * read-only for a separate worker account (#828), kept **only** so this * version can retire it (#895). * * Nothing renders or installs it any more: worker sessions launch under the * fleet account again (#894), so the bind serves nobody, and a `.mount` unit * an earlier release enabled stays enabled until something disables it. This * constant is that something's target — see {@link HostRuntimePlan.retire}. * * The name is not a choice: systemd derives a mount unit's name from its mount * point and refuses to load one under any other, so this is the escaped form of * that path — `/` becomes `-`, and a literal `-` inside a component becomes * `\x2d` (what `systemd-escape -p --suffix=mount` prints). */ export const HARNESS_MOUNT_UNIT_NAME = "var-lib-omp\\x2dworker\\x2dharness-node_modules.mount"; /** * Host state a previous release installed that this one retires (#895). * * Kept as one value rather than three so a caller cannot execute the steps * without being able to say what they retire (the CLI prints `units`/`staged`, * the reconcile reports them, and `formatHostRuntimePlan` shows them before * the confirm). */ export interface HostRetirement { /** Installed unit paths this version no longer ships, still on disk. */ units: readonly string[]; /** Staged renders of those units still sitting in the state directory. */ staged: readonly string[]; /** The ordered steps that retire them: disable/unmount, remove, reload. */ steps: readonly PrivilegedStep[]; } export type PlannedWrite = { path: string; action: "create" | "update" | "keep"; content: string; value: T; }; /** * The `AGENTS.md` symlink setup places in a project's fleet cwd so the pane's * session auto-loads the composed brief instead of staying brief-less. * * {@link action} is what the plan will *do*, compared against what sits at * {@link path}: * - `create` — no entry yet, link it; * - `update` — a symlink occupies {@link path} but points elsewhere, replace it; * - `keep` — an already-correct symlink, leave untouched (idempotent re-run); * - `skip` — a regular file occupies {@link path}, never overwrite an * operator's file (see {@link skippedReason}). */ export interface BriefLinkPlan { /** `AGENTS.md` inside the project's fleet cwd ({@link tickCwdForProject}). */ path: string; /** Relative target inside the symlink — what the link resolves to. */ target: string; action: "create" | "update" | "keep" | "skip"; /** Why {@link action} is `skip`; present only then. */ skippedReason?: string; } /** * Why a host-global install (no `--project`) wrote no per-project tail, and * what the operator does about it. */ export interface HostRuntimeNoProject { /** The project-scoped files the host-global install deliberately left unwritten. */ skipped: readonly string[]; /** The command that writes them: `setup host` naming one project. */ how: string; } export interface HostRuntimePlan { service: PlannedWrite; /** * The staged Herdr session-server unit. Absent when Herdr is not installed * ({@link ServiceRuntime.herdr} is `undefined`) — a host driving Herdr some * other way must stay supported. `upgrade` and `start` treat * {@link DEFAULT_HERDR_UNIT} as a first-class dependency, and this is the * rendering that provisions it (see {@link renderHerdrUnit}). */ herdrUnit?: PlannedWrite; /** * The fleet recovery oneshot (#485): staged always, because the daemon unit * (which every fleet has) carries `OnFailure=` to it. Rendered by * {@link renderRecoverUnit}. */ recoverUnit: PlannedWrite; /** * The recovery playbook body, staged from the shipped package file and * installed to {@link RECOVER_SCRIPT_INSTALL_PATH} so the unit's ExecStart * is a stable absolute path. */ recoverScript: PlannedWrite; /** * `[terminal] default_shell` staged into the session's herdr config, the * belt-and-braces behind the unit's `SHELL=` (see {@link renderHerdrConfig}). * Present exactly when {@link herdrUnit} is, unless the login shell is not a * usable path ({@link usableHerdrShell}) or the rendered config would not * parse — either one leaves {@link herdrConfigProblem} set and writes * nothing. * * This is the **staged** copy, under the state directory like every other * file the plan writes: herdr's live config is a file conductor does not own, * so it is only replaced post-consent by the install step naming * {@link herdrConfigTarget} (#513). */ herdrConfig?: PlannedWrite; /** The live herdr config {@link herdrConfig} merges into, post-consent. */ herdrConfigTarget?: string; /** Why no pane-shell key is planned (unusable login shell, …). */ herdrConfigProblem?: string; /** * A loud note when the pane-shell merge kept an explicit * `resume_agents_on_restore = true` — {@link renderHerdrConfig} preserves * that value on purpose (it may be a desktop host), and the plan says so * rather than silently overriding a deliberate choice. */ herdrConfigWarning?: string; /** * The herdr-conductor plugin's `config.env`, staged like the pane-shell * config and merged into the live plugin config dir post-consent: the file * recover.sh sources (`herdr/bin/recover.sh`) — TARGET_SESSION, FLEET_CWDS * (derived from every configured project), TELEGRAM_ENV, ACCESS_JSON. * Present exactly when {@link herdrUnit} is; conductor-owned keys are * patched in place, operator keys survive untouched. */ herdrEnv?: PlannedWrite; /** The live plugin config.env {@link herdrEnv} merges into, post-consent. */ herdrEnvTarget?: string; /** * The Herdr session this host's unit serves, present exactly when * {@link herdrUnit} is. Carried on the plan because the install has to know * which session to prove supervision of (#893), and re-deriving it there * would let the unit's `--session` and the session probed disagree. */ herdrSession?: string; /** * When {@link planTick} rewrote a shared-default `agentName`, the live herdr * pane still carries the old identity — the ticking decline that names the * fix (`herdr agent rename `). Setup performs the rename after * the install when the session is reachable and exactly one pane matches; * anything else surfaces the exact command (never guesses a pane). */ agentRename?: { session: string; from: string; to: string }; tick?: PlannedWrite; /** The `AGENTS.md` — composed-brief symlink — placed in the fleet cwd. */ briefLink?: BriefLinkPlan; /** * The operator console's workspace: its own cwd under the state root, the * rendered console floor, and the `AGENTS.md` link that makes the session * load it. Planned per project like {@link briefLink}, and deliberately * carrying no tick config — the console cwd must stay a directory the tick * extension cannot activate in. */ console?: ConsolePlan; /** * Set when an install ran with no project named (a host-global install): * the per-project tail was deliberately not written. Names the files and * how to write them, so the operator is told what is missing and how to get * it rather than silently installing host-global units that leave a project * brief-less and tick-less. */ noProject?: HostRuntimeNoProject; /** * The exact privileged steps this plan will run, in order — recovery first, * so the daemon unit that follows names a recovery unit systemd already has. * This is the single list: {@link installCommands} and `runHostInstall` are * both derived from it, so a step can never appear in the printed plan and * be absent from the run (#509). */ steps: readonly PrivilegedStep[]; /** * The same steps as `sudo …` shell lines, for humans to read. A projection * of {@link steps} rather than a parallel list, so nothing executes a step * the printed plan does not show. */ installCommands: readonly string[]; cliSource: "global" | "plugin"; /** Absolute path of the unit systemd actually reads. */ installedPath: string; /** * What installing would do to the unit **systemd reads**, which is a different * question from {@link HostRuntimePlan.service}'s action — that one compares the * staged copy under the state directory. * * The distinction is load-bearing: staging has always happened during setup, * so on any fleet configured before the install was executed the staged file is * already current (`service.action === "keep"`) while `/etc/systemd/system` * holds nothing at all. Gating the install offer on the staged action therefore * skipped exactly the fleets that had never installed the unit, and told them * the installed unit matched. */ installedAction: PlannedWrite["action"]; /** * The installed destinations whose live bytes differ from this version's * render — exactly the files the privileged steps would rewrite. `upgrade` * names them after a release that re-rendered a template, and * {@link currentInstall} is `true` iff this list is empty, so the two can * never disagree (#598). */ drift: readonly string[]; /** * Host state this version no longer ships but a previous one installed, and * the ordered steps that retire it (#895). Absent when there is nothing to * retire, which is every host that never ran the separate worker boundary * and every host that has already been converged. * * Retirement is planned rather than merely dropped because deleting the * provisioning code cannot unmount anything: the `.mount` unit an earlier * release enabled stays enabled across upgrades, and a bind of the * operator's install would keep being mounted at a path nothing reads. So * the steps run FIRST in {@link steps}, and while any of them are owed * {@link currentInstall} is false — a re-run that reported "already current" * with the obsolete mount still active is exactly the half-applied * retirement this exists to prevent. */ retire?: HostRetirement; /** * True when every file the privileged install steps would write is already * at its destination with the current bytes. `runHostInstall` uses it to * make a no-op re-run of `setup host` neither restage nor restart anything. */ currentInstall: boolean; } export interface ServiceRuntime { username: string; home: string; path: string; bun: string; cli?: string; packageCli: string; conductorHome: string; telegramStateDir: string; /** * The account's login shell. Rendered into the Herdr session unit as * `Environment=SHELL=` (and into the herdr config as `[terminal] * default_shell`), so every pane in the fleet session runs this shell instead * of the `/bin/sh` Herdr falls back to when `$SHELL` is unset (the fleet's * dash-pane symptom, #456). A systemd service has no `$SHELL`, so this is * load-bearing, not decoration. */ shell: string; /** * Absolute path of the `herdr` binary on this host, or `undefined` when Herdr * is not installed. When absent, setup stages no Herdr unit and no pane-shell * config — a host driving Herdr some other way stays supported (#456). */ herdr?: string; /** * Non-default Herdr session to pin into the unit (#394). Omitted when the * host uses {@link DEFAULT_HERDR_SESSION}: an always-present line would show * as drift on every existing host and add a redundant default. */ herdrSession?: string; } function systemdQuote(value: string): string { if (/\r|\n/.test(value)) throw new Error("systemd values cannot contain newlines"); return `"${value.replaceAll("\\", "\\\\").replaceAll('"', '\\"').replaceAll("%", "%%")}"`; } /** * A bare path for the settings that take exactly one, never a quoted word list. * * `Environment=` and `ExecStart=` unquote their values, so {@link systemdQuote} * is right there. `WorkingDirectory=` does not: systemd reads the whole value as * the path, so a quoted one fails `systemd-analyze verify` with "path is not * absolute" and the unit refuses to start — a landmine that only fires on the * next restart, because the running daemon keeps the unit it started with. * Spaces need no quoting here; `%` still does, or it reads as a specifier. */ function systemdPath(value: string): string { if (/\r|\n/.test(value)) throw new Error("systemd values cannot contain newlines"); return value.replaceAll("%", "%%"); } function actionFor(path: string, content: string): PlannedWrite["action"] { if (!existsSync(path)) return "create"; try { return readFileSync(path, "utf8") === content ? "keep" : "update"; } catch { return "update"; } } /** * Which account this host's fleet belongs to, and how we know. * * Two sources, in order of authority: the installed unit's own `User=` is what * systemd will actually run as, and the owner of `$OMP_CONDUCTOR_HOME` is what * owns the config, the store and the state directory. Either one disagreeing * with the invoking account means staging would write the wrong identity. */ export interface FleetAccount { name: string | undefined; source: "installed unit" | "$OMP_CONDUCTOR_HOME owner" | "nothing on this host"; } export interface EscalationDeps { invokingUser(): string; /** `$SUDO_USER`, which `sudo` sets and `sudo -i` / `su -` clear. */ sudoUser(): string | undefined; /** `User=` from the installed unit, or `undefined` when there is no unit. */ unitUser(): string | undefined; /** Owner of `$OMP_CONDUCTOR_HOME`, or `undefined` when it does not exist. */ homeOwner(): string | undefined; conductorHome(): string; } /** uid → account name. `id -un` because no Node builtin maps a foreign uid. */ function accountName(uid: number): string { const ran = spawnSync("id", ["-un", String(uid)], { encoding: "utf8" }); const name = ran.status === 0 ? (ran.stdout ?? "").trim() : ""; return name === "" ? `uid ${uid}` : name; } function installedUnitUser(): string | undefined { try { const unit = readFileSync(join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME), "utf8"); const match = /^User=(.*)$/m.exec(unit); if (match === null) return undefined; // Unquoted in what we generate, but systemd accepts quotes and a // hand-edited unit is exactly the case this guard has to read correctly. const value = match[1]?.trim().replace(/^"(.*)"$/, "$1") ?? ""; return value === "" ? undefined : value; } catch { return undefined; } } export const DEFAULT_ESCALATION_DEPS: EscalationDeps = { invokingUser: () => userInfo().username, sudoUser: () => process.env["SUDO_USER"], unitUser: installedUnitUser, homeOwner: () => { try { return accountName(statSync(dirname(configPath())).uid); } catch { return undefined; } }, conductorHome: () => dirname(configPath()), }; /** The account this host's fleet runs as, from whichever source knows. */ export function resolveFleetAccount(deps: EscalationDeps = DEFAULT_ESCALATION_DEPS): FleetAccount { const fromUnit = deps.unitUser(); if (fromUnit !== undefined) return { name: fromUnit, source: "installed unit" }; const fromHome = deps.homeOwner(); if (fromHome !== undefined) return { name: fromHome, source: "$OMP_CONDUCTOR_HOME owner" }; return { name: undefined, source: "nothing on this host" }; } export type EscalationVerdict = { kind: "ok" } | { kind: "refuse"; message: string }; /** * Refuses an *escalated* invocation of any setup path — and only that. * * The distinction is the whole point. Staging derives the unit's `User=` and * `HOME=` from the invoking account ({@link defaultServiceRuntime}), so a * `sudo omp-conductor setup host` writes `User=root` + `HOME=/root` into a unit * for a fleet that runs as somebody else, and the config it loads, the state * directory it writes and the indexes it points at all resolve as root too. * Nothing about that announces itself: the unit starts, the timer goes green, * and the fleet reads none of it. So it is refused rather than accommodated. * * `uid 0` is *not* the test. A fleet that legitimately runs as root — root owns * `$OMP_CONDUCTOR_HOME` and the unit says `User=root` — is the case this verb * exists for on single-tenant boxes, and refusing there would break exactly the * hosts it targets. Two checks instead: * * - `$SUDO_USER` is set, which is `sudo` announcing the escalation itself; * - the invoking account disagrees with the fleet account, which is what * catches `sudo -i` and `su -` — both clear `$SUDO_USER`, so the * environment check alone is not sufficient. */ export function checkEscalation( verb: string, deps: EscalationDeps = DEFAULT_ESCALATION_DEPS, ): EscalationVerdict { const invoking = deps.invokingUser(); const sudoUser = deps.sudoUser(); const fleet = resolveFleetAccount(deps); if (sudoUser !== undefined && sudoUser !== "") { return { kind: "refuse", message: refusal(verb, invoking, sudoUser, fleet, deps.conductorHome()), }; } if (fleet.name !== undefined && fleet.name !== invoking) { return { kind: "refuse", message: refusal(verb, invoking, undefined, fleet, deps.conductorHome()), }; } return { kind: "ok" }; } function refusal( verb: string, invoking: string, sudoUser: string | undefined, fleet: FleetAccount, home: string, ): string { const fleetName = fleet.name ?? sudoUser ?? "the fleet's own account"; return [ `omp-conductor: run ${verb} as the account the fleet runs as, not escalated.`, sudoUser === undefined ? `This is running as "${invoking}", but the fleet runs as "${fleetName}" (${fleet.source}).` : `This is running as "${invoking}" under sudo from "${sudoUser}"` + (fleet.name === undefined ? "." : `, and the fleet runs as "${fleet.name}" (${fleet.source}).`), `As "${invoking}" the config, ${home}, ~/.cache and the unit's own User= all resolve`, `as "${invoking}" instead, and the result is a unit that starts and a fleet that reads`, "none of it. Nothing has been written.", "", `Run it as "${fleetName}". Only the individual install steps need root, and`, `${verb} runs those for you with sudo after showing you each one.`, ].join("\n"); } /** * Whether a planned step list restarts the Herdr session unit — read off the * exact argv the transaction would run, never a step title, so a renamed step * cannot silently disarm the guard below. */ function restartsHerdrUnit(steps: readonly PrivilegedStep[]): boolean { return steps.some( (step) => step.argv[0] === "systemctl" && step.argv[1] === "restart" && step.argv[2] === DEFAULT_HERDR_UNIT, ); } /** * Refuses a host transaction that would kill the shell running it (#834). * * Herdr panes are children of {@link DEFAULT_HERDR_UNIT}, so the step list's * `systemctl restart` of that unit terminates every pane in the session — * including the one that invoked `setup host`. Measured 2026-08-20: the install * reached `[20/20] restart herdr-fleet.service` and completed, while the calling * OMP pane died at exit 143 with `stopReason: aborted`. The install was fine and * *looked* like a failure, because the process that would have reported and * verified it no longer existed. `upgrade` already refuses this context * outright; this is the same refusal for the other transaction that restarts * the same unit. * * Two conditions, both required, so the refusal is exactly as narrow as the * hazard: * * - the plan actually restarts the session unit (a host with no herdr unit, * or a no-op re-run that installs nothing, cannot kill anybody); * - the caller is inside a Herdr pane (`$HERDR_ENV`), which is the same * signal {@link upgradeConductor} keys on. * * Refusing *before* any mutation is what makes it safe to refuse at all: * nothing is staged, nothing is installed, and the operator re-runs the whole * transaction from a context that survives it. */ export function checkSessionRestartContext( verb: string, steps: readonly PrivilegedStep[], env: Record = process.env, ): EscalationVerdict { const inSession = env["HERDR_ENV"] !== undefined && env["HERDR_ENV"] !== ""; if (!inSession || !restartsHerdrUnit(steps)) return { kind: "ok" }; return { kind: "refuse", message: [ `omp-conductor: run ${verb} from a shell outside the target Herdr session.`, `This transaction restarts ${DEFAULT_HERDR_UNIT}, and this shell is a pane inside it`, "($HERDR_ENV is set). The restart would kill this process mid-install, before it could", "report the outcome or verify anything — a completed install that looks like a failure.", "Nothing has been staged or installed.", "", "Run the same command from a context the restart cannot reach:", ` - a plain login shell on this host (ssh, or a terminal outside herdr), or`, ` - a detached transient unit: systemd-run --collect --unit=omp-conductor-setup-host \\`, ` --property=Type=oneshot --uid="$(id -un)" omp-conductor ${verb} …`, "", "The fleet keeps running until you do; this refusal changed nothing.", ].join("\n"), }; } /** * Where the fleet's own binaries are looked for, independent of whoever invoked * this process (#890). * * `Bun.which` searches `process.env.PATH` by default, which is how #879's fix * stayed half-done: the staged PATH stopped *containing* caller entries, but * the two directories it derives from `omp-conductor` and `herdr` were still * resolved through the caller's PATH. Under a reduced startup PATH — a systemd * unit, a cron shell, `ssh host ''` — `herdr` resolves to null, the staged * unit silently loses its directory (and its `herdr` line entirely), and * `doctor` re-deriving the render in a richer shell reports drift on an install * nobody touched. * * So resolution answers to the host's install layout instead, in this order: * * 1. the directory of the Bun that is executing right now — a global `bun add` * install puts `omp-conductor` beside it; * 2. this package's own `node_modules/.bin`, when it is installed as a package * rather than run from a checkout — the binaries that ship with *this* code; * 3. the two conventional per-user bin directories; * 4. the standard system directories the staged unit already carries. * * The caller's PATH is deliberately absent: including it as a fallback would * make the resolved value depend on the invoking shell again, which is the whole * defect. A binary installed somewhere none of these cover is not resolved, and * that renders identically from every shell — a wrong-but-stable answer a * `doctor` finding can name, rather than an answer that changes per caller. */ export function canonicalBinSearchPath(home: string = homedir()): string { const packageBin = ((): string[] => { // `/node_modules/omp-conductor/omp/src` → `/node_modules/.bin` const marker = `${sep}node_modules${sep}`; const at = import.meta.dir.lastIndexOf(marker); return at === -1 ? [] : [join(import.meta.dir.slice(0, at), "node_modules", ".bin")]; })(); return [ dirname(process.execPath), ...packageBin, join(home, ".bun", "bin"), join(home, ".local", "bin"), ...SERVICE_SYSTEM_PATH.split(":"), ] .filter((value, index, all) => value.length > 0 && all.indexOf(value) === index) .join(":"); } export function defaultServiceRuntime( telegramStateDir: string, // The one resolved session name (#976). Passed in so tests stay hermetic and // the renderer never shells out — same threading as telegramStateDir. herdrSession: string = resolveHerdrSession(), // The canonical search path (#890), injectable for the same reason: a test // must be able to prove the render does not move when the CALLER's PATH does, // which means controlling what "installed here" means without touching the // process environment the assertion is about. searchPath: string = canonicalBinSearchPath(), ): ServiceRuntime { const home = homedir(); const bun = process.execPath; // Resolved against the canonical install layout, never `process.env.PATH`: // the whole point of #890 is that these two directories are facts about the // host, not about the shell that happened to run setup or doctor. const globalCli = Bun.which("omp-conductor", { PATH: searchPath }); const herdr = Bun.which("herdr", { PATH: searchPath }); const loginShell = userInfo().shell; // #879: the staged PATH is a canonical host-runtime value — the directories // of the binaries this runtime resolved plus SERVICE_SYSTEM_PATH — never the // invoking shell's PATH. Caller entries (/root/bin, /opt/puppetlabs/bin, …) // once leaked into the staged units, so `doctor` re-deriving the render in // its own process reported PATH drift against byte-identical installs, and // rerunning setup reproduced the finding its own fix text prescribes away. const pathParts = [ dirname(bun), ...(globalCli === null ? [] : [dirname(globalCli)]), ...(herdr === null ? [] : [dirname(herdr)]), ...SERVICE_SYSTEM_PATH.split(":"), ].filter((value, index, all) => value.length > 0 && all.indexOf(value) === index); return { username: userInfo().username, home, path: pathParts.join(":"), bun, ...(globalCli === null ? {} : { cli: globalCli }), packageCli: join(import.meta.dir, "cli.ts"), conductorHome: dirname(configPath()), telegramStateDir, shell: loginShell ?? "/bin/sh", // Only when Herdr is actually installed: a host running Herdr some other // way (or none yet) gets no Herdr unit rendered, ever. ...(herdr === null ? {} : { herdr }), // Only pin a non-default session. Default hosts keep a unit with no // HERDR_SESSION line so setup/upgrade do not invent drift (#394). ...(herdrSession !== DEFAULT_HERDR_SESSION ? { herdrSession } : {}), }; } export function totalConfiguredWorkers(cfg: ConductorConfig = loadConfig()): number { return cfg.projects.reduce( (total, project) => total + resolveCaps(project, cfg.defaults).maxConcurrentWorkers, 0, ); } export function renderDaemonService(runtime: ServiceRuntime, totalWorkers: number): string { const command = runtime.cli === undefined ? [runtime.bun, runtime.packageCli, "daemon", "--port", String(DEFAULT_PORT)] : [runtime.cli, "daemon", "--port", String(DEFAULT_PORT)]; const memoryMax = totalWorkers <= 1 ? "3G" : "5G"; return [ "[Unit]", "Description=omp-conductor dispatch daemon", "Documentation=https://github.com/TerrifiedBug/conductor", "After=network-online.target", "Wants=network-online.target", // #485: a crash-looped daemon used to sit in `failed` indefinitely, in // silence — no unit was watching it. The recovery oneshot runs outside // this unit's cgroup, collects evidence durably, and pages tier-2. `OnFailure=${RECOVER_SERVICE_NAME}`, "", "[Service]", "Type=simple", `User=${runtime.username}`, `Environment=${systemdQuote(`HOME=${runtime.home}`)}`, `Environment=${systemdQuote(`PATH=${runtime.path}`)}`, `Environment=${systemdQuote(`OMP_CONDUCTOR_HOME=${runtime.conductorHome}`)}`, `Environment=${systemdQuote(`OMP_TELEGRAM_STATE_DIR=${runtime.telegramStateDir}`)}`, // Escape hatch for hosts still on the pre-#320 "fleet" session (#394). Only // when set: an always-on line would drift every default host on every setup. ...(runtime.herdrSession === undefined ? [] : [`Environment=${systemdQuote(`HERDR_SESSION=${runtime.herdrSession}`)}`]), `WorkingDirectory=${systemdPath(stateDir())}`, `ExecStart=${command.map(systemdQuote).join(" ")}`, // #546: restart on any exit — clean, signalled or crashed — except an // explicit `systemctl stop`, which systemd records as intentional and does // not undo. A stray SIGTERM (a worker's `bun test`) then costs seconds of // downtime instead of leaving the fleet down until a human. A crash loop // still trips the start-limit burst and reaches `OnFailure=` above. "Restart=always", "SuccessExitStatus=0 143", "MemoryAccounting=yes", `MemoryMax=${memoryMax}`, "", "[Install]", "WantedBy=multi-user.target", "", ].join("\n"); } /** * The Herdr session-server unit, in the same shape as the daemon unit. * * The fleet's 24/7 guarantee (`upgrade`'s restart + pane-recovery leg, `start`, * the recovery plugin) treats {@link DEFAULT_HERDR_UNIT} as a first-class * dependency, yet nothing rendered it — every operator hand-wrote it from a bare * `ExecStart=` line. This provisions it, and it is where the dash-pane bug dies: * a systemd service has no `$SHELL`, so without `Environment=SHELL=` Herdr * resolved every pane as `/bin/sh`→dash with no readline and no `.bashrc`. The * account's login shell is pinned here, and again in the herdr config * ({@link renderHerdrConfig}) as belt and braces. */ export function renderHerdrUnit(runtime: ServiceRuntime): string { if (runtime.herdr === undefined) throw new Error("cannot render a herdr unit without a herdr binary"); const session = runtime.herdrSession ?? DEFAULT_HERDR_SESSION; const configPath = join(runtime.home, ".config", "herdr", "config.toml"); return [ "[Unit]", "Description=herdr fleet session server", "Documentation=https://github.com/TerrifiedBug/conductor", "After=network-online.target", "Wants=network-online.target", // #485: the fleet's two units are each other's only plausible watcher and // neither watched the other. The recovery oneshot is the watcher, and it // must fire for the orchestrator host too — a dead herdr means no tick, so // nobody else would ever report it. `OnFailure=${RECOVER_SERVICE_NAME}`, "", "[Service]", "Type=simple", `User=${runtime.username}`, `Environment=${systemdQuote(`HOME=${runtime.home}`)}`, `Environment=${systemdQuote(`PATH=${runtime.path}`)}`, `Environment=${systemdQuote(`HERDR_CONFIG_PATH=${configPath}`)}`, `Environment=${systemdQuote(`SHELL=${runtime.shell}`)}`, `WorkingDirectory=${systemdPath(runtime.conductorHome)}`, `ExecStart=${systemdQuote(runtime.herdr)} --session ${systemdQuote(session)} server`, // #546: same exposure as the daemon — restart on any exit except an // explicit `systemctl stop`; a crash loop still trips the start limit. "Restart=always", "RestartSec=5", "", "[Install]", "WantedBy=multi-user.target", "", ].join("\n"); } /** * The fleet recovery unit (#485): a oneshot every fleet unit's `OnFailure=` * names. * * It runs as root, outside the fleet units' cgroups and process trees, so the * restart it performs can never take it down, and it carries an explicit PATH — * on a typical host the non-interactive ssh PATH lacks `omp`, `herdr` and * `omp-conductor`, and the playbook needs all three. * * The bounds are part of the unit text so `doctor` drift sees them: * `StartLimitIntervalSec`/`StartLimitBurst` cap failed recovery starts at the * systemd layer (the playbook keeps its own durable consecutive-attempt * counter, agreeing on the same window), and there is deliberately **no** * `OnFailure=` of its own and no `[Install]` — nothing activates recovery * except the two fleet units' failure, so recovery can never re-trigger * recovery. */ export function renderRecoverUnit(runtime: ServiceRuntime, projectName: string | undefined): string { // `RECOVER_PROJECT` scopes the recovery action and optional re-arm. It is // host-global and installed once, so a static action scope is legitimate // only on a single-project host. With several projects it stays unset; the // playbook independently derives one deterministic outbox owner from the // live config and names every affected project in the report (#1027). return [ "[Unit]", "Description=omp-conductor fleet recovery (OnFailure handler)", "Documentation=https://github.com/TerrifiedBug/conductor", // The oneshot bound: a recovery that fails twice in the window stops // trying, and the playbook's durable counter uses the same window. "StartLimitIntervalSec=1800", "StartLimitBurst=2", "", "[Service]", "Type=oneshot", `Environment=${systemdQuote(`PATH=${runtime.path}`)}`, `Environment=${systemdQuote(`OMP_CONDUCTOR_HOME=${runtime.conductorHome}`)}`, `Environment=${systemdQuote(`RECOVER_STATE_DIR=${stateDir()}`)}`, ...(projectName === undefined ? [] : [`Environment=${systemdQuote(`RECOVER_PROJECT=${projectName}`)}`]), `ExecStart=${RECOVER_SCRIPT_INSTALL_PATH}`, "", ].join("\n"); } /** * Whether a login-shell value may be pinned into herdr's config at all. * * `userInfo().shell` degrades to the literal `unknown` where the passwd entry * cannot be read (the same root as #511's drift check), and `unknown` is not a * path — a pane told to exec it dies on start, which is a stopped fleet. Only * an absolute path that exists on this host qualifies; anything else must mean * the caller writes **no** key and lets herdr's own `$SHELL → /bin/sh → * passwd` fallback apply. */ export function usableHerdrShell(shell: string): boolean { if (!shell.startsWith("/")) return false; try { return existsSync(shell); } catch { return false; } } /** * Idempotently put `[terminal] default_shell` into a herdr config, preserving * everything else verbatim. * * Belt-and-braces behind the unit's `Environment=SHELL=`: a pane comes up as the * account's login shell even if an operator later edits the unit and drops the * `SHELL` line. Herdr falls back `$SHELL → /bin/sh` and skips the passwd entry, * so without this the only guarantee would be the unit's environment. * * A shell that is not a usable executable path ({@link usableHerdrShell}) * writes **no** key at all — the file passes through byte-identical, because * "pin the shell" must never mean "write the word `unknown`". */ export interface HerdrConfigRender { content: string; /** * Set when the merged config kept an explicit `resume_agents_on_restore = * true` — preserved on purpose (it may be a deliberate desktop config), but * loudly flagged because the headless shape herdr's own README calls * recoverable requires `false`. When absent, the value is `false` (written * or already present) and nothing needs saying. */ warning?: string; } export function renderHerdrConfig(shell: string, existing?: string): HerdrConfigRender { const text = existing ?? ""; // Unusable login shell → byte-identical pass-through, exactly as before: the // "pin the shell" guarantee is all-or-nothing, and the fleet docs' // `resume_agents_on_restore` key rides in on the same config merge, so an // operator whose passwd entry cannot be resolved still gets nothing written. if (!usableHerdrShell(shell)) return { content: text }; const quoted = `"${shell.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`; const line = `default_shell = ${quoted}`; // One anchored, comment-aware match, used for both the guard and the replace // so they can never disagree: a *real* key starts its line (after whitespace), // while `# default_shell = …` starts with a comment and is not a key. The // old guard was unanchored, so it matched a commented line the anchored // replace did not — a config whose only mention was a comment took the // "replace" branch, replaced nothing, and silently wrote nothing at all. // Any existing key keeps its position and gets the new value in place — // even when the value already matches, so a re-run stays idempotent instead // of duplicating the key. const keyLine = /^([ \t]*default_shell[ \t]*=\s*).*$/m; let content: string; if (keyLine.test(text)) { content = text.replace(keyLine, `$1${quoted}`); } else { const lines = text.split("\n"); const table = lines.findIndex((l) => /^\[terminal\]\s*$/.test(l)); if (table === -1) { const body = text.replace(/\s+$/, ""); content = `${body.length > 0 ? body + "\n" : ""}[terminal]\n${line}\n`; } else { // The table exists but has no default_shell; drop the key right under it. lines.splice(table + 1, 0, line); content = lines.join("\n"); } } // [session] resume_agents_on_restore = false, the same anchored, comment-aware // merge as default_shell: a real key is kept at its position, a commented-out // mention is not a key. The one difference is explicit `true` (the value an // operator pinned on a desktop host): it is preserved deliberately, and the // caller warns about it rather than silently flipping a deliberate choice. const resumeLine = "resume_agents_on_restore = false"; const resumeKey = /^[ \t]*resume_agents_on_restore[ \t]*=\s*([^#\n]*)/m; const resumeMatch = resumeKey.exec(content); const resumeValue = resumeMatch?.[1]?.trim().split(/\s+/)[0]; if (resumeMatch !== null) { if (resumeValue === "false") { content = content.replace(/^([ \t]*resume_agents_on_restore[ \t]*=\s*)[^#\n]*/m, `$1false`); } else if (resumeValue !== undefined && resumeValue.length > 0) { return { content, warning: `explicit resume_agents_on_restore = ${resumeValue} kept as-is (could be a deliberate desktop config); ` + "a headless fleet host leaves no live terminal for restored panes unless it is false", }; } } else { const lines = content.split("\n"); const sessionTable = lines.findIndex((l) => /^\[session\]\s*$/.test(l)); if (sessionTable === -1) { const body = content.replace(/\s+$/, ""); content = `${body.length > 0 ? body + "\n" : ""}[session]\n${resumeLine}\n`; } else { // The table exists but has no resume key; drop the key right under it. lines.splice(sessionTable + 1, 0, resumeLine); content = lines.join("\n"); } } return { content }; } /** * The values `setup host` owns in the herdr-conductor plugin's `config.env`. * * recover.sh sources the file (`herdr/bin/recover.sh`), and conductor writes * exactly the keys that file reads to resolve a fleet. `FLEET_CWDS` is the * multi-fleet form of the legacy single `FLEET_CWD`, so the renderer always * writes the former; the legacy key is an operator key and survives untouched. */ export interface HerdrConductorEnvValues { /** The Herdr session that owns the fleet: TARGET_SESSION. */ targetSession: string; /** Every configured project's fleet cwd, in config order (FLEET_CWDS). */ fleetCwds: readonly string[]; /** Absolute path of omp-telegram's `.env` (TELEGRAM_ENV). */ telegramEnv: string; /** Absolute path of omp-telegram's `access.json` (ACCESS_JSON). */ accessJson: string; } /** The keys `setup` patches; every other line in the file is operator-owned. */ const CONDUCTOR_ENV_KEYS = ["TARGET_SESSION", "FLEET_CWDS", "TELEGRAM_ENV", "ACCESS_JSON"] as const; const ENV_VALUE_ESCAPE = /[\\"$]/g; /** One `KEY=value` line, with the value shell-quoted so recover.sh's `.` * sourcing reads paths with spaces or `$` literally. */ function envLine(key: string, value: string): string { return `${key}="${value.replace(ENV_VALUE_ESCAPE, "\\$&")}"`; } /** * Render herdr-conductor's `config.env` — create it when missing; when present * patch only conductor-owned keys and pass every other line through verbatim * (an operator's `RECOVER_RECHECK_SECONDS`, `BOOTSTRAP_RESUME`, AGENT_NAME, * FLEET_CWD … must survive a re-run). * * A conductor-owned key that already exists keeps its position and gets the new * value; a missing one is appended at the end. The anchored `^[ \t]*KEY=` guard * mirrors the TOML merge's comment-awareness: a line starting with `#` is a * comment, not a key, so a commented draft is neither patched nor mistaken for * an existing real key (which would block the append). */ export function renderHerdrConductorEnv(values: HerdrConductorEnvValues, existing?: string): string { const lines = (existing ?? "").split("\n"); // The split of a file ending in a newline leaves one empty trailing element // (and a fresh file has exactly one empty element, nothing more); both are // the join's terminator, not file content — the single join below re-adds it. if (lines.length > 0 && lines[lines.length - 1] === "") lines.pop(); const headers: Record<(typeof CONDUCTOR_ENV_KEYS)[number], string> = { TARGET_SESSION: values.targetSession, FLEET_CWDS: values.fleetCwds.join(":"), TELEGRAM_ENV: values.telegramEnv, ACCESS_JSON: values.accessJson, }; const seen = new Set(); for (const index of lines.keys()) { const line = lines[index]!; const match = /^([ \t]*)([A-Za-z_][A-Za-z0-9_]*)([ \t]*=[ \t]*).*$/.exec(line); if (match === null) continue; const key = match[2] as (typeof CONDUCTOR_ENV_KEYS)[number]; if (!(key in headers)) continue; seen.add(key); lines[index] = `${match[1]}${envLine(key, headers[key])}`; } for (const key of CONDUCTOR_ENV_KEYS) { if (!seen.has(key)) lines.push(envLine(key, headers[key])); } return `${lines.join("\n")}\n`; } /** * Decide whether the fleet pane may be renamed, and if so what to run. * * The contract is written for the one-identity case and refuses to guess: * - exactly one pane carries `from` (the restamped-away identity) → rename it; * - zero, several, or an unreadable agent list → surface the exact command * and the reason; never invent a pane. * * `command` is always the one-line remedy an operator can paste; on ambiguity * it carries the generic pane placeholder because no specific pane may be * guessed. It is the same shape the tick's decline path prints (`herdr agent * rename `), born from the session so the same command works from * outside herdr too. */ export type AgentRenameVerdict = | { kind: "rename"; pane: string; command: string } | { kind: "flag"; command: string; reason: string }; export function agentRenameVerdict( list: HerdrAgentList, from: string, to: string, session: string, ): AgentRenameVerdict { const commandFor = (pane: string): string => `herdr --session ${session} agent rename ${pane} ${to}`; if (list.kind !== "ok") { return { kind: "flag", command: commandFor(""), reason: `herdr agent list is unreadable (${list.problem}) — cannot verify which pane to rename`, }; } const matches = list.agents.filter((a) => a.name === from); if (matches.length === 1) { return { kind: "rename", pane: matches[0]!.paneId, command: commandFor(matches[0]!.paneId) }; } if (matches.length === 0) { return { kind: "flag", command: commandFor(""), reason: `no live pane is registered as agent "${from}" — nothing was renamed (an unregistered fleet pane must be named by hand)`, }; } return { kind: "flag", command: commandFor(""), reason: `${matches.length} panes (${matches.map((a) => a.paneId).join(", ")}) are registered as "${from}" — refusing to guess which is the fleet pane`, }; } /** The live herdr CLI seam setup-host shells out for the one-shot rename. */ export interface AgentRenameDeps { /** `herdr --session agent list`'s one JSON line. */ list(session: string): HerdrAgentList; /** `herdr --session agent rename `. */ rename(session: string, pane: string, name: string): { ok: boolean; problem?: string }; } /** The production wiring: the same CLI `recover.sh` and the tick both use. */ export const DEFAULT_AGENT_RENAME_DEPS: AgentRenameDeps = { list: (session) => { const bin = Bun.which("herdr") ?? "herdr"; try { const run = spawnSync(bin, ["--session", session, "agent", "list"], { encoding: "utf8", timeout: 3000, stdio: ["ignore", "pipe", "pipe"], }); if (run.error !== undefined) return { kind: "unavailable", problem: run.error.message }; if (run.status !== 0) return { kind: "unavailable", problem: `exit ${String(run.status)}: ${(run.stderr ?? "").trim().split("\n")[0] ?? ""}` }; return parseHerdrAgents(run.stdout ?? ""); } catch (err) { return { kind: "unavailable", problem: err instanceof Error ? err.message : String(err) }; } }, rename: (session, pane, name) => { const bin = Bun.which("herdr") ?? "herdr"; try { const run = spawnSync(bin, ["--session", session, "agent", "rename", pane, name], { encoding: "utf8", timeout: 3000, stdio: ["ignore", "pipe", "pipe"], }); if (run.error !== undefined) return { ok: false, problem: run.error.message }; if (run.status !== 0) { return { ok: false, problem: `exit ${String(run.status)}: ${(run.stderr ?? "").trim().split("\n")[0] ?? ""}` }; } return { ok: true }; } catch (err) { return { ok: false, problem: err instanceof Error ? err.message : String(err) }; } }, }; /** The directory `herdr plugin config-dir herdr-conductor` prints. */ export function herdrConductorPluginConfigDir(home: string): string { return join(home, ".config", "herdr", "plugins", "config", "herdr-conductor"); } /** * The fleet cwd of every configured project, in config order — the FLEET_CWDS * value recover.sh walks to find each fleet's `.conductor-tick.json`. * * Tries the real config first; when it cannot be read (a not-yet-configured * box, install tests staging before a config exists), the named project's own * cwd stands in — the same guarded fallback shape as `hostInstallWorkers` and * `hostMultiProject`. Multi-project hosts get the full list when the config * loads, which is the case the recovery path exists for. */ export function configuredFleetCwds(project: ProjectConfig | undefined): string[] { try { return loadConfig().projects.map((p) => tickCwdForProject(p)); } catch { return project === undefined ? [] : [tickCwdForProject(project)]; } } function tickSearchRoots(project: ProjectConfig): string[] { const roots = [stateDir(), dirname(project.workspaceRoot), project.workspaceRoot]; return roots.filter((root, index) => roots.indexOf(root) === index); } /** * The tick config for one project's fleet cwd. * * Everything per-project here used to be shared, and that was two bugs: one * `/armed` marker meant arming project A armed B as well, and one * `agentName` meant every pane claimed the identity `fleet`, so herdr recovery * and tick ownership could not tell two fleets apart. * * An existing config keeps an explicit `armedFile`/`agentName` only when it * differs from those shared defaults. A value equal to a shared default cannot * have been a deliberate per-project choice — it *is* the collision — so it is * rewritten; anything else is operator intent and survives untouched. */ /** * This project's tick config, wherever it actually lives. A config stamped for * another project is that project's file: the search roots overlap, and * restamping it here would hand this project's identity to the other fleet's cwd. */ function findProjectTick( project: ProjectConfig, ): { root: string; path: string; config: TickConfig } | undefined { for (const root of tickSearchRoots(project)) { const result = readTickConfig(root); if (result.kind === "invalid") { throw new Error(`tick config invalid at ${result.path}: ${result.problem}; fix or remove it before setup`); } if (result.kind === "ok" && tickConfigMatchesProject(result.config, project.name)) { return { root, path: result.path, config: result.config }; } } return undefined; } /** * The directory this project's fleet pane runs in, which is **not** always * `workspaceRoot`. * * A fleet that predates per-project roots keeps its `.conductor-tick.json` in the * state dir, and that is the cwd its pane and `FLEET_CWD` already point at. Naming * `workspaceRoot` instead sends `recover.sh` somewhere with no tick config, where * `tick_agent_name` falls back to `fleet` and no longer matches the renamed pane — * silently costing a live project its recovery. Falls back to `workspaceRoot`, * which is where a brand-new project's config is written. */ export function tickCwdForProject(project: ProjectConfig): string { try { return findProjectTick(project)?.root ?? project.workspaceRoot; } catch { // An invalid config is setup's problem to report, not the handoff's. return project.workspaceRoot; } } /** * Where this project's `AGENTS.md` link should point and what setup will do * about it. The link lives in the **fleet cwd** — the directory the pane runs * in — pointing at the composed brief under `workspaceRoot`, by a relative * target so the tree stays movable. * * A fleet whose tick config predates per-project roots keeps its cwd at the * state dir, so the link is `/AGENTS.md -> worktrees/ORCHESTRATOR.md`; * a brand-new project links `AGENTS.md -> ORCHESTRATOR.md` in its own cwd. * * Never a clobber: a symlink already at the brief is left alone, a stale * symlink is replaceable, and a *regular file* is an operator's own brief — * reported and left untouched. */ function planBriefLink(project: ProjectConfig): BriefLinkPlan { const path = join(tickCwdForProject(project), AGENTS_BRIEF_NAME); const target = relative(dirname(path), join(project.workspaceRoot, ORCHESTRATOR_BRIEF_NAME)); let st; try { st = lstatSync(path); } catch { // ENOENT (no link yet) and any unreadable path (no permission on a parent) // both collapse to a best-effort create; a refused filesystem surfaces as // a warning at write time. return { path, target, action: "create" }; } if (st.isSymbolicLink()) { return { path, target, action: readlinkSync(path, "utf8") === target ? "keep" : "update", }; } return { path, target, action: "skip", skippedReason: `an existing file at ${path} is not ours to overwrite; left untouched`, }; } /** * Read-only facts about who is serving the fleet's Herdr session. * * Deliberately two independent reads rather than one: herdr answers whether the * *session* is served, systemd answers whether the *unit* is serving anything, * and the incident this exists for is exactly the case where those two * disagree. */ export interface HerdrOwnershipFacts { /** * Whether the target session has a live server, from herdr's own session * list. `undefined` when that could not be read — never `false`, because * "herdr did not answer" and "no server" lead to opposite actions. */ sessionRunning: boolean | undefined; /** The canonical unit as systemd reports it, or `undefined` when unreadable. */ unit: | { activeState: string; subState: string; mainPid: number; /** How many times systemd has restarted it — a loop's own tell. */ nRestarts: number; } | undefined; } export type HerdrOwnership = /** The canonical unit is up and is the only thing that can be serving it. */ | { kind: "unit"; detail: string } /** A live session the canonical unit is not serving: the #893 collision. */ | { kind: "unmanaged"; detail: string } /** No live session at all — a fresh host, or one whose server is down. */ | { kind: "absent"; detail: string } /** A read failed. Never treated as either of the decided answers. */ | { kind: "unknown"; detail: string }; /** * Who owns the fleet session (#893). * * `setup host` can complete its whole transaction successfully while * `herdr-fleet.service` sits in `activating (auto-restart)`: an unmanaged * `herdr --session server` already holds the session, so every start of * the unit exits 1, the panes stay under the unmanaged process, and dispatch * stays paused after a green setup. Measured 2026-08-22T09:01:31Z: exit 0, * `NRestarts=2`, `ExecStart` status 1, the real fleet under * `herdr session attach` ancestry. A `systemctl restart` returning 0 is * therefore not evidence that supervision transferred — `Type=simple` returns * before the child discovers the collision and dies. * * The reasoning behind the verdicts, since only one of them is an observation: * * - A live session with the unit **not** active is decided: something outside * the unit is serving it. Nothing else can be true. * - A live session with the unit active *is* the unit's, because two servers * cannot hold one session — that mutual exclusion is precisely what makes * the collision fatal, so it is also what makes this inference sound. * - Either read failing is `unknown`, which callers must treat as a refusal * rather than as "probably fine": the destructive remedy here is stopping * somebody's live session. */ export function herdrOwnership(facts: HerdrOwnershipFacts): HerdrOwnership { if (facts.sessionRunning === undefined) { return { kind: "unknown", detail: "herdr could not say whether the fleet session has a live server", }; } if (facts.unit === undefined) { return { kind: "unknown", detail: "systemctl could not say what state the fleet session unit is in", }; } const { activeState, subState, mainPid, nRestarts } = facts.unit; const unitServing = activeState === "active" && subState === "running" && mainPid > 0; if (!facts.sessionRunning) { return { kind: "absent", detail: unitServing ? `the unit is ${activeState} (${subState}) but no session server answers — it is coming up` : `no live fleet session server (unit ${activeState}/${subState})`, }; } if (unitServing) { return { kind: "unit", detail: `served by ${DEFAULT_HERDR_UNIT} (pid ${mainPid})` }; } return { kind: "unmanaged", detail: `the fleet session has a live server, but ${DEFAULT_HERDR_UNIT} is ${activeState} (${subState})` + `${nRestarts > 0 ? `, restarted ${nRestarts} time(s)` : ""} with no main pid — ` + "the session is held outside systemd, so every start of the unit exits 1", }; } /** * What this version retires from the host, read from disk (#895). * * One entry today — the worker harness bind mount 0.18.1 installed — and the * shape is deliberately a list, because "the release that removes a unit" is a * recurring event and the alternative is a bespoke check each time. * * Ordering is the whole content of the step list: * * 1. `disable --now` first, which stops (for a `.mount`: unmounts) and drops * the enablement symlinks in one call. Doing it before the file is gone * matters — systemd cannot disable a unit whose file it can no longer * read, so removing first would strand the enablement symlink and leave * the mount live until reboot. * 2. Then remove the installed file, then one `daemon-reload` so systemd * forgets the unit rather than keeping it loaded-but-absent. * 3. Then `rmdir` the empty mount point, best-effort: it refuses a non-empty * directory by construction, so it can never destroy anything, and a host * where the unmount left files behind keeps them for an operator to see. * * Nothing here touches the `omp-worker` account or its home. An unused account * is inert, and deleting a Unix account (with whatever a previous release put * in its home) to tidy up is a destructive act this retirement deliberately * refuses — the same judgement #895 records. */ function planRetirement(unitDir: string): HostRetirement | undefined { const installed = join(unitDir, HARNESS_MOUNT_UNIT_NAME); const staged = join(stateDir(), HARNESS_MOUNT_UNIT_NAME); const units = existsSync(installed) ? [installed] : []; const stagedFiles = existsSync(staged) ? [staged] : []; if (units.length === 0 && stagedFiles.length === 0) return undefined; return { units, staged: stagedFiles, steps: [ ...(units.length === 0 ? [] : [ { title: `disable and unmount ${HARNESS_MOUNT_UNIT_NAME} (retired: worker sessions run as the fleet account again)`, argv: ["systemctl", "disable", "--now", HARNESS_MOUNT_UNIT_NAME], }, { title: `remove ${installed}`, argv: ["rm", "-f", installed] }, { title: "reload systemd", argv: ["systemctl", "daemon-reload"] }, { title: `remove the now-unused mount point ${WORKER_HARNESS_DIR}`, argv: ["rmdir", "--ignore-fail-on-non-empty", WORKER_HARNESS_DIR], bestEffort: true, }, ]), // The staged render is the fleet account's own file, so it needs no // escalation — and it goes last: a failed disable must not leave the // state directory looking converged. ...stagedFiles.map((path) => ({ title: `remove the staged ${HARNESS_MOUNT_UNIT_NAME} render`, argv: ["rm", "-f", path], unprivileged: true, })), ], }; } function planTick(project: ProjectConfig, telegramStateDir: string): { write: PlannedWrite; /** Set when the config was restamped from the shared default: the live herdr * pane still carries that old identity, and the plan must rename it * (or flag the exact command) after the tick lands. */ previousAgentName?: string; } { const found = findProjectTick(project); const existing = found === undefined ? undefined : { path: found.path, config: found.config }; const armedFile = join(stateDir(), `armed-${project.name}`); const path = existing?.path ?? join(project.workspaceRoot, TICK_CONFIG_FILE); const config: TickConfig = existing === undefined ? { intervalSeconds: DEFAULT_TICK_INTERVAL_SECONDS, project: project.name, armedFile, accessFile: join(telegramStateDir, "access.json"), agentName: project.name, } : { ...existing.config, project: project.name, armedFile: existing.config.armedFile === undefined || existing.config.armedFile === legacyArmedMarkerPath() ? armedFile : existing.config.armedFile, accessFile: existing.config.accessFile ?? join(telegramStateDir, "access.json"), agentName: existing.config.agentName === undefined || existing.config.agentName === DEFAULT_FLEET_AGENT_NAME ? project.name : existing.config.agentName, }; const content = `${JSON.stringify(config, null, 2)}\n`; return { write: { path, action: actionFor(path, content), content, value: config }, // The rename only fires for a *re*-stamp: the old agentName was the shared // default (or absent, which every reader resolves to the same default), and // the new one is the project's own name. An already-project-stamped config // is not a collision and needs no rename. A brand-new config (no existing // tick at all) is not a restamp either: there was no previous identity for // a pane to be stuck with. ...(existing !== undefined && (existing.config.agentName === undefined || existing.config.agentName === DEFAULT_FLEET_AGENT_NAME) ? { previousAgentName: DEFAULT_FLEET_AGENT_NAME } : {}), }; } export function planHostRuntime( /** * The project the per-project tail (tick config, brief link, the recovery * unit's RECOVER_PROJECT) is planned for. `undefined` plans the host-global * units only — the same staged paths and systemd destinations, one shared * daemon — and a `noProject` note naming the per-project files it left for * a named run (#530). */ project: ProjectConfig | undefined, _caps: Caps, telegramStateDir: string, runtime: ServiceRuntime = defaultServiceRuntime(telegramStateDir), // Callers that know the fleet pass the sum of resolved maxConcurrentWorkers. // Defaulting through loadConfig() would throw in install tests that stage // files before a config exists, and would hide a missing total at the call site. totalWorkers: number = 1, // The directory the privileged steps install units into. `runHostInstall` // injects a test-hermetic directory so the executed argv is assertable; the // real install always resolves to `/etc/systemd/system`. unitDir: string = SYSTEMD_UNIT_DIR, // Where the recovery playbook installs; injectable like {@link unitDir} so // the idempotency gate is testable without touching the real `/usr/local/sbin`. recoverScriptInstallPath: string = RECOVER_SCRIPT_INSTALL_PATH, // True when the host configures more than one project. The recovery unit is // host-global, so on a multi-project host it must not encode one project's // name — and a no-project install never does (#510/#530). multiProject: boolean = false, ): HostRuntimePlan { const servicePath = join(stateDir(), STAGED_SERVICE_NAME); const serviceContent = renderDaemonService(runtime, totalWorkers); const service: PlannedWrite = { path: servicePath, action: actionFor(servicePath, serviceContent), content: serviceContent, value: serviceContent, }; const herdrUnitPath = join(stateDir(), DEFAULT_HERDR_UNIT); const herdrConfigPath = join(runtime.home, ".config", "herdr", "config.toml"); // The pane-shell config is staged like every other file; the live herdr // config — a file conductor does not own — is only written by the install // step, after consent (#513). A declined `setup host` must leave herdr's // config byte-for-byte untouched, and the merge must land before the unit // restart that accompanies it. const herdrConfigStagedPath = join(stateDir(), "herdr-config.toml"); const herdrEnvStagedPath = join(stateDir(), "herdr-conductor.env"); const herdrEnvTarget = join(herdrConductorPluginConfigDir(runtime.home), "config.env"); let herdrUnit: PlannedWrite | undefined; let herdrConfig: PlannedWrite | undefined; let herdrConfigTarget: string | undefined; let herdrConfigProblem: string | undefined; let herdrConfigWarning: string | undefined; let herdrEnv: PlannedWrite | undefined; // The recovery playbook ships in this package's systemd/ dir; the same // bytes go into the staged copy, so version control is the single source. const recoverScriptPath = join(stateDir(), RECOVER_SCRIPT_FILE); let recoverScriptContent: string; try { recoverScriptContent = readFileSync(join(import.meta.dir, "..", "systemd", RECOVER_SCRIPT_FILE), "utf8"); } catch (err) { throw new Error( `cannot read the recovery playbook at ${join(import.meta.dir, "..", "systemd", RECOVER_SCRIPT_FILE)} — it ships with this package, and setup cannot stage it without it: ${err instanceof Error ? err.message : String(err)}`, ); } if (runtime.herdr !== undefined) { const unitContent = renderHerdrUnit(runtime); herdrUnit = { path: herdrUnitPath, action: actionFor(herdrUnitPath, unitContent), content: unitContent, value: unitContent, }; // Belt and braces: read the session's herdr config so `[terminal] // default_shell` merges into whatever the operator already has, preserving // it verbatim. A missing or unreadable config starts from scratch. let existing: string | undefined; try { existing = readFileSync(herdrConfigPath, "utf8"); } catch { existing = undefined; } // An unresolvable login shell (userInfo() renders the literal "unknown") // plans nothing and says why: never a placeholder, and never a value that // is not an executable path — herdr's own `$SHELL → /bin/sh → passwd` // fallback applies instead. if (!usableHerdrShell(runtime.shell)) { herdrConfigProblem = `login shell ${JSON.stringify(runtime.shell)} is not an absolute path that exists on this host — ` + "no [terminal] default_shell is written, so herdr's own $SHELL → /bin/sh → passwd fallback applies"; } else { const rendered = renderHerdrConfig(runtime.shell, existing); const configContent = rendered.content; if (rendered.warning !== undefined) herdrConfigWarning = rendered.warning; // The rendered file is parse-checked before anything may replace herdr's // live config. An input that is already broken (a duplicated key, say) // renders broken — the replace touches every real key, so two keys stay // two keys — and writing that file is exactly the stopped-fleet outcome // this must prevent. A parse failure aborts the whole plan here, before // a single file has been staged. if (actionFor(herdrConfigPath, configContent) !== "keep") { try { Bun.TOML.parse(configContent); } catch (err) { throw new Error( `refusing to write herdr's config at ${herdrConfigPath}: it would not parse ` + `(${err instanceof Error ? err.message : String(err)}). Nothing has been staged — ` + "fix the file by hand (herdr's `config check` names the line), then re-run setup host.", ); } } herdrConfig = { path: herdrConfigStagedPath, action: actionFor(herdrConfigStagedPath, configContent), content: configContent, value: configContent, }; herdrConfigTarget = herdrConfigPath; } // The herdr-conductor plugin's config.env (#541): recover.sh sources // `$HERDR_PLUGIN_CONFIG_DIR/config.env` and shrinks to legacy /root paths // when it is missing — on a host whose fleet lives elsewhere the recovery // still runs, only the page silently goes nowhere. Setup owns // TARGET_SESSION/FLEET_CWDS/TELEGRAM_ENV/ACCESS_JSON; the renderer patches // only those keys, so an operator's BOOTSTRAP_RESUME or // RECOVER_RECHECK_SECONDS survive a re-run. let existingEnv: string | undefined; try { existingEnv = readFileSync(herdrEnvTarget, "utf8"); } catch { existingEnv = undefined; } const envContent = renderHerdrConductorEnv( { targetSession: runtime.herdrSession ?? DEFAULT_HERDR_SESSION, // Derived from every configured project's tick cwd, not from the // host-global unit the install runs for: FLEET_CWDS is what recover.sh // walks to find each fleet's `.conductor-tick.json`. fleetCwds: configuredFleetCwds(project), telegramEnv: join(telegramStateDir, ".env"), accessJson: join(telegramStateDir, "access.json"), }, existingEnv, ); herdrEnv = { path: herdrEnvStagedPath, action: actionFor(herdrEnvStagedPath, envContent), content: envContent, value: envContent, }; } const installedPath = join(unitDir, STAGED_SERVICE_NAME); const installedHerdr = join(unitDir, DEFAULT_HERDR_UNIT); const recoverUnitPath = join(stateDir(), RECOVER_SERVICE_NAME); // The recovery unit is host-global: one shared unit, installed once. // `RECOVER_PROJECT` scopes recovery action/re-arm only and is therefore // omitted on a multi-project host. Report ownership is a separate runtime // decision: the playbook reads the configured project set, chooses its first // project as outbox owner, and lists every project as affected (#1027). const recoverProject = project === undefined || multiProject ? undefined : project.name; const recoverUnitContent = renderRecoverUnit(runtime, recoverProject); const recoverUnit: PlannedWrite = { path: recoverUnitPath, action: actionFor(recoverUnitPath, recoverUnitContent), content: recoverUnitContent, value: recoverUnitContent, }; const recoverScript: PlannedWrite = { path: recoverScriptPath, action: actionFor(recoverScriptPath, recoverScriptContent), content: recoverScriptContent, value: recoverScriptContent, }; // The worker identity plan (#798): account creation, the agent-config bind, // the runtime provision, and the search/read ACL grants are host changes // the same install consent covers, so they join the same single step list. // What an earlier release left on this host and this one takes away (#895), // read from disk exactly like every other currentness fact in this plan. const retire = planRetirement(unitDir); // Recovery first: the daemon unit that follows names it in OnFailure=, so // the restart below must never point at a unit systemd cannot load. This is // the single list `runHostInstall` executes and {@link installCommands} // renders from, so no step can be in one and missing from the other (#509). const installSteps: PrivilegedStep[] = [ { title: "install the recovery playbook", argv: ["install", "-m", "0755", recoverScriptPath, recoverScriptInstallPath], }, { title: `install ${RECOVER_SERVICE_NAME}`, argv: ["install", "-m", "0644", recoverUnitPath, join(unitDir, RECOVER_SERVICE_NAME)], }, { title: "reload systemd", argv: ["systemctl", "daemon-reload"] }, { title: `install ${STAGED_SERVICE_NAME}`, argv: ["install", "-m", "0644", servicePath, installedPath], }, { title: "reload systemd", argv: ["systemctl", "daemon-reload"] }, { title: `enable ${STAGED_SERVICE_NAME}`, argv: ["systemctl", "enable", STAGED_SERVICE_NAME] }, { title: `restart ${STAGED_SERVICE_NAME}`, argv: ["systemctl", "restart", STAGED_SERVICE_NAME] }, // The herdr session server, provisioned alongside the daemon (#456). Only // when herdr is installed and the plan therefore staged a unit. ...(herdrUnit === undefined ? [] : [ { title: `install ${DEFAULT_HERDR_UNIT}`, argv: ["install", "-m", "0644", herdrUnitPath, installedHerdr], }, { title: "reload systemd", argv: ["systemctl", "daemon-reload"] }, { title: `enable ${DEFAULT_HERDR_UNIT}`, argv: ["systemctl", "enable", DEFAULT_HERDR_UNIT] }, // The pane-shell merge lands *before* the restart it accompanies, so // the session server comes up with the pinned shell, not the stale // config. It runs without sudo on purpose: the file is the fleet // account's own (the escalation guard guarantees setup runs as that // account), and a root-owned copy would break herdr's next rewrite. // Nothing writes herdr's live config until this consent-gated step. ...(herdrConfig === undefined || actionFor(herdrConfigPath, herdrConfig.content) === "keep" ? [] : [ { title: `write the pane shell into ${herdrConfigPath}`, argv: ["install", "-m", "0644", herdrConfigStagedPath, herdrConfigPath], unprivileged: true, }, ]), // The herdr-conductor config.env (#541) rides the same consent-gated, // unprivileged, pre-restart lane: recover.sh sources it on every // startup, so the session server must come up already pointing at // this host's fleet cwds and telegram files. ...(herdrEnv === undefined || actionFor(herdrEnvTarget, herdrEnv.content) === "keep" ? [] : [ { title: `write the herdr-conductor config.env into ${herdrEnvTarget}`, argv: ["install", "-m", "0644", herdrEnvStagedPath, herdrEnvTarget], unprivileged: true, }, ]), { title: `restart ${DEFAULT_HERDR_UNIT}`, argv: ["systemctl", "restart", DEFAULT_HERDR_UNIT] }, ]), ]; const installStepsAll: PrivilegedStep[] = [ // Retirement first (#895): the obsolete bind is unmounted before anything // restarts, so no unit comes up while a mount nothing reads is still live, // and an operator watching the plan sees the removal before the installs. ...(retire === undefined ? [] : retire.steps), ...installSteps, ]; // Everything the privileged steps install is already at its destination with // the current bytes, so a re-run of `setup host` has nothing to install and // nothing to restart. The herdr unit is absent on a host without herdr, and // an absent unit that would not be provisioned is nothing to do. const installedAction = actionFor(installedPath, serviceContent); // The same comparison as a destination list, so `upgrade` can name the // drifted files and `currentInstall` can never disagree with that list: a // plan is current exactly when no installed destination differs from the // render. const drift: string[] = []; const noteDrift = (path: string, content: string | undefined): void => { if (content !== undefined && actionFor(path, content) !== "keep") drift.push(path); }; noteDrift(installedPath, serviceContent); noteDrift(join(unitDir, RECOVER_SERVICE_NAME), recoverUnitContent); noteDrift(recoverScriptInstallPath, recoverScriptContent); if (herdrUnit !== undefined) noteDrift(installedHerdr, herdrUnit.content); // The pane-shell file is a destination like the units: a plan with all // units current but the config merge still pending must not report // "nothing to install" and skip the very write it exists to make. if (herdrConfig !== undefined) noteDrift(herdrConfigPath, herdrConfig.content); // Same for the herdr-conductor config.env: a pending env merge is pending // work, not an already-current install. if (herdrEnv !== undefined) noteDrift(herdrEnvTarget, herdrEnv.content); // Owed retirement is pending work like any drifted file (#895): a host whose // obsolete mount is still enabled has something left to do, and a re-run that // reported "already current" there would be the half-applied retirement this // slice exists to prevent. const currentInstall = drift.length === 0 && retire === undefined; return { service, // The herdr unit and pane-shell config stand and fall together: no herdr, no // session to supervise, nothing for a pane shell to belong to. (The pane // shell is omitted too when the login shell is unusable or the rendered // config would not parse — those plans carry {@link herdrConfigProblem}.) ...(herdrUnit === undefined ? {} : { herdrUnit, herdrSession: runtime.herdrSession ?? DEFAULT_HERDR_SESSION }), ...(herdrConfig === undefined ? {} : { herdrConfig, herdrConfigTarget }), ...(herdrConfigProblem === undefined ? {} : { herdrConfigProblem }), ...(herdrConfigWarning === undefined ? {} : { herdrConfigWarning }), ...(herdrEnv === undefined ? {} : { herdrEnv, herdrEnvTarget }), recoverUnit, recoverScript, ...(project === undefined ? { // Host-global install: the per-project tail entities are not written, // and the plan says exactly which files and how to write them. noProject: { skipped: [TICK_CONFIG_FILE, AGENTS_BRIEF_NAME, CONSOLE_BRIEF_NAME], how: "re-run `omp-conductor setup host ` (or --project NAME) to write them for one project", }, } : { briefLink: planBriefLink(project), // The console workspace is per-project like the brief link, and // planned for every project — embedded or external. An embedded // orchestrator still has an operator to answer. console: planConsole(project), ...(project.escalation.orchestrator === "external" ? (() => { const planned = planTick(project, telegramStateDir); return { ...(planned.previousAgentName === undefined || runtime.herdr === undefined ? {} : { // The restamp that made the tick's agentName no longer // the shared default: the live pane (if any) still // carries the old name, and setup will rename it after // the install — or print the exact command. agentRename: { session: runtime.herdrSession ?? DEFAULT_HERDR_SESSION, from: planned.previousAgentName, to: project.name, }, }), tick: planned.write, }; })() : {}), }), steps: installStepsAll, // For humans to read; the executed form is {@link steps}, in argv. installCommands: installStepsAll.map((s) => formatStep(s)), cliSource: runtime.cli === undefined ? "plugin" : "global", installedPath, installedAction, drift, ...(retire === undefined ? {} : { retire }), currentInstall, }; } export function formatHostRuntimePlan(plan: HostRuntimePlan): string { const lines = [ "host runtime", ` service ${plan.service.action} ${plan.service.path}`, ` daemon entry ${plan.cliSource === "global" ? "installed omp-conductor CLI" : "current installed plugin"}`, ` recovery ${plan.recoverUnit.action} ${plan.recoverUnit.path}`, ` recovery exec ${plan.recoverScript.action} ${plan.recoverScript.path} -> ${RECOVER_SCRIPT_INSTALL_PATH}`, ...(plan.retire === undefined ? [] : [ ` retire ${[...plan.retire.units, ...plan.retire.staged].join(", ")}`, ` (the ${HARNESS_MOUNT_UNIT_NAME.replace(/\\x2d/g, "-")} bind an earlier release installed; worker sessions run as the fleet account)`, ]), ...(plan.herdrUnit === undefined ? [" herdr session skipped — herdr not installed on this host"] : [ ` herdr session ${plan.herdrUnit.action} ${plan.herdrUnit.path}`, ...(plan.herdrConfig === undefined ? [` pane shell skipped — ${plan.herdrConfigProblem ?? "login shell unusable"}`] : [` pane shell ${plan.herdrConfig.action} ${plan.herdrConfig.path} ([terminal] default_shell)`]), ...(plan.herdrConfigWarning === undefined ? [] : [` pane resume warn — ${plan.herdrConfigWarning}`]), ...(plan.herdrEnv === undefined ? [] : [` herdr env ${plan.herdrEnv.action} ${plan.herdrEnv.path} -> ${plan.herdrEnvTarget}`]), ...(plan.agentRename === undefined ? [] : [` agent rename ${plan.agentRename.from} -> ${plan.agentRename.to} in session ${plan.agentRename.session}`]), ]), ]; if (plan.tick !== undefined) { lines.push( ` heartbeat ${plan.tick.action} ${plan.tick.path}`, ` interval ${plan.tick.value.intervalSeconds}s`, ` arm gate ${plan.tick.value.armedFile}`, ` channel gate ${plan.tick.value.accessFile}`, ); } else { lines.push(" heartbeat embedded orchestrator — no external tick config"); } if (plan.noProject !== undefined) { lines.push( ` per-project skipped (no project named): ${plan.noProject.skipped.join(", ")}`, ` ${plan.noProject.how}`, ); } if (plan.briefLink !== undefined) { lines.push( plan.briefLink.action === "skip" ? ` brief link ${plan.briefLink.path} — ${plan.briefLink.skippedReason}` : ` brief link ${plan.briefLink.action} ${plan.briefLink.path} -> ${plan.briefLink.target}`, ); } if (plan.console !== undefined) { lines.push( ` console brief ${plan.console.brief.action} ${plan.console.brief.path}`, plan.console.link.action === "skip" ? ` console link ${plan.console.link.path} — ${plan.console.link.skippedReason}` : ` console link ${plan.console.link.action} ${plan.console.link.path} -> ${plan.console.link.target}`, ); } lines.push(" install staged only; the final result prints the systemd install commands"); return lines.join("\n"); } function atomicWrite(path: string, content: string, mode: number): void { mkdirSync(dirname(path), { recursive: true }); const tmp = `${path}.${process.pid}.${Date.now()}.tmp`; try { writeFileSync(tmp, content, { mode }); chmodSync(tmp, mode); renameSync(tmp, path); chmodSync(path, mode); } catch (err) { rmSync(tmp, { force: true }); throw err; } } /** What {@link writeHostRuntime} wrote, plus anything it could not. */ export interface HostRuntimeWrite { /** Paths that were (re)written. */ wrote: string[]; /** Best-effort failures the caller should surface; setup continues regardless. */ warnings: string[]; } export function writeHostRuntime(plan: HostRuntimePlan): HostRuntimeWrite { const wrote: string[] = []; const warnings: string[] = []; if (plan.service.action !== "keep") { atomicWrite(plan.service.path, plan.service.content, 0o644); wrote.push(plan.service.path); } if (plan.herdrUnit !== undefined && plan.herdrUnit.action !== "keep") { atomicWrite(plan.herdrUnit.path, plan.herdrUnit.content, 0o644); wrote.push(plan.herdrUnit.path); } if (plan.herdrConfig !== undefined && plan.herdrConfig.action !== "keep") { atomicWrite(plan.herdrConfig.path, plan.herdrConfig.content, 0o644); wrote.push(plan.herdrConfig.path); } if (plan.herdrEnv !== undefined && plan.herdrEnv.action !== "keep") { atomicWrite(plan.herdrEnv.path, plan.herdrEnv.content, 0o644); wrote.push(plan.herdrEnv.path); } if (plan.recoverUnit.action !== "keep") { atomicWrite(plan.recoverUnit.path, plan.recoverUnit.content, 0o644); wrote.push(plan.recoverUnit.path); } if (plan.recoverScript.action !== "keep") { atomicWrite(plan.recoverScript.path, plan.recoverScript.content, 0o755); wrote.push(plan.recoverScript.path); } if (plan.tick !== undefined && plan.tick.action !== "keep") { atomicWrite(plan.tick.path, plan.tick.content, 0o600); wrote.push(plan.tick.path); } // `skip` is a regular file the plan already told the operator would be left // alone; `keep` is an already-correct link. Only create/update touch disk — // and never on the plan's word: the entry can change across the consent gap // (an operator may drop a regular file where a stale symlink was planned for // replacement). Re-lstat at write time and act on *current* state, so a file // that appears mid-flight is preserved, never unlinked by a stale plan. // A host-global plan (no project) has no briefLink to write at all. if (plan.briefLink !== undefined && (plan.briefLink.action === "create" || plan.briefLink.action === "update")) { const { path, target } = plan.briefLink; try { mkdirSync(dirname(path), { recursive: true }); let st: Stats | undefined; try { st = lstatSync(path); } catch { st = undefined; // ENOENT — nothing there yet; also best-effort if a parent refuses. } if (st !== undefined && !st.isSymbolicLink()) { // A regular file occupies the path (present at plan time, appeared, or // swapped in since). Never clobber an operator's file: warn and carry on. warnings.push(`${path} is a regular file, not ours to overwrite; AGENTS.md brief link skipped`); } else if (st !== undefined && readlinkSync(path, "utf8") === target) { // Already the correct link — leave it untouched, however the plan read it. } else { // Missing, or a stale symlink: (re)link it. Only a *current* symlink is // removed; an operator's file that appears mid-flight survives the write. if (st !== undefined) rmSync(path, { force: true }); symlinkSync(target, path); wrote.push(path); } } catch (err) { // Best effort: a filesystem that refuses symlinks must not fail setup. warnings.push(`could not link ${path} -> ${target}: ${err instanceof Error ? err.message : String(err)}`); } } // One writer for the console workspace, shared with `omp-conductor console` // (see {@link writeConsole}): a second copy of the never-clobber link // discipline is a second place for it to be got wrong. Warnings come back // rather than throwing, exactly like the brief link above. if (plan.console !== undefined) { const written = writeConsole(plan.console); wrote.push(...written.wrote); warnings.push(...written.warnings); } return { wrote, warnings }; } // ---------------------------------------------------------- rollback (#652) -- // // The inverse of the writes above, for the setup apply's mutation inventory: // every path the apply can write is captured byte-for-byte (or as absence, or // as unreadable — never conflated) before the first mutation, and restored on // any later failure. `capturePathState` is the read half, `restorePathState` // the write half; the transaction that owns them lives in `setup-wizard.ts`. /** One path's pre-entry state, three-valued so absence is never confused with * unreadability: * - `absent` — nothing at the path; rollback removes what the apply created. * - `bytes` — readable on entry (a regular file with its mode, or a symlink * with its target); rollback writes those exact bytes/mode or re-links it. * - `unreadable` — something exists but cannot be read. The transaction must * fail closed before changing anything: collapsing this to absence and * later deleting the path would remove state the apply never saw. */ export type CapturedPathState = | { path: string; kind: "absent" } | { path: string; kind: "bytes"; bytes: string; mode: number; symlink: boolean } | { path: string; kind: "unreadable" }; /** Reads {@link CapturedPathState} for one path. Symlinks are captured as * their target (readlink), never followed — the brief link is a symlink by * contract, and a dangling one is still state worth restoring. */ export function capturePathState(path: string): CapturedPathState { let st: Stats; try { st = lstatSync(path); } catch { return { path, kind: "absent" }; } try { if (st.isSymbolicLink()) { return { path, kind: "bytes", bytes: readlinkSync(path, "utf8"), mode: st.mode & 0o7777, symlink: true }; } return { path, kind: "bytes", bytes: readFileSync(path, "utf8"), mode: st.mode & 0o7777, symlink: false }; } catch { return { path, kind: "unreadable" }; } } /** Options for {@link restorePathState}. */ export interface RestorePathOptions { /** * Never remove or replace a regular file: only a symlink (or absence) is * touched, so an operator's file that appeared mid-transaction survives a * rollback that cannot clobber it — the same guard the brief-link write * itself applies (#652). A regular file where the entry state said * "absent"/"symlink" therefore reports a restoration failure: the prior * state is not coherent and the operator must be told. */ preserveRegularFile?: boolean; } /** Restores {@link CapturedPathState} captured before the first mutation: * writes the entry bytes back (with their mode), re-links a symlink's entry * target, or removes what the apply created when nothing was there. Returns * `undefined` on a complete restore, else the first failure's message — a * rollback that could not restore a path must be reported, never implied * complete. */ export function restorePathState(state: CapturedPathState, opts: RestorePathOptions = {}): string | undefined { const { path } = state; try { if (state.kind === "bytes" && state.symlink) { // Re-link the entry target. Under `preserveRegularFile` a regular file // that replaced the link mid-flight is left alone — and reported — so a // rollback never clobbers an operator's file it did not create. let current: Stats | undefined; try { current = lstatSync(path); } catch { current = undefined; } if (current !== undefined && !current.isSymbolicLink() && opts.preserveRegularFile === true) { return `could not restore the symlink at ${path}: a regular file now occupies it`; } rmSync(path, { force: true }); mkdirSync(dirname(path), { recursive: true }); symlinkSync(state.bytes, path); return undefined; } if (state.kind === "bytes") { mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, state.bytes); // Mode is part of the pre-entry state (a staged unit 0644, the tick // config 0600, the recovery playbook 0755): writeFileSync preserves an // existing file's mode, so chmod explicitly restores the captured one. chmodSync(path, state.mode); return undefined; } // Absent at entry: remove what the apply created. A regular file that // appeared where the entry was absent is the operator's own unless the // caller says otherwise. let current: Stats | undefined; try { current = lstatSync(path); } catch { return undefined; // already gone — a complete restore. } if (current.isSymbolicLink() || opts.preserveRegularFile !== true) { rmSync(path, { force: true }); return undefined; } return `could not restore ${path}: a regular file appeared where nothing was at entry`; } catch (err) { return `could not restore ${path}: ${err instanceof Error ? err.message : String(err)}`; } } export interface SetupSmokeResult { mode: "temporary" | "existing"; /** * Whether the smoke itself proved the daemon answers `/healthz` on * {@link daemon}.port. A record-less active unit has no port to probe, so * its "existing" smoke is unproven: it cannot count as passed until the * caller restarts the daemon through the lifecycle seam, whose * `waitForOwnedDaemon` proves MainPID + `/healthz` (#651, review #2). */ healthProven: boolean; status: StatusSnapshot; daemon: DaemonRecord; } export interface SetupSmokeDeps { paused(project: string): boolean; runOnce(project: string): Promise; living(): DaemonRecord | undefined; /** * systemd ownership of the `omp-conductor.service` supervisor unit, probed * independently of the pidfile/runtime record. An active unit is a live * supervised daemon even when its record is missing or unreadable (#618): * it must never be run beside a `--once` tick, and `start()` must never be * aimed at it. `unknown` is preserved — never collapsed into `inactive` — * so a possibly-live supervised daemon is not run beside (#651, review #2). */ unitOwnership(): UnitOwnership; health(port: number): Promise<{ ok: boolean; body?: string }>; start(project: string): Promise; stop(): Promise; status(project: string): StatusSnapshot; } const DEFAULT_SMOKE_DEPS: SetupSmokeDeps = { paused: isPaused, runOnce: async (project) => await runDaemon({ once: true, project }), living: livingDaemon, unitOwnership: () => probeUnit(SYSTEMD_UNIT), health: healthCheck, start: async (project) => await startDaemon({ project }), stop: stopDaemon, status: statusSnapshot, }; /** The daemon record a smoke attributes to a live but record-less supervisor. * The port is unknown (the record would have named it); the "existing" mode * caller restarts the daemon through the lifecycle seam and recomputes the * real /healthz line from that restart, so this satisfies the result shape * without pretending to know a port. `healthProven: false` marks that this * is a placeholder until that restart proves the unit's MainPID answers. */ function unrecordedDaemon(mainPid: number): DaemonRecord { return { pid: 0, port: 0, startedAt: 0, logFile: ``, }; } export async function runSetupSmoke( project: string, deps: SetupSmokeDeps = DEFAULT_SMOKE_DEPS, ): Promise { if (!deps.paused(project)) throw new Error("setup smoke requires paused dispatch"); // Detect a live daemon BEFORE the mutating `--once` tick (#618). A paused // `--once` run is itself a dispatcher: it settles rows, projects labels, // admits and salvages workers. Running one beside a live daemon orphans and // salvages the runs the live process owns — exactly what the incident did to // three workers before failing its restart check. An already-active daemon // needs only its health proving (the caller restarts it through the lifecycle // seam so the new config takes effect) — no `--once` process ever runs beside // it, and no `systemctl start` is aimed at a unit that is already active. const existing = deps.living(); if (existing !== undefined) { const health = await deps.health(existing.port); if (!health.ok) throw new Error(`existing daemon on :${existing.port} did not answer /healthz`); return { mode: "existing", healthProven: true, status: deps.status(project), daemon: existing }; } // The pidfile record is absent, but that is not "no daemon": an active // systemd-owned unit is a live supervised dispatcher even when its record // went missing (a crash raced a re-record, or an unmanaged start never wrote // one). Without this independent probe the smoke runs a second `--once` // beside it, and `start()` targets the already-active unit — two dispatchers // owning the same state, the #618 incident shape. Treat it as an existing // daemon: no competing tick, no activation; the caller restarts the unit so // the new config takes effect (#651, review #1). const ownership = deps.unitOwnership(); if (ownership.kind === "unknown") { // A probe failure is not the confirmed negative `inactive` claims to be: // collapsing `unknown` here is how a unit-owned daemon gets a second one // started beside it (#651, review #2). The mutating `--once` tick refuses // instead. throw new Error( `cannot prove whether a daemon is running: systemd ownership probe failed (${ownership.reason}) — ` + "refusing to run the setup smoke beside a possibly-live daemon", ); } if (ownership.kind === "active") { // No record to read a port from, so no /healthz proof is possible here: // the smoke reports `existing` honestly as *unproven*, and the caller // must restart the unit through the lifecycle seam — whose // `waitForOwnedDaemon` proves MainPID + `/healthz` — before the smoke // counts as passed (#651, review #2). return { mode: "existing", healthProven: false, status: deps.status(project), daemon: unrecordedDaemon(ownership.pid), }; } // Confirmed inactive (or failed, which owns no process): a paused `--once` // smoke can run by itself, then a temporary daemon proves the new runtime, // then it is stopped again. await deps.runOnce(project); const daemon = await deps.start(project); try { const health = await deps.health(daemon.port); if (!health.ok) throw new Error(`temporary daemon on :${daemon.port} did not answer /healthz`); return { mode: "temporary", healthProven: true, status: deps.status(project), daemon }; } finally { await deps.stop(); } }