/** * The durable journal for fleet-initiated upgrades (#486). * * `omp-conductor upgrade` has always been a transaction *inside* one process: * it paused, drained, installed, reloaded and verified, and the process that * verified was the process that restarted the fleet. The fleet-installs-itself * path replaces that wrapper, not the transaction: a detached transient unit * performs the install while this module records every surface it touches, and * the first tick of the post-restart daemon verifies against the journal — * a process that did not run the install, which is the only witness worth * trusting after the restarts. * * Everything here is a leaf on purpose: the verb server, the daemon's * privileged actions and the upgrade engine all need the same journal and the * same transient-unit naming, and importing `upgrade.ts` from both verbs and * daemon would close import cycles that have no business existing. The module * holds no state of its own; the journal is append-only JSONL under the * conductor state directory, so a process that died mid-write leaves every * line before the torn one readable. */ import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { stateDir } from "./config.ts"; import type { ResolvedGrants } from "./types.ts"; /** * Where the fleet's upgrade journal lives: one JSONL file per host, because * the transaction it records is host-wide — one daemon serves every configured * project, and the install replaces every surface at once. */ export const UPGRADE_JOURNAL_FILE = "upgrade-journal.jsonl"; /** The packages this fleet never leaves on mixed versions. */ export interface JournalSurfaces { cliVersion: string; ompVersion?: string; herdrSource?: string; } /** What the fleet looked like when the install began, for delta verification. */ export interface JournalFleetState { dispatch: string; ticks: string; pane: string; herdr: string; } /** One named verification check, recorded so the outcome is evidence, not prose. */ export interface UpgradeCheck { name: string; ok: boolean; detail?: string; } export interface UpgradeJournalEntry { /** ISO timestamp of the write. */ at: string; kind: "request" | "snapshot" | "phase" | "outcome"; /** Always present for request/outcome; phases carry them for searchability. */ version?: string; gitHead?: string; /** The transient unit performing the install, when it named itself. */ unit?: string; /** The step name: paused, drain, install, brief, restart, awaiting-verification, … */ phase?: string; /** The surface one phase touched: cli/omp/herdr/brief/herdr-service/daemon/dispatch. */ surface?: string; ok: boolean; detail?: string; /** The durable pre-install identities, for the process that must roll back. */ previous?: InstallSnapshotSurfaces; configBackup?: string; /** The pause sentinel the install owns, for the verifier that must lift it. */ pauseKey?: string; /** Whether dispatch was already paused when the request began. */ initialPaused?: boolean; /** Creation instant of the exact pause sentinel this transaction acquired. */ pauseSince?: number; /** The fleet layers the install began under, for delta verification. */ initial?: JournalFleetState; /** The projects the install covers, in the form the upgrade engine uses. */ selectors?: readonly (string | null)[]; /** Host-global install authority resolved before the detached unit started. */ installHolder?: ResolvedGrants["install"]; /** Every configured project affected by the host-global install, report-only. */ installProjects?: readonly string[]; /** Verification evidence when the post-restart tick closes the transaction. */ checks?: readonly UpgradeCheck[]; } /** Paths and roots discovered before the first install mutation (#1019). */ export interface InstallSnapshotManagement { cli: { executable: string; target: string; root: string; }; omp: { root: string; lock: string; }; herdr: { root?: string; installRoot: string; }; } /** The installed identities a rollback must restore, verbatim. */ export interface InstallSnapshotSurfaces { cliVersion: string; ompVersion?: string; herdrSource?: string; management?: InstallSnapshotManagement; } export function upgradeJournalPath(root = stateDir()): string { return join(root, UPGRADE_JOURNAL_FILE); } export function readUpgradeJournal(root = stateDir()): UpgradeJournalEntry[] { const path = upgradeJournalPath(root); if (!existsSync(path)) return []; let text: string; try { text = readFileSync(path, "utf8"); } catch { // An unreadable journal is the same answer as an absent one for the read // side; the writer keeps appending and the operator can recover the file. return []; } const entries: UpgradeJournalEntry[] = []; for (const line of text.split("\n")) { if (line.length === 0) continue; try { const entry = JSON.parse(line) as UpgradeJournalEntry; if (entry !== null && typeof entry === "object") entries.push(entry); } catch { // One torn line (the process died mid-append) does not hide the earlier // evidence; it is exactly the "interrupted transaction" the journal was // built to diagnose. } } return entries; } /** * Append one line. Each write is one `appendFileSync`, so a crash cannot * interleave two entries; callers order their own phases. */ export function appendJournal(root: string, entry: UpgradeJournalEntry): void { mkdirSync(root, { recursive: true }); appendFileSync( join(root, UPGRADE_JOURNAL_FILE), `${JSON.stringify(entry)}\n`, { mode: 0o600 }, ); } /** * The transient systemd unit that performs an install or its rollback. The * unit lives in the system manager, not in the pane's process tree and not in * the daemon's cgroup, so `systemctl restart herdr-fleet.service` and the * daemon restart both pass it by. The version is slugged into the unit name * because the unit names the transaction; a second request for the same * version collides with the first on purpose (see the launch guard). */ export function transientUnitName(version: string, kind: "install" | "rollback"): string { const slug = version.replace(/[^a-zA-Z0-9_.-]/g, "-"); return `omp-conductor-upgrade-${kind}-${slug}`; } /** * The explicit PATH a detached unit must carry. On this fleet's host `omp`, * `herdr` and `omp-conductor` are absent from the non-interactive ssh PATH, * and a transient unit starts with systemd's own minimal PATH — so the * executor prepends the two well-known runtime dirs and keeps whatever the * daemon itself ran with. Deduplicated so a dir already on the path is not * listed twice. */ const EXECUTOR_PATH_DIRS = ["/root/.bun/bin", "/root/.local/bin"]; export function executorPath(env: NodeJS.ProcessEnv): string { const existing = typeof env["PATH"] === "string" ? env["PATH"] : ""; const parts = [...EXECUTOR_PATH_DIRS, ...existing.split(":").filter((part) => part.length > 0)]; return [...new Set(parts)].join(":"); } export type UnitSpawnResult = { code: number; stdout: string; stderr: string }; export type UnitSpawnFn = (argv: readonly string[]) => Promise; export type UnitCommand = | { kind: "install"; version: string; holder?: ResolvedGrants["install"]; projects?: readonly string[]; } | { kind: "rollback"; version: string }; export interface LaunchedUnit { ok: true; unit: string; } export interface UnitLaunchRefusal { ok: false; stderr: string; } /** * Start one detached transient unit that performs an install or a rollback. * * The environment is explicit and minimal on purpose: `PATH` is composed by * {@link executorPath} so the unit can find `omp`, `herdr` and * `omp-conductor`, and the requesting context's `HERDR_SESSION` (if any) is * passed through so the fleet the unit talks to is the fleet the request * meant. The unit is `--collect`ed so a finished unit leaves no failed-unit * litter behind. The spawning process keeps the unit name; the unit is told * through `OMP_CONDUCTOR_UNIT` so its journal records which unit ran. */ export async function launchTransientUnit( spawn: UnitSpawnFn, env: NodeJS.ProcessEnv, command: UnitCommand, ): Promise { const unit = transientUnitName(command.version, command.kind); const setenv = [`PATH=${executorPath(env)}`, `OMP_CONDUCTOR_UNIT=${unit}`]; const session = env["HERDR_SESSION"]; if (session !== undefined && session.length > 0) setenv.push(`HERDR_SESSION=${session}`); if (command.kind === "install" && command.holder !== undefined) { setenv.push(`OMP_CONDUCTOR_INSTALL_HOLDER=${command.holder}`); } if (command.kind === "install" && command.projects !== undefined) { setenv.push(`OMP_CONDUCTOR_INSTALL_PROJECTS=${command.projects.join(",")}`); } const argv = [ "systemd-run", `--unit=${unit}`, "--collect", ...setenv.map((pair) => `--setenv=${pair}`), "omp-conductor", command.kind === "install" ? "upgrade-install" : "upgrade-rollback", ...(command.kind === "install" ? ["--to", command.version] : []), ]; const ran = await spawn(argv); if (ran.code !== 0) { const detail = ran.stderr.trim() || ran.stdout.trim() || `exit ${ran.code}`; return { ok: false, stderr: `could not start ${unit} through systemd-run: ${detail}. ` + "Nothing was installed and dispatch was not touched; run `omp-conductor upgrade` from a shell instead.", }; } return { ok: true, unit }; }