/** * The bounded unit-template reconcile (#905). * * `upgrade` swaps the package that *renders* the host units, which silently * invalidates the installed copies on every release that touches a template — * and then hands the operator a chore: re-run `omp-conductor setup host` * yourself. That advisory (#598) is worse than a chore, because `setup host` * restarts `herdr-fleet.service` at the end and kills its own calling pane * (#834). This module is the other half of that finding: the smallest thing * that makes the installed units match the render, and nothing else. * * Two properties make it safe to run inside the upgrade transaction: * * - **It is bounded to unit templates.** The staged copies of the daemon * unit, the recovery unit and its playbook, the herdr session unit and the * harness mount unit; then `install` for exactly the destinations whose * live bytes differ, then one `daemon-reload`. No account creation, no * ACLs, no `enable`, no `restart`, no herdr config merge, no tick config, * no brief link — every one of those stays `setup host`'s, and a change * that needs them is reported rather than attempted. * - **It never overwrites an operator's edit.** The caller names the * destinations that were *already* drifting before the new package landed; * those are hand edits rather than drift this upgrade created, and they are * skipped and reported. A host whose units already match is left untouched * byte for byte, because the render-and-compare in * {@link HostRuntimePlan.drift} is what decides. * * It lives in its own module, and runs as its own CLI verb, because the render * is *code*: `upgradeConductor` replaces the package underneath itself, so * every in-process render after the install phase is still the OLD version's * templates. Only a freshly spawned process can render what the new release * actually ships, which is why the transaction shells out to this verb instead * of calling `writeHostRuntime` directly. */ import { writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { HARNESS_MOUNT_UNIT_NAME, RECOVER_SCRIPT_INSTALL_PATH, RECOVER_SERVICE_NAME, STAGED_SERVICE_NAME, type HostRuntimePlan, } from "./setup-host.ts"; import { DEFAULT_HERDR_UNIT } from "./fleet.ts"; import { runPrivileged, type PrivilegedDeps, type PrivilegedStep } from "./privileged.ts"; import type { WizardUi } from "./wizard-ui.ts"; /** The confirm key, so a non-interactive caller can answer it up front. */ export const UNIT_RECONCILE_ANSWER_KEY = "reconcile-units"; /** One staged file and the destination it belongs at. */ export interface UnitDestination { /** What this destination is, for the plan and the log. */ label: string; /** The staged copy under the state directory — the render's own bytes. */ source: string; /** The live path the system reads. */ installed: string; /** `install -m` mode: units are data, the playbook is executed. */ mode: string; /** The rendered content, so the staged copy can be written before install. */ content: string; } /** * Every destination this reconcile is allowed to touch, in dependency order: * the recovery playbook and unit before the daemon unit that names it in * `OnFailure=`, so systemd never reloads a unit referencing one it cannot * load. * * Deliberately absent, and not an oversight: the herdr pane-shell config and * the herdr-conductor `config.env` (fleet-account files whose refresh implies * a herdr restart), the worker identity, the tick config and the brief link. * Those are `setup host`'s to own. */ export function unitDestinations(plan: HostRuntimePlan): UnitDestination[] { const unitDir = dirname(plan.installedPath); const destinations: UnitDestination[] = [ { label: "recovery playbook", source: plan.recoverScript.path, installed: RECOVER_SCRIPT_INSTALL_PATH, mode: "0755", content: plan.recoverScript.content, }, { label: RECOVER_SERVICE_NAME, source: plan.recoverUnit.path, installed: join(unitDir, RECOVER_SERVICE_NAME), mode: "0644", content: plan.recoverUnit.content, }, { label: STAGED_SERVICE_NAME, source: plan.service.path, installed: plan.installedPath, mode: "0644", content: plan.service.content, }, ]; if (plan.herdrUnit !== undefined) { destinations.push({ label: DEFAULT_HERDR_UNIT, source: plan.herdrUnit.path, installed: join(unitDir, DEFAULT_HERDR_UNIT), mode: "0644", content: plan.herdrUnit.content, }); } return destinations; } export interface UnitReconcilePlan { /** Destinations that will be refreshed, in execution order. */ refresh: UnitDestination[]; /** * Host state this version retires, from {@link HostRuntimePlan.retire} (#895). * * It rides here rather than in a verb of its own because `upgrade` already * spawns this reconcile as a child of the just-installed CLI — the only * process that can render what the new release ships, and therefore the only * one that knows what it stopped shipping. Retirement is NOT drift: a unit * this version no longer renders cannot be compared against a render, so it * is never protected by `--protect` and never counted as out of scope. */ retire: readonly string[]; /** Drifted destinations left alone because the caller protected them. */ protectedDrift: string[]; /** Drifted destinations outside this reconcile's remit, named so the caller * can say precisely what still needs `setup host` — never a blanket * "re-run setup host". */ outOfScope: string[]; /** The privileged batch: one `install` per refreshed destination, then one * `daemon-reload`. Empty when nothing drifts. */ steps: PrivilegedStep[]; } /** * What reconciling this plan would do. Pure: it renders no file and runs no * command, so a caller can print it, test it, or decide there is nothing to do. * * `protect` names destinations whose installed bytes already differed from the * *previous* version's render — operator hand edits, which this never * overwrites. */ export function planUnitReconcile( plan: HostRuntimePlan, protect: readonly string[] = [], /** `false` retires only — the posture `upgrade` uses when it could not read * a drift baseline, so a refresh cannot tell an operator's edit from its * own, but a retirement still must not be skipped (#895). */ refreshDestinations = true, ): UnitReconcilePlan { const protectedSet = new Set(protect); const drifted = new Set(plan.drift); const known = unitDestinations(plan); const refresh: UnitDestination[] = []; const protectedDrift: string[] = []; for (const destination of known) { if (!refreshDestinations) continue; if (!drifted.has(destination.installed)) continue; if (protectedSet.has(destination.installed)) { protectedDrift.push(destination.installed); continue; } refresh.push(destination); } const mine = new Set(known.map((d) => d.installed)); const outOfScope = refreshDestinations ? plan.drift.filter((path) => !mine.has(path)) : []; const retireSteps = plan.retire?.steps ?? []; const steps: PrivilegedStep[] = refresh.length === 0 && retireSteps.length === 0 ? [] : [ // Retirement first, and before any install: an obsolete unit is // disabled and unmounted while the daemon this reconcile serves is // still the one the caller drained for. ...retireSteps, ...refresh.map((destination) => ({ title: `install ${destination.label} (${destination.installed})`, argv: ["install", "-m", destination.mode, destination.source, destination.installed], })), // One reload for the whole batch: systemd re-reads every unit file, // so per-unit reloads would be the same work repeated. No `enable` // and no `restart` — the units are already enabled, and restarting // is the caller's decision, taken inside its own drain window. { title: "reload systemd", argv: ["systemctl", "daemon-reload"] }, ]; const retire = plan.retire === undefined ? [] : [...plan.retire.units, ...plan.retire.staged]; return { refresh, retire, protectedDrift, outOfScope, steps }; } export type UnitReconcileOutcome = /** Nothing drifted and nothing to retire: the host already matches this * version. */ | { kind: "current"; protectedDrift: string[]; outOfScope: string[] } /** The batch ran. `refreshed` are the destinations now matching the render, * and `retired` the host state this version removed (#895). */ | { kind: "reconciled"; refreshed: string[]; retired: string[]; protectedDrift: string[]; outOfScope: string[] } /** The confirm was declined; nothing was written or installed. */ | { kind: "declined"; pending: string[] } /** A step failed. `failed` names it; nothing after it ran. */ | { kind: "failed"; failed: string; detail: string; remaining: string[] }; /** * Stage the rendered unit files and install the ones whose live bytes differ. * * The staged writes happen first and unprivileged (they land under the state * directory the fleet account owns), and only for destinations that will * actually be installed — so a declined confirm leaves the state directory as * it was, and a current host is not rewritten just to prove it is current. */ export async function reconcileUnits( plan: HostRuntimePlan, ui: WizardUi, options: { protect?: readonly string[]; /** `false` retires only, refreshing nothing (#895). */ refresh?: boolean; /** Pre-answered confirm for a non-interactive caller (the upgrade * transaction, whose operator already consented to the whole upgrade). */ deps?: PrivilegedDeps; /** Injected in tests; defaults to a real file write. */ write?: (path: string, content: string) => void; } = {}, ): Promise { const reconcile = planUnitReconcile(plan, options.protect ?? [], options.refresh ?? true); if (reconcile.steps.length === 0) { return { kind: "current", protectedDrift: reconcile.protectedDrift, outOfScope: reconcile.outOfScope }; } const write = options.write ?? ((path: string, content: string) => writeFileSync(path, content)); const outcome = await runPrivileged(reconcile.steps, ui, { ...(options.deps === undefined ? {} : { deps: options.deps }), title: "Install the re-rendered host units now?", answerKey: UNIT_RECONCILE_ANSWER_KEY, preamble: [ ...(reconcile.refresh.length === 0 ? [] : ["This release renders host unit files that differ from the installed copies."]), ...(reconcile.retire.length === 0 ? [] : [`This release retires host state an earlier one installed: ${reconcile.retire.join(", ")}.`]), "Units only: nothing is enabled, restarted, or re-provisioned here.", ], // Staged copies are written under the same consent as the install that // reads them, and after it — never before a declined confirm. beforeRun: async () => { for (const destination of reconcile.refresh) write(destination.source, destination.content); }, }); if (outcome.kind === "declined") { return { kind: "declined", pending: [...reconcile.retire, ...reconcile.refresh.map((d) => d.installed)] }; } if (outcome.kind === "failed") { return { kind: "failed", failed: outcome.step.title, detail: outcome.stderr.trim().split("\n")[0] ?? `exit ${outcome.exitCode}`, remaining: outcome.remaining.map((step) => step.title), }; } return { kind: "reconciled", refreshed: reconcile.refresh.map((d) => d.installed), retired: [...reconcile.retire], protectedDrift: reconcile.protectedDrift, outOfScope: reconcile.outOfScope, }; } /** One line per outcome, for the CLI and the upgrade journal. */ export function formatUnitReconcile(outcome: UnitReconcileOutcome): string { const tail = (protectedDrift: readonly string[], outOfScope: readonly string[]): string => [ protectedDrift.length === 0 ? "" : ` left alone (modified outside conductor): ${protectedDrift.join(", ")}.`, outOfScope.length === 0 ? "" : ` still owed to \`omp-conductor setup host\`: ${outOfScope.join(", ")}.`, ].join(""); switch (outcome.kind) { case "current": return `host units: already match this version's render.${tail(outcome.protectedDrift, outcome.outOfScope)}`; case "reconciled": return ( (outcome.refreshed.length === 0 ? "host units: nothing to refresh." : `host units: reconciled ${outcome.refreshed.length} destination(s) — ${outcome.refreshed.join(", ")}.`) + (outcome.retired.length === 0 ? "" : ` Retired: ${outcome.retired.join(", ")}.`) + tail(outcome.protectedDrift, outcome.outOfScope) ); case "declined": return `host units: not reconciled (declined) — still drifted: ${outcome.pending.join(", ")}.`; case "failed": return ( `host units: reconcile failed at "${outcome.failed}" (${outcome.detail}); ` + `${outcome.remaining.length} step(s) did not run.` ); } }