/** * `doctor` — mechanically check the deployment faults that have actually cost * debugging sessions, before the fleet hits them (#287, legibility 1/5). * * Every production incident this reports on was environmental, silent, and * cheap to detect: a systemd unit whose user/working-directory did not match * the host (#152/#153 class), case-mismatched routing labels documented to * fail silently (README Limitations), spend telemetry absent so the USD cap * never fired, `gh` auth expiring under a live daemon. None of them had a * check before the incident; this module is that check. * * Two properties are load-bearing and enforced here rather than documented: * * - **Read-only by default.** The store is opened as a `readonly` sqlite * connection; nothing writes config, labels, systemd, or the database. * The single side effect anywhere is the self-identified Telegram probe * message, and only `--probe-telegram` enables it. * - **Every probe is injectable.** The runner composes probes through * {@link DoctorDeps}; the same seams the rest of the package uses — the * tracker's own `gh` transport, `probeTelegramHealth`, the report * outbox's send, the `renderDaemonService` runtime plan — are what a * default run wires, so `doctor` cannot disagree with the daemon about a * shared fact. Tests inject every probe, and no test touches the host's * systemd unit, daemon, Telegram, GitHub account, or production state * (the #399 boundary). */ import { accessSync, constants, existsSync, readdirSync, readFileSync, statSync, } from "node:fs"; import { spawnSync } from "node:child_process"; import { homedir } from "node:os"; import { dirname, join, sep } from "node:path"; import type { Stats } from "node:fs"; import { Database } from "bun:sqlite"; import { stringify } from "yaml"; import { configBackupDir, configPath, dbBackupDirFor, findProject, loadConfig, resolveArmProof, resolveCaps, stateDir, } from "./config.ts"; import { observeArmChallenge } from "./arm-challenge.ts"; import { ompSettingsOverlay, sessionRootDir } from "./omp-settings.ts"; import { DEFAULT_HERDR_UNIT, probeTelegramHealth, resolveHerdrSession, resolveTickConfig, sessionsRoot, sessionDirForCwd, telegramStateDir, } from "./fleet.ts"; import { errText } from "./log.ts"; import { formatDownDuration } from "./orchestrator-down.ts"; import { pauseInstance, pauseInstanceAt, pausedPath } from "./pause.ts"; import type { TelegramHealth } from "./status-render.ts"; import { claimIsLive, claimedTelegramTopics, lockPidAlive, pidAlive, killOf, readTelegramChannel, readTelegramDmOwner, readTelegramMisroutes, readTelegramPollState, resolveClaimedSessionFile, resolveProjectClaim, resolveProjectTopicId, telegramPlumbingVerdict, telegramTopicsTidy, TELEGRAM_LOCK_FRESH_MS, type ClaimedTopicsResult, type TelegramChannelState, type TelegramDmOwnerState, type TelegramPollState, type TelegramMisroute, } from "./escalate.ts"; import { herdrConductorPluginConfigDir, planHostRuntime, RECOVER_SERVICE_NAME, STAGED_SERVICE_NAME, SYSTEMD_UNIT_DIR, tickCwdForProject, totalConfiguredWorkers, } from "./setup-host.ts"; import { checkTokenScopes, type ScopeCheck } from "./setup.ts"; import { DB_SNAPSHOT_STEM, dbPath, LIVE_STATES } from "./store.ts"; import { telegramReportSend, type ReportSend } from "./reports.ts"; import { fetchRateLimit, GhError, gh } from "./tracker/github.ts"; import { repoSlugFor } from "./gitops.ts"; import { DEFAULT_FLEET_AGENT_NAME, parseHerdrAgents, readTickConfig, type HerdrAgentList, } from "./orchestrator-tick.ts"; import type { Caps, ConductorConfig, Escalation, ProjectConfig, RepoTarget, RunState } from "./types.ts"; import { DEFAULT_ARM_PROOF, type ArmProof } from "./types.ts"; import { DEFAULT_DEPS as UPGRADE_DEPS, expectedHerdrSource, inspectSurfaces, releaseIdentity, type InstalledSurfaces, } from "./upgrade.ts"; import { TELEGRAM_PACKAGE, checkTelegramFreshness, type TelegramFreshness, } from "./telegram-freshness.ts"; import { bindingAllowanceWindow, sharedUsageSource } from "./usage.ts"; import { judgeSpendTelemetry, spendTelemetryDetail, type SpendBilling } from "./spend-telemetry.ts"; /** One read of this host's installed surfaces: the three identities, or why * they could not be read. A failed read is a finding (`warn`, "unverified"), * never a thrown doctor run and never a silent pass (#904). */ export type SurfaceRead = | { ok: true; surfaces: InstalledSurfaces } | { ok: false; detail: string }; /** * `doctor`'s finding vocabulary. `pass`/`warn`/`fail` — a warning renders the * same but never flips the exit code; exit is nonzero iff any finding fails. */ export type FindingStatus = "pass" | "warn" | "fail"; export type ReportStatus = "pass" | "warn" | "fail"; /** One checked fact: id (stable for CI), status, summary, and a one-line fix. */ export interface Finding { /** Stable identifier — the JSON shape is a CI contract, so ids never move. */ id: string; status: FindingStatus; summary: string; /** The one-line remedy; absent when the finding passes. */ fix?: string; } /** * The stable report shape `--json` prints and the human renderer reads. * * `checkedAt` is the UTC ISO instant the run started; `status` is `fail` iff * any finding is `fail`, else `warn` iff any is `warn`, else `pass`. */ export interface DoctorReport { project: string; checkedAt: string; status: ReportStatus; findings: Finding[]; } /** How many *working* runs the spend-telemetry judgement wants. */ export const SPEND_SAMPLE_RUNS = 5; /** * How many rows to read to find them (#970). * * Over-sampled deliberately: the judgement discards 0-turn runs, and on this * project 26 of 48 zero-spend rows are exactly those. Reading only * `SPEND_SAMPLE_RUNS` rows would let a burst of administrative kills push the * real evidence out of the window and report "nothing to judge yet" on a fleet * that had plenty to judge. */ export const SPEND_SAMPLE_ROWS = SPEND_SAMPLE_RUNS * 4; /** The past incidents this doctor flags, quoted in the finding so an operator * who never lived them knows which failure mode the check exists to prevent. */ const INCIDENTS = { systemd: "the unit/user mismatch class that once killed workers at turn 0 with an empty agent.db (#152/#153)", labels: "label matching is exact and case-sensitive, and a case-mismatched label is documented to fail silently (README Limitations)", spend: "spend telemetry was once absent, so the USD cap never fired — $0.00 spend is not proof of no spend", ghauth: "gh auth expired under a live daemon and every tracker call failed silently", } as const; // ------------------------------------------------------------------ dependencies export interface GhRepoRead { ok: boolean; detail?: string; } export interface GhLabelsRead { ok: boolean; labels?: string[]; detail?: string; } /** The canonical units `setup host`/`upgrade` would stage right now. */ export interface CanonicalUnits { daemon: string; herdr?: string; /** The account the daemon unit renders with, for the ownership checks. */ username: string; } /** One sampled run row: `state` keeps spend from in-flight rows, which are not * a telemetry statement yet, and `turns` separates a run that reported nothing * from one that genuinely did nothing (#970). */ export interface RunSpendRow { state: RunState; spendUsd: number; turns: number; } /** Injectable seams. Every field defaults to the production wiring, which * reuses the package's existing transports — no second GitHub client, no new * Telegram HTTP call, no re-parsed status. */ export interface DoctorDeps { /** Defaults to {@link loadConfig}; a loader that throws is a finding, not a crash. */ loadConfig?: () => ConductorConfig; /** `gh auth status` scope read — the wizard's own check, so setup consent * and `doctor` cannot disagree about what the token may do. */ scopes?: () => Promise; /** `gh api rate_limit` succeeds — the tracker's own liveness probe. */ rateLimitOk?: () => Promise; /** Whether `gh` can read one configured repo (scopes cover it). */ repoReadable?: (repo: string) => Promise; /** The exact labels present in one repo, for the exact-case check. */ repoLabels?: (repo: string) => Promise; /** True when this is a systemd host (`/etc/systemd/system` exists). */ hasSystemd?: () => boolean; /** The installed unit text at an absolute path, or undefined when absent. */ readUnit?: (path: string) => string | undefined; /** Whether one installed systemd unit is currently failed; undefined when unreadable. */ unitFailed?: (name: string) => boolean | undefined; /** stat one path; undefined when it does not exist. */ stat?: (path: string) => Stats | undefined; /** uid of a username (`id -u`), or undefined when it cannot be resolved. */ uidOf?: (name: string) => number | undefined; /** `PRAGMA integrity_check` over the store, read-only. ok=true for "ok". */ dbIntegrity?: (path: string) => { ok: boolean; detail?: string }; /** Whether the configured db-snapshot directory exists and is writable, so a * snapshot can land. Injected so a test never chmods the host (#399). */ snapshotDirState?: (dir: string) => { exists: boolean; writable: boolean }; /** Whether the run-session root — where the omp settings overlay is * materialised at dispatch (#537) — can hold a new `run-` directory. A * missing `sessions/` parent is not a fault (dispatch's recursive `mkdir` * creates it), so the writable question falls back to the state dir itself. */ sessionRootState?: (root: string) => { exists: boolean; writable: boolean }; /** The most recent completed runs for one project, newest first, limited. */ recentRuns?: (project: string, limit: number) => RunSpendRow[]; /** The bot health probe `status` uses (getMe etc.). */ telegramHealth?: (projectName: string) => Promise; /** The report-transport send, per project; only ever invoked with * `--probe-telegram`. Defaults to the report outbox's own transport. */ telegramSend?: (project: ProjectConfig) => ReportSend; /** The canonical units to compare the installed units against. */ canonicalUnits?: (project: ProjectConfig, cfg: ConductorConfig) => CanonicalUnits; /** Whether herdr is installed on this host (a `herdr` on PATH). */ herdrInstalled?: () => boolean; /** * Whether a worker session could load its harness at all (#910), exercised * the way a worker does: the installed peer resolved from this package's own * directory under `bun --no-install`, so neither import can fall through to * Bun's ambient cache. `ok: false` means no worker on this host can start. */ workerHarness?: () => Promise<{ ok: true; version: string } | { ok: false; detail: string }>; /** * The pause sentinel for one project as an instance (#938): who set it, why, * when, and — for a fence whose lifetime is one process's — the owning pid. */ pauseFence?: (project?: string) => { source: string; reason?: string; since: number; owner?: number } | undefined; /** Whether one pid is a live process. Injected so the abandoned-fence * finding is testable without spawning anything. */ pidLive?: (pid: number) => boolean; /** * Whether any conductor process is running an upgrade transaction right * now (#1113) — the liveness fact an owner-less `upgrade` sentinel needs, * because it records no pid of its own. Injected so the orphaned-upgrade * fault is testable without scanning /proc. */ upgradeLive?: () => boolean; /** Live `herdr --session agent list`, parsed through the tick's own * parser (the same "one JSON line on stdout" contract recover.sh reads). */ herdrAgents?: (session: string) => HerdrAgentList; /** The fleet session name, for the rename fix line. */ herdrSession?: () => string; /** The live herdr config at `~/.config/herdr/config.toml`, or undefined. */ herdrConfig?: () => string | undefined; /** The live herdr-conductor plugin `config.env`, or undefined. */ herdrEnv?: () => string | undefined; /** Live omp-telegram topic claims (threads.json), for the #600 probes. * `unavailable` when the registry cannot be read or does not parse — the * caller must not read that as "no claims" (#626). */ claimedTopics?: () => ClaimedTopicsResult; /** Recorded flat-chat deliveries this project's stale pin already caused * (#1094), read from conductor state. The topic-pin row renders them as * observed evidence until the pin is repaired — rows match the *current* * pin id, so re-pinning is what retires them. */ topicMisroutes?: () => readonly TelegramMisroute[]; /** Whether the bridge's access.json runs per-session topic tidy on. */ topicsTidy?: () => boolean; /** The typed bot.lock poll-ownership read, told apart so absent/malformed * state is a #612 failure — the lock, not daemon.json, is what says whether * the bridge is actually polling. */ pollState?: () => TelegramPollState; /** The typed dm-owner.json read — the untopiced-DM recipient a flat * challenge's reply routes through (`/telegram own`). Absent/malformed is a * named "no pinned owner", not a pass. */ dmOwner?: () => TelegramDmOwnerState; /** The paired inbound channel state, probed exactly as armTicks probes it * (readTelegramChannel over the tick's accessFile). */ channelState?: (project: ProjectConfig) => TelegramChannelState; /** Whether the project can ride inbound at all — a readable tick config * naming an accessFile. The tick runtime treats a missing accessFile as no * fleet bridge, so that shape is neutral, never a fault. */ projectArms?: (project: ProjectConfig) => boolean; /** The topic id an arm challenge would send into for this project — * resolveProjectTopicId's pin-or-live-claim; undefined means flat chat. */ armSendTopic?: (project: ProjectConfig) => number | undefined; /** The session surface the claim-only plumbing verdict judges for this * project — tick-cwd-derived plus the live claim's own directory. The * challenge proof reads conductor state instead (#614), so this feeds * claim-only verdict checks only. */ armScanDirs?: (project: ProjectConfig) => readonly string[]; /** Whether a recorded claim or dm-owner pid is live, with omp-telegram's * topics.ts semantics (EPERM is dead). The one claim-liveness fact in this * package: topic-pin's dead partition, its exact-pin check, project * resolution and the telegram-plumbing verdict all derive from it, so two * rows cannot answer one claim differently (#987). */ pidAlive?: (pid: number) => boolean; /** The provider allowance window nearest its ceiling, from `omp usage --json` * — what binds a subscription-billed fleet instead of a dollar cap (#984). * `undefined` when the provider reports no comparable window. */ allowanceWindow?: () => Promise; /** Whether a bot.lock owner pid is live, with omp-telegram's api.ts * semantics (EPERM is live). Distinct from {@link pidAlive} because the * bridge itself uses two rules. */ lockPidAlive?: (pid: number) => boolean; /** Whether a lock-file mtime is younger than LOCK_FRESH_MS — the bridge's * own liveness rule: a fresh heartbeat keeps the poll live even when the * recorded pid is gone. */ lockFresh?: (mtimeMs: number) => boolean; /** The fleet agent name the tick config of one project names, or undefined * when there is no (readable) tick — the expected live herdr pane identity. */ tickAgentName?: (project: ProjectConfig) => string | undefined; /** The three installed identities this host carries, read through * `upgrade`'s own seam; a failed read is a named finding, never a throw. */ installedSurfaces?: () => Promise; /** The `omp-telegram` install/daemon/published triple (#961). */ telegramFreshness?: () => Promise; /** The commit one published version was cut from, for judging the herdr * plugin's pin; `undefined` when the registry cannot be read. */ releaseGitHead?: (version: string) => Promise; /** Clock, so a run is deterministic in tests. */ now?: () => number; /** The one opt-in side effect: send one self-identified Telegram probe. */ probeTelegram?: boolean; } /** The resolved probe set a run uses (defaults merged with the overrides). */ export type Probes = Required> & Pick; function messageOf(err: unknown): string { return err instanceof Error ? err.message : String(err); } function firstLineOf(stderr: string): string { const line = stderr.split("\n").find((l) => l.trim().length > 0)?.trim() ?? ""; return line.length > 120 ? `${line.slice(0, 120)}…` : line; } /** A gh failure → a bounded, human line (never a credential or a stack). */ function ghFailureDetail(err: unknown): string { if (err instanceof GhError) return firstLineOf(err.stderr); return messageOf(err); } // --------------------------------------------------------------- production wiring async function defaultRepoReadable(repo: string): Promise { try { await gh(["api", `repos/${repo}`]); return { ok: true }; } catch (err) { return { ok: false, detail: ghFailureDetail(err) }; } } async function defaultRepoLabels(repo: string): Promise { try { const raw = await gh(["label", "list", "--repo", repo, "--limit", "1000", "--json", "name"]); const parsed: unknown = JSON.parse(raw); if (!Array.isArray(parsed)) return { ok: false, detail: "gh label list returned an unexpected shape" }; const labels: string[] = []; for (const row of parsed) { const name = (row as { name?: unknown } | null)?.name; if (typeof name === "string" && name.length > 0) labels.push(name); } return { ok: true, labels }; } catch (err) { return { ok: false, detail: ghFailureDetail(err) }; } } function defaultReadUnit(path: string): string | undefined { try { return readFileSync(path, "utf8"); } catch { return undefined; } } export function systemdStateFailed(state: string): boolean | undefined { const normalized = state.trim(); if (normalized === "") return undefined; return normalized === "failed"; } function defaultUnitFailed(name: string): boolean | undefined { const ran = spawnSync("systemctl", ["is-failed", name], { encoding: "utf8" }); if (ran.error !== undefined) return undefined; return systemdStateFailed(ran.stdout ?? ""); } function defaultStat(path: string): Stats | undefined { try { return statSync(path); } catch { return undefined; } } function defaultUidOf(name: string): number | undefined { const ran = spawnSync("id", ["-u", name], { encoding: "utf8" }); if (ran.status !== 0) return undefined; const uid = Number.parseInt((ran.stdout ?? "").trim(), 10); return Number.isInteger(uid) && uid >= 0 ? uid : undefined; } /** * `PRAGMA integrity_check`, opened **read-only** so the check itself can never * mutate the store it is inspecting. A missing file is a fresh host, not a * fault; a file that is not sqlite, or a check that reports anything but * `ok`, is. */ function defaultDbIntegrity(path: string): { ok: boolean; detail?: string } { if (!existsSync(path)) return { ok: true, detail: "no runs store yet — nothing to check" }; let database: Database; try { database = new Database(path, { readonly: true }); } catch (err) { return { ok: false, detail: `store cannot be opened read-only: ${messageOf(err)}` }; } try { const rows = database.query<{ integrity_check: string }, []>("PRAGMA integrity_check").all(); for (const row of rows) { if (row["integrity_check"] !== "ok") { return { ok: false, detail: `integrity_check: ${row["integrity_check"]}` }; } } return { ok: true }; } catch (err) { return { ok: false, detail: `integrity_check failed: ${messageOf(err)}` }; } finally { database.close(); } } /** * Whether the configured db-snapshot directory exists and is writable. No * write is performed — the check is stat + access, keeping doctor read-only. * A directory that does not exist reads `writable: false` and lets the probe * report it as "no snapshot yet" rather than inventing one, while a read-only * or non-directory path reads as a genuine unwritable failure. */ function defaultSnapshotDirState(dir: string): { exists: boolean; writable: boolean } { try { const st = statSync(dir); if (!st.isDirectory()) return { exists: true, writable: false }; try { accessSync(dir, constants.W_OK); return { exists: true, writable: true }; } catch { return { exists: true, writable: false }; } } catch { return { exists: false, writable: false }; } } /** * Whether the run-session root can hold a new session directory. No write is * performed — the same stat + access as {@link defaultSnapshotDirState}, which * keeps `doctor` read-only. A missing `sessions/` parent is normal on a fresh * host (dispatch's recursive `mkdir` creates it), so the writable question * falls back to the state dir it would be created under; a state dir that * cannot even be read is a genuine unwritable failure. */ function defaultSessionRootState(root: string): { exists: boolean; writable: boolean } { const writable = (dir: string): boolean => { try { const st = statSync(dir); if (!st.isDirectory()) return false; accessSync(dir, constants.W_OK); return true; } catch { return false; } }; try { const st = statSync(root); return { exists: true, writable: st.isDirectory() && writable(root) }; } catch { return { exists: false, writable: writable(dirname(root)) }; } } /** * The most recent completed runs for one project, newest first — read on the * same read-only connection the integrity check opens, because the store's own * methods mutate on open and `doctor` is read-only. The state filter uses the * store's own `LIVE_STATES` vocabulary, so `doctor` never re-decides what * "completed" means. */ function defaultRecentRuns(project: string, limit: number): RunSpendRow[] { const path = dbPath(); if (!existsSync(path)) return []; let database: Database; try { database = new Database(path, { readonly: true }); } catch { return []; } try { const placeholders = [...LIVE_STATES].map(() => "?").join(", "); const rows = database .query<{ state: string; spendUsd: number; turns: number }, [string, ...string[], number]>( `SELECT state, spendUsd, turns FROM runs WHERE project = ? AND state NOT IN (${placeholders}) ORDER BY startedAt DESC, rowid DESC LIMIT ?`, ) .all(project, ...LIVE_STATES, limit); const out: RunSpendRow[] = []; for (const row of rows) { const state = row.state as RunState; // A state vocabulary the store never had is not "completed". if (!LIVE_STATES.includes(state)) { out.push({ state, spendUsd: row.spendUsd, turns: row.turns }); } } return out; } catch { return []; } finally { database.close(); } } /** The canonical units — exactly what `setup host`/`upgrade` stage. */ function defaultCanonicalUnits(project: ProjectConfig, cfg: ConductorConfig): CanonicalUnits { const caps = resolveCaps(project, cfg.defaults); const plan = planHostRuntime(project, caps, telegramStateDir(), undefined, totalConfiguredWorkers(cfg)); const username = /^User=(.*)$/m.exec(plan.service.value)?.[1]?.trim() ?? ""; return { daemon: plan.service.value, ...(plan.herdrUnit === undefined ? {} : { herdr: plan.herdrUnit.value }), username, }; } // --------------------------------------------------------------- finding helpers function passFinding(id: string, summary: string): Finding { return { id, status: "pass", summary }; } function warnFinding(id: string, summary: string, fix: string): Finding { return { id, status: "warn", summary, fix }; } function failFinding(id: string, summary: string, fix: string): Finding { return { id, status: "fail", summary, fix }; } // ------------------------------------------------------------------ probes function configProbe(problem: string | undefined): Finding { if (problem === undefined) return passFinding("config", "config.json loads and validates"); return failFinding( "config", `config.json does not load: ${problem}`, "repair it (run `omp-conductor setup`) before any other check can mean anything", ); } /** * The config backup (#193): a pre-write copy at least as fresh as the file it * protects. A config that was never written through the atomic writer — or * that was hand-edited after the last save — has no backup for the bad next * write to be rolled back from. */ function backupProbe(probes: Probes): Finding { const backups = (() => { try { return readdirSync(configBackupDir()).filter((name) => name.startsWith("config.json.bak-")); } catch { return []; } })(); const config = probes.stat(configPath()); if (config === undefined) { return backups.length === 0 ? passFinding("config-backup", "no config.json yet — nothing to back up") : passFinding("config-backup", `${backups.length} config backup(s) present`); } if (backups.length === 0) { return warnFinding( "config-backup", "config.json has no backup — the next bad config write destroys the only copy", "re-save the config (run `omp-conductor setup`) so writeConfigRaw files a pre-write backup", ); } const newestMtime = backups .map((name) => probes.stat(join(configBackupDir(), name))?.mtimeMs ?? 0) .reduce((max, m) => Math.max(max, m), 0); if (config.mtimeMs > newestMtime + 60_000) { return warnFinding( "config-backup", `config.json is newer than every backup (${backups.length} file(s)) — the newest edit was never backed up`, "save config.json through `omp-conductor setup` so a fresh pre-write backup lands", ); } return passFinding("config-backup", `config backup is fresh (${backups.length} file(s))`); } /** The store's `PRAGMA integrity_check` — corruption was previously silent. */ function dbProbe(probes: Probes): Finding { const result = probes.dbIntegrity(dbPath()); if (result.ok) { return passFinding( "db", result.detail === undefined ? "conductor.db integrity is ok" : `conductor.db: ${result.detail}`, ); } return failFinding( "db", `conductor.db failed its integrity check: ${result.detail ?? "unknown"} — a corrupt store has been silent until nothing dispatches`, "stop the daemon, restore the store from a backup, or rebuild it", ); } /** * The conductor.db snapshot store (#579): the configured dir is writable so a * snapshot can land, and a snapshot at least as fresh as the store protects * it. The store is the carrier of the verb ledger, the decision rows, the run * rows and the material-event ledger; a store with no restorable copy has no * recovery path, so absence is a warning and an unwritable configured dir a * failure. */ function dbBackupProbe(probes: Probes, cfg: ConductorConfig | undefined): Finding { const dir = dbBackupDirFor(cfg); const state = probes.snapshotDirState(dir); if (state.exists && !state.writable) { return failFinding( "db-backup", `db backup dir ${dir} is not writable — a conductor.db snapshot cannot land there`, `make ${dir} writable by the conductor account, or set dbBackupDir in config to a writable directory`, ); } const snapshots = (() => { try { return readdirSync(dir).filter((name) => name.startsWith(DB_SNAPSHOT_STEM)); } catch { return []; } })(); const store = probes.stat(dbPath()); if (store === undefined) { return snapshots.length === 0 ? passFinding("db-backup", "no conductor.db yet — nothing to snapshot") : passFinding("db-backup", `${snapshots.length} conductor.db snapshot(s) present`); } if (snapshots.length === 0) { return warnFinding( "db-backup", "conductor.db has no snapshot — losing it loses the audit trail with no recovery path", "run `omp-conductor snapshot-db` (the daemon cadence uses the same SQLite snapshot primitive)", ); } const newestMtime = snapshots .map((name) => probes.stat(join(dir, name))?.mtimeMs ?? 0) .reduce((max, m) => Math.max(max, m), 0); if (store.mtimeMs > newestMtime + 60_000) { return warnFinding( "db-backup", `conductor.db is newer than every snapshot (${snapshots.length} file(s)) — the newest writes were never snapshot`, "run `omp-conductor snapshot-db` so restore-db has the current state", ); } return passFinding("db-backup", `conductor.db snapshot is fresh (${snapshots.length} file(s))`); } /** Every repo the config names: each project's tracker plus routing targets, * as GitHub slugs, deduplicated, in config order. */ function configuredRepos(cfg: ConductorConfig | undefined): string[] { if (cfg === undefined) return []; const repos: string[] = []; for (const project of cfg.projects) { for (const repo of [ project.tracker.repo, ...Object.values(project.routing.repos).map((r: RepoTarget) => repoSlugFor(r)), ]) { if (!repos.includes(repo)) repos.push(repo); } } return repos; } /** * gh auth: token alive, rate-limit readable, effective scopes cover every * configured repo. The daemon's tracker reads go through the same transport, * so a failure here is the expired-token class with the daemon already live. */ async function ghAuthProbe(probes: Probes, repos: string[]): Promise { const [scopes, rateOk] = await Promise.all([probes.scopes(), probes.rateLimitOk()]); const failures: string[] = []; const notes: string[] = []; if (scopes !== undefined) { if (scopes.ok) { notes.push(scopes.login === undefined ? "authenticated" : `authenticated as ${scopes.login}`); notes.push(`scopes: ${scopes.scopes.join(", ") || "none listed"}`); } else { failures.push( scopes.login === undefined ? "`gh auth status` says not authenticated" : `token for ${scopes.login} is missing scopes ${scopes.missing.join(", ") || "(none listed)"}`, ); } } else { failures.push("`gh auth status` did not answer"); } if (rateOk) notes.push("rate-limit API readable"); else failures.push("`gh api rate_limit` fails — the daemon's tracker calls would fail the same way"); const reads = await Promise.all(repos.map(async (repo) => ({ repo, read: await probes.repoReadable(repo) }))); for (const r of reads) { if (r.read.ok) notes.push(`${r.repo} readable`); else failures.push(`configured repo ${r.repo} is not readable with this token${r.read.detail === undefined ? "" : ` (${r.read.detail})`}`); } if (failures.length === 0) { return passFinding("gh-auth", `gh is valid ${notes.length === 0 ? "" : `— ${notes.join("; ")}`}`.trim()); } return failFinding( "gh-auth", `${failures.join("; ")} — ${INCIDENTS.ghauth}`, "run `gh auth login` and grant the scopes the wizard requires (setup's checkTokenScopes names them)", ); } /** * Every label the dispatch loop depends on, with exact case — a case-mismatched * label is documented to fail silently, so doctor names the near-miss instead * of just the absence. */ async function labelProbe(probes: Probes, project: ProjectConfig): Promise { const wanted: { kind: string; name: string }[] = [ { kind: "queue", name: project.queueLabel }, { kind: "state", name: project.stateLabels.inProgress }, { kind: "state", name: project.stateLabels.blocked }, { kind: "state", name: project.stateLabels.failed }, ...Object.keys(project.routing.repos).map((key) => ({ kind: "routing", name: `${project.routing.labelPrefix}${key}`, })), ]; const read = await probes.repoLabels(project.tracker.repo); if (!read.ok) { return failFinding( "labels", `[${project.name}] cannot read the labels of ${project.tracker.repo}${read.detail === undefined ? "" : ` (${read.detail})`}`, "check `gh` can list that repo's labels — the daemon reads them the same way", ); } const present = new Set(read.labels ?? []); const norm = (s: string): string => s.toLowerCase().replace(/[-_\s:]+/g, ""); const nearByName = new Map(); for (const label of read.labels ?? []) { if (!nearByName.has(norm(label))) nearByName.set(norm(label), label); } const missing = wanted.filter((w) => !present.has(w.name)); if (missing.length === 0) { return passFinding("labels", `[${project.name}] ${wanted.length} configured label(s) on ${project.tracker.repo} match exactly`); } const detail = missing.map((m) => { const near = nearByName.get(norm(m.name)); return near === undefined || near === m.name ? `"${m.name}" (${m.kind})` : `"${m.name}" (${m.kind}) — near-miss present: "${near}"`; }); const allNearMisses = missing.every((m) => { const near = nearByName.get(norm(m.name)); return near !== undefined && near !== m.name; }); return failFinding( "labels", `[${project.name}] ${detail.join("; ")} — ${INCIDENTS.labels}`, allNearMisses ? "fix the label case/format in the tracker repo (or change the config to the actual spelling)" : "create the missing label(s) in the tracker repo (`gh label create`, or re-run `omp-conductor setup`)", ); } /** * A unit line whose canonical value is a property of the *rendering process*, * not the deployment. The herdr unit's `Environment="SHELL=…"` carries the * account's login shell as it resolved in the process that staged it (#463 — * the pane-shell pin that keeps panes off dash). Doctor cannot re-derive that * value: in a non-login/tick context `userInfo().shell` resolves to nothing and * renders `SHELL=unknown`, while the unit that was actually staged carries * `SHELL=/bin/bash`. Comparing against a value that answers to "whoever asked" * can never pass, so it is excluded from the drift comparison and reported as * not-comparable, never as drift (#511). */ const PROCESS_DERIVED_UNIT_LINE = /^Environment="SHELL=/; /** * Whether a canonical unit's pane-shell value could not be resolved — the * `SHELL=` pin rendered when the invoking process has no login shell (an empty * value, or the literal `unknown` userInfo returns in a non-login context). * Excluded from the drift comparison, such a value is reported as * not-comparable rather than implied to match (#511). */ function unresolvableShell(unit: string): boolean { const line = unit .split("\n") .map((l) => l.trim()) .find((l) => PROCESS_DERIVED_UNIT_LINE.test(l)); if (line === undefined) return false; const value = /^Environment="SHELL=(.*)"$/.exec(line)?.[1] ?? ""; return value === "" || value === "unknown"; } /** * Line-level drift between an installed unit and the canonical render, ignoring * lines whose canonical value is not re-derivable by doctor (the * {@link PROCESS_DERIVED_UNIT_LINE} pane-shell pin). The pane shell is host * state resolved at staging time, so a rendered value doctor cannot reproduce * is excluded from both sides rather than reported as a difference — an * unresolvable canonical is "cannot compare", never drift (#511). * * The service `PATH=` line has no such exemption: since #879 it is canonical * on both sides (`defaultServiceRuntime` composes it from the resolved fleet * binaries plus the standard system directories, never from the rendering * process's environment), so a differing installed value is genuine drift and * must fail here with the value named. */ export function unitDrift(installed: string, canonical: string): { differences: string[] } { const want = canonical .split("\n") .map((l) => l.trim()) .filter((l) => l.length > 0 && !PROCESS_DERIVED_UNIT_LINE.test(l)); const have = installed .split("\n") .map((l) => l.trim()) .filter((l) => l.length > 0 && !PROCESS_DERIVED_UNIT_LINE.test(l)); const wantSet = new Set(want); const haveSet = new Set(have); const differences = [ ...want.filter((l) => !haveSet.has(l)).map((l) => `missing/differs: ${l}`), ...have.filter((l) => !wantSet.has(l)).map((l) => `extra: ${l}`), ]; return { differences }; } function driftLines(name: string, installed: string, canonical: string): string[] { const { differences } = unitDrift(installed, canonical); if (differences.length === 0) return []; const shown = differences.slice(0, 4).join("; "); const more = differences.length > 4 ? ` (+${differences.length - 4} more)` : ""; return [`${name} differs from the staged unit: ${shown}${more}`]; } /** * Installed unit vs the canonical staged rendering: any drift in User=, * Environment (save the process-derived pane shell, which is not comparable), * WorkingDirectory, ExecStart, Restart/SuccessExitStatus, or memory lines is * the unit-drift class that once killed workers at turn 0. */ function unitProbe(probes: Probes, project: ProjectConfig | undefined, cfg: ConductorConfig | undefined): Finding { if (!probes.hasSystemd()) return passFinding("systemd-unit", "no systemd directory on this host — nothing to check"); if (project === undefined || cfg === undefined) { return passFinding("systemd-unit", "config unreadable — the canonical staged unit is unknown, nothing to compare"); } const canonical = probes.canonicalUnits(project, cfg); const installedDaemon = probes.readUnit(join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME)); if (installedDaemon === undefined) { return warnFinding("systemd-unit", `${STAGED_SERVICE_NAME} is not installed`, "install it: run `omp-conductor setup host` from the fleet account"); } const problems = driftLines(STAGED_SERVICE_NAME, installedDaemon, canonical.daemon); const uncomparable: string[] = []; if (canonical.herdr !== undefined) { const installedHerdr = probes.readUnit(join(SYSTEMD_UNIT_DIR, DEFAULT_HERDR_UNIT)); if (installedHerdr === undefined) { problems.push(`${DEFAULT_HERDR_UNIT} is not installed (the staged plan provisions it)`); } else { if (unresolvableShell(canonical.herdr)) { uncomparable.push("the pane-shell SHELL value is not comparable (it resolves from this process, not the deployment)"); } problems.push(...driftLines(DEFAULT_HERDR_UNIT, installedHerdr, canonical.herdr)); } } if (problems.length === 0) { const note = uncomparable.length === 0 ? "" : ` — ${uncomparable.join("; ")}`; return passFinding("systemd-unit", `installed units match the staged render${note}`); } return failFinding( "systemd-unit", [...problems, INCIDENTS.systemd].join("; "), "re-stage and reinstall with `omp-conductor setup host` from the fleet account", ); } /** The `OnFailure=` targets an installed unit names, each a systemd unit file. */ function onFailureTargets(unitText: string): string[] { return (unitText.match(/^OnFailure=(.*)$/gm) ?? []).flatMap((line) => line.slice("OnFailure=".length).trim().split(/\s+/).filter(Boolean), ); } /** * Every `OnFailure=` target an installed fleet unit names must itself be * installed. The fleet units carry the line from day one (#485); the failure * this check exists for is the line present while the recovery oneshot it * names is not installed — so systemd silently refuses to enqueue the job and * a failed unit never recovers (#509). */ function recoveryProbe(probes: Probes): Finding { if (!probes.hasSystemd()) return passFinding("systemd-recovery", "no systemd directory on this host — nothing to check"); const problems: string[] = []; const fleetUnits: readonly [string, string][] = [ [STAGED_SERVICE_NAME, join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME)], [DEFAULT_HERDR_UNIT, join(SYSTEMD_UNIT_DIR, DEFAULT_HERDR_UNIT)], ]; for (const [name, path] of fleetUnits) { const text = probes.readUnit(path); if (text === undefined) continue; // not installed — the unit/ownership probes name it for (const target of onFailureTargets(text)) { if (probes.readUnit(join(SYSTEMD_UNIT_DIR, target)) === undefined) { problems.push(`${name} names OnFailure=${target}, which is not installed`); } } } if (problems.length === 0) { return passFinding("systemd-recovery", "every OnFailure= target names an installed unit"); } return failFinding( "systemd-recovery", `${problems.join("; ")} — systemd cannot enqueue the recovery a failed fleet unit needs (#485/#509)`, "install the recovery unit and playbook: run `omp-conductor setup host` from the fleet account", ); } function recoveryStateProbe(probes: Probes): Finding { const id = "systemd-recovery-state"; if (!probes.hasSystemd()) return passFinding(id, "no systemd directory on this host — nothing to check"); if (probes.readUnit(join(SYSTEMD_UNIT_DIR, RECOVER_SERVICE_NAME)) === undefined) { return passFinding(id, "recovery unit is not installed — systemd-recovery owns that finding"); } const failed = probes.unitFailed(RECOVER_SERVICE_NAME); if (failed === false) return passFinding(id, `${RECOVER_SERVICE_NAME} is not failed`); const fix = `inspect \`systemctl status ${RECOVER_SERVICE_NAME}\` and \`journalctl -u ${RECOVER_SERVICE_NAME}\`; ` + `fix the root cause, then run \`systemctl reset-failed ${RECOVER_SERVICE_NAME}\` and rerun doctor`; return failed === true ? warnFinding(id, `${RECOVER_SERVICE_NAME} is in failed state — its last recovery did not complete`, fix) : warnFinding(id, `${RECOVER_SERVICE_NAME} state could not be read`, fix); } function installedUnitUser(probes: Probes): string | undefined { const unit = probes.readUnit(join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME)); if (unit === undefined) return undefined; const match = /^User=(.*)$/m.exec(unit); if (match === null) return undefined; const value = match[1]?.trim().replace(/^"(.*)"$/, "$1") ?? ""; return value === "" ? undefined : value; } /** * Ownership and modes of the unit file and the runtime dirs: a daemon that * cannot write its own state is the empty-agent.db class, and a unit someone * else can edit is a takeover. */ function ownershipProbe(probes: Probes): Finding { if (!probes.hasSystemd()) return passFinding("systemd-ownership", "no systemd directory on this host — nothing to check"); const failures: string[] = []; const warns: string[] = []; const daemonPath = join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME); const daemonStat = probes.stat(daemonPath); if (daemonStat !== undefined) { if (daemonStat.uid !== 0) failures.push(`${STAGED_SERVICE_NAME} is not owned by root`); if ((daemonStat.mode & 0o022) !== 0) warns.push(`${STAGED_SERVICE_NAME} is group/other-writable`); } const unitUser = installedUnitUser(probes); const fleetUid = unitUser === undefined ? process.getuid?.() : probes.uidOf(unitUser); for (const [label, dir] of [["state dir", stateDir()], ["telegram state dir", telegramStateDir()]] as const) { const st = probes.stat(dir); if (st === undefined) continue; if (fleetUid !== undefined && st.uid !== fleetUid) { failures.push( `${label} (${dir}) is owned by uid ${st.uid}, but the ${unitUser === undefined ? "current user" : `unit's account "${unitUser}"`} has uid ${fleetUid}`, ); } if ((st.mode & 0o022) !== 0) warns.push(`${label} (${dir}) is group/other-writable`); } if (failures.length === 0 && warns.length === 0) { return passFinding("systemd-ownership", "unit file and runtime dirs are owned and permitted as deployed"); } const summary = [...failures, ...warns].join("; "); const fix = "reinstall the unit as root and chown the state dirs to the account the unit runs as"; if (failures.length === 0) { return warnFinding("systemd-ownership", summary, fix); } return failFinding("systemd-ownership", summary, fix); } /** IANA timezone in reporting config — an invalid zone silently mis-schedules * the availability window and the daily digest (#273). */ function timezoneProbe(project: ProjectConfig | undefined): Finding { if (project === undefined) return passFinding("reporting-timezone", "no project resolved — nothing to check"); const tzs: [string, string | undefined][] = [ ["reporting.availability.timezone", project.reporting?.availability?.timezone], ["reporting.digest.timezone", project.reporting?.digest?.timezone], ]; const configured = tzs.filter(([, tz]) => tz !== undefined); const bad = configured.filter(([, tz]) => tz !== undefined && !isKnownTimezone(tz)); if (bad.length === 0) { return passFinding( "reporting-timezone", configured.length === 0 ? `[${project.name}] no reporting timezone configured` : `[${project.name}] reporting timezone(s) valid: ${configured.map(([, tz]) => `"${tz}"`).join(", ")}`, ); } return failFinding( "reporting-timezone", `[${project.name}] ${bad.map(([label, tz]) => `${label}: "${tz}" is not a known IANA timezone`).join("; ")} — an invalid zone silently skips the availability window and the daily digest`, "set a valid IANA timezone (e.g. Europe/London) in reporting.availability or reporting.digest", ); } /** The same acceptance the config validator uses (`Intl.DateTimeFormat`), so * `doctor` and `config.ts` cannot disagree about a zone. */ export function isKnownTimezone(tz: string): boolean { try { new Intl.DateTimeFormat("en-GB", { timeZone: tz }); return true; } catch { return false; } } /** * The fleet-owned omp settings overlay (#537): reports the *effective* overlay * every worker session of the project will load — the `ompSettings` map plus * the retry keys derived from `modelFallbacks`, rendered exactly as the * materialiser writes them, never the raw config snippet `setup` would echo — * and turns red when that overlay cannot land. The file is written at dispatch * under `/sessions/run-/`, so doctor cannot prove the write * without writing: it probes the root the materialisation would land in with * the same stat + access the db-backup probe uses, keeping the check * read-only (#287). The session root is a host-wide fact, probed once and * shared across the per-project findings. */ function ompSettingsProbe( project: ProjectConfig, sessionRoot: { exists: boolean; writable: boolean }, ): Finding { const overlay = ompSettingsOverlay(project); if (overlay === undefined) { return passFinding( "omp-settings", `[${project.name}] no omp settings overlay configured — workers inherit the daemon account's global settings unchanged`, ); } if (!sessionRoot.writable) { return failFinding( "omp-settings", `[${project.name}] the omp settings overlay cannot be materialised: ${sessionRootDir()} cannot hold a session directory, so every worker would dispatch without its staged settings`, `make ${sessionRootDir()} writable by the conductor account`, ); } return passFinding( "omp-settings", `[${project.name}] worker sessions load this omp settings overlay:\n${stringify(overlay).trimEnd()}`, ); } /** The self-identified probe message `--probe-telegram` delivers through the * same report transport a real report would take. */ function probeMessage(project: string, checkedAt: string): string { return [ "omp-conductor doctor · Telegram delivery probe", `project: ${project} · started ${checkedAt}`, "Sent by `omp-conductor doctor --probe-telegram` to verify delivery end to end.", "No action needed — you can ignore this message.", ].join("\n"); } /** * Telegram delivery health — and, when `--probe-telegram` is set, one * self-identified message through the report transport. A dead bot token is * silent: reports leave the outbox and never land. */ async function telegramProbe(probes: Probes, project: ProjectConfig | undefined, checkedAt: number): Promise { if (project === undefined) return passFinding("telegram", "no project resolved — nothing to check"); const chatId = (project.escalation.telegramChatId ?? "").trim(); if (chatId === "") { return passFinding("telegram", `[${project.name}] no escalation.telegramChatId configured — nothing to probe`); } const health = await probes.telegramHealth(project.name); const failures: string[] = []; const notes: string[] = [`[${project.name}]`]; if (health.kind === "down") failures.push(`bot health: down (${health.detail ?? "getMe failed"})`); else if (health.kind === "unconfigured") notes.push(`bot health: unconfigured (${health.detail ?? "no token"})`); else if (health.kind === "degraded") notes.push(`bot health: degraded (${health.detail ?? ""})`); else notes.push(`bot health: ok (${health.detail ?? ""})`); if (probes.probeTelegram) { const send = probes.telegramSend?.(project); if (send === undefined) { failures.push("probe requested but no send transport is wired"); } else { try { await send(probeMessage(project.name, new Date(checkedAt).toISOString())); notes.push("probe message delivered"); } catch (err) { failures.push(`probe send failed: ${messageOf(err)}`); } } } if (failures.length === 0) { return passFinding("telegram", notes.join("; ")); } return failFinding( "telegram", `[${project.name}] ${failures.join("; ")}`, "fix the bot token / omp-telegram install; `doctor --probe-telegram` proves delivery end to end", ); } /** * Is any economic control actually governing this project? (#985) * * Two exist, and on this fleet neither was active. `caps.planUsage` was `null`, * so the allowance gate that admission already applies — it holds every routed * candidate and escalates once (`admission.ts`) — was configured off. And * `caps.dailySpendUsd` was a ceiling that #984 established cannot fire under * subscription billing, because a subscription request carries no per-request * price. So the fleet ran with no ceiling of any kind, and no surface said so: * `status` reported `unmetered — no plan allowance cap configured`, which reads * as a fact about the plan rather than as a missing control. * * This row states the gap once, per project, with the remedy that closes it. * It is deliberately NOT a new gate: the gate exists and works. What was * missing was anyone being told it is switched off. */ function economicGateProbe( project: ProjectConfig, caps: Caps, subscriptionBilled: boolean, allowanceWindow: string | undefined, ): Finding { const allowanceGate = caps.planUsage !== null; // A dollar cap governs only if it can fire, and under subscription billing it // cannot — that is #984's finding, reused here rather than re-derived. const dollarGate = caps.dailySpendUsd !== null && !subscriptionBilled; if (allowanceGate) { return passFinding( "economic-gate", `[${project.name}] the provider allowance gate is configured (window ${caps.planUsage?.windowId ?? "?"}, ` + `hold at ${String(Math.round((caps.planUsage?.maxUsedFraction ?? 0) * 100))}% used)` + (dollarGate ? ` beside a $${caps.dailySpendUsd?.toFixed(2) ?? "?"} daily cap` : ""), ); } if (dollarGate) { return passFinding( "economic-gate", `[${project.name}] the $${caps.dailySpendUsd?.toFixed(2) ?? "?"} daily spend cap governs; ` + `no provider allowance cap is configured`, ); } return warnFinding( "economic-gate", `[${project.name}] no economic control is active: caps.planUsage is null, and ` + (caps.dailySpendUsd === null ? "no daily spend cap is set" : `the $${caps.dailySpendUsd.toFixed(2)} daily cap cannot fire on subscription billing`) + ` — admission has nothing to weigh and will keep claiming until the provider itself refuses` + (allowanceWindow === undefined ? "" : `; the window that would bind is ${allowanceWindow}`), allowanceWindow === undefined ? "run `omp usage --json` on the fleet host and copy an allowance `id` into `caps.planUsage.windowId` (with `maxUsedFraction`), or set a `caps.dailySpendUsd` the billing model can actually spend" : `set caps.planUsage to { windowId: "${allowanceWindow.split(" ")[0] ?? ""}", maxUsedFraction: 0.9 } — the gate is already wired into admission, it is simply switched off`, ); } /** * Spend telemetry, through the shared judgement (#970). * * The predicate this replaced fired only when *every* sampled run reported * $0.00, and counted 0-turn kills as evidence. Replayed over this fleet's whole * history it would have fired 14 times while staying silent through 37 windows * that had lost a majority of their telemetry — so partial loss, the common * case, was the invisible one. See `spend-telemetry.ts` for the measurements. */ function spendProbe(rows: RunSpendRow[], limit: number, billing: SpendBilling = {}): Finding { const verdict = judgeSpendTelemetry(rows, limit, billing); const detail = spendTelemetryDetail(verdict); // A subscription-billed fleet's zeros are the truth, so this passes and names // the constraint that does bind (#984). It still says the sentence: a reader // who expected a dollar figure needs to know why there isn't one. What it // must never do is warn forever with advice that cannot be acted on — there // is nothing to repair, and a permanent warning is one nobody reads. if (verdict.kind === "subscription") { return passFinding("spend-telemetry", detail ?? "subscription-billed; no USD cap applies"); } if (detail !== undefined) { return warnFinding( "spend-telemetry", `${detail} — ${INCIDENTS.spend}`, "verify harness spend reporting (the per-run spendUsd column / `omp usage --json`); a USD cap on top of zeros never fires", ); } if (verdict.kind === "insufficient") { return passFinding( "spend-telemetry", `${verdict.worked} completed run(s) did any work — fewer than ${verdict.needed}, nothing to judge yet`, ); } // Healthy. Naming the metered share rather than only the total: a window with // one missing row is fine and saying so is how a reader learns the baseline // is not zero. const healthy = verdict as Extract; return passFinding( "spend-telemetry", `spend observed on ${healthy.worked - healthy.missing} of the last ${healthy.worked} working runs ` + `($${healthy.totalUsd.toFixed(2)} total)`, ); } /** Live `herdr --session agent list` through the project's own * parser — the published "one JSON line on stdout" interface, never the * socket. Same bound as the tick's own query so a hung herdr cannot hang * doctor. */ function defaultHerdrAgents(session: string): HerdrAgentList { try { const run = spawnSync("herdr", ["--session", session, "agent", "list"], { encoding: "utf8", timeout: 3000, stdio: ["ignore", "pipe", "pipe"], }); if (run.error !== undefined) return { kind: "unavailable", problem: run.error.message }; if (run.status !== 0) { return { kind: "unavailable", problem: `herdr agent list exited ${String(run.status)}: ${(run.stderr ?? "").trim().split("\n")[0] ?? ""}`, }; } return parseHerdrAgents(run.stdout ?? ""); } catch (err) { return { kind: "unavailable", problem: err instanceof Error ? err.message : String(err) }; } } /** Live herdr config, defaulting to the fleet account's own path. */ function defaultHerdrConfig(): string | undefined { try { return readFileSync(join(homedir(), ".config", "herdr", "config.toml"), "utf8"); } catch { return undefined; } } /** Live herdr-conductor plugin `config.env`, same path setup provisions. */ function defaultHerdrEnv(): string | undefined { try { return readFileSync(join(herdrConductorPluginConfigDir(homedir()), "config.env"), "utf8"); } catch { return undefined; } } /** * #541 check 1 — live herdr agent name equals the tick config's agentName. * * The paneOwnership decline the fleet actually hit names the exact fix (`herdr * agent rename `); doctor checks the same premise the tick does * (is a pane registered under the configured agent name?) and reports drift * with the one-line remedy. */ function herdrAgentNameProbe(probes: Probes, p: ProjectConfig): Finding { if (!probes.herdrInstalled()) { return passFinding("herdr-agent-name", `[${p.name}] herdr not installed — nothing to check`); } const want = probes.tickAgentName(p); if (want === undefined) { return passFinding("herdr-agent-name", `[${p.name}] no readable tick config — nothing to check`); } const session = probes.herdrSession(); const agents = probes.herdrAgents(session); if (agents.kind !== "ok") { return failFinding( "herdr-agent-name", `[${p.name}] could not read the live herdr agent list (${agents.problem}) — cannot verify the fleet agent name`, `restore the herdr session (check \`herdr status\`), then re-run doctor`, ); } const holders = agents.agents.filter((a) => a.name === want); if (holders.length > 0) { return passFinding("herdr-agent-name", `[${p.name}] live pane(s) ${holders.map((a) => a.paneId).join(", ")} are agent "${want}"`); } // Drift: no pane carries the configured identity. Exactly one pane pinned to // the shared default is the safe candidate from the decline turnover — name // it when unambiguous, else give the generic command. const stale = agents.agents.filter((a) => a.name === DEFAULT_FLEET_AGENT_NAME); const pane = stale.length === 1 ? stale[0]!.paneId : ""; return failFinding( "herdr-agent-name", `[${p.name}] no live pane is agent "${want}" — every tick will be declined until the fleet pane carries this name (the 2026-08-15 restamp class)`, `herdr --session ${session} agent rename ${pane} ${want}`, ); } /** * #614 check — the state of this project's arming acknowledgement transaction. * * The challenge proof is conductor-owned state now: the host records a pending * challenge and waits for the acknowledgement the orchestrator's inbound * adapter writes. No transcript location is diagnosed any more — where (or * whether) a session file lives is not part of arming — so what is worth * reporting is a transaction left open: a challenge still pending and whether * it has been acknowledged, or an expired record nothing settled. Every shape * is inert by construction (a reply past expiry, or for a replaced id, is * refused), so this informs; only the stale leftover warns. */ function armAckProbe(probes: Probes, p: ProjectConfig): Finding { const now = probes.now(); const sighting = observeArmChallenge(p.name); if (sighting === undefined) { return passFinding("arm-ack", `[${p.name}] no pending arm challenge — the acknowledgement handshake is idle`); } const window = sighting.sentAt === undefined || sighting.expiresAt === undefined ? "an unreadable record window" : `sent ${new Date(sighting.sentAt).toISOString()}, expires ${new Date(sighting.expiresAt).toISOString()}`; if (sighting.expiresAt !== undefined && now >= sighting.expiresAt) { return warnFinding( "arm-ack", `[${p.name}] an arming challenge from ${window} expired without being settled — it is inert (replies past expiry are refused); the fleet session notifies the operator and clears it once its notice goes out, or the next arm replaces it`, `the code is dead — no reply to it can arm. Re-run the ceremony: \`omp-conductor arm --project ${p.name}\``, ); } if (sighting.acknowledgedAt !== undefined) { return passFinding( "arm-ack", `[${p.name}] an arm challenge is pending (${window}) and was acknowledged ${new Date(sighting.acknowledgedAt).toISOString()} — the host consumes it on settle`, ); } return passFinding( "arm-ack", `[${p.name}] an arm challenge is pending (${window}), not yet acknowledged — live if \`arm\` is running right now, otherwise its window simply runs out`, ); } /** * #600 check — a pinned escalation.telegramTopicId that no live claim carries. * * The wizard pins from the claims at setup time, but the bridge re-claims * topics across restarts and conductor restarts the service itself (#407), so * the pin drifts. A stale pin is not itself a fault: {@link * resolveProjectTopicId} follows the project's one unambiguous live claim in * that shape (#407/#412), so routing is healthy exactly when a claim answers * to the project. The pin is a fault only when no live claim can take its * place — then sends keep the stale pin and degrade to the flat chat, and the * fix is to re-pin. Removing the pin is never the fix: a missing pin disables * substitution entirely and opts the project into flat chat (#626). * * A registry that cannot be read is neither proof of health nor a stale pin: * the bridge may be fine and the pin live, but nothing can be verified, so the * finding warns and names restoring the registry — never "the pin stands". * * On a host with `topicsTidy` on, a pin drifts *structurally*: every pane exit * closes the topic, so no re-pin survives and the durable fix is to turn tidy * off and re-pin. */ function topicPinProbe(probes: Probes, p: ProjectConfig): Finding { const pinned = p.escalation.telegramTopicId; if (pinned === undefined) { return passFinding( "topic-pin", `[${p.name}] no pinned topic id — sends go to the flat chat (live-claim substitution only follows a stale pin)`, ); } // #1094: sends that already rode the flat chat because of THIS pin turn the // warnings below from hypothetical into observed. Rows are keyed to the pin // id, so re-pinning retires them — nothing deletes them. const misroutes = probes.topicMisroutes().filter( (row) => row.project === p.name && row.staleTopicId === pinned, ); const misrouteNote = misroutes.length === 0 ? "" : ` It has happened: ${misroutes.length} send(s) were delivered to the flat chat because of this pin, last at ${new Date(misroutes[misroutes.length - 1]!.at).toISOString()}`; const result = probes.claimedTopics(); if (result.kind !== "ok") { return warnFinding( "topic-pin", `[${p.name}] pinned topic ${pinned} — ` + (result.kind === "missing" ? `omp-telegram has no claim registry (threads.json in its state dir), so doctor cannot tell whether the pin is live or a claim answers to this project${misrouteNote}` : `omp-telegram's claim registry is unreadable (${result.problem}), so doctor cannot tell whether the pin is live or a claim answers to this project${misrouteNote}`), `check omp-telegram's claim registry (threads.json in its state dir) is readable and the bridge is running, then re-run doctor — until then sends keep the pinned topic and degrade to the flat chat on a missing thread (#318)`, ); } // One liveness fact for everything this probe decides — the dead partition, // the exact-pin check below and identity resolution all read // `probes.pidAlive` through the same helper, so diverging injections cannot // split the rows' answer again (#987). Before #987 every registry row was // treated as live here, so a pin aimed at a corpse read PASS while the // plumbing row called the same claim dead. const isLive = (claim: { pid?: number }): boolean => claimIsLive(claim, killOf(probes.pidAlive)); const all = result.claims; const claims = all.filter(isLive); const dead = all.filter((claim) => !isLive(claim)); // Named, never dropped: the row is the only index to the remote topic, so // deleting it strands the topic instead of cleaning it up. const deadNote = dead.length === 0 ? "" : ` (dead claims still recorded, and left alone: ${dead .map((c) => `${c.threadId}${c.pid === undefined ? " no pid" : ` pid ${c.pid}`}`) .join(", ")} — run the bridge's /cleanup to remove those topics)`; if (claims.length === 0) { if (dead.length > 0) { return warnFinding( "topic-pin", `[${p.name}] pinned topic ${pinned} — every claim in omp-telegram's registry is dead${deadNote}${misrouteNote}`, `run /cleanup in Telegram so the bridge closes those topics and drops their rows itself, then re-run setup to re-pin escalation.telegramTopicId to a live claim — never delete rows from threads.json by hand, because the row is the only index to the remote topic (#987)`, ); } return passFinding( "topic-pin", `[${p.name}] pinned topic ${pinned} — no live claims to compare (the bridge has claimed no topics yet); the pin stands`, ); } // `claims` is already the live partition, so a dead pinned row cannot win // this check (#987). if (claims.some((claim) => claim.threadId === pinned)) { return passFinding("topic-pin", `[${p.name}] pinned topic ${pinned} is a live claim${deadNote}`); } const match = resolveProjectClaim(claims, p.name, probes.pidAlive, p.workspaceRoot); if (match.kind === "match") { return passFinding( "topic-pin", `[${p.name}] pinned topic ${pinned} is not among omp-telegram's live claims, but the ` + `${match.claim.workspaceLabel === p.name ? "herdr space claim" : "claim"} for this project ` + `(topic ${match.claim.threadId}) is live — sends follow it, so topic routing is healthy`, ); } if (match.kind === "ambiguous") { // Several live claims answer to the project, so substitution would be a // coin toss and sends keep the stale pin. This is not the no-answer case: // a claim IS waiting for the project, and naming the claimants tells the // operator exactly which panes to disambiguate (#626). return warnFinding( "topic-pin", `[${p.name}] pinned topic ${pinned} is not among omp-telegram's live claims, and several live claims answer to this project ` + `(topics ${match.claimants.map((c) => c.threadId).join(", ")}) — substitution would be a coin toss, so sends keep the stale pin${misrouteNote}`, `make exactly one live claim answer to this project: rename the other panes' herdr spaces or re-claim them one at a time, then re-run doctor — live-claim substitution follows the unique claim from there`, ); } if (probes.topicsTidy()) { return warnFinding( "topic-pin", `[${p.name}] pinned topic ${pinned} is not among omp-telegram's live claims, no live claim answers ` + `to this project, and this host runs topicsTidy — the pin is vestigial and can never be live again ` + `(every pane exit closes its topic); sends keep the stale pin${misrouteNote}`, `turn topicsTidy off, then re-run setup to re-pin escalation.telegramTopicId to a currently claimed topic`, ); } return warnFinding( "topic-pin", `[${p.name}] pinned topic ${pinned} is not among omp-telegram's live claims (${claims.map((c) => c.threadId).join(", ")}), ` + `and no live claim answers to this project — sends keep the stale pin and degrade to the flat chat${misrouteNote}`, `update escalation.telegramTopicId to a currently claimed topic, or re-run setup to re-pin`, ); } /** * #612/#636 check — whether omp-telegram plumbing is live, *before* the arm * challenge, on the route the challenge actually rides. Read-only — the state * files and liveness probes, no `getUpdates`, no transcript scan, no network. * Renders the shared verdict as its own finding so a dead recipient or dead * poll is immediate and named, not discovered 300 seconds into an arm * challenge. * * Gated on the project's *real* inbound requirement, never on * `escalation.telegramTopicId` (an outbound send destination, not the inbound * shape) and never on `escalation.telegramChatId`, which `armTicks` ignores — * it sends the challenge to the paired owner from the tick's `accessFile`. * Only a project that can arm — a readable tick config naming an `accessFile` * — rides inbound at all, so only that shape runs the verdict: an armed * project with no chat id must still be verified, and a manually pinned * outbound-only target must not fail for absent claim/poll state. The route * is the forum topic the send would use (`resolveProjectTopicId` — the pin, * or the live claim a stale pin substitutes) when one resolves; with no topic * the challenge is unthreaded and the reply routes through the bridge's DM * owner, or — after `/telegram own clear` — the polling session itself. The * finding always renders, so the stable report shape and exit code are * unchanged. */ function telegramPlumbingProbe(probes: Probes, p: ProjectConfig): Finding { if (!probes.projectArms(p)) { return passFinding( "telegram-plumbing", `[${p.name}] outbound-only paging — no tick accessFile to arm, so no live claim, DM owner or poll ownership is required`, ); } // Transcript-surface identity checks belong to the claim-only proof alone: // since #614 the challenge proof reads conductor acknowledgement state, so // judging it by where a transcript lives could report arming impossible // while `arm` succeeds. An unreadable config keeps the default proof — // fail-safe to the stricter verdict, exactly like armTicks. let proof: ArmProof = DEFAULT_ARM_PROOF; try { proof = resolveArmProof(p); } catch { /* unreadable config — keep today's stricter default */ } const verdict = telegramPlumbingVerdict( probes.armSendTopic(p), proof === "claim-only" ? { dirs: probes.armScanDirs(p) } : undefined, { channel: probes.channelState(p), registry: probes.claimedTopics(), poll: probes.pollState(), dmOwner: probes.dmOwner(), alive: probes.pidAlive, lockAlive: probes.lockPidAlive, fresh: probes.lockFresh, }, ); if (verdict.ok) { const named = verdict.proof.includes("claim") ? "verified live claim + live poll" : verdict.proof.includes("dm-owner") ? "verified live dm-owner + live poll" : "verified live local poller session"; return passFinding( "telegram-plumbing", `[${p.name}] live omp-telegram plumbing: ${named} (${verdict.proof.join(" + ")})`, ); } switch (verdict.reason) { case "channel-down": { const channel = probes.channelState(p); const reason = channel.kind === "down" ? channel.reason : "unexpected channel state"; return failFinding( "telegram-plumbing", `[${p.name}] the tick's accessFile does not pair an enabled bridge (${reason}) — an arm challenge could not be sent`, `pair the bot (/telegram pair) and enable the bridge (/telegram on), then re-run doctor`, ); } case "registry-unreadable": return failFinding( "telegram-plumbing", `[${p.name}] omp-telegram's threads.json is absent, unreadable or malformed — the pinned-or-claimed topic's liveness cannot be verified`, `check the bridge state dir for a valid threads.json (the bridge writes it when a forum pane claims a topic)`, ); case "no-topic-claim": return failFinding( "telegram-plumbing", `[${p.name}] no live omp-telegram claim carries the topic arming would send into — the pin is stale and no live claim takes its place`, `re-pin escalation.telegramTopicId to a currently claimed topic (or remove it to route the challenge flat), then re-run doctor`, ); case "claim-dead": return failFinding( "telegram-plumbing", `[${p.name}] the topic's omp-telegram claim records no live pid — the pane behind it is gone`, `restart the claiming pane so the bridge re-claims it with a live pid`, ); case "claim-session-outside": return failFinding( "telegram-plumbing", `[${p.name}] the topic's claim names a session file outside the session directories arm scans — the challenge reply could never be seen`, `resume the claiming pane's session under ${sessionsRoot()}, or from the tick cwd ${sessionDirForCwd(tickCwdForProject(p))}, then re-run doctor`, ); case "no-dm-owner": return failFinding( "telegram-plumbing", `[${p.name}] no flat-chat challenge could pass: no dm-owner.json, and bot.lock names no session inside the arm scan surface — the private reply has no recipient arm can see`, `pin the orchestrator session as the DM owner (/telegram own in that pane), or pin escalation.telegramTopicId so the challenge rides a claimed forum topic`, ); case "dm-owner-dead": return failFinding( "telegram-plumbing", `[${p.name}] omp-telegram's dm-owner.json records no live pid — the pane that receives flat replies is gone`, `restart the pane that owns dm-owner.json so the bridge re-pins it with a live pid`, ); case "dm-owner-unrelated": return failFinding( "telegram-plumbing", `[${p.name}] the dm-owner.json session sits outside the arm scan surface — a flat reply would land in a transcript no challenge can read`, `pin the orchestrator session as the DM owner (/telegram own in this project's pane), or pin escalation.telegramTopicId so the challenge rides a claimed forum topic`, ); case "no-poller-state": return failFinding( "telegram-plumbing", `[${p.name}] omp-telegram's bot.lock is absent, unreadable or malformed — nothing owns the poll, so nothing is receiving inbound`, `check the bridge state dir for a bot.lock held by a live poller (the daemon releases it on a getMe or poller failure)`, ); case "poller-dead": return failFinding( "telegram-plumbing", `[${p.name}] omp-telegram's bot.lock records no live owner and no fresh heartbeat — Telegram isn't being polled`, `restart the omp-telegram bridge so a poller re-acquires bot.lock`, ); } } /** * #541 check 2 — `[session] resume_agents_on_restore` on the live herdr config. * * herdr's own README: the default `true` leaves a restored omp pane with a * deferred resume plan and no live terminal, which is the configuration that * pages instead of recovering. Absent ~ the unrecoverable default; an explicit * `true` is a deliberate (desktop) choice and only warned. */ /** * A dispatch fence whose owning process is gone (#938). * * Only a fence that *declared* an owner can be judged: `setPaused` records * `owner=` for a setup transaction, whose lifetime is that process's, and * deliberately not for an operator `hold` or the fail-closed hold a failed * apply leaves behind — those are meant to outlive every process, and calling * one "abandoned" would be worse than saying nothing about it. * * Measured 2026-08-21T13:24Z: an operator abandoned a setup that looked hung; * the policy change had applied, and dispatch sat under * `source=setup reason="setup apply fence"` until somebody thought to run * `resume`. Nothing anywhere said the process that wrote it was gone. * * This never clears anything, and that is deliberate. A pid is reusable, a * paused fleet is the safe state, and resuming dispatch nobody authorised is a * worse outcome than a fence that needs one command. So it names the command. */ function fenceProbe(probes: Probes, project: string | undefined): Finding { const id = project === undefined ? "dispatch-fence" : `dispatch-fence:${project}`; const fence = probes.pauseFence(project); if (fence === undefined) return passFinding(id, "dispatch is not held by a fence"); const why = fence.reason === undefined ? "" : ` — "${fence.reason}"`; if (fence.owner === undefined) { // An operator hold, or the durable hold a failed apply leaves on purpose. // Both are somebody's decision rather than a leak, whatever their age. return passFinding(id, `dispatch held by ${fence.source}${why} (no owning process; nothing to expire)`); } if (probes.pidLive(fence.owner)) { return passFinding(id, `dispatch held by a live ${fence.source} (pid ${fence.owner})`); } return warnFinding( id, `dispatch is held by ${fence.source}${why}, whose process (pid ${fence.owner}) is gone: the fence outlived ` + `the transaction that set it, and has stood since ${new Date(fence.since).toISOString()}`, `nothing here clears it — confirm no setup is running, then \`omp-conductor resume${project === undefined ? "" : ` --project ${project}`}\``, ); } /** * Whether any conductor process is running an upgrade transaction (#1113). * * The upgrade sentinel records no `owner=` — unlike a setup fence, whose pid * is precise, the drain spans subprocesses and the detached installer is a * transient unit — so the honest question is whether ANY conductor process is * mid-upgrade at all. Read from `/proc` cmdlines, never inferred from journal * state: a journal outlives its crash, which is exactly the ambiguity this * check exists to cut through. The whole transaction family matches * (`upgrade`, the detached `upgrade-install`, `upgrade-rollback`); unreadable * entries are skipped, and a zombie's empty cmdline cannot match. */ function upgradeProcessLive(): boolean { try { for (const entry of readdirSync("/proc")) { if (!/^\d+$/.test(entry)) continue; let raw: string; try { raw = readFileSync(join("/proc", entry, "cmdline"), "utf8"); } catch { continue; // vanished mid-scan, or not ours to read } const argv = raw.split("\0").filter((arg) => arg.length > 0); const conductor = argv.some((arg) => /(^|\/)(omp-conductor|cli\.ts)$/.test(arg)); if (conductor && argv.some((arg) => arg === "upgrade" || arg.startsWith("upgrade-"))) return true; } } catch { return false; } return false; } /** * The pause-state judgement shared by the doctor row and the daemon watch * (#1113), so the two surfaces cannot drift apart about which pauses still * have a plausible owner. The rules, in the order they are applied: * * - a recorded `owner=` is the precise fact whenever one exists (#938); * - `hold` is supported at any age — stopping the fleet deliberately is not * a fault, and must never nag; * - `upgrade` owns its drain by existing: a live transaction is fine, an * exited one is the one fault class, because no future event can ever * clear that sentinel; * - `spend-cap` clears itself when spend falls back below the cap; * - every other durable hold is somebody's decision, reported as such. */ export type HeldPause = { kind: "held"; verdict: | "operator-hold" | "self-expiring" | "durable" | "live-owner" | "live-upgrade" | "dead-owner" | "orphaned-upgrade"; source: string; reason?: string; since: number; owner?: number; }; export type PauseVerdict = { kind: "unpaused" } | HeldPause; export interface PauseLiveness { upgradeLive(): boolean; pidLive(pid: number): boolean; } export function classifyPause( pause: { source: string; reason?: string; since: number; owner?: number } | undefined, liveness: PauseLiveness, ): PauseVerdict { if (pause === undefined) return { kind: "unpaused" }; const base: Omit = { kind: "held", source: pause.source, ...(pause.reason === undefined ? {} : { reason: pause.reason }), since: pause.since, ...(pause.owner === undefined ? {} : { owner: pause.owner }), }; if (pause.owner !== undefined) { return { ...base, verdict: liveness.pidLive(pause.owner) ? "live-owner" : "dead-owner" }; } if (pause.source === "hold") return { ...base, verdict: "operator-hold" }; if (pause.source === "upgrade") { return { ...base, verdict: liveness.upgradeLive() ? "live-upgrade" : "orphaned-upgrade" }; } if (pause.source === "spend-cap") return { ...base, verdict: "self-expiring" }; return { ...base, verdict: "durable" }; } /** Fleet-scoped pages carry issue `0` — there is no tracker issue for this. */ const PAUSE_PAGE_ISSUE = 0; /** * The orphaned-upgrade page (#1113). Anchored on the sentinel's `since`, like * every once-per-incident page here: the notification ledger dedupes on the * summary, so a still-orphaned sentinel pages exactly once while each newly * recreated sentinel — a fresh `since` — pages again. */ export function orphanedPauseEscalation(project: string, pause: HeldPause): Escalation { const since = new Date(pause.since).toISOString(); return { tier: 2, category: "fleet-stopped", project, issue: PAUSE_PAGE_ISSUE, summary: `Orphaned upgrade pause holds dispatch since ${since} — no upgrade process is running (${project})`, detail: [ `Dispatch has been paused by the upgrade sentinel since ${since}` + (pause.reason === undefined ? "" : ` — "${pause.reason}"`) + ".", "The upgrade transaction that wrote it has exited, so no future event will", "ever clear it: dispatch stays stopped until someone resumes.", "", "Remedy: `omp-conductor resume --all`. Nothing auto-resumes — reporting is", "the whole contract (#1113); resuming a pause nobody authorised stays yours.", ].join("\n"), }; } /** One pause sentinel read: who set it, why, when — and any recorded owner. */ type PauseInstance = { source: string; reason?: string; since: number; owner?: number }; export interface OrphanedPauseDeps { /** The project this daemon serves; the effective sentinel for it reads * through the same precedence the doctor row uses (project wins over the * legacy global file). */ project: string; escalate(e: Escalation): Promise; log?(msg: string): void; pause?: () => PauseInstance | undefined; /** This project's own `paused-` sentinel, told apart from the * legacy global one so a genuinely per-project orphan pages for its own * project while a shared one pages exactly once (#1113 review round 1). */ projectSentinel?: () => PauseInstance | undefined; globalSentinel?: () => PauseInstance | undefined; upgradeLive?: () => boolean; pidLive?: (pid: number) => boolean; /** Every configured project name — from the shared config.json every * daemon on this host reads, which is what makes the canonical owner * below one identity everywhere. Defaults to {@link loadConfig}. */ configuredProjects?: () => string[]; } /** * Best-effort page delivery, mirroring `safeEscalate`: a transport failure is * logged, never thrown, and cannot take the tick down. */ async function deliverOrphanPage( deps: OrphanedPauseDeps, write: (msg: string) => void, held: HeldPause, ): Promise { let delivered = true; try { await deps.escalate(orphanedPauseEscalation(deps.project, held)); } catch (err) { delivered = false; write(`orphaned-pause escalation could not be delivered: ${errText(err)}`); } write( `ERROR: an orphaned upgrade pause holds dispatch (since ${new Date(held.since).toISOString()})` + (delivered ? " — paged once for this sentinel" : " — page unsent"), ); } /** * The daemon half of #1113: raise the orphaned-upgrade case through the * ordinary escalation path instead of waiting for a tick to read `status` — * a stopped fleet is precisely the condition that cannot assume anything is * watching. Called above the pause gate on every pass; nothing here ever * resumes. * * "Once" needs an ownership boundary, because the ledger that dedupes pages * is per-project store while the upgrade sentinel usually is not: it writes * the legacy GLOBAL file, and every project's effective read falls back to * it. So the two sentinel kinds are told apart before paging: * * - this project's own `paused-` orphan → this project pages as * itself, whatever else is configured; * - the legacy global orphan → exactly one daemon owns the incident: the * one serving the alphabetically-first configured project name — the same * name on every daemon, because they all read the same config.json. A * daemon that cannot read that config assumes it is alone and pages * anyway; staying silent on a failed read would be the exact silence * #1113 exists to end. * * Within either owner, the ledger dedupes on the since-anchored summary, so * a still-orphaned sentinel pages once and a recreated one (fresh `since`) * pages again. */ export async function reconcileOrphanedPause(deps: OrphanedPauseDeps): Promise { const write = deps.log ?? ((msg: string) => process.stderr.write(`[conductor ${new Date().toISOString()}] ${msg}\n`)); const liveness: PauseLiveness = { upgradeLive: deps.upgradeLive ?? upgradeProcessLive, pidLive: deps.pidLive ?? pidAlive, }; const verdict = classifyPause((deps.pause ?? (() => pauseInstance(deps.project)))(), liveness); if (verdict.kind !== "held" || verdict.verdict !== "orphaned-upgrade") return; // A per-project sentinel of its own: this project owns that page outright. const ownVerdict = classifyPause( (deps.projectSentinel ?? (() => pauseInstanceAt(pausedPath(deps.project))))(), liveness, ); if (ownVerdict.kind === "held" && ownVerdict.verdict === "orphaned-upgrade") { await deliverOrphanPage(deps, write, ownVerdict); return; } // The shared global sentinel: gate on the canonical owner before paging. let configured: string[] | undefined; try { configured = deps.configuredProjects?.() ?? loadConfig().projects.map((p) => p.name); } catch { configured = undefined; } const names = configured ?? []; const owner = names.length === 0 ? deps.project : [...names].sort()[0]; if (deps.project !== owner) { write(`orphaned upgrade pause holds dispatch — ${owner} owns the page for the global sentinel`); return; } const globalVerdict = classifyPause( (deps.globalSentinel ?? (() => pauseInstanceAt(pausedPath())))(), liveness, ); if (globalVerdict.kind === "held" && globalVerdict.verdict === "orphaned-upgrade") { await deliverOrphanPage(deps, write, globalVerdict); } } /** * The dispatch-pause row itself (#1113), beside {@link fenceProbe}: the fence * row judges a declared owning process, this one judges the pause — `pass` * when unpaused, and when paused naming the source, the age, and whether * anything still plausibly owns it. One fault class only: the orphaned * upgrade sentinel, which can never clear itself. Nothing here resumes * anything; reporting is the whole contract. */ function pauseProbe(probes: Probes, project: string | undefined): Finding { const id = project === undefined ? "dispatch-pause" : `dispatch-pause:${project}`; const v = classifyPause(probes.pauseFence(project), { upgradeLive: probes.upgradeLive, pidLive: probes.pidLive, }); if (v.kind === "unpaused") return passFinding(id, "dispatch is not paused"); const why = v.reason === undefined ? "" : ` — "${v.reason}"`; const age = formatDownDuration(Math.max(0, probes.now() - v.since)); const base = `dispatch paused by ${v.source}${why}, ${age} ago`; switch (v.verdict) { case "orphaned-upgrade": return failFinding( id, `${base}, and no upgrade process is running — this pause can never clear itself`, "run `omp-conductor resume --all` (nothing here resumes it)", ); case "dead-owner": return warnFinding( id, `${base} — its owning process (pid ${v.owner}) is gone`, `confirm nothing is running, then \`omp-conductor resume${project === undefined ? "" : ` --project ${project}`}\``, ); case "operator-hold": return passFinding(id, `${base} — a deliberate hold, supported at any age`); case "live-upgrade": return passFinding(id, `${base} — the upgrade transaction is draining, and resumes dispatch when it finishes`); case "live-owner": return passFinding(id, `${base} — owned by live pid ${v.owner}`); case "self-expiring": return passFinding(id, `${base} — clears itself when spend falls below the cap`); case "durable": return passFinding(id, `${base} — no owning process; held until someone resumes it`); } } function herdrResumeProbe(probes: Probes): Finding { if (!probes.herdrInstalled()) { return passFinding("herdr-resume", "herdr not installed — nothing to check"); } const live = probes.herdrConfig(); if (live === undefined) { return failFinding( "herdr-resume", "no herdr config at " + join(homedir(), ".config", "herdr", "config.toml") + " — a restored omp pane gets no live terminal (herdr's own README)", "run \`omp-conductor setup host\` from the fleet account (writes [session] resume_agents_on_restore = false), or add it by hand", ); } let parsed: { session?: { resume_agents_on_restore?: unknown } }; try { parsed = Bun.TOML.parse(live) as { session?: { resume_agents_on_restore?: unknown } }; } catch (err) { return failFinding( "herdr-resume", `the live herdr config does not parse (${err instanceof Error ? err.message : String(err)}) — herdr itself refuses it`, "run `herdr config check` to name the line, fix, then re-run doctor", ); } const value = parsed.session?.resume_agents_on_restore; if (value === false) { return passFinding("herdr-resume", "resume_agents_on_restore is false — restored panes come up as shells, not deferred plans"); } if (value === true) { return warnFinding( "herdr-resume", "resume_agents_on_restore is explicitly true — possibly a deliberate desktop config, but a headless fleet restores panes with no live terminal", "set it to false for a headless host ([session] resume_agents_on_restore = false), or run `omp-conductor setup host`", ); } return failFinding( "herdr-resume", "resume_agents_on_restore is absent — herdr defaults it to true, the configuration its own README calls unrecoverable", "run `omp-conductor setup host` from the fleet account (writes the key), or add [session] resume_agents_on_restore = false by hand", ); } /** * #541 check 3 — herdr-conductor `config.env` FLEET_CWDS vs the configured * projects. Absent, a virgin host recovers against legacy `/root/fleet` paths * that do not exist: recovery still runs, only the page silently goes nowhere. */ function herdrEnvProbe(probes: Probes, cfg: ConductorConfig | undefined): Finding { if (!probes.herdrInstalled()) { return passFinding("herdr-config-env", "herdr not installed — nothing to check"); } if (cfg === undefined) { return passFinding("herdr-config-env", "config unreadable — nothing to compare"); } const live = probes.herdrEnv(); if (live === undefined) { return failFinding( "herdr-config-env", "no herdr-conductor config.env — recovery falls back to legacy /root-based single-tenant paths this host does not use, so pages silently skip", "run `omp-conductor setup host` from the fleet account (writes config.env)", ); } const expected = new Set(cfg.projects.map((p) => tickCwdForProject(p))); const actual = parseFleetCwds(live); const coherent = actual !== undefined && actual.size === expected.size && [...actual].every((cwd) => expected.has(cwd)); if (coherent) { return passFinding("herdr-config-env", `config.env FLEET_CWDS matches the configured fleets (${[...expected].join(":")})`); } return failFinding( "herdr-config-env", `config.env FLEET_CWDS (${actual === undefined ? "unset" : [...actual].join(":")}) does not match the configured fleets (${[...expected].join(":")})`, "re-run `omp-conductor setup host`, which derives FLEET_CWDS from the configured projects", ); } /** FLEET_CWDS (colon- or space-separated, per recover.sh) or undefined. */ function parseFleetCwds(text: string): Set | undefined { const value = parseEnvKey(text, "FLEET_CWDS"); if (value !== undefined) { return new Set(value.split(/[\s:]+/).filter((w) => w.length > 0)); } // Legacy single-fleet form, the same fallback recover.sh applies. const legacy = parseEnvKey(text, "FLEET_CWD"); return legacy === undefined ? undefined : new Set([legacy]); } /** The first unquoted-or-quoted `KEY=value` on its own line, comments ignored. */ function parseEnvKey(text: string, key: string): string | undefined { for (const line of text.split("\n")) { const match = new RegExp(`^[ \\t]*${key}[ \\t]*=[ \\t]*"?([^"\\n]*)"?[ \\t]*$`).exec(line); if (match !== null) return match[1]; } return undefined; } /** * Install-surface parity (#904). omp-conductor installs onto three surfaces — * the discovered CLI package tree, the omp plugin, and the herdr recovery * plugin — and only a live `upgrade` invocation ever compared them. On a * fleet whose installs are manual they diverge silently: measured on this * host on 2026-08-22, the omp plugin sat on the withdrawn 0.18.1 release for * about a day beside a 0.18.0 daemon, with no finding, no status line and no * escalation. * * The question is inter-surface agreement on the host, never staleness * against the registry: a fleet may deliberately sit on an older release, * but never on two at once. `expectedGitHead` is the commit the CLI's own * version was published from, so the herdr pin can be judged against the * same release; `undefined` means the registry could not be read and the pin * is reported as unverified rather than as a mismatch. */ /** * The bootstrap deadlock (#910), which is a conjunction rather than a fault: * * - no worker on this host can start, because the *installed* code cannot * load its harness; and * - the fix for that ships through a release, and a release needs a merged * worker pull request — so the broken install is what blocks its own * replacement. * * Kept as its own finding, never folded into the stale-setup ones, because the * two states call for opposite actions. "Host runtime differs — run setup host" * is *correct advice* for a stale install and *useless* here: no amount of * re-running setup replaces the installed package, and diagnosing that cost * real time on 2026-08-20..22. So this names the one command that breaks the * cycle (#908's bootstrap identity), and an ordinary stale host still reads * exactly as it did. */ function bootstrapDeadlockProbe( harness: { ok: true; version: string } | { ok: false; detail: string }, ): Finding { if (harness.ok) { return passFinding("bootstrap-deadlock", `workers can load the installed harness (${harness.version})`); } return failFinding( "bootstrap-deadlock", `no worker can start: the installed conductor cannot load its harness — ${harness.detail}. ` + "A release cannot publish the fix either, because publishing needs a merged worker pull request, " + "so the installed package blocks its own replacement", "install the fix by exact commit instead of by version: `omp-conductor upgrade --bootstrap " + "--source ` (its checks run against the source first, and it refuses rather " + "than installing a partly verified tree)", ); } /** * Is the plugin a mandated conductor contract runs on actually current? (#961) * * Sibling to {@link surfaceParityProbe}, and deliberately a separate finding: * that one is about the three surfaces of *this* package agreeing with each * other, this one is about the one peer the floor's reply-in-topic instruction * depends on being new enough to honour it. * * A `warn`, never a `fail`. Nothing here is broken on this host's own terms — * the fleet dispatches, merges and reports fine on a stale plugin. What breaks * is one documented instruction, and the remedy is an install the operator owns, * so failing the run would make `doctor` red for something conductor must not * fix itself. */ export function telegramFreshnessProbe(freshness: TelegramFreshness): Finding { const id = "telegram-plugin-freshness"; const { installed, daemon, published } = freshness.surfaces; const named = `installed=${installed.kind === "version" ? installed.version : installed.kind}, ` + `daemon=${daemon.kind === "version" ? daemon.version : daemon.kind}, ` + `published=${published.kind === "version" ? published.version : published.kind}`; switch (freshness.state) { case "current": return passFinding(id, `${TELEGRAM_PACKAGE} is current (${named})`); case "installed-behind": return warnFinding( id, `${freshness.detail} (${named})`, `install ${TELEGRAM_PACKAGE}@${published.kind === "version" ? published.version : "latest"} and restart its daemon; a targetless telegram_send needs the newer target ladder to reply in the topic a message arrived in (#882)`, ); case "daemon-stale": return warnFinding( id, `${freshness.detail} (${named})`, `restart the ${TELEGRAM_PACKAGE} daemon so it serves the installed version`, ); case "not-installed": return warnFinding( id, `${freshness.detail} (${named})`, `install ${TELEGRAM_PACKAGE}, or stop relying on Telegram delivery for this fleet`, ); case "unknown": // Not a finding about the plugin: a finding about this check. An // unreachable registry must never read as "behind" *or* as "current". return warnFinding( id, `${TELEGRAM_PACKAGE} freshness unverified: ${freshness.detail ?? "no surface answered"} (${named})`, "no action needed if this host has no outbound npm access; otherwise re-run doctor when it does", ); } } export function surfaceParityProbe( read: SurfaceRead, expectedGitHead: string | undefined, herdrExpected: boolean, ): Finding { const id = "install-surfaces"; if (!read.ok) { return warnFinding( id, `installed surfaces unreadable: ${read.detail} — parity between the CLI, the omp plugin and the herdr plugin is unverified`, "check that `omp-conductor`, `omp` and (when this host runs herdr) `herdr` are on PATH for this user, then re-run doctor", ); } const { cliVersion, ompVersion, herdrSource } = read.surfaces; const named = `cli=${cliVersion}, omp=${ompVersion ?? "absent"}, herdr=${herdrSource ?? "absent"}`; // A proven disagreement first: two different releases live on one host is // the fault this probe exists for, and it outranks anything absent. if (ompVersion !== undefined && ompVersion !== cliVersion) { return failFinding( id, `installed surfaces disagree: the omp plugin is ${ompVersion} while the CLI/daemon tree is ${cliVersion} (${named})`, `run \`omp-conductor upgrade --to ${cliVersion}\` (or to the release this fleet should be on) so all three surfaces carry one identity`, ); } if (herdrExpected && herdrSource !== undefined && expectedGitHead !== undefined) { if (herdrSource.startsWith("local:")) { return warnFinding( id, `the herdr plugin is linked to a local checkout, not a released pin (${named})`, `run \`omp-conductor upgrade --to ${cliVersion}\` to replace the link with the released pin, or keep the link deliberately while developing`, ); } const expected = expectedHerdrSource(expectedGitHead); if (herdrSource !== expected) { return failFinding( id, `installed surfaces disagree: the herdr plugin is pinned to ${herdrSource} while ${cliVersion} was published from ${expectedGitHead} (${named})`, `run \`omp-conductor upgrade --to ${cliVersion}\` so the herdr recovery plugin matches the release the rest of the host runs`, ); } } // Nothing disagrees. Anything absent is named as absent — a surface this // host does not carry is not a mismatch, and a pin that could not be // verified is not a pass in disguise. const absent: string[] = []; if (ompVersion === undefined) absent.push("omp plugin not installed — a tick cannot arm without it"); if (herdrExpected && herdrSource === undefined) absent.push("herdr plugin not installed — pane recovery cannot run"); if (herdrExpected && herdrSource !== undefined && expectedGitHead === undefined) { absent.push(`herdr pin ${herdrSource} unverified — the registry did not answer what ${cliVersion} was published from`); } if (absent.length > 0) { return warnFinding( id, `${absent.join("; ")} (${named})`, `run \`omp-conductor upgrade --to ${cliVersion}\` to install every surface this host needs from one release`, ); } return passFinding(id, `one identity across every installed surface (${named})`); } /** * Run every probe and assemble the stable report. * * Host-wide facts (config backup, store integrity, gh auth, systemd units, * ownership, spend telemetry) are probed once; the per-project facts (labels, * reporting timezone, telegram) are probed for every resolved project, each * named (#530). * * @param projectName the `--project NAME` value; undefined checks every * configured project * @param opts injected seams; every probe defaults to the production wiring */ export async function runDoctor(projectName: string | undefined, opts: DoctorDeps = {}): Promise { const probes: Probes = { ...defaultProbes(), ...opts }; const checkedAt = probes.now(); const checkedAtIso = new Date(checkedAt).toISOString(); let cfg: ConductorConfig | undefined; let configProblem: string | undefined; try { cfg = probes.loadConfig(); } catch (err) { configProblem = messageOf(err); } // Resolve the project set: the named project when `--project NAME` is given, // otherwise *(every)* configured project. Doctor is fleet-scoped and covers // the whole host, so a no-name run checks every project's per-project facts // once each, not the first one and not nothing (#530). let projects: ProjectConfig[] = []; let projectProblem: string | undefined; if (cfg !== undefined) { if (projectName !== undefined) { try { projects = [findProject(cfg, projectName)]; } catch (err) { projectProblem = messageOf(err); } } else { projects = cfg.projects; } } const findings: Finding[] = []; findings.push(configProbe(configProblem)); if (cfg === undefined) { findings.push(passFinding("project", "config unreadable — no project to resolve")); } else if (projectProblem !== undefined) { findings.push( failFinding( "project", `no project resolved: ${projectProblem}`, "pass --project NAME (doctor checks the named project; with no name it checks every configured project)", ), ); } else if (projects.length === 1) { findings.push(passFinding("project", projects[0]!.name)); } else { findings.push(passFinding("project", `${projects.map((p) => p.name).join(", ")} (${projects.length} projects)`)); } // Host-wide facts, probed once per run — never once per project (the daemon // unit, the store and the token are shared across projects, so re-probing // them per project and deduping the output would hide a real per-project // difference behind a constant count). findings.push(backupProbe(probes)); findings.push(dbProbe(probes)); findings.push(dbBackupProbe(probes, cfg)); findings.push(await ghAuthProbe(probes, configuredRepos(cfg))); // Install-surface parity is host-wide, like the unit and the store: one // read of the three identities, judged once (#904). const surfaces = await probes.installedSurfaces(); const cliVersion = surfaces.ok ? surfaces.surfaces.cliVersion : undefined; const expectedGitHead = cliVersion === undefined ? undefined : await probes.releaseGitHead(cliVersion); findings.push(surfaceParityProbe(surfaces, expectedGitHead, probes.herdrInstalled())); // Host-wide for the same reason: one plugin install serves every project, so // it is judged once rather than repeated per project (#961). findings.push(telegramFreshnessProbe(await probes.telegramFreshness())); // Host-wide like the surfaces: one install, one harness, one answer (#910). findings.push(bootstrapDeadlockProbe(await probes.workerHarness())); // The run-session root is one host-wide fact — where every run's session dir // (and the omp settings overlay inside it) lands — probed once and shared // across the per-project omp-settings findings. const sessionRoot = sessionRootDir(); const sessionRootState = probes.sessionRootState(sessionRoot); // The collected run sample across the resolved project set, newest-first; // spend telemetry is a single host-wide finding, not one per project. const spendRows: RunSpendRow[] = []; for (const p of projects) spendRows.push(...probes.recentRuns(p.name, SPEND_SAMPLE_ROWS)); // Per-project probes run for every resolved project, each named in its // finding so a multi-project run stays legible. When nothing resolved // (config unreadable, or --project named an unknown project), a no-op pass // keeps the stable report shape rather than silently dropping the row. if (projects.length === 0) { findings.push( passFinding("labels", projectProblem === undefined ? "no project resolved — nothing to check" : `labels uncheckable: ${projectProblem}`), ); findings.push( passFinding("herdr-agent-name", projectProblem === undefined ? "no project resolved — nothing to check" : `agent name uncheckable: ${projectProblem}`), ); findings.push( passFinding("arm-ack", projectProblem === undefined ? "no project resolved — nothing to check" : `arm acknowledgement uncheckable: ${projectProblem}`), ); findings.push( passFinding("topic-pin", projectProblem === undefined ? "no project resolved — nothing to check" : `topic pin uncheckable: ${projectProblem}`), ); findings.push( passFinding("telegram-plumbing", projectProblem === undefined ? "no project resolved — nothing to check" : `telegram plumbing uncheckable: ${projectProblem}`), ); findings.push( passFinding("omp-settings", projectProblem === undefined ? "no project resolved — nothing to check" : `omp settings overlay uncheckable: ${projectProblem}`), ); } else { for (const p of projects) { findings.push(await labelProbe(probes, p)); findings.push(herdrAgentNameProbe(probes, p)); findings.push(armAckProbe(probes, p)); findings.push(topicPinProbe(probes, p)); findings.push(telegramPlumbingProbe(probes, p)); findings.push(ompSettingsProbe(p, sessionRootState)); } } // The installed-unit check is host-global: one shared daemon. Any project // renders the same canonical units (the daemon/units carry no project), so // the first one stands in for the rendering seam. findings.push(unitProbe(probes, projects[0], cfg)); findings.push(recoveryProbe(probes)); findings.push(recoveryStateProbe(probes)); findings.push(ownershipProbe(probes)); // #541 seam checks, host-global: the live herdr config and the plugin's // config.env are single files on the host, not per-project facts. findings.push(herdrResumeProbe(probes)); findings.push(herdrEnvProbe(probes, cfg)); if (projects.length === 0) { findings.push(timezoneProbe(undefined)); findings.push(await telegramProbe(probes, undefined, checkedAt)); findings.push(fenceProbe(probes, undefined)); findings.push(pauseProbe(probes, undefined)); } else { for (const p of projects) findings.push(timezoneProbe(p)); for (const p of projects) findings.push(await telegramProbe(probes, p, checkedAt)); // Per project, because the sentinel is: one fleet's fence must never be // reported against another's name. for (const p of projects) findings.push(fenceProbe(probes, p.name)); // The pause state itself (#1113), per project for the same reason. for (const p of projects) findings.push(pauseProbe(probes, p.name)); } // The billing class the spend row needs (#984). Declared only when EVERY // project requires a subscription credential: on a mixed fleet the zeros of // an API-key project are still a fault, and one project's declaration must // not excuse another's telemetry loss. const declaredSubscription = projects.length > 0 && projects.every((p) => (p.requireOauthProviders ?? []).length > 0); const allowanceWindow = probes.allowanceWindow === undefined ? undefined : await probes.allowanceWindow(); const billing = { declaredSubscription, ...(allowanceWindow === undefined ? {} : { allowanceWindow }), }; findings.push(spendProbe(spendRows, SPEND_SAMPLE_RUNS, billing)); // Per project, because the caps are: one project's configured ceiling says // nothing about another's (#985). const subscriptionBilled = judgeSpendTelemetry(spendRows, SPEND_SAMPLE_RUNS, billing).kind === "subscription"; if (cfg !== undefined) { const defaults = cfg.defaults; for (const p of projects) { findings.push( economicGateProbe(p, resolveCaps(p, defaults), subscriptionBilled, allowanceWindow), ); } } const status: ReportStatus = findings.some((f) => f.status === "fail") ? "fail" : findings.some((f) => f.status === "warn") ? "warn" : "pass"; return { project: projects.length === 0 ? projectName ?? "(none resolved)" : projects.length === 1 ? projects[0]!.name : projects.map((p) => p.name).join(", "), checkedAt: checkedAtIso, status, findings, }; } /** * Exercise the worker's real import contract out of process (#910). * * `harness-loader.ts` is the module a worker session loads its harness through, * and it is runnable — so this runs exactly that, with `--no-install`, and reads * its verdict rather than re-deriving one. Anything else would be a second * opinion about the launch path, which is how a doctor comes to disagree with * the thing it is checking. */ async function defaultWorkerHarness(): Promise<{ ok: true; version: string } | { ok: false; detail: string }> { const loader = join(import.meta.dir, "harness-loader.ts"); try { const child = Bun.spawn(["bun", "--no-install", loader], { stdin: "ignore", stdout: "pipe", stderr: "pipe", env: process.env, }); const stdout = new Response(child.stdout).text(); const stderr = new Response(child.stderr).text(); const code = await child.exited; if (code !== 0) { const detail = ((await stderr).trim() || (await stdout).trim()).split("\n")[0] ?? `exit ${code}`; return { ok: false, detail }; } const parsed: unknown = JSON.parse((await stdout).trim()); const version = parsed !== null && typeof parsed === "object" ? Reflect.get(parsed, "version") : undefined; return typeof version === "string" && version.length > 0 ? { ok: true, version } : { ok: false, detail: "the harness loaded but reported no version" }; } catch (err) { return { ok: false, detail: err instanceof Error ? err.message : String(err) }; } } /** The default wiring — every production transport the rest of the package uses. */ export function defaultProbes(): Probes { return { loadConfig, scopes: checkTokenScopes, rateLimitOk: async () => (await fetchRateLimit()) !== undefined, repoReadable: defaultRepoReadable, repoLabels: defaultRepoLabels, hasSystemd: () => existsSync(SYSTEMD_UNIT_DIR), readUnit: defaultReadUnit, unitFailed: defaultUnitFailed, stat: defaultStat, uidOf: defaultUidOf, dbIntegrity: defaultDbIntegrity, snapshotDirState: defaultSnapshotDirState, sessionRootState: defaultSessionRootState, recentRuns: defaultRecentRuns, telegramHealth: (projectName) => probeTelegramHealth(projectName), telegramSend: telegramReportSend, canonicalUnits: defaultCanonicalUnits, herdrInstalled: () => Bun.which("herdr") !== null, pauseFence: (project) => pauseInstance(project), upgradeLive: upgradeProcessLive, workerHarness: defaultWorkerHarness, pidLive: (pid) => pidAlive(pid), herdrAgents: defaultHerdrAgents, herdrSession: () => resolveHerdrSession(), herdrConfig: defaultHerdrConfig, herdrEnv: defaultHerdrEnv, claimedTopics: () => claimedTelegramTopics(), topicMisroutes: () => readTelegramMisroutes(), topicsTidy: () => telegramTopicsTidy(), pollState: () => readTelegramPollState(), dmOwner: () => readTelegramDmOwner(), channelState: (p) => { // The exact tick accessFile armTicks probes — never the state-dir // default, so a project whose tick pairs a different bridge is judged // against its own file. const tick = resolveTickConfig(p.name); if (tick.kind !== "ok" || tick.config.accessFile === undefined) { return { kind: "down", reason: "no tick accessFile" }; } return readTelegramChannel(tick.config.accessFile); }, projectArms: (p) => { // A tick config only carries inbound when it names an accessFile: the // tick runtime treats a missing one as no fleet bridge (and armTicks // throws without it), so a parseable config with none is no more inbound // than no config at all. const tick = resolveTickConfig(p.name); return tick.kind === "ok" && tick.config.accessFile !== undefined; }, armSendTopic: (p) => resolveProjectTopicId(p), armScanDirs: (p) => { // The same session surface armTicks builds from the tick cwd and the // live claim's file: the cwd-derived session directory, plus the claim's // own directory when resume moved the transcript elsewhere (#600). const cwdDir = sessionDirForCwd(tickCwdForProject(p)); const claimed = resolveClaimedSessionFile(p); if (claimed === undefined) return [cwdDir]; const root = sessionsRoot(); const claimDir = dirname(claimed); if (claimDir !== root && !claimDir.startsWith(join(root, sep))) return [cwdDir]; return claimDir === cwdDir ? [cwdDir] : [cwdDir, claimDir]; }, pidAlive, allowanceWindow: async () => bindingAllowanceWindow(await sharedUsageSource().read()), lockPidAlive, lockFresh: (mtimeMs) => Date.now() - mtimeMs < TELEGRAM_LOCK_FRESH_MS, tickAgentName: (p) => { const tick = readTickConfig(tickCwdForProject(p)); if (tick.kind !== "ok") return undefined; return tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME; }, installedSurfaces: async () => { // `upgrade`'s own reader, so doctor cannot disagree with the transaction // about what is installed. A host with no herdr is asked only about the // two surfaces it has, rather than failing the whole read (#904). try { const surfaces = await inspectSurfaces(UPGRADE_DEPS, { readHerdr: Bun.which("herdr") !== null }); return { ok: true, surfaces }; } catch (err) { return { ok: false, detail: messageOf(err) }; } }, telegramFreshness: () => // The module's own reader, so `doctor` and `status` cannot disagree about // what is installed — the same one-seam rule `installedSurfaces` follows. checkTelegramFreshness({ run: async (cmd, args) => { const r = await UPGRADE_DEPS.run(cmd, args); return { code: r.code, stdout: r.stdout }; }, }), releaseGitHead: async (version) => { try { return (await releaseIdentity(UPGRADE_DEPS, version)).gitHead; } catch { // An unreadable registry makes the herdr pin unverifiable, never // wrong: the probe says so instead of inventing a mismatch. return undefined; } }, now: Date.now, probeTelegram: false, }; }