import { loadLiveTransports, isMarkerLive, isPidAlive, markerHoldsLiveProcess } from "./registry.js"; import { registerTmuxHost, activeTransport, HERDR, type TmuxHost, isLocallyProbeable, isTmuxKind, paneExists, tmuxVersion, targetOf, tmuxAvailable, } from "../transports/index.js"; import { newestMtimeUnder, onDiskBuildMtime, onDiskSourceMtime, SERVER_BUILD_MTIME, SERVER_BUILD_SHA, BUILD_DIR } from "../build.js"; import { prefixOf, prefixVerdict } from "../prefix.js"; import { execFileSync } from "node:child_process"; import { registerTool } from "./registry.js"; import { roleInputSchema, type RoleArg } from "../roles.js"; import { attributeWriter, isGitRepo, lastWriterOf, loadScopes, ownsDocument } from "./scopes.js"; // Moved to a LEAF so `status` can report capabilities without a cycle — see // src/server-identity.ts. Re-exported so every existing importer is untouched. export { resolveServerIdentity } from "../server-identity.js"; import { resolveServerIdentity } from "../server-identity.js"; // ONE-WAY EDGE, and the direction is the point: `status` CALLS the capabilities // verb rather than re-deriving its probes. Two probes of the same thing is the // four-matchers defect committed deliberately, which is why 17.4 was deferred // behind 13.7 instead of built beside it. import { capabilitiesTool } from "../capabilities.js"; import { sendMessageTool, readMessagesTool } from "./messaging.js"; import { randomUUID } from "node:crypto"; import { existsSync, readFileSync, watch } from "node:fs"; import { promises as fsp } from "node:fs"; import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import { z } from "zod"; import { seatBuildOf, installedFrom, psReader } from "./seat-build.js"; import path from "node:path"; import { AGENTS_FILE, CURSOR_DIR, DEFAULT_ROOM, INBOX_DIR, ROOT, ROOM_FILE, ROOMS_DIR, ROOMS_FILE, STATUS_FILE, addMember, appendJsonl, cursorFile, deleteFile, ensureRoom, fileSize, getRooms, inboxFile, listCursorFiles, listInboxFiles, listSessionFiles, listTransportFiles, type SessionBinding, logFile, memberRooms, normalizeRoom, pidFile, readJson, readJsonl, receiptFile, listReceiptFiles, removeMember, rewriteJsonl, roomFile, rotateAgentToken, setRoomMeta, stashHistory, retrieveHistory, pruneHistory, transportFile, TRANSPORT_DIR, updateJson, type RoomRegistry, } from "../store.js"; import { type AgentEntry, type AgentRegistry, type Message, type StatusEntry, type Cursor, type Source, type TransportMarker, sourceFile, getOffset, setOffset, sysMsg, moveFile, agentIdFromCursorFilename, STALE_MS, EVICT_MS, MAX_WAIT_MS, } from "./shared.js"; // ---------- ping ---------- export const pingSchema = { from: z.string().min(1), to: z.string().min(1), echo: z.boolean().optional(), }; // Liveness probe answered entirely from server-side state — registry entry, // transport marker, pusher pid, tmux pane. It never touches the target's // session, so a fleet-wide sweep costs zero model tokens on the targets. // Distinct from `heartbeat` (the target refreshing its own activity // timestamp): ping is a third party asking "would a DM land right now?". // echo=true is the one exception — it drops a PING DM into the target's inbox // (normal delivery, so the target's model DOES wake); opt-in, default off. export async function pingTool(args: { from: string; to: string; echo?: boolean }) { const t0 = process.hrtime.bigint(); const now = Date.now(); const latencyMs = () => Math.round(Number(process.hrtime.bigint() - t0) / 1e3) / 1e3; const reg = await readJson(AGENTS_FILE, {}); const entry = reg[args.to]; if (!entry) { return { ok: true, to: args.to, alive: false, reachable: false, reason: "unregistered", latencyMs: latencyMs() }; } const marker = await readJson(transportFile(args.to), null); const heartbeatAgeSec = Math.floor((now - entry.lastHeartbeat) / 1000); const heartbeatFresh = now - entry.lastHeartbeat < STALE_MS; let transportLive = false; let paneAlive: boolean | undefined; if (marker) { transportLive = isMarkerLive(marker, reg, now); if (transportLive && isLocallyProbeable(marker.transport) && targetOf(marker)) { // The pusher can outlive its pane (agent window closed) — probe the pane. // Why `has-session` and not `display-message`, with the positive control // both ways, is documented once on `paneExists`. paneAlive = paneExists(targetOf(marker)!); } } const reachable = transportLive && paneAlive !== false; // heartbeatFresh is only valid EVIDENCE where something writes a heartbeat. // A local tmux-push transport's liveness is its pid (isPidAlive) — // hooks/tmux-pusher.mjs never calls `heartbeat`, so the field measures time // since JOIN, not activity (Task 13.3/13.4, stall.ts's own established // rule for exactly this transport type). Crediting it here for a // tmux-push agent produced the incident this fix exists for: `alive` and // `reachable` read fully healthy, `heartbeatFresh` sat in `checks` reading // false, and the top-line boolean never surfaced the disagreement — a // status layer discarding what its own lower layer already reported, the // same shape as `stall_clock_status` before Task 13.1. A REMOTE pusher, or // an agent with no probeable marker at all, has no pid to fall back on — // there heartbeat genuinely IS the liveness mechanism, unchanged. const heartbeatIsValidSignal = !marker || !isLocallyProbeable(marker.transport); const alive = reachable || (heartbeatIsValidSignal && heartbeatFresh); let echoSent = false; if (args.echo && alive) { // ⟨q-138f4b78⟩ — the echo is an agent→agent DM, so past the typed-record // cutover it MUST carry a record or the send is refused; and `echoSent` // reports what the send SAID, never that a call was made — measured on the // cutover morning: this reported true while nothing was written. const sent = await sendMessageTool({ from: args.from, to: args.to, text: `PING: echo requested by ${args.from} — DM back if responsive.`, record: { type: "fyi", payload: { summary: `PING echo requested by ${args.from}` } }, }); echoSent = sent.ok === true; } return { ok: true, to: args.to, alive, reachable, ...(alive ? {} : { reason: marker ? "transport-dead" : "heartbeat-stale" }), checks: { registered: true, heartbeatFresh, heartbeatAgeSec, // Whether heartbeatFresh above is real evidence for this transport, or // just time-since-join. A reader who sees `heartbeatFresh: false` next // to `heartbeatValid: false` should not read that as a dissenting // signal — there is no signal there to dissent. heartbeatValid: heartbeatIsValidSignal, transport: marker?.transport ?? null, transportLive, ...(paneAlive !== undefined ? { paneAlive, tmuxTarget: marker?.tmuxTarget } : {}), }, ...(args.echo ? { echoSent } : {}), latencyMs: latencyMs(), }; } // ---------- send_command (context-management control commands) ---------- // The only slash commands a lead may inject into a sub-agent's CLI. Locked on // purpose: these wipe/compact context or reload the harness's own skill files // (cheap, reversible-by-the-agent, harness-local), nothing that mutates the repo // or the bus. `reload-skills` added 2026-08-19 on a DAVID_DECISION (option 1: // that one command only; re-decide per command). Stored WITHOUT the leading // slash; the wire text is `/${cmd}`. export const CONTROL_COMMANDS = ["clear", "compact", "reload-skills"] as const; // Transports whose pusher can actually TYPE a slash command into a live CLI. // A control command is meaningless to a plain MCP poller, so send_command is // gated to agents currently attached over one of these. // Normalize "clear" / "/clear" / " /Clear " → "clear"; null if not allowlisted. function normalizeControlCommand(raw: string): string | null { const c = raw.trim().replace(/^\/+/, "").toLowerCase(); return (CONTROL_COMMANDS as readonly string[]).includes(c) ? c : null; } // Live transports filtered to the tmux-push family (local + remote). async function liveTmuxTargets(): Promise> { const all = await loadLiveTransports(); const out = new Map(); for (const [id, m] of all) if (isTmuxKind(m.transport)) out.set(id, m); return out; } export const sendCommandSchema = { from: z.string().min(1), to: z.string().optional(), room: z.string().optional(), command: z.string().min(1), // Default 3000ms. After /clear, schedules an identity-reminder DM to each // recipient so a freshly-wiped worker re-anchors on its agentId and bus // attach state. Set 0 to opt out. Ignored for non-/clear commands. reminderMs: z.number().int().min(0).max(60_000).optional(), // Override the auto-generated reminder body if you want something specific. reminderText: z.string().optional(), // Block until the receiving pusher confirms it actually typed the command // into the pane (out-of-band receipt poll — zero added agent context). // Default true: a control command you can't confirm is the bug this fixes. // Set false for fire-and-forget. The wait is bounded by deliveryTimeoutMs. waitForDelivery: z.boolean().optional(), deliveryTimeoutMs: z.number().int().min(0).max(30_000).optional(), }; // A receipt as the pusher writes it. `submitted` is present only on control // receipts from a pusher new enough to VERIFY submission (v0.19.0+). // `scriptMtime` is the reporting pusher's build identity — the same // module-graph stamp the transport marker carries (newest mtime across the // pusher's entry file AND its hooks/ imports, per #28: the stale part is as // likely submit.mjs as the entrypoint). It is honesty, not security: a lying // pusher defeats it, exactly like report_transport. The value is that a // "confirmed" can be tied to the code that did the confirming. type Receipt = { id: string; ts: number; control?: boolean; submitted?: boolean; verified?: boolean; reason?: string; scriptMtime?: number; }; // Poll an agent's receipt log until a receipt for `msgId` appears or the // deadline passes. Returns the receipt, or null on timeout. File-only — no // agent context. // // A receipt proves the pusher TYPED the payload into the pane. For a control // command that is NOT proof it ran: the command can sit in the input behind an // autocomplete menu, delivered and inert. `submitted` is the field that // distinguishes them, and `deliveryOutcome` below is the only place allowed to // turn a receipt into a "confirmed". async function waitForReceipt(agentId: string, msgId: string, timeoutMs: number): Promise { const file = receiptFile(agentId); const deadline = Date.now() + timeoutMs; // First check is immediate; then poll on a short interval. for (;;) { const receipts = await readJsonl(file); const hit = receipts.find((r) => r.id === msgId); if (hit) return { ...hit, ts: hit.ts ?? Date.now() }; if (Date.now() >= deadline) return null; await new Promise((res) => setTimeout(res, 150)); } } // Turn a receipt (or its absence) into the delivery verdict for a CONTROL // command. Only `submitted === true` earns "confirmed" — anything else is // pending with a reason the caller can act on. Reporting an unverified // submission as confirmed is the defect this exists to remove: a check that // cannot fail loudly is worse than no check. // // `pusherSourceMtime` (the caller passes newestPusherSourceMtime()) lets a // CONFIRMED verdict carry a note when the reporting pusher's build identity // is behind the on-disk pusher source, or absent entirely. The note never // downgrades the verdict — the command demonstrably ran — it says whose // verification logic said so. Absence of the stamp reads as UNKNOWN, never // as fresh (same ruling as doctor's stale-pusher-script / provenance // checks: absence is not exemption), and the absence note is issued before // the on-disk comparison so an unstattable hooks dir cannot silence it. export function deliveryOutcome( agentId: string, receipt: Receipt | null, timeoutMs: number, pusherSourceMtime?: number, ): { delivery: "confirmed" | "pending"; at?: number; reason?: string; note?: string } { if (!receipt) { return { delivery: "pending", reason: `no delivery receipt from '${agentId}' within ${timeoutMs}ms — the command was written but may not have reached the pane (stale/wedged pusher). Run doctor or re-attach the agent.`, }; } if (receipt.submitted === true) { let note: string | undefined; if (receipt.scriptMtime === undefined) { note = `'${agentId}' confirmed the submission, but its pusher carries no build-identity stamp — the pusher predates receipt provenance and this confirmation cannot be tied to any known code; re-attach the agent (detach_agent + attach_agent) to upgrade it.`; } else if (pusherSourceMtime !== undefined && receipt.scriptMtime < pusherSourceMtime - 1) { const loaded = new Date(receipt.scriptMtime).toISOString(); const ondisk = new Date(pusherSourceMtime).toISOString(); note = `'${agentId}' confirmed the submission, but its pusher loaded its code at ${loaded} and the on-disk pusher source is newer (${ondisk}) — the verification logic behind this confirmation predates the current code; re-attach the agent (detach_agent + attach_agent) to upgrade it.`; } return { delivery: "confirmed", at: receipt.ts, ...(note ? { note } : {}) }; } if (receipt.submitted === false) { return { delivery: "pending", at: receipt.ts, reason: receipt.reason ?? `'${agentId}' pasted the command but could not confirm it was submitted — it may be sitting in the input.`, }; } // No `submitted` field: a pre-v0.19.0 pusher that stamps on paste. It may // well have worked — but it cannot tell us, and guessing "confirmed" is the // lie we are removing. Say what is actually known. return { delivery: "pending", at: receipt.ts, reason: `'${agentId}' typed the command into its pane, but its pusher predates submit verification and cannot confirm the command ran — re-attach the agent (detach_agent + attach_agent) to upgrade it.`, }; } function defaultReminderText(agentId: string): string { return ( `[agent-coord] context reset by /clear. ` + `Your bus identity is '${agentId}'. You remain registered and attached — call ` + `status({agentId:"${agentId}"}) to re-orient (role, transport, unread) and ` + `list_rooms() for the channels you're in. Any DM or channel post you receive ` + `next is your new task context.` ); } /* * REMINDERS ARE OBSERVABLE, because "scheduled" is not "delivered". * * `send_command` returns `reminderScheduled` the instant the timer is set, and * the delivery is an async lockfile-protected append whose only failure path * was a line on stderr. So a reminder could be LATE or LOST and the caller had * already been told it was handled — the same claim-without-a-completion-signal * shape as an empty check rollup reading green. * * MEASURED: `withLock` is configured `retries: 10, minTimeout: 20, * maxTimeout: 200`, so the append is PERMITTED well over a second of backoff. * Under sustained inbox contention the reminder lands at ~420ms. Any assertion * with a wall-clock budget below the lock's own policy is asserting something * the system never promised — which is what made tools.test.mjs:156 fail under * full-suite load and pass everywhere else. * * A longer sleep would convert that race into a slower race. This gives the * work a real completion signal instead, so callers and tests can await the * thing itself rather than guess at a duration. */ type PendingReminder = { done: Promise; timer: NodeJS.Timeout }; const pendingReminders = new Set(); /** Reminders whose delivery FAILED, kept so a drop is observable, not stderr-only. */ export const reminderFailures: { to: string; error: string; at: number }[] = []; /** * Resolves once every scheduled reminder has been delivered or has failed. * * Re-`ref`s the pending timers for the duration of the wait. The reminder timer * is deliberately `unref`d so it never holds the MCP server open on its own — * but that also means the event loop will not wait for it, and awaiting an * unref'd timer's promise hangs until the loop drains. A caller that has * explicitly asked to wait is stating the opposite intent, so the ref is * restored for exactly that window and dropped again afterwards. * * (The unref has a production consequence worth naming: if the server's * transport closes before the timer fires, the reminder is dropped. That is a * separate defect from the one this function fixes — reported, not silently * papered over.) */ export function remindersSettled(): Promise { const pending = [...pendingReminders]; for (const p of pending) p.timer.ref(); return Promise.all(pending.map((p) => p.done)).then(() => { for (const p of pending) p.timer.unref(); }); } function scheduleReminders( from: string, recipients: string[], delayMs: number, override: string | undefined, ): void { // Registered BEFORE the timer fires, so awaiting settlement covers the delay // as well as the write — otherwise a caller could observe "nothing pending" // during the window between scheduling and firing. let markDone: () => void = () => {}; const done = new Promise((resolve) => { markDone = resolve; }); let entry: PendingReminder; const t = setTimeout(async () => { try { for (const r of recipients) { try { const reminder: Message = { id: randomUUID(), ts: Date.now(), from, to: r, text: override ?? defaultReminderText(r), // A just-cleared agent is contextless until this lands — it must // push immediately, never queue behind the routine tier. urgent: true, }; await appendJsonl(inboxFile(r), reminder); } catch (e) { const error = (e as Error)?.message ?? String(e); // Recorded, not just printed: a dropped reminder leaves a just-cleared // agent contextless, and stderr is not somewhere anyone looks for that. reminderFailures.push({ to: r, error, at: Date.now() }); process.stderr.write(`[send_command] post-/clear reminder to '${r}' failed: ${error}\n`); } } } finally { pendingReminders.delete(entry); markDone(); } }, delayMs); entry = { done, timer: t }; pendingReminders.add(entry); // Don't keep the event loop alive solely for the reminder — the MCP server's // transport already holds it open as long as it's connected. if (typeof t.unref === "function") t.unref(); } // Inject a context-management slash command into a sub-agent's live tmux // session. Writes a control-flagged message the pushers deliver RAW (no banner, // no `[DM …]` prefix) so the receiving CLI runs it as a real slash command. // Hard-gated to tmux: refuses unless the target(s) have a live tmux-push(-remote) // transport, so a command never rots unexecuted in an offline inbox. export async function sendCommandTool(args: { from: string; to?: string; room?: string; command: string; reminderMs?: number; reminderText?: string; waitForDelivery?: boolean; deliveryTimeoutMs?: number; }) { const cmd = normalizeControlCommand(args.command); if (!cmd) { return { ok: false, error: `unsupported command '${args.command}'. Allowed: ${CONTROL_COMMANDS.map((c) => "/" + c).join(", ")}`, }; } if (!args.to && !args.room) { return { ok: false, error: "specify 'to' (a single agent) or 'room' (a channel's tmux-attached members)" }; } if (args.to && args.room) { return { ok: false, error: "specify only one of 'to' or 'room'" }; } const text = `/${cmd}`; const liveTmux = await liveTmuxTargets(); // DM: target must itself be tmux-attached. if (args.to) { const marker = liveTmux.get(args.to); // Phase 5.4 Task 4 — a herdr seat takes the control IN-PROCESS: no inbox message, no // pusher receipt; the transport types it, verifies it left the input, and answers. const herdrMarker = marker ? undefined : (await loadLiveTransports()).get(args.to); const activeT = activeTransport(); if (herdrMarker?.transport === HERDR) { if (activeT?.kind !== HERDR) { return { ok: false, error: `'${args.to}' is attached through herdr but this server's active transport is ${activeT?.kind ?? "none"} — a mixed fleet; the server configured for herdr must send this control` }; } const r = await activeT.sendControl(herdrMarker, cmd as "clear" | "compact" | "reload-skills"); const reminderMs = cmd === "clear" ? args.reminderMs ?? 3000 : 0; if (r.ok && reminderMs > 0) scheduleReminders(args.from, [args.to], reminderMs, args.reminderText); const extra = r as { enters?: number; note?: string }; return { ok: r.ok, command: text, delivered: r.ok ? [args.to] : [], transport: HERDR, delivery: r.ok ? "confirmed" : "refused", ...(r.error ? { error: r.error } : {}), ...(extra.enters !== undefined ? { enters: extra.enters } : {}), ...(extra.note ? { note: extra.note } : {}) }; } if (!marker) { return { ok: false, error: `'${args.to}' has no live tmux-push transport — control commands can only be injected into a tmux session. Attach it (join/attach_agent) or target an attached agent.`, }; } const msg: Message = { id: randomUUID(), ts: Date.now(), from: args.from, to: args.to, text, control: true, }; const target = inboxFile(args.to); await appendJsonl(target, msg); // Confirm the pusher actually typed it into the pane before we report success // (unless explicitly fire-and-forget). Out-of-band receipt poll — the // confirmation rides back in THIS tool result, costing no extra agent context. const wait = args.waitForDelivery ?? true; const deliveryTimeoutMs = args.deliveryTimeoutMs ?? 8000; const receipt = wait ? await waitForReceipt(args.to, msg.id, deliveryTimeoutMs) : null; const outcome = wait ? deliveryOutcome(args.to, receipt, deliveryTimeoutMs, newestPusherSourceMtime()) : null; const confirmed = outcome?.delivery === "confirmed"; // After /clear the receiver forgets its identity and that it's bus-attached // (the system prompt isn't re-applied because /clear isn't a session // start). Schedule a follow-up DM as a re-anchor; opt out with reminderMs:0. const reminderMs = cmd === "clear" ? args.reminderMs ?? 3000 : 0; if (reminderMs > 0) scheduleReminders(args.from, [args.to], reminderMs, args.reminderText); return { ok: true, id: msg.id, command: text, target, delivered: [args.to], transport: marker.transport, // delivery: confirmed = pusher typed it into the pane; pending = written but // unconfirmed within the timeout (stale/wedged pusher — run doctor). Absent // when waitForDelivery:false. ...(wait && outcome ? { delivery: outcome.delivery, confirmed, ...(outcome.at !== undefined ? { deliveredAt: outcome.at } : {}), // A confirmed delivery can still warn: the note names a reporting // pusher whose build identity is stale or absent. `confirmed` // stays true — the command ran; the warning is about who said so. ...(confirmed ? (outcome.note ? { warning: outcome.note } : {}) : { warning: outcome.reason }), } : {}), ...(reminderMs > 0 ? { reminderScheduled: { delayMs: reminderMs, recipients: [args.to] } } : {}), }; } // Room: broadcast to every tmux-attached member (never the sender itself). const chan = normalizeRoom(args.room); const rooms = await getRooms(); const members = rooms[chan]?.members ?? []; const delivered = members.filter((m) => m !== args.from && liveTmux.has(m)); if (delivered.length === 0) { return { ok: false, error: `no tmux-attached members in #${chan} to receive '${text}' (${members.length} member(s) total). Control commands only fire in a live tmux session.`, }; } const skipped = members.filter((m) => m !== args.from && !liveTmux.has(m)); const msg: Message = { id: randomUUID(), ts: Date.now(), from: args.from, room: chan, text, control: true, }; const target = roomFile(chan); await appendJsonl(target, msg); // Confirm each member's pusher typed it in (same msg.id lands in every // member's own receipt file). Poll all in parallel within one timeout. const wait = args.waitForDelivery ?? true; const deliveryTimeoutMs = args.deliveryTimeoutMs ?? 8000; let confirmed: string[] = []; let pending: string[] = []; let pendingReasons: string[] = []; let confirmNotes: string[] = []; if (wait) { const sourceMtime = newestPusherSourceMtime(); const results = await Promise.all( delivered.map(async (m) => ({ m, outcome: deliveryOutcome(m, await waitForReceipt(m, msg.id, deliveryTimeoutMs), deliveryTimeoutMs, sourceMtime), })), ); confirmed = results.filter((r) => r.outcome.delivery === "confirmed").map((r) => r.m); pending = results.filter((r) => r.outcome.delivery !== "confirmed").map((r) => r.m); pendingReasons = results .filter((r) => r.outcome.delivery !== "confirmed") .map((r) => `${r.m}: ${r.outcome.reason}`); confirmNotes = results .filter((r) => r.outcome.delivery === "confirmed" && r.outcome.note) .map((r) => r.outcome.note!); } // Same post-/clear re-anchor as the DM path — one reminder per delivered // member, in their own inbox, with their own agentId in the body. const reminderMs = cmd === "clear" ? args.reminderMs ?? 3000 : 0; if (reminderMs > 0) scheduleReminders(args.from, delivered, reminderMs, args.reminderText); return { ok: true, id: msg.id, command: text, target, room: chan, delivered, skipped: skipped.length ? skipped : undefined, ...(wait ? { delivery: pending.length === 0 ? "confirmed" : "partial", confirmed, ...(pending.length ? { pending, warning: `not confirmed as submitted within ${deliveryTimeoutMs}ms — ${pendingReasons.join(" | ")}`, } : {}), // Confirmed members whose reporting pusher is stale or unstamped — // the confirmations stand, the notes say whose code issued them. ...(confirmNotes.length ? { notes: confirmNotes } : {}), } : {}), ...(reminderMs > 0 ? { reminderScheduled: { delayMs: reminderMs, recipients: delivered } } : {}), }; } // Does `pid` actually belong to one of our tmux pushers? A transport marker // records a pid, but a marker can outlive its process and pids get recycled — // so "pid is alive" is NOT evidence the pid is still the pusher. Anything that // SIGTERMs a marker's pid must confirm identity first or it will eventually // kill an unrelated process on the user's machine. // // Pushers are spawned as ` <.../hooks/tmux-pusher.mjs>` (see // attachAgentTool), so the script path in the process's argv is the signature. // Returns false when we cannot confirm — including when `ps` is unavailable. // Refusing to kill an unverifiable pid is the safe failure: a wedged pusher // that survives is a nuisance, a wrong SIGTERM is not. export function isPusherProcess(pid: number): boolean { if (!Number.isInteger(pid) || pid <= 0) return false; const ps = spawnSync("ps", ["-o", "command=", "-p", String(pid)], { encoding: "utf8" }); if (ps.status !== 0) return false; // pid gone, or no usable ps return (ps.stdout ?? "").includes(path.basename(resolvePusherPath())); } // ---------- attach_agent / detach_agent (tmux push transport) ---------- export const attachAgentSchema = { agentId: z.string().min(1), tmuxTarget: z.string().optional(), includeRoom: z.boolean().optional(), allowlist: z.array(z.string()).optional(), debounceMs: z.number().int().positive().max(60_000).optional(), }; export async function attachAgentTool(args: { agentId: string; tmuxTarget?: string; includeRoom?: boolean; allowlist?: string[]; debounceMs?: number; }) { // Resolve target: explicit arg > MCP server's own TMUX_PANE env. // Phase 5.4 Task 4 — a fleet configured for herdr attaches through the transport: no // pusher is spawned, the marker carries pid 0 and a herdr pane id, and every refusal // names herdr (an absent binary is never a silent fall-through to tmux). const activeT = activeTransport(); if (activeT?.kind === HERDR) { const existing = await readJson(transportFile(args.agentId), null); if (existing) await deleteFile(transportFile(args.agentId)); let marker: TransportMarker; try { marker = await activeT.attach({ agentId: args.agentId, target: args.tmuxTarget, includeRoom: args.includeRoom, allowlist: args.allowlist, debounceMs: args.debounceMs }); } catch (e) { return { ok: false, error: (e as Error).message }; } marker = { ...marker, serverBuildMtime: SERVER_BUILD_MTIME }; await fsp.mkdir(path.dirname(transportFile(args.agentId)), { recursive: true }); await updateJson(transportFile(args.agentId), marker, () => marker); return { ok: true, agentId: args.agentId, transport: HERDR, target: marker.target, pid: marker.pid, pidWhy: marker.pidWhy, rooms: marker.rooms, note: "herdr socket transport: no pusher process — delivery is made in-process by this server through herdr's socket API; liveness is herdr's own pane status", }; } // ⟨q-ec020f6a⟩ slice C — local tmux-push DELETED outright: 0 of 13 live seats used it, // and keeping it meant a second delivery mechanism (hooks/tmux-pusher.mjs, now removed) // plus a role-card precondition that halted every live herdr seat on a false requirement. // attach_agent handles exactly one kind now. A remote seat never came through here in // the first place — it registers via report_transport (scripts/coord-pusher.mjs) — so // this refuses rather than silently trying (and failing) to spawn a script that no // longer exists. return { ok: false, error: `attach_agent only attaches the herdr transport now (this fleet's active transport is ` + `${activeT ? `'${activeT.kind}'` : "unconfigured"}) — local tmux-push was removed (⟨q-ec020f6a⟩). ` + `A remote seat registers via report_transport, not attach_agent.`, }; } export const detachAgentSchema = { agentId: z.string().min(1), }; export async function detachAgentTool(args: { agentId: string }) { const marker = await readJson(transportFile(args.agentId), null); let killed = false; let unverified = false; if (marker?.transport === HERDR) { // No pusher to kill: the socket transport has no process of its own. The marker is // the whole attachment, and removing it is the detach. await deleteFile(transportFile(args.agentId)); return { ok: true, agentId: args.agentId, killed: false, hadMarker: true, transport: HERDR, note: "herdr socket transport: no pusher process to stop; marker removed" }; } if (marker && isPidAlive(marker.pid)) { // ALIVE IS NOT ENOUGH — VERIFY IT IS A PUSHER BEFORE SIGNALLING IT. // // A pid is only meaningful while the process that owned it is running: pids // are recycled, and a stale marker pointing at a recycled pid makes this a // SIGTERM at an unrelated process. Doctor's reaper already gates its kill on // `isPusherProcess` for exactly this reason (see the wedged-pusher fix); this // path never took the correction, and `unregister`/`quit` inherit it. // // UNVERIFIABLE MEANS CLEAR THE MARKER, NEVER SIGNAL. The marker is ours to // delete and the process is not ours to kill — those are different // authorities, and conflating them is how a detach becomes someone else's // outage. Reported rather than silent, so a pusher that genuinely needs // killing is not left running unnoticed. if (isPusherProcess(marker.pid)) { try { process.kill(marker.pid, "SIGTERM"); killed = true; } catch { // already gone } } else { unverified = true; } } await deleteFile(transportFile(args.agentId)); await deleteFile(pidFile(args.agentId, "pusher")); return { ok: true, agentId: args.agentId, killed, hadMarker: marker !== null, ...(unverified ? { warning: `pid ${marker!.pid} is alive but is NOT a pusher process — the marker was cleared and NOTHING was signalled. ` + `A stale marker pointing at a recycled pid would otherwise make this a SIGTERM at an unrelated process. ` + `If a pusher is still running for '${args.agentId}', find it with \`doctor\` and let its reaper handle it.`, } : {}), }; } function resolvePusherPath(): string { // transport.js (compiled) lives in dist/tools/; pusher lives in hooks/ at repo root. const here = path.dirname(fileURLToPath(import.meta.url)); return path.resolve(here, "..", "..", "hooks", "tmux-pusher.mjs"); } /** An epoch-ms stamp as ISO-8601 UTC, or an explicit absence — never a blank. */ function stamp(ms: number | undefined): string { return ms === undefined ? "(no stamp)" : new Date(ms).toISOString(); } // Freshness basis for the stale-pusher-script mechanism: the newest mtime // across the pusher's source dir (hooks/*.mjs), NOT just the entry file. The // pusher imports submit.mjs / tier.mjs / roles.mjs, so a fix touching only an // import (the #21/#25 control-submit fixes did) leaves tmux-pusher.mjs's own // mtime unchanged — a single-file stamp/compare reports ok on a pusher running // exactly the code the fix replaced. Used by both the attach-time stamp and // doctor's on-disk comparison so the two sides can never drift apart. // AGENT_COORD_HOOKS_DIR is a test seam only: it redirects what freshness // MEASURES (against a temp copy of hooks/) so tests never touch the mtimes of // real sources shared with live pushers — it never changes what attach SPAWNS. function newestPusherSourceMtime(): number | undefined { const dir = process.env.AGENT_COORD_HOOKS_DIR ?? path.dirname(resolvePusherPath()); return newestMtimeUnder(dir, [".mjs"]); } // ---------- status / whoami ---------- export const statusSchema = { agentId: z.string().min(1) }; export async function statusTool(args: { agentId: string }) { const reg = await readJson(AGENTS_FILE, {}); const entry = reg[args.agentId]; const transports = await loadLiveTransports(); const transport = transports.get(args.agentId); const inbox = await readJsonl(inboxFile(args.agentId)); const cursor = await readJson(cursorFile(args.agentId), {}); const inboxOffset = cursor.inboxOffset ?? 0; const unread = Math.max(0, inbox.length - inboxOffset); // WHAT THIS PROCESS CAN ACTUALLY DO, embedded WHOLE. // // A new tool otherwise lands silently: an agent reading an old listing // asserts the tool does not exist, and it is right about its own listing and // wrong about the world. Three wrong conclusions in two days came from // exactly that, each about which build a pane was running. // // The full report travels, not a summary. Flattening it to a boolean would // hand the caller a tidier answer and a worse one: `versionLabel` stays // CONTEXT rather than evidence, a THROWN probe stays an absent capability // rather than a broken check, and `answeredBy` keeps naming which pid // answered — two servers disagreeing mid-rollout is the normal case, and it // is the case every other instrument here reports as if the fleet were one // thing. const capabilities = await capabilitiesTool(); // ⟨q-8a3f1c05⟩ — THIS SEAT'S BUILD STATE, both halves: the pusher from its marker's // pid via ps (keyed by --agent), the server from THIS process — status is called // by the seat about itself, which is the one case the server half is knowable. const id = resolveServerIdentity(); const installed = installedFrom(id.path); const build = seatBuildOf({ agentId: args.agentId, marker: transport, installed, ps: psReader, server: { pid: process.pid, startedAt: Date.now() - Math.round(process.uptime() * 1000), buildMtime: installed.buildMtime }, }); return { agentId: args.agentId, build, registered: !!entry, entry, attached: !!transport, transport, inboxDepth: inbox.length, inboxUnread: unread, inTmux: !!process.env.TMUX_PANE, tmuxPane: process.env.TMUX_PANE, capabilities, }; } // ---------- join (combo: register + auto-attach + read inbox) ---------- const joinAttachOptionsSchema = z.object({ tmuxTarget: z.string().optional(), includeRoom: z.boolean().optional(), allowlist: z.array(z.string()).optional(), debounceMs: z.number().int().positive().max(60_000).optional(), }); export const joinSchema = { agentId: z.string().min(1), project: z.string().optional(), // Free text, or a declared identity ({roleId, displayName}) — see // roleInputSchema. A frozen roleId cannot be changed by re-joining. role: roleInputSchema.optional(), // attach: undefined → auto-attach if $TMUX_PANE is set; true → always try; // false → never; object → attach with overrides. attach: z.union([z.boolean(), joinAttachOptionsSchema]).optional(), readInbox: z.boolean().optional(), // First-claim guard overrides (server.ts guardFirstClaim): claiming an id // that is PROVABLY LIVE on the bus refuses unless the call presents that // agent's token (tokens.json / coord-token) — `force` alone no longer // overrides a provably live incumbent (q-314e0187). `force` still bypasses // the guard when liveness cannot be verified, or the id is verified absent. // Both ignored once bound. token: z.string().optional(), force: z.boolean().optional(), // Prose-only exemption from the typed-record rule — see registerSchema. // Declared here too because `join` is the call every card actually makes. proseOnly: z.union([z.boolean(), z.object({ reason: z.string().min(1) })]).optional(), }; export async function joinTool(args: { agentId: string; project?: string; role?: RoleArg; proseOnly?: boolean | { reason: string }; attach?: boolean | { tmuxTarget?: string; includeRoom?: boolean; allowlist?: string[]; debounceMs?: number }; readInbox?: boolean; }) { const reg = await registerTool({ agentId: args.agentId, project: args.project, role: args.role, proseOnly: args.proseOnly, }); // A refused role update (frozen roleId) fails the whole join rather than // silently attaching a transport under the wrong identity. if (!reg.ok) return reg; // Decide attach behavior. const wantAttach = args.attach === false ? false : args.attach === true || typeof args.attach === "object" ? true : !!process.env.TMUX_PANE; // undefined → auto-detect // Always present as object | null so callers can branch on a single key // instead of "did I pass attach?" — per agent-pa's API review. let attach: Awaited> | null = null; if (wantAttach) { const opts = typeof args.attach === "object" ? args.attach : {}; attach = await attachAgentTool({ agentId: args.agentId, ...opts }); } const readInbox = args.readInbox ?? true; let inbox: Awaited> | null = null; if (readInbox) { inbox = await readMessagesTool({ agentId: args.agentId, source: "inbox" }); } // Surface the default channel's topic + MOTD (room rules) in the same // round-trip, so a connecting agent sees them without a separate call. const rooms = await getRooms(); const def = rooms[DEFAULT_ROOM]; return { ok: true, registered: reg.agent, attached: !!attach && attach.ok !== false, attach, inbox, defaultRoom: { room: DEFAULT_ROOM, topic: def?.topic, motd: def?.motd }, inTmux: !!process.env.TMUX_PANE, }; } // ---------- transport markers (for remote pushers) ---------- export const reportTransportSchema = { agentId: z.string().min(1), transport: z.string().min(1), tmuxTarget: z.string().optional(), host: z.string().optional(), since: z.number().optional(), // mtime of the script the remote daemon loaded into memory at spawn time // (epoch ms). Lets doctor() flag the remote pusher as stale if its on-disk // counterpart has been upgraded since. The remote pusher passes // `(await fsp.stat(__filename)).mtimeMs`; absent → doctor skips the check. scriptMtime: z.number().optional(), // Whether this pusher carries room traffic as well as DMs. Absent → the // marker cannot say, and every reader reports UNKNOWN rather than assuming // a full transport (the worker-2 shape). rooms: z.boolean().optional(), }; // Called by an external push daemon (typically scripts/coord-pusher.mjs on a // remote machine) to publish a transport marker so list_agents reflects the // attachment. The local tmux-push path writes the marker directly inside // attach_agent; this is the wire-callable equivalent for remote pushers. export async function reportTransportTool(args: { agentId: string; transport: string; tmuxTarget?: string; host?: string; since?: number; scriptMtime?: number; rooms?: boolean; }) { const marker: TransportMarker = { agentId: args.agentId, transport: args.transport, pid: 0, // not meaningful for remote; liveness comes from heartbeat tmuxTarget: args.tmuxTarget, host: args.host, since: args.since ?? Date.now(), scriptMtime: args.scriptMtime, rooms: args.rooms, }; await updateJson(transportFile(args.agentId), marker, () => marker); return { ok: true, marker }; } export const clearTransportSchema = { agentId: z.string().min(1), }; // Idempotent remote-counterpart to detach_agent: just deletes the marker. Used // by the remote pusher on graceful shutdown so list_agents stops showing it // attached. (Does NOT try to kill any process — there's nothing local to kill.) export async function clearTransportTool(args: { agentId: string }) { const removed = await deleteFile(transportFile(args.agentId)); return { ok: true, removed }; } export const reportReceiptSchema = { agentId: z.string().min(1), id: z.string().min(1), from: z.string().optional(), control: z.boolean().optional(), submitted: z.boolean().optional(), verified: z.boolean().optional(), reason: z.string().optional(), // Build identity of the reporting pusher: newest mtime across its loaded // module graph (entry file + hooks/ imports), sampled once at its startup — // the same basis report_transport's scriptMtime uses. Absent → the receipt's // provenance is UNKNOWN and deliveryOutcome says so; the server never // defaults it (a default here would be assume-fresh, the twin of the // assume-success `submitted` refuses to invent). scriptMtime: z.number().optional(), }; // Wire-callable counterpart to the local pusher's receipt stamp (writeReceipts // in hooks/tmux-pusher.mjs). A remote pusher types into a pane on ANOTHER // machine and cannot append to this host's receipts/.jsonl, so before this // existed a control command to a tmux-push-remote agent was never confirmable: // send_command waited out deliveryTimeoutMs and reported delivery:"pending" // even when the command demonstrably ran. // // The receipt is appended in the exact shape the local pusher writes, so // waitForReceipt/deliveryOutcome need no remote-specific branch. `submitted` // is recorded only when the caller reports it — absence means "typed but // unverified", which deliveryOutcome refuses to call confirmed. The server // cannot see the remote pane, so it stores what the pusher observed and // nothing more; defaulting the field here would recreate assume-success one // layer up. Trust matches report_transport: the identity gate binds agentId // to the session, so a pusher can only stamp its own agent's receipt file. export async function reportReceiptTool(args: { agentId: string; id: string; from?: string; control?: boolean; submitted?: boolean; verified?: boolean; reason?: string; scriptMtime?: number; }) { const receipt: Receipt & { agentId: string; from?: string } = { id: args.id, agentId: args.agentId, ts: Date.now(), ...(args.from !== undefined ? { from: args.from } : {}), control: args.control === true, ...(args.submitted !== undefined ? { submitted: args.submitted } : {}), ...(args.verified !== undefined ? { verified: args.verified } : {}), ...(args.reason !== undefined ? { reason: args.reason } : {}), ...(args.scriptMtime !== undefined ? { scriptMtime: args.scriptMtime } : {}), }; await appendJsonl(receiptFile(args.agentId), receipt); return { ok: true, receipt }; } // ---------- doctor (bus-wide health check) ---------- type DoctorLevel = "ok" | "warn" | "error"; type DoctorFinding = { check: string; level: DoctorLevel; detail: string; fixable: boolean; items?: string[]; }; // Count non-empty lines vs successfully-parsed entries in a JSONL file. // Offsets index the PARSED entries (see readJsonl), so `parsed` is the figure // cursor math is compared against; `malformed` is the desync risk. async function scanJsonl(file: string): Promise<{ lines: number; parsed: number; malformed: number }> { if (!existsSync(file)) return { lines: 0, parsed: 0, malformed: 0 }; const raw = await fsp.readFile(file, "utf8"); let lines = 0; let parsed = 0; for (const line of raw.split("\n")) { if (!line.trim()) continue; lines++; try { JSON.parse(line); parsed++; } catch { // malformed } } return { lines, parsed, malformed: lines - parsed }; } // Find leftover proper-lockfile lock dirs (`.lock`) across the state // dirs. Anything older than the threshold is almost certainly orphaned by a // crashed writer (withLock's stale window is 5s). async function scanStaleLocks(olderThanMs: number, now: number): Promise<{ path: string; ageMs: number }[]> { const out: { path: string; ageMs: number }[] = []; const dirs = [ROOT, INBOX_DIR, CURSOR_DIR, ROOMS_DIR, TRANSPORT_DIR]; for (const dir of dirs) { if (!existsSync(dir)) continue; let names: string[]; try { names = await fsp.readdir(dir); } catch { continue; } for (const name of names) { if (!name.endsWith(".lock")) continue; const p = path.join(dir, name); try { const st = await fsp.stat(p); const ageMs = now - st.mtimeMs; if (ageMs > olderThanMs) out.push({ path: p, ageMs }); } catch { // vanished mid-scan } } } return out; } /* ── wiring the seam's TMUX HOST (Phase 5.4 Task 3) ─────────────────────────── */ /** * The process-layer half of `TmuxTransport`, supplied by the module that owns * pusher spawn, receipts and marker files. * * ⚠ SCOPE, STATED RATHER THAN IMPLIED: Task 3 puts the seam in the IDENTITY and * DIAGNOSIS path — `capabilities` asks the live transport what it is, and the * mixed-fleet check reads markers through it. DELIVERY STILL RUNS THROUGH THE * EXISTING CODE PATHS. Task 2's gate was that nothing changes, and rerouting * every push through a new object would change the thing most likely to break * quietly. So the four delivery methods below THROW rather than no-op: a host * that silently accepted a push and dropped it would be the one failure this * fleet cannot observe, and an explicit throw is reachable only from code that * has not been written yet. */ const TMUX_HOST: TmuxHost = { attach: async () => { throw new Error("TmuxTransport.attach is not the delivery path yet — call attachAgentTool (Phase 5.4 Task 3 wires identity only)"); }, detach: async () => { throw new Error("TmuxTransport.detach is not the delivery path yet — call detachAgentTool"); }, push: async () => { throw new Error("TmuxTransport.push is not the delivery path yet — delivery runs through the pusher process"); }, sendControl: async () => { throw new Error("TmuxTransport.sendControl is not the delivery path yet — call sendCommandTool"); }, /** * Is the pusher behind this marker still running? Reuses `isPusherProcess`, * which checks the COMMAND of the pid rather than merely that a pid exists — * a recycled pid belonging to something else is not a live pusher. */ pusherAlive: (marker) => isPusherProcess(marker.pid), killPusher: (marker) => { try { process.kill(marker.pid, "SIGTERM"); return true; } catch { return false; } }, }; registerTmuxHost(TMUX_HOST); export const doctorSchema = { fix: z.boolean().optional(), maxFileBytes: z.number().int().positive().optional(), }; export async function doctorTool(args: { fix?: boolean; maxFileBytes?: number }) { const fix = args.fix ?? false; const maxBytes = args.maxFileBytes ?? 5 * 1024 * 1024; const now = Date.now(); const findings: DoctorFinding[] = []; const fixed: string[] = []; const reg = await readJson(AGENTS_FILE, {}); const known = new Set(Object.keys(reg)); const rooms = await getRooms(); const channels = Object.keys(rooms); // 1. Orphan transport markers (dead local pid, or stale remote heartbeat). { const dead: string[] = []; for (const fname of await listTransportFiles()) { const file = path.join(TRANSPORT_DIR, fname); const marker = await readJson(file, null); if (!marker || !isMarkerLive(marker, reg, now)) { dead.push(file); if (fix) { await deleteFile(file); fixed.push(`deleted stale transport marker ${fname}`); } } } findings.push({ check: "orphan-transport-markers", level: dead.length ? "warn" : "ok", detail: dead.length ? `${dead.length} stale transport marker(s) (dead pid or expired remote heartbeat)` : "no stale transport markers", fixable: true, items: dead.length ? dead.map((f) => path.basename(f)) : undefined, }); } // 1b. Stale pusher daemons — a long-running pusher loaded its script into // memory at spawn time, so when the on-disk script is later upgraded // (npm i -g a new version), the still-running pid is on the OLD code. // Pre-v0.8.2 pushers had no `control:true` awareness and silently // dropped /clear /compact at the slash-guard — ack:true with no // keystrokes ever reaching the pane. Comparing the marker's stamped // scriptMtime against the newest on-disk mtime across hooks/*.mjs // catches it — the whole module graph, not just the entry file, because // the control-submit logic lives in submit.mjs and a fix landing there // alone leaves tmux-pusher.mjs's mtime (and a single-file stamp) intact. // Local tmux-push only — for tmux-push-remote the script lives on a // different host so we can't stat it from here. { const localPusherMtime = newestPusherSourceMtime(); const stale: string[] = []; const unverifiable: string[] = []; for (const fname of await listTransportFiles()) { const file = path.join(TRANSPORT_DIR, fname); const marker = await readJson(file, null); if (!marker || !isMarkerLive(marker, reg, now)) continue; if (!isLocallyProbeable(marker.transport)) continue; // remote = can't verify (documented limit: can't stat another host) if (marker.scriptMtime === undefined) { // ABSENCE IS NOT EXEMPTION. The field's own writer once dropped it, // and the silent skip here meant the check was disabled by the very // thing it monitors — a live pusher we cannot verify is a warn, not // an ok (credit agent-coordination-david-dev). Same flip, same // commit, as serverBuildMtime below: the two checks must never // disagree about what absence means. unverifiable.push(`${marker.agentId} (pid ${marker.pid}, no scriptMtime stamp — cannot verify; detach_agent + attach_agent to re-stamp)`); continue; } if (localPusherMtime === undefined) continue; if (marker.scriptMtime < localPusherMtime - 1) { // -1ms slack for fs mtime rounding const loaded = new Date(marker.scriptMtime).toISOString(); const ondisk = new Date(localPusherMtime).toISOString(); stale.push(`${marker.agentId} (pid ${marker.pid}, loaded ${loaded}, on-disk now ${ondisk})`); } } const bad = [...stale, ...unverifiable]; findings.push({ check: "stale-pusher-script", level: bad.length ? "warn" : "ok", detail: bad.length ? `${stale.length} attached pusher(s) running pre-upgrade code and ${unverifiable.length} whose freshness cannot be verified (no stamp) — control commands (/clear, /compact) may be silently dropped. Run detach_agent + attach_agent for each, or have the agent relaunch.` : "all attached pushers are running the current on-disk script", fixable: false, items: bad.length ? bad : undefined, }); } // 1b². The same staleness class one layer up: doctor itself runs inside an // MCP server process that imported dist/ at startup. `npm run build` // rewrites dist/ under the still-running server, which then keeps // spawning pushers and stamping markers with logic the rebuild // replaced — merging is not deploying, and until the session restarts // no on-disk artifact reflects what this process will actually do. // Self-scoped by construction: each session's doctor reports on the // server it is running in, which is the only process whose loaded // build it can truthfully know. { const onDisk = onDiskBuildMtime(); const drifted = SERVER_BUILD_MTIME !== undefined && onDisk !== undefined && SERVER_BUILD_MTIME < onDisk - 1; const identity = `${SERVER_BUILD_SHA ?? "unknown-sha"} @ ${BUILD_DIR}`; findings.push({ check: "server-build-drift", level: drifted ? "warn" : "ok", detail: drifted ? `this MCP server loaded its build at ${new Date(SERVER_BUILD_MTIME!).toISOString()} but the on-disk build is newer (${new Date(onDisk!).toISOString()}) — the session is running pre-rebuild code and everything it stamps or spawns uses replaced logic. Restart this agent's session. (${identity})` : `server is running the current on-disk build (${identity})`, fixable: false, }); } // 1b³. WHERE A NEW GLOBAL INSTALL WOULD LAND, which is a different question // from which copy is running (1b² above) and neither substitutes. On // this box `npm prefix -g` and the fleet's actual load path pointed at // two different nvm versions, so an install that printed success would // have updated a copy nothing loads. { const loadPrefix = prefixOf(BUILD_DIR); let installPrefix: string | null = null; let npmPath: string | undefined; try { installPrefix = execFileSync("npm", ["prefix", "-g"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim() || null; npmPath = execFileSync("which", ["npm"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim() || undefined; } catch { installPrefix = null; } const v = prefixVerdict(loadPrefix, installPrefix, npmPath); findings.push({ check: "install-prefix", level: v.level, detail: v.detail, fixable: false }); } // 1b²ᵇ. The affirmative catch for merged-but-never-rebuilt: src/ newer than // the compiled build means no restart can help — the artifact every // future session will load is already behind the code. Distinct from // 1b² (a process behind its dist); this is the DISK being behind // itself, which is why it can fire on a bus with zero live sessions. // Not inferred from marker state: both sides are statted directly. { const srcMtime = onDiskSourceMtime(); const distMtime = onDiskBuildMtime(); const behind = srcMtime !== undefined && distMtime !== undefined && distMtime < srcMtime - 1; findings.push({ check: "dist-behind-source", level: behind ? "warn" : "ok", detail: behind ? `src/ is newer than the compiled build (src ${new Date(srcMtime!).toISOString()}, dist ${new Date(distMtime!).toISOString()}) — the checkout was updated but never rebuilt, so every session (current and future) runs pre-update code. \`npm run build\`, then restart sessions.` : srcMtime === undefined ? "no src/ to compare (packaged install) — dist is the only artifact" : "compiled build is at least as new as src/", fixable: false, }); } // 1b³. Marker provenance — which server BUILD stamped each marker. A live // pusher can be perfectly fresh while the marker's stamps were // computed by an outdated server (observed live 2026-07-29: a stale // server's attach stamped single-file freshness that agreed with the // new on-disk check only because tmux-pusher.mjs happened to be the // newest hooks file). Local tmux-push only, same as 1b. { const onDisk = onDiskBuildMtime(); const unverifiable: string[] = []; for (const fname of await listTransportFiles()) { const file = path.join(TRANSPORT_DIR, fname); const marker = await readJson(file, null); if (!marker || !isMarkerLive(marker, reg, now)) continue; if (!isLocallyProbeable(marker.transport)) continue; // remote = can't verify (documented limit: can't stat another host) if (marker.serverBuildMtime === undefined) { // ABSENCE IS NOT EXEMPTION — flipped in the same commit as the // scriptMtime absence above, so the two checks can never disagree // about what a missing stamp means. An unstamped marker was written // by a pre-provenance server (or by hand): precisely the population // this check exists to police, and the one it must not exempt. // Transition is deliberately correct-and-loud: after the upgrade, // every pre-existing marker warns at once, each cleared by a session // restart + re-attach — the burst is also the only external view of // which sessions still run pre-provenance servers, since a stale // server cannot self-report (see 1b²). unverifiable.push(`${marker.agentId} (no serverBuildMtime stamp — stamped by a pre-provenance server; restart that session, then detach_agent + attach_agent)`); continue; } if (onDisk === undefined) continue; if (marker.serverBuildMtime < onDisk - 1) { // A COMPONENT CAN REPORT ITS OWN FRESHNESS; A SIBLING'S RECORD OF IT // GOES STALE SILENTLY. That asymmetry is the whole correction here. // // `serverBuildMtime` is stamped inside `attach_agent` — by the PUSHER's // attach, once — so it records what the server was AT THE LAST ATTACH. // A server-only restart (David's `/mcp`) reloads the server and touches // no marker, so this stamp cannot be refreshed by the thing it // describes and CANNOT DISTINGUISH a restarted server from an // un-restarted one. // // Measured on three agents: each answered `list_subscriptions` — a tool // that does not exist before 0.26.7 — while its marker still stamped a // pre-0.26.7 build. The stamp was 2.2 hours behind the installed dist // on a server demonstrably running the new code. // // So this is reported as what it IS: the marker predates the build, and // the server's actual state is UNKNOWN from here. Calling it an // outdated server is a claim this record cannot support, and it priced // a restart decision on a count where at least three entries had // already restarted. // // The pusher half is untouched and remains valid: `scriptMtime` is // self-reported by the process it describes. const stamped = new Date(marker.serverBuildMtime).toISOString(); const current = new Date(onDisk).toISOString(); unverifiable.push( `${marker.agentId} (marker stamped at the last ATTACH from server build ${stamped}, on-disk build ${current} — this says the MARKER is behind, NOT that the server is: a server-only restart refreshes no marker. Exercise the server to settle it, or detach_agent + attach_agent to refresh the stamp)`, ); } } // `outdated` is gone, not emptied: nothing can populate it any more, and a // bucket that always reports 0 is a claim the reader still has to discount. const bad = unverifiable; findings.push({ check: "marker-server-provenance", level: bad.length ? "warn" : "ok", detail: bad.length ? `${bad.length} transport marker(s) cannot report their server's build: the stamp is written by attach_agent, so it records the server AT THE LAST ATTACH and a server-only restart refreshes nothing. This says the MARKER is behind or absent — NOT that the server is stale. Exercise the server to settle it (a tool that exists only in the newer build), or detach_agent + attach_agent to refresh the stamp. Restart the session too if the server itself is old.` : "every local transport marker was stamped at an attach against the current build — which says the markers are current, not that every server is", fixable: false, items: bad.length ? bad : undefined, }); } // 1b³. THE SPLIT: one pane, two components, ONE version number. // // The two stamps above are each checked, and each in its own finding — which // means the state that actually bit us has no name. `detach_agent` + // `attach_agent` respawns the PUSHER and reloads its script; **it does not // restart the MCP SERVER.** So a pane can run a current pusher against a // two-day-old server, and both existing checks are individually correct while // nothing says "this agent is split". Measured across this machine: all three // panes ran a 2026-08-27 pusher hook against a 2026-08-25 server build. // // WHY IT IS A DISPLAY TASK AND NOT A DESIGN ONE: both facts were already // written into every marker. Nothing needed adding; the pair needed reading // together, and reading them apart is what let "we restarted the transports" // be heard as "we restarted the fleet". // // The classification is the deliverable, because the REMEDIES differ: SPLIT // needs a session restart (re-attach will not touch the server), fully-stale // needs both, fresh needs nothing. Reporting two independent warnings leaves // the reader to infer which — and the inference that was actually made was the // optimistic one. // // Every timestamp is ISO-8601 UTC (`Z`), so no reading depends on the reader's // timezone. { const onDiskScript = newestPusherSourceMtime(); const onDiskServer = onDiskBuildMtime(); const split: string[] = []; const bothStale: string[] = []; const unknown: string[] = []; let fresh = 0; for (const fname of await listTransportFiles()) { const file = path.join(TRANSPORT_DIR, fname); const marker = await readJson(file, null); if (!marker || !isMarkerLive(marker, reg, now)) continue; if (!isLocallyProbeable(marker.transport)) continue; // remote: the script lives on another host const { scriptMtime, serverBuildMtime, agentId } = marker; if (scriptMtime === undefined || serverBuildMtime === undefined || onDiskScript === undefined || onDiskServer === undefined) { // A pane missing either stamp cannot be classified. Saying so is the // point: "unclassifiable" and "fresh" are different answers, and the // whole finding is about not collapsing them. unknown.push(`${agentId} (pusher ${stamp(scriptMtime)} · server ${stamp(serverBuildMtime)} — one stamp missing, cannot classify)`); continue; } const scriptStale = scriptMtime < onDiskScript - 1; const serverStale = serverBuildMtime < onDiskServer - 1; const pair = `${agentId} (pusher ${stamp(scriptMtime)} · server ${stamp(serverBuildMtime)})`; if (scriptStale && serverStale) bothStale.push(pair); else if (scriptStale !== serverStale) { split.push(`${pair} — ${scriptStale ? "pusher stale, server current" : "pusher CURRENT, server STALE"}`); } else fresh++; } const bad = [...split, ...bothStale, ...unknown]; findings.push({ check: "transport-build-split", level: split.length || bothStale.length ? "warn" : unknown.length ? "warn" : "ok", detail: bad.length ? `${split.length} pane(s) SPLIT (the two components are at different builds under one version number — a re-attach reloads the pusher and NOT the server, so this needs a SESSION restart), ` + `${bothStale.length} fully stale, ${unknown.length} unclassifiable, ${fresh} fully current. On-disk now: pusher ${stamp(onDiskScript)} · server ${stamp(onDiskServer)}. Times are ISO-8601 UTC.` : `${fresh} local pane(s) fully current — pusher and server both at the on-disk build (ISO-8601 UTC)`, fixable: false, items: bad.length ? bad : undefined, }); } // 1c. Wedged local pushers (pid-alive, pane-dead). v0.8.0 made pushers // self-exit when their own tmux-target probe finds the pane gone, but // that only fires from inside the pusher's own poll loop — if the pane // is killed in a way that loop never observes (or the loop itself is // wedged), the pid stays alive, isMarkerLive's pid-alive check keeps // treating it as live, and list_agents reports it "live" while nothing // can actually be delivered. Local tmux-push only — a tmux-push-remote // marker's pane lives on a different host, unprobeable from here. { // Without a tmux binary we can't tell "wedged" from "can't probe" — skip // rather than flag every local marker as dead. const tmuxIsAvailable = tmuxAvailable(); const wedged: { agentId: string; pid: number; file: string; target: string; isPusher: boolean }[] = []; if (tmuxIsAvailable) { for (const fname of await listTransportFiles()) { const file = path.join(TRANSPORT_DIR, fname); const marker = await readJson(file, null); if (!marker || !isMarkerLive(marker, reg, now)) continue; if (!isLocallyProbeable(marker.transport)) continue; // remote = no local pane to probe if (!marker.tmuxTarget) continue; // no target recorded, can't probe // has-session actually validates the target and fails on a dead // pane/session; `display-message -p -t ` does NOT // (tmux 3.6b exits 0 for any target, even a just-killed one, when // the format string has no #{...} needing that target resolved). const probe = { status: paneExists(targetOf(marker)!) ? 0 : 1 }; if (probe.status === 0) continue; // pane alive // The marker's pid being alive does not make it OUR pid — see // isPusherProcess. Record the verdict now so `fix` only ever signals // a confirmed pusher. wedged.push({ agentId: marker.agentId, pid: marker.pid, file, target: marker.tmuxTarget, isPusher: isPusherProcess(marker.pid), }); } } if (fix) { for (const w of wedged) { // Clearing the marker is always safe — the pane is gone either way, so // nothing can be delivered through it. Signalling is not: an // unverifiable pid is some other process that inherited this number. if (w.isPusher) { try { process.kill(w.pid, "SIGTERM"); } catch { /* already gone */ } } await deleteFile(w.file); fixed.push( w.isPusher ? `reaped wedged pusher for ${w.agentId} (pid ${w.pid}, tmux target '${w.target}' gone)` : `cleared stale transport marker for ${w.agentId} (tmux target '${w.target}' gone; pid ${w.pid} is not a tmux-pusher — not signalled)`, ); } } findings.push({ check: "wedged-local-pushers", level: wedged.length ? "warn" : "ok", detail: wedged.length ? `${wedged.length} local pusher(s) alive (pid) but their tmux pane is gone — looks attached, delivers nothing. ${fix ? "Reaped (SIGTERM + marker cleared)." : "Run doctor with fix:true to SIGTERM and clear the marker."}` : tmuxIsAvailable ? "no wedged local pushers (pid-alive, pane-dead)" : "tmux not available — skipped wedged-pusher pane probe", fixable: true, items: wedged.length ? wedged.map( (w) => `${w.agentId} (pid ${w.pid}, tmux target '${w.target}')${w.isPusher ? "" : " — pid is not a tmux-pusher, marker will be cleared without signalling"}`, ) : undefined, }); } // 1d. Duplicate session bindings — two live MCP sessions bound to one agent // id means two processes are ACTING as the same agent (the // `-liaison` shape: a dev session bound onto a live worker's id; // force/token make that possible on purpose, this makes it visible). // Bindings are per-process closure state, so this reads the on-disk // session markers stdio servers write at bind time. A marker whose pid // is dead is litter from a killed session (default signal death skips // exit handlers) — cleaned under fix. Live duplicates are NOT auto- // fixable: doctor cannot know which of two running sessions is the // impostor; the wrong session should `quit` (its marker clears on exit). { const byAgent = new Map(); const stale: string[] = []; for (const file of await listSessionFiles()) { const s = await readJson(file, null); if (!s || typeof s.pid !== "number" || !s.agentId || !isPidAlive(s.pid)) { stale.push(path.basename(file)); if (fix) { await deleteFile(file); fixed.push(`deleted stale session binding ${path.basename(file)}`); } continue; } const list = byAgent.get(s.agentId) ?? []; list.push({ pid: s.pid, via: s.via ?? "unknown", boundAt: s.boundAt ?? 0 }); byAgent.set(s.agentId, list); } const dupes: string[] = []; for (const [id, list] of byAgent) { if (list.length < 2) continue; dupes.push( `${id} — ${list .map((b) => `pid ${b.pid} (via ${b.via}, bound ${b.boundAt ? new Date(b.boundAt).toISOString() : "unknown"})`) .join(" AND ")}`, ); } findings.push({ check: "duplicate-session-binding", level: dupes.length ? "warn" : "ok", detail: dupes.length ? `${dupes.length} agent id(s) bound by more than one live session — two processes are acting as the same agent. Decide which is legitimate; the other should quit (its binding clears on exit).` : stale.length ? `no duplicate session bindings (${stale.length} stale binding file(s) from dead sessions${fix ? " — cleaned" : "; run doctor with fix:true to clean"})` : "no duplicate session bindings", fixable: true, items: dupes.length ? dupes : undefined, }); } // 2. Orphan room memberships (member not in the registry). { const orphans = new Set(); for (const e of Object.values(rooms)) { for (const m of e.members ?? []) if (!known.has(m)) orphans.add(m); } if (fix && orphans.size) { await updateJson(ROOMS_FILE, {}, (cur) => { for (const e of Object.values(cur)) { if (e.members?.length) e.members = e.members.filter((m) => known.has(m)); } return cur; }); fixed.push(`dropped ${orphans.size} orphan membership(s): ${[...orphans].join(", ")}`); } findings.push({ check: "orphan-room-memberships", level: orphans.size ? "warn" : "ok", detail: orphans.size ? `${orphans.size} channel member(s) not in the registry` : "all channel members are registered", fixable: true, items: orphans.size ? [...orphans] : undefined, }); } // 3. Orphan inbox / cursor files (owner not registered). { const orphanInbox: string[] = []; for (const fname of await listInboxFiles()) { const id = fname.replace(/\.jsonl$/, ""); if (!known.has(id)) { orphanInbox.push(id); if (fix) { await deleteFile(path.join(INBOX_DIR, fname)); fixed.push(`deleted orphan inbox ${fname}`); } } } const orphanCursor: string[] = []; for (const fname of await listCursorFiles()) { const { id } = agentIdFromCursorFilename(fname); if (!known.has(id)) { orphanCursor.push(id); if (fix) { await deleteFile(path.join(CURSOR_DIR, fname)); fixed.push(`deleted orphan cursor ${fname}`); } } } const total = orphanInbox.length + orphanCursor.length; findings.push({ check: "orphan-inboxes-cursors", level: total ? "warn" : "ok", detail: total ? `${orphanInbox.length} inbox + ${orphanCursor.length} cursor file(s) for unregistered ids` : "no orphan inbox/cursor files", fixable: true, items: total ? [...new Set([...orphanInbox, ...orphanCursor])] : undefined, }); } // Precompute parsed line counts for cursor + malformed checks. const counts = new Map(); const countFor = async (file: string) => { if (!counts.has(file)) counts.set(file, await scanJsonl(file)); return counts.get(file)!; }; /** A PUSH cursor's bound: bytes on disk, the same figure `advancePushCursor` writes. ⟨q-f018dd51⟩ */ const sizes = new Map(); const byteSizeOf = async (file: string) => { if (!sizes.has(file)) { let n = 0; try { n = (await fsp.stat(file)).size; } catch { n = 0; } sizes.set(file, n); } return sizes.get(file)!; }; // 4. Cursor offsets past end-of-file (would return [] forever). // // q-40449919: `id` must be the AGENT's id, not the cursor filename minus // one extension — a push cursor's filename is `.push.json`, and // `inboxFile(id)` on the unfixed `.push` resolves to a file that // never existed, so `inboxMax` reads 0 and the cursor's real, correct // offset compares as past-EOF against an empty file that was never the // agent's inbox in the first place. { const broken: string[] = []; const lagging: string[] = []; for (const fname of await listCursorFiles()) { const { id, kind } = agentIdFromCursorFilename(fname); const cursorPath = path.join(CURSOR_DIR, fname); const cursor = await readJson(cursorPath, {}); const overflow: string[] = []; // ⛔⛔ THE TWO CURSOR KINDS ARE IN DIFFERENT UNITS, AND CONFLATING THEM ARMED A DESTRUCTIVE FIX. // A READ cursor (`.json`) indexes PARSED ENTRIES — a message count. A PUSH cursor // (`.push.json`) is a BYTE OFFSET: `advancePushCursor` writes `toOffset ?? statSync(file).size`, // "the byte just past the line that was delivered". Measuring a byte offset against a message // count makes EVERY push cursor on a non-trivial inbox read as past EOF — and because this // check is `fixable`, the "clamp" then rewinds it to the count. ⟨q-f018dd51⟩: on 2026-09-17 a // `doctor {fix:true}` did exactly that to 11 of 13 seats across two fleets in one call — // one coordinator seat went from byte 3,413,506 to byte 992, i.e. to the START of its inbox. // Nothing replayed only because the running tails held an in-process high-water mark; the // damage was LATENT and would have fired on the next restart, when that mark is gone. const boundOf = async (file: string) => (kind === "push" ? await byteSizeOf(file) : (await countFor(file)).parsed); const unit = kind === "push" ? "bytes" : "entries"; const inboxMax = await boundOf(inboxFile(id)); if ((cursor.inboxOffset ?? 0) > inboxMax) overflow.push(`inboxOffset ${cursor.inboxOffset}>${inboxMax} ${unit}`); const roomMax = await boundOf(ROOM_FILE); if ((cursor.roomOffset ?? 0) > roomMax) overflow.push(`roomOffset ${cursor.roomOffset}>${roomMax} ${unit}`); const statusMax = await boundOf(STATUS_FILE); if ((cursor.statusOffset ?? 0) > statusMax) overflow.push(`statusOffset ${cursor.statusOffset}>${statusMax} ${unit}`); for (const [chan, off] of Object.entries(cursor.roomOffsets ?? {})) { const max = await boundOf(roomFile(chan)); if (off > max) overflow.push(`roomOffsets[${chan}] ${off}>${max} ${unit}`); } // ⭐ THE DIRECTION THAT WAS NEVER TESTED, AND THE ONE THAT ACTUALLY BIT. Everything above asks // "is the cursor PAST the end" — an over-run. A seat goes silently deaf from the opposite // shape: its push cursor sits BEHIND a grown inbox, so mail is owed and never typed // (⟨q-cdb5b007⟩). doctor reported "all cursor offsets are within bounds" for seats 10KB // behind, all day, while a worker missed its GO and idled 40 minutes. // ⛔ REPORTED, NEVER "FIXED": advancing a lagging cursor to EOF SKIPS the backlog rather than // delivering it. The remedy is a working tail; a clamp here would destroy the evidence and // the mail. This is why lag is deliberately absent from the `fix` branch below. if (kind === "push") { const size = await byteSizeOf(inboxFile(id)); const behind = size - Number(cursor.inboxOffset ?? 0); if (behind > 0) lagging.push(`${id}: ${behind} byte(s) owed (push cursor ${cursor.inboxOffset ?? 0}/${size})`); } if (overflow.length) { broken.push(`${id}${kind === "push" ? " (push)" : ""}: ${overflow.join(", ")}`); if (fix) { await updateJson(cursorPath, {}, (c) => { if ((c.inboxOffset ?? 0) > inboxMax) c.inboxOffset = inboxMax; if ((c.roomOffset ?? 0) > roomMax) c.roomOffset = roomMax; if ((c.statusOffset ?? 0) > statusMax) c.statusOffset = statusMax; if (c.roomOffsets) { for (const chan of Object.keys(c.roomOffsets)) { // Same per-kind bound as the detection above — a clamp that used the other unit // would be the very rewind this row exists to stop. const max = kind === "push" ? (sizes.get(roomFile(chan)) ?? 0) : (counts.get(roomFile(chan))?.parsed ?? 0); if (c.roomOffsets[chan] > max) c.roomOffsets[chan] = max; } } return c; }); fixed.push(`clamped cursor offsets for ${id}${kind === "push" ? " (push)" : ""}`); } } } findings.push({ check: "cursor-past-eof", level: broken.length ? "error" : "ok", detail: broken.length ? `${broken.length} cursor(s) with an offset past EOF — delivery stalled` : "all cursor offsets are within bounds (push cursors measured in BYTES, read cursors in parsed entries)", fixable: true, items: broken.length ? broken : undefined, }); // A SEPARATE FINDING, AND DELIBERATELY `fixable: false`. Lag is the ⟨q-cdb5b007⟩ deafness shape: // mail owed and never typed. It is a symptom of a tail that is not running or not typing, and // the remedy is that tail — advancing the cursor would SKIP the backlog, not deliver it. findings.push({ check: "push-cursor-lag", level: lagging.length ? "warn" : "ok", detail: lagging.length ? `${lagging.length} seat(s) with mail owed but not yet typed into their pane — a tail that is not running, or running and holding. Check herdrTail: delivered/held tells you which, and they have opposite fixes. NOT auto-repairable: advancing a lagging cursor SKIPS the backlog instead of delivering it.` : "no push cursor is behind its inbox", fixable: false, items: lagging.length ? lagging : undefined, }); } // 5. Malformed JSONL lines (silently desync offset math between server + hooks). { const jsonlFiles = [ ROOM_FILE, STATUS_FILE, ...channels.filter((c) => c !== DEFAULT_ROOM).map((c) => roomFile(c)), ...(await listInboxFiles()).map((f) => path.join(INBOX_DIR, f)), ]; const bad: string[] = []; for (const file of jsonlFiles) { const c = await countFor(file); if (c.malformed > 0) { bad.push(`${path.basename(file)} (${c.malformed})`); if (fix) { await fsp.copyFile(file, file + ".bak"); await rewriteJsonl(file, () => true); // drops unparseable lines fixed.push(`rewrote ${path.basename(file)} dropping ${c.malformed} malformed line(s) (backup: ${path.basename(file)}.bak)`); } } } findings.push({ check: "malformed-jsonl", level: bad.length ? "warn" : "ok", detail: bad.length ? `${bad.length} file(s) contain unparseable lines` : "no malformed JSONL lines", fixable: true, items: bad.length ? bad : undefined, }); } // 6. Stale agents (registered, no live transport, heartbeat past EVICT_MS). Report only. // // Task 5.3/22.1: "27 stale" collapsed two different populations that look // identical in a flat heartbeat-age list — 14 agents whose PANE was still // genuinely current (their marker's pid had died, e.g. a `/mcp` reconnect // spawned a new process in the SAME pane, but the pane itself never went // anywhere) and 13 agents that were ORPHANS with no pane behind them at // all, aged 19–45h. An orphan is stale by construction and can NEVER // clear — reporting it beside a merely-slow-heartbeat agent manufactures a // permanent false alarm in the same bucket as a real one. // // THE DISCRIMINATOR: a marker file that still exists on disk (even though // its own pid/heartbeat check already failed `isMarkerLive`) and names a // `tmuxTarget` `tmux has-session` still confirms. `has-session` checks the // PANE, not the recorded pid — so it survives exactly the reconnect shape // above, the same probe `wedged-local-pushers` already uses one direction // over (there: a live pid whose pane died; here: a dead/expired marker // whose pane did not). No marker at all, or a marker whose pane is // confirmed gone, is the only shape left — and that is a genuine orphan. { // Compute liveness WITHOUT deleting dead markers — loadLiveTransports // prunes as a side effect, which would make this read-only check mutate // state (and pre-empt the orphan-marker fix in check 1). const live = new Set(); const markerByAgent = new Map(); for (const fname of await listTransportFiles()) { const marker = await readJson(path.join(TRANSPORT_DIR, fname), null); if (!marker) continue; markerByAgent.set(marker.agentId, marker); if (isMarkerLive(marker, reg, now)) live.add(marker.agentId); } // Without a tmux binary we cannot probe a pane at all — every stale // agent is reported as an orphan candidate rather than silently split, // same posture `wedged-local-pushers` takes. const tmuxIsAvailable = tmuxAvailable(); const paneAlive = (target: string): boolean => tmuxIsAvailable && paneExists(target); const paneConfirmed: string[] = []; const orphans: string[] = []; for (const [id, a] of Object.entries(reg)) { if (live.has(id)) continue; if (now - a.lastHeartbeat <= EVICT_MS) continue; const age = `${Math.floor((now - a.lastHeartbeat) / 3600000)}h`; const marker = markerByAgent.get(id); const markerTarget = marker ? targetOf(marker) : undefined; if (isLocallyProbeable(marker?.transport) && markerTarget && paneAlive(markerTarget)) { paneConfirmed.push(`${id} (${age}, pane '${markerTarget}' still current)`); } else { orphans.push(`${id} (${age})`); } } const stale = [...paneConfirmed, ...orphans]; findings.push({ check: "stale-agents", level: stale.length ? "warn" : "ok", detail: stale.length ? `${stale.length} agent(s) past the eviction window — next list_agents will drop them ` + `(${paneConfirmed.length} pane-confirmed current, ${orphans.length} genuine orphan(s) with no live pane behind them)` : "no stale agents", fixable: false, items: stale.length ? stale : undefined, }); // A SEPARATE finding, not a sub-line: "stale-agents" answers "how many // will list_agents drop", which is true of both buckets equally — an // orphan and a pane-confirmed agent are dropped from the SAME list the // same way. "orphan-agents" answers the different question this task is // actually about — which of those, if any, can never clear on their // own — and reports it even when it is empty, so a checker that only // speaks when it fires cannot be told from one that never ran. findings.push({ check: "orphan-agents", level: orphans.length ? "warn" : "ok", detail: orphans.length ? `${orphans.length} stale agent(s) have no live tmux pane behind them — permanent, will not clear on their own (unregister or let eviction drop them)` : tmuxIsAvailable ? `no orphans among ${stale.length} stale agent(s)${stale.length ? " — all pane-confirmed current" : ""}` : "tmux not available — could not distinguish orphans from pane-confirmed stale agents", fixable: false, items: orphans.length ? orphans : undefined, }); } // 6b. STALENESS HAS TWO CAUSES AND THEY LOOK IDENTICAL IN A FLAT LIST. // // Check 6 reports "these N agents are stale", which is the detection half and // leaves the reader to infer the cause. The inference is the expensive part: a // coordinator misdiagnosed a billing outage as a coordination failure for want // of exactly this distinction, and spent the incident chasing agents. // // UNIFORM every agent stale by roughly the same interval. Agents do not // fail in lockstep — something they SHARE did: the machine slept, // the API key expired, billing lapsed, the host lost network. The // remedy is the environment, and touching the agents does nothing. // DIVERGENT some stale, some fresh, on the same bus at the same moment. The // shared substrate is demonstrably working, so the fault is the // stale agent's. The remedy is that agent. // // CLASSIFICATION IS THE DELIVERABLE, NOT DETECTION — the same finding as // `transport-build-split`: two states with different remedies reported as one // warning leave the reader to guess, and the guess is made under incident // pressure. // // WITH FEWER THAN TWO AGENTS THE QUESTION IS UNANSWERABLE and it says so. One // stale agent on a bus of one is uniform and divergent simultaneously; there is // no second observation to compare against. Naming a cause there would be the // confident-wrong-answer shape this whole class is about. { const live = new Set(); for (const fname of await listTransportFiles()) { const marker = await readJson(path.join(TRANSPORT_DIR, fname), null); if (marker && isMarkerLive(marker, reg, now)) live.add(marker.agentId); } const entries = Object.entries(reg).map(([id, a]) => ({ id, live: live.has(id), ageMs: now - a.lastHeartbeat, stale: !live.has(id) && now - a.lastHeartbeat > STALE_MS, })); const stale = entries.filter((e) => e.stale); const fresh = entries.filter((e) => !e.stale); const mins = (ms: number) => Math.round(ms / 60000); let level: "ok" | "warn" = "ok"; let detail: string; let items: string[] | undefined; if (entries.length === 0) { detail = "no registered agents — nothing to classify"; } else if (stale.length === 0) { detail = `${entries.length} agent(s), none stale (threshold ${mins(STALE_MS)}m)`; } else if (entries.length < 2) { level = "warn"; detail = `1 agent and it is stale (${mins(stale[0]!.ageMs)}m) — UNCLASSIFIABLE. Uniform and divergent are the same ` + `picture with one observation, so the cause is not named here rather than guessed.`; items = [`${stale[0]!.id} (${mins(stale[0]!.ageMs)}m)`]; } else if (fresh.length === 0) { // Spread across the stale set decides it: a shared cause stops everything at // once, so the ages cluster. Independent failures do not. const ages = stale.map((e) => e.ageMs).sort((a, b) => a - b); const spread = ages[ages.length - 1]! - ages[0]!; const tight = spread <= Math.max(2 * 60 * 1000, ages[ages.length - 1]! * 0.1); level = "warn"; detail = tight ? `UNIFORM: all ${stale.length} agent(s) stale within ${mins(spread)}m of each other (${mins(ages[0]!)}–${mins(ages[ages.length - 1]!)}m). ` + `Agents do not fail in lockstep — suspect something they SHARE (machine asleep, credentials, billing, network). ` + `Restarting agents will not help.` : `ALL ${stale.length} agent(s) are stale but their ages span ${mins(spread)}m — NOT uniform, so a single shared cause does not explain it. ` + `Treat as ${stale.length} independent failures until something ties them together.`; items = stale.map((e) => `${e.id} (${mins(e.ageMs)}m)`); } else { level = "warn"; detail = `DIVERGENT: ${stale.length} stale, ${fresh.length} fresh on the same bus at the same moment. ` + `The shared substrate is demonstrably working — the fault is the stale agent's, not the environment's.`; items = stale.map((e) => `${e.id} (${mins(e.ageMs)}m stale)`); } findings.push({ check: "staleness-class", level, detail, fixable: false, ...(items ? { items } : {}) }); } // 7. Oversized JSONL files. Report only (suggest prune). { const big: string[] = []; const candidates = [ ROOM_FILE, STATUS_FILE, ...channels.filter((c) => c !== DEFAULT_ROOM).map((c) => roomFile(c)), ...(await listInboxFiles()).map((f) => path.join(INBOX_DIR, f)), ]; for (const file of candidates) { const sz = await fileSize(file); if (sz > maxBytes) big.push(`${path.basename(file)} (${(sz / 1024 / 1024).toFixed(1)}MB)`); } findings.push({ check: "oversized-files", level: big.length ? "warn" : "ok", detail: big.length ? `${big.length} file(s) over ${(maxBytes / 1024 / 1024).toFixed(0)}MB — consider prune` : "no oversized files", fixable: false, items: big.length ? big : undefined, }); } // 8. Stale lock dirs from crashed writers. { const locks = await scanStaleLocks(60_000, now); for (const l of locks) { if (fix) { try { await fsp.rm(l.path, { recursive: true, force: true }); fixed.push(`removed stale lock ${path.basename(l.path)}`); } catch { // ignore } } } findings.push({ check: "stale-locks", level: locks.length ? "warn" : "ok", detail: locks.length ? `${locks.length} lock dir(s) older than 60s — likely from a crashed writer` : "no stale locks", fixable: true, items: locks.length ? locks.map((l) => `${path.basename(l.path)} (${Math.floor(l.ageMs / 1000)}s)`) : undefined, }); } // 9. Channel/registry consistency: rooms/.jsonl files without a registry entry. { const orphanFiles: string[] = []; if (existsSync(ROOMS_DIR)) { let names: string[] = []; try { names = await fsp.readdir(ROOMS_DIR); } catch { // ignore } for (const name of names) { if (!name.endsWith(".jsonl")) continue; const chan = name.replace(/\.jsonl$/, ""); if (!rooms[chan]) { orphanFiles.push(name); if (fix) { await ensureRoom(chan, "doctor"); fixed.push(`registered channel '${chan}' (had a JSONL file but no registry entry)`); } } } } findings.push({ check: "channel-registry-consistency", level: orphanFiles.length ? "warn" : "ok", detail: orphanFiles.length ? `${orphanFiles.length} channel file(s) with no registry entry` : "channel files and registry agree", fixable: true, items: orphanFiles.length ? orphanFiles : undefined, }); } // 9b. Document scope drift (Phase 8 Task 4). For each document declared in // scopes.json, compare git's last writer against the declared owner. // // DETECTION ONLY, never fixable — rewriting or reverting someone else's // file is not a safe automatic repair, and the bus cannot prevent the // write in the first place (agents edit these with ordinary file tools; // enforcement waits for Task 5). Skips silently when no scopes.json // exists (opt-in) or when the declared repo isn't a git checkout, the // same way the wedged-pusher check skips without tmux — a check that // can't observe anything must not guess. { const scopes = await loadScopes(); const drift: string[] = []; const unattributed: string[] = []; let detail: string; if (!scopes.documents.length) { detail = scopes.configured ? `${path.basename(scopes.file)} declares no documents` : `no ${path.basename(scopes.file)} — document scopes are opt-in and none are declared`; } else if (!isGitRepo(scopes.repo)) { detail = `${scopes.documents.length} document(s) declared but '${scopes.repo}' is not a git checkout — last writer is unknowable, check skipped`; } else { for (const doc of scopes.documents) { const writer = lastWriterOf(scopes.repo, doc.path); if (!writer) continue; // never committed — nothing has written it yet const who = attributeWriter(writer, reg); if (!who) { // Commits are authored by humans/machine accounts, not agent ids, so // an unmappable author is the normal case — reported, never flagged. unattributed.push(`${doc.path}: last written by '${writer.author}' (${writer.commit}), not attributable to a registered agent`); continue; } if (ownsDocument(who.agentId, reg[who.agentId], doc.owner)) continue; drift.push( `${doc.path}: declared owner '${doc.owner}' (${doc.mode}) but last written by '${who.agentId}'` + `${who.roleId ? ` [role ${who.roleId}]` : ""} in ${writer.commit} (${writer.when})`, ); } detail = drift.length ? `${drift.length} document(s) last written by someone other than their declared owner — advisory: coordinate ownership, doctor will not rewrite anyone's file` : `${scopes.documents.length} declared document(s) agree with their scope`; } findings.push({ check: "document-scope-drift", level: drift.length ? "warn" : "ok", detail, fixable: false, items: drift.length ? drift : unattributed.length ? unattributed : undefined, }); } // 10. Environment sanity. Report only. Path + version come from the running // module's package.json so a stale/decoy bin is named (LESSONS #35). Git // identity is walk-up-to-.git + rev-parse (kit monorepo root); omitted // entirely when there is no checkout — never invented. { const tmuxReported = tmuxVersion(); const tmuxOk = tmuxReported !== undefined; const ident = resolveServerIdentity(); const loc = ident.branch && ident.sha ? `path=${ident.path} version=${ident.version} ${ident.branch}@${ident.sha.slice(0, 12)}` : `path=${ident.path} version=${ident.version}`; const items = [ `root=${ROOT}`, `execPath=${process.execPath}`, `inTmux=${!!process.env.TMUX_PANE}`, `path=${ident.path}`, `version=${ident.version}`, ]; if (ident.branch) items.push(`branch=${ident.branch}`); if (ident.sha) items.push(`sha=${ident.sha}`); findings.push({ check: "environment", level: tmuxOk ? "ok" : "warn", detail: tmuxOk ? `root=${ROOT}; node=${process.execPath}; ${loc}; tmux=${tmuxReported || "present"}` : `root=${ROOT}; node=${process.execPath}; ${loc}; tmux NOT on PATH — the tmux-push transport will not work`, fixable: false, items, }); } const summary = { ok: findings.filter((f) => f.level === "ok").length, warn: findings.filter((f) => f.level === "warn").length, error: findings.filter((f) => f.level === "error").length, }; return { ok: true, healthy: summary.warn === 0 && summary.error === 0, fixApplied: fix, root: ROOT, findings, fixed: fix ? fixed : undefined, summary, }; }