/** * The daemon process itself: what `omp-conductor daemon` actually runs. * * Everything in this directory above it is a pure-ish pass over a `Deps`. * This is where the real ones are built — the tracker, the store, the verb * socket, the orchestrator session, the pool and the registries — and where the * only long-lived state in the dispatcher lives: `Bun.serve`, the timers, the * integrity baseline recorded at boot, and the shutdown that has to drain them * all in the right order. * * Keeping it last and separate is what lets every pass be tested without a * process. Nothing here is called by a pass; the arrow runs one way, from the * runtime into the passes, which is also why `runDaemon` is the only place a * dep is wired to its production implementation. */ import { mkdirSync } from "node:fs"; import { join } from "node:path"; import { findProject, loadConfig, resolveCaps, resolveReleaseGrants, resolveSharedInstallAuthority, stateDir } from "../config.ts"; import { probeCredentialClass } from "../credential-class.ts"; import { createEscalator, escalationIssueRef } from "../escalate.ts"; import { releaseOrphanedWorkerPane } from "../fleet.ts"; import { probeCriticalBase, probeRunLane, pushRunBranch } from "../gitops.ts"; import { pendingCodeGraph, probeCodeGraph, type CodeGraphHealth } from "../graph-health.ts"; import { ensureHttpToken } from "../http-token.ts"; import { acquireOnceLease, livingDaemon } from "../lifecycle.ts"; import { errText, log } from "../log.ts"; import { reconcileOrchestratorDown } from "../orchestrator-down.ts"; import { startOrchestrator, type OrchestratorHandle } from "../orchestrator.ts"; import { isPaused } from "../pause.ts"; import { recordReleaseBlock } from "../release-policy.ts"; import { createReportOutbox, type ReportOutbox } from "../reports.ts"; import { reconcileOrphanedRuns } from "../settlement.ts"; import { dbPath, openStore, utcDay } from "../store.ts"; import { checkTelegramFreshness } from "../telegram-freshness.ts"; import { GraphqlBreaker, makeTracker } from "../tracker/github.ts"; import { RELEASE_SHAPES, type Escalation, type ProjectConfig, type ResolvedGrants, type RunRecord } from "../types.ts"; import { runCommand } from "../upgrade-verify.ts"; import { inspectSurfaces } from "../upgrade.ts"; import { sharedUsageSource } from "../usage.ts"; import { githubVerbActions } from "../verbs/actions.ts"; import { listenVerbChannel, type VerbListener } from "../verbs/server.ts"; import { ensureVerbSocketDir, peerCredentialReader, transportBanner, verbSocketPath } from "../verbs/socket.ts"; import type { RunPublisher } from "../worktree.ts"; import { createWorkerPool } from "./admission-pass.ts"; import { DEFAULT_PORT, GRAPH_HEALTH_INTERVAL_MS, PACKAGE_SRC_DIR, REPORT_DELIVERY_INTERVAL_MS, TICK_INTERVAL_MS, verbDeps, type DaemonOpts, type Deps, type DrainSignal, type IntegrityGate } from "./deps.ts"; import { createTurnLimitRegistry, createWorkerControlRegistry } from "./dispatch.ts"; import { daemonHttpResponse } from "./http.ts"; import { packageManifest } from "./integrity.ts"; import { workspaceOwnership } from "./panes.ts"; import { reconcileCrashedReviewRevisions } from "./review.ts"; import { reconcileHistoricalInfra } from "./settle-pass.ts"; import { tick } from "./tick.ts"; import { daemonHealth, daemonHealthSnapshot } from "./views.ts"; export interface ProjectRuntime { d: Deps; outbox: ReportOutbox; orchestrator?: OrchestratorHandle; orchestratorVerbs?: VerbListener; codeGraph: CodeGraphHealth; graphProbe?: Promise; reportPass?: Promise; } export function orchestratorStandingOrders( project: ProjectConfig, projects: readonly ProjectConfig[], ): { brief: string; releaseGrants: ResolvedGrants; } { const installAuthority = resolveSharedInstallAuthority(projects); const releaseGrants = { ...resolveReleaseGrants(project), install: installAuthority.holder ?? "human", }; const grantedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] === "orchestrator"); const deniedShapes = RELEASE_SHAPES.filter((shape) => releaseGrants[shape] !== "orchestrator"); return { releaseGrants, brief: [ `You are the omp-conductor orchestrator for project "${project.name}".`, `Tracker: ${project.tracker.repo}. Pass --repo ${project.tracker.repo} to every gh command:`, "this working directory is the conductor's state directory, not a checkout.", `Labels: queue=${project.queueLabel}, running=${project.stateLabels.inProgress}, ` + `blocked=${project.stateLabels.blocked}, failed=${project.stateLabels.failed}.`, "The dispatcher claims queue-labelled issues, runs one worker session per attempt in its own", "worktree under hard turn/wallclock/spend caps, and escalates to you when a worker blocks or", "fails twice, its gates stay red, its branch conflicts, or a tripwire fires.", "Your job when that happens: re-brief the issue (comment what the next worker must do", `differently, then put ${project.queueLabel} back on it), file follow-up issues, or promote to`, "tier 2 and let the human decide.", project.authority.merge === "orchestrator" ? "You never edit product code or push a branch — a worker session does that. Merging is yours: one PR at " + "a time, freshness-checked against the base branch, per the Releases section of your POLICY.md." : "You never edit product code, push a branch, or merge a PR — a worker session edits and pushes, and a " + "human merges.", `Release tool gate: ${ grantedShapes.length === 0 ? "every release and deploy shape is mechanically blocked for you" : `you may invoke ${grantedShapes.join(", ")} — and only by the procedure in your POLICY.md` }.` + (deniedShapes.length === 0 ? "" : ` Blocked: ${deniedShapes.join(", ")}.`), ...(installAuthority.holder === undefined ? [ `Host-global install authority conflicts: ${installAuthority.entries .map((entry) => `${entry.project}=${entry.holder}`) .join(", ")}. Install is blocked.`, ] : []), "Handle each escalation below before the next one.", ].join("\n"), }; } /** * Pacing for the resident dispatch loop, interruptible so an operator action — * `resume` — does not wait out the full interval (#380). * * `requestWake` is sticky until a pass absorbs it: while the long sleep is in * flight it is cut short, and when a pass is mid-flight the loop skips the * *following* sleep. Either way the next pass starts without the interval * wait. The single boolean coalesces any number of near-simultaneous wakes * into one prompt pass, and the strictly sequential loop means a wake can * never start a pass on top of one already running. */ export interface DispatchPace { /** Ask for a prompt pass; cuts an in-flight sleep short. Coalesces. */ requestWake(): void; /** Absorb a pending request — true when the pass that just finished should * skip its sleep. */ takeWake(): boolean; /** The long interval sleep; resolves early when {@link requestWake} fires. */ sleep(ms: number): Promise; } export function createDispatchPace(): DispatchPace { let pendingWake = false; let interruptSleep: (() => void) | undefined; return { requestWake() { pendingWake = true; interruptSleep?.(); }, takeWake() { const pending = pendingWake; pendingWake = false; return pending; }, async sleep(ms) { // A wake that landed in the gap between the loop's takeWake check and // this call is absorbed here: skip the interval, and consume the flag // so the pass that follows it is the only prompt one. if (pendingWake) { pendingWake = false; return; } await new Promise((resolve) => { const timer = setTimeout(resolve, ms); interruptSleep = () => { // The interruption *is* the absorption: the pass that follows the // sleep is the prompt pass this wake asked for, so the loop sleeps // normally after it instead of running a second immediate pass. pendingWake = false; clearTimeout(timer); resolve(); }; // A wake can land between the loop's takeWake check and this // constructor; resolve immediately instead of letting it wait out // the full interval. if (pendingWake) interruptSleep(); }); interruptSleep = undefined; }, }; } export interface DispatchLoopOptions { /** One dispatch pass over every configured project runtime. */ tick(): Promise; /** True once a stop is requested; checked between passes. */ shouldStop(): boolean; pace: DispatchPace; /** The scheduled interval between passes (`TICK_INTERVAL_MS` in production). */ tickIntervalMs: number; } /** * The resident daemon's dispatch loop: one pass immediately, then a * `tickIntervalMs` sleep the {@link DispatchPace} can cut short. Exported so * the resume-wake behaviour can be pinned deterministically without a live * daemon (#380). */ export async function runDispatchLoop(o: DispatchLoopOptions): Promise { const { tick, shouldStop, pace, tickIntervalMs } = o; while (!shouldStop()) { await tick(); if (shouldStop()) break; // A wake that landed while the pass was in flight skips the following // sleep: that pass made its admission decisions against pre-resume state, // and the resumed project must not wait a full interval for the next one. if (pace.takeWake()) continue; await pace.sleep(tickIntervalMs); } } /** What the daemon knows about its tier-1 orchestrator at diversion time. */ export interface Tier1DiversionFacts { /** The project's configured tier-1 transport is the external orchestrator * session: issue comments are the *configured* destination, not a * fallback. */ external: boolean; /** Why the daemon-owned orchestrator session failed to start, when it did. */ startError?: string; /** Whether the daemon-owned orchestrator session is alive right now * (`OrchestratorHandle.alive()` — event-driven, false once the session * exited). */ alive: boolean; /** Whether the daemon-owned orchestrator session is mid-turn right now * (`OrchestratorHandle.busy()`). */ busy: boolean; } /** * The one-line record of a tier-1 escalation that landed on the issue-comment * fallback instead of the orchestrator injection (#1068). * * "while the orchestrator was down" is reserved for what it actually means — * a session that failed to start or one that has exited. The #1062 shape was * neither: the orchestrator was mid-tick, its session did not take the * injection, and a reader of "diverted … while the orchestrator was down" * went looking for a dead session that did not exist. When the session is * alive the line says where the escalation was delivered and why it was not * injected; when the project runs an external orchestrator the comments *are* * the configured transport and the line says so instead of calling it a * diversion. */ export function tier1DiversionLine(e: Escalation, facts: Tier1DiversionFacts): string { const ref = escalationIssueRef(e.issue); if (facts.external) { return `orchestrator: tier-1 escalation on ${ref} posted as an issue comment (external orchestrator — the configured tier-1 transport)`; } if (facts.startError !== undefined || !facts.alive) { return ( `orchestrator: tier-1 escalation on ${ref} diverted to issue comments while the orchestrator was down` + (facts.startError !== undefined ? ` (start failed: ${facts.startError})` : " (session exited)") ); } return ( `orchestrator: tier-1 escalation on ${ref} posted as an issue comment — the orchestrator session ` + `is alive but did not take the injection${facts.busy ? " (mid-turn)" : ""}` ); } export async function runDaemon(o: DaemonOpts = {}): Promise { // A `--once` tick is still a dispatcher: it settles rows, projects labels, // admits and launches workers, so it must hold the same exclusive daemon/ // state ownership as the long-running dispatcher for its whole lifecycle // (#431). This is the one choke point every once tick passes through (both // `daemon --once` and the `setup host` smoke), so ownership is acquired here // — before any settlement, label, admission, worktree or worker mutation — // and refused (having done none of those) while another live daemon owns the // project or another live `--once` holds the lease. const onceLease = o.once ? acquireOnceLease({ port: o.port, project: o.project }) : undefined; if (onceLease !== undefined && !onceLease.ok) { throw new Error( `another daemon is alive (pid ${onceLease.pid}); stop it first — a --once tick refuses to run beside it`, ); } const cfg = loadConfig(); const projects = o.project === undefined ? cfg.projects : [findProject(cfg, o.project)]; const store = openStore(dbPath()); const verbPeerReader = peerCredentialReader(); const verbDir = ensureVerbSocketDir(stateDir()); const integrity: IntegrityGate = { baseline: packageManifest(), paged: false }; const usage = sharedUsageSource(); const turnLimits = createTurnLimitRegistry((runId, maxTurns) => { store.updateRun(runId, { maxTurns }); }); const workerControls = createWorkerControlRegistry(); // Created before the pool so a worker's completion can poke the loop it // shares with the interval sleep (#878). const pace = createDispatchPace(); const workers = createWorkerPool(() => { // Sticky and coalescing by construction: several workers settling together // produce one pass, and a pass already running absorbs the request rather // than overlapping. No second owner is created and no gate is skipped. pace.requestWake(); }); const alive = livingDaemon(); const runtimes: ProjectRuntime[] = []; // The stop fence (#374): one signal shared by every project's tick. // `stopping` gates the run loop between whole ticks; `drain.draining` is // the same event visible *inside* a tick already in flight, so a pass that // was mid-admission when the stop landed re-checks it before claiming or // launching. Created here, before the loop, so `stop` below and every // project's `Deps` reference the same object. const drain: DrainSignal = { draining: false }; log(`verb transport: ${transportBanner(verbDir, verbPeerReader)}`); log(`package integrity baseline: ${integrity.baseline.size} files under ${PACKAGE_SRC_DIR}`); for (const project of projects) { const projectLog = (message: string): void => { log(projects.length === 1 ? message : `[${project.name}] ${message}`); }; const caps = resolveCaps(project, cfg.defaults); const host = cfg.host; // One transient-server-error breaker per project (#642): admission's // GraphQL checks and the orchestrator mutation commands share it, so a 503 // observed by either side gates both instead of one provider outage being // re-asked by every candidate AND every mutation. const graphqlBreaker = new GraphqlBreaker(); const tracker = makeTracker(project, undefined, { onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"), onRefusal: (at) => store.recordGhRefusal?.(at), onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"), }, { graphqlBreaker }); const verbActions = githubVerbActions(project, undefined, undefined, graphqlBreaker); if (alive === undefined || alive.pid === process.pid) { const orphanPublisher = (run: RunRecord): RunPublisher => { const repo = project.routing.repos[run.repo]; return async (branch) => repo === undefined ? { ok: false, stderr: `run ${String(run.id)} names repo "${run.repo}", ` + `which project "${project.name}" no longer routes`, } : pushRunBranch(project, { repo, runRepoPath: run.worktree, branch }); }; for (const run of await reconcileOrphanedRuns(store, project.name, orphanPublisher)) { // The workspace outlives the process, so a restart inherits panes whose // workers are gone (#842). A session-host child dies with the daemon that // owned its socket, so an orphaned row's pane is authoritatively dead // whatever its recorded pid says. Ownership is proven before the close // (#1035 review): pane ids are reused after a Herdr restart, so only a // pane inside a workspace conductor owns — token-marked or store-recorded, // which is what survives Herdr dropping tokens on restore — whose // reported identity still agrees with this row is retired. An unowned // hit closes nothing and says so. const orphanPane = releaseOrphanedWorkerPane( { paneId: run.paneId, paneLabel: run.paneLabel, runId: run.id, project: run.project }, { ownership: workspaceOwnership(store) }, ); if (orphanPane.kind === "released") { projectLog( `#${run.issue} herdr pane ${orphanPane.paneId} retired: its worker died with the previous daemon`, ); } else if (orphanPane.kind === "unowned") { projectLog(`#${run.issue} herdr pane ${orphanPane.paneId} left alone: ${orphanPane.reason}`); } else if (orphanPane.kind === "failed") { projectLog(`#${run.issue} herdr pane ${orphanPane.paneId} release failed: ${orphanPane.reason}`); } projectLog( `#${run.issue} orphaned by a previous daemon (attempt ${run.attempt}, was ${run.state}, ` + `worktree ${run.worktree}) — slot freed; the ${project.stateLabels.inProgress} label stays ` + "until the orchestrator triages what the worker left", ); } } else if (runtimes.length === 0) { log(`skipping orphan reconciliation: daemon pid ${alive.pid} is alive and owns the active runs`); } const { brief, releaseGrants } = orchestratorStandingOrders(project, cfg.projects); let orchestrator: OrchestratorHandle | undefined; let orchestratorVerbs: VerbListener | undefined; /** First start-failure cause, surfaced by the orchestrator-down incident (#288). */ let orchestratorStartError: string | undefined; if (project.escalation.orchestrator === "external") { projectLog( "orchestrator: external — tier-1 escalations post as issue comments for the external session's drain duty", ); } else { try { const orchCwd = join( stateDir(), projects.length === 1 ? "orchestrator" : `orchestrator-${project.name}`, ); mkdirSync(orchCwd, { recursive: true }); orchestratorVerbs = await listenVerbChannel( verbDeps({ project, store, tracker, verbActions }), { kind: "orchestrator", path: verbSocketPath(verbDir, `orchestrator-${project.name}`), project: project.name, role: "orchestrator", }, { ...(verbPeerReader === undefined ? {} : { peerReader: verbPeerReader }) }, ); orchestrator = await startOrchestrator({ cwd: orchCwd, brief, releaseGrants, socketPath: join(orchCwd, "ipc.sock"), verbSocketPath: orchestratorVerbs.path, onSpawn: (pid) => { orchestratorVerbs?.bindPid(pid); }, onChildLog: (line) => { projectLog(`orchestrator ${line}`); }, onReleaseBlocked: (shape, context) => recordReleaseBlock(project.name, "orchestrator", shape, context), }); const transcript = orchestrator.sessionFile(); const loaded = orchestrator.extensionVersion(); projectLog( `orchestrator session ready${transcript === undefined ? "" : ` · ${transcript}`}` + `${loaded === undefined ? "" : ` · loaded omp-conductor ${loaded}`}`, ); } catch (err) { orchestratorStartError = errText(err); projectLog( "WARNING: orchestrator session failed to start; tier-1 escalations will fall back to issue " + `comments: ${orchestratorStartError}`, ); await orchestratorVerbs?.close(); orchestratorVerbs = undefined; } } let runtimeDeps: Deps | undefined; const currentProject = (): ProjectConfig => runtimeDeps?.project ?? project; const deliveryPolicyValid = (): boolean => runtimeDeps?.deliveryPolicyValid === true; const escalator = createEscalator( currentProject, tracker, store, orchestrator, Date.now, deliveryPolicyValid, (e) => { // Every tier-1 escalation that lands on the issue-comment fallback was // diverted from the orchestrator. Count it durably on the open incident // (a no-op when none is open), so the page and status name how much // the outage diverted (#288). store.bumpOrchestratorDiverted(project.name, 1); projectLog( tier1DiversionLine(e, { external: project.escalation.orchestrator === "external", startError: orchestratorStartError, alive: orchestrator?.alive() ?? false, busy: orchestrator?.busy() ?? false, }), ); }, ); const outbox = createReportOutbox({ project: currentProject, store, escalate: (event) => escalator.escalate(event), log: projectLog, deliveryAllowed: deliveryPolicyValid, }); if (alive === undefined || alive.pid === process.pid) { for (const report of outbox.recover(Date.now())) { projectLog( `report ${report.id} was left mid-send by a previous daemon (attempt ${report.attempts}) — ` + "retrying; its message will say it may be a repeat", ); } } const d: Deps = { project, caps, host, tracker, store, drain, deliveryPolicyValid: false, usage, escalate: (event) => escalator.escalate(event), turnLimits, workerControls, integrity, stall: { paged: false }, ...(orchestrator === undefined ? {} : { orchestrator }), cleanup: { next: 0 }, probeCriticalBase: (repo, markers, branch) => probeCriticalBase(project, repo, branch, markers), // The same seam `doctor` and `upgrade` read (#904/#919), never a second // implementation. `readHerdr` stays on: a host that runs herdr is the // case where a stale recovery pin matters, and a host without it answers // "absent", which no cheap surface treats as a fault. probeInstallSurfaces: () => inspectSurfaces({ run: runCommand, log, env: process.env }), probeTelegramFreshness: () => // The module's own reader, so the daemon, `doctor` and `status` cannot // come to disagree about what is installed (#961). checkTelegramFreshness({ run: async (cmd, args) => { const r = await runCommand(cmd, args); return { code: r.code, stdout: r.stdout }; }, }), probeWorktreeLane: (input) => probeRunLane(input), // The credential-class fence's reader (#852): the runnable probe module, // spawned against the live credential store. Wired once here so the // admission gate and the launch fence ask the same question through the // same transport — two readers would eventually disagree, and the one that // disagreed by passing is the expensive one. probeCredentialClass: (provider) => probeCredentialClass(provider), // A cross-repo Depends-on prerequisite reads through the same GitHub // credential/accounting seams as the project tracker — a fresh tracker // scoped to the referenced repo, reusing the daemon's gh hooks so API // spend and refusals are counted exactly as the main tracker's are. probeIssueIn: (ownerRepo, issue) => makeTracker({ ...project, tracker: { kind: "github", repo: ownerRepo } }, undefined, { onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"), onRefusal: (at) => store.recordGhRefusal?.(at), }).issueSnapshot(issue), // The body twin of `probeIssueIn`, for the dependency-graph cycle pass // (#421): reads a reachable routed prerequisite's body through the same // per-repo tracker/credential/accounting seams. probeBodyIn: (ownerRepo, issue) => makeTracker({ ...project, tracker: { kind: "github", repo: ownerRepo } }, undefined, { onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"), onRefusal: (at) => store.recordGhRefusal?.(at), }).issueBody(issue), ...(verbPeerReader === undefined ? {} : { verbPeerReader }), verbActions, }; runtimeDeps = d; // Review-revision restart recovery (#692): a revision the previous daemon // claimed and lost — its run is `running`/`orphaned` by the orphan sweep // above — is restored to `pushed-green` and re-queued so this process's // first dispatch pass resumes the exact session. Runs under the same // dead-daemon guard as `reconcileOrphanedRuns`, and after it, so the // revision's worktree has already been salvaged to the branch before it is // cleared for a fresh reattach. A round that cannot be restored was // settled and escalated by the reconcile itself. if (alive === undefined || alive.pid === process.pid) { try { for (const revision of await reconcileCrashedReviewRevisions(d)) { projectLog( `#${revision.issue} review round ${revision.round} recovered across restart — ` + `run ${revision.runId} is pushed-green again and awaits its next dispatch pass`, ); } } catch (err) { projectLog(`review revision restart recovery failed: ${errText(err)}`); } } // Adjudication restart recovery (#932). An adjudication is `running` only // while a session it launched is alive, and that session died with the // previous process — nothing resumes it, because an adjudicator has no // branch, no worktree and no transcript worth continuing: its whole output // is a verdict it never produced. // // So it settles `failed`, which is honest and terminal, rather than being // left `running` forever (a row nothing would ever touch again) or silently // re-queued (a second launch for one head, which the one-shot exists to // prevent). The one shot for that head is spent, and #876's disposition path // reads `failed` exactly as it reads any other non-clear verdict. if (alive === undefined || alive.pid === process.pid) { for (const adjudication of store.openReviewAdjudications(project.name)) { if (adjudication.state !== "running") continue; store.settleReviewAdjudication( adjudication.id, "failed", "the daemon restarted while this adjudication was running; no verdict was produced", Date.now(), ); projectLog( `#${adjudication.issue} adjudication failed across restart — its session died with the previous daemon`, ); } } // Startup reconciliation: close an incident carried over from a previous // process when the orchestrator is up (one recovery notice), or open one // when it failed to start (one down page). A daemon restarted while still // down rediscovers the open incident and does not re-page it. await reconcileOrchestratorDown({ project, store, orchestrator, escalate: (event) => d.escalate(event), ...(orchestratorStartError === undefined ? {} : { startCause: orchestratorStartError }), log: projectLog, }); runtimes.push({ d, outbox, ...(orchestrator === undefined ? {} : { orchestrator }), ...(orchestratorVerbs === undefined ? {} : { orchestratorVerbs }), codeGraph: pendingCodeGraph(project), }); } // Historical budget repair (#638): one bounded pass per daemon process, // before the first tick, so admission and `unblock` read the repaired counts // from the moment the daemon starts. A classifier fix changes future // verdicts but not rows already settled under the old one; re-fetching their // failed check logs and reclassifying the exact closed infrastructure // signature returns those attempts without re-animating the runs. Best-effort // per project — one unreachable project must not stop the rest from booting. for (const runtime of runtimes) { try { await reconcileHistoricalInfra(runtime.d); } catch (err) { log(`historical infra reconciliation failed: ${errText(err)}`); } } if (o.once) { try { for (const runtime of runtimes) await tick(runtime.d, workers); await workers.drain(); for (const runtime of runtimes) await runtime.outbox.deliverDue(); } finally { // Drop exclusivity only while it is still ours; a manager that claimed // the record mid-tick is left alone. A lease acquired but never released // (config/lease failure before this branch) is stale recovery's job. if (onceLease !== undefined && onceLease.ok) onceLease.lease.release(); for (const runtime of runtimes.toReversed()) { await runtime.orchestrator?.dispose(); await runtime.orchestratorVerbs?.close(); } store.close(); } return; } const refreshCodeGraph = (runtime: ProjectRuntime): void => { if (runtime.graphProbe !== undefined) return; runtime.graphProbe = probeCodeGraph(runtime.d.project) .then((health) => { runtime.codeGraph = health; }) .catch(() => { log( `[${runtime.d.project.name}] code-graph health probe failed unexpectedly; ` + "retaining the previous bounded result", ); }) .finally(() => { runtime.graphProbe = undefined; }); }; for (const runtime of runtimes) refreshCodeGraph(runtime); const graphTimer = setInterval(() => { for (const runtime of runtimes) refreshCodeGraph(runtime); }, GRAPH_HEALTH_INTERVAL_MS); const drainReports = (runtime: ProjectRuntime): void => { if (runtime.reportPass !== undefined) return; runtime.reportPass = runtime.outbox .deliverDue() .then(() => {}) .catch((err: unknown) => { log(`[${runtime.d.project.name}] report delivery pass failed: ${errText(err)}`); }) .finally(() => { runtime.reportPass = undefined; }); }; for (const runtime of runtimes) drainReports(runtime); const reportTimer = setInterval(() => { for (const runtime of runtimes) drainReports(runtime); }, REPORT_DELIVERY_INTERVAL_MS); let stopping = false; // The wake surface (#380): `resume` POSTs /wake, `stop` interrupts the same // sleep. A wake request is sticky and coalescing — several resumes in quick // succession produce one prompt pass, never an overlapping one. const stop = (): void => { if (stopping) return; stopping = true; // Close the admission fence before the loop is told, so a tick already in // flight — parked at an await when the signal landed — sees `draining` // when it resumes and cannot create the work this shutdown is about to // wait for (#374). drain.draining = true; log("shutdown requested — draining dispatch; live worker sessions are not waited for"); pace.requestWake(); }; process.on("SIGINT", stop); process.on("SIGTERM", stop); // This loopback surface exposes liveness (open) and bounded live-run controls // (authenticated). Repository and tracker mutations stay on credentialled verb // sockets, where project and role come from the channel rather than a payload. // // The mutating half used to be open, on the reasoning that binding 127.0.0.1 // is itself the boundary. It is not, on a shared host: every local process has // the same loopback — including the worker sessions this daemon launches — so // anything running here could pause, stop or re-ceiling any run, or wake // dispatch in a loop. Phase 4 closes that with a bearer token minted here, // before the server exists, so no request can ever be served against a token // that had not been decided yet. `GET /healthz` deliberately stays open; the // reason is at {@link daemonHttpResponse} and it is load-bearing for // `requireDaemonControl`. const httpToken = ensureHttpToken(); const server = Bun.serve({ hostname: "127.0.0.1", port: o.port ?? DEFAULT_PORT, fetch: (req) => daemonHttpResponse(req, { token: httpToken, projects: runtimes.map((runtime) => ({ project: runtime.d.project.name, store, caps: () => runtime.d.caps, // The same project prefix the runtime's own journal lines carry, so // a pause audit reads like every other daemon line (#997). log: (line: string) => log(runtimes.length === 1 ? line : `[${runtime.d.project.name}] ${line}`), })), turnLimits, workerControls, wake: () => { log("dispatch wake requested — an immediate pass is starting"); pace.requestWake(); }, health: () => daemonHealth( runtimes.map((runtime) => { const orch = runtime.orchestrator; const mode = runtime.d.project.escalation.orchestrator; return daemonHealthSnapshot( store, runtime.d.project.name, isPaused(runtime.d.project.name), runtime.codeGraph, workerControls, orch === undefined ? // External mode is the pane's session: the daemon hosts no // child to attest. An embedded orchestrator that failed to // start is a live outage of the surface the upgrade must // promise reloaded (#832) — say so out loud. { mode: mode === "external" ? "external" : "failed" } : { mode: "embedded", ...(orch.extensionVersion() === undefined ? {} : { loaded: orch.extensionVersion() }), ...(orch.sessionFile() === undefined ? {} : { sessionFile: orch.sessionFile() }), alive: orch.alive(), }, ); }), ), }), }); log( projects.length === 1 ? `serving /healthz on :${server.port}, project ${projects[0]!.name}` : `serving /healthz on :${server.port}, projects ${projects.map(({ name }) => name).join(", ")}`, ); try { await runDispatchLoop({ tick: async () => { for (const runtime of runtimes) { try { await tick(runtime.d, workers); } catch (err) { log( `${projects.length === 1 ? "" : `[${runtime.d.project.name}] `}` + `tick failed: ${errText(err)}`, ); } } }, shouldStop: () => stopping, pace, tickIntervalMs: TICK_INTERVAL_MS, }); } finally { process.off("SIGINT", stop); process.off("SIGTERM", stop); clearInterval(graphTimer); clearInterval(reportTimer); for (const runtime of runtimes) await runtime.reportPass; await workers.drain(); await server.stop(true); for (const runtime of runtimes.toReversed()) { await runtime.orchestrator?.dispose(); await runtime.orchestratorVerbs?.close(); } store.close(); log("stopped"); process.exitCode = 0; } }