/** * The seam every dispatch pass is written against. * * `Deps` is the reason this directory can exist at all: a pass takes the one * struct and touches nothing else, so it can move to its own module without * learning about its neighbours. That also makes this module the test of * whether a cut is in the right place — a pass that needs something no `Deps` * field can reach was split along the wrong line. * * Three kinds of thing live here and nothing else: * * - `Deps` itself, the gate and registry interfaces it is made of, and the * intervals `runDaemon` schedules against. The gate types sit here rather * than beside their implementations so `Deps` never imports a pass, and no * pass imports another merely to name a field it already holds. * - `verbDeps` and `recordDaemonSessionSpend`: the two projections of `Deps` * that more than one pass performs. Duplicated they would be two answers to * one question; parked inside one pass they would make the others import it * sideways. * - the pure identity and classification leaves the passes share — `repoSlug`, * `knowledgeRepoKey`, `githubRepo`, `completionLastError`, * `exhaustedSessionReason`. A shared floor with no upstream of its own is * what keeps the rest of this directory acyclic. * * `PACKAGE_SRC_DIR` is the one declaration here that is not a move. Modules * under `daemon/` sit one directory deeper than the code they came from, so * every `import.meta.dir` read that meant "the installed `src/`" — the brief * templates, the integrity manifest root, the two lines that print the package * root — resolves through this constant rather than silently re-deriving a * path one level too deep. */ import { join } from "node:path"; import { findProject, loadConfig } from "../config.ts"; import type { CredentialClassProbeResult } from "../credential-class.ts"; import type { HerdrRun } from "../fleet.ts"; import { probeRunLane, readBaseChain, type CriticalBaseProbe, type RunLaneProbe } from "../gitops.ts"; import { errText, log } from "../log.ts"; import type { OrchestratorHandle } from "../orchestrator.ts"; import { isPaused, pausedAt } from "../pause.ts"; import type { UnroutableReason } from "../routing.ts"; import type { TelegramFreshness } from "../telegram-freshness.ts"; import type { Caps, Escalation, FailureClass, HostConstraints, IssueSnapshot, ProjectConfig, RepoTarget, RunRecord, Store, Tracker } from "../types.ts"; import type { InstalledSurfaces } from "../upgrade.ts"; import type { UsageSource } from "../usage.ts"; import { githubVerbActions } from "../verbs/actions.ts"; import type { VerbActions, VerbDeps } from "../verbs/server.ts"; import type { PeerReader } from "../verbs/socket.ts"; import type { AdjudicationResult, AdjudicatorOpts, RunWorkerDeps, ToSpecScoutOpts, ToSpecScoutResult, WorkerPauseControl, WorkerPausePhase } from "../worker.ts"; /** * The installed `src/` — the directory `daemon.ts` used to be `import.meta.dir` * of, before its passes moved one level down into `daemon/`. * * Three reads depend on that exact path and would break silently one directory * deeper: the brief templates under `src/briefs/`, the integrity manifest root * (which is the package's self-portrait, so a wrong root would hash a tenth of * it and call the rest unchanged), and the two operator-facing lines that print * the package root. One constant so all five sites keep the same answer. */ export const PACKAGE_SRC_DIR = join(import.meta.dir, ".."); /** Long enough that the tracker is not polled raw, short enough that a human * who labels an issue sees it picked up within a coffee break. */ export const TICK_INTERVAL_MS = 5 * 60_000; export const GRAPH_HEALTH_INTERVAL_MS = 60_000; /** Report delivery runs on its own timer, not the five-minute dispatch tick: a * report an operator is waiting on must not sit in the outbox for the length of * a poll interval, and delivery is owed even while claiming is paused (#123). */ export const REPORT_DELIVERY_INTERVAL_MS = 30_000; export const DEFAULT_PORT = 8787; /** Fleet-wide escalations still need an issue number in the payload; 0 is the * sentinel that reads as "no issue" in every renderer. */ export const NO_ISSUE = 0; export const UNROUTABLE_TEXT: Record = { "no-repo-label": "it carries no repo label", "multiple-repo-labels": "it carries more than one repo label", "unknown-repo": "its repo label maps to no configured repo", }; export interface DaemonOpts { once?: boolean; port?: number; project?: string; } /** Everything one tick touches. `project` and `caps` are re-resolved at each * tick boundary so an operator's config edit applies on the next tick rather * than the next daemon restart (#170); a tick and the runs it admits see one * consistent snapshot, and a mid-run edit never changes a live run's labels, * model or caps — `handleIssue` destructures them at dispatch time. The rest * are resolved once at startup so a tick never re-reads config mid-flight and * changes its own limits underneath itself. */ export interface DrainSignal { draining: boolean; } export interface Deps { project: ProjectConfig; caps: Caps; /** The typed host-constraints block (#721), re-resolved at the tick * boundary like `project`/`caps`, and rendered into every brief the tick * dispatches. Optional so tests that never exercise it can omit it. */ host?: HostConstraints; tracker: Tracker; store: Store; /** * Set true the moment a daemon stop (SIGTERM/SIGINT) begins. The run loop * only sees `stopping` between whole ticks, so a pass that was already in * flight when the stop landed must re-check this flag itself — before it * claims and before it launches — or it creates exactly the work the * shutdown is about to wait for and then lose to the stop timeout (#374). * Optional only so tests that never exercise shutdown can omit it; * `runDaemon` always wires the real one. */ drain?: DrainSignal; /** False after a live config reload fails; autonomous delivery then holds * fail-closed until a later tick validates the config again. */ deliveryPolicyValid?: boolean; /** Provider-reported plan allowance, cached with a TTL. Resolved once at * startup like every other dep so a tick cannot swap its own meter. */ usage: UsageSource; escalate(e: Escalation): Promise; turnLimits: TurnLimitRegistry; workerControls: WorkerControlRegistry; /** Session seam for lifecycle integration tests; production uses the real harness. */ workerDeps?: RunWorkerDeps; /** Harness per-process logs used to classify terminal worker routes. */ harnessLogDir?: string; /** * The Herdr runner for the worker-representation surface (#1035). Production * leaves it absent — the real `herdr` CLI answers; lifecycle tests inject a * fake so convergence is proven without a terminal. */ herdrRun?: HerdrRun; integrity: IntegrityGate; stall: StallGate; /** * Stops the wedged orchestrator by exact identity, for the bounded * auto-restart ({@link restartWedgedOrchestrator}). Production leaves it * absent and `stopConductorPane` answers — the same resolver `halt --pane` * uses, so there is exactly one implementation of "which pane is the * orchestrator". A test injects a fake, which is what makes the counter, the * bound and the record-before-attempt ordering assertable without a terminal. */ restartOrchestrator?: ( projectName: string, ) => Promise<{ stopped: "herdr-agent" | "already-gone"; detail: string; agentName: string }>; /** * The embedded orchestrator session handle when one started; absent when it * failed to start or the project uses an external orchestrator. Feeds the * orchestrator-down reconcile ({@link reconcileOrchestratorDown}) so a * crashed session pages once per incident instead of degrading quietly. */ orchestrator?: OrchestratorHandle; cleanup?: RetainedCleanupCursor; /** * Reads the connecting uid off a verb socket (#126). Resolved once at startup * so the mechanism is logged before the first socket exists; absent on a host * exposing no peer-credential call, where the `0600` socket under the * daemon-owned `0711` parent is all that keeps other local accounts out, and * `status` says so. It never told two sessions apart — they share the daemon's * uid. */ verbPeerReader?: PeerReader; /** * The privileged half the verbs call once their checks pass. Optional only so * a test can build a `Deps` without a repository; `runDaemon` always wires * the real one. */ verbActions?: VerbActions; /** * The fleet-installs-itself verification pass (#486), run on the first tick * after a detached upgrade. The default is the journal-gated * {@link verifyPendingUpgradeTick}; a test injects its own to drive the tick * without touching the host's packages, journal or outbox. */ upgradeVerifier?: (d: Deps) => Promise; /** * Answers whether a preserved continuation branch contains every configured * critical-base marker, for the stale-base admission hold (#428). Wired by * `runDaemon` to the mirror-backed {@link probeCriticalBase}; a test injects * a fake. Absent, the stale-base gate fails closed whenever a project names * a marker (a safety interlock must not silently weaken). */ probeCriticalBase?: CriticalBaseProbe; /** * Reads one active run's file lane for the admission file-lane interlock * (#555): the union of its uncommitted worktree changes and its branch-vs-base * diff. Reconciliation is not occupancy: while the run merges its base, the * probe reports only files it has actually diverged on, never the files the * merge merely staged (#684). Wired by `runDaemon` to the mirror/worktree-backed * {@link probeRunLane}; a test injects a fake. Absent, the interlock is inert * (no lane is ever known occupied), which is the issue's "fail open": the * gate adds holds, it never refuses a well-formed issue for lack of this * probe the way `criticalBase` does. */ probeWorktreeLane?: RunLaneProbe; /** * Reads one provider's current credential class from the harness, out of * process (#852). Wired by `runDaemon` to {@link probeCredentialClass}; a test * injects a fake. * * Absent, both fences refuse a project that declares `requireOauthProviders` — * this one fails CLOSED, unlike `probeWorktreeLane`, because an unverified * credential class costs exactly what a wrong one costs. A project declaring * nothing never calls it, so an unwired test dispatches as it always has. */ probeCredentialClass?: (provider: string) => Promise; /** * Reads one issue's tracker state in a repository the admission tracker is * not bound to — the cross-repo Depends-on interlock (#420). Wired by * `runDaemon` to a repo-scoped tracker; a test injects a fake. Absent, * admission fails a routed cross-repo prerequisite closed. */ probeIssueIn?: (repo: string, issue: number) => Promise; /** * Reads one issue's BODY in a repository the admission tracker is not bound * to — the dependency-graph cycle pass (#421). Wired by `runDaemon` to a * repo-scoped tracker; a test injects a fake. Absent, a routed reachable * body fails that branch closed rather than synthesising a cycle. */ probeBodyIn?: (repo: string, issue: number) => Promise; /** * Reads the three install identities this host carries (#919). Wired by * `runDaemon` to `inspectSurfaces`, the same seam `doctor` and `upgrade` * read, so nothing re-implements the probe. A test injects its own; absent, * the pass records nothing and every cheap surface honestly says "not * observed yet" rather than claiming agreement. */ probeInstallSurfaces?: () => Promise; /** The `omp-telegram` install/daemon/published triple, read on the same * periodic pass as the surfaces above (#961). */ probeTelegramFreshness?: () => Promise; /** * Runs one review-ceiling adjudication (#932). Production is * {@link runAdjudicator}; a test injects a fake so the assertion can be the * launch arguments and the assembled brief — which is what this pass actually * produces — rather than a stored flag. */ runAdjudicatorImpl?: (opts: AdjudicatorOpts) => Promise; /** * Runs one to-spec grooming pass (#1041). Production is * {@link runToSpecScout}; a test injects a fake, which is what makes the * whole mechanical pipeline — select, claim, persist, gate, promote, wake — * assertable end to end without a session or a network. */ runToSpecScoutImpl?: (opts: ToSpecScoutOpts) => Promise; /** * The dispatch wake fired after a queue-label hand-back or a mechanical * promotion (#1041) — the same seam `settlement.ts` reads, so one injected * spy covers every requeue path a tick runs. Production leaves it absent and * the real loopback client answers: from inside the daemon that is a POST to * its own `/wake`, which is deliberate rather than a shortcut — the wake is * one coalescing flag with one owner, and a second in-process path to it * would be a second answer to "is a pass pending". */ wake?: (projectName: string) => Promise; } /** * The daemon-side view a verb call decides against (#126). * * `project` is a thunk, not the resolved value the rest of a tick uses. That is * the whole "fail closed on an unreadable config" rule: a config read once at * boot can never become unreadable, and it cannot pick up an operator who has * just taken merge authority back either. Every verb pays one file read for * the property that its answer reflects the config as it is *now*. * * `fleetStop` is read the same way and for the same reason — inside the verb, * after the model decided to call it. `hold` and `halt` both set the pause * sentinel alongside disarming ticks, so this one read covers both. */ export function verbDeps(d: Pick): VerbDeps { return { project: () => findProject(loadConfig(), d.project.name), projects: () => loadConfig().projects, store: d.store, tracker: d.tracker, actions: d.verbActions ?? githubVerbActions(d.project), fleetStop: () => isPaused(d.project.name) ? "claiming is paused for this fleet (omp-conductor hold or stop)" : undefined, pausedAt: () => pausedAt(d.project.name), log, now: () => Date.now(), chain: { readBaseChain }, lane: { probeRunLane }, }; } /** * Whether the wedged-orchestrator page has gone out, and when it last went out, * for the stall currently on disk. */ export interface StallGate { paged: boolean; lastPagedAt?: number; } /** * What the daemon booted with, and whether it has already paged about losing * it. Lives exactly as long as one `runDaemon()` call — which is the whole * trick: a restart re-records both. */ export interface IntegrityGate { baseline: Map; paged: boolean; } /** * `owner/repo` for `gh`, derived from the clone URL. * * ponytail: RepoTarget has no explicit slug, so it is parsed off the URL and * falls back to the routing name. Upgrade path is an optional `slug` field once * a non-GitHub remote actually shows up. */ export function repoSlug(repo: RepoTarget): string { const m = /(?:[:/])([^/:]+\/[^/]+?)(?:\.git)?$/.exec(repo.cloneUrl); return m?.[1] ?? repo.name; } /** * The identity the per-repo knowledge overlay is filed under: the canonical * `owner/repo` when the clone URL names one, else the routed name. * * Deliberately NOT {@link repoSlug}: that one is a loose parse whose job is to * *say something* in a brief for any URL shape, so a filesystem path yields a * plausible-looking pair of directory names. A knowledge file keyed off that * would be keyed off where a mirror happens to live. One function, used by both * the writer at settlement and the reader in the brief, so the two can never * file and read under different names. */ export function knowledgeRepoKey(repo: RepoTarget): string { return githubRepo(repo.cloneUrl) ?? repo.name; } export type ExtendTurnLimitResult = | { kind: "extended"; runId: string; maxTurns: number } | { kind: "not-increase"; runId: string; maxTurns: number } | { kind: "not-active" }; export interface TurnLimitController { maxTurns(): number; close(): void; } export interface TurnLimitRegistry { open(project: string, issue: number, runId: string, maxTurns: number): TurnLimitController; extend(project: string, issue: number, maxTurns: number): ExtendTurnLimitResult; } export type WorkerControlResult = | { kind: "ok"; runId: string; phase: WorkerPausePhase } | { kind: "stopped"; runId: string; reason: string } | { kind: "refused"; runId: string; error: string } | { kind: "not-active" }; export interface WorkerControlSlot { install(control: WorkerPauseControl): void; /** Stop accepted before the session controller exists. */ requestedStop(): string | undefined; close(): void; } export interface WorkerControlRegistry { /** * `onPhase` is called after a pause/resume/stop that actually changed this * worker's phase (#842) — the authoritative transition, from the same control * that performed it. It exists so a surface outside this registry (the run's * Herdr representation) can follow the phase without polling and without * inventing a second notion of "paused". */ open( project: string, issue: number, runId: string, onPhase?: (phase: WorkerPausePhase) => void, ): WorkerControlSlot; /** `source` names who asked (board/cli/dashboard) — recorded as pause * provenance so "who paused this and when" is answerable later (#997). */ pause(project: string, issue: number, source: string): Promise; resume(project: string, issue: number): WorkerControlResult; stop(project: string, issue: number, reason: string): Promise; /** Live runs whose phase is not `running` — what /healthz and the board * show — each carrying its pause provenance when one was recorded (#997). */ snapshot(project: string): { issue: number; runId: string; phase: WorkerPausePhase; source?: string; pausedAtMs?: number; }[]; } // =============================================== daemon-owned session accounting // (Phase 4, attribution.) Grooming and adjudication are real sessions the daemon // launches on its own initiative: they resolve a model, take turns, and cost // money. Until now their `{turns, spendUsd}` was printed to the journal and // thrown away, so the only spend conductor could attribute was worker spend, and // every question of the form "what did the pipeline cost, not just the workers" // had no answer at all. // // They are NOT run rows. `computeStats` walks `runs` as issue journeys keyed on // `run.issue`, deciding merged/failed/settled from the sequence of states for // one issue — a groom row landing in that table would be counted as an attempt // on the issue and would corrupt the merge counts it feeds. Hence a dedicated // role-tagged table (Lane A's `recordSessionSpend`). /** * Persist one daemon-owned session's accounting, telling unmetered from free. * * The honesty rule this exists to enforce: the harness reports `spendUsd: 0` * both for a session that genuinely cost nothing and for one whose provider * reports no per-request price at all (a subscription credential). Writing 0 for * the second would render as "$0.00" and read as free, which is exactly the * mistake `spend-telemetry.ts` was written to stop — "$0.00 spend is not proof * of no spend" (#46, #984). * * So the discrimination is the same one {@link judgeSpendTelemetry} makes: a * session that took turns is a *working* session, and a working session * reporting zero is UNMETERED — the column is omitted and the stats lane renders * it as unknown. A session that took no turns did not run (an unresolvable role, * a harness that would not start), and zero is then the true cost. * * Recorded before any of the caller's verdict branching, so every early return * still accounts for the money that was already spent. */ export function recordDaemonSessionSpend( d: Pick, row: { role: "groom" | "adjudicator"; issue?: number; /** The role/model pattern conductor asked for. */ model?: string; /** What actually ran it, when the session reported back. */ resolvedModel?: string; turns: number; spendUsd: number; at: number; }, ): void { const metered = row.turns === 0 || row.spendUsd > 0; try { d.store.recordSessionSpend({ project: d.project.name, role: row.role, ...(row.issue === undefined ? {} : { issue: row.issue }), ...(row.model === undefined ? {} : { model: row.model }), ...(row.resolvedModel === undefined ? {} : { resolvedModel: row.resolvedModel }), turns: row.turns, ...(metered ? { spendUsd: row.spendUsd } : {}), at: row.at, }); } catch (err) { // Accounting must never lose the session it accounts for: the verdict this // run produced is worth more than its price tag. log(`${row.role} session spend not recorded: ${errText(err)}`); } } /** * The terminal classes that mean "the transcript this round would resume * cannot usefully take another turn". * * Enumerated rather than inferred: each of these is a *cap or a provider * refusal*, not a logic failure, so resuming buys a session that dies the same * way. The value is the clause the log line and the correction brief read, so * the reason a fresh session exists is never "the daemon decided so". */ export const EXHAUSTED_SESSION_CLASSES: Partial> = { "turn-cap-progress": "it was killed by the turns cap", "turn-cap-spinning": "it was killed by the turns cap while spinning", "wall-clock-cap-progress": "it was killed by the wall-clock cap", "wall-clock-cap-spinning": "it was killed by the wall-clock cap while spinning", "model-empty-stop": "the provider returned empty turns until the harness retry cap ended it", }; /** * Why the session recorded on this run cannot be resumed, or `undefined` when * it can (#1047). * * Two facts, both durable on the row and both readable without opening a * transcript: * * - the row's own terminal classification, when it names a cap or the * provider empty-stop; and * - the turn ledger: a session at or past its recorded ceiling has no turns * left whatever ended it. This is the fact that catches the rounds #1035 * actually burnt — the review path raises the ceiling by one round's * allowance per round (#747), so a round that spends its whole allowance * leaves `turns >= maxTurns` on the row even after the claim has cleared * the classification. * * Shared with the answered-block continuation path, which needs the same * question answered about the same kind of row: resuming an exhausted session * is exactly as useless there. */ export function exhaustedSessionReason( run: Pick, ): string | undefined { const classed = run.failureClass === undefined ? undefined : EXHAUSTED_SESSION_CLASSES[run.failureClass]; if (classed !== undefined) return `${classed} (${run.failureClass})`; if (run.turns >= run.maxTurns) { return `it has spent its whole turn ceiling (${run.turns}/${run.maxTurns} turns)`; } return undefined; } /** Canonical `owner/repo` identity from a configured network clone URL. */ export function githubRepo(cloneUrl: string): string | undefined { const normalized = cloneUrl.replace(/\/$/, "").replace(/\.git$/, ""); const match = /^(?:https?:\/\/[^/]+\/|ssh:\/\/git@[^/]+\/|git@[^:]+:)([^/\s]+\/[^/\s]+)$/.exec( normalized, ); return match?.[1]; } export interface RetainedCleanupCursor { next: number; } /** A provider refusal a session recorded before dying, or undefined for none. */ export interface SessionError { status?: number; message: string; } export function completionLastError( providerCredit: string | undefined, providerTransient: string | undefined, verifiedReason: string | undefined, sessionErr: SessionError | undefined, ): string | undefined { return providerCredit ?? providerTransient ?? verifiedReason ?? sessionErr?.message; }