import type { WorkflowRun, WorkflowRunStore, WorkflowRunStatus, WorkflowRunSummary } from "@sema-agent/core"; import { type FleetEventBus } from "../fleet/fleet-bus.js"; import { type WorkflowAgentSessionIndex } from "./workflow-agent-session-index.js"; /** A workflow completion notification's terminal payload — the bounded, redacted shape core's notifier seam * carries, re-derivable from a durable {@link import("@sema-agent/core").WorkflowRun} on the recovery path. */ export interface WorkflowCompletionPayload { runId: string; status: "completed" | "failed"; /** Redacted + length-bounded — never host-internal paths/tokens (parity with core's `boundedSummary`). */ summary: string; sourceTaskId?: string; principal?: string; /** core 1.208: the ORIGINATING session id, closed over by the Runner at RunWorkflow mount time — * the lookup-free routing key for the completion inbox. Present on the LIVE notify path; ABSENT on the * crash-recovery path (the durable `WorkflowRun` does not carry it — relayed to core to thread it there too, * parity with `sourceTaskId`'s 1.187 story), where the delivery falls back to the `sourceTaskId` run-row lookup. */ originatingSessionId?: string; } /** The downstream delivery the journal guards (the deployment's real completion route — log+meter today, a full * re-invoke into the originator's stream/inbox tomorrow). Idempotent on `runId` by contract; the journal still * de-dups so it is invoked AT MOST once per run in steady state and at most one extra time across a crash. */ export type WorkflowCompletionDelivery = (p: WorkflowCompletionPayload) => Promise | void; /** One journal entry: the durable to-do record for a single run's completion notify. `acked` flips true ONLY * after delivery succeeds (deliver-then-ack — a crash mid-delivery re-delivers on recovery). */ export interface WorkflowNotifyJournalEntry { runId: string; /** The run's scope (= creating principal) — needed to re-fetch / owner-attribute on recovery. */ scope: string; /** Whether the completion has been DELIVERED to the downstream sink (and may be GC'd). */ acked: boolean; /** The originating task id, threaded to the delivery (parity with core's notifier input). */ sourceTaskId?: string; /** The originating principal, threaded to the delivery. */ principal?: string; /** When the run was journaled (epoch ms) — for retention + observability. */ createdAt: number; /** When the entry was acked (epoch ms) — set with `acked`. */ ackedAt?: number; } /** * The pluggable durable seam for the notify journal. Intentionally tiny: record a started run, mark it acked, * and enumerate the un-acked backlog the recovery sweep walks. `listPending` is the one method the * `WorkflowRunStore` cannot provide (it has no cross-scope enumeration) and is the whole reason the journal is a * separate store. A backend (the File default, or a follow-on TiDB/PG) implements this; the gate logic in * {@link WorkflowNotifyGate} is backend-agnostic. */ export interface WorkflowNotifyJournalStore { /** Record a STARTED run as pending-notify. Idempotent on `runId`: a second `record` for the same run is a * no-op (it must NOT reset an already-`acked` entry back to pending — that would re-deliver forever). */ record(entry: { runId: string; scope: string; sourceTaskId?: string; principal?: string; createdAt: number; }): Promise; /** Mark a run's completion DELIVERED. Idempotent: acking an already-acked or missing run is a no-op. */ ack(runId: string, ackedAt: number): Promise; /** Every NOT-`acked` entry — the recovery backlog. Order is unspecified (the sweep handles each independently). */ listPending(): Promise; /** Read one entry (`null` when absent) — lets the gate check `acked` before a live delivery (the dedup). */ get(runId: string): Promise; } export declare function recoverySummary(status: WorkflowRunStatus, error?: string): string; /** * The AT-LEAST-ONCE completion-notify gate. Wrap the deployment's real delivery once at boot, then: * - {@link buildNotifier} hands core a {@link import("@sema-agent/core").WorkflowCompletionNotifier} whose `notify` * runs the LIVE deliver-then-ack path (the in-process at-most-once notify, now journaled into at-least-once). * - {@link onWorkflowStart} journals a run the moment it starts (so a crash BEFORE its terminal notify still * leaves a pending entry the recovery sweep finds). Call it from wherever the runId becomes known. * - {@link recover} runs ONCE at boot: re-derives every pending run's terminal state from the * {@link WorkflowRunStore} and re-delivers + acks the ones that are terminal. * * The gate is the SINGLE writer of `acked`, and dedups every delivery against the journal, so the wrapped * delivery sink is invoked at most once per run in steady state (and at most one extra time across a crash — * the at-least-once / idempotent contract). */ export declare class WorkflowNotifyGate { private readonly journal; private readonly runStore; private readonly deliver; private readonly opts; constructor(journal: WorkflowNotifyJournalStore, runStore: WorkflowRunStore, deliver: WorkflowCompletionDelivery, opts?: { now?: () => number; /** Surfaced on a failure (so a transient downstream/journal error is observable, NOT swallowed silently — * it stays pending + retries on the next recovery sweep). Never throws back into core's notify. `record` = * the start-time journal write failed; `deliver` = the downstream delivery threw; `recover` = a per-entry * recovery-sweep error. */ onError?: (stage: "record" | "deliver" | "recover", runId: string, err: unknown) => void; }); /** * The runIds whose journal entry THIS process recorded — the boot-orphan judgment's authoritative anchor * (round-1 review, MEDIUM). A wall-clock cutoff alone is not sound: after a BACKWARD clock step the entries * this process records land BELOW the boot cutoff, and the sweep would finalize runs whose in-process * executor is alive and running (a false `failed` notify for a workflow that then keeps going — worse than * the absent row it was fixing). Membership here is a fact about THIS incarnation, unforgeable by any clock. * Retired on ack AND the moment a sweep observes the run terminal (the anchor only ever gates the `running` * arm), so a delivery/ack outage cannot pile up entries for runs that are no longer executing. */ private readonly recordedThisIncarnation; private now; /** Record a journal entry AND remember that this incarnation is the one that recorded it (see the field). */ private recordPending; /** Ack an entry + drop its incarnation mark (acked ⇒ never scanned again, so the mark has no further use). */ private ackDelivered; /** * Journal a STARTED run as pending-notify. Call this with the synchronous `runId` from `startWorkflow` / * `run_workflow` — BEFORE the workflow can reach terminal — so a crash mid-run still leaves a recoverable * to-do. Best-effort + idempotent: a journal throw is reported, never propagated (it must not break the * workflow start; worst case the recovery sweep won't know about this run — degraded, not corrupting). */ onWorkflowStart(input: { runId: string; scope: string; sourceTaskId?: string; principal?: string; }): Promise; /** * The core-facing notifier. core calls this exactly once per run on terminal (its own at-most-once, runId-dedup * `fire`). We turn it into at-least-once: ensure the run is journaled (the start hook may have been skipped), * then deliver-then-ack. Idempotent: if the run is ALREADY acked (a recovery sweep beat us, or a duplicate * core fire), skip — at most one delivery. */ buildNotifier(): { notify: (input: WorkflowCompletionPayload & { scope?: string; }) => Promise; }; /** * Deliver a run's completion exactly once (idempotent on the journal). Ensures the entry exists (record is a * no-op if it already does), SKIPS if already acked, otherwise delivers THEN acks (a crash between deliver and * ack re-delivers on recovery — at-least-once). A delivery throw is reported and re-thrown to the LIVE caller * only as a swallowed report (core's `fire` already `.catch`es), leaving the entry pending for the next sweep. */ private deliverOnce; /** * RECOVERY sweep — run at boot AND PERIODICALLY (wired into the service reaper), BEFORE/while serving traffic. * For every pending (not-acked) journal entry, re-fetch the run from the durable {@link WorkflowRunStore}: * - TERMINAL (completed/failed): its in-process notify was lost to a crash (or it finished while we were * down) → re-deliver + ack (idempotent on the receiver). * - `running` but STALE (run.startedAt older than `orphanGraceMs`): a crash-orphaned run — core never resumes * a prior `running` row (resume mints a NEW runId) and never reaps a `running` row, so NOTHING will ever * flip it terminal. Finalize-as-ABANDONED: deliver a `failed` completion (so the originator learns the * workflow died) + ack, instead of leaking the entry forever. This closes the exact crash topology SVC-1 * exists for (a replica SIGKILLed mid-run). * - `running` and FRESH (within the grace window): genuinely in flight → leave pending (the owning process * delivers its terminal notify, or the next sweep catches it once it goes terminal or stale). The sweep * publishes NOTHING here — see the single-writer invariant below. * - MISSING (reaped / never persisted): ack-as-abandoned so the journal doesn't chase a ghost forever. * `orphanGraceMs` MUST exceed the max expected workflow runtime (the run store has no cross-replica liveness * signal, so age is the only orphan proxy). Returns a tally. A per-entry throw is isolated so one bad entry * can't abort the sweep. * * BOTH abandoned arms (boot-orphan + stale-past-grace) flip the DURABLE row to `failed` FIRST (CAS on rev), * so the run store, the fleet panel and the delivered notify give ONE answer — previously the notify said * `failed` while `/workflows` kept saying `running` forever ([ref]: republishing without finalizing would * have turned "panel empty" into "panel shows a row that never moves"). A lost CAS means the run moved under * the sweep (e.g. its real terminal landed concurrently) — skip this pass; the entry stays pending and the * next sweep handles the NEW state. * * 🔴 SINGLE-WRITER INVARIANT for the fleet row (round-1 review, two HIGH findings): the sweep only ever * publishes a **TERMINAL** fleet frame (the flip above, via `publishTerminalFleetRow` → final frame + remove). * It NEVER publishes a `running` row. Two reasons, both "a row we mint here can become one nobody can retire": * 1. cross-replica (SQL journal): a pending `running` entry may belong to ANOTHER replica. Its terminal * update lands on that replica's own (replica-local) fleet bus, and its ack removes the entry from the * shared journal — so this replica would never see the run again and its minted row would sit `running` * forever. (The terminal-redelivery arm deliberately doesn't publish either — it is a notify path.) * 2. same-replica: `runStore.get` is a READ-TIME SNAPSHOT. If the live run commits its terminal (and the * wrapper removes the row) between that read and the publish, a `running` republish resurrects a row * that will never be removed again. * A live row's ONE writer is this replica's `put`/`update` observation point ({@link JournalingWorkflowRunStore}), * which by construction sees every transition including the terminal one. */ recover(opts?: { orphanGraceMs?: number; /** Replica-local (File/in-memory) journal ONLY: a pending entry recorded BEFORE this timestamp belongs to * a previous incarnation of THIS process — workflow executors are in-process, so they died with it and a * still-`running` run is a boot-orphan: finalize it NOW instead of leaving the user's "Waiting for * workflow" pointed at a run that can never finish for the whole grace window ([ref] 顺带①/[ref]§二). * NEVER pass this for a cross-replica (SQL) journal — there a pending running run may be genuinely alive * on another replica, and age (`orphanGraceMs`) is the only sound orphan proxy. * ⚠️ This wall-clock cutoff is a NECESSARY, not sufficient, condition — {@link recordedThisIncarnation} * is the authoritative one (a clock rollback after boot would otherwise stamp THIS process's own new * entries below the cutoff and finalize live runs). */ finalizeStartedBeforeMs?: number; /** Retire a fleet row for a run this sweep just finalized ([ref]/[ref]): called ONLY with a TERMINAL run, * so the {@link JournalingWorkflowRunStore} projection publishes the final frame and then removes the row — * the store, the panel and the notify end up saying the same thing. Never called with a `running` run (see * the single-writer invariant above). Late-bound because the wrapper is constructed after this gate. */ publishTerminalFleetRow?: (id: string, run: WorkflowRun) => void; }): Promise<{ scanned: number; redelivered: number; stillRunning: number; abandoned: number; }>; } /** * Default zero-dependency, crash-safe {@link WorkflowNotifyJournalStore}. An append-only JSONL ledger replayed * at boot (last-writer-wins per `runId`), the same durable posture as core's `FileWorkflowRunStore` — fsync'd * appends, atomic single-line records. Single-instance / TOC-local: a cross-replica TiDB/PG port is a follow-on * (the recovery sweep + gate are backend-agnostic). * * The ledger only ever grows by one line per `record`/`ack`; an acked entry's line is not deleted (the replay * folds it), so retention is a future compaction concern — the backlog `listPending` walks is bounded by * IN-FLIGHT + un-swept runs, not history. */ export declare class FileWorkflowNotifyJournalStore implements WorkflowNotifyJournalStore { private readonly fsyncEnabled; private readonly ledgerPath; private readonly entries; private fd; constructor(root: string, fsyncEnabled?: boolean); /** Replay the ledger into the authoritative map (last-writer-wins by line order). Tolerates a torn final line * (a crash mid-append) by dropping an unparsable trailing record — the same crash-safety FileWorkflowRunStore * relies on (a partial append is simply not yet committed state). */ private replay; /** Commit one entry: append the JSON line (fsync) THEN flip the in-memory map (crash-safe ordering — the * durable record lands before the in-memory state the caller observes). */ private commit; record(input: { runId: string; scope: string; sourceTaskId?: string; principal?: string; createdAt: number; }): Promise; ack(runId: string, ackedAt: number): Promise; listPending(): Promise; get(runId: string): Promise; /** Release the append handle (best-effort) — called on shutdown. */ close(): void; /** Test/inspection: total entries (acked + pending). */ get size(): number; } /** In-process {@link WorkflowNotifyJournalStore} — tests / single-instance ephemeral. Does NOT survive a * restart (so it provides NO crash-recovery on its own); use {@link FileWorkflowNotifyJournalStore} or a * durable backend for the real at-least-once guarantee. Useful to unit-test the gate's live + idempotency * paths without touching disk, and to simulate a crash by handing a FRESH store to a new gate. */ export declare class InMemoryWorkflowNotifyJournalStore implements WorkflowNotifyJournalStore { private readonly entries; record(input: { runId: string; scope: string; sourceTaskId?: string; principal?: string; createdAt: number; }): Promise; ack(runId: string, ackedAt: number): Promise; listPending(): Promise; get(runId: string): Promise; get size(): number; } /** * A {@link WorkflowRunStore} DECORATOR that journals a run the instant core persists it at START. core's * `startWorkflow` `put`s the run row (`status: "running"`) synchronously before the body runs, carrying both the * `runId` and the `scope` — the one synchronous start-time hook the service gets WITHOUT a new core seam (core * exposes no `onWorkflowStart`). Wrapping the run store here observes that `put`, journals the started run as * pending-notify, then delegates to the wrapped store. Every other method is a pure pass-through (the run store * stays core's source of truth for `/workflows`). This is the [[core-service-boundary]]-clean wiring: reuse the * core store seam, observe writes through it, never fork a parallel run model. * * Journaling on `put` (not on notify) is what makes recovery cover a crash BEFORE terminal: the entry exists the * moment the run starts, so the boot sweep finds it even if the process died mid-run. */ export declare class JournalingWorkflowRunStore implements WorkflowRunStore { private readonly inner; /** [ref] optional since the fleet decoupling: undefined ⇒ no start-time journal (ephemeral memory * backend has nothing to recover), the FLEET half still publishes — the panel is replica-local UI * state and must not be gated on the durable-notify machinery (the regression that emptied clay's * workflow panel on memory-backend deployments). */ private readonly gate?; /** MF-Fleet (data contract): publish the workflow's fleet row on every put/update (the same write-observation * point as the notify journal). Optional — undefined ⇒ no fleet publish. */ private readonly fleetBus?; /** S-185 车CM:workflow 出身 park 的 join 索引。**本装饰器是它全仓唯一的写者** —— 索引是 `WorkflowRun` * 的派生投影,写点必须与 fleet 行投影/notify journal 同一个(见 index 模块的文件头「单写者」段)。 * 可选:缺席 ⇒ 一位不维护(与 fleet 半场同姿势,行为逐字不变)。 */ private readonly agentSessionIndex?; constructor(inner: WorkflowRunStore, /** [ref] optional since the fleet decoupling: undefined ⇒ no start-time journal (ephemeral memory * backend has nothing to recover), the FLEET half still publishes — the panel is replica-local UI * state and must not be gated on the durable-notify machinery (the regression that emptied clay's * workflow panel on memory-backend deployments). */ gate?: Pick | undefined, /** MF-Fleet (data contract): publish the workflow's fleet row on every put/update (the same write-observation * point as the notify journal). Optional — undefined ⇒ no fleet publish. */ fleetBus?: FleetEventBus | undefined, /** S-185 车CM:workflow 出身 park 的 join 索引。**本装饰器是它全仓唯一的写者** —— 索引是 `WorkflowRun` * 的派生投影,写点必须与 fleet 行投影/notify journal 同一个(见 index 模块的文件头「单写者」段)。 * 可选:缺席 ⇒ 一位不维护(与 fleet 半场同姿势,行为逐字不变)。 */ agentSessionIndex?: WorkflowAgentSessionIndex | undefined); /** 上一次**本进程**为某条 run 写进索引的行集 —— 变更检测,不是缓存。 * * 为什么需要:workflow 的进度写是每只 agent 状态变化一次,而 park 是罕见事件 ⇒ 无条件同步会给 * 「一条 park 都没有」的绝大多数 run 每次写多打一发删除(SQL)或多追加一行账本(file,无界增长)。 * 判据是**行集是否变化**,与后端无关 ⇒ 规则只有一条、两个后端同享。 * 跨副本安全性:一条 run 的写者按 core 契约唯一(`store` 的 owner),而同步只发生在一次**成功 CAS 之后**; * 失败切换后新 owner 的表是空的 ⇒ 它的第一次写必落盘,收敛。 * * 🔴 **空集也记**(7.69.0 合并重扫 c0):此前成功臂写作「空集 delete、非空集 set」,于是「一条 park * 都没有」的那条 run —— 也就是上面这段头注**点名要短路的绝大多数形** —— memo 恒回 `undefined`、短路 * 判据恒假,每一次 put/update 都重跑一遍 `syncRun(id, [])`(file 车道每次多一行账本 + 一次 fsync,而 * 这本账本按设计没有压缩腿;SQL 车道每次多一发 `DELETE`)。代码与自己成文的判据相反。 * 当时那条 delete 臂承担的是**有界性**:memo 若无条件记,表会随本进程见过的每一条 run 单调增长。 * 现在有界性由下面那条**终态驱逐**独立承担(run 终态 ⇒ 它不会再被写 ⇒ 摘表),两件事各归各位: * 记不记 = 变更检测的正确性;摘不摘 = 表的有界性。**不**加「首次空集跳过」这类依赖进程内历史的特判。 */ private readonly indexedRows; /** * 写腿的**不变式守卫**(codex r1 [medium],验真后修):`indexedRows` 只保留一次**我们亲眼看着落地** * 的写。持久化抛错时这条 run 的盘上状态未知(而 core 会吞掉这次失败、这条 run 往往再也不被写), * 若把 memo 留着,那一条 `{runId → rows}` 就永久留在表里 —— 终态驱逐等不到它的终态写。 * * ⚠️ 如实登记的残余:一条**最后一次写成功、随后被放弃**的 run(进程崩前没跑到终态)仍会在本进程的 * memo 里留一条空集。它与 fleet 活行登记簿同一量级(都随本世进程消亡),刻意**不**为它加清扫腿: * 那是给一个不产生错误答案的形加机制。 */ private forgetIndexMemoIfWriteOutcomeUnknown; /** 索引的一次收敛(put/update 成功后调用)。 * * **自吞错**:索引是观察面的派生物,它的故障绝不能把一次已成功的 run 持久化变成失败 —— 与 * `publishFleet` 的姿态同源。但自吞 ≠ 静默([ref]):留痕走**登记过的** fail-open tag * `server.workflow-park-index.sync-failed`(P-DEBT;射程、后果与欠账的唯一属主 = 那条词表项的 * `note`,census 里有对应行)。 */ private syncAgentSessionIndex; /** Derive + publish the MF-Fleet workflow row from a WorkflowRun (doneCount/totalCount from agents, tokens from * stats); a terminal workflow LEAVES the fleet (publish terminal then remove), a running one stays/updates. */ private publishFleet; /** [ref] recovery projection seam: the same fleet projection as the put/update observation points, exposed so * the recovery sweep can RETIRE a row for a run it just finalized (terminal input ⇒ final frame, then remove). * Wire it as `recover`'s `publishTerminalFleetRow` hook — and only ever hand it a TERMINAL run: a `running` * row published from outside the wrapper's own write path has no guaranteed retirement (see the invariant on * {@link WorkflowNotifyGate.recover}). */ republishFleet(id: string, run: WorkflowRun): void; put(id: string, run: WorkflowRun): Promise; update(id: string, scope: string, run: WorkflowRun, expect?: { rev: number; }): Promise; get(id: string): Promise; listByScope(scope: string, opts?: { status?: WorkflowRunStatus; limit?: number; }): Promise; reap(scope: string, now: number, opts?: { maxAgeMs?: number; keep?: number; }): Promise; } //# sourceMappingURL=workflow-notify-journal.d.ts.map