/** * Async-promote queue worker supervisor. * * PR #3 of the async-promote-queue series. PR #1 shipped the queue * library; PR #2 exposed it to the agent + HTTP routes. This module * drains the queue: N worker loops, each polling `claimNext` on a * short interval, calling `agent.assertion.promote(...)` on the * claimed job, then recording success or classified failure back into * the queue. * * Open questions from the plan §10 are resolved here: * * 1. **Worker model**: in-process `setInterval(claimNext, pollIntervalMs)` * × `workerConcurrency` instances. AsyncLift's on-demand * `processNext` model is rejected because the promote queue has no * external trigger — the daemon owns its own clock. * * 2. **Backoff curve**: defined in `async-promote-queue-utils.ts`, * inherited by the queue itself; the worker just calls `fail()` with * a classification and the queue handles backoff bookkeeping. * * 3. **Error classification**: see `classifyPromoteError` below. * Seeded from the rc.10 Graphify import patterns (see * `INTEGRATION_NOTES_GRAPHIFY.md` and `dkg-graphify-rc10-test/FINDINGS_v2.md`). * * 4. **Telemetry**: the supervisor emits `memoryGraphChanged` on every * `succeeded` transition that promoted >0 triples, mirroring the * sync `/promote` route. State transitions to `queued`/`running`/ * `failed_retrying` are NOT emitted — they're queue internals. * * Shutdown semantics: RFC §6.2 says do NOT mark `running → queued` * on shutdown. We stop polling, let in-flight jobs complete (or * timeout), and rely on `recoverOnStartup()` at the next boot to * decide what to do with any leases the old worker held. */ import type { DKGAgent } from '@origintrail-official/dkg-agent'; import { type AsyncPromoteQueue, type PromoteFailureClassification, type PromoteJob, type PromoteRequest } from '@origintrail-official/dkg-publisher'; /** * Convenience type for the daemon's existing `emitMemoryGraphChanged` * callback. Kept local so this module doesn't have to import the * `MemoryGraphChangedEvent` shape from `routes/context.ts` and pull in * its full route surface. */ export interface PromoteMemoryGraphChangedEvent { contextGraphId: string; layers: ('wm' | 'swm')[]; subGraphName?: string; operation: string; source: string; counts?: { triples?: number; }; } export interface PromoteWorkerConfig { /** The host DKG agent — provides the queue + the sync `promote` call. */ agent: DKGAgent; /** Number of concurrent worker loops (default 4 — RFC §4.5). */ workerConcurrency?: number; /** Polling interval for claimNext (default 100ms). */ pollIntervalMs?: number; /** * Heartbeat interval. Must be SHORTER than the queue's `leaseMs` * (default 5min); default 60s = 5× safety margin. */ heartbeatIntervalMs?: number; /** * Max time `stop()` will wait for in-flight jobs to complete before * returning. After the timeout, the in-flight `agent.assertion.promote` * continues in the background but the supervisor stops tracking it — * the next boot's `recoverOnStartup()` reconciles. */ shutdownTimeoutMs?: number; /** Deterministic time source for tests. */ now?: () => number; /** Defaults to `console.warn`. The daemon passes its own logger. */ log?: (msg: string) => void; /** Defaults to a no-op. The daemon passes its `memoryGraphChanged` emitter. */ emitMemoryGraphChanged?: (event: PromoteMemoryGraphChangedEvent) => void; /** * Worker-id prefix used when minting each loop's `workerId`. Defaults * to `daemon-`. Tests inject a stable prefix. */ workerIdPrefix?: string; } export interface PromoteWorkerSupervisor { /** Run `recoverOnStartup()` once, then spawn the polling loops. */ start(): Promise; /** Stop polling and wait (up to `shutdownTimeoutMs`) for in-flight jobs. */ stop(): Promise; /** * Test-only: drive one full poll across every worker slot * synchronously. Returns the number of jobs picked up by this round. */ tickOnce(): Promise; /** * Observability — counts of completed runs since start. Counters reset * on every `start()` call (a new supervisor lifecycle). */ getCounters(): PromoteWorkerCounters; } export interface PromoteWorkerCounters { succeeded: number; failedTerminal: number; failedRetrying: number; /** * Codex #665: jobs whose promote ran successfully but whose post-promote * bookkeeping (commit-marker write / queue.succeed) failed mid-flight. * These remain in `running` state until next startup recovery; operators * MUST inspect SWM/VM before any explicit `/recover`. */ partialPromoteAmbiguity: number; /** Number of `runJob` invocations that started (regardless of outcome). */ attempted: number; /** Set when shuttingDown was hit mid-job; ops can correlate with abandoned counts at next startup. */ interruptedAtShutdown: number; } export type ClassifiedPromoteError = { classification: PromoteFailureClassification; retryable: boolean; message?: string; }; /** * Map a promote error message to a `PromoteAttemptError` classification. * Seeded from the three rc.10 Graphify import patterns documented in * `dkg-graphify-rc10-test/FINDINGS_v2.md`. Returns `fatal` for unknown * patterns — the operator can re-classify and call `/recover` after * inspecting the failure. * * Exported so the daemon supervisor (and future tooling like an * operator dashboard) can preview the verdict without going through * the worker. */ export declare function classifyPromoteError(err: unknown): ClassifiedPromoteError; /** * Per-job execution — extracted so tests can exercise the exact * try/catch/heartbeat shape without spinning the supervisor. * * Resolves once the job's lifecycle transition (succeed or fail) has * been written back to the queue. Heartbeats run in the background * for the duration; both `succeed` and `fail` clear the lease so any * in-flight heartbeat after that point throws a `PromoteJobLeaseError`, * which the heartbeat catcher logs and ignores. */ export declare function runPromoteJob(args: { job: PromoteJob; queue: AsyncPromoteQueue; workerId: string; runPromote: (request: PromoteRequest, markPromoteStarted: () => Promise) => Promise<{ promotedCount: number; }>; now: () => number; heartbeatIntervalMs: number; log: (msg: string) => void; emitMemoryGraphChanged?: (event: PromoteMemoryGraphChangedEvent) => void; }): Promise<{ outcome: 'succeeded' | 'failed_retrying' | 'failed_terminal' | 'partial_promote_ambiguity'; error?: ClassifiedPromoteError; }>; export declare function createPromoteWorkerSupervisor(config: PromoteWorkerConfig): PromoteWorkerSupervisor; //# sourceMappingURL=async-promote-worker.d.ts.map