import { EventEmitter } from 'node:events'; import { DatabaseInterface } from '@happyvertical/sql'; import { RetentionSweeperOptions } from './retention.js'; import { SmrtJob } from './smrt-job.js'; import { SmrtJobEvent } from './smrt-job-event.js'; /** * TaskRunner configuration */ export interface TaskRunnerConfig { /** Worker ID (auto-generated if not provided) */ id?: string; /** Number of concurrent jobs to process */ concurrency?: number; /** Queues to process (default: ['default']) */ queues?: string[]; /** Polling interval in milliseconds */ pollInterval?: number; /** Heartbeat interval in milliseconds */ heartbeatInterval?: number; /** Maximum time to wait for jobs to complete on shutdown */ shutdownTimeout?: number; /** * @deprecated No longer used. Recovery keys on worker liveness, not per-job * heartbeat staleness (#1474). Use {@link leaseTtlMs} / {@link leaseTickMs}. */ staleJobThresholdMs?: number; /** Worker liveness lease time-to-live in milliseconds */ leaseTtlMs?: number; /** How often to renew the worker liveness lease, in milliseconds */ leaseTickMs?: number; /** * Periodic system-table retention sweep (#2375). * * A running worker is the framework's only always-on scheduler, so it is * also where the retention sweep is scheduled by default. Set to `false` to * opt out entirely (for example when a cron job runs `smrt db:prune` * instead), or pass options to change the cadence or the policy. The first * sweep runs one interval after `start()`, never at start. */ retention?: RetentionSweeperOptions | false; } /** * TaskRunner events */ export interface TaskRunnerEvents { 'job:started': (job: SmrtJob) => void; 'job:event': (job: SmrtJob, event: SmrtJobEvent) => void; 'job:progress': (job: SmrtJob, event: SmrtJobEvent) => void; 'job:completed': (job: SmrtJob, result: unknown) => void; 'job:failed': (job: SmrtJob, error: Error) => void; 'job:retrying': (job: SmrtJob, error: Error, delay: number) => void; 'runner:started': () => void; 'runner:stopped': () => void; 'runner:error': (error: Error) => void; } /** * Raised when a job exceeds its timeout under `timeoutBehavior` `'fail'`/`'kill'`. * * Distinguished from an ordinary handler error so the failure path can choose * NOT to auto-retry: the original handler keeps running after a timeout (JS * can't preempt it), so re-queueing the row as `pending` while the original * still executes guarantees concurrent duplicate execution (see #2 in the * #1401 review). A timed-out job is therefore failed terminally rather than * retried, shrinking — though, given the at-least-once contract, not fully * eliminating — the overlap window. */ export declare class JobTimeoutError extends Error { constructor(message: string); } /** * TaskRunner processes SMRT jobs by invoking methods on SmrtObjects * * Features: * - Executes jobs via SmrtObject method invocation * - Configurable concurrency and timeout behavior * - Automatic retry with configurable strategies * - Job context logging for visibility * - Embedded mode (in-process) or standalone (CLI) */ export declare class TaskRunner extends EventEmitter { readonly id: string; /** * Per-incarnation-unique worker key. Stored as the `worker_id` on claimed * jobs and in `_smrt_workers`, so a restart of a runner sharing the same * configured `id` does not look like it still owns the previous * incarnation's orphaned jobs. The human-facing {@link id} stays stable for * events/logs. */ private readonly workerKey; private readonly config; private readonly effectiveLeaseTtlMs; private collection; private eventCollection; private workerCollection; private workersTableVerified; private lastRecoverySweepAt; private running; private activeJobs; private pollTimer; private heartbeatTimer; private leaseTimer; /** Periodic system-table retention sweep, when not opted out (#2375). */ private retentionSweeper; private livenessWorker; private shutdownPromise; private db; private logger; constructor(config?: TaskRunnerConfig); /** * Initialize the runner with database connection */ initialize(db: DatabaseInterface): Promise; /** * Start processing jobs */ start(): Promise; /** * Stop processing jobs (graceful shutdown) */ stop(): Promise; /** * Check if runner is running */ isRunning(): boolean; /** * Get count of active jobs */ activeJobCount(): number; /** * Start the polling loop */ private startPolling; /** * Poll for and process jobs */ private poll; /** * Process a single job. * * AT-LEAST-ONCE EXECUTION CONTRACT: a `timeout` only races the handler's * promise — JavaScript cannot preempt an already-running handler, so on a * `'fail'` (or `'kill'`) timeout the original handler keeps executing in the * background while the job row is failed. Timeouts are NOT auto-retried (see * handleJobError) precisely so a still-running handler is not duplicated by a * retry; but the orphaned handler's own side effects still happen, and an * ordinary (non-timeout) failure IS retried and re-claimable by any worker. * Handlers invoked from a job MUST be idempotent (e.g. keyed by * `context.job.jobId` or a caller-supplied idempotency key); do not rely on a * job body running exactly once. See AGENTS.md "Timeouts & at-least-once". */ private processJob; /** * Execute a job honoring its {@link SmrtJob.timeoutBehavior}. * * - `'fail'` (default) and `'kill'`: race the handler against a timeout. On * timeout the race rejects and the caller fails/retries the job. * `'kill'` cannot actually preempt the handler in-process (JavaScript has * no thread interruption), so it is treated identically to `'fail'` — the * handler keeps running in the background; only the job row is failed. This * is documented in AGENTS.md so `'kill'` is honest about what it does * rather than silently behaving like a no-op. * - `'warn'`: do NOT fail on timeout. Arm a one-shot warning (logged + emitted * as a job event) at the deadline, but await the handler to completion so a * slow-but-successful handler still completes. This makes `'warn'` honest: * previously every timeout was treated as `'fail'` regardless of the * persisted/UI-shown behavior. */ private runWithTimeout; /** * Apply a terminal/retry state transition to a job only if this worker still * owns it and it is still `running`. Returns whether the write applied. * * This closes the completion-vs-recovery race: if recovery already failed a * job out from under a finishing handler (a genuine zombie), the handler's * outcome is dropped rather than resurrecting the row. */ private writeOwnedJob; /** * Execute a job by invoking the method on the SmrtObject */ private executeJob; /** * Handle job execution error */ private handleJobError; private createExecutionContext; private appendJobEvent; /** * Whether the `_smrt_workers` table exists. Cached once positive — the table * never disappears mid-run, so this avoids a probe query on every poll. */ private workersTableReady; /** * Recover jobs orphaned by dead/restarted workers. * * A `running` job is recovered only when its owning worker is *not alive* * (issue #1474): not live in this process and holding no fresh lease in * `_smrt_workers`. This is independent of the handler event loop, so a worker * whose handler holds the loop synchronously keeps a fresh lease (renewed off * the loop by the liveness thread) or stays in this process's live set, and is * never false-recovered. The live set takes precedence over a stale lease, and * a runner never recovers its own active jobs. * * Recovery is swept at most once per lease tick (not every poll), since * detection is TTL-bound anyway — this bounds the per-poll database load. */ private recoverStaleJobs; /** * Renew this worker's liveness lease. * * In Stage 1 this runs on the main event loop, so it provides cross-process * detection no weaker than the old per-job heartbeat. Stage 2 moves the * renewal to an off-loop worker thread so a synchronous handler can no longer * starve it. Same-process correctness never depends on this timer — the * in-memory live set covers it. */ private startLeaseRenewal; /** * Spawn the off-loop liveness thread. It opens its own connection and renews * this worker's lease on its own thread (unstarvable by handler CPU). Returns * false if the thread can't be resolved or fails to start, so the caller can * fall back to main-loop renewal. */ private startLivenessThread; private handleLivenessThreadLoss; /** Stop the liveness thread (graceful, with a short bound), if running. */ private stopLivenessThread; /** * Per-job heartbeat loop — telemetry only ("last activity" for the UI). It no * longer gates recovery (that is the worker lease), so a blocked loop missing * a heartbeat is harmless. */ private startHeartbeat; /** * Wait for active jobs to complete with timeout */ private waitForActiveJobs; } /** * Create a TaskRunner instance */ export declare function createTaskRunner(config?: TaskRunnerConfig): TaskRunner; export default TaskRunner; //# sourceMappingURL=runner.d.ts.map