// The two queue gauges an alert can actually be written against. `queue_depth` and `jobs_total` // live in `@ultimat3/core`'s `runtime-metrics.ts` because the deploy chart scales on them; these // two are not scaling signals, they are the ones every queue team pages on, and neither existed: // // "page if the oldest job in payments is older than 5 minutes" — no series carried // `oldestReadyMs`, which `QueueStats` computes, `inspectQueues` renders and nothing published. // `queue_depth` cannot tell 10 jobs stuck for an hour from 10 enqueued a second ago. // "page if the dead-letter queue is not empty" — `jobs_total{outcome="dead"}` is a COUNTER, so // a DLQ that filled overnight and stopped growing has a flat rate and alerts on nothing. // // Declared here rather than in core because they are this package's facts and core is another // package's file; if they move next to `queueDepth` later, these two functions are the only // callers to redirect. import type { Gauge } from '@ultimat3/core'; import { gauge } from '@ultimat3/core'; import type { JobOutcome } from './execute'; /** * `JobOutcome` -> the `jobs_total` label, and `null` for the outcomes that are not one. `suspended` * is deliberately unmapped: parking a run is control flow, so counting it would make every * `step.sleep` read as a finished job and make the failure ratio meaningless. `interrupted` for * the same reason: a deploy cutting a job short is the process's doing, and a failure ratio that * spikes on every rollout is a page nobody answers. `refused` is `failed` because the ROW is: the * counter says what `x jobs list --state failed` says, and never `dead` — a key refusal is not in * the dead-letter queue, so it must not move the gauge beside this one. Read by the worker's * `recordJob` sites. */ export const JOB_OUTCOME_LABELS = Object.freeze< Record >({ completed: 'ok', suspended: null, retried: 'failed', 'dead-lettered': 'dead', dropped: 'failed', interrupted: null, refused: 'failed', }); /** Seconds and not milliseconds: every Prometheus duration is seconds, and the alert is `> 300`. */ export const queueOldestReady: Gauge = gauge('queue_oldest_ready_seconds', { unit: 's', description: 'Age of the oldest job that is ready and due, by queue — 0 when none is', }); export const queueDeadJobs: Gauge = gauge('queue_dead_jobs', { unit: '{job}', description: 'Jobs sitting in the dead-letter state, by queue', }); export function recordQueueOldestReady(queue: string, oldestReadyMs: number): void { queueOldestReady.record(oldestReadyMs / 1000, { queue }); } export function recordQueueDeadJobs(queue: string, dead: number): void { queueDeadJobs.record(dead, { queue }); } /** * What an autoscaler sizes a worker fleet with: `ceil((queue_depth + Σ in_flight) / capacity)` * pods. Unlabelled — one series per PROCESS, which is what `maxInFlight` caps and what one worker * pod is; the scrape's own `instance` label tells pods apart, and `sum` across them is the fleet. */ export const workerInFlight: Gauge = gauge('job_worker_in_flight', { unit: '{job}', description: 'Jobs this worker process is running right now, across every queue it serves', }); export const workerCapacity: Gauge = gauge('job_worker_capacity', { unit: '{job}', description: 'The most jobs this worker process runs at once: maxInFlight, or the sum of its queue slots when tighter or unset', }); export function recordWorkerInFlight(held: number): void { workerInFlight.record(held); } export function recordWorkerCapacity(capacity: number): void { workerCapacity.record(capacity); }