// Durable job records: a small on-disk representation of "a job that outlives
// the client that launched it". A supervisor process owns the record; a client
// only reads it. A bridge drop, a closed terminal, or a killed client
// therefore interrupts the *view* of the job, never the job itself.
//
// Layout, under a caller-supplied base directory:
//
// /jobs//job.json immutable description (argv, cwd, resource)
// /jobs//status.json mutable state, rewritten atomically
// /jobs//job.log merged child stdout + stderr
//
// Classification is fail-closed, mirroring qa-status: liveness is proven by the
// PID, never assumed from the status document. A non-terminal state whose PID
// is dead classifies as "dead", never as "running".
//
// Toolkit tier: this module must not import src/core (layering check).
import { randomUUID } from "node:crypto";
import {
existsSync,
mkdirSync,
readdirSync,
readFileSync,
renameSync,
writeFileSync,
} from "node:fs";
import { join, resolve } from "node:path";
export const DURABLE_JOB_SCHEMA_VERSION = 1;
/** How often the supervisor rewrites status.json while a job runs. */
export const DURABLE_JOB_HEARTBEAT_MS = 15_000;
/** A running job whose heartbeat is older than this earns a staleness warning,
* but stays "running" while its PID is alive. */
export const DURABLE_JOB_HEARTBEAT_STALE_MS = 120_000;
export const DURABLE_JOB_DOCUMENT_FILENAME = "job.json";
export const DURABLE_JOB_STATUS_FILENAME = "status.json";
export const DURABLE_JOB_LOG_FILENAME = "job.log";
const JOBS_DIRNAME = "jobs";
/** States a supervisor writes. "completed" is the only terminal one. */
export type DurableJobState = "launching" | "queued" | "running" | "completed";
/** States a reader can observe. "dead" is derived, never written. */
export type DurableJobObservedState = DurableJobState | "dead";
const WRITTEN_STATES = new Set(["launching", "queued", "running", "completed"]);
export interface DurableJobDocument {
schema_version: number;
job_id: string;
/** Admission resource the job queues on. */
resource: string;
/** Concurrent holders allowed on that resource. */
capacity: number;
/** Human-readable description shown in listings. */
label: string;
/** Argument vector of the wrapped command; argv[0] is the executable. */
argv: string[];
/** Directory the wrapped command runs in. */
cwd: string;
created_at: string;
}
export interface DurableJobStatus {
schema_version: number;
job_id: string;
/** PID of the supervisor, not of the wrapped command. */
pid: number;
state: DurableJobState;
started_at: string;
updated_at: string;
/** Present while the supervisor waits for an admission slot. */
queue?: { resource: string; waiting_since: string };
exit_code?: number | null;
signal?: string | null;
}
export interface DurableJobReport {
state: DurableJobObservedState;
/** The job reached "completed"; exit_code and signal are final. */
terminal: boolean;
job_id: string | null;
pid: number | null;
resource: string | null;
label: string | null;
argv: string[] | null;
exit_code: number | null;
signal: string | null;
created_at: string | null;
started_at: string | null;
updated_at: string | null;
/** now minus status.updated_at; null without a parseable heartbeat. */
heartbeat_age_ms: number | null;
queue?: { resource: string; waiting_since: string };
job_dir: string;
log_path: string;
warnings: string[];
}
export interface DurableJobDeps {
/** PID liveness probe (injectable for tests). Default: process.kill(pid, 0). */
pidAlive?: (pid: number) => boolean;
/** Evaluation instant in epoch milliseconds. Default: Date.now(). */
now?: () => number;
}
export type DurableJobClassification =
| { ok: true; report: DurableJobReport }
| { ok: false; error: string };
export interface DurableJobListEntry {
job_id: string;
job_dir: string;
created_at: string | null;
document: DurableJobDocument | null;
report: DurableJobReport | null;
/** Set when the directory exists but its status document is unusable. */
error?: string;
}
function isRecord(value: unknown): value is Record {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
/** Default PID liveness probe. EPERM counts as alive; only ESRCH is dead. */
function defaultPidAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch (err: unknown) {
return (err as NodeJS.ErrnoException).code !== "ESRCH";
}
}
/** Directory holding every job record under a base directory. */
export function jobsRoot(base: string): string {
return join(resolve(base), JOBS_DIRNAME);
}
/** Directory of one job record. Does not create it. */
export function jobDirFor(base: string, jobId: string): string {
return join(jobsRoot(base), jobId);
}
/** Create (idempotently) the directory for one job and return its path. */
export function createJobDir(base: string, jobId: string): string {
const dir = jobDirFor(base, jobId);
mkdirSync(dir, { recursive: true });
return dir;
}
export function jobLogPath(jobDir: string): string {
return join(resolve(jobDir), DURABLE_JOB_LOG_FILENAME);
}
function writeAtomic(path: string, body: string): void {
const tmp = `${path}.${randomUUID().slice(0, 8)}.tmp`;
writeFileSync(tmp, body);
renameSync(tmp, path);
}
/** Write the immutable job description. */
export function writeJobDocument(jobDir: string, document: DurableJobDocument): void {
writeAtomic(
join(resolve(jobDir), DURABLE_JOB_DOCUMENT_FILENAME),
`${JSON.stringify(document, null, 2)}\n`,
);
}
/** Replace the status document atomically, so a reader never sees a torn file. */
export function writeJobStatus(jobDir: string, status: DurableJobStatus): void {
writeAtomic(
join(resolve(jobDir), DURABLE_JOB_STATUS_FILENAME),
`${JSON.stringify(status, null, 2)}\n`,
);
}
function readJson(path: string): unknown {
return JSON.parse(readFileSync(path, "utf8"));
}
/** Read the job description, or null when it is missing or unusable. */
export function readJobDocument(jobDir: string): DurableJobDocument | null {
const path = join(resolve(jobDir), DURABLE_JOB_DOCUMENT_FILENAME);
let parsed: unknown;
try {
parsed = readJson(path);
} catch {
return null;
}
if (!isRecord(parsed)) return null;
if (typeof parsed.job_id !== "string") return null;
return {
schema_version:
typeof parsed.schema_version === "number"
? parsed.schema_version
: DURABLE_JOB_SCHEMA_VERSION,
job_id: parsed.job_id,
resource: typeof parsed.resource === "string" ? parsed.resource : "",
capacity: typeof parsed.capacity === "number" ? parsed.capacity : 1,
label: typeof parsed.label === "string" ? parsed.label : "",
argv: Array.isArray(parsed.argv)
? parsed.argv.filter((a): a is string => typeof a === "string")
: [],
cwd: typeof parsed.cwd === "string" ? parsed.cwd : "",
created_at: typeof parsed.created_at === "string" ? parsed.created_at : "",
};
}
/** Read the status document, or null when it is missing or unusable. */
export function readJobStatus(jobDir: string): DurableJobStatus | null {
const path = join(resolve(jobDir), DURABLE_JOB_STATUS_FILENAME);
let parsed: unknown;
try {
parsed = readJson(path);
} catch {
return null;
}
if (!isRecord(parsed)) return null;
if (typeof parsed.job_id !== "string") return null;
if (typeof parsed.state !== "string" || !WRITTEN_STATES.has(parsed.state)) return null;
const queue = parsed.queue;
return {
schema_version:
typeof parsed.schema_version === "number"
? parsed.schema_version
: DURABLE_JOB_SCHEMA_VERSION,
job_id: parsed.job_id,
pid: typeof parsed.pid === "number" ? parsed.pid : 0,
state: parsed.state as DurableJobState,
started_at: typeof parsed.started_at === "string" ? parsed.started_at : "",
updated_at: typeof parsed.updated_at === "string" ? parsed.updated_at : "",
...(isRecord(queue) &&
typeof queue.resource === "string" &&
typeof queue.waiting_since === "string"
? { queue: { resource: queue.resource, waiting_since: queue.waiting_since } }
: {}),
...(typeof parsed.exit_code === "number" || parsed.exit_code === null
? { exit_code: parsed.exit_code as number | null }
: {}),
...(typeof parsed.signal === "string" || parsed.signal === null
? { signal: parsed.signal as string | null }
: {}),
};
}
/**
* Classify one job directory, fail-closed. "completed" is terminal and carries
* the wrapped command's exit code. Any other written state is trusted only
* while the supervisor PID is alive; otherwise the job is "dead" — the record
* outlived the process that owned it.
*/
export function classifyJob(jobDir: string, deps: DurableJobDeps = {}): DurableJobClassification {
const isAlive = deps.pidAlive ?? defaultPidAlive;
const nowMs = deps.now ? deps.now() : Date.now();
const abs = resolve(jobDir);
if (!existsSync(abs)) return { ok: false, error: `no such job directory: ${abs}` };
const statusPath = join(abs, DURABLE_JOB_STATUS_FILENAME);
if (!existsSync(statusPath)) {
return {
ok: false,
error: `${abs} carries no ${DURABLE_JOB_STATUS_FILENAME}: not a job directory`,
};
}
const status = readJobStatus(abs);
if (!status) {
return { ok: false, error: `${statusPath} is not a readable job status document` };
}
const document = readJobDocument(abs);
let heartbeatAgeMs: number | null = null;
const updatedMs = Date.parse(status.updated_at);
if (!Number.isNaN(updatedMs)) heartbeatAgeMs = Math.max(0, nowMs - updatedMs);
const base = {
job_id: status.job_id,
pid: status.pid > 0 ? status.pid : null,
resource: document?.resource ?? null,
label: document?.label ?? null,
argv: document?.argv ?? null,
created_at: document?.created_at ?? null,
started_at: status.started_at || null,
updated_at: status.updated_at || null,
heartbeat_age_ms: heartbeatAgeMs,
job_dir: abs,
log_path: jobLogPath(abs),
};
if (status.state === "completed") {
return {
ok: true,
report: {
...base,
state: "completed",
terminal: true,
exit_code: status.exit_code ?? null,
signal: status.signal ?? null,
warnings: [],
},
};
}
if (!(status.pid > 0 && isAlive(status.pid))) {
return {
ok: true,
report: {
...base,
state: "dead",
terminal: false,
exit_code: null,
signal: null,
...(status.queue ? { queue: status.queue } : {}),
warnings: [],
},
};
}
const warnings: string[] = [];
if (
status.state === "running" &&
heartbeatAgeMs !== null &&
heartbeatAgeMs > DURABLE_JOB_HEARTBEAT_STALE_MS
) {
warnings.push(
`heartbeat stale: last update ${Math.round(heartbeatAgeMs / 1000)}s ago ` +
`(threshold ${DURABLE_JOB_HEARTBEAT_STALE_MS / 1000}s), but pid ${status.pid} is alive`,
);
}
return {
ok: true,
report: {
...base,
state: status.state,
terminal: false,
exit_code: null,
signal: null,
...(status.queue ? { queue: status.queue } : {}),
warnings,
},
};
}
/**
* Every job record under a base directory, newest first. Ordering keys on the
* description's created_at, falling back to the status document's started_at,
* then to the directory name, so a record missing its description still lists.
*/
export function listJobs(base: string, deps: DurableJobDeps = {}): DurableJobListEntry[] {
const root = jobsRoot(base);
let names: string[];
try {
names = readdirSync(root, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name);
} catch {
return [];
}
const entries: { entry: DurableJobListEntry; sortKey: number }[] = [];
for (const name of names) {
const dir = join(root, name);
const document = readJobDocument(dir);
const classified = classifyJob(dir, deps);
const createdAt =
document?.created_at || (classified.ok ? classified.report.started_at : null) || null;
const parsed = createdAt !== null ? Date.parse(createdAt) : Number.NaN;
entries.push({
entry: {
job_id: document?.job_id ?? name,
job_dir: dir,
created_at: createdAt,
document,
report: classified.ok ? classified.report : null,
...(classified.ok ? {} : { error: classified.error }),
},
sortKey: Number.isNaN(parsed) ? 0 : parsed,
});
}
entries.sort((a, b) => {
if (b.sortKey !== a.sortKey) return b.sortKey - a.sortKey;
return b.entry.job_id.localeCompare(a.entry.job_id);
});
return entries.map((item) => item.entry);
}
/** Describe a job's outcome as a process exit code: the wrapped command's own
* code, 1 when it died on a signal, 4 when the record is dead. */
export function jobExitCode(report: DurableJobReport): number {
if (report.state === "dead") return 4;
if (!report.terminal) return 5;
if (report.signal !== null) return 1;
return report.exit_code ?? 1;
}
/** Format a duration the way the status commands do. */
export function formatJobAge(ms: number): string {
const totalSeconds = Math.max(0, Math.round(ms / 1000));
if (totalSeconds < 60) return `${totalSeconds}s`;
const minutes = Math.floor(totalSeconds / 60);
if (minutes < 60) return `${minutes}m ${totalSeconds % 60}s`;
const hours = Math.floor(minutes / 60);
return `${hours}h ${minutes % 60}m`;
}