import { randomUUID as nodeRandomUUID } from "node:crypto"; import { lstat, mkdir, readdir, rm } from "node:fs/promises"; import { isAbsolute, join, normalize, relative, sep } from "node:path"; import { writeWorkflowJson } from "./atomic-storage.js"; import { runDirectoryForRequest } from "./run-store.js"; import { WorkflowLeaseManager } from "./workflow-lease.js"; import type { RunProjectionV2, WorkerAttemptProjection } from "./workflow-types.js"; export interface CleanupItem { readonly kind: "run" | "worker-session" | "artifact"; readonly path: string; readonly bytes: number; } export interface CleanupCandidate { readonly requestKey: string; readonly runId: string; readonly updatedAt: string; readonly ageMs: number; readonly items: readonly CleanupItem[]; readonly reclaimableBytes: number; } export interface CleanupSkip { readonly requestKey: string; readonly runId: string; readonly reason: | "not-old-enough" | "active-run" | "ambiguous-run" | "active-worker" | "held-session-lease" | "corrupt-session-lease"; readonly diagnostic: string; } export interface RunCleanupPlan { readonly dryRun: true; readonly olderThanMs: number; readonly candidates: readonly CleanupCandidate[]; readonly skipped: readonly CleanupSkip[]; readonly reclaimableBytes: number; } export interface RunCleanupResult extends Omit { readonly dryRun: false; readonly auditEventPath: string | null; readonly deletedRunIds: readonly string[]; } interface CleanupRunCatalog { listRuns(): Promise; } interface CleanupLeaseAuthority { withCurrentSupervisor(generation: string, operation: () => Promise): Promise; inspectWorkerSession(sessionPath: string): ReturnType; } export interface WorkflowRunCleanupOptions { readonly namespaceDirectory: string; readonly store: CleanupRunCatalog; readonly leaseManager: CleanupLeaseAuthority; readonly supervisorGeneration: string; readonly now?: () => Date; readonly randomUUID?: () => string; readonly removePath?: (path: string) => Promise; } export class CleanupConfirmationError extends Error { constructor(message: string) { super(message); this.name = "CleanupConfirmationError"; } } export class WorkflowRunCleanup { readonly #namespaceDirectory: string; readonly #store: CleanupRunCatalog; readonly #leaseManager: CleanupLeaseAuthority; readonly #supervisorGeneration: string; readonly #now: () => Date; readonly #randomUUID: () => string; readonly #removePath: (path: string) => Promise; constructor(options: WorkflowRunCleanupOptions) { if (!isAbsolute(options.namespaceDirectory)) { throw new TypeError("Workflow cleanup namespace must be absolute"); } this.#namespaceDirectory = normalize(options.namespaceDirectory); this.#store = options.store; this.#leaseManager = options.leaseManager; this.#supervisorGeneration = requireNonEmpty( options.supervisorGeneration, "workflow supervisor generation", ); this.#now = options.now ?? (() => new Date()); this.#randomUUID = options.randomUUID ?? nodeRandomUUID; this.#removePath = options.removePath ?? (async (path) => { assertOwnedPath(this.#namespaceDirectory, path); await rm(path, { recursive: true, force: true }); }); } async plan(olderThanMs: number): Promise { const age = positiveInteger(olderThanMs, "cleanup age"); const nowMs = this.#now().getTime(); const candidates: CleanupCandidate[] = []; const skipped: CleanupSkip[] = []; for (const run of await this.#store.listRuns()) { const evaluated = await this.#evaluate(run, age, nowMs); if ("reason" in evaluated) skipped.push(evaluated); else candidates.push(evaluated); } candidates.sort((left, right) => left.runId.localeCompare(right.runId)); skipped.sort((left, right) => left.runId.localeCompare(right.runId)); return { dryRun: true, olderThanMs: age, candidates, skipped, reclaimableBytes: candidates.reduce((sum, candidate) => sum + candidate.reclaimableBytes, 0), }; } async execute(input: { readonly olderThanMs: number; readonly confirmed: boolean; }): Promise { if (!input.confirmed) { throw new CleanupConfirmationError("Workflow cleanup deletion requires explicit confirmation"); } return this.#leaseManager.withCurrentSupervisor(this.#supervisorGeneration, async () => { const plan = await this.plan(input.olderThanMs); if (plan.candidates.length === 0) { return { ...plan, dryRun: false, auditEventPath: null, deletedRunIds: [] }; } const auditEventPath = await this.#recordAudit(plan); const deletedRunIds: string[] = []; for (const candidate of plan.candidates) { for (const session of candidate.items.filter((item) => item.kind === "worker-session")) { await this.#removePath(session.path); } const run = candidate.items.find((item) => item.kind === "run"); if (!run) throw new Error(`Cleanup candidate ${candidate.runId} has no run directory`); await this.#removePath(run.path); deletedRunIds.push(candidate.runId); } return { ...plan, dryRun: false, auditEventPath, deletedRunIds }; }); } async #evaluate( run: RunProjectionV2, olderThanMs: number, nowMs: number, ): Promise { const ageMs = Math.max(0, nowMs - Date.parse(run.updatedAt)); if (ageMs < olderThanMs) { return skip(run, "not-old-enough", `run age ${ageMs}ms is below ${olderThanMs}ms`); } if (run.state === "ambiguous" || run.state === "needs-attention" || run.unresolvedAmbiguities.length > 0) { return skip(run, "ambiguous-run", `run state is ${run.state}`); } if (run.state !== "done" && run.state !== "cancelled") { return skip(run, "active-run", `run state is ${run.state}`); } const attempts = Object.values(run.workers).flatMap((worker) => worker.attempts); if (attempts.some(isActiveAttempt)) { return skip(run, "active-worker", "at least one worker attempt is active or lacks exit evidence"); } const sessionPaths = [...new Set(attempts.map((attempt) => attempt.session.path))].sort(); for (const sessionPath of sessionPaths) { const lease = await this.#leaseManager.inspectWorkerSession(sessionPath); if (lease.kind === "owned") { return skip( run, "held-session-lease", `session ${sessionPath} is held by generation ${lease.owner.generation}`, ); } if (lease.kind === "corrupt") { return skip(run, "corrupt-session-lease", lease.diagnostic); } } const runPath = runDirectoryForRequest(this.#namespaceDirectory, run.requestKey); const artifactPath = join(runPath, "artifacts"); const runBytes = await pathBytes(runPath); const items: CleanupItem[] = [{ kind: "run", path: runPath, bytes: runBytes }]; for (const sessionPath of sessionPaths) { assertOwnedPath(this.#namespaceDirectory, sessionPath); items.push({ kind: "worker-session", path: sessionPath, bytes: await pathBytes(sessionPath) }); } const artifactBytes = await pathBytes(artifactPath); if (artifactBytes > 0) items.push({ kind: "artifact", path: artifactPath, bytes: artifactBytes }); return { requestKey: run.requestKey, runId: run.runId, updatedAt: run.updatedAt, ageMs, items, reclaimableBytes: runBytes + items .filter((item) => item.kind === "worker-session") .reduce((sum, item) => sum + item.bytes, 0), }; } async #recordAudit(plan: RunCleanupPlan): Promise { const directory = join(this.#namespaceDirectory, "cleanup-ledger", "events"); await mkdir(directory, { recursive: true, mode: 0o700 }); const eventId = `cleanup_${requireNonEmpty(this.#randomUUID(), "cleanup event UUID")}`; const path = join(directory, `${this.#now().getTime()}.${safePathPart(eventId)}.json`); await writeWorkflowJson(path, { schema: 1, eventId, type: "cleanup-started", occurredAt: this.#now().toISOString(), supervisorGeneration: this.#supervisorGeneration, olderThanMs: plan.olderThanMs, candidates: plan.candidates, reclaimableBytes: plan.reclaimableBytes, }, { createOnly: true }); return path; } } export async function confirmCleanupDeletion(input: { readonly dryRun: boolean; readonly yes: boolean; readonly tui: boolean; readonly confirm?: () => Promise; }): Promise { if (input.dryRun) return false; if (input.tui) { if (!input.confirm || !await input.confirm()) { throw new CleanupConfirmationError("Workflow cleanup was not confirmed; no state changed"); } return true; } if (!input.yes) { throw new CleanupConfirmationError("Headless workflow cleanup requires explicit yes=true"); } return true; } export function parseCleanupAge(value: string): number { const match = /^(\d+)(ms|s|m|h|d|w)$/.exec(value.trim()); if (!match) throw new TypeError("Cleanup age must use "); const amount = Number(match[1]); const multiplier = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000, w: 604_800_000 }[ match[2] as "ms" | "s" | "m" | "h" | "d" | "w" ]; const result = amount * multiplier; return positiveInteger(result, "cleanup age"); } function isActiveAttempt(attempt: WorkerAttemptProjection): boolean { return attempt.pidStartEvidence.observedExitAt === null || attempt.pidStartEvidence.exitStatus === null || attempt.state === "pending" || attempt.state === "running" || attempt.state === "blocked" || attempt.state === "ambiguous"; } function skip( run: RunProjectionV2, reason: CleanupSkip["reason"], diagnostic: string, ): CleanupSkip { return { requestKey: run.requestKey, runId: run.runId, reason, diagnostic }; } async function pathBytes(path: string): Promise { let info; try { info = await lstat(path); } catch (error) { if (isNodeError(error) && error.code === "ENOENT") return 0; throw error; } if (!info.isDirectory()) return info.size; let total = info.size; for (const entry of await readdir(path, { withFileTypes: true })) { total += await pathBytes(join(path, entry.name)); } return total; } function assertOwnedPath(namespaceDirectory: string, path: string): void { const child = relative(namespaceDirectory, normalize(path)); if (child.length === 0 || child === ".." || child.startsWith(`..${sep}`) || isAbsolute(child)) { throw new Error(`Workflow cleanup path escapes its namespace: ${path}`); } } function positiveInteger(value: number, name: string): number { if (!Number.isSafeInteger(value) || value <= 0) throw new TypeError(`${name} must be positive`); return value; } function requireNonEmpty(value: string, name: string): string { if (value.length === 0) throw new TypeError(`${name} is required`); return value; } function safePathPart(value: string): string { return value.replace(/[^A-Za-z0-9._-]/g, "_"); } function isNodeError(error: unknown): error is NodeJS.ErrnoException { return error instanceof Error && "code" in error; }