import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; import { spawn, spawnSync } from 'node:child_process'; import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { closeDb } from '../db/client'; import { runMigrations } from '../db/migrate'; import { resetTestDbPath } from '../test-utils/db-path'; import { InFlightError, OPERATIONS_SWEEP_PATTERN, OPERATIONS_SWEEP_SUBSCRIBER, OPERATION_TTL_MS, checkInFlight, completeOperation, ensureOperationsSweepSubscriber, failOperation, isPidRunnable, refuseIfInFlight, startOperation, } from './module-operations'; describe('module-operations', () => { let dir: string; beforeEach(async () => { dir = mkdtempSync(join(tmpdir(), 'celilo-ops-test-')); process.env.CELILO_DB_PATH = join(dir, 'celilo.db'); await runMigrations(process.env.CELILO_DB_PATH); }); afterEach(() => { closeDb(); resetTestDbPath(); try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } }); describe('startOperation / completeOperation / failOperation', () => { it('records an in-progress row on start, transitions to completed on complete', () => { const id = startOperation('homebridge', 'deploy'); const inFlight = checkInFlight(); expect(inFlight).toHaveLength(1); expect(inFlight[0].operation.id).toBe(id); expect(inFlight[0].operation.status).toBe('in_progress'); expect(inFlight[0].operation.operation).toBe('deploy'); expect(inFlight[0].operation.moduleId).toBe('homebridge'); completeOperation(id); expect(checkInFlight()).toHaveLength(0); }); it('transitions to failed with error message on failOperation', () => { const id = startOperation('caddy', 'backup'); failOperation(id, new Error('disk full')); expect(checkInFlight()).toHaveLength(0); // Re-querying directly to verify the failed row is recorded const id2 = startOperation('caddy', 'backup'); const inFlight = checkInFlight(); expect(inFlight).toHaveLength(1); expect(inFlight[0].operation.id).toBe(id2); }); it('failOperation accepts non-Error values', () => { const id = startOperation('caddy', 'restore'); failOperation(id, 'string error reason'); expect(checkInFlight()).toHaveLength(0); }); }); /** * celilo#737. Recording an outcome is BOOKKEEPING; the caller's error is the * information. On celilo-mgr a `SQLITE_BUSY` inside `failOperation` propagated * out of the catch block that called it and REPLACED the deploy's own error, * so the operator was shown a database-locking problem and never learned what * the deploy did wrong — the original was destroyed and is unrecoverable. * * `breakOperationsTable` stands in for any write failure. The mechanism does * not matter; what matters is that no failure of the write can reach the * caller. */ describe('recording an outcome cannot destroy what it records (#737)', () => { function breakOperationsTable(): void { const { getDb } = require('../db/client'); getDb().$client.run('DROP TABLE module_operations'); } it('failOperation does not replace the error it was called to record', () => { const id = startOperation('caddy', 'deploy'); breakOperationsTable(); const original = new Error('ansible task failed on step 7'); let surfaced: unknown; // Exactly the shape every call site uses (module-deploy.ts, // module-remove.ts): record the failure, then rethrow the original. try { try { throw original; } catch (err) { failOperation(id, err); throw err; } } catch (err) { surfaced = err; } expect(surfaced).toBe(original); }); it('failOperation does not throw when the write fails', () => { const id = startOperation('caddy', 'deploy'); breakOperationsTable(); expect(() => failOperation(id, new Error('original'))).not.toThrow(); }); it('completeOperation does not throw when the write fails', () => { // The mirror bug: a deploy that SUCCEEDED reporting a database error, and // skipping the `emitDeployCompleted` that follows the call. const id = startOperation('caddy', 'deploy'); breakOperationsTable(); expect(() => completeOperation(id)).not.toThrow(); }); it('startOperation still throws — its row IS the in-flight lock', () => { // Deliberately NOT swallowed. `checkInFlight` reads this row to refuse a // backup during a deploy, so a silently-missing row would let the two run // together. Failing before any work happens is honest; failing after it // is what #737 is about. breakOperationsTable(); expect(() => startOperation('caddy', 'deploy')).toThrow(); }); }); describe('checkInFlight', () => { it('returns empty when no operations are in flight', () => { expect(checkInFlight()).toHaveLength(0); }); it('describes conflicts with module + operation + pid', () => { const id = startOperation('caddy', 'deploy'); const conflicts = checkInFlight(); expect(conflicts).toHaveLength(1); expect(conflicts[0].describe).toBe(`deploy of caddy (pid ${process.pid})`); completeOperation(id); }); it('excludes the operation matching excludeOperationId', () => { const own = startOperation('caddy', 'backup'); const other = startOperation('homebridge', 'deploy'); const conflicts = checkInFlight(own); expect(conflicts).toHaveLength(1); expect(conflicts[0].operation.id).toBe(other); completeOperation(own); completeOperation(other); }); it('ignores rows whose pid is no longer alive', () => { // Spawn a short-lived process, capture its pid, wait for it to exit. // // `process.execPath` (the bun binary running this suite), NOT a bare // `node`: this repo is bun-based and nothing guarantees a node on PATH. // Where there was none, spawnSync returned `pid: undefined` and the // assertion below failed with "Expected and actual values must be numbers // or bigints" — which reads as a broken pid check rather than a missing // interpreter. const child = spawnSync(process.execPath, ['-e', 'process.exit(0)']); const deadPid = child.pid; expect(deadPid).toBeGreaterThan(0); expect(isPidRunnable(deadPid)).toBe(false); // Insert a fake row with the dead pid via raw SQL (bypasses pid=process.pid in startOperation). const { getDb } = require('../db/client'); const { moduleOperations } = require('../db/schema'); const db = getDb(); db.insert(moduleOperations) .values({ id: 'fake-dead-row', moduleId: 'orphan', operation: 'deploy', status: 'in_progress', pid: deadPid, }) .run(); const conflicts = checkInFlight(); expect(conflicts).toHaveLength(0); // dead pid filtered out }); }); describe('refuseIfInFlight', () => { it('throws InFlightError when conflicts exist', () => { const id = startOperation('caddy', 'deploy'); expect(() => refuseIfInFlight()).toThrow(InFlightError); try { refuseIfInFlight(); } catch (err) { expect(err).toBeInstanceOf(InFlightError); expect((err as InFlightError).conflicts).toHaveLength(1); expect((err as Error).message).toContain('deploy of caddy'); } completeOperation(id); }); it('is a no-op when no conflicts exist', () => { expect(() => refuseIfInFlight()).not.toThrow(); }); it('respects excludeOperationId', () => { const own = startOperation('caddy', 'backup'); expect(() => refuseIfInFlight(own)).not.toThrow(); completeOperation(own); }); }); describe('isPidRunnable', () => { it('returns true for the current process', () => { expect(isPidRunnable(process.pid)).toBe(true); }); it('returns false for a dead pid', () => { const child = spawnSync('node', ['-e', 'process.exit(0)']); expect(isPidRunnable(child.pid as number)).toBe(false); }); // The bug this whole module exists to prevent: a Ctrl-Z'd `module deploy` // is still "alive" by kill(pid, 0) and held the backup lock for 20 days. it('returns false for a STOPPED process, which kill(pid, 0) calls alive', () => { const child = spawn('sleep', ['60'], { stdio: 'ignore' }); const pid = child.pid as number; try { expect(isPidRunnable(pid)).toBe(true); child.kill('SIGSTOP'); // Wait for the state change to land in the process table. for (let i = 0; i < 100 && isPidRunnable(pid); i++) spawnSync('sleep', ['0.01']); // Still passes the old liveness test... let existsByKill = true; try { process.kill(pid, 0); } catch { existsByKill = false; } expect(existsByKill).toBe(true); // ...but is correctly reported as unable to make progress. expect(isPidRunnable(pid)).toBe(false); } finally { child.kill('SIGCONT'); child.kill('SIGKILL'); } }); }); describe('abandonment by age', () => { function insertRow(id: string, pid: number, startedAt: Date): void { const { getDb } = require('../db/client'); const { moduleOperations } = require('../db/schema'); getDb() .insert(moduleOperations) .values({ id, moduleId: 'ancient', operation: 'deploy', status: 'in_progress', pid, startedAt, }) .run(); } it('ignores a row older than the TTL even though its process is alive', () => { // process.pid is unquestionably running, so age is the only thing that // can release this row. This is the pid-reuse case: an old row whose // number now belongs to some unrelated healthy process. insertRow('ancient-row', process.pid, new Date(Date.now() - OPERATION_TTL_MS - 60_000)); expect(checkInFlight()).toHaveLength(0); }); it('still blocks on a young row whose process is alive', () => { insertRow('fresh-row', process.pid, new Date(Date.now() - 60_000)); expect(checkInFlight()).toHaveLength(1); }); }); }); describe('ensureOperationsSweepSubscriber', () => { it('registers the hourly reclaim against the existing clear command', () => { const calls: Array<{ name: string; pattern: string; handler: string; registeredBy?: string }> = []; ensureOperationsSweepSubscriber({ subscribe: (options) => calls.push(options) }); expect(calls).toEqual([ { name: OPERATIONS_SWEEP_SUBSCRIBER, pattern: OPERATIONS_SWEEP_PATTERN, handler: 'celilo module operations clear', registeredBy: 'celilo-module-operations', }, ]); // Finer than the TTL, so a wedged row never survives long. expect(OPERATIONS_SWEEP_PATTERN).toBe('timer.tick.1h'); }); });