import { spawn, type ChildProcess } from 'child_process'; import http from 'http'; import fs from 'fs'; import path from 'path'; import { PKG_DIR, WORKSPACE_DIR } from '../shared/paths.js'; import { log } from '../shared/logger.js'; let child: ChildProcess | null = null; let restarts = 0; let lastSpawnTime = 0; let intentionallyStopped = false; // Hard backstop against an orphaned backend. SIGTERM/SIGINT go through stopBackend() for a graceful // stop, but process.exit() (server EADDRINUSE handler, self-update relaunch, fatal errors) bypasses // those handlers AND can't run async cleanup — so the backend child would survive as an orphan // (PPID→1) still holding BACKEND_PORT, EADDRINUSE-ing every later backend spawn until killPort // happens to reclaim it. 'exit' fires on EVERY exit path (including process.exit) and allows the one // synchronous kill we need. (A SIGKILL of the supervisor itself can't be caught — killPort covers it // on the next startup.) process.on('exit', () => { try { if (child && child.exitCode === null) child.kill('SIGKILL'); } catch {} }); // True once the backend has crash-looped past MAX_RESTARTS and given up — i.e. it's down and // will NOT come back without the user fixing the code. The supervisor shows the "backend down" // interstitial in this state. Cleared on every spawn attempt (a deliberate restart is "trying again"). let gaveUp = false; const MAX_RESTARTS = 3; const STABLE_THRESHOLD = 30_000; // 30s — if backend ran this long, it wasn't a crash loop // Rolling-window backstop: the 30s "stable" rule resets the consecutive counter, so a backend // that crashes every ~35s would otherwise restart forever. Give up if it crashes too often within // the window (kept generous so a legitimately long-running-then-crashing backend isn't penalized). let crashTimes: number[] = []; const CRASH_WINDOW_MS = 5 * 60_000; // 5 min const CRASH_WINDOW_MAX = 6; // > this many crashes in the window → give up regardless of the 30s reset // Called once when the backend gives up, so the supervisor can tell the live chat (which exists // precisely so the user can be told to fix it). Set via setBackendGiveUpHandler; logging-only default. let onGiveUp: (() => void) | null = null; /** Extra env vars injected into every backend spawn (e.g. BLOBY_AGENT_SECRET) */ let extraEnv: Record = {}; /** Set extra environment variables for the backend process. Applies to all future spawns including auto-restarts. */ export function setBackendEnv(env: Record): void { extraEnv = { ...extraEnv, ...env }; } /** Register a callback fired once when the backend gives up (crash-looped past the limits). * The supervisor wires this to broadcast a chat event so the user is told to fix it. */ export function setBackendGiveUpHandler(fn: () => void): void { onGiveUp = fn; } const LOG_FILE = path.join(WORKSPACE_DIR, '.backend.log'); // Holds the LAST crashed run's output. spawnBackend truncates LOG_FILE on every (re)spawn, so an // agent reading .backend.log right after an auto-restart would otherwise see only the fresh (often // empty) run and lose the originating error. The crash exit handler copies LOG_FILE here first. const LOG_FILE_PREV = LOG_FILE + '.prev'; export function getBackendPort(basePort: number): number { return basePort + 4; } export function spawnBackend(port: number): ChildProcess { // Self-guard against double-spawn. Several restart paths (file watcher, bot:turn-complete, // scheduler pulse, channel manager) each chain their own stopBackend().then(spawnBackend); // two genuinely-concurrent triggers could otherwise both spawn onto the contended // BACKEND_PORT. The normal restart is unaffected: stopBackend() nulls `child` before its // promise resolves, and the crash auto-restart runs only after exit (exitCode !== null), // so this guard no-ops only when a live backend already exists. if (child && child.exitCode === null) { log.warn('Backend already running — skipping duplicate spawn'); return child; } const backendPath = path.join(WORKSPACE_DIR, 'backend', 'index.ts'); lastSpawnTime = Date.now(); intentionallyStopped = false; gaveUp = false; // Clear log file on each restart — only keeps current run try { fs.writeFileSync(LOG_FILE, ''); } catch {} // Wrap the backend in an inline loader that: // 1. Registers module resolution hooks to prevent walk-up to parent node_modules // 2. Dynamically imports the user's backend (tsx handles .ts compilation) // 3. Adds a keepalive timer so the event loop never drains (fixes exit code 0 under systemd) // 4. Catches and logs import errors instead of silently exiting const backendUrl = 'file://' + backendPath.replace(/\\/g, '/'); const boundary = WORKSPACE_DIR + path.sep; const wrapper = [ // Workspace isolation: block module resolution outside workspace/node_modules. // tsx loads via --import BEFORE this code runs, so it resolves from root (correct). // All subsequent imports (the backend code) are restricted to workspace. `const nodeModule = await import('node:module');`, `const { fileURLToPath } = await import('node:url');`, `if (nodeModule.registerHooks) {`, ` const BOUNDARY = ${JSON.stringify(boundary)};`, ` nodeModule.registerHooks({`, ` resolve(specifier, context, nextResolve) {`, ` const result = nextResolve(specifier, context);`, ` if (result.url.startsWith('node:') || result.url.startsWith('data:')) return result;`, ` if (fileURLToPath(result.url).startsWith(BOUNDARY)) return result;`, ` throw new Error('[workspace-isolate] "' + specifier + '" resolved outside workspace: ' + fileURLToPath(result.url));`, ` }`, ` });`, `} else {`, ` console.warn('[backend] Node 22.15+ required for workspace isolation — running without module boundaries');`, `}`, `import('${backendUrl}')`, ` .catch(e => { console.error('[backend] Fatal:', e); process.exit(1); });`, `setInterval(() => {}, 60000);`, ].join('\n'); child = spawn(process.execPath, ['--import', 'tsx/esm', '--input-type=module', '-e', wrapper], { cwd: WORKSPACE_DIR, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, ...extraEnv, BACKEND_PORT: String(port) }, }); child.stdout?.on('data', (d) => { process.stdout.write(d); try { fs.appendFileSync(LOG_FILE, d); } catch {} }); child.stderr?.on('data', (d) => { process.stderr.write(d); try { fs.appendFileSync(LOG_FILE, d); } catch {} }); child.on('exit', (code) => { // Supervisor called stopBackend() — don't auto-restart if (intentionallyStopped) return; // Preserve the just-crashed run's output before the next spawnBackend truncates LOG_FILE, so a // post-bounce read (agent or interstitial) can still fetch the originating error via ?prev=1. // Only crashes reach here (intentional stops returned above), so .prev always holds the last crash. try { fs.copyFileSync(LOG_FILE, LOG_FILE_PREV); } catch {} // Any unexpected exit (crash, SIGTERM, OOM, null code) — restart log.warn(`Backend exited unexpectedly (code ${code})`); // Track crashes in a rolling window (backstop for the 30s-reset crash-loop hole). const now = Date.now(); crashTimes = crashTimes.filter((t) => now - t < CRASH_WINDOW_MS); crashTimes.push(now); const windowExceeded = crashTimes.length > CRASH_WINDOW_MAX; // If backend was alive for >30s, it's not a crash loop — reset the consecutive counter. if (now - lastSpawnTime > STABLE_THRESHOLD) { restarts = 0; } if (!windowExceeded && restarts < MAX_RESTARTS) { restarts++; const delay = Math.min(1000 * restarts, 5000); log.info(`Restarting backend (${restarts}/${MAX_RESTARTS}, delay ${delay}ms)...`); setTimeout(() => spawnBackend(port), delay); } else { gaveUp = true; log.error(`Backend failed too many times${windowExceeded ? ` (${crashTimes.length} crashes in ${CRASH_WINDOW_MS / 60000}min)` : ''}. Use Bloby chat to debug.`); try { onGiveUp?.(); } catch {} } }); log.ok(`Backend spawned on port ${port}`); return child; } /** Stop the backend and wait for the process to fully exit before resolving. * This prevents port collisions when restarting (old process must release the port first). * Concurrent calls return the same promise to avoid double-spawn races. */ let stopPromise: Promise | null = null; export function stopBackend(): Promise { if (stopPromise) return stopPromise; if (!child || child.exitCode !== null) { child = null; return Promise.resolve(); } intentionallyStopped = true; const dying = child; child = null; const promise = new Promise((resolve) => { let killTimer: ReturnType | null = null; let finished = false; const done = () => { if (finished) return; // exit + SIGKILL paths can both fire; run once finished = true; if (killTimer) clearTimeout(killTimer); // Only release the shared guard if it still points at THIS stop. A later stopBackend() // may already have installed its own promise; the 3s safety timer (or a late exit) must // never null a *different* stop's guard — that would make isBackendStopping() lie and let // a concurrent spawn race the in-flight kill for the port. if (stopPromise === promise) stopPromise = null; resolve(); }; dying.once('exit', done); dying.kill(); // Safety: force kill after 3s if SIGTERM doesn't land. killTimer = setTimeout(() => { try { dying.kill('SIGKILL'); } catch {} done(); }, 3000); }); stopPromise = promise; return promise; } let restartInFlight: Promise | null = null; let rerunRequested = false; /** Serialized + coalescing backend restart — the single funnel for every deliberate restart * (file watcher, turn-complete, scheduler pulse, channel manager). Concurrent callers share * one in-flight restart; a request that arrives mid-restart triggers exactly one more * stop→spawn cycle afterward, so the final backend was spawned after the latest request. This * removes the double-spawn-onto-contended-port race of independent stopBackend().then(spawn) chains. */ export function restartBackend(port: number): Promise { if (restartInFlight) { rerunRequested = true; return restartInFlight; } restartInFlight = (async () => { do { rerunRequested = false; resetBackendRestarts(); await stopBackend(); spawnBackend(port); } while (rerunRequested); })().finally(() => { restartInFlight = null; }); return restartInFlight; } export function isBackendAlive(): boolean { return child !== null && child.exitCode === null; } /** True when the backend has crash-looped past MAX_RESTARTS and given up — down and not * coming back without a code fix. Drives the supervisor's "backend down" interstitial. */ export function isBackendDead(): boolean { return gaveUp; } /** Read the tail of the backend log (default 100 lines) for the "copy logs" debug helper and the * agent's GET /__bloby/control/logs/backend endpoint. Pass prev=true to read the last CRASHED run * (.backend.log.prev) — useful right after an auto-restart, when the live log is a fresh run. */ export function readBackendLogTail(maxLines = 100, prev = false): string { try { const text = fs.readFileSync(prev ? LOG_FILE_PREV : LOG_FILE, 'utf-8'); const lines = text.split('\n'); return lines.slice(-maxLines).join('\n').trim(); } catch { return ''; } } /** True if the backend was (re)spawned within the last ~2s — so callers can tell the agent that a * near-empty log tail is a fresh-spawn artifact, not the absence of an error. */ export function backendJustSpawned(): boolean { return Date.now() - lastSpawnTime < 2000; } /** Resolve true as soon as the backend's HTTP port is ACCEPTING connections (any response — even a * 404 — means the port is bound and serving), false if it never comes up within timeoutMs or the * backend gives up first. This is the REAL readiness signal that the restart-and-verify endpoint * returns to the agent: isBackendAlive() only means the child process was spawned, not that it has * bound its port, so it reports "alive" during the startup window when requests still 503. */ export function probeBackendReady(port: number, timeoutMs = 15000): Promise { const deadline = Date.now() + timeoutMs; return new Promise((resolve) => { const attempt = () => { if (gaveUp) return resolve(false); // crash-looped past the limit — it's not coming up const req = http.request( { host: '127.0.0.1', port, path: '/', method: 'GET', timeout: 2000 }, (res) => { res.resume(); resolve(true); }, // any HTTP response = port is listening ); req.on('error', () => { if (Date.now() >= deadline) return resolve(false); setTimeout(attempt, 250); }); req.on('timeout', () => { try { req.destroy(); } catch {} }); // → 'error' → retry/deadline req.end(); }; attempt(); }); } export function isBackendStopping(): boolean { return stopPromise !== null; } export function resetBackendRestarts(): void { restarts = 0; // A deliberate restart (file edit, user fix, scheduler) is a fresh attempt — clear the rolling // crash window too so a just-fixed backend gets a clean slate (deliberate stops never record a // crash, so this only matters right after a give-up + fix). crashTimes = []; }