/** * Backup restore service. * Downloads, decrypts, extracts backup archives and executes on_restore hooks. * For system state backups, restores the Celilo database. */ import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { eq } from 'drizzle-orm'; import { extract as tarExtract } from 'tar'; import { getDbPath } from '../config/paths'; import { closeDb, getDb } from '../db/client'; import { moduleConfigs, modules, secrets as secretsTable } from '../db/schema'; import type { Backup } from '../db/schema'; import { invokeHook } from '../hooks/executor'; import { createHookStores } from '../hooks/hook-store'; import { createConsoleLogger } from '../hooks/logger'; import type { ModuleManifest } from '../manifest/schema'; import { decryptSecret } from '../secrets/encryption'; import { getOrCreateMasterKey } from '../secrets/master-key'; import { decryptFileToFile } from './backup-cipher'; import { assertCompatibleSchema, parseManifest } from './backup-manifest'; import { createStorageProvider } from './backup-storage'; import { applyCrossModuleWriteRoot, moduleHasCrossModuleRead } from './cross-module-read'; import { getModuleSystems } from './deployed-systems'; import { completeOperation, failOperation, refuseIfInFlight, startOperation, } from './module-operations'; import { remoteAccessPolicy } from './remote-access'; export interface RestoreResult { success: boolean; error?: string; healthCheckPassed?: boolean; } /** * Restore a system state backup (replaces the Celilo database) */ export async function restoreSystemStateBackup(backup: Backup): Promise { // Refuse if any module operation is in flight (deploy/uninstall/backup/ // restore). Caller surfaces the InFlightError to the operator. refuseIfInFlight(); // Start an operation row so concurrent attempts in another process see // this restore as in-flight. The row's value is short-lived: once we // replace the DB file below, the new DB won't have this row. That's // acceptable — a system restore is a single-process operation by design, // and a parallel CLI invocation would be a foot-gun in any case. startOperation('__system__', 'restore'); const provider = await createStorageProvider(backup.storageId); const tempDir = join(tmpdir(), `celilo-restore-${backup.id}`); try { mkdirSync(tempDir, { recursive: true }); // Download encrypted archive const encryptedPath = join(tempDir, 'system.enc'); await provider.download(backup.storagePath, encryptedPath); // Decrypt → tar, streamed. Reads both the current format and the legacy // JSON envelope (see backup-cipher.ts). const masterKey = await getOrCreateMasterKey(); const envelopeDir = join(tempDir, 'envelope'); mkdirSync(envelopeDir, { recursive: true }); const tarPath = join(tempDir, 'envelope.tar'); await decryptFileToFile(encryptedPath, tarPath, masterKey); await tarExtract({ file: tarPath, cwd: envelopeDir }); // Read + validate manifest BEFORE touching the live DB. An // incompatible artifact must not get past this point. const manifestPath = join(envelopeDir, 'manifest.json'); if (!existsSync(manifestPath)) { return { success: false, error: 'System restore failed: artifact has no manifest.json. It may be from an older celilo (pre-envelope format) or corrupted.', }; } const manifest = parseManifest(readFileSync(manifestPath, 'utf-8')); assertCompatibleSchema(manifest); if (manifest.kind !== 'system') { return { success: false, error: `System restore failed: artifact kind is '${manifest.kind}', expected 'system'.`, }; } const restoredDbPath = join(envelopeDir, 'celilo.db'); if (!existsSync(restoredDbPath)) { return { success: false, error: 'System restore failed: artifact has no celilo.db.', }; } // Close current database connection. After this point, the operation // row from startOperation() above is unreachable (replaced by the // restored DB's content) and we cannot meaningfully complete/fail it // — but the restore IS the completion. closeDb(); // Replace the database file const dbPath = getDbPath(); copyFileSync(restoredDbPath, dbPath); // Remove WAL/SHM files (they're from the old DB state) try { rmSync(`${dbPath}-wal`, { force: true }); } catch { // May not exist } try { rmSync(`${dbPath}-shm`, { force: true }); } catch { // May not exist } return { success: true }; } catch (error) { return { success: false, error: `System restore failed: ${error instanceof Error ? error.message : String(error)}`, }; } finally { try { rmSync(tempDir, { recursive: true, force: true }); } catch { // Best effort cleanup } } } /** * Build config and secret maps for a module */ async function buildModuleContext(moduleId: string): Promise<{ configMap: Record; secretMap: Record; }> { const db = getDb(); const configs = db.select().from(moduleConfigs).where(eq(moduleConfigs.moduleId, moduleId)).all(); const configMap: Record = {}; for (const c of configs) { configMap[c.key] = c.valueJson ? JSON.parse(c.valueJson) : c.value; } const secretRecords = db .select() .from(secretsTable) .where(eq(secretsTable.moduleId, moduleId)) .all(); const masterKey = await getOrCreateMasterKey(); const secretMap: Record = {}; for (const s of secretRecords) { secretMap[s.name] = decryptSecret( { encryptedValue: s.encryptedValue, iv: s.iv, authTag: s.authTag }, masterKey, ); } return { configMap, secretMap }; } /** * Restore a module data backup. * Downloads, decrypts, extracts, then executes the module's on_restore hook. * Optionally runs a health check after restore. */ export async function restoreModuleBackup( backup: Backup, options: { runHealthCheck?: boolean } = {}, ): Promise { if (!backup.moduleId) { return { success: false, error: 'Backup has no associated module' }; } // Captured because the narrowing above does not survive into the hookStores // callback below, where TypeScript widens the parameter back to string|null. const moduleId = backup.moduleId; const db = getDb(); const mod = db.select().from(modules).where(eq(modules.id, backup.moduleId)).get(); if (!mod) { return { success: false, error: `Module not found: ${backup.moduleId}` }; } const manifest = mod.manifestData as unknown as ModuleManifest; const hookDef = manifest.hooks?.on_restore; if (!hookDef) { return { success: false, error: `Module '${mod.id}' has no on_restore hook defined`, }; } // Refuse if any module operation is in flight. Checked after we've // validated the backup + module, so the operator gets the more useful // error first. refuseIfInFlight(); const opId = startOperation(backup.moduleId, 'restore'); const provider = await createStorageProvider(backup.storageId); const tempDir = join(tmpdir(), `celilo-restore-${backup.id}`); const envelopeDir = join(tempDir, 'envelope'); try { mkdirSync(envelopeDir, { recursive: true }); // Download encrypted archive const encryptedPath = join(tempDir, 'backup.tar.enc'); await provider.download(backup.storagePath, encryptedPath); // Decrypt → tar, streamed. Reads both the current format and the legacy // JSON envelope (see backup-cipher.ts). Extract into envelopeDir; the // envelope contains: // manifest.json - validated below // data/ - on_backup hook artifacts (passed to on_restore) const masterKey = await getOrCreateMasterKey(); const tarPath = join(tempDir, 'envelope.tar'); await decryptFileToFile(encryptedPath, tarPath, masterKey); await tarExtract({ file: tarPath, cwd: envelopeDir }); // Read + validate envelope manifest BEFORE invoking the hook. const manifestPath = join(envelopeDir, 'manifest.json'); if (!existsSync(manifestPath)) { const err = 'Module restore failed: artifact has no manifest.json. It may be from an older celilo (pre-envelope format) or corrupted.'; failOperation(opId, err); return { success: false, error: err }; } const envelopeManifest = parseManifest(readFileSync(manifestPath, 'utf-8')); assertCompatibleSchema(envelopeManifest); if (envelopeManifest.kind !== 'module') { const err = `Module restore failed: artifact kind is '${envelopeManifest.kind}', expected 'module'.`; failOperation(opId, err); return { success: false, error: err }; } if (envelopeManifest.moduleId && envelopeManifest.moduleId !== backup.moduleId) { const err = `Module restore failed: artifact was created for module '${envelopeManifest.moduleId}', not '${backup.moduleId}'.`; failOperation(opId, err); return { success: false, error: err }; } // The on_restore hook reads its data from envelope/data/ — that's // where on_backup wrote it. const restoreDataDir = join(envelopeDir, 'data'); if (!existsSync(restoreDataDir)) { const err = 'Module restore failed: artifact has no data/ directory.'; failOperation(opId, err); return { success: false, error: err }; } // Build context for hook execution const { configMap, secretMap } = await buildModuleContext(mod.id); const logger = createConsoleLogger(mod.id, 'on_restore'); // Cross-module-write privilege: if the manifest declares // cross_module_read, hand the hook a writable staging dir. After // a successful hook return, applyCrossModuleWriteRoot atomically // moves each module's subtree into live storage. If the hook // crashes/fails, the staging dir is discarded with no side effects. const hookInputs: Record = { restore_dir: restoreDataDir, schema_version: envelopeManifest.dataSchemaVersion ?? backup.schemaVersion ?? '', }; let crossModuleWriteRoot: string | undefined; if (moduleHasCrossModuleRead(manifest)) { crossModuleWriteRoot = join(tempDir, 'cross-module-write'); mkdirSync(crossModuleWriteRoot, { recursive: true }); hookInputs.cross_module_write_root = crossModuleWriteRoot; } // Execute on_restore hook. Pass the data dir + the data-schema-version // from the envelope manifest so the hook can migrate its own data if // needed. Fall back to the backup record's stored value (kept for // backups created before the envelope landed in this celilo build). const hookResult = await invokeHook( mod.sourcePath, 'on_restore', manifest.celilo_contract, hookDef, hookInputs, configMap, secretMap, logger, { debug: false, systems: getModuleSystems(backup.moduleId, db), remoteAccess: remoteAccessPolicy(backup.moduleId, db), hookStores: () => createHookStores(db, moduleId), }, ); if (!hookResult.success) { const errMsg = hookResult.error ?? 'on_restore hook failed'; failOperation(opId, errMsg); return { success: false, error: errMsg, }; } // Hook succeeded — apply the cross-module staging dir back onto // live storage. Atomic per module (rename live → live.old + rename // staged → live, with rollback on mid-loop failure). If the apply // itself fails, fail the operation but leave the artifact intact — // the operator can investigate and re-run. if (crossModuleWriteRoot) { try { const applyResult = applyCrossModuleWriteRoot(crossModuleWriteRoot); if (applyResult.applied.length > 0) { logger.info(`Cross-module restore applied for: ${applyResult.applied.join(', ')}`); } if (applyResult.skipped.length > 0) { logger.warn( `Cross-module restore skipped (no staged data) for: ${applyResult.skipped.join(', ')}`, ); } } catch (applyErr) { const errMsg = `cross_module_write_root apply failed: ${applyErr instanceof Error ? applyErr.message : String(applyErr)}`; failOperation(opId, errMsg); return { success: false, error: errMsg }; } } // Run health check if requested and the module has one let healthCheckPassed: boolean | undefined; if (options.runHealthCheck && manifest.hooks?.health_check) { try { const { runModuleHealthCheck } = await import('./health-runner'); const healthResult = await runModuleHealthCheck(mod.id, db, { noInteractive: true, }); healthCheckPassed = healthResult.status === 'healthy'; } catch { healthCheckPassed = false; } } completeOperation(opId); return { success: true, healthCheckPassed, }; } catch (error) { failOperation(opId, error); return { success: false, error: `Restore failed: ${error instanceof Error ? error.message : String(error)}`, }; } finally { try { rmSync(tempDir, { recursive: true, force: true }); } catch { // Best effort cleanup } } }