/** * Celilo's own lifecycle events. Emitted on every `module deploy` * (started / completed / failed) so subscribers can react — production * smoke tests, alerting, dashboards, etc. * * The event types are dot-segmented with the module id as the last * segment: `deploy.started.lunacycle`, `deploy.completed.lunacycle`. * This lets subscribers target one module (`deploy.completed.lunacycle`) * or fan out (`deploy.completed.*`). * * Bus errors are best-effort: a sick bus shouldn't break a deploy. Each * helper opens-emits-closes; per-event overhead is a few ms. */ import { defineEvents, openBus } from '@celilo/event-bus'; import { z } from 'zod'; import { getEventBusPath } from '../config/paths'; const NO_SCHEMAS = defineEvents({}); export interface DeployStartedPayload { module: string; startedAt: number; } export interface DeployCompletedPayload { module: string; startedAt: number; durationMs: number; } export interface DeployFailedPayload { module: string; startedAt: number; durationMs: number; error: string; } export interface HealthCheckFailedPayload { module: string; reason: string; } export interface UninstallStartedPayload { module: string; startedAt: number; } export interface UninstallCompletedPayload { module: string; startedAt: number; durationMs: number; } export interface UninstallFailedPayload { module: string; startedAt: number; durationMs: number; error: string; } /** * A deployed system (a machine or container-service instance hosting a * module) has come up / been torn down. Carries the system's identity so a * subscriber (e.g. a dns_internal provider) can register/deregister internal * DNS records without re-querying the deploying module's config. * openspec/specs/internal-dns-split-horizon/spec.md D5. */ export interface SystemCreatedPayload { /** The module whose deploy produced this system (lets handlers resolve zones, etc.). */ module: string; /** Bare hostname (e.g. "dns-int"), no domain. */ hostname: string; /** IPv4 address, CIDR stripped (e.g. "192.168.0.53"). */ targetIp: string; } export type SystemDestroyedPayload = SystemCreatedPayload; const DeployStartedSchema = z.object({ module: z.string().min(1), startedAt: z.number().int().nonnegative(), }); const DeployCompletedSchema = z.object({ module: z.string().min(1), startedAt: z.number().int().nonnegative(), durationMs: z.number().int().nonnegative(), }); const DeployFailedSchema = z.object({ module: z.string().min(1), startedAt: z.number().int().nonnegative(), durationMs: z.number().int().nonnegative(), error: z.string(), }); const HealthCheckFailedSchema = z.object({ module: z.string().min(1), reason: z.string(), }); const UninstallStartedSchema = z.object({ module: z.string().min(1), startedAt: z.number().int().nonnegative(), }); const UninstallCompletedSchema = z.object({ module: z.string().min(1), startedAt: z.number().int().nonnegative(), durationMs: z.number().int().nonnegative(), }); const UninstallFailedSchema = z.object({ module: z.string().min(1), startedAt: z.number().int().nonnegative(), durationMs: z.number().int().nonnegative(), error: z.string(), }); const SystemCreatedSchema = z.object({ module: z.string().min(1), hostname: z.string().min(1), targetIp: z.string().min(1), }); const SystemDestroyedSchema = SystemCreatedSchema; export function emitDeployStarted(payload: DeployStartedPayload): void { DeployStartedSchema.parse(payload); emitBest(`deploy.started.${payload.module}`, payload); } export function emitDeployCompleted(payload: DeployCompletedPayload): void { DeployCompletedSchema.parse(payload); emitBest(`deploy.completed.${payload.module}`, payload); } export function emitDeployFailed(payload: DeployFailedPayload): void { DeployFailedSchema.parse(payload); emitBest(`deploy.failed.${payload.module}`, payload); } export function emitHealthCheckFailed(payload: HealthCheckFailedPayload): void { HealthCheckFailedSchema.parse(payload); emitBest(`health-check.failed.${payload.module}`, payload); } export function emitUninstallStarted(payload: UninstallStartedPayload): void { UninstallStartedSchema.parse(payload); emitBest(`uninstall.started.${payload.module}`, payload); } export function emitUninstallCompleted(payload: UninstallCompletedPayload): void { UninstallCompletedSchema.parse(payload); emitBest(`uninstall.completed.${payload.module}`, payload); } export function emitUninstallFailed(payload: UninstallFailedPayload): void { UninstallFailedSchema.parse(payload); emitBest(`uninstall.failed.${payload.module}`, payload); } /** * Emit `system.created.` (D5). Last segment is the module id so * subscribers can fan out (`system.created.*`) or target one. */ export function emitSystemCreated(payload: SystemCreatedPayload): void { SystemCreatedSchema.parse(payload); emitBest(`system.created.${payload.module}`, payload); } /** Emit `system.destroyed.` (D5). */ export function emitSystemDestroyed(payload: SystemDestroyedPayload): void { SystemDestroyedSchema.parse(payload); emitBest(`system.destroyed.${payload.module}`, payload); } /** * Emit `public_web.routes_changed` (ISS-0035). A coarse "the route table * changed, re-read it" signal: a consumer registered or unregistered a route, * so the public_web PROVIDER (caddy) should reconcile its running config from * web_routes. The provider subscribes via a manifest `subscriptions:` entry. * Best-effort — a failed emit never breaks register_route; the route is already * persisted in web_routes and the provider's next deploy reconciles it anyway. */ export function emitWebRoutesChanged(triggeredBy: string): void { emitBest('public_web.routes_changed', { triggeredBy }); } const ROUTES_CHANGED_TYPE = 'public_web.routes_changed'; /** * Emit `public_web.routes_changed` and block until the provider's reconcile * delivery settles (ISS-0035). This is what `register_route` / `unregister_routes` * await so they return only once the route is actually live — restoring the * synchronous guarantee the old SSH path gave, provider-agnostically. A * consuming module's `health_check` runs right after it registers its route, so * without this wait it races the ~1s async reconcile and sees the placeholder. * * Captures the high-water id BEFORE emitting so the wait targets exactly the * event this call produced. Degrades safely: no dispatcher → returns without * waiting; no provider subscribed → no deliveries → returns immediately. */ export async function emitWebRoutesChangedAndWait( triggeredBy: string, opts?: { timeoutMs?: number; pollMs?: number }, ): Promise { const since = routesChangedHighWater(); emitWebRoutesChanged(triggeredBy); return waitForRouteReconcile(since, opts); } /** * Outcome of {@link waitForRouteReconcile}. `events` is how many * `public_web.routes_changed` events the deploy emitted; `succeeded`/`failed` * count their settled deliveries (one per subscribing provider). `timedOut` * is true when the deadline hit with deliveries still pending/running. */ export interface RouteReconcileWaitResult { events: number; succeeded: number; failed: number; timedOut: boolean; /** * True when no event dispatcher was running, so the `routes_changed` event * was persisted but never DELIVERED to the provider (caddy). The route is * therefore NOT live. Authoritative callers (ISS-0081) must treat this as a * failure, not a benign "0 deliveries" success. */ noDispatcher: boolean; } /** * Highest `public_web.routes_changed` event id at this instant, or 0 if none * exist yet. A deploy captures this before it runs so {@link waitForRouteReconcile} * can tell which route-change events the deploy itself produced. */ export function routesChangedHighWater(): number { let bus: ReturnType | undefined; try { bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS }); return bus.recentEvents({ type: ROUTES_CHANGED_TYPE, limit: 1 })[0]?.id ?? 0; } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.warn(`[celilo] failed to read routes-changed high-water: ${msg}`); return 0; } finally { bus?.close(); } } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } /** * Block until every `public_web.routes_changed` event emitted after * `sinceEventId` has had all its deliveries settle (succeeded / failed / * abandoned), or until `timeoutMs` elapses. This is what makes * `module deploy` return only once the public_web PROVIDER (caddy) has * reconciled its running config from web_routes — closing the race where * deploy returned while the route was still the placeholder (ISS-0035). * * When the deploy registered no routes, or no provider subscribes to the * event, there are zero deliveries and this returns immediately. With no * dispatcher running, nothing will ever deliver the reconcile, so the wait is * skipped rather than burning the whole deadline (the deploy-time dispatcher * gate, ISS-0042, is the upstream guard). The wait is advisory: it never turns * a bus error into a deploy failure. The caller inspects the result and * decides what to surface. */ export async function waitForRouteReconcile( sinceEventId: number, opts: { timeoutMs?: number; pollMs?: number } = {}, ): Promise { const timeoutMs = opts.timeoutMs ?? 90_000; const pollMs = opts.pollMs ?? 250; const deadline = Date.now() + timeoutMs; let bus: ReturnType | undefined; try { bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS }); const b = bus; // The deploy's emits already happened, so the set of route-change events // is fixed; only their deliveries transition as the dispatcher works. const events = b .recentEvents({ type: ROUTES_CHANGED_TYPE, limit: 100 }) .filter((e) => e.id > sinceEventId); if (events.length === 0) { return { events: 0, succeeded: 0, failed: 0, timedOut: false, noDispatcher: false }; } if (b.health().status === 'no_dispatcher') { console.warn( `[celilo] route-reconcile: no dispatcher running — not waiting; ${events.length} change event(s) persisted, the provider will reconcile on its next run`, ); return { events: events.length, succeeded: 0, failed: 0, timedOut: false, noDispatcher: true, }; } while (true) { const deliveries = events.flatMap((e) => b.deliveriesForEvent(e.id)); const settled = (status: string) => deliveries.filter((d) => d.status === status).length; const pending = deliveries.filter( (d) => d.status === 'pending' || d.status === 'running', ).length; if (pending === 0 || Date.now() >= deadline) { return { events: events.length, succeeded: settled('succeeded'), failed: settled('failed') + settled('abandoned'), timedOut: pending > 0, noDispatcher: false, }; } await sleep(pollMs); } } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.warn(`[celilo] route-reconcile wait errored: ${msg}`); return { events: 0, succeeded: 0, failed: 0, timedOut: false, noDispatcher: false }; } finally { bus?.close(); } } /** * Open the bus, emit, close. Errors are caught and logged so a * misbehaving bus never wedges the caller. The empty-registry mode * skips bus-side payload validation; we validate above with our own * schemas before calling. */ function emitBest(type: string, payload: unknown): void { let bus: ReturnType | undefined; try { bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS }); bus.emitRaw(type, payload); } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.warn(`[celilo] failed to emit ${type}: ${msg}`); } finally { bus?.close(); } }