{"version":3,"file":"factory-Bt7TWQje.mjs","names":[],"sources":["../src/storage/outage.ts","../src/storage/factory.ts"],"sourcesContent":["/**\n * outage.ts — is the storage backend actually reachable right now?\n *\n * The daemon retries a dead backend forever, which is correct: a Postgres\n * container that is down will usually come back, and giving up would lose the\n * work queue. What was wrong is that it did so in complete silence.\n *\n * Observed 2026-07-26: the container was down for roughly two days. The daemon\n * logged \"Postgres unavailable\" 144 times over 36 minutes, the work queue backed\n * up, session notes for the whole period were never written — and\n * `pai daemon status` reported \"Index: idle\" throughout. The one command anyone\n * would run to check said everything was fine.\n *\n * So the outage is recorded where the status command can see it, and escalated\n * once through the notification channels that were already configured and\n * already unused for this.\n */\n\nexport interface BackendOutage {\n  backend: string;\n  /** When the current run of failures began. */\n  since: number;\n  /** Consecutive failed attempts so far. */\n  attempts: number;\n  lastError: string;\n}\n\nlet current: BackendOutage | null = null;\n\n/** Record that the backend is currently unreachable. */\nexport function setBackendOutage(outage: BackendOutage): void {\n  current = outage;\n}\n\n/**\n * Record that the backend answered.\n *\n * Called on every successful connection, including the first — so a daemon that\n * never had a problem reports none, and one that recovered stops reporting an\n * outage that has ended.\n */\nexport function clearBackendOutage(): void {\n  current = null;\n}\n\n/** The current outage, or null when the backend is answering. */\nexport function getBackendOutage(): BackendOutage | null {\n  return current;\n}\n\n/**\n * Human-readable elapsed time, for a status line rather than a log.\n *\n * Exported because the status command needs the duration on its own, to colour\n * it separately from the rest of the sentence. Without this it kept a second\n * copy of the same rounding, which is how two renderings of one outage drift.\n */\nexport function humanDuration(ms: number): string {\n  const mins = Math.max(1, Math.round(ms / 60_000));\n  return mins < 60 ? `${mins} min` : `${(mins / 60).toFixed(1)} h`;\n}\n\n/** Human-readable duration, for a status line rather than a log. */\nexport function describeOutage(o: BackendOutage, now = Date.now()): string {\n  return `${o.backend} unreachable for ${humanDuration(now - o.since)}, ${o.attempts} attempts — ${o.lastError}`;\n}\n","/**\n * Storage backend factory.\n *\n * Reads the daemon config and returns the appropriate StorageBackend.\n *\n * When Postgres is the configured backend we NEVER silently fall back to\n * SQLite — doing so would split the corpus across two databases. Instead:\n *  - Daemon (waitForPostgres: true) retries Postgres forever with capped\n *    backoff until it comes up (handles the boot race where launchd starts\n *    the daemon before Docker Desktop / Postgres is ready).\n *  - CLI / one-shot callers (default) retry a few times, then throw a clear\n *    error rather than returning a wrong/empty SQLite database.\n */\n\nimport type { PaiDaemonConfig } from \"../daemon/config.js\";\nimport type { StorageBackend } from \"./interface.js\";\nimport { setBackendOutage, clearBackendOutage } from \"./outage.js\";\n\nexport interface StorageBackendOptions {\n  /**\n   * When true, retry Postgres indefinitely instead of giving up. Used by the\n   * long-lived daemon so a not-yet-ready Postgres at boot is tolerated.\n   * Defaults to false (one-shot CLI behaviour: bounded retries, then throw).\n   */\n  waitForPostgres?: boolean;\n}\n\n/** Backoff schedule (ms) for the bounded CLI retry path. */\nconst CLI_RETRY_DELAYS_MS = [500, 1_000, 2_000];\n\n/** Backoff cap (ms) for the daemon's infinite retry path. */\nconst DAEMON_RETRY_CAP_MS = 15_000;\n\n/**\n * Create and return the configured StorageBackend.\n *\n * Auto-behaviour:\n *  - storageBackend = \"sqlite\"   → SQLiteBackend always\n *  - storageBackend = \"postgres\" → PostgresBackend (retried; never falls back)\n */\nexport async function createStorageBackend(\n  config: PaiDaemonConfig,\n  opts: StorageBackendOptions = {}\n): Promise<StorageBackend> {\n  if (config.storageBackend === \"postgres\") {\n    return await connectPostgres(config, opts.waitForPostgres ?? false);\n  }\n\n  // Default: SQLite\n  return createSQLiteBackend();\n}\n\n/**\n * Attempt a single Postgres connection (ensure DB + test). Returns the live\n * backend on success, or an error string describing why it failed.\n */\nasync function attemptPostgres(\n  config: PaiDaemonConfig\n): Promise<{ backend: StorageBackend } | { error: string }> {\n  const { PostgresBackend } = await import(\"./postgres.js\");\n  const pgConfig = config.postgres ?? {};\n\n  let backend: InstanceType<typeof PostgresBackend> | null = null;\n  try {\n    // Ensure the per-user database exists and has the schema applied.\n    await PostgresBackend.ensureDatabase(pgConfig);\n\n    backend = new PostgresBackend(pgConfig);\n    const err = await backend.testConnection();\n    if (err) {\n      await backend.close().catch(() => {});\n      return { error: err };\n    }\n    return { backend };\n  } catch (e) {\n    if (backend) await backend.close().catch(() => {});\n    return { error: e instanceof Error ? e.message : String(e) };\n  }\n}\n\n/**\n * Consecutive failures before the outage is escalated to the user.\n *\n * Not the first failure: a container restarting, or the daemon starting before\n * Docker is up, recovers within a few seconds and is not worth a notification.\n * By the fifth attempt the backoff has already spent tens of seconds, which is\n * long enough that something is actually wrong.\n */\nconst ESCALATE_AFTER_ATTEMPTS = 5;\n\n/** Tell the user the backend is down, through whatever channels are configured. */\nasync function notifyBackendDown(attempts: number, lastError: string): Promise<void> {\n  try {\n    const { routeNotification } = await import(\"../notifications/router.js\");\n    const { loadConfig } = await import(\"../daemon/config.js\");\n    await routeNotification(\n      {\n        event: \"error\",\n        title: \"PAI: storage backend unreachable\",\n        message:\n          `Postgres has not answered in ${attempts} attempts (${lastError}). ` +\n          `Indexing, session notes and the work queue are stalled until it returns. ` +\n          `Check the container, then \\`pai daemon status\\`.`,\n      },\n      loadConfig().notifications\n    );\n  } catch {\n    // A notification that cannot be sent must never take the daemon down with\n    // it — the daemon retrying is still the useful behaviour here.\n  }\n}\n\n/** And say when it comes back, so the alert is not left hanging. */\nasync function notifyBackendRecovered(\n  attempts: number,\n  since: number | null\n): Promise<void> {\n  try {\n    const { routeNotification } = await import(\"../notifications/router.js\");\n    const { loadConfig } = await import(\"../daemon/config.js\");\n    const mins = since ? Math.max(1, Math.round((Date.now() - since) / 60_000)) : null;\n    await routeNotification(\n      {\n        event: \"completion\",\n        title: \"PAI: storage backend back\",\n        message:\n          `Postgres answered after ${attempts} attempts` +\n          (mins ? `, ${mins} min down` : \"\") +\n          `. The queue will drain on its own.`,\n      },\n      loadConfig().notifications\n    );\n  } catch {\n    /* same reasoning as above */\n  }\n}\n\nasync function connectPostgres(\n  config: PaiDaemonConfig,\n  waitForever: boolean\n): Promise<StorageBackend> {\n  let attempt = 0;\n  let lastError = \"unknown error\";\n  let outageSince: number | null = null;\n  let escalated = false;\n\n  // eslint-disable-next-line no-constant-condition\n  while (true) {\n    attempt++;\n    const result = await attemptPostgres(config);\n    if (\"backend\" in result) {\n      if (attempt > 1) {\n        process.stderr.write(\n          `[pai-daemon] Connected to PostgreSQL backend (after ${attempt} attempts).\\n`\n        );\n      } else {\n        process.stderr.write(\"[pai-daemon] Connected to PostgreSQL backend.\\n\");\n      }\n      // An outage that ended must stop being reported, or the status command\n      // trades one wrong answer for another.\n      clearBackendOutage();\n      if (escalated) void notifyBackendRecovered(attempt, outageSince);\n      return result.backend;\n    }\n\n    lastError = result.error;\n\n    if (!waitForever && attempt > CLI_RETRY_DELAYS_MS.length) {\n      // Bounded CLI path exhausted — fail loudly, never silently use SQLite.\n      throw new Error(\n        `Postgres backend unreachable after ${attempt} attempts: ${lastError}. ` +\n          `Is Docker Desktop / Postgres running? Refusing to fall back to SQLite ` +\n          `(would split the corpus). Start Postgres and retry.`\n      );\n    }\n\n    const delayMs = waitForever\n      ? Math.min(DAEMON_RETRY_CAP_MS, 1_000 * 2 ** Math.min(attempt - 1, 4))\n      : CLI_RETRY_DELAYS_MS[attempt - 1];\n\n    process.stderr.write(\n      `[pai-daemon] Postgres unavailable (${lastError}). ` +\n        `Retry ${attempt}${waitForever ? \"\" : `/${CLI_RETRY_DELAYS_MS.length + 1}`} ` +\n        `in ${delayMs}ms...\\n`\n    );\n\n    // Publish the outage so `pai daemon status` can report it. Without this the\n    // daemon retries silently forever and status still reads \"idle\" — which is\n    // what happened for two days in July: 144 retries over 36 minutes, session\n    // notes never written, and the one command anyone would run to check\n    // reporting that everything was fine.\n    setBackendOutage({\n      backend: \"postgres\",\n      since: outageSince ?? (outageSince = Date.now()),\n      attempts: attempt,\n      lastError: String(lastError),\n    });\n\n    // And escalate once, out loud, rather than only into a log nobody tails.\n    // Once — not per retry — because a notification that repeats every few\n    // seconds is filtered within a minute and stops being a signal at all.\n    if (waitForever && attempt === ESCALATE_AFTER_ATTEMPTS && !escalated) {\n      escalated = true;\n      void notifyBackendDown(attempt, String(lastError));\n    }\n\n    await new Promise((r) => setTimeout(r, delayMs));\n  }\n}\n\nasync function createSQLiteBackend(): Promise<StorageBackend> {\n  const { openFederation } = await import(\"../memory/db.js\");\n  const { SQLiteBackend } = await import(\"./sqlite.js\");\n  const db = openFederation();\n  return new SQLiteBackend(db);\n}\n"],"mappings":";AA2BA,IAAI,UAAgC;;AAGpC,SAAgB,iBAAiB,QAA6B;AAC5D,WAAU;;;;;;;;;AAUZ,SAAgB,qBAA2B;AACzC,WAAU;;;AAIZ,SAAgB,mBAAyC;AACvD,QAAO;;;;;;;;;AAUT,SAAgB,cAAc,IAAoB;CAChD,MAAM,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,KAAK,IAAO,CAAC;AACjD,QAAO,OAAO,KAAK,GAAG,KAAK,QAAQ,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;;;;;;AC/B/D,MAAM,sBAAsB;CAAC;CAAK;CAAO;CAAM;;AAG/C,MAAM,sBAAsB;;;;;;;;AAS5B,eAAsB,qBACpB,QACA,OAA8B,EAAE,EACP;AACzB,KAAI,OAAO,mBAAmB,WAC5B,QAAO,MAAM,gBAAgB,QAAQ,KAAK,mBAAmB,MAAM;AAIrE,QAAO,qBAAqB;;;;;;AAO9B,eAAe,gBACb,QAC0D;CAC1D,MAAM,EAAE,oBAAoB,MAAM,OAAO;CACzC,MAAM,WAAW,OAAO,YAAY,EAAE;CAEtC,IAAI,UAAuD;AAC3D,KAAI;AAEF,QAAM,gBAAgB,eAAe,SAAS;AAE9C,YAAU,IAAI,gBAAgB,SAAS;EACvC,MAAM,MAAM,MAAM,QAAQ,gBAAgB;AAC1C,MAAI,KAAK;AACP,SAAM,QAAQ,OAAO,CAAC,YAAY,GAAG;AACrC,UAAO,EAAE,OAAO,KAAK;;AAEvB,SAAO,EAAE,SAAS;UACX,GAAG;AACV,MAAI,QAAS,OAAM,QAAQ,OAAO,CAAC,YAAY,GAAG;AAClD,SAAO,EAAE,OAAO,aAAa,QAAQ,EAAE,UAAU,OAAO,EAAE,EAAE;;;;;;;;;;;AAYhE,MAAM,0BAA0B;;AAGhC,eAAe,kBAAkB,UAAkB,WAAkC;AACnF,KAAI;EACF,MAAM,EAAE,sBAAsB,MAAM,OAAO;EAC3C,MAAM,EAAE,eAAe,MAAM,OAAO;AACpC,QAAM,kBACJ;GACE,OAAO;GACP,OAAO;GACP,SACE,gCAAgC,SAAS,aAAa,UAAU;GAGnE,EACD,YAAY,CAAC,cACd;SACK;;;AAOV,eAAe,uBACb,UACA,OACe;AACf,KAAI;EACF,MAAM,EAAE,sBAAsB,MAAM,OAAO;EAC3C,MAAM,EAAE,eAAe,MAAM,OAAO;EACpC,MAAM,OAAO,QAAQ,KAAK,IAAI,GAAG,KAAK,OAAO,KAAK,KAAK,GAAG,SAAS,IAAO,CAAC,GAAG;AAC9E,QAAM,kBACJ;GACE,OAAO;GACP,OAAO;GACP,SACE,2BAA2B,SAAS,cACnC,OAAO,KAAK,KAAK,aAAa,MAC/B;GACH,EACD,YAAY,CAAC,cACd;SACK;;AAKV,eAAe,gBACb,QACA,aACyB;CACzB,IAAI,UAAU;CACd,IAAI,YAAY;CAChB,IAAI,cAA6B;CACjC,IAAI,YAAY;AAGhB,QAAO,MAAM;AACX;EACA,MAAM,SAAS,MAAM,gBAAgB,OAAO;AAC5C,MAAI,aAAa,QAAQ;AACvB,OAAI,UAAU,EACZ,SAAQ,OAAO,MACb,uDAAuD,QAAQ,eAChE;OAED,SAAQ,OAAO,MAAM,kDAAkD;AAIzE,uBAAoB;AACpB,OAAI,UAAW,CAAK,uBAAuB,SAAS,YAAY;AAChE,UAAO,OAAO;;AAGhB,cAAY,OAAO;AAEnB,MAAI,CAAC,eAAe,UAAU,oBAAoB,OAEhD,OAAM,IAAI,MACR,sCAAsC,QAAQ,aAAa,UAAU,6HAGtE;EAGH,MAAM,UAAU,cACZ,KAAK,IAAI,qBAAqB,MAAQ,KAAK,KAAK,IAAI,UAAU,GAAG,EAAE,CAAC,GACpE,oBAAoB,UAAU;AAElC,UAAQ,OAAO,MACb,sCAAsC,UAAU,WACrC,UAAU,cAAc,KAAK,IAAI,oBAAoB,SAAS,IAAI,MACrE,QAAQ,SACjB;AAOD,mBAAiB;GACf,SAAS;GACT,OAAO,gBAAgB,cAAc,KAAK,KAAK;GAC/C,UAAU;GACV,WAAW,OAAO,UAAU;GAC7B,CAAC;AAKF,MAAI,eAAe,YAAY,2BAA2B,CAAC,WAAW;AACpE,eAAY;AACZ,GAAK,kBAAkB,SAAS,OAAO,UAAU,CAAC;;AAGpD,QAAM,IAAI,SAAS,MAAM,WAAW,GAAG,QAAQ,CAAC;;;AAIpD,eAAe,sBAA+C;CAC5D,MAAM,EAAE,mBAAmB,MAAM,OAAO;CACxC,MAAM,EAAE,kBAAkB,MAAM,OAAO;AAEvC,QAAO,IAAI,cADA,gBAAgB,CACC"}