/** * Database Layer * * Handles SQLite database initialization and connection management. */ import { SqliteDatabase, SqliteBackend } from './sqlite-adapter'; import { SchemaVersion } from '../types'; export { SqliteDatabase, SqliteBackend } from './sqlite-adapter'; /** * WAL size past which `healOversizedWal` (run at every `open`) checkpoints and * truncates the file, and to which `journal_size_limit` clips the WAL after any * resetting checkpoint. A SIGKILL'd process (the #850 liveness watchdog, OOM, * crash) can leave an arbitrarily large WAL behind — a whole deferred-sync * run's worth (#1248) — and before #1431 no later session ever shrank it: the * file just grew, killed session after killed session, until the disk filled * (25.6 GB observed). 64 MB is far above anything a healthy open ever sees * (a clean close deletes the WAL) yet small enough to cap the leak. * Override with `LATTICE_SENSOR_WAL_HEAL_MB` (also feeds `journal_size_limit`). */ export declare const WAL_HEAL_THRESHOLD_BYTES: number; /** Resolve the heal threshold from the env override (MB); invalid ⇒ 64 MB. */ export declare function resolveWalHealBytes(envVal: string | undefined): number; /** * Database connection wrapper with lifecycle management */ export declare class DatabaseConnection { private db; private dbPath; private backend; /** * `dev:ino` of the DB file at the moment we opened it (or null when the * platform/filesystem reports no usable inode). Lets us notice when the file * we hold open has been unlinked and REPLACED by a new file at the same path * — a git worktree removed and re-added, or `.lattice/sensor/` deleted and * re-`init`ed under a long-lived server — at which point our fd reads a now * dead inode forever (#925). See `isReplacedOnDisk`. */ private openedInode; private constructor(); /** * Initialize a new database at the given path */ static initialize(dbPath: string): DatabaseConnection; /** * Open an existing database */ static open(dbPath: string): DatabaseConnection; /** * FTS maintenance triggers dropped/recreated around a bulk load. * Names must match schema.sql. */ private static readonly FTS_TRIGGER_NAMES; /** * Enter bulk-load mode: drop the per-row FTS sync triggers so mass node * inserts skip per-row tokenization. MUST be paired with endBulkNodeLoad() * (use try/finally); a crash inside the window is healed on the next open(). * The window is DB-wide (triggers are schema objects), which is safe because * endBulkNodeLoad() rebuilds nodes_fts from the nodes table wholesale — any * row written by anyone during the window is captured by the rebuild. */ beginBulkNodeLoad(): void; /** * Leave bulk-load mode: rebuild the whole FTS index from the nodes table in * one pass (far cheaper than per-row trigger firings), then recreate the * triggers by re-running schema.sql (idempotent — everything in it is * IF NOT EXISTS). */ endBulkNodeLoad(): void; /** * NON-UNIQUE secondary indexes maintained per-row during the parse phase's * bulk inserts — the store-architecture arc's first lever (plan §4d: dubbo's * parse-loop wall is 94% store-writer busy, and the #1320 post-mortem showed * statement batching and sorted inserts are ~zero on this path because * B-TREE MAINTENANCE is the floor). A fresh init writes every row of * nodes/unresolved_refs/files exactly once and reads none of them until * resolution, so the parse window can drop all of these and rebuild each in * one table scan afterwards — the same measured trade as the resolution * phase's edge-index window (2.8s → 1.1s inserting, ~0.3s recreating). * Primary keys and UNIQUE constraints stay (upserts and OR-IGNORE dedup * conflict on them). */ private static readonly BULK_PARSE_INDEX_NAMES; /** * Enter bulk-parse-load mode (FRESH-INIT ONLY — the caller gates on a fresh * DB, because an incremental index deletes per-file rows mid-phase and needs * the file_path indexes): drop every parse-lane secondary index, including * the four non-unique edge indexes (parse inserts contains-edges too; the * UNIQUE identity index stays for INSERT OR IGNORE dedup, and its `source` * prefix keeps source-keyed reads indexed, as in the edge window). MUST be * paired with endBulkParseLoad(); a crash inside the window is healed on the * next DatabaseConnection open (schema.sql re-applies CREATE INDEX IF NOT * EXISTS). */ beginBulkParseLoad(): void; /** * Leave bulk-parse-load mode: recreate everything the window dropped, one * table scan per index, with a yield between statements (same * liveness-watchdog rationale as endBulkEdgeLoad — at kernel scale each * build is a long synchronous scan). The edge indexes are rebuilt here too, * so paths that never enter the resolution phase's own bulk-edge window * (small runs) are left with a complete schema; the batched resolver's * beginBulkEdgeLoad simply re-drops them (DROP IF EXISTS — idempotent). */ endBulkParseLoad(): Promise; /** * unresolved_refs secondary indexes NOT read by the batched resolution * loop. The loop pages pending refs by keyset (`status='pending' AND id>?` * — the status index + PK), deletes resolved rows by id, and parks failures * with a status UPDATE; every other ref index serves SYNC-time paths * (per-file re-index deletes, name-keyed retry, failed-tail heal). Each * per-batch DELETE maintains all of them — the biggest single main-thread * stage on the dubbo profile (deletes 1.2s of a 5.4s resolution phase) — * so the batched loop drops them and rebuilds at the end, where the table * holds only the surviving FAILED refs (resolved rows are gone), making * the recreate near-free. */ private static readonly BULK_REF_INDEX_NAMES; /** * Enter bulk-ref mode for the batched resolution loop — see * BULK_REF_INDEX_NAMES. MUST be paired with endBulkRefLoad(); a crash * inside the window heals on the next open (schema.sql re-applies * CREATE INDEX IF NOT EXISTS). */ beginBulkRefLoad(): void; /** Leave bulk-ref mode: recreate each index in one scan (yield between). */ endBulkRefLoad(): Promise; /** * Names of the NON-UNIQUE edge indexes dropped for a bulk edge load. * idx_edges_identity deliberately stays: INSERT OR IGNORE's dedup conflicts * on it (#1034), and its leftmost column is `source`, so the source-keyed * reads resolution makes mid-window (supertype walks over * `implements`/`extends`) keep an index via its prefix — verified with * EXPLAIN QUERY PLAN. Target-keyed and kind-keyed reads (traversal, * synthesis) happen only after endBulkEdgeLoad(). */ private static readonly BULK_EDGE_INDEX_NAMES; /** * Enter bulk-edge-load mode: drop the non-unique edge indexes so the mass * INSERT OR IGNORE stream pays one B-tree (the identity index) instead of * five — measured 2.8s → 1.1s inserting a 224k-edge resolution set, with * recreation costing ~0.3s. MUST be paired with endBulkEdgeLoad(); a crash * inside the window is healed on the next DatabaseConnection open (schema.sql * re-applies CREATE INDEX IF NOT EXISTS). */ beginBulkEdgeLoad(): void; /** * Leave bulk-edge-load mode: recreate the dropped indexes in one pass each * over the (now fully loaded) edges table — far cheaper than maintaining * them per-insert. DDL is extracted from schema.sql so it cannot drift. * * Async with a yield BETWEEN the four CREATE INDEX statements: each build is * a synchronous scan of the whole edges table (~20s apiece at Linux-kernel * scale, 79s total measured), and running them back-to-back is a single * event-loop stall longer than the #850 liveness watchdog's 60s window — a * daemon-triggered re-index would be SIGKILLed right after doing the work. * One yield per statement keeps every stall to a single index build, which * stays inside the window. */ endBulkEdgeLoad(): Promise; /** Recreate the FTS triggers + rebuild if a bulk-load window never closed. */ private healBulkNodeLoad; /** * Recreate the FTS sync triggers from schema.sql — extracted from the file * rather than duplicated here so the DDL cannot drift from the schema. * (Re-execing the whole schema is not an option: it contains data INSERTs * that are not idempotent, e.g. schema_versions.) */ private recreateFtsTriggers; /** * Get the underlying database instance */ getDb(): SqliteDatabase; /** * Get the SQLite backend serving this connection. Per-instance so * MCP cross-project queries report the right backend even when * multiple project DBs are open in the same process. */ getBackend(): SqliteBackend; /** * Get database file path */ getPath(): string; /** * The journal mode actually in effect (e.g. 'wal', 'delete'). * * SQLite silently keeps the prior mode if WAL can't be enabled — e.g. on * filesystems without shared-memory support (some network/virtualized mounts, * WSL2 /mnt). So the effective mode can differ * from what `configureConnection` requested. Surfaced in `lattice sensor status` so * a "database is locked" report is triageable: 'wal' ⇒ readers never block on a * writer; anything else ⇒ they can. See issue #238. */ getJournalMode(): string; /** * Get current schema version */ getSchemaVersion(): SchemaVersion | null; /** * Execute a function within a transaction */ transaction(fn: () => T): T; /** * Get database file size in bytes */ getSize(): number; /** * Size of the `-wal` sidecar file in bytes. 0 when it doesn't exist (non-WAL * journal mode, in-memory DB, or no write since the last checkpoint+reset). */ getWalSizeBytes(): number; /** Size of the main DB file in bytes (0 for in-memory/unknown) — the WAL * valve scales its fold caps with it (resolveWalValveMb). */ getDbFileSizeBytes(): number; /** Current `wal_autocheckpoint` interval in pages (0 = disabled). */ getWalAutocheckpoint(): number; /** * Set the connection's `wal_autocheckpoint` interval (pages; 0 disables). * Bulk indexing defers checkpoints entirely (#1231): the default 1000-page * auto-checkpoint re-writes hot B-tree/FTS pages into the main DB file over * and over — measured at ~95% of ALL disk I/O during a bulk index, and the * difference between 45s and 19+ minutes on HDD-class storage. During * deferral a {@link WalCheckpointValve} bounds WAL growth off-thread. */ setWalAutocheckpoint(pages: number): void; /** * `PRAGMA wal_checkpoint(PASSIVE)` on a worker thread with its own * connection. PASSIVE never blocks the writer, and running it off-thread * means the main thread — and the #850 watchdog heartbeat — keep turning * even when the backfill is minutes of I/O on slow storage (a synchronous * checkpoint that exceeds the watchdog's 60s window gets a healthy index * SIGKILLed — observed in the #1231 repro). * * Returns SQLite's checkpoint result row — `log === checkpointed` with * `busy === 0` means the ENTIRE WAL was backfilled, so the writer's next * commit restarts the WAL from the top and the file stops growing. The * WAL valve needs that signal because a WAL file's SIZE never shrinks: * after the first wrap, raw file size says nothing about the un-backfilled * backlog. Best-effort: returns null on any failure (including worker * threads being unavailable — a potentially minutes-long checkpoint must * never run inline on the main thread). */ checkpointWalPassive(): Promise<{ busy: number; log: number; checkpointed: number; } | null>; /** * `PRAGMA wal_checkpoint(TRUNCATE)` — same off-thread pattern as PASSIVE, * but on success the WAL FILE is chopped to zero. A completed passive * backfill bounds the un-checkpointed backlog, yet the FILE only stops * growing when a commit finds ZERO readers holding WAL marks — rare while * pool workers cycle, so at kernel scale a fully-backfilled WAL still * accreted the phase's whole write volume on disk (§7a.1: 22GB). The valve * calls this exactly at a parked barrier (writer parked, pool drained, * backfill complete) where the no-reader condition is guaranteed rather * than lucky. The worker sets a short busy_timeout so a racing reader * degrades this to a no-op (busy=1) instead of a stall. */ checkpointWalTruncate(): Promise<{ busy: number; log: number; checkpointed: number; } | null>; /** * Shrink a leftover oversized WAL (#1431). A SIGKILL'd session — the #850 * liveness watchdog, OOM, a crash — leaves its WAL on disk, the next session * appends to the same file, and (pre-#1431) nothing ever truncated it: * PASSIVE checkpoints fold frames but keep the file at its high-water mark, * and the one shrinking path (a clean last-connection close) is exactly what * the killed world never takes. Unbounded growth until the disk fills. * * Called fire-and-forget from every `open()`: cost is one statSync when the * WAL is small (the overwhelmingly common case). Past the threshold it runs * the off-thread PASSIVE fold then TRUNCATE — both on worker connections * with a busy_timeout, so a racing writer degrades this to a no-op that the * next open retries rather than a stall. */ healOversizedWal(): Promise<{ healed: boolean; beforeBytes: number; afterBytes: number; }>; private walHeal; private runWalHeal; private checkpointWal; /** * Optimize database (vacuum and analyze) */ optimize(): void; /** * Lightweight maintenance to run after bulk writes (indexAll, sync). * Two operations: * * - `PRAGMA optimize` — incremental ANALYZE; SQLite only re-analyzes * tables whose row counts changed materially since the last * ANALYZE. Without it, the query planner has no statistics on the * freshly-bulk-loaded tables and can pick suboptimal indexes. * * - `PRAGMA wal_checkpoint(PASSIVE)` — fold pending WAL pages back * into the main database file so the WAL file doesn't grow * unboundedly between automatic checkpoints (auto-fires at 1000 * pages by default; large indexAll runs blow past that). * * Runs on a WORKER THREAD with its own connection: on a multi-GB index * these pragmas are minutes of synchronous IO (a 95k-file kernel index * left a 593MB WAL whose checkpoint alone blew the #850 watchdog's 60s * window and got a COMPLETED index SIGKILLed at the finish line). WAL * checkpointing from a second connection is standard SQLite; `PRAGMA * optimize` persists its statistics in sqlite_stat tables, so the main * connection benefits the same. The main thread just awaits a message, * so the event loop — and the watchdog heartbeat — keep turning. * * Everything is silently swallowed on failure — best-effort * optimization, never load-bearing for correctness. If worker threads * are unavailable, falls back to a bounded in-line `PRAGMA optimize` * and SKIPS the checkpoint (the final close() checkpoints after the * CLI has already disarmed its watchdog). */ runMaintenance(): Promise; /** * Run pragmas on a worker thread against its own connection to this DB * (shared machinery for {@link runMaintenance} and * {@link checkpointWalPassive}). Each pragma is individually best-effort; * the whole call is best-effort. `inlineFallback` (if any) runs on THIS * connection only when worker threads are unavailable — keep it to pragmas * that are safe to run synchronously on the main thread. */ private runPragmasOffThread; /** * Close the database connection */ close(): void; /** * Check if the database connection is open */ isOpen(): boolean; /** * True when the DB file at our path has been REPLACED on disk since we opened * it — a different inode now lives at the same path, so the fd we still hold * points at a now-unlinked inode that can never receive new writes (#925). * The trigger is removing and recreating `.lattice/sensor/` at the same path under * a long-lived process (`git worktree remove` + re-add, or `rm -rf * .lattice/sensor` + `lattice sensor init`). Returns false when the inode is unchanged, * when the file is momentarily absent (mid-recreate — nothing to reopen onto * yet), or when the platform doesn't report a usable inode (Windows can't * unlink an open file and its st_ino is unreliable, so this never fires there). */ isReplacedOnDisk(): boolean; } /** * Default database filename */ export declare const DATABASE_FILENAME = "sensor.db"; /** * Get the default database path for a project */ export declare function getDatabasePath(projectRoot: string): string; /** * Delete a database file and its WAL sidecars (`-wal`/`-shm`). * * This is how a FULL re-index discards an existing database — rather than * opening the old graph and DELETE-ing every row. On a large or pre-fix * poisoned index (e.g. an old graph that scanned an ignored gitlink corpus into * ~1.6M nodes with a multi-GB WAL, #1065) the per-row `nodes_fts` delete-trigger * churn blocks the main thread long enough to trip the #850 liveness watchdog * before indexing even starts, so the rebuild could never recover the bad state * (#1067). Unlinking is O(1) regardless of DB size and also reclaims the disk * the bloated WAL would otherwise keep. * * POSIX removes the directory entry even while another process (a daemon/MCP * server) still holds the file open; that holder heals via `reopenIfReplaced` * (#925). On Windows a live holder can make the unlink fail with EBUSY/EPERM — * that is thrown for the caller to surface ("stop the other process and retry"). * The `-wal`/`-shm` sidecars are best-effort: SQLite recreates them on the next * open, so a leftover sidecar is harmless. */ export declare function removeDatabaseFiles(dbPath: string): void; //# sourceMappingURL=index.d.ts.map