/** * Unified migration manager for SQLite databases. * * Consolidates duplicated reconciliation, bootstrap, retry, and column-safety * logic that was previously copy-pasted between sqlite.ts (tasks.db) and * memory-sqlite.ts (brain.db). Both modules now delegate to these shared functions. * * @task T132 * @see https://github.com/anthropics/cleo/issues/82 * @see https://github.com/anthropics/cleo/issues/63 * @see https://github.com/anthropics/cleo/issues/65 */ import type { DatabaseSync } from 'node:sqlite'; import type { MigrationConfig, MigrationMeta } from 'drizzle-orm/migrator'; import type { NodeSQLiteDatabase } from 'drizzle-orm/node-sqlite'; import { isSqliteBusy } from './with-retry.js'; /** * Re-export {@link isSqliteBusy} from its canonical home so existing * imports like `import { isSqliteBusy } from './migration-manager.js'` * and the public re-export chain `sqlite.ts → migration-manager.ts` * continue to compile unchanged. * * The implementation now lives in `./with-retry.ts` (gh#391). */ export { isSqliteBusy }; /** Required column definition for ensureColumns(). */ export interface RequiredColumn { name: string; /** ALTER TABLE ADD COLUMN DDL suffix (e.g., 'text', 'integer DEFAULT 0'). */ ddl: string; } /** * Strip SQL line (`-- …`) and block (`/* … *​/`) comments from a migration's SQL * before scanning it for DDL targets. * * Migration files routinely carry prose comments describing the change — phrases * like "the project-side CREATE TABLE half of that move" or "CREATE TABLE IF NOT * EXISTS ensures it exists". A DDL-extraction regex run over the RAW SQL captures * bogus "table" names from those comments (`half`, `IF`, `ensures`), making a * fully-satisfied migration look un-applied. The journal then omits it, Drizzle * re-runs its bare `CREATE TABLE …` against a DB where the table already exists, * and the open fails (root cause of the "Task database not initialized" poison). * * Always strip comments via this helper BEFORE any `createTableRegex` / * `createIndexRegex` / `alterColumnRegex` scan. The pair of replaces matches the * idiom already used by {@link isExecutableStatement} and Scenario 3's * comment-only baseline detection. * * @param sql - Raw migration SQL (one statement or many joined with `\n`) * @returns The SQL with all `--` line comments and `/* … *​/` block comments removed */ export declare function stripSqlComments(sql: string): string; /** * Compute the set of tables that a migration lineage CREATEs and a LATER * migration permanently ELIMINATES (DROP TABLE with no subsequent recreate). * * ## Why (bug #2) * * The DDL probe ({@link probeAndMarkApplied}) marks a `CREATE TABLE foo` * migration applied only if `foo` exists in the live schema. That breaks when a * later migration DROPs `foo` for good: e.g. `t033`/`initial` create * `release_manifests`, which `t9686b2` later DROPs (superseded by `releases`). * On a DB that has run the WHOLE lineage, `release_manifests` is correctly * absent — but the probe then refuses to mark `t033`/`initial` applied, so * drizzle's `migrate()` re-runs their bare `CREATE TABLE release_manifests` * against the live DB and throws ("table already exists" / cascade) — surfacing * as the opaque E_NOT_INITIALIZED / E_INTERNAL on a consolidated `cleo.db`. * * Pre-computing the eliminated set lets the probe treat a CREATE-of-an- * eliminated-table as already satisfied: the table is *supposed* to be gone, so * its absence is evidence the lineage ran, not that it didn't. * * ## Disposition tracking * * Walks migrations in `folderMillis` order and, WITHIN each migration, processes * statements IN ORDER so the SQLite rebuild/recreate idioms resolve correctly: * * - `CREATE TABLE x` (or `… RENAME TO x`) → `x` is present. * - `DROP TABLE x` → `x` is dropped. * * The rebuild idiom (`CREATE __new_x; DROP x; ALTER __new_x RENAME TO x`) and the * drop-then-recreate idiom (`DROP x; CREATE x`) both end with `x` PRESENT because * the final in-order statement for `x` is a create/rename. A table is * "eliminated" only if the LAST statement touching it across the whole lineage is * a DROP. Transient intermediates (`__new_*`) are naturally classified by the * same rule and are irrelevant to the probe (which already redirects renames to * the final name). * * @param migrations - All local migrations of one lineage (from readMigrationFiles) * @returns The set of permanently-eliminated final table names * @task T11553 */ export declare function computeEliminatedTables(migrations: ReadonlyArray<{ folderMillis: number; sql?: string | string[]; }>): Set; /** * The timestamp-prefix of the consolidation cutover migration * (`20260531000001_t11363-consolidation-cleo-{project,global}`), used as the * fallback when the consolidation migration cannot be located on disk. * * Migration folder names start with a 14-char numeric timestamp prefix * (`YYYYMMDDHHMMSS`). Comparing those prefixes lexicographically is equivalent * to comparing the encoded timestamps, so this string is the cutover boundary. * * @task T11553 */ export declare const CONSOLIDATION_CUTOVER_PREFIX = "20260531000001"; /** * Resolve the consolidation cutover timestamp-prefix for a lineage being * reconciled, by locating the `*-consolidation-cleo-*` migration in the SIBLING * consolidated folders (`drizzle-cleo-project` / `drizzle-cleo-global`) next to * the supplied `migrationsFolder`. * * The legacy lineages (`drizzle-tasks` / `drizzle-brain`) do NOT themselves * contain the consolidation migration — it lives in the consolidated folders — * so we scan the siblings. Best-effort: if neither sibling is present/readable * (partial install) the function returns {@link CONSOLIDATION_CUTOVER_PREFIX}. * * @param migrationsFolder - Absolute path to the lineage folder being reconciled * @returns The 14-char timestamp prefix of the consolidation cutover migration * @task T11553 */ export declare function resolveConsolidationCutoverPrefix(migrationsFolder: string): string; /** * Check whether a table exists in a SQLite database. */ export declare function tableExists(nativeDb: DatabaseSync, tableName: string): boolean; /** * Create a pre-migration safety backup of the database file. * * Only creates the backup once (idempotent). Non-fatal on failure. */ export declare function createSafetyBackup(dbPath: string): void; /** * Bootstrap and reconcile the Drizzle migration journal. * * Handles four scenarios: * 1. Tables exist but no __drizzle_migrations — bootstrap baseline as applied * 2. Journal has orphaned hashes (from older CLEO version) — clear and re-mark all as applied * 3. Journal exists but is missing entries for migrations whose DDL has already been applied * (e.g., ALTER TABLE ADD COLUMN ran but journal entry was never written — happens when * migrations are cherry-picked from worktrees or the process crashes mid-migration). * Auto-inserts the missing journal entry so Drizzle skips the migration instead of * re-running ALTER TABLE and crashing on "duplicate column name". * 4. Journal entries exist but have null `name` — Drizzle v1 beta identifies applied * migrations by name, so entries without a name are invisible to it, causing already- * applied migrations to be re-run and fail with "duplicate column name". Backfills * the name from the local migration file matched by hash. * * ## Cross-lineage orphan-deletion guard (T11829 — OOM root fix) * * The consolidated `cleo.db` carries ONE shared `__drizzle_migrations` journal but * is reconciled by MULTIPLE physically-coexisting migration lineages * (`drizzle-tasks`, `drizzle-cleo-project`, `drizzle-nexus`, `drizzle-brain`, * `drizzle-agent-registry`, `drizzle-conduit`). Each lineage only knows its OWN * folder, so a naive Sub-case B classified EVERY sibling lineage's journal rows as * "true orphans" and DELETEd them — the next sibling open then deleted THIS * lineage's rows, so the journal NEVER converged (it oscillated and re-ran * BEGIN/COMMIT migrate transactions on every open, holding the WAL writer lock and * stacking 300-550 MB-per-connection opens until the host OOM-killed). * * The fix: an entry is a TRUE orphan only when its hash belongs to NO lineage that * physically shares this DB. Callers that share the journal pass * `siblingMigrationsFolders` (the OTHER lineages' folders); the orphan-DELETE * decision then uses the UNION of every sibling lineage's hashes plus this * lineage's own. A hash present in any sibling is preserved (it is that sibling's * legitimately-applied migration, not an orphan). Standalone DBs (a single * lineage) pass no siblings and behave exactly as before. * * @param nativeDb - Native SQLite database handle * @param migrationsFolder - Path to the drizzle migrations folder * @param existenceTable - A table name used to detect if the DB has data (e.g., 'tasks' or 'brain_decisions') * @param logSubsystem - Logger subsystem name for reconciliation warnings * @param siblingMigrationsFolders - OTHER migration-lineage folders that share this * `__drizzle_migrations` journal inside the same consolidated `cleo.db` (T11829). * Their hashes are added to the orphan-deletion union so a sibling lineage's rows * are never deleted as cross-lineage orphans. Omit (or pass `[]`) for a DB with a * single lineage. Unreadable/empty sibling folders are skipped defensively. */ export declare function reconcileJournal(nativeDb: DatabaseSync, migrationsFolder: string, existenceTable: string, logSubsystem: string, siblingMigrationsFolders?: readonly string[]): void; /** * Reconcile + apply the standalone `drizzle-brain` migrations against a CONSOLIDATED * `cleo.db` whose prefixed `brain_*` tables were already created by the * consolidated migration (T11647). For each not-yet-journaled `drizzle-brain` * migration, this either: * * 1. **Marks it applied without running** when its net effect is ALREADY present: * every `CREATE TABLE` / `RENAME TO` result table exists, OR it is an * ALTER-ADD-COLUMN-only migration whose columns all already exist (the * consolidated migration created the prefixed tables with their FINAL * columns). This avoids the `CREATE TABLE`/rename collisions and * "duplicate column" errors a wholesale re-run would hit. * 2. **Executes it directly** (native `exec`, then journals it) when it is * genuinely missing — the UNPREFIXED legacy runtime tables (`deriver_queue`, * `sticky_tags`, `session_narrative`, `brain_task_observations`) the * consolidated migration omits but the runtime queries. Those migrations are * all `IF NOT EXISTS` DDL + idempotent `INSERT OR IGNORE` backfills, so a * direct exec is safe. * * ## Why NOT {@link reconcileJournal} + drizzle `migrate()` * * The brain `__drizzle_migrations` journal is SHARED with the TASKS domain inside * the same consolidated `cleo.db`. On a first consolidated open the brain hashes * are not yet journaled, so `reconcileJournal`'s Scenario-2 path classifies the * TASKS-domain hashes as "orphans" of the brain migration set and DELETES them — * which forces the tasks domain to re-run its table-rebuild migrations on the * NEXT tasks open and crash (`tasks_new RENAME TO tasks` fires the * `task_relations_non_containment_insert` trigger against a mid-rebuild `tasks`). * And drizzle's `migrate()` wraps the brain DDL in a `BEGIN`/`COMMIT` over the * shared handle. This function is **additive-only** (it never deletes a journal * row) and uses plain `nativeDb.exec()`, so it cannot corrupt the shared journal * or engage the cross-domain transaction. * * Idempotent: dedups by hash (the journal has no UNIQUE on `hash`, so an explicit * presence check is required). On a brand-new / standalone `brain.db` (no * prefixed brain tables yet) every migration is "genuinely missing" and is * executed in order — equivalent to a full migrate. * * @param nativeDb - Native SQLite handle (the consolidated `cleo.db`). * @param migrationsFolder - The `drizzle-brain` migrations folder. * @returns `{ marked, applied }` — counts of migrations journaled-without-running * vs executed-and-journaled. * * @task T11647 */ export declare function reconcileBrainMigrationsForConsolidatedDb(nativeDb: DatabaseSync, migrationsFolder: string): { marked: number; applied: number; }; /** * Check whether an error is a SQLite "duplicate column name" error. * * These are thrown when an ALTER TABLE ADD COLUMN statement is re-executed * after the column was already added (Scenario 3 in reconcileJournal). * * Inspects the full `.cause` chain so a Drizzle-wrapped error (whose real SQLite * message lives in `.cause.message`) is still recognised. */ export declare function isDuplicateColumnError(err: unknown): boolean; /** * T9174: Check for "table X already exists" SQLite error. * Occurs when migration SQL lacks IF NOT EXISTS but startup DDL already ran. * * Inspects the full `.cause` chain so a Drizzle-wrapped error (whose real SQLite * message lives in `.cause.message`) is still recognised. */ export declare function isTableAlreadyExistsError(err: unknown): boolean; /** * Filter whitespace-only and comment-only statements from drizzle's readMigrationFiles output. * * Guards against two classes of non-executable statements: * * 1. **Whitespace-only** — caused by trailing `--> statement-breakpoint` markers. * drizzle-orm splits migration files on that marker, producing array entries * that are purely whitespace (e.g., `"\n"`). Passing that to * `session.run(sql.raw("\n"))` causes a "Failed to run the query '\n'" crash. * * 2. **Comment-only** — the T1165 probe-and-skip baseline marker pattern. A * migration.sql that contains only SQL comments (no executable DDL) is the * designated anchor for drizzle-kit's snapshot chain. On fresh installs the * baseline migration should be a no-op; on existing installs reconcileJournal * Scenario 3 pre-marks it applied before migrate() is called. Either way, * drizzle's session.run() must never receive a comment-only SQL string — * node:sqlite's `prepare()` rejects it with `ERR_INVALID_STATE`. * * This function is idempotent: migrations with only real DDL pass through unchanged. * * @param migrations - Array returned by readMigrationFiles * @returns A new array where every migration's `.sql` property has been * filtered to remove whitespace-only and comment-only entries. */ export declare function sanitizeMigrationStatements(migrations: MigrationMeta[]): MigrationMeta[]; /** * Run drizzle migrations after sanitizing whitespace-only SQL statements. * * This is a drop-in replacement for drizzle's `migrate()` from * `drizzle-orm/node-sqlite/migrator`. It reads migration files via * `readMigrationFiles`, filters any empty/whitespace-only statement chunks, * then delegates to `db.dialect.migrate()` directly — bypassing the * re-read that drizzle's `migrate()` would perform internally. * * Use this at every call site instead of drizzle's `migrate()` to defend * against malformed migration files regardless of authoring discipline. * * @param db - Drizzle NodeSQLiteDatabase instance * @param config - Migration config (migrationsFolder required) */ export declare function migrateSanitized(db: NodeSQLiteDatabase, config: MigrationConfig): void; /** * Run Drizzle migrations with SQLITE_BUSY retry and exponential backoff. * * Also handles "duplicate column name" errors (Scenario 3): if Drizzle tries to * re-apply a migration whose DDL columns already exist (journal entry missing), * this function calls reconcileJournal again to insert the missing entry and * retries migrate() once more. This is the belt-and-suspenders safety net for * any partial migration that slips through the proactive reconcileJournal check. * * Uses migrateSanitized internally to filter whitespace-only SQL statements * before they reach drizzle's session.run() (T1159 defense-in-depth). * * @param db - Drizzle database instance * @param migrationsFolder - Path to the drizzle migrations folder * @param nativeDb - Optional native SQLite handle for duplicate-column auto-reconcile * @param existenceTable - Optional existence-check table name for auto-reconcile * @param logSubsystem - Optional logger subsystem name for auto-reconcile warnings */ export declare function migrateWithRetry(db: NodeSQLiteDatabase, migrationsFolder: string, nativeDb?: DatabaseSync, existenceTable?: string, logSubsystem?: string): void; /** * Ensure all required columns exist on a table. * * Uses PRAGMA table_info to inspect the schema and adds any missing columns * via ALTER TABLE ADD COLUMN. Safety net for databases where Drizzle migrations * could not run due to journal corruption or version skew. * * The `context` parameter (T9169) categorizes the call site so log levels can * reflect intent: * * - **`'legacy-upgrade'`** (default): the call is a compatibility safety net * for a DB created by an older CLEO version. Missing columns are EXPECTED * and the repair is informational. Logs at WARN level. * * - **`'fresh'`**: the call follows a successful migration run on a * newly-created DB. Missing columns indicate a migration-chain DEFECT * (missing forward migration or breakpoint truncation). Logs at ERROR * level so the CI schema-warning gate (T9170) can fail the build. * * @param nativeDb - Native SQLite database handle * @param tableName - Table to check (e.g., 'tasks') * @param requiredColumns - Columns that must exist * @param logSubsystem - Logger subsystem name * @param context - 'legacy-upgrade' (default, WARN) or 'fresh' (ERROR) */ export declare function ensureColumns(nativeDb: DatabaseSync, tableName: string, requiredColumns: RequiredColumn[], logSubsystem: string, context?: 'legacy-upgrade' | 'fresh'): void; //# sourceMappingURL=migration-manager.d.ts.map