/** * pi-loom: Shared Types & Helpers * * Extracted from store.ts to reduce file size and improve modularity. * All types and helper functions live here — no database dependency. */ import { randomBytes } from "node:crypto"; import { existsSync, mkdirSync } from "node:fs"; import { createRequire } from "node:module"; import { join } from "node:path"; import Database from "better-sqlite3"; import { EMBED_DIM } from "./embed.js"; // ═══════════════════════════════════════════════════════════════ // Types // ═══════════════════════════════════════════════════════════════ export type MemStatus = "active" | "expired" | "archived"; /** How a memory was created — replaces the old JSON provenance object. */ export type Provenance = | "direct" // explicit loom_store | "auto_captured" // auto-capture pattern match | "extracted_fact" // loom_extract atomic fact | "consolidated_pattern" // RecMem consolidation | "dream_insight" // Dream Engine insight (also used for entity_profile sub-type) | "entity_profile"; // deprecated — merged into dream_insight, kept for migration /** A link in the derivation chain: what this memory was derived from. */ export interface DerivationLink { id: string; // event_id or memory_id type: "event" | "memory"; weight: number; // 0.0–1.0 } export interface MemRow { id: string; content: string; fact_summary: string | null; entity_id: string | null; kind: string | null; scope_type: string | null; scope_id: string | null; confidence: number | null; visibility: string | null; valid_at: string; expire_at: string | null; importance: number; status: MemStatus; tags: string | null; content_hash: string | null; created_at: string; recall_count: number; last_recalled_at: string | null; hit_count: number; last_hit_at: string | null; session_ids: string | null; provenance: string | null; // Provenance enum as string, or legacy JSON for old rows derivation: string | null; // JSON array of DerivationLink } export interface RawEventRow { id: string; session_id: string; event_type: string; payload: string; created_at: string; } export interface SessionSummaryRow { id: string; session_id: string; summary: string; decisions: string | null; errors: string | null; changes: string | null; unfinished: string | null; memory_ids: string | null; created_at: string; } export interface EpisodeRow { episode_id: string; session_id: string; summary: string | null; entity_id: string | null; token_count: number | null; created_at: string; } export interface EntityEdgeRow { edge_id: string; source_entity: string; target_entity: string; relation_type: string; memory_id: string | null; episode_id: string | null; confidence: number; created_at: string; } export interface MemoryEdgeRow { edge_id: string; source_id: string; target_id: string; relation: string; confidence: number; created_at: string; } export interface ConstraintRow { id: string; entity_id: string; description: string; path_condition: string | null; enforcement: "warn" | "block" | "log"; state: "active" | "satisfied" | "violated" | "archived"; created_at: string; last_checked_at: string | null; } // ═══════════════════════════════════════════════════════════════ // Helpers // ═══════════════════════════════════════════════════════════════ export function parseTags(tags: string | null): string[] { if (!tags) return []; try { return JSON.parse(tags); } catch { return []; } } export function genId(): string { return randomBytes(12).toString("hex"); } const CHARS_PER_TOKEN = 4; export function estimateTokens(text: string): number { return Math.ceil(text.length / CHARS_PER_TOKEN); } // ═══════════════════════════════════════════════════════════════ // Database helpers // ═══════════════════════════════════════════════════════════════ export function getDbDir(): string { if (process.env.PI_LOOM_DIR) return process.env.PI_LOOM_DIR; return join(process.cwd(), ".pi-loom"); } export function getDbPath(): string { return join(getDbDir(), "loom.db"); } export function openDb(): Database.Database { const dir = getDbDir(); if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); const db = new Database(getDbPath()); db.pragma("journal_mode = WAL"); db.pragma("foreign_keys = ON"); // Load sqlite-vec extension (best-effort, synchronous) try { const _require = createRequire(import.meta.url); const { load: loadVec } = _require("sqlite-vec"); loadVec(db); } catch { // sqlite-vec not available — vector search disabled, FTS5 still works } db.exec(` CREATE TABLE IF NOT EXISTS memories ( id TEXT PRIMARY KEY, content TEXT NOT NULL, fact_summary TEXT, entity_id TEXT, kind TEXT NOT NULL DEFAULT 'memory', scope_type TEXT NOT NULL DEFAULT 'repo', scope_id TEXT, confidence REAL NOT NULL DEFAULT 1.0, visibility TEXT NOT NULL DEFAULT 'project', valid_at TEXT NOT NULL DEFAULT (datetime('now')), expire_at TEXT, importance REAL NOT NULL DEFAULT 0.5, status TEXT NOT NULL DEFAULT 'active' CHECK(status IN ('active','expired','archived')), tags TEXT, content_hash TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')), recall_count INTEGER NOT NULL DEFAULT 0, last_recalled_at TEXT, hit_count INTEGER NOT NULL DEFAULT 1, last_hit_at TEXT, session_ids TEXT NOT NULL DEFAULT '[]', provenance TEXT NOT NULL DEFAULT 'direct', derivation TEXT NOT NULL DEFAULT '[]' ); CREATE INDEX IF NOT EXISTS idx_mem_entity ON memories(entity_id); CREATE INDEX IF NOT EXISTS idx_mem_status ON memories(status); CREATE INDEX IF NOT EXISTS idx_mem_expire ON memories(expire_at); CREATE INDEX IF NOT EXISTS idx_mem_importance ON memories(importance DESC); CREATE INDEX IF NOT EXISTS idx_mem_hash ON memories(content_hash); -- FTS5 full-text search: porter stemming for keyword normalization CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5( content, fact_summary, tokenize='porter unicode61' ); -- v1.0: Evidence-first vec_memories (replaces legacy memories_emb) -- Uses rowid as primary key to align with FTS5 — single source of truth. -- sqlite-vec KNN requires the embedding column MATCH syntax. CREATE VIRTUAL TABLE IF NOT EXISTS vec_memories USING vec0( embedding float[${EMBED_DIM}] ); -- Raw event audit log: first observed → processed → aggregated CREATE TABLE IF NOT EXISTS raw_events ( id TEXT PRIMARY KEY, session_id TEXT NOT NULL, event_type TEXT NOT NULL, payload TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS idx_raw_event_session ON raw_events(session_id); CREATE INDEX IF NOT EXISTS idx_raw_event_type ON raw_events(event_type); -- Session summaries (LLM-generated — decisions, errors, changes, unfinished) CREATE TABLE IF NOT EXISTS session_summaries ( id TEXT PRIMARY KEY, session_id TEXT NOT NULL, summary TEXT NOT NULL, decisions TEXT, errors TEXT, changes TEXT, unfinished TEXT, memory_ids TEXT, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); -- Episodes: session-level temporal entities CREATE TABLE IF NOT EXISTS episodes ( episode_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, summary TEXT, entity_id TEXT, token_count INTEGER, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); -- Entity graph edges (typed relations between entities) CREATE TABLE IF NOT EXISTS entity_edges ( edge_id TEXT PRIMARY KEY, source_entity TEXT NOT NULL, target_entity TEXT NOT NULL, relation_type TEXT NOT NULL, memory_id TEXT, episode_id TEXT, confidence REAL DEFAULT 0.5, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS idx_edges_source ON entity_edges(source_entity); CREATE INDEX IF NOT EXISTS idx_edges_target ON entity_edges(target_entity); -- Generic memory graph edges. Entity edges remain as a compatibility projection. CREATE TABLE IF NOT EXISTS memory_edges ( edge_id TEXT PRIMARY KEY, source_id TEXT NOT NULL, target_id TEXT NOT NULL, relation TEXT NOT NULL, confidence REAL DEFAULT 0.5, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS idx_memory_edges_source ON memory_edges(source_id); CREATE INDEX IF NOT EXISTS idx_memory_edges_target ON memory_edges(target_id); CREATE INDEX IF NOT EXISTS idx_memory_edges_relation ON memory_edges(relation); -- Path-conditioned constraints (v1.0) CREATE TABLE IF NOT EXISTS constraints ( id TEXT PRIMARY KEY, entity_id TEXT NOT NULL, description TEXT NOT NULL, path_condition TEXT, enforcement TEXT NOT NULL DEFAULT 'warn', state TEXT NOT NULL DEFAULT 'active', created_at TEXT NOT NULL DEFAULT (datetime('now')), last_checked_at TEXT ); `); // ── Schema migrations (additive, safe to re-run) ── // Migration: add fact_summary column if missing const hasFacts = db .prepare("SELECT COUNT(*) as cnt FROM pragma_table_info('memories') WHERE name = 'fact_summary'") .get() as { cnt: number }; if (hasFacts.cnt === 0) { db.exec(`ALTER TABLE memories ADD COLUMN fact_summary TEXT`); console.error("[pi-loom] Migrated: added fact_summary column to memories"); } // Migration: Phase 3 — subconscious tracking (hit_count + last_hit_at) const migrations3: Array<[string, string, string]> = [ ["hit_count", "INTEGER NOT NULL DEFAULT 1", "subconscious hit counter"], ["last_hit_at", "TEXT", "most recent hit timestamp"], ]; for (const [col, def, desc] of migrations3) { const has = db.prepare("SELECT COUNT(*) as cnt FROM pragma_table_info('memories') WHERE name = ?").get(col) as { cnt: number; }; if (has.cnt === 0) { db.exec(`ALTER TABLE memories ADD COLUMN ${col} ${def}`); console.error(`[pi-loom] Migrated: added ${col} column to memories (${desc})`); } } // Migration: cross-session tracking (session_ids) { const has = db .prepare("SELECT COUNT(*) as cnt FROM pragma_table_info('memories') WHERE name = ?") .get("session_ids") as { cnt: number }; if (has.cnt === 0) { db.exec("ALTER TABLE memories ADD COLUMN session_ids TEXT NOT NULL DEFAULT '[]'"); } } // Migration: memory provenance chain { const has = db .prepare("SELECT COUNT(*) as cnt FROM pragma_table_info('memories') WHERE name = ?") .get("provenance") as { cnt: number }; if (has.cnt === 0) { db.exec("ALTER TABLE memories ADD COLUMN provenance TEXT NOT NULL DEFAULT '{}'"); } } // Migration: derivation DAG (v1.0 Evidence) { const has = db .prepare("SELECT COUNT(*) as cnt FROM pragma_table_info('memories') WHERE name = ?") .get("derivation") as { cnt: number }; if (has.cnt === 0) { db.exec("ALTER TABLE memories ADD COLUMN derivation TEXT NOT NULL DEFAULT '[]'"); } } // Migration: MemoryNode metadata (kind/scope/confidence/visibility) const nodeMigrations: Array<[string, string]> = [ ["kind", "TEXT NOT NULL DEFAULT 'memory'"], ["scope_type", "TEXT NOT NULL DEFAULT 'repo'"], ["scope_id", "TEXT"], ["confidence", "REAL NOT NULL DEFAULT 1.0"], ["visibility", "TEXT NOT NULL DEFAULT 'project'"], ]; for (const [col, def] of nodeMigrations) { const has = db .prepare("SELECT COUNT(*) as cnt FROM pragma_table_info('memories') WHERE name = ?") .get(col) as { cnt: number }; if (has.cnt === 0) { db.exec(`ALTER TABLE memories ADD COLUMN ${col} ${def}`); } } db.exec(` CREATE INDEX IF NOT EXISTS idx_mem_kind ON memories(kind); CREATE INDEX IF NOT EXISTS idx_mem_scope ON memories(scope_type, scope_id); CREATE TABLE IF NOT EXISTS memory_edges ( edge_id TEXT PRIMARY KEY, source_id TEXT NOT NULL, target_id TEXT NOT NULL, relation TEXT NOT NULL, confidence REAL DEFAULT 0.5, created_at TEXT NOT NULL DEFAULT (datetime('now')) ); CREATE INDEX IF NOT EXISTS idx_memory_edges_source ON memory_edges(source_id); CREATE INDEX IF NOT EXISTS idx_memory_edges_target ON memory_edges(target_id); CREATE INDEX IF NOT EXISTS idx_memory_edges_relation ON memory_edges(relation); `); // Migration: repair FTS5 index if out of sync const ftsCount = (db.prepare("SELECT COUNT(*) as cnt FROM memories_fts").get() as { cnt: number }).cnt; const memCount = (db.prepare("SELECT COUNT(*) as cnt FROM memories WHERE status = 'active'").get() as { cnt: number }) .cnt; if (ftsCount < memCount) { const missing = db .prepare( "SELECT rowid, content, fact_summary FROM memories WHERE status = 'active' AND rowid NOT IN (SELECT rowid FROM memories_fts)", ) .all() as Array<{ rowid: number; content: string; fact_summary: string | null }>; for (const m of missing) { db.prepare("INSERT INTO memories_fts(rowid, content, fact_summary) VALUES (?, ?, ?)").run( m.rowid, m.content, m.fact_summary ?? "", ); } } else if (ftsCount === 0 && memCount > 0) { db.exec( `INSERT INTO memories_fts(rowid, content, fact_summary) SELECT rowid, content, fact_summary FROM memories WHERE status = 'active'`, ); } return db; }