/** * pi-loom: Fact Extraction Engine * * Mem0-style atomic fact extraction from stored memories. * Facts are stored in the memories table with "extracted-fact" tag, * making them automatically searchable via FTS5 and context injection. * * Bridge to ESR: facts are anchored to the same entity_id as their parent. */ import type { ExtensionContext } from "@earendil-works/pi-coding-agent"; import { callLLMForJSON, hasLLM, resolveMcpAuth, resolveModelForLLM } from "./model.js"; import type { LoomStore, MemRow } from "./store.js"; /** * Regex-based fact extraction — zero-dependency fallback when no LLM is available. * * Extracts: * - Preference statements: "I prefer/like/dislike X" * - Quantified facts: "X is N units" * - Name/value pairs: "X is Y" * - Action confirmations: "[Agent] has done X" * - File paths: "src/file.ts" * - Entity IDs: "task-xxx", "issue-xxx" * - Dates: "2026-01-15" * * This is intentionally simple — LLM extraction is always preferred when available. */ function regexExtractFacts(memories: MemRow[]): Array<{ fact: string; entities: string[] }> { const facts: Array<{ fact: string; entities: string[] }> = []; for (const mem of memories) { const text = mem.content; // Preference patterns for (const m of text.matchAll( /(?:I|User|The user)\s+(prefers?|likes?|dislikes?|wants?|needs?)\s+(.{10,120}?)(?:\.|$)/gi, )) { facts.push({ fact: m[0].trim().replace(/\.$/, ""), entities: [] }); } // File path patterns for (const m of text.matchAll(/(?:\/[\w./-]+\.[a-z]{2,4})/gi)) { if (!facts.some((f) => f.fact.includes(m[0]))) { facts.push({ fact: `File referenced: ${m[0]}`, entities: [] }); } } // Entity ID patterns for (const m of text.matchAll(/\b(?:loom|esr|task|issue|bug|feature|fix|pr|commit)[-_:]\w+/gi)) { facts.push({ fact: `Entity referenced: ${m[0]}`, entities: [m[0]] }); } // Quantified facts: "X is N units" for (const m of text.matchAll(/(\w+(?:\s+\w+){0,3})\s+(?:is|was|has|had)\s+(\d+(?:\.\d+)?)\s*(\w+)?/gi)) { const fact = m[0].trim().replace(/\.$/, ""); if (fact.length > 10 && !facts.some((f) => f.fact === fact)) { facts.push({ fact, entities: [] }); } } // Date mentions for (const m of text.matchAll(/\b(20\d{2}-\d{2}-\d{2})\b/g)) { if (!facts.some((f) => f.fact.includes(m[0]))) { facts.push({ fact: `Date mentioned: ${m[0]}`, entities: [] }); } } } // Deduplicate const seen = new Set(); return facts.filter((f) => { const key = f.fact.toLowerCase().slice(0, 80); if (seen.has(key)) return false; seen.add(key); return true; }); } const FACT_EXTRACT_SYSTEM = `You are an atomic fact extractor for a memory system. Given conversation dialogue, extract ALL atomic, self-contained facts. Rules: - Extract AT LEAST 1 fact from EVERY conversation turn, even if it's simple - Each fact must be one sentence, self-contained (resolve pronouns) - Preserve names, dates, quantities, preferences exactly as stated - Extract BOTH user facts AND assistant confirmations/actions - For preferences: "User prefers X", "User likes Y", "User dislikes Z" - For knowledge updates: include the latest value - For temporal info: include dates when available - Do NOT summarize or paraphrase — keep the exact key information Output JSON array: [{"fact": "...", "entities": ["keyword1", "keyword2"]}] Examples: Input: "I graduated with a degree in Business Administration last year" Output: [{"fact": "Graduated with a degree in Business Administration"}] Input: "My daily commute to work is 45 minutes each way" Output: [{"fact": "Daily commute to work is 45 minutes each way"}] Input: "Just got back from Target, they had a $5 coupon on coffee creamer" Output: [{"fact": "Redeemed $5 coupon on coffee creamer at Target"}] Input: "Assistant: I've booked your flight for March 3rd to Chicago" Output: [{"fact": "Assistant booked flight to Chicago on March 3rd"}] `; export interface FactExtractConfig { modelProvider?: string; modelId?: string; /** Enable dedup against existing facts. Default: true. */ } export interface FactExtractResult { memory_count: number; fact_count: number; fact_ids: string[]; /** Count of duplicate facts skipped during dedup. */ } /** * Extract atomic facts from memories and store them as searchable fact memories. * * Pipeline: raw memories → LLM extract atomic facts → store as memories → FTS5 indexed → searchable. * Facts inherit entity_id + importance from parent, with "extracted-fact" + "entity:..." tags. */ export async function extractFacts( store: LoomStore, memories: MemRow[], ctx: ExtensionContext | null, config: FactExtractConfig = {}, ): Promise { if (memories.length === 0) { return { memory_count: 0, fact_count: 0, fact_ids: [] }; } // ── Zero-dependency fallback: regex extraction when no LLM ── if (!hasLLM(ctx)) { const facts = regexExtractFacts(memories); const factIds: string[] = []; for (const f of facts) { if (f.fact.length < 5) continue; const parent = memories[0]; const factMem = store.storeFact(parent, f.fact); factIds.push(factMem.id); } console.error( `[pi-loom] ExtractFacts (regex fallback): ${facts.length} facts from ${memories.length} memories (no LLM available).`, ); return { memory_count: memories.length, fact_count: factIds.length, fact_ids: factIds }; } const model = resolveModelForLLM(ctx, config, "PI_FACT_MODEL", "deepseek/deepseek-v3.1"); if (!model) { console.error("[pi-loom] ExtractFacts: no model available."); return { memory_count: memories.length, fact_count: 0, fact_ids: [] }; } const auth = ctx ? await ctx.modelRegistry.getApiKeyAndHeaders(model) : resolveMcpAuth(model); if (!auth.ok || !auth.apiKey) { return { memory_count: memories.length, fact_count: 0, fact_ids: [] }; } const factIds: string[] = []; const BATCH_SIZE = 10; for (let i = 0; i < memories.length; i += BATCH_SIZE) { const batch = memories.slice(i, i + BATCH_SIZE); const batchLines = batch.map((m, j) => `[M${i + j + 1}] ${m.content.slice(0, 400)}`); const prompt = [ FACT_EXTRACT_SYSTEM, "", "## Conversation Turns", batchLines.join("\n\n"), "", "Extract ALL atomic facts from these conversation turns.", 'Output as JSON array: [{"fact": "...", "entities": [...]}]', ].join("\n"); type FactJSON = { fact?: string; entities?: string[] }; const parsed = await callLLMForJSON(model, prompt, { apiKey: auth.apiKey!, headers: auth.headers, maxTokens: 2048, }); if (!parsed) continue; for (let k = 0; k < parsed.length; k++) { const fact = parsed[k].fact; if (!fact || fact.length < 10) continue; const source = batch[Math.min(k, batch.length - 1)]; const factMem = store.storeFact(source, fact); factIds.push(factMem.id); } } return { memory_count: memories.length, fact_count: factIds.length, fact_ids: factIds }; }