/** * index.ts — RAPTOR orchestrator (Sprint 13, Phase 6 → promoted to live in S42B). * * Entry point that turns a session's leaves into a hierarchical summary tree. * The tree is BUILT, PERSISTED to raptor_nodes, and SERVED in recall queries * when RAPTOR_ENABLED=true (default). Set RAPTOR_SHADOW_MODE=true to build + * persist only without serving (transition/eval use). * * PREVENT-PI-004: no network here. Any Ollama call lives in summarizer.ts * (localhost-only, annotated). This module is pure orchestration. */ import type { Embedder } from "../../embedder.js"; import { defaultEmbedder } from "../../embedder.js"; import { buildRaptorTree, type Leaf, type RaptorTree } from "./tree.js"; import { incrementRaptorTree } from "./incremental.js"; import { stagedExpansion } from "./retrieval.js"; import { Logger } from "../../log.js"; import { saveRaptorTree, listRaptorNodes } from "../../store/sqlite.js"; import { insertBuildHistory, computeCoherenceScore } from "./buildHistory.js"; import { loadDedupConfig } from "../../config/dedup.js"; /** * Shadow mode default OFF: the tree is built, persisted, AND served when * RAPTOR_ENABLED=true (the Setup tab toggle). Set RAPTOR_SHADOW_MODE=true to * build + persist only (transition/eval use without serving live recall). */ export function isShadowMode(): boolean { return process.env.RAPTOR_SHADOW_MODE === "true"; } export interface RaptorOrchestratorOptions { embedder?: Embedder; stateDir: string; sessionId: string; budgetMs?: number; clustersPerLevel?: number; consistencyThreshold?: number; /** Best-effort logger for shadow events. */ logger?: Logger; /** S25: epoch ms to stamp on every node (freshness guard). Defaults to now. */ builtAt?: number; } /** * Build the RAPTOR tree for a session's leaves. Per shadow-mode rules, the tree * is persisted + logged regardless; whether it is served is the caller's choice * (Sprint 13: it is built but NOT injected into recallAndInline). * * Returns the built tree (in-memory) for eval/tests, and persists it to the * store. Never throws — on any build error it logs and returns null. */ export function runRaptor( leaves: Leaf[], opts: RaptorOrchestratorOptions, ): RaptorTree | null { const embedder = opts.embedder ?? defaultEmbedder(); const logger = opts.logger; const startedAt = opts.builtAt ?? Date.now(); const builtAt = startedAt; try { // Try incremental update first when the flag is ON (Sprint 26, #7). // The incremental path reads the existing tree, diffs by leaf id, and // inserts only new leaves into the nearest clusters. Falls back to a full // rebuild when the tree is missing, corrupted, or >50% of nodes are new. const cfg = loadDedupConfig(); let tree: RaptorTree | null = null; if (cfg.RAPTOR_INCREMENTAL) { tree = incrementRaptorTree(leaves, leaves, { embedder, stateDir: opts.stateDir, sessionId: opts.sessionId, budgetMs: opts.budgetMs, clustersPerLevel: opts.clustersPerLevel, consistencyThreshold: opts.consistencyThreshold, logger, builtAt, }); } // Fall back to full rebuild when incremental returned null (not attempted, // tree missing, ratio exceeded, or internal error). if (!tree) { tree = buildRaptorTree(leaves, { embedder, budgetMs: opts.budgetMs, clustersPerLevel: opts.clustersPerLevel, consistencyThreshold: opts.consistencyThreshold, }); saveRaptorTree(opts.sessionId, tree, builtAt, opts.stateDir); } logger?.info("raptor_build", { sessionId: opts.sessionId, nodes: tree.nodes.size, levels: tree.levels, rootId: tree.rootId, timedOut: tree.timedOut, shadow: isShadowMode(), }); // S42D: record structured build history (coherence score + leaf count) so // the freshness check can skip the next rebuild when the tree is recent // and the chunk count is stable. Best-effort: a history-write failure must // never break the build. try { // Map leaf id → embedding so computeCoherenceScore can score real // intra-cluster spread (without this, leaves collapse to parent centroids // and the score inflates to ~1.0 for every tree). const leafEmbs = new Map(); for (const l of leaves) if (l.embedding.length > 0) leafEmbs.set(l.id, l.embedding); const coherence = computeCoherenceScore(tree, leafEmbs); insertBuildHistory( { sessionId: opts.sessionId, stateDir: opts.stateDir, startedAt, completedAt: Date.now(), nodeCount: tree.nodes.size, leafCount: leaves.length, depth: tree.levels, configJson: JSON.stringify({ budgetMs: opts.budgetMs, clustersPerLevel: opts.clustersPerLevel, consistencyThreshold: opts.consistencyThreshold, }), coherenceScore: coherence, timedOut: tree.timedOut, }, opts.stateDir, ); } catch (histErr) { logger?.warn("raptor_build_history_failed", { sessionId: opts.sessionId, error: String(histErr instanceof Error ? histErr.message : histErr), }); } if (isShadowMode()) { // Build + log only. Do NOT replace retrieval. logger?.info("raptor_shadow", { sessionId: opts.sessionId, served: false }); } return tree; } catch (e) { logger?.error("raptor_build_failed", { sessionId: opts.sessionId, error: String(e instanceof Error ? e.message : e), }); return null; } } /** * Staged retrieval over a persisted/session tree. Only meaningful when RAPTOR * is promoted (Sprint 14); provided here so eval can measure it in shadow. * * Returns the leaf ids the staged expansion would serve for `query`. */ export function recallRaptor( query: string, sessionId: string, opts: { embedder?: Embedder; stateDir: string; k?: number; topM?: number }, ): string[] { const embedder = opts.embedder ?? defaultEmbedder(); const tree = rehydrateRaptorTree(sessionId, opts.stateDir); if (!tree) return []; return stagedExpansion(query, tree, { embedder, k: opts.k, topM: opts.topM }); } /** * Rehydrate a persisted RAPTOR tree from raptor_nodes (Fix D): rebuild the * in-memory RaptorTree + parent links so vectorStore.search can serve it live. * Returns null when no tree exists (caller falls back to the flat path). */ export function rehydrateRaptorTree( sessionId: string, stateDir: string, ): RaptorTree | null { const nodes = listRaptorNodes(sessionId, stateDir); if (nodes.length === 0) return null; // S25: derive freshness + fallback metadata from the persisted nodes. // builtAt = max node built_at (0 when unknown → caller treats as stale). // timedOut = the tree's root is the extractive-fallback marker (level 99). const builtAt = nodes.reduce((max, n) => Math.max(max, n.builtAt), 0); const root = nodes.reduce( (best, n) => (!best || n.level > (best?.level ?? -1) ? n : best), null, ); const timedOut = root != null && root.level >= 99; const tree: RaptorTree = { nodes: new Map( nodes.map((n) => [ n.id, { id: n.id, level: n.level, parentId: n.parentId, children: n.children, summary: n.summary, embedding: n.embedding, qualityMarker: n.qualityMarker as any, tokenEstimate: n.tokenEstimate, }, ]), ), rootId: root?.id ?? null, levels: Math.max(1, ...nodes.map((n) => n.level + 1)), timedOut, builtAt, }; return tree; } /** * Return the RAPTOR root summary for a session, if a tree has been built. * Used by the durable-trim driver (Fix B/D) to supply pi a session-level * compressed summary instead of one slice's extractive summary. Returns * undefined when no tree exists yet (caller falls back to the slice summary). */ export function recallRaptorRootSummary( sessionId: string, stateDir: string, ): string | undefined { const nodes = listRaptorNodes(sessionId, stateDir); if (nodes.length === 0) return undefined; // Highest-level node = the root (covers all leaves). const root = nodes.reduce<(typeof nodes)[number] | null>( (best, n) => (!best || n.level > best.level ? n : best), null, ); return root?.summary || undefined; }