/** * embedder.ts — pluggable text embedding for the local vector store. * * Default embedder is a zero-dependency, deterministic hashed n-gram bag * encoder — no native build, no network, no external library, works offline. * It is heuristic-strength (good enough to rank "which checkpoint is relevant * to this query?"), not RAG-grade. A stronger LOCAL embedding backend (your own * localhost ONNX/TEI/Ollama server) can be plugged in via MEGACOMPACT_EMBEDDING_URL * — see httpEmbedder.ts. The `Embedder` interface is the seam both implement; * this extension ships no model and makes no remote call (PREVENT-PI-004). */ import { HttpEmbedder, embeddingConfigFromEnv } from "./httpEmbedder.js"; export type Vector = number[]; /** Common embedding contract. Implementations must be deterministic. */ export interface Embedder { /** Dimensionality of vectors this embedder produces. */ readonly dim: number; /** Discriminator: "trigram" | "http". Lets callers branch on the embedder * backend without instanceof checks (e.g. HyDE is http-only). */ readonly kind: string; embed(text: string): Vector; } /** Normalize a vector to unit length (cosine-sim safe). Returns a new array. */ export function l2Normalize(v: Vector): Vector { let sumSq = 0; for (const x of v) sumSq += x * x; const norm = Math.sqrt(sumSq); if (norm === 0) return v.map(() => 0); return v.map((x) => x / norm); } /** Cosine similarity in [-1, 1]. Assumes inputs are same dim. */ export function cosineSimilarity(a: Vector, b: Vector): number { if (a.length !== b.length) return 0; let dot = 0; let na = 0; let nb = 0; for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; } if (na === 0 || nb === 0) return 0; return dot / (Math.sqrt(na) * Math.sqrt(nb)); } /** Stable 32-bit string hash (FNV-1a). */ function fnv1a(str: string): number { let h = 0x811c9dc5; for (let i = 0; i < str.length; i++) { h ^= str.charCodeAt(i); h = Math.imul(h, 0x01000193); } return h >>> 0; } /** * Default embedder: character 3-gram bag-of-counts, hashed into a fixed-dim * vector, L2-normalized. Captures local lexical/structure overlap well enough * for checkpoint relevance ranking. * * S53B: includes a 256-entry (configurable) FIFO cache keyed by FNV-1a(text). * Each cache entry holds a DEFENSIVE COPY of the embedding to prevent mutation * from corrupting stored vectors. Set MEGACOMPACT_EMBED_CACHE=0 to disable. */ export class TrigramEmbedder implements Embedder { readonly kind = "trigram"; readonly dim: number; private readonly seed: number; /** @type {Map} */ private readonly _cache: Map; private readonly _cacheSize: number; private _hits = 0; private _misses = 0; constructor(dim = 512, seed = 0x9e3779b9) { this.dim = dim; this.seed = seed >>> 0; const cap = Number(process.env.MEGACOMPACT_EMBED_CACHE ?? "256"); this._cacheSize = cap > 0 ? Math.floor(cap) : 0; this._cache = new Map(); } /** Cache statistics — always { hits: 0, misses: 0 } when cache is disabled. */ getEmbedCacheStats(): { hits: number; misses: number } { return { hits: this._hits, misses: this._misses }; } embed(text: string): Vector { if (this._cacheSize === 0) { return this._embedRaw(text); } const key = fnv1a(text).toString(36); const cached = this._cache.get(key); if (cached !== undefined) { this._hits++; // Defensive copy: caller mutating the returned vector must not corrupt cache. return cached.slice() as Vector; } this._misses++; const vec = this._embedRaw(text); // Store a defensive copy (slice) so caller mutation cannot corrupt stored value. this._cache.set(key, vec.slice() as Vector); // FIFO eviction: remove the oldest entry when at capacity. if (this._cache.size > this._cacheSize) { // Map insertion order is guaranteed — first key is the oldest. const firstKey = this._cache.keys().next().value; if (firstKey !== undefined) this._cache.delete(firstKey); } return vec; } // guardrails-allow PREVENT-MOCK-001: TrigramEmbedder is a documented lexical 3-gram bag-of-counts projection (FNV-1a into a 512-dim L2-normalized vector), not a semantic model; captured as exchange-rate-limited lexical recall (accuracy floor acknowledged) /** Raw embed computation (no cache logic). Public so tests can bypass cache. */ _embedRaw(text: string): Vector { const vec = new Array(this.dim).fill(0); const norm = text.toLowerCase().replace(/\s+/g, " "); if (norm.length === 0) return l2Normalize(vec); vec[fnv1a(norm) % this.dim] += 1; for (const word of norm.split(" ")) { if (word.length === 0) continue; vec[fnv1a(word) % this.dim] += 1; for (let i = 0; i + 3 <= word.length; i++) { const gram = word.slice(i, i + 3); const idx = (fnv1a(gram) ^ this.seed) % this.dim; vec[idx] += 1; } } if (norm.length < 3) vec[fnv1a(norm) % this.dim] += 1; return l2Normalize(vec); } } /** * Select the default embedder used by VectorStore. * * - If MEGACOMPACT_EMBEDDING_URL points at a localhost server, use HttpEmbedder * (your own local embedding backend — ONNX/TEI/Ollama/etc). This is the * PREVENT-PI-004-sanctioned "bring your own" path: the endpoint is a * user-spawned loopback server, so conversation content never leaves the box. * - Otherwise the TrigramEmbedder is the shipped default: zero-dependency, * deterministic, GPU-free, cross-platform, fully offline. * * The `Embedder` interface is the seam for any LOCAL embedder. Never point it * at a remote provider — that would violate PREVENT-PI-004. A user wanting * semantic-grade dedup should run a local embedding server and set the URL. */ export function defaultEmbedder(): Embedder { const cfg = embeddingConfigFromEnv(); if (cfg) return new HttpEmbedder(cfg); return new TrigramEmbedder(); }