/** * pi-loom: Embedding Client * * Two-tier embedding strategy: * 1. API (primary): EMBED_API_KEY → OpenAI-compatible API (text-embedding-3-small, 1536d) * 2. Local (fallback): character n-gram TF vector when no API key is set (256d, zero deps) * * Configurable via env: * EMBED_API_KEY / EMBED_API_BASE / EMBED_MODEL * Falls back to DEEPSEEK_API_KEY → OPENAI_API_KEY. * * For production-quality semantic search without an API key: * npm install @xenova/transformers * Set EMBED_LOCAL_MODEL=all-MiniLM-L6-v2 (or any HF sentence-transformers model). */ const EMBED_API_KEY = process.env.EMBED_API_KEY || process.env.DEEPSEEK_API_KEY || process.env.OPENAI_API_KEY || ""; const EMBED_API_BASE = process.env.EMBED_API_BASE || process.env.OPENAI_API_BASE || "https://router.shengsuanyun.com/api/v1"; const EMBED_MODEL = process.env.EMBED_MODEL || "text-embedding-3-small"; /** Dimension of the embedding model. API default: 1536, local fallback: 256. */ export const EMBED_DIM = EMBED_API_KEY ? parseInt(process.env.EMBED_DIM || "1536", 10) : 256; let _cacheWarned = false; let _localModel: any = null; let _localModelLoading = false; // ═══════════════════════════════════════════════════════ // Tier 1: API-based embedding (OpenAI-compatible) // ═══════════════════════════════════════════════════════ /** * Generate embeddings for one or more texts via API. */ export async function embedTextsAPI(texts: string[]): Promise { if (texts.length === 0) return []; if (!EMBED_API_KEY?.trim()) return []; const truncated = texts.map((t) => t.slice(0, 8000)); const url = `${EMBED_API_BASE.replace(/\/$/, "")}/embeddings`; const body = JSON.stringify({ model: EMBED_MODEL, input: truncated }); try { const resp = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${EMBED_API_KEY}`, }, body, }); if (!resp.ok) { const errText = await resp.text(); console.error(`[pi-loom] Embed API error ${resp.status}: ${errText.slice(0, 200)}`); return []; } const data = (await resp.json()) as any; if (!data.data || !Array.isArray(data.data)) { console.error("[pi-loom] Embed API unexpected response:", JSON.stringify(data).slice(0, 200)); return []; } const sorted = (data.data as Array<{ index: number; embedding: number[] }>).sort((a, b) => a.index - b.index); return sorted.map((d) => d.embedding); } catch (err) { console.error("[pi-loom] Embed API call failed:", err instanceof Error ? err.message : err); return []; } } // ═══════════════════════════════════════════════════════ // Tier 2a: Local Transformers.js (optional dep) // ═══════════════════════════════════════════════════════ /** * Try to load @xenova/transformers for production-quality local embeddings. * Returns null if not installed. */ async function loadLocalTransformer(): Promise { if (_localModel) return _localModel; if (_localModelLoading) return null; _localModelLoading = true; try { // @ts-expect-error — optional dependency, may not be installed const { pipeline } = await import("@xenova/transformers"); const modelName = process.env.EMBED_LOCAL_MODEL || "Xenova/all-MiniLM-L6-v2"; const extractor = await pipeline("feature-extraction", modelName); _localModel = { extractor, dim: 384, type: "transformers" as const }; console.error(`[pi-loom] Local embedding: ${modelName} loaded`); return _localModel; } catch (_err) { console.error("[pi-loom] @xenova/transformers not available. Install with:\n npm install @xenova/transformers"); return null; } finally { _localModelLoading = false; } } // ═══════════════════════════════════════════════════════ // Tier 2b: N-gram TF fallback (zero-dependency, always available) // ═══════════════════════════════════════════════════════ const NGRAM_DIM = 256; const NGRAM_SIZE = 3; // trigrams /** * Generate a 256-dim character n-gram TF vector. * * Uses character trigrams as features, hashed into a fixed-size vector. * While not semantically rich, this provides: * - Deterministic representation (same text → same vector) * - Partial similarity (similar n-grams → similar vector) * - Zero dependencies, instant, no model download * * For production use, prefer API embeddings or @xenova/transformers. */ function ngramEmbed(text: string): number[] { const lower = text.toLowerCase(); const vec = new Array(NGRAM_DIM).fill(0); // Slide a trigram window over the text for (let i = 0; i <= lower.length - NGRAM_SIZE; i++) { const gram = lower.slice(i, i + NGRAM_SIZE); // Hash the trigram into a bucket [0, NGRAM_DIM) let hash = 0; for (let j = 0; j < gram.length; j++) { hash = ((hash << 5) - hash + gram.charCodeAt(j)) | 0; } const bucket = ((hash % NGRAM_DIM) + NGRAM_DIM) % NGRAM_DIM; vec[bucket] += 1; } // Normalize to unit vector const norm = Math.sqrt(vec.reduce((s, v) => s + v * v, 0)) + 1e-10; for (let i = 0; i < NGRAM_DIM; i++) { vec[i] /= norm; } return vec; } // ═══════════════════════════════════════════════════════ // Unified API // ═══════════════════════════════════════════════════════ /** * Generate embeddings for one or more texts. * * Priority: * 1. API (EMBED_API_KEY set) → OpenAI-compatible, 1536d * 2. Local Transformers.js (if installed) → all-MiniLM-L6-v2, 384d * 3. N-gram TF fallback → character trigrams, 256d * * Returns empty array if no embedding method is available. */ export async function embedTexts(texts: string[]): Promise { if (texts.length === 0) return []; // Tier 1: API if (EMBED_API_KEY?.trim()) { return embedTextsAPI(texts); } // Tier 2a: Local Transformers.js const local = await loadLocalTransformer(); if (local?.type === "transformers") { const truncated = texts.map((t) => t.slice(0, 512)); // MiniLM max tokens const results = await local.extractor(truncated, { pooling: "mean", normalize: true }); return results.tolist ? (Array.from(results.tolist()) as number[][]) : []; } // Tier 2b: N-gram TF fallback if (!_cacheWarned) { console.error( "[pi-loom] No embed API key set — using local n-gram TF embedding (256d).\n" + " Set EMBED_API_KEY for production-quality embeddings, or\n" + " npm install @xenova/transformers for local MiniLM-L6-v2.", ); _cacheWarned = true; } return texts.map((t) => ngramEmbed(t)); } /** Shorthand for single text. */ export async function embedText(text: string): Promise { const results = await embedTexts([text]); return results[0] ?? []; }