/** * Semantic deduplication for extracted insights. * * Before inserting a new insight into the DB, we check if a semantically * equivalent insight already exists. If cosine similarity > DEDUP_THRESHOLD, * the existing insight receives a corroboration boost (+0.1 trust, capped at * +0.2 total) rather than creating a duplicate card. * * Embedding is done with the same local embedder used for card search, * keeping all inference in-process at zero external API cost. */ import type { ExtractedInsight } from "./extractor.js"; export interface DeduplicatedInsight { insight: ExtractedInsight; /** ID of existing insight if this is a corroboration (no new card), undefined if new */ corroboratesId?: string; /** Adjusted trust_score after corroboration boost */ trustScore: number; } export interface StoredInsightEmbedding { id: string; statement: string; embedding: Float32Array; trustScore: number; corroborationCount: number; } /** * Computes cosine similarity between two Float32Array vectors. */ export declare function cosineSimilarity(a: Float32Array, b: Float32Array): number; /** * Deduplicates a batch of new insights against the existing stored embeddings. * * For each new insight: * - Embed the statement * - Compare against existing embeddings * - If similarity > DEDUP_THRESHOLD: return as corroboration (no new card) * - Otherwise: return as new (insert new card) * * **Side-effect**: `existing` is mutated in-place. Newly accepted insights are * appended to `existing` so that later insights in the same batch can * corroborate them. Callers that share the `existing` array across calls must * be aware that it grows after each invocation. */ export declare function deduplicateInsights(newInsights: ExtractedInsight[], existing: StoredInsightEmbedding[]): Promise; //# sourceMappingURL=dedup.d.ts.map