/** * 4-layer dedup (Task 308). * * Strict superset of the previous nodeId-only dedup. Catches duplicate * representations the typed-edge hook (Task 305) and document-ingest * pipeline will inevitably create: slug variants, canonical-name * variants, and content-hash duplicates. * * Order: hits are sorted by descending score FIRST, then walked. The * highest-score representative wins on every layer; later entries with * the same key on any active layer are dropped. Missing values on a * layer (no `slug`, no `canonicalName`, no derivable hash) are not * collisions — they simply skip that layer for that row. */ import { createHash } from "node:crypto"; export type DedupLayer = "nodeId" | "slug" | "canonicalName" | "contentHash"; export const DEFAULT_LAYERS: DedupLayer[] = [ "nodeId", "slug", "canonicalName", "contentHash", ]; interface DedupHit { nodeId: string; properties: Record; score: number; } function readSlug(props: Record): string | null { const slug = props["slug"]; return typeof slug === "string" && slug.length > 0 ? slug : null; } function readCanonicalName(props: Record): string | null { const canonical = props["canonicalName"]; if (typeof canonical === "string" && canonical.length > 0) { return canonical.toLowerCase(); } const name = props["name"]; if (typeof name === "string" && name.length > 0) { return name.toLowerCase(); } return null; } function readContentHash(props: Record): string | null { const compiled = props["compiledTruth"]; const content = props["content"]; const source = (typeof compiled === "string" && compiled.length > 0) ? compiled : (typeof content === "string" && content.length > 0 ? content : null); if (source === null) return null; return createHash("sha256").update(source).digest("hex").slice(0, 16); } /** * Dedup `hits` across the requested layers. Returns a new array with the * highest-scoring representative per collision class, in original * descending-score order. */ export function dedup( hits: T[], layers: DedupLayer[] = DEFAULT_LAYERS, ): T[] { const sorted = [...hits].sort((a, b) => b.score - a.score); const seen = new Map>(); for (const layer of layers) seen.set(layer, new Set()); const out: T[] = []; for (const hit of sorted) { let collision = false; const keys: Array<[DedupLayer, string]> = []; for (const layer of layers) { let key: string | null; switch (layer) { case "nodeId": key = hit.nodeId; break; case "slug": key = readSlug(hit.properties); break; case "canonicalName": key = readCanonicalName(hit.properties); break; case "contentHash": key = readContentHash(hit.properties); break; } if (key === null) continue; if (seen.get(layer)!.has(key)) { collision = true; break; } keys.push([layer, key]); } if (collision) continue; for (const [layer, key] of keys) seen.get(layer)!.add(key); out.push(hit); } return out; }