/** * Verbatim-overlap detector (bd startsim-768w.18.16.1). * * Measures how much of a generated passage is copied VERBATIM from its source * articles — the guardrail against plagiarism / SEO duplicate-content now that the * writer is fed real article text. Pure + framework-free so it unit-tests in node. * * Method: word-shingle coverage. Both sides are normalised (lowercased, punctuation * stripped, whitespace collapsed) to words; every `minRun`-word shingle of the * sources goes in a set; each candidate position covered by an in-source shingle is * marked "copied". From that we report the LONGEST consecutive copied run (words) * and the fraction of the candidate that is copied. Short shared phrases (< minRun * words) are deliberately invisible — quoting a handful of words is fine. */ export interface OverlapResult { /** Longest run of consecutive candidate words copied verbatim from a source. */ longestRunWords: number; /** Fraction (0–1) of candidate words that sit inside a copied run. */ overlapRatio: number; /** The longest copied span (normalised words), for a human detail. '' when none. */ sample: string; } /** Lowercase, strip punctuation, collapse whitespace → word list. */ export function normWords(s: string): string[] { return String(s ?? '') .toLowerCase() .replace(/[^\p{L}\p{N}\s]/gu, ' ') .split(/\s+/) .filter(Boolean); } /** * Verbatim overlap of `candidate` against `sources`. `minRun` (default 6) is the * shingle size — the shortest copied run that counts (so ≤5-word coincidences / * short quotes are ignored). Returns zeros for empty candidate or no sources. */ export function verbatimOverlap( candidate: string, sources: string[], minRun = 6, ): OverlapResult { const cand = normWords(candidate); const n = cand.length; const empty: OverlapResult = { longestRunWords: 0, overlapRatio: 0, sample: '' }; if (n === 0 || minRun < 1) return empty; const shingles = new Set(); for (const src of sources ?? []) { const w = normWords(src); for (let i = 0; i + minRun <= w.length; i++) { shingles.add(w.slice(i, i + minRun).join(' ')); } } if (shingles.size === 0) return empty; const covered = new Array(n).fill(false); for (let i = 0; i + minRun <= n; i++) { if (shingles.has(cand.slice(i, i + minRun).join(' '))) { for (let j = i; j < i + minRun; j++) covered[j] = true; } } let longest = 0; let coveredCount = 0; let cur = 0; let curStart = 0; let bestStart = -1; let bestEnd = -1; for (let i = 0; i < n; i++) { if (covered[i]) { coveredCount++; if (cur === 0) curStart = i; cur++; if (cur > longest) { longest = cur; bestStart = curStart; bestEnd = i; } } else { cur = 0; } } return { longestRunWords: longest, overlapRatio: coveredCount / n, sample: longest > 0 ? cand.slice(bestStart, bestEnd + 1).join(' ') : '', }; }