import type { ParsedArticle } from '../parsers/feed'; import { MAX_ARTICLE_GUID_SIZE, MAX_ARTICLE_LINK_SIZE } from '../utils/constants'; import { parseHttpUrl } from '../utils/validation'; /** * The "deduplication identity" of an Article (CONTEXT.md, ADR-0010). * `(guid, link)` together — GUID preferred, link fallback. Either side may be * null when the source feed omits one. */ export interface ExistingIdentity { guid: string | null; link: string | null; } /** * Decide which incoming articles are new (not already present) given the set * of identities already stored for the same Feed. Implements ADR-0010: * * - A candidate is a duplicate if its GUID matches an existing GUID **or** * its link matches an existing link. GUID is preferred because some feeds * rotate links. * - Candidates without a link are dropped (link is required for insertion). * - Empty-string GUIDs are treated as absent (never match). * - Intra-payload dedupe: a second candidate that matches an earlier one in * the same payload is also dropped. * * Pure: no I/O. Returns a fresh array in payload order. */ export type LinkedArticle = ParsedArticle & { link: string }; /** Canonicalize an absolute Article resource identity without truncating it. */ export function canonicalizeArticleLink(link: string | null | undefined): string | undefined { if (!link || link.length > MAX_ARTICLE_LINK_SIZE) return undefined; const parsed = parseHttpUrl(link); if (!parsed.valid || parsed.value.length > MAX_ARTICLE_LINK_SIZE) return undefined; return parsed.value; } export function partitionNewArticles(candidates: ParsedArticle[], existing: ExistingIdentity[]): LinkedArticle[] { const seenGuids = new Set(); const seenLinks = new Set(); for (const e of existing) { if (e.guid && e.guid.length <= MAX_ARTICLE_GUID_SIZE) seenGuids.add(e.guid); const link = canonicalizeArticleLink(e.link); if (link) seenLinks.add(link); } const fresh: LinkedArticle[] = []; for (const candidate of candidates) { const link = canonicalizeArticleLink(candidate.link); if (!link) continue; const guid = candidate.guid && candidate.guid.length <= MAX_ARTICLE_GUID_SIZE ? candidate.guid : undefined; if (guid && seenGuids.has(guid)) continue; if (seenLinks.has(link)) continue; const normalized: LinkedArticle = { ...candidate, link }; if (candidate.guid !== undefined && candidate.guid.length > MAX_ARTICLE_GUID_SIZE) delete normalized.guid; fresh.push(normalized); if (guid) seenGuids.add(guid); seenLinks.add(link); } return fresh; }