/** * Incremental crawl diff — compute stable content hashes and skip unchanged screens. * * Each crawled page gets a content fingerprint (murmurhash-free, pure JS). * On the next crawl of the same project, pages with the same fingerprint are * marked SKIPPED instead of re-processed, drastically reducing crawl time and * LLM token spend for sites with mostly-static content. * * Design: "same content hash" means the meaningful text hasn't changed. * Layout/CSS/JS changes that don't affect visible text are ignored intentionally. */ export interface ContentHash { url: string; hash: string; textLength: number; computedAt: string; } export interface DiffResult { url: string; changed: boolean; previousHash?: string; currentHash: string; reason: 'new' | 'changed' | 'unchanged'; } /** * Compute a stable 64-bit-ish hash from text content. * djb2 variant — fast, no deps, collision-resistant enough for content fingerprinting. */ export declare function hashContent(text: string): string; /** * Normalize text before hashing — strips whitespace-only lines, trims each line, * collapses runs of blank lines. Makes hash stable across minor formatting changes. */ export declare function normalizeText(text: string): string; export declare function computeContentHash(url: string, markdownText: string): ContentHash; /** * Compare a page's current content against prior hashes. * Returns { changed: false } for unchanged pages — caller should skip re-processing. */ export declare function diffContent(url: string, currentMarkdown: string, priorHashes: Map): DiffResult; /** * Build a hash map from prior screen records (from DB query result). * screenRecords: Array<{ url: string; elementHash?: string | null }> */ export declare function buildPriorHashMap(screenRecords: Array<{ url: string; elementHash?: string | null; }>): Map; /** * Filter a batch of discovered URLs against prior hashes. * Returns { toProcess, toSkip } — caller enqueues toProcess, logs toSkip count. * * Note: filtering at URL level only works if the prior crawl already visited the URL. * For new URLs, they're always in toProcess. */ export declare function partitionByChange(urls: string[], priorHashes: Map): { toProcess: string[]; toSkip: string[]; };