/**
 * Content deduplication — detect near-duplicate pages during crawl.
 *
 * Uses MinHash-inspired approach: instead of full MinHash (needs many hash functions),
 * we use 8 structural fingerprints extracted from the markdown to detect pages that
 * share the same layout/template even if the data differs.
 *
 * Typical duplicates: /products?page=1 vs /products?page=2 (same template, different data).
 * These waste crawl budget and generate redundant test cases.
 */

/** Extract structural fingerprints from markdown — layout-sensitive, data-insensitive */
function extractFingerprints(markdown: string): string[] {
  const lines = markdown.split('\n').filter(l => l.trim().length > 0);
  const fps: string[] = [];

  // FP1: heading structure (H1-H6 count distribution)
  const headings = lines.filter(l => /^#{1,6} /.test(l));
  const hCounts = [0, 0, 0, 0, 0, 0];
  for (const h of headings) {
    const lvl = h.match(/^(#{1,6}) /)?.[1].length ?? 1;
    hCounts[lvl - 1]++;
  }
  fps.push('h:' + hCounts.join(','));

  // FP2: link count (grouped into buckets)
  const linkCount = (markdown.match(/\[.+?\]\(.+?\)/g) ?? []).length;
  fps.push('lk:' + Math.floor(linkCount / 5));

  // FP3: list item count (grouped)
  const listCount = lines.filter(l => /^[\-\*\+] |^\d+\. /.test(l)).length;
  fps.push('li:' + Math.floor(listCount / 3));

  // FP4: code block presence
  fps.push('code:' + (markdown.includes('```') ? '1' : '0'));

  // FP5: table presence
  fps.push('tbl:' + (markdown.includes('|') ? '1' : '0'));

  // FP6: approximate paragraph count
  const paraCount = markdown.split(/\n{2,}/).filter(p => p.trim().length > 30).length;
  fps.push('p:' + Math.floor(paraCount / 2));

  // FP7: first heading text (normalized)
  const firstH = headings[0]?.replace(/^#{1,6} /, '').toLowerCase().replace(/[^a-z0-9]/g, '') ?? '';
  fps.push('fh:' + firstH.slice(0, 20));

  // FP8: word count bucket
  const wordCount = markdown.split(/\s+/).length;
  fps.push('wc:' + Math.floor(wordCount / 50));

  return fps;
}

function similarity(a: string[], b: string[]): number {
  if (a.length !== b.length) return 0;
  let matches = 0;
  for (let i = 0; i < a.length; i++) {
    if (a[i] === b[i]) matches++;
  }
  return matches / a.length;
}

export interface DedupResult {
  isDuplicate: boolean;
  similarUrl?: string;
  similarityScore: number;
}

export class ContentDeduplicator {
  private seen = new Map<string, { fps: string[]; url: string }>();
  private readonly threshold: number;

  constructor(threshold = 0.85) {
    this.threshold = threshold;
  }

  check(url: string, markdown: string): DedupResult {
    const fps = extractFingerprints(markdown);

    for (const [key, entry] of this.seen) {
      const score = similarity(fps, entry.fps);
      if (score >= this.threshold) {
        return { isDuplicate: true, similarUrl: entry.url, similarityScore: score };
      }
    }

    this.seen.set(url, { fps, url });
    return { isDuplicate: false, similarityScore: 1.0 };
  }

  size(): number {
    return this.seen.size;
  }

  reset(): void {
    this.seen.clear();
  }
}