/**
 * HTML → Markdown extraction for crawled pages — matches crawl4AI's markdown
 * output capability. Two-tier, mirroring crawl4AI's "markdown" vs "fit markdown":
 *
 *  - "fit" markdown: Readability's best guess at the primary content (title +
 *    article body), stripped of nav/ads/chrome. Ideal for content/marketing
 *    pages and RAG chunking. null when Readability finds no article (most
 *    app UI screens — dashboards, forms — legitimately have none).
 *  - "full" markdown: the whole visible page converted straight to Markdown,
 *    always present. The fallback for app screens where there's no "article".
 *
 * Runs entirely on an already-captured `page.content()` string — no extra
 * network fetch, no extra page navigation.
 */
import { parseHTML } from 'linkedom';
import { Readability } from '@mozilla/readability';
import TurndownService from 'turndown';

const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
turndown.remove(['script', 'style', 'noscript', 'svg', 'iframe'] as unknown as (keyof HTMLElementTagNameMap)[]);

export interface ExtractedMarkdown {
  markdown: string;
  method: 'readability' | 'full-page';
  title: string | null;
  wordCount: number;
}

function stripNoiseElements(document: Document): void {
  const noisy = document.querySelectorAll('script, style, noscript, svg, iframe, [aria-hidden="true"]');
  noisy.forEach((el: any) => el.remove());
}

/** Best-effort HTML→Markdown for one crawled page. Never throws. */
export function extractMarkdown(html: string, url: string): ExtractedMarkdown | null {
  try {
    const { document } = parseHTML(html);
    stripNoiseElements(document);

    let readabilityResult: { title: string; content: string; textContent: string } | null = null;
    try {
      // Readability mutates the DOM it's given — clone via re-parsing so the
      // full-page fallback below still has an intact document.
      const { document: readerDoc } = parseHTML(html);
      stripNoiseElements(readerDoc);
      readabilityResult = new Readability(readerDoc as any, { charThreshold: 50 }).parse() as any;
    } catch {
      readabilityResult = null;
    }

    if (readabilityResult?.content && readabilityResult.textContent.trim().length >= 50) {
      const markdown = turndown.turndown(readabilityResult.content);
      return {
        markdown: `# ${readabilityResult.title || document.title || url}\n\n${markdown}`,
        method: 'readability',
        title: readabilityResult.title || document.title || null,
        wordCount: readabilityResult.textContent.trim().split(/\s+/).length,
      };
    }

    const body = document.body;
    if (!body) return null;
    const markdown = turndown.turndown(body.innerHTML || '');
    const text = markdown.trim();
    if (!text) return null;
    const title = document.title || null;
    return {
      markdown: title ? `# ${title}\n\n${markdown}` : markdown,
      method: 'full-page',
      title,
      wordCount: text.split(/\s+/).length,
    };
  } catch {
    return null;
  }
}