/**
 * Sitemap.xml discovery — seeds the crawl queue from a project's sitemap
 * instead of (or alongside) plain link-following, matching crawl4AI's
 * sitemap-seeding capability. Best-effort throughout: most sites don't have
 * a sitemap, or it 404s, or it's malformed — none of that should ever fail
 * a crawl, it just means sitemap seeding contributes nothing this run.
 */
import { XMLParser } from 'fast-xml-parser';
import { ProxyAgent } from 'undici';
import { assertCrawlUrlSafe } from '@detiq/core';
import { logger } from './logger.js';

const FETCH_TIMEOUT_MS = 10_000;
const MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)
const MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index

async function getSitemapMaxUrls(): Promise<number> {
  const { getSystemConfig } = await import('@detiq/core');
  const cfg = await getSystemConfig('crawl_tuning');
  return parseInt(String(cfg?.sitemapMaxUrls ?? '5000'), 10);
}

function toArray<T>(x: T | T[] | undefined | null): T[] {
  if (x == null) return [];
  return Array.isArray(x) ? x : [x];
}

/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */
function hostKey(hostname: string): string {
  return hostname.replace(/^www\./i, '').toLowerCase();
}

/**
 * Same-site check that tolerates www/non-www and http/https differences — a sitemap on
 * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).
 */
export function sameSite(candidateUrl: string, siteOrigin: string): boolean {
  try {
    const originHost = /^https?:\/\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;
    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);
  } catch {
    return false;
  }
}

/**
 * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap
 * document (either a <urlset> or a <sitemapindex>), filtered to same-origin
 * and deduped. Pure function — no I/O — for direct unit testing without a
 * real HTTP fetch.
 */
export function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls = 5000): { pageUrls: string[]; subSitemapUrls: string[] } {
  const rawPageUrls: string[] = [];
  let subSitemapUrls: string[] = [];

  if (parsed?.sitemapindex) {
    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)
      .map((s: any) => s?.loc)
      .filter((loc: any): loc is string => typeof loc === 'string')
      .slice(0, MAX_SUB_SITEMAPS);
  }

  if (parsed?.urlset) {
    rawPageUrls.push(
      ...toArray(parsed.urlset.url)
        .map((u: any) => u?.loc)
        .filter((loc: any): loc is string => typeof loc === 'string'),
    );
  }

  // Same-site only (www/protocol-tolerant) — a sitemap listing external URLs shouldn't seed this project's queue.
  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));

  return { pageUrls: [...new Set(sameOrigin)].slice(0, maxUrls), subSitemapUrls };
}

async function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {
  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the
  // stricter webhook-oriented assertResolvesToPublicAddress) — a sitemap
  // living on a tenant's internal/VPN-only staging server is a legitimate
  // crawl target, not an attack; only loopback/cloud-metadata are blocked.
  try {
    assertCrawlUrlSafe(url);
  } catch {
    return null;
  }
  try {
    // Redirects followed normally here (unlike the webhook-dispatch.ts
    // 'manual' pattern) — an http→https or bare→www redirect on the
    // sitemap is completely ordinary, and Playwright's actual page
    // navigation later in the crawl already follows redirects to whatever
    // this same site sends it to; a sitemap fetch isn't a more sensitive
    // operation than that.
    const res = await fetch(url, {
      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },
      ...(dispatcher ? { dispatcher } : {}),
    });
    if (!res.ok) return null;
    const text = await res.text();
    return new XMLParser().parse(text);
  } catch {
    return null;
  }
}

/**
 * Fetches and parses a sitemap (or sitemap index, recursed up to a small
 * depth) into a flat, deduped list of same-origin URLs.
 */
export async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any, maxUrls?: number): Promise<string[]> {
  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];
  if (maxUrls === undefined) maxUrls = await getSitemapMaxUrls();
  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
  if (!parsed) return [];

  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin, maxUrls);
  const collected = [...pageUrls];

  if (subSitemapUrls.length > 0 && collected.length < maxUrls) {
    const subResults = await Promise.all(
      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher, maxUrls))
    );
    for (const sub of subResults) collected.push(...sub);
  }

  return [...new Set(collected)].slice(0, maxUrls);
}

/**
 * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
 * explicit override) and returns the URLs it lists. Never throws.
 */
export async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {
  let origin: string;
  try {
    origin = new URL(appUrl).origin;
  } catch {
    return [];
  }
  const target = sitemapUrl || `${origin}/sitemap.xml`;
  try {
    return await fetchSitemapUrls(target, origin);
  } catch (err) {
    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');
    return [];
  }
}

/**
 * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because
 * DataImpulse-style usernames contain commas (country codes) that break new URL().
 * Returns undefined for no/auto proxy — the caller then fetches directly.
 */
export function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {
  if (!proxyUrl || proxyUrl === 'auto') return undefined;
  try {
    const m = proxyUrl.match(/^(https?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
    if (!m) return new ProxyAgent(proxyUrl);
    const [, scheme, user, pass, hostPort] = m;
    const uri = `${scheme}://${hostPort}`;
    if (user) {
      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');
      return new ProxyAgent({ uri, token });
    }
    return new ProxyAgent({ uri });
  } catch {
    return undefined;
  }
}

/** Extract same-origin `Sitemap:` directives from a robots.txt body. */
export function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {
  const out: string[] = [];
  for (const line of robotsTxt.split(/\r?\n/)) {
    const m = line.match(/^\s*sitemap:\s*(\S+)/i);
    if (m && sameSite(m[1], origin)) out.push(m[1]);
  }
  return [...new Set(out)];
}

export interface UrlCandidate {
  url: string;
  source: 'sitemap' | 'robots' | 'link';
}

/**
 * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap
 * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped
 * candidate list the user can review and seed before launching the Playwright crawl.
 * Never throws — bot-protected sites (Akamai/Cloudflare) simply yield an empty list.
 */
export async function discoverProjectUrls(
  appUrl: string,
  opts?: { sitemapUrl?: string; proxyUrl?: string },
): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {
  let origin: string;
  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }

  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
  const maxUrls = await getSitemapMaxUrls();

  // 1. robots.txt → declared sitemaps
  let robotsSitemaps: string[] = [];
  try {
    assertCrawlUrlSafe(`${origin}/robots.txt`);
    const res = await fetch(`${origin}/robots.txt`, {
      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },
      ...(dispatcher ? { dispatcher } : {}),
    });
    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);
  } catch { /* best-effort */ }

  // 2. sitemap targets: explicit override + robots-declared + conventional default
  const robotsSet = new Set(robotsSitemaps);
  const sitemapTargets = [...new Set([
    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),
    ...robotsSitemaps,
    `${origin}/sitemap.xml`,
  ])];

  const seen = new Map<string, UrlCandidate>();
  let sitemapFound = false;
  for (const sm of sitemapTargets) {
    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher, maxUrls);
    if (urls.length) sitemapFound = true;
    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';
    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });
  }

  return { candidates: [...seen.values()].slice(0, maxUrls), sitemapFound };
}