/**
 * HTTP-only crawl engine — 10-50x faster than browser for SSR/static sites.
 *
 * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.
 * No browser overhead. Falls back to browser engine on JS-heavy sites.
 *
 * Features:
 *   - HTTP/2 native (Node 20 built-in fetch)
 *   - ETag / Last-Modified response caching
 *   - 429 / 503 exponential backoff with jitter
 *   - BFS / DFS traversal
 *   - URL regex filtering
 *   - Robots.txt compliance
 *   - Link discovery (href, action, src)
 *   - Markdown extraction (Readability + Turndown)
 *   - OG / schema.org / meta extraction
 *   - Table → JSON extraction
 *   - JSON Lines, CSV, raw HTML output formats
 *   - Streaming progress via onProgress callback
 */

import { createHash } from 'node:crypto';
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
import { join } from 'node:path';
import { Readability } from '@mozilla/readability';
import TurndownService from 'turndown';
import { parseHTML } from 'linkedom';
import { RobotsCache } from './robots-txt.js';
import { ResponseCache } from './response-cache.js';
import { extractMetadata, extractTables, extractImages } from './extractors/advanced.js';
import { CHROME_CIPHERS, CHROME_CURVES } from './tls-fingerprint.js';
import { renderingPredictor } from './engine-router.js';

// Chrome-level TLS dispatcher: impit (true JA3/JA4 BoringSSL match) with undici fallback
let chromeFetchDispatcher: any;
async function getChromeFetchDispatcher() {
  if (!chromeFetchDispatcher) {
    try {
      // impit provides native Chrome BoringSSL TLS fingerprint — true JA3/JA4 match
      // @ts-ignore — impit is an optional dependency
      const impitMod = await import('impit');
      const ImpitClass = impitMod.Impit ?? impitMod.default;
      if (ImpitClass) {
        chromeFetchDispatcher = new ImpitClass({ browser: 'chrome' });
        return chromeFetchDispatcher;
      }
    } catch { /* impit not installed — fall through to undici */ }
    const { Agent } = await import('undici');
    chromeFetchDispatcher = new Agent({
      connect: {
        ciphers: CHROME_CIPHERS,
        ecdhCurve: CHROME_CURVES,
        minVersion: 'TLSv1.2' as any,
        maxVersion: 'TLSv1.3' as any,
      },
      keepAliveTimeout: 10_000,
      keepAliveMaxTimeout: 30_000,
    });
  }
  return chromeFetchDispatcher;
}

const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });

// Pages unlikely to have testable UI — skip from HTTP crawl queue
const _skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA
  ? process.env.CRAWL_SKIP_PATH_EXTRA.split(',').map((p) => p.trim()).filter(Boolean)
  : [];
const SKIP_PATH = _skipPathExtra.length > 0
  ? new RegExp(`/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility|${_skipPathExtra.join('|')})([-/]|$)`, 'i')
  : /\/(terms|privacy|dpa|legal|cookie|changelog|blog|help|docs|about|contact|careers|press|imprint|gdpr|compliance|acceptable-use|security-policy|accessibility)([-/]|$)/i;

async function getHttpConfig() {
  const { getSystemConfig } = await import('@detiq/core');
  const cfg = await getSystemConfig('crawl_tuning');
  return {
    retryLimit:   parseInt(String(cfg?.httpRetryLimit    ?? '5'),     10),
    backoffCapMs: parseInt(String(cfg?.httpBackoffCapMs  ?? '60000'), 10),
    retry5xx:     parseInt(String(cfg?.http5xxRetryLimit ?? '3'),     10),
    timeoutMs:    parseInt(String(cfg?.httpTimeoutMs     ?? '15000'), 10),
    concurrency:  parseInt(String(cfg?.httpConcurrency   ?? '5'),     10),
  };
}

export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';

export interface HttpScreen {
  url: string;
  title: string | null;
  markdown: string;
  html?: string;
  metadata: {
    description: string | null;
    ogTitle: string | null;
    ogDescription: string | null;
    ogImage: string | null;
    canonical: string | null;
    schemaTypes: string[];
    lang: string | null;
  };
  tables: Array<Record<string, string>[]>;
  links: string[];
  images: Array<{ src: string; alt: string }>;
  statusCode: number;
  fromCache: boolean;
  requiresBrowser?: boolean;
  durationMs: number;
}

export interface HttpCrawlOpts {
  appUrl: string;
  maxScreens?: number;            // default 100
  urlFilter?: string;             // regex pattern — only crawl matching URLs
  traversalStrategy?: 'BFS' | 'DFS'; // default BFS
  outputFormats?: HttpOutputFormat[];
  cacheDir?: string;              // default '/tmp/zeta-cache'
  userAgent?: string;
  headers?: Record<string, string>;
  respectRobotsTxt?: boolean;     // default true
  includeRawHtml?: boolean;
  timeoutMs?: number;             // per-request timeout, default 15000
  concurrency?: number;           // default 5 parallel fetches
  onProgress?: (update: { url: string; screensFound: number; total: number; cached: boolean }) => void;
}

export interface HttpCrawlResult {
  screens: HttpScreen[];
  stats: {
    total: number;
    cached: number;
    errors: number;
    rateLimited: number;
    durationMs: number;
    cacheHitRate: number;
  };
  jsonLines?: string;
  csv?: string;
}

function canonicalize(url: string): string {
  try {
    const u = new URL(url);
    u.hash = '';
    // Strip tracking params
    for (const p of ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content', 'utm_term', 'fbclid', 'gclid', 'ref', 'referrer']) {
      u.searchParams.delete(p);
    }
    return u.href.toLowerCase().replace(/\/$/, '');
  } catch {
    return url.toLowerCase();
  }
}

function extractLinks(html: string, baseUrl: string): string[] {
  const links: string[] = [];
  const seen = new Set<string>();
  try {
    const base = new URL(baseUrl);
    const { document } = parseHTML(html);
    const anchors = Array.from(document.querySelectorAll('a[href]'));
    for (const a of anchors) {
      const href = (a as any).getAttribute('href') ?? '';
      if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:') || href.startsWith('javascript:')) continue;
      try {
        const resolved = new URL(href, base).href;
        // Same origin only
        if (new URL(resolved).origin !== base.origin) continue;
        const c = canonicalize(resolved);
        if (!seen.has(c)) { seen.add(c); links.push(resolved); }
      } catch { /* skip */ }
    }
  } catch { /* non-fatal */ }
  return links;
}

function extractMarkdown(html: string, url: string): { markdown: string; title: string | null } {
  try {
    const { document } = parseHTML(html);
    // Remove script/style tags before Readability
    for (const tag of Array.from(document.querySelectorAll('script, style, noscript'))) (tag as any).remove();
    const reader = new Readability(document as any, { charThreshold: 50 });
    const article = reader.parse();
    if (article && article.textContent && article.textContent.split(/\s+/).length >= 30) {
      const md = turndown.turndown(article.content ?? html);
      return { markdown: md, title: article.title ?? null };
    }
  } catch { /* fall through */ }
  // Full-page fallback
  try {
    const { document } = parseHTML(html);
    for (const tag of Array.from(document.querySelectorAll('script, style, noscript, nav, footer'))) (tag as any).remove();
    const bodyHtml = (document as any).body?.innerHTML ?? html;
    const md = turndown.turndown(bodyHtml);
    const titleEl = (document as any).querySelector('title');
    return { markdown: md, title: titleEl?.textContent?.trim() ?? null };
  } catch {
    return { markdown: '', title: null };
  }
}

interface FetchResult {
  body: string;
  statusCode: number;
  contentType: string;
  fromCache: boolean;
  durationMs: number;
  etag?: string;
  lastModified?: string;
}

async function fetchWithBackoff(
  url: string,
  cache: ResponseCache,
  opts: { userAgent: string; timeoutMs: number; extraHeaders?: Record<string, string> },
  retryCount = 0,
  httpCfg?: Awaited<ReturnType<typeof getHttpConfig>>
): Promise<FetchResult> {
  if (!httpCfg) httpCfg = await getHttpConfig();
  const t0 = performance.now();

  if (cache.isFresh(url)) {
    const cached = cache.get(url)!;
    return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: 0 };
  }

  const validationHeaders = cache.getValidationHeaders(url);

  const headers: Record<string, string> = {
    'User-Agent': opts.userAgent,
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.9',
    'Accept-Encoding': 'gzip, deflate, br, zstd',
    // Chrome-like security headers reduce bot-detection fingerprint
    'sec-ch-ua': `"Google Chrome";v="${process.env.CRAWLER_CHROME_VERSION ?? '120'}", "Chromium";v="${process.env.CRAWLER_CHROME_VERSION ?? '120'}", "Not-A.Brand";v="99"`,
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'none',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    ...validationHeaders,
    ...(opts.extraHeaders ?? {}),
  };

  let response: Response;
  try {
    const dispatcher = await getChromeFetchDispatcher();
    response = await (fetch as any)(url, {
      headers,
      signal: AbortSignal.timeout(opts.timeoutMs),
      redirect: 'follow',
      dispatcher,
    });
  } catch (e: any) {
    throw new Error(`[http-engine] fetch failed for ${url}: ${e.message}`);
  }

  const durationMs = performance.now() - t0;

  // 304 Not Modified — use cached body
  if (response.status === 304) {
    const cached = cache.get(url);
    if (cached) {
      cache.markHit(url);
      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs };
    }
  }

  // 429 / 503 — exponential backoff with jitter
  if (response.status === 429 || response.status === 503) {
    if (retryCount >= httpCfg.retryLimit) throw new Error(`[http-engine] rate-limited after ${httpCfg.retryLimit} retries: ${url}`);
    const retryAfter = parseInt(response.headers.get('retry-after') ?? '0') || 0;
    const baseDelay = retryAfter > 0 ? retryAfter * 1000 : Math.min(httpCfg.backoffCapMs, 1_000 * Math.pow(2, retryCount));
    const jitter = Math.random() * 1000;
    console.log(`[http-engine] ${response.status} on ${url} — backing off ${Math.round(baseDelay + jitter)}ms (retry ${retryCount + 1}/${httpCfg.retryLimit})`);
    await new Promise(r => setTimeout(r, baseDelay + jitter));
    return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
  }

  // 5xx — retry up to 3 times
  if (response.status >= 500) {
    if (retryCount < httpCfg.retry5xx) {
      await new Promise(r => setTimeout(r, 2_000 * (retryCount + 1)));
      return fetchWithBackoff(url, cache, opts, retryCount + 1, httpCfg);
    }
    if (cache.staleIfError(url)) {
      const cached = cache.get(url)!;
      return { body: cached.body, statusCode: 200, contentType: cached.contentType, fromCache: true, durationMs: performance.now() - t0 };
    }
  }

  const contentType = response.headers.get('content-type') ?? '';
  const body = await response.text();
  const etag = response.headers.get('etag') ?? undefined;
  const lastModified = response.headers.get('last-modified') ?? undefined;
  const cacheControl = response.headers.get('cache-control') ?? undefined;

  // Cache successful HTML responses
  if (response.ok && contentType.includes('html')) {
    cache.set(url, { url, etag, lastModified, cacheControl, body, contentType, statusCode: response.status });
  }

  return { body, statusCode: response.status, contentType, fromCache: false, durationMs, etag, lastModified };
}

export async function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult> {
  const startTime = performance.now();
  const httpCfg = await getHttpConfig();
  const maxScreens = opts.maxScreens ?? 100;
  const cacheDir = opts.cacheDir ?? '/tmp/zeta-cache';
  const timeoutMs = opts.timeoutMs ?? httpCfg.timeoutMs;
  const userAgent = opts.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)';
  const cache = new ResponseCache(cacheDir);
  const robots = opts.respectRobotsTxt !== false ? new RobotsCache() : null;
  const urlFilterRe = opts.urlFilter ? (() => { try { return new RegExp(opts.urlFilter!, 'i'); } catch { return null; } })() : null;

  const visited = new Set<string>();
  const queue: string[] = [opts.appUrl];
  const queued = new Set<string>([canonicalize(opts.appUrl)]);
  const screens: HttpScreen[] = [];
  let cachedCount = 0;
  let errorCount = 0;
  let rateLimitedCount = 0;

  const batchFetch = async (urls: string[]): Promise<void> => {
    await Promise.all(urls.map(async (url) => {
      const canonUrl = canonicalize(url);
      if (visited.has(canonUrl)) return;
      visited.add(canonUrl);

      // Robots.txt check
      if (robots && !(await robots.isAllowed(url).catch(() => true))) {
        console.log(`[http-engine] robots.txt blocked: ${url}`);
        return;
      }

      // Skip content/legal pages
      try {
        const { pathname } = new URL(url);
        if (SKIP_PATH.test(pathname)) return;
      } catch { /* skip */ }

      // URL filter
      if (urlFilterRe) {
        try {
          const { pathname, search } = new URL(url);
          if (!urlFilterRe.test(pathname + search)) {
            console.log(`[http-engine] filtered out: ${url}`);
            return;
          }
        } catch { /* skip */ }
      }

      let fetchResult: FetchResult;
      try {
        fetchResult = await fetchWithBackoff(url, cache, { userAgent, timeoutMs, extraHeaders: opts.headers }, 0, httpCfg);
        if (fetchResult.statusCode === 429 || fetchResult.statusCode === 503) rateLimitedCount++;
      } catch (err: any) {
        console.error(`[http-engine] error fetching ${url}: ${err.message}`);
        errorCount++;
        return;
      }

      if (!fetchResult.contentType.includes('html')) return; // skip non-HTML
      if (fetchResult.statusCode >= 400) return;
      if (fetchResult.fromCache) cachedCount++;

      const { markdown, title } = extractMarkdown(fetchResult.body, url);
      const fullMeta = extractMetadata(fetchResult.body, url);
      const links = extractLinks(fetchResult.body, url);
      const tables = extractTables(fetchResult.body);
      const images = extractImages(fetchResult.body, url);

      const screen: HttpScreen = {
        url,
        title: title ?? fullMeta.title,
        markdown,
        html: opts.includeRawHtml ? fetchResult.body : undefined,
        metadata: {
          description: fullMeta.description,
          ogTitle: fullMeta.ogTitle,
          ogDescription: fullMeta.ogDescription,
          ogImage: fullMeta.ogImage,
          canonical: fullMeta.canonical,
          schemaTypes: fullMeta.schemaTypes,
          lang: fullMeta.lang,
        },
        tables,
        links,
        images: images.map(i => ({ src: i.src, alt: i.alt })),
        statusCode: fetchResult.statusCode,
        fromCache: fetchResult.fromCache,
        requiresBrowser: renderingPredictor.getCached(url) === 'browser',
        durationMs: fetchResult.durationMs,
      };
      screens.push(screen);

      // Record rendering prediction: SPA signals → browser needed; content-rich → http engine works
      renderingPredictor.predict(url, fetchResult.body, fetchResult.statusCode);

      opts.onProgress?.({ url, screensFound: screens.length, total: maxScreens, cached: fetchResult.fromCache });

      // Enqueue discovered links
      for (const link of links) {
        const c = canonicalize(link);
        if (!queued.has(c) && !visited.has(c) && screens.length + queue.length < maxScreens * 2) {
          queued.add(c);
          if (opts.traversalStrategy === 'DFS') queue.unshift(link);
          else queue.push(link);
        }
      }
    }));
  };

  const concurrency = opts.concurrency ?? httpCfg.concurrency;

  while (queue.length > 0 && screens.length < maxScreens) {
    const batch = queue.splice(0, concurrency);
    await batchFetch(batch);
  }

  // Output formats
  let jsonLines: string | undefined;
  let csv: string | undefined;

  const formats = opts.outputFormats ?? [];
  if (formats.includes('jsonlines')) {
    jsonLines = screens.map(s => JSON.stringify({ url: s.url, title: s.title, markdown: s.markdown, metadata: s.metadata })).join('\n');
  }
  if (formats.includes('csv')) {
    const header = 'url,title,description,ogTitle,schemaTypes,statusCode,fromCache,durationMs';
    const rows = screens.map(s =>
      [s.url, s.title ?? '', s.metadata.description ?? '', s.metadata.ogTitle ?? '', s.metadata.schemaTypes.join('|'), s.statusCode, s.fromCache, Math.round(s.durationMs)].map(v => `"${String(v).replace(/"/g, '""')}"`).join(',')
    );
    csv = [header, ...rows].join('\n');
  }

  const cacheStats = cache.stats();
  return {
    screens,
    stats: {
      total: screens.length,
      cached: cachedCount,
      errors: errorCount,
      rateLimited: rateLimitedCount,
      durationMs: performance.now() - startTime,
      cacheHitRate: cacheStats.hitRate,
    },
    jsonLines,
    csv,
  };
}