/**
 * Single-page fetch → clean markdown (+ llms.txt), with a short-lived cache.
 *
 * Extracted pages are cached for up to an hour (keyed by url + maxChars) so that
 * repeat reads — per-page RAG follow-ups, multi-hop link-following, or the same
 * URL surfacing across searches — don't re-fetch or re-extract. Mirrors the
 * fetch/extract logic web_search and web_fetch both need; the SSRF guard lives in
 * safe-fetch.ts and applies to every fetch here.
 */

import { fetchHtml, MAX_HTML_BYTES } from "./safe-fetch.ts";
import { type Extractor, toMarkdown } from "../content/extract.ts";
import { fetchLlmstxt } from "./llms-txt.ts";
import type { AssembledPage } from "../content/assemble.ts";
import type { SearchResult } from "../search/search.ts";
import { TtlCache } from "../util.ts";
import { browserAvailable, browserDepError, renderWithBrowser } from "./browser.ts";
import { fetchViaReader, type ReaderMode } from "./reader.ts";
import { recordBlock } from "../session/block-ledger.ts";

const PAGE_TTL_MS = 60 * 60_000; // 1 hour
const PAGE_CACHE_MAX = 256;
const _pageCache = new TtlCache<AssembledPage>(PAGE_TTL_MS, PAGE_CACHE_MAX);

export interface FetchPageOptions {
  extractor: Extractor;
  maxChars: number;
  userAgent: string;
  acceptLanguage?: string;
  timeoutMs: number;
  allowPrivateNetwork: boolean;
  llmsTxtEnabled: boolean;
  llmsTxtMaxChars: number;
  llmsTxtFetchFull: boolean;
  // Bot-block escalation tiers (see fetchPageUncached).
  browserFallbackEnabled?: boolean;
  browserTimeoutMs?: number;
  browserNoSandbox?: boolean;
  readerEndpoint?: string;
  readerMode?: ReaderMode;
  signal?: AbortSignal;
}

interface Escalation {
  html?: string;
  markdown?: string;
  blocked?: true;
  tiersTried?: string;
  note?: string;
}

/** Collapse blank lines + char-truncate reader-supplied markdown (mirrors toMarkdown). */
function capMarkdown(md: string, maxChars: number): { markdown: string; truncated: boolean } {
  const clean = md.replace(/\n{3,}/g, "\n\n").trim();
  if (clean.length > maxChars) return { markdown: clean.slice(0, maxChars), truncated: true };
  return { markdown: clean, truncated: false };
}

/**
 * Escalate a bot-blocked fetch: try a sandboxed headless browser, then an
 * optional self-hosted reader. Returns rendered HTML (for the normal extractor
 * path), reader markdown (used as-is), or `{blocked:true}` when every tier fails.
 */
async function escalate(url: string, o: FetchPageOptions): Promise<Escalation> {
  const tried: string[] = [];
  let note: string | undefined;

  if (o.browserFallbackEnabled !== false) {
    if (browserAvailable()) {
      tried.push("browser-render");
      const br = await renderWithBrowser(url, {
        timeoutMs: o.browserTimeoutMs ?? 20000,
        allowPrivateNetwork: o.allowPrivateNetwork,
        maxBytes: MAX_HTML_BYTES,
        noSandbox: o.browserNoSandbox === true,
        userAgent: o.userAgent,
        acceptLanguage: o.acceptLanguage,
        signal: o.signal,
      });
      if (br.html) return { html: br.html };
      if (br.sandboxFailure || br.error) note = br.error;
    } else {
      note = browserDepError();
    }
  }

  const endpoint = o.readerEndpoint?.trim();
  if (endpoint) {
    tried.push("reader");
    const rd = await fetchViaReader(url, {
      endpoint,
      mode: o.readerMode ?? "auto",
      timeoutMs: o.browserTimeoutMs ?? 20000,
      userAgent: o.userAgent,
      acceptLanguage: o.acceptLanguage,
      allowPrivateNetwork: o.allowPrivateNetwork,
      signal: o.signal,
    });
    if (rd.markdown) return { markdown: rd.markdown };
    if (rd.html) return { html: rd.html };
    if (rd.error) note = rd.error;
  }

  return { blocked: true, tiersTried: tried.join(" and "), note };
}

/**
 * Fetch one page → AssembledPage (clean markdown + optional llms.txt). Probes
 * llms.txt concurrently with the page fetch/extract. Never throws: a failure is
 * captured in the returned page's `.error`.
 */
async function fetchPageUncached(r: SearchResult, o: FetchPageOptions): Promise<AssembledPage> {
  const llmsPromise = o.llmsTxtEnabled
    ? fetchLlmstxt(r.url, {
        userAgent: o.userAgent,
        maxChars: o.llmsTxtMaxChars,
        fetchFull: o.llmsTxtFetchFull,
        allowPrivateNetwork: o.allowPrivateNetwork,
        signal: o.signal,
      }).catch(() => undefined)
    : Promise.resolve(undefined);

  const res = await fetchHtml(r.url, {
    timeoutMs: o.timeoutMs,
    userAgent: o.userAgent,
    acceptLanguage: o.acceptLanguage,
    allowPrivateNetwork: o.allowPrivateNetwork,
    signal: o.signal,
  });

  let html = res.html;

  if (!html && res.blocked) {
    // Bot-blocked: escalate (headless browser → reader). Only a terminal block
    // (all tiers exhausted) is recorded in the ledger and surfaced as `blocked`.
    const esc = await escalate(r.url, o);
    if (esc.markdown !== undefined) {
      const { markdown, truncated } = capMarkdown(esc.markdown, o.maxChars);
      return { ...r, markdown, truncated, llmsTxt: await llmsPromise };
    }
    if (esc.html) {
      html = esc.html;
    } else {
      const lastError = res.error ?? "blocked";
      const fullError = esc.note ? `${lastError} — ${esc.note}` : lastError;
      recordBlock(r.url, { status: res.status, reason: res.blockReason, lastError: fullError });
      return {
        ...r,
        error: fullError,
        blocked: true,
        status: res.status,
        blockReason: res.blockReason,
        tiersTried: esc.tiersTried,
        llmsTxt: await llmsPromise,
      };
    }
  } else if (res.error || !html) {
    // Generic (non-block) failure: do NOT escalate, do NOT touch the ledger.
    return { ...r, error: res.error ?? "no content", llmsTxt: await llmsPromise };
  }

  if (!html) return { ...r, error: "no content", llmsTxt: await llmsPromise };
  try {
    const { title, markdown, truncated } = await toMarkdown(o.extractor, html, r.url, o.maxChars);
    return { ...r, title: title || r.title, markdown, truncated, llmsTxt: await llmsPromise };
  } catch (e) {
    return { ...r, error: `extract failed: ${e instanceof Error ? e.message : String(e)}`, llmsTxt: await llmsPromise };
  }
}

/**
 * Cached single-page fetch. A fresh, successfully-extracted page is reused within
 * the TTL; failures are not cached (so a transient error can be retried). The
 * cache key includes maxChars so a larger request isn't served a truncated hit.
 */
export function cachedFetchPage(r: SearchResult, o: FetchPageOptions): Promise<AssembledPage> {
  const key = `${r.url}|${o.maxChars}`;
  const hit = _pageCache.get(key);
  if (hit) return hit;
  const promise = fetchPageUncached(r, o).then((page) => {
    // Don't cache failures — evict so the next call retries.
    if (page.error || !page.markdown) _pageCache.delete(key);
    return page;
  });
  _pageCache.set(key, promise);
  return promise;
}