/**
 * llms.txt (https://llmstxt.org/) discovery, caching, and structure-aware
 * truncation.
 *
 * Probes the standardized /llms.txt for a URL's origin (and, when opted in,
 * walks parent dirs for subpath-hosted files). Content is validated (content-
 * type + sniff in safe-fetch's fetchTextFile, plus the markdown heuristic here)
 * so HTML soft-404s / JSON blobs are never injected into context. Per-origin
 * caching + in-flight dedup avoids re-probing the same domain within a search.
 *
 * The parsing/truncation helpers are pure; only the cache + fetch touch I/O.
 */

import { fetchTextFile } from "./safe-fetch.ts";

export interface LlmstxtResult {
  content: string;
  truncated: boolean;
}

// Short, dedicated timeout for llms.txt probes so a slow/non-existent file
// never holds up the main results.
export const LLMS_TXT_TIMEOUT_MS = 3000;
// Hard cap on raw content kept in memory/cache per origin.
const LLMS_TXT_RAW_CAP = 200_000;
// In-process cache TTL and size bound.
const LLMS_TXT_TTL_MS = 10 * 60_000;
const LLMS_TXT_CACHE_MAX = 256;

interface LlmsTxtCacheEntry {
  at: number;
  promise: Promise<string | undefined>;
}
const _llmsTxtCache = new Map<string, LlmsTxtCacheEntry>();

function pruneLlmsTxtCache(): void {
  const now = Date.now();
  for (const [k, v] of _llmsTxtCache) {
    if (now - v.at >= LLMS_TXT_TTL_MS) _llmsTxtCache.delete(k);
  }
  while (_llmsTxtCache.size > LLMS_TXT_CACHE_MAX) {
    const oldest = _llmsTxtCache.keys().next().value;
    if (oldest === undefined) break;
    _llmsTxtCache.delete(oldest);
  }
}

/**
 * Heuristic: does this text look like a real llms.txt (markdown) rather than an
 * HTML soft-404, JSON error blob, or other garbage? Per the spec, llms.txt is
 * markdown that starts with an H1, a `>` summary, or a markdown link list.
 */
export function looksLikeLlmsTxt(text: string): boolean {
  const head = text.slice(0, 2000);
  if (/<!doctype\s+html|<html[\s>]|<head[\s>]|<body[\s>]|<script[\s>]|<\?xml/i.test(head)) {
    return false;
  }
  const firstLine = text.split("\n").map((l) => l.trim()).find((l) => l.length > 0) ?? "";
  if (!firstLine) return false;
  return (
    /^#{1,6}\s+\S/.test(firstLine) || // markdown heading
    firstLine.startsWith(">") || // blockquote summary
    /^[-*]\s*\[.+\]\(.+\)/.test(firstLine) // markdown link list item
  );
}

// Directory chain from a page's directory up to root, capped to bound requests.
// e.g. "/docs/tutorials/" -> ["/docs/tutorials/", "/docs/", "/"]
export function parentDirs(basePath: string, maxLevels: number): string[] {
  const dirs: string[] = [];
  let p = basePath && basePath.startsWith("/") ? basePath : "/";
  if (!p.endsWith("/")) p += "/";
  while (dirs.length < maxLevels) {
    if (!dirs.includes(p)) dirs.push(p);
    if (p === "/") break;
    p = p.replace(/[^/]+\/$/, "") || "/"; // strip last path segment
  }
  if (!dirs.includes("/")) dirs.push("/");
  return dirs;
}

/** Split llms.txt into its preamble (H1 + summary + intro) and `## ` sections. */
export function splitLlmsTxtSections(
  content: string,
): { preamble: string; sections: { name: string; body: string }[] } {
  const lines = content.split("\n");
  const idxs: number[] = [];
  for (let i = 0; i < lines.length; i++) {
    if (/^##\s+/.test(lines[i])) idxs.push(i);
  }
  if (idxs.length === 0) return { preamble: content, sections: [] };
  const preamble = lines.slice(0, idxs[0]).join("\n");
  const sections = idxs.map((start, k) => {
    const end = k + 1 < idxs.length ? idxs[k + 1] : lines.length;
    return {
      name: lines[start].replace(/^##\s+/, "").trim().toLowerCase(),
      body: lines.slice(start, end).join("\n").trim(),
    };
  });
  return { preamble, sections };
}

export function hardSlice(s: string, maxChars: number): string {
  if (s.length <= maxChars) return s;
  let cut = s.lastIndexOf("\n", maxChars);
  if (cut < maxChars * 0.6) cut = maxChars; // avoid cutting too aggressively
  return s.slice(0, cut).trim();
}

/**
 * Structure-aware truncation: drop the spec's `## Optional` section first, keep
 * whole sections in order until the budget is reached, and only then fall back
 * to a line-boundary hard slice — avoids cutting mid-link/mid-section.
 */
export function truncateLlmsTxt(content: string, maxChars: number): LlmstxtResult {
  if (content.length <= maxChars) return { content, truncated: false };

  const { preamble, sections } = splitLlmsTxtSections(content);
  if (sections.length === 0) {
    return { content: hardSlice(content, maxChars), truncated: true };
  }

  const kept = sections.filter((s) => s.name !== "optional");
  const droppedOptional = kept.length !== sections.length;

  let out = preamble.trim();
  let usedSections = 0;
  for (const sec of kept) {
    if (out.length + sec.body.length + 2 <= maxChars) {
      out += `\n\n${sec.body}`;
      usedSections++;
    } else {
      break;
    }
  }

  const droppedSections = kept.length - usedSections;
  if (out.length > maxChars) out = hardSlice(out, maxChars); // preamble alone overflowed

  const notes: string[] = [];
  if (droppedOptional) notes.push("optional section omitted");
  if (droppedSections > 0) notes.push(`${droppedSections} section(s) omitted`);
  if (notes.length) out += `\n\n_[llms.txt truncated: ${notes.join("; ")} to fit budget]_`;

  return { content: out.trim(), truncated: true };
}

export interface LlmsTxtFetchOptions {
  userAgent: string;
  maxChars: number;
  fetchFull: boolean;
  allowPrivateNetwork?: boolean;
  signal?: AbortSignal;
}

async function fetchLlmsTxtRawUncached(
  origin: string,
  basePath: string,
  opts: LlmsTxtFetchOptions,
): Promise<string | undefined> {
  const candidates: string[] = [];
  if (opts.fetchFull) {
    // Deeper discovery: walk the page's directory tree (capped), preferring the
    // fuller file then the index at each level — catches subpath-hosted files.
    const dirs = parentDirs(basePath, 3);
    for (const d of dirs) candidates.push(`${origin}${d}llms-full.txt`);
    for (const d of dirs) candidates.push(`${origin}${d}llms.txt`);
  }
  candidates.push(`${origin}/llms.txt`); // standardized root path
  const unique = [...new Set(candidates)];

  let combined = "";
  for (const llmsUrl of unique) {
    const text = await fetchTextFile(llmsUrl, {
      userAgent: opts.userAgent,
      timeoutMs: LLMS_TXT_TIMEOUT_MS,
      allowPrivateNetwork: opts.allowPrivateNetwork,
      signal: opts.signal,
    });
    if (text && looksLikeLlmsTxt(text)) {
      combined += combined ? `\n\n---\n\n${text}` : text;
      if (combined.length >= LLMS_TXT_RAW_CAP) {
        combined = combined.slice(0, LLMS_TXT_RAW_CAP);
        break;
      }
    }
  }
  return combined.trim() ? combined.trim() : undefined;
}

/** Cached + in-flight-deduped raw fetch keyed by origin (+ base/full when relevant). */
function getLlmsTxtRaw(origin: string, basePath: string, opts: LlmsTxtFetchOptions): Promise<string | undefined> {
  pruneLlmsTxtCache();
  const key = `${origin}|full=${opts.fetchFull}|base=${opts.fetchFull ? basePath : ""}`;
  const now = Date.now();
  const hit = _llmsTxtCache.get(key);
  if (hit && now - hit.at < LLMS_TXT_TTL_MS) {
    // Refresh LRU position so a frequently-reused fresh entry isn't evicted
    // before colder ones once the cache exceeds its size bound.
    _llmsTxtCache.delete(key);
    _llmsTxtCache.set(key, hit);
    return hit.promise;
  }

  const promise = (async () => {
    try {
      // Negative results (clean 404 -> undefined) are intentionally cached to
      // avoid re-probing domains without llms.txt on every search.
      return await fetchLlmsTxtRawUncached(origin, basePath, opts);
    } catch {
      // Abort/transient error: evict so it can be retried later.
      _llmsTxtCache.delete(key);
      return undefined;
    }
  })();
  _llmsTxtCache.set(key, { at: now, promise });
  return promise;
}

/**
 * Fetch /llms.txt (and optionally /llms-full.txt) for a URL's origin, with
 * per-origin caching/dedup and structure-aware truncation. Returns undefined
 * when the URL is malformed or no llms.txt exists. Callers gate on the
 * llmsTxtEnabled config before calling.
 */
export async function fetchLlmstxt(url: string, opts: LlmsTxtFetchOptions): Promise<LlmstxtResult | undefined> {
  let origin: string;
  let basePath: string;
  try {
    const u = new URL(url);
    origin = u.origin;
    basePath = u.pathname.replace(/\/[^/]*$/, "/"); // the page's directory
    if (basePath === "/") basePath = ""; // root page: no separate subpath probe
  } catch {
    return undefined;
  }

  const raw = await getLlmsTxtRaw(origin, basePath, opts);
  if (!raw) return undefined;
  return truncateLlmsTxt(raw, opts.maxChars);
}