import { Readability } from "@mozilla/readability"; import { parseHTML } from "linkedom"; import TurndownService from "turndown"; import pLimit from "p-limit"; import { fetchRemoteUrl, validateRemoteUrl } from "./ssrf.ts"; import { getSsrfAllowRanges } from "./config.ts"; const DEFAULT_TIMEOUT_MS = 30_000; const CONCURRENT_LIMIT = 3; const MIN_USEFUL_CONTENT = 500; const MAX_RESPONSE_BYTES = 5 * 1024 * 1024; const MAX_PDF_BYTES = 20 * 1024 * 1024; const BROWSER_HEADERS: Record = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", // Prefer server-negotiated markdown, fall back to HTML (acceptmarkdown.com convention). Accept: "text/markdown, text/html;q=0.9, application/xhtml+xml;q=0.9, */*;q=0.8", "Accept-Language": "en-US,en;q=0.9", "Sec-Fetch-Dest": "document", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Site": "none", "Upgrade-Insecure-Requests": "1", }; const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" }); const fetchLimit = pLimit(CONCURRENT_LIMIT); export interface FetchedContent { url: string; title: string; content: string; error: string | null; renderer: "markdown" | "http" | "playwright" | "pdf" | null; } function errorMessage(err: unknown): string { return err instanceof Error ? err.message : String(err); } /** Title from the first markdown heading, falling back to the URL basename. */ function firstHeadingTitle(text: string, url: string): string { return ( text.match(/^#{1,6}\s+(.+)/m)?.[1]?.trim() || urlBasename(url) ); } function urlBasename(url: string): string { try { return new URL(url).pathname.split("/").filter(Boolean).pop() || url; } catch { return url; } } function isPdf(url: string, mediaType: string): boolean { return mediaType === "application/pdf" || urlBasename(url).toLowerCase().endsWith(".pdf"); } /** Extract PDF text page-by-page (with page markers) via unpdf, lazily loaded. */ async function extractPdfText(buffer: ArrayBuffer, url: string): Promise { const { getDocumentProxy } = await import("unpdf"); const pdf = await getDocumentProxy(new Uint8Array(buffer)); const lines: string[] = [`# ${urlBasename(url)}`, "", `> Source: ${url}`, `> Pages: ${pdf.numPages}`, "", "---", ""]; for (let i = 1; i <= pdf.numPages; i++) { const page = await pdf.getPage(i); const content = await page.getTextContent(); const pageText = content.items .map((item) => (item as { str?: string }).str ?? "") .join(" ") .replace(/\s+/g, " ") .trim(); if (!pageText) continue; if (i > 1) lines.push("", ``, ""); lines.push(pageText); } return lines.join("\n"); } function isAbortError(err: unknown): boolean { return errorMessage(err).toLowerCase().includes("abort"); } /** Heuristic: little text but many scripts suggests client-side rendering. */ function isLikelyJSRendered(html: string): boolean { const body = html.match(/]*>([\s\S]*?)<\/body>/i)?.[1] ?? ""; const text = body .replace(//gi, "") .replace(//gi, "") .replace(/<[^>]+>/g, "") .replace(/\s+/g, " ") .trim(); const scriptCount = (html.match(/