/** * Optional self-hosted reader fallback (last escalation tier). * * Routes a blocked URL through a USER-CONFIGURED reader endpoint that fetches it * server-side and returns clean content. Two shapes are supported: * - "jina": a Jina-Reader-style service — GET `${endpoint}/${url}` → markdown. * - "flaresolverr": FlareSolverr — POST `${endpoint}` `{cmd:"request.get",url}` * → JSON `{solution:{response:""}}` (HTML, handed back for extraction). * * There is NO third-party default: `readerEndpoint` is unset by default and is * global-config-only (egress-sensitive, like `kagiEndpoint`). The request to the * endpoint goes through the same SSRF guard (`safeFetch`) under * `allowPrivateNetwork`, since a self-hosted reader is often on a private host. */ import { MAX_HTML_BYTES, readCappedText, safeFetch } from "./safe-fetch.ts"; export type ReaderMode = "jina" | "flaresolverr" | "auto"; export interface ReaderOptions { endpoint: string; mode: ReaderMode; timeoutMs: number; userAgent: string; acceptLanguage?: string; allowPrivateNetwork: boolean; signal?: AbortSignal; } export interface ReaderResult { markdown?: string; html?: string; error?: string; } function resolveMode(endpoint: string, mode: ReaderMode): "jina" | "flaresolverr" { if (mode === "jina" || mode === "flaresolverr") return mode; // auto: FlareSolverr is conventionally reached at `.../v1`. return /\/v1\/?$/.test(endpoint) || /flaresolverr/i.test(endpoint) ? "flaresolverr" : "jina"; } /** Fetch `url` via the configured reader endpoint. Never throws. */ export async function fetchViaReader(url: string, opts: ReaderOptions): Promise { const endpoint = opts.endpoint.trim(); if (!endpoint) return { error: "no reader endpoint configured" }; const mode = resolveMode(endpoint, opts.mode); try { if (mode === "flaresolverr") { const res = await safeFetch(endpoint, { method: "POST", body: JSON.stringify({ cmd: "request.get", url, maxTimeout: opts.timeoutMs }), headers: { "Content-Type": "application/json", Accept: "application/json" }, timeoutMs: opts.timeoutMs + 5_000, // FlareSolverr solves the challenge before replying signal: opts.signal, allowPrivateNetwork: opts.allowPrivateNetwork, }); if (!res.ok) { await res.body?.cancel().catch(() => {}); return { error: `reader HTTP ${res.status} ${res.statusText}` }; } const json = JSON.parse(await readCappedText(res, MAX_HTML_BYTES)) as { solution?: { response?: string }; }; const html = json?.solution?.response; return typeof html === "string" && html.trim() ? { html } : { error: "reader returned no content" }; } // jina-style: GET endpoint with the target URL appended. const target = `${endpoint.replace(/\/+$/, "")}/${url}`; const res = await safeFetch(target, { headers: { "User-Agent": opts.userAgent, "Accept-Language": opts.acceptLanguage ?? "en-US,en;q=0.9", Accept: "text/markdown,text/plain,*/*", "X-Return-Format": "markdown", }, timeoutMs: opts.timeoutMs + 5_000, signal: opts.signal, allowPrivateNetwork: opts.allowPrivateNetwork, }); if (!res.ok) { await res.body?.cancel().catch(() => {}); return { error: `reader HTTP ${res.status} ${res.statusText}` }; } const text = (await readCappedText(res, MAX_HTML_BYTES)).trim(); return text ? { markdown: text } : { error: "reader returned no content" }; } catch (e) { return { error: `reader failed: ${e instanceof Error ? e.message : String(e)}` }; } }