import { collapseWhitespace } from "./html-to-md.ts";
import { checkUrlAccess, MAX_SIGNAL_TIMEOUT_MS, normalizeUrlScheme, type WebsitePolicy } from "./web-access.ts";
import { parseLocalPath, resolveAndValidateHost, truncatePageText } from "./web-fetch.ts";
const DEFAULT_RENDER_TIMEOUT_MS = 60_000;
const JINA_READER_URL = "https://r.jina.ai/";
const CANCELLED_MESSAGE = "Failed to render URL: cancelled.";
const TIMED_OUT_MESSAGE = "Failed to render URL: timed out.";
const LOCAL_FILE_MESSAGE = "Blocked: web_render cannot fetch local files.";
const EMPTY_READER_MESSAGE = "Failed to render URL: the Jina Reader returned no content.";
export interface RenderPageOptions {
timeoutMs?: number;
maxChars?: number;
signal?: AbortSignal;
websitePolicy?: WebsitePolicy | null;
apiKey?: string;
}
interface JinaReaderData {
title?: string;
url?: string;
content?: string;
}
interface JinaReaderPayload {
data?: JinaReaderData;
}
function clampTimeout(value: number | undefined): number {
if (value === undefined || !Number.isFinite(value)) return DEFAULT_RENDER_TIMEOUT_MS;
return Math.min(MAX_SIGNAL_TIMEOUT_MS, Math.max(1, Math.floor(value)));
}
function budgetMessage(caller: AbortSignal | undefined, signal: AbortSignal): string | null {
if (caller?.aborted) return CANCELLED_MESSAGE;
if (signal.aborted) return TIMED_OUT_MESSAGE;
return null;
}
async function readerErrorDetail(response: Response): Promise {
try {
const body = (await response.json()) as { readableMessage?: unknown; message?: unknown; detail?: unknown };
const detail = [body.readableMessage, body.message, body.detail].find(
(value) => typeof value === "string" && value.trim().length > 0,
);
if (detail) return `${response.status} ${detail}`;
} catch {}
return `${response.status} ${response.statusText}`.trim();
}
function formatRenderedPage(data: JinaReaderData, fallbackUrl: string, maxChars: number | undefined): string {
const title = collapseWhitespace(String(data.title ?? ""));
const finalUrl = collapseWhitespace(String(data.url ?? "")) || fallbackUrl;
const content = String(data.content ?? "").trim();
if (!content) return truncatePageText("", maxChars);
const header: string[] = [];
if (title) header.push(`Title: ${title}`);
header.push(`URL: ${finalUrl}`);
header.push("Rendered via the Jina Reader (r.jina.ai).");
return truncatePageText(`${header.join("\n")}\n\n${content}`, maxChars);
}
export async function renderPageText(url: string, options: RenderPageOptions = {}): Promise {
const target = typeof url === "string" ? url.trim() : "";
const policy = options.websitePolicy ?? null;
if (!target) return checkUrlAccess("", policy)[1];
if (parseLocalPath(target) !== null) return LOCAL_FILE_MESSAGE;
const normalized = normalizeUrlScheme(target);
const [allowed, reason, hostname] = checkUrlAccess(normalized, policy);
if (!allowed) return reason;
const timeoutSignal = AbortSignal.timeout(clampTimeout(options.timeoutMs));
const signal = options.signal ? AbortSignal.any([options.signal, timeoutSignal]) : timeoutSignal;
const resolved = await resolveAndValidateHost(hostname, signal, false);
const resolutionBudget = budgetMessage(options.signal, signal);
if (resolutionBudget !== null) return resolutionBudget;
if (!resolved.ok) return resolved.reason;
const headers: Record = { Accept: "application/json" };
const apiKey = options.apiKey?.trim();
if (apiKey) headers["Authorization"] = `Bearer ${apiKey}`;
let response: Response;
try {
response = await fetch(JINA_READER_URL + encodeURIComponent(normalized), { headers, signal });
} catch (err) {
const budget = budgetMessage(options.signal, signal);
if (budget !== null) return budget;
return `Failed to render URL: ${err instanceof Error ? err.message : String(err)}`;
}
if (!response.ok) return `Failed to render URL: ${await readerErrorDetail(response)}`;
let payload: JinaReaderPayload;
try {
payload = (await response.json()) as JinaReaderPayload;
} catch {
const budget = budgetMessage(options.signal, signal);
if (budget !== null) return budget;
return `Failed to render URL: the Jina Reader returned a non-JSON response (HTTP ${response.status}).`;
}
if (!payload || !payload.data) return EMPTY_READER_MESSAGE;
return formatRenderedPage(payload.data, normalized, options.maxChars);
}