/** * Shared types and utilities for web-fetch handlers */ import { ptree } from "@gajae-code/utils"; import type { AgentStorage } from "../../session/agent-storage"; import { ToolAbortError } from "../../tools/tool-errors"; import type { AddressResolver } from "../insane/url-guard"; import { guardedPublicFetch } from "../insane/url-guard"; export { formatNumber } from "@gajae-code/utils"; export interface RenderResult { url: string; finalUrl: string; contentType: string; method: string; content: string; fetchedAt: string; truncated: boolean; notes: string[]; } export type SpecialHandler = ( url: string, timeout: number, signal?: AbortSignal, storage?: AgentStorage | null, ) => Promise; export const MAX_OUTPUT_CHARS = 500_000; export const MAX_BYTES = 50 * 1024 * 1024; const USER_AGENTS = [ "curl/8.0", "Mozilla/5.0 (compatible; TextBot/1.0)", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", ]; const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); function isBotBlocked(status: number, content: string): boolean { if (status === 403 || status === 503) { const lower = content.toLowerCase(); return ( lower.includes("cloudflare") || lower.includes("captcha") || lower.includes("challenge") || lower.includes("blocked") || lower.includes("access denied") || lower.includes("bot detection") ); } return false; } /** * Truncate and cleanup output */ export function finalizeOutput(content: string): { content: string; truncated: boolean } { const cleaned = content.replace(/\n{3,}/g, "\n\n").trim(); const truncated = cleaned.length > MAX_OUTPUT_CHARS; return { content: cleaned.slice(0, MAX_OUTPUT_CHARS), truncated, }; } export interface LoadPageOptions { timeout?: number; headers?: Record; method?: string; body?: string; maxBytes?: number; signal?: AbortSignal; publicUrlGuard?: boolean; resolver?: AddressResolver; maxRedirects?: number; } export interface LoadPageResult { content: string; contentType: string; finalUrl: string; ok: boolean; status?: number; error?: string; } function shouldRewriteRedirectMethod(status: number, method: string): boolean { const normalized = method.toUpperCase(); return status === 303 || ((status === 301 || status === 302) && normalized === "POST"); } /** * Fetch a page with timeout and size limit */ export async function loadPage(url: string, options: LoadPageOptions = {}): Promise { const { timeout = 20, headers = {}, maxBytes = MAX_BYTES, signal, method = "GET", body, publicUrlGuard = true, resolver, maxRedirects = 10, } = options; const initialUrl = url; attempts: for (let attempt = 0; attempt < USER_AGENTS.length; attempt++) { if (signal?.aborted) { throw new ToolAbortError(); } const userAgent = USER_AGENTS[attempt]; const requestSignal = ptree.combineSignals(signal, timeout * 1000); let currentUrl = initialUrl; let currentMethod = method; let currentBody = body; try { for (let redirectCount = 0; redirectCount <= maxRedirects; redirectCount++) { const requestInit: RequestInit = { signal: requestSignal, method: currentMethod, headers: { "User-Agent": userAgent, Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "identity", // Cloudflare Markdown-for-Agents returns corrupted bytes when compression is negotiated ...headers, }, redirect: "manual", }; if (currentBody !== undefined) { requestInit.body = currentBody; } const dial = publicUrlGuard ? await guardedPublicFetch(currentUrl, requestInit, { resolver }) : { ok: true as const, response: await fetch(currentUrl, requestInit), logicalUrl: new URL(currentUrl) }; if (!dial.ok) { return { content: "", contentType: "", finalUrl: dial.logicalUrl, ok: false, error: `Blocked URL fetch: target URL is not public HTTP(S): ${dial.reason}`, }; } const { response } = dial; const logicalUrl = dial.logicalUrl.toString(); if (REDIRECT_STATUSES.has(response.status)) { const location = response.headers.get("location"); if (!location) { return { content: "", contentType: "", finalUrl: logicalUrl, ok: false, status: response.status, error: "Redirect response missing Location header", }; } currentUrl = new URL(location, logicalUrl).toString(); if (shouldRewriteRedirectMethod(response.status, currentMethod)) { currentMethod = "GET"; currentBody = undefined; } continue; } const contentType = response.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() ?? ""; const finalUrl = logicalUrl; const reader = response.body?.getReader(); if (!reader) { return { content: "", contentType, finalUrl, ok: false, status: response.status }; } const chunks: Uint8Array[] = []; let totalSize = 0; while (true) { const { done, value } = await reader.read(); if (done) break; chunks.push(value); totalSize += value.length; if (totalSize > maxBytes) { reader.cancel(); break; } } const content = Buffer.concat(chunks).toString("utf-8"); if (isBotBlocked(response.status, content) && attempt < USER_AGENTS.length - 1) { continue attempts; } if (!response.ok) { return { content, contentType, finalUrl, ok: false, status: response.status }; } return { content, contentType, finalUrl, ok: true, status: response.status }; } return { content: "", contentType: "", finalUrl: currentUrl, ok: false, error: `Too many redirects (${maxRedirects})`, }; } catch { if (signal?.aborted) { throw new ToolAbortError(); } if (attempt === USER_AGENTS.length - 1) { return { content: "", contentType: "", finalUrl: currentUrl, ok: false }; } } } return { content: "", contentType: "", finalUrl: initialUrl, ok: false }; } export { htmlToBasicMarkdown } from "./html-to-markdown"; /** * Build a RenderResult from markdown content. Calls finalizeOutput internally. */ export function buildResult( md: string, opts: { url: string; finalUrl?: string; method: string; fetchedAt: string; notes?: string[]; contentType?: string }, ): RenderResult { const output = finalizeOutput(md); return { url: opts.url, finalUrl: opts.finalUrl ?? opts.url, contentType: opts.contentType ?? "text/markdown", method: opts.method, content: output.content, fetchedAt: opts.fetchedAt, truncated: output.truncated, notes: opts.notes ?? [], }; } /** * Format a date value as YYYY-MM-DD. Returns empty string on invalid input. */ export function formatIsoDate(value?: string | number | Date): string { if (value == null) return ""; if (typeof value === "string") { const datePrefix = value.match(/^\d{4}-\d{2}-\d{2}/); if (datePrefix) return datePrefix[0]; } try { return new Date(value).toISOString().split("T")[0]; } catch { return ""; } } /** * Decode common HTML entities. */ export function decodeHtmlEntities(text: string): string { return text .replace(/</g, "<") .replace(/>/g, ">") .replace(/&/g, "&") .replace(/"/g, '"') .replace(/�?39;/g, "'") .replace(/'/g, "'") .replace(///g, "/") .replace(/ /g, " "); } /** * Format seconds into HH:MM:SS or MM:SS. */ export function formatMediaDuration(totalSeconds: number): string { const hours = Math.floor(totalSeconds / 3600); const minutes = Math.floor((totalSeconds % 3600) / 60); const secs = Math.floor(totalSeconds % 60); if (hours > 0) return `${hours}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")}`; return `${minutes}:${String(secs).padStart(2, "0")}`; } /** * Extract localized text, preferring en-US/en. */ export type LocalizedText = string | Record | null | undefined; export function getLocalizedText(value: LocalizedText, defaultLocale?: string): string | undefined { if (value == null) return undefined; if (typeof value === "string") return value; if (defaultLocale && value[defaultLocale]) return value[defaultLocale]; return ( value["en-US"] ?? value.en_US ?? value.en ?? Object.values(value).find(v => typeof v === "string") ?? undefined ); } /** * Check if content looks like HTML by inspecting the leading tag. */ export function looksLikeHtml(content: string): boolean { const trimmed = content.trim().toLowerCase(); return ( trimmed.startsWith("