/** * `fetch_url` — fetch a URL, returning extracted text for HTML/JSON or saving * binary content (validated by magic bytes) into the uploads workspace. */ import { randomUUID } from "node:crypto"; import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; import { readBodyLimited, ResponseTooLargeError, } from "../../../util/http-body.js"; import { advertisedBinaryKind, decodeText, detectBinaryType, extractText, isHtmlContent, isTextContent, matchesBinaryKind, } from "../../../util/web-content.js"; import { dirs } from "../../../util/paths.js"; import type { SharedActionHandlers } from "./types.js"; const MAX_RESPONSE_MB = 50; const MAX_RESPONSE_BYTES = MAX_RESPONSE_MB * 1024 * 1024; const MAX_TEXT_CHARS = 50_000; /** Cap returned text, marking the cut so truncation is never silent. */ function capText(text: string): string { if (text.length <= MAX_TEXT_CHARS) return text; return `${text.slice(0, MAX_TEXT_CHARS)}\n\n[Content truncated at ${MAX_TEXT_CHARS} characters]`; } export const fetchUrlHandlers: SharedActionHandlers = { fetch_url: async (body) => { const url = String(body.url ?? ""); if (!url) return { ok: false, error: "Missing URL" }; try { const parsed = new URL(url); if (!["http:", "https:"].includes(parsed.protocol)) { return { ok: false, error: "URL must use http or https protocol" }; } } catch { return { ok: false, error: "Invalid URL" }; } try { const resp = await fetch(url, { signal: AbortSignal.timeout(15_000), headers: { "User-Agent": "Talon/1.0" }, redirect: "follow", }); if (!resp.ok) return { ok: false, error: `HTTP ${resp.status}` }; const ct = resp.headers.get("content-type") ?? ""; // Reject oversized responses before downloading the body. // The Content-Length header is advisory but saves bandwidth when present. const contentLength = resp.headers.get("content-length"); if (contentLength && Number(contentLength) > MAX_RESPONSE_BYTES) { return { ok: false, error: `File too large (${(Number(contentLength) / 1024 / 1024).toFixed(0)}MB, max ${MAX_RESPONSE_MB}MB)`, }; } const mimeType = ct.split(";")[0].trim().toLowerCase(); let buffer: Buffer; try { buffer = await readBodyLimited(resp, MAX_RESPONSE_BYTES); } catch (err) { if (err instanceof ResponseTooLargeError) { return { ok: false, error: `Response too large (max ${MAX_RESPONSE_MB}MB)`, }; } throw err; } if (isTextContent(mimeType, buffer)) { const trimmed = decodeText(buffer, ct).trim(); if (!trimmed) return { ok: true, text: "(Page has no readable content)" }; // extractText is a DOM extractor — running it on JSON/XML/JavaScript/ // plain text strips small payloads like {"status":"ok"} to nothing, // so only HTML (declared or sniffed) goes through it. if (!isHtmlContent(mimeType, trimmed)) { return { ok: true, text: capText(trimmed) }; } const text = extractText(trimmed, Number.POSITIVE_INFINITY); if (text.length < 20) return { ok: true, text: "(Page has no readable content)" }; return { ok: true, text: capText(text) }; } if (buffer.length === 0) return { ok: false, error: "Empty response (0 bytes)" }; const detected = await detectBinaryType(buffer); const advertised = advertisedBinaryKind(mimeType); // Do not save an error page or arbitrary bytes under a trusted-looking // image/PDF/ZIP extension merely because the server advertised one. if (advertised && !matchesBinaryKind(advertised, detected, buffer)) { const text = extractText(decodeText(buffer, ct), 500); return { ok: false, error: `Server returned invalid ${advertised} content.${text ? ` Content: ${text}` : ""}`, }; } const uploadsDir = dirs.uploads; if (!existsSync(uploadsDir)) mkdirSync(uploadsDir, { recursive: true }); const filePath = resolve( uploadsDir, `${Date.now()}-${randomUUID().slice(0, 8)}-fetched.${detected?.ext ?? "bin"}`, ); writeFileSync(filePath, buffer); const typeLabel = detected?.mime.startsWith("image/") ? "image" : (detected?.ext ?? ct.split("/")[1]?.split(";")[0] ?? "file"); return { ok: true, text: `Downloaded ${typeLabel} (${(buffer.length / 1024).toFixed(0)}KB) to: ${filePath}\nRead it with the Read tool or send it with send(type="file", file_path="${filePath}").`, }; } catch (err) { return { ok: false, error: `Fetch failed: ${err instanceof Error ? err.message : err}`, }; } }, };