// web_fetch tool — fetch and extract content from a URL // // Routes to the best available provider for content extraction, with SSRF // protection, large-response spillover, and optional raw HTML mode. import { mkdtemp, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { StringEnum } from "@earendil-works/pi-ai"; import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult, truncateHead, } from "@earendil-works/pi-coding-agent"; import { Text } from "@earendil-works/pi-tui"; import { Type } from "typebox"; import { loadConfig, resolveApiKey } from "./config.js"; import { buildFetchChain, createAvailableProviders, fetchPromptGuidelines, fetchProviderNames, PROVIDERS, } from "./providers/index.js"; import type { FetchResponse, Provider } from "./providers/types.js"; import { assertSafeDns, cleanHtmlToMarkdown, fetchWithRetry, SafeMemoryCache } from "./utils.js"; // 全局抓取缓存实例，TTL 为 5 分钟，最大 100 条 export const fetchCache = new SafeMemoryCache<{ result: FetchResponse; provider: string }>(300000, 100); const FETCH_TEMP_PREFIX = "pisearch-fetch-"; const FETCH_TEMP_FILE = "content.txt"; export interface FetchExecuteOpts { provider?: string; } export async function executeFetch( providers: Map, url: string, opts: FetchExecuteOpts, signal?: AbortSignal, ): Promise<{ result: FetchResponse; provider: string }> { // SSRF guard await assertSafeDns(url); const errors: string[] = []; // Build try chain let chain: string[]; if (opts.provider) { if (!providers.has(opts.provider)) { throw new Error( `Requested provider "${opts.provider}" is not available. Configure its API key or omit provider.`, ); } // LLM explicit provider — try only it (option A: no fallback) chain = [opts.provider]; } else { // No provider specified — cost-priority fallback chain chain = buildFetchChain().filter((name) => providers.has(name)); } for (const name of chain) { const p = providers.get(name); if (!p?.capabilities.contentExtraction || !p.fetch) continue; try { const result = await p.fetch(url, signal); return { result, provider: name }; } catch (err) { errors.push(`${name}: ${err instanceof Error ? err.message : String(err)}`); } // If LLM specified a provider, stop here (don't fall back) if (opts.provider) break; } throw new Error(`All providers failed to fetch ${url}:\n${errors.join("\n")}`); } interface FetchDetails { url: string; provider: string; title?: string; contentType?: string; raw?: boolean; truncation?: TruncationResult; fullOutputPath?: string; } export function normalizeDirectResponse(rawText: string, contentType: string, raw: boolean): FetchResponse { if (raw) return { text: rawText, contentType: contentType || undefined }; const trimmed = rawText.trim().toLowerCase(); if ( contentType.toLowerCase().includes("text/html") || contentType.toLowerCase().includes("application/xhtml+xml") || trimmed.startsWith(" { const dir = await mkdtemp(join(tmpdir(), FETCH_TEMP_PREFIX)); const file = join(dir, FETCH_TEMP_FILE); await writeFile(file, content, "utf8"); return file; } function formatTruncationFooter(t: TruncationResult, tempFile: string): string { const skippedLines = t.totalLines - t.outputLines; const skippedBytes = t.totalBytes - t.outputBytes; return ( `\n\n[Truncated: ${t.outputLines}/${t.totalLines} lines` + ` (${formatSize(t.outputBytes)}/${formatSize(t.totalBytes)}).` + ` ${skippedLines} lines (${formatSize(skippedBytes)}) omitted.` + ` Full content: ${tempFile}]` ); } export function registerWebFetchTool(pi: ExtensionAPI): void { const providerNames = fetchProviderNames(); pi.registerTool({ name: "web_fetch", label: "Web Fetch", description: "Fetch and extract the full content of a URL. Returns clean text/markdown. Use after web_search when you need to read a specific page in detail. Use the provider guidance in promptGuidelines or omit provider to use the cost-priority fallback chain. Set raw=true for direct raw HTML (do not combine raw with provider).", promptSnippet: "Fetch and read the full content of a URL — use after web_search to drill into specific pages.", get promptGuidelines() { return fetchPromptGuidelines(); }, parameters: Type.Object({ url: Type.String({ description: "The URL to fetch. Must start with http:// or https://." }), provider: Type.Optional( StringEnum(providerNames, { description: `Directly specify the extraction provider. Omit to auto-select via cost-priority fallback chain (${buildFetchChain().join(" → ")}).`, }), ), raw: Type.Optional( Type.Boolean({ description: "Return raw HTML via direct HTTP. Cannot be combined with provider. Default: false.", default: false, }), ), }), async execute(_toolCallId, params, signal, onUpdate) { const url = params.url as string; const raw = (params.raw ?? false) as boolean; const requestedProvider = params.provider as string | undefined; if (raw && requestedProvider) { throw new Error("'raw=true' uses direct HTTP and cannot be combined with 'provider'."); } const cacheKey = `${url}_${raw}_${requestedProvider ?? "auto"}`; const cached = fetchCache.get(cacheKey); let result: FetchResponse; let provider: string; if (cached) { onUpdate?.({ content: [{ type: "text", text: `Fetching ${url}... (Cache Hit!)` }], details: { url }, }); result = cached.result; provider = cached.provider; } else { // Build providers from config const config = loadConfig(); const apiKeys: Record = {}; for (const meta of PROVIDERS) { apiKeys[meta.name] = resolveApiKey(meta.name, meta.envVar, config); } const providers = createAvailableProviders(apiKeys); // Raw mode: skip provider chain, go direct HTTP if (raw) { onUpdate?.({ content: [{ type: "text", text: `Fetching ${url} (raw)...` }], details: { url }, }); try { const { ip: safeIp } = await assertSafeDns(url); const res = await fetchWithRetry(url, { signal }, 1, safeIp); if (!res.ok) throw new Error(`HTTP ${res.status}`); const rawText = await res.text(); const originalContentType = res.headers.get("content-type") ?? ""; result = normalizeDirectResponse(rawText, originalContentType, raw); provider = "direct"; } catch (err) { throw new Error(`Raw fetch failed for ${url}: ${err instanceof Error ? err.message : String(err)}`); } } else { onUpdate?.({ content: [{ type: "text", text: `Fetching ${url}...` }], details: { url }, }); const fetched = await executeFetch(providers, url, { provider: requestedProvider }, signal); result = fetched.result; provider = fetched.provider; } // 抓取成功，塞入内存缓存 fetchCache.set(cacheKey, { result, provider }); } const truncation = truncateHead(result.text, { maxLines: DEFAULT_MAX_LINES, maxBytes: DEFAULT_MAX_BYTES, }); const details: FetchDetails = { url, provider, raw, title: result.title, contentType: result.contentType }; let output = truncation.content; if (truncation.truncated) { const tempFile = await spillToTemp(result.text); details.truncation = truncation; details.fullOutputPath = tempFile; output += formatTruncationFooter(truncation, tempFile); } const header = [`**URL:** ${url}`, `**Provider:** ${provider}`]; if (result.title) header.push(`**Title:** ${result.title}`); if (raw) header.push("**Mode:** raw"); header.push(""); return { content: [{ type: "text", text: header.join("\n") + output }], details, }; }, renderCall(args, theme) { const u = args.url as string; const raw = args.raw ? " [raw]" : ""; return new Text(theme.fg("toolTitle", theme.bold("Fetch ")) + theme.fg("accent", u + raw), 0, 0); }, renderResult(result, { isPartial }, theme, context) { if (isPartial) return new Text(theme.fg("warning", "Fetching..."), 0, 0); const d = result.details as FetchDetails | undefined; // 处理错误状态（Codex 风格 200 字符限制） if (context?.isError) { const content = result.content[0]; const errMsg = content?.type === "text" ? content.text : "Fetch failed"; const cleanErrMsg = errMsg.split("\n")[0] || "Fetch failed"; return new Text(theme.fg("error", `✗ fetch failed: ${cleanErrMsg.slice(0, 200)}`), 0, 0); } // 正常运行状态的单行状态摘要（Title 放宽至 120 字符） let summary = theme.fg("success", `✓ ${d?.provider ?? "fetched"}`); if (d?.title) summary += theme.fg("muted", `: ${d.title.slice(0, 120)}`); if (d?.truncation?.truncated) summary += theme.fg("warning", " (truncated)"); if (d?.raw) summary += theme.fg("muted", " raw"); return new Text(summary, 0, 0); }, }); }