import type { AgentToolResult, ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { Text } from "@earendil-works/pi-tui"; import { StringEnum } from "@earendil-works/pi-ai/compat"; import { Type } from "typebox"; import type { ExtractedContent, ExtractOptions } from "../fetch/types.ts"; import { normalizeFetchContentParams } from "./fetch-params.ts"; import { getMaxInlineContentChars } from "../config/store.ts"; import { runWithProxy } from "../net/proxy.ts"; import { writeTempContent } from "./temp-output.ts"; const MAX_FETCH_URLS = 10; let fetchModule: Promise | undefined; async function fetchAllContent(urls: string[], signal?: AbortSignal, options?: ExtractOptions): Promise { const module = await (fetchModule ??= import("../fetch/router.ts")); return module.fetchAllContent(urls, signal, options); } function initialSlice(content: string, limit: number): string { if (content.length <= limit) return content; let end = limit; const lineBreak = content.lastIndexOf("\n", limit); if (lineBreak >= Math.floor(limit * 0.8)) end = lineBreak + 1; return content.slice(0, end); } type PresentedFetch = { result: ExtractedContent; text: string; file?: { path: string; chars: number; bytes: number }; }; async function presentResult(result: ExtractedContent, limit: number, raw: boolean): Promise { if (result.error || result.content.length <= limit) return { result, text: result.content }; const file = await writeTempContent(result.content, raw ? ".txt" : ".md"); const preview = initialSlice(result.content, limit); return { result, file, text: `${preview}\n\n---\nShowing ${preview.length} of ${result.content.length} characters. Full content saved to: ${file.path}`, }; } function detailsFor(items: PresentedFetch[]) { const files = items.flatMap(({ result, file }) => file ? [{ url: result.url, ...file }] : []); return { urlCount: items.length, successful: items.filter(item => !item.result.error).length, totalChars: items.reduce((total, item) => total + item.result.content.length, 0), ...(items.length === 1 ? { title: items[0].result.title, mimeType: items[0].result.mimeType, status: items[0].result.status, truncated: !!items[0].file, ...(items[0].file ? { filePath: items[0].file.path, fileChars: items[0].file.chars, fileBytes: items[0].file.bytes, } : {}), } : files.length > 0 ? { tempFiles: files } : {}), }; } export function registerWebFetch(pi: ExtensionAPI): void { pi.registerTool({ name: "web_fetch", label: "Web Fetch", description: "Fetch HTTP(S) pages as readable Markdown or raw text. Oversized content is saved under /tmp.", promptSnippet: "Use to retrieve web page content. Read filePath when the result is truncated.", parameters: Type.Object({ url: Type.Optional(Type.String({ description: "Single URL" })), urls: Type.Optional(Type.Array(Type.String(), { maxItems: MAX_FETCH_URLS, description: "Multiple URLs" })), mode: Type.Optional(StringEnum(["readable", "raw"])), proxy: Type.Optional(Type.String({ description: "Optional HTTP(S) proxy URL" })), }), async execute(_callId, params, signal, onUpdate): Promise>> { let normalized: ReturnType; try { normalized = normalizeFetchContentParams(params); } catch (error) { const message = error instanceof Error ? error.message : String(error); return { content: [{ type: "text", text: `Error: ${message}` }], details: { error: message } }; } if (normalized.urlList.length === 0) return { content: [{ type: "text", text: "Error: provide url or urls." }], details: { error: "No URL provided" } }; if (normalized.urlList.length > MAX_FETCH_URLS) return { content: [{ type: "text", text: `Error: at most ${MAX_FETCH_URLS} URLs are allowed.` }], details: { error: "Too many URLs" } }; return runWithProxy(normalized.options.proxy, async () => { onUpdate?.({ content: [{ type: "text", text: `Fetching ${normalized.urlList.length} URL(s)...` }], details: { phase: "fetch", progress: 0 } }); const fetched = await fetchAllContent(normalized.urlList, signal, { mode: normalized.options.mode, proxy: normalized.options.proxy, toolNames: { webSearch: "web_search", fetchContent: "web_fetch" }, }); const presented = await Promise.all(fetched.map(result => presentResult(result, getMaxInlineContentChars(), normalized.options.mode === "raw"))); const details = detailsFor(presented); if (presented.length === 1) { const item = presented[0]; return item.result.error ? { content: [{ type: "text", text: `Error: ${item.result.error}` }], details: { ...details, error: item.result.error, urls: normalized.urlList } } : { content: [{ type: "text", text: item.text }], details: { ...details, urls: normalized.urlList } }; } const sections = presented.map(({ result, text }) => result.error ? `## ${result.url}\n\nError: ${result.error}` : `## ${result.title || result.url}\n\nSource: ${result.url}\n\n${text}`); return { content: [{ type: "text", text: sections.join("\n\n---\n\n") }], details: { ...details, urls: normalized.urlList } }; }); }, renderCall(args, theme) { const { urlList } = normalizeFetchContentParams(args); return new Text(theme.fg("toolTitle", theme.bold("fetch ")) + theme.fg("accent", urlList.length === 1 ? urlList[0] : `${urlList.length} URLs`), 0, 0); }, renderResult(result, { isPartial, expanded }, theme) { if (isPartial) return new Text(theme.fg("accent", "fetching..."), 0, 0); const details = result.details as { successful?: number; urlCount?: number; truncated?: boolean; error?: string }; const status = details.error ? theme.fg("error", "fetch failed") : theme.fg("success", `${details.successful ?? 0}/${details.urlCount ?? 0} URLs`) + (details.truncated ? theme.fg("warning", " [saved to /tmp]") : ""); const text = result.content.find(part => part.type === "text")?.text ?? ""; if (!text) return new Text(status, 0, 0); const limit = expanded ? text.length : 600; const preview = text.length > limit ? `${text.slice(0, limit)}...` : text; const hint = !expanded && text.length > limit ? theme.fg("muted", "\n\n... (ctrl+o to expand)") : ""; return new Text(`${status}\n${theme.fg(details.error ? "error" : "toolOutput", preview)}${hint}`, 0, 0); }, }); }