import { StringEnum } from "@earendil-works/pi-ai"; import { keyText, type ExtensionAPI, type Theme } from "@earendil-works/pi-coding-agent"; import { Text } from "@earendil-works/pi-tui"; import { type Static, Type } from "typebox"; import { ArtifactStore, type FetchArtifactFormat } from "./artifacts.ts"; import { fetchWithImpit } from "./impit-fetch.ts"; import { DEFAULT_PAGE_LINES, MAX_PAGE_LINES, formatTextPage, paginateText } from "./paging.ts"; export const WebFetchParameters = Type.Object( { url: Type.String({ description: "HTTP(S) URL to fetch." }), format: Type.Optional( StringEnum(["source", "text", "markdown", "html"] as const, { description: "View returned to the agent. If omitted, HTML uses Markdown; other sources preserve their native form or return a local binary path.", }), ), offset: Type.Optional(Type.Integer({ minimum: 1, description: "First line to return (1-based). Default: 1." })), limit: Type.Optional( Type.Integer({ minimum: 1, maximum: MAX_PAGE_LINES, description: `Maximum lines to return. Default: ${DEFAULT_PAGE_LINES}; maximum: ${MAX_PAGE_LINES}.`, }), ), }, { additionalProperties: false }, ); export type WebFetchInput = Static; export interface WebFetchDetails { phase: "fetching" | "completed"; url: string; finalUrl?: string; format: FetchArtifactFormat; title?: string; status?: number; fetchId?: string; contentKind?: string; sourceFormat?: string; parseWarning?: string; startLine?: number; endLine?: number; totalLines?: number; nextOffset?: number; rawBytes?: number; markdownBytes?: number; rawPath?: string; sourcePath?: string; textPath?: string; markdownPath?: string; } function compactUrl(value: string, max = 90): string { return value.length <= max ? value : `${value.slice(0, max - 3)}...`; } export function resolveWebFetchFormat( requested: FetchArtifactFormat | undefined, contentKind: string, ): FetchArtifactFormat { return requested ?? (contentKind === "html" ? "markdown" : "source"); } function renderFetchCall(args: WebFetchInput, theme: Theme): Text { const format = args.format ?? "auto"; const offset = args.offset ?? 1; const limit = args.limit ?? DEFAULT_PAGE_LINES; let text = `${theme.fg("toolTitle", theme.bold("web_fetch "))}${theme.fg("accent", compactUrl(args.url))}`; text += theme.fg("muted", ` [${format} ${offset}-${offset + limit - 1}]`); return new Text(text, 0, 0); } export function registerWebFetchTool(pi: ExtensionAPI): void { pi.registerTool({ name: "web_fetch", label: "Web Fetch", description: "Fetch any HTTP(S) resource with Impit. HTML defaults to readable Markdown; other text preserves its source format, while images and binary files are saved for model access by path.", parameters: WebFetchParameters, executionMode: "parallel", async execute(_toolCallId, params, signal, onUpdate, ctx) { const requestedFormat = params.format; onUpdate?.({ content: [{ type: "text", text: `Fetching ${params.url} with Impit...` }], details: { phase: "fetching", url: params.url, format: requestedFormat ?? "source" }, }); const snapshot = await fetchWithImpit(params.url, { signal }); const format = resolveWebFetchFormat(requestedFormat, snapshot.kind); const store = new ArtifactStore(ctx.sessionManager.getSessionId()); await store.cleanup(); const saved = await store.saveFetch({ requestedUrl: snapshot.requestedUrl, finalUrl: snapshot.finalUrl, status: snapshot.status, title: snapshot.title, contentType: snapshot.contentType, contentKind: snapshot.kind, sourceFormat: snapshot.sourceFormat, extension: snapshot.extension, charset: snapshot.charset, headers: snapshot.headers, redirects: snapshot.redirects, rawBody: snapshot.rawBody, sourceText: snapshot.sourceText, text: snapshot.text, markdown: snapshot.markdown, parseWarning: snapshot.parseWarning, }); const selected = await store.readFetch(saved.id, format); const page = paginateText(selected.text, params); const details: WebFetchDetails = { phase: "completed", url: snapshot.requestedUrl, finalUrl: snapshot.finalUrl, format, title: snapshot.title, status: snapshot.status, fetchId: saved.id, startLine: page.startLine, endLine: page.endLine, totalLines: page.totalLines, nextOffset: page.nextOffset, contentKind: snapshot.kind, sourceFormat: snapshot.sourceFormat, parseWarning: snapshot.parseWarning, rawBytes: saved.metadata.rawBytes, markdownBytes: saved.metadata.markdownBytes, rawPath: saved.paths.raw, sourcePath: saved.paths.source, textPath: saved.paths.text, markdownPath: saved.paths.markdown, }; return { content: [ { type: "text", text: formatTextPage(page, { artifactId: saved.id, view: format, source: snapshot.finalUrl, continuationTool: "web_fetch_read", continuationArgs: { fetch_id: saved.id, format }, }), }, ], details, }; }, renderCall(args, theme) { return renderFetchCall(args, theme); }, renderResult(result, { expanded, isPartial }, theme) { const details = result.details; if (isPartial || details?.phase === "fetching") { return new Text(theme.fg("warning", `Fetching ${compactUrl(details?.url ?? "web page")}`), 0, 0); } if (!details?.fetchId) { const error = result.content.find((item) => item.type === "text")?.text ?? "web_fetch failed"; return new Text(theme.fg("error", error), 0, 0); } let text = theme.fg("success", details.title || details.finalUrl || details.url); const hint = `${theme.fg("dim", keyText("app.tools.expand"))}${theme.fg("muted", expanded ? " to collapse" : " to expand")}`; text += `${theme.fg("muted", " (")}${hint}${theme.fg("muted", ")")}`; if (expanded) { text += `\n${theme.fg("muted", `Original: ${details.rawPath}`)}`; if (details.sourcePath) text += `\n${theme.fg("muted", `Source: ${details.sourcePath}`)}`; if (details.textPath) text += `\n${theme.fg("muted", `Text: ${details.textPath}`)}`; if (details.markdownPath) text += `\n${theme.fg("muted", `Markdown: ${details.markdownPath}`)}`; if (details.parseWarning) text += `\n${theme.fg("warning", details.parseWarning)}`; const output = result.content.find((item) => item.type === "text")?.text; if (output) text += `\n\n${theme.fg("toolOutput", output)}`; } return new Text(text, 0, 0); }, }); }