import { Markdown } from "@earendil-works/pi-tui"; import { isProgress, type Chunk, type PiToolShell, type ToolContext } from "../../types.ts"; import { toolExpandHint, toolFreshnessLabel, toolSessionNotice } from "../tool-labels.ts"; import { formatChecklistText as toolChecklistText, toolProgressLayout, progressStartedAtMs, } from "../tool-progress.ts"; import { toolResourceStatus, formatBytes, formatDuration } from "../tool-resource.ts"; import { buildToolResultTree, toolResultTree, type ToolResultGroup } from "../tool-result-tree.ts"; import { toolResultFileDetails, stringValue } from "../tool-result.ts"; import { currentSpinnerFrame } from "../tool-spinner.ts"; import { toolStatusDot, toolStatus } from "../tool-status.ts"; import { activity, failure, getMarkdownTheme, muted, separator, success } from "../tui.ts"; import type { RenderComponent, RenderTheme } from "../types.ts"; export function renderWebScrapeResult( result: PiToolShell, expanded = false, theme?: RenderTheme, ): RenderComponent { const envelope = result.details as Partial>>; if (isProgress(envelope)) { const url = envelope.url ?? "unknown URL"; const status = envelope.state === "error" ? "error" : envelope.state === "done" ? "done" : "loading"; const startedAtMs = progressStartedAtMs(envelope) ?? Date.now(); return toolProgressLayout({ renderContent(width) { const row = toolResourceStatus({ url, label: status, state: status, width, theme, startedAtMs, restoreBg: "toolPendingBg", }); const summary = `web_scrape ${envelope.state}${separator(theme)}${muted(toolExpandHint.text, theme)}`; const lines = [row, "", summary]; if (expanded && envelope.checklist?.length) lines.push("", ...envelope.checklist.map(toolChecklistText)); if (status === "loading") lines.push("", `${currentSpinnerFrame()} Working...`); return lines.join("\n"); }, padToWidth: true, }); } const stale = envelope.cache?.staleness; const sourceLabel = envelope.cache?.cached ? activity(`\u21BB cache hit${stale ? ` ${stale}` : ""}`, theme) : success("\u21BB fresh fetch", theme); const summary = envelope.error ? toolStatus([`${envelope.mode ?? ""} mode`, envelope.format ?? ""], theme) : toolStatus( [ `${toolStatusDot(envelope.status, theme)} ${envelope.status ?? ""}`, `${envelope.mode ?? ""} mode`, envelope.format, sourceLabel, { text: formatDuration(envelope.timing?.durationMs) ?? "", tone: "muted" }, toolFreshnessLabel(envelope), expanded ? undefined : toolExpandHint, ], theme, ); const field = envelope.data?.markdown ?? envelope.data?.text ?? envelope.data?.title; const preview = stringValue(envelope.answerContext ?? field ?? result.content.at(0)?.text) ?? ""; const url = envelope.finalUrl ?? envelope.url ?? "unknown URL"; const state = envelope.error ? "error" : "done"; return toolProgressLayout( { body: (width) => toolResourceStatus({ url, label: state, state, width, theme, }), summary, expanded, notice: toolSessionNotice(envelope), expandedSections: (width) => { const ct = envelope.contentType ?? ""; if ( /^(?:application\/octet-stream|application\/pdf|image\/|audio\/|video\/)/u.test(ct) || !!(envelope.data && typeof envelope.data === "object" && "fileSize" in envelope.data) ) return [toolResultFileDetails(envelope, theme).render(width).join("\n")]; const out = [toolResultTree(buildScrapeSections(envelope, theme), width, theme)]; if (preview && !markdownPreviewComponent(envelope.format, preview, theme)) out.push( (envelope.format === "json" || envelope.format === "html" || envelope.format === "ax-tree" ? `\`\`\`${envelope.format}\n${preview}\n\`\`\`` : preview ).slice(0, 1200), ); return out; }, markdownPreview: () => markdownPreviewComponent(envelope.format, preview, theme), responseId: envelope.responseId, hasError: !!envelope.error, }, theme, ); } function markdownPreviewComponent( format: string | undefined, preview: string | undefined, theme?: RenderTheme, ): RenderComponent | undefined { if (format !== "markdown" || !preview || preview.length <= 100) return; return new Markdown(preview.slice(0, 1200), 0, 0, getMarkdownTheme(theme)); } function buildScrapeSections( envelope: Partial>>, theme?: RenderTheme, ): ReturnType { const headers = envelope.headers; const hasHeaders = !!headers && Object.keys(headers).length > 0; const groups = new Map(); const add = addScrapeRow.bind(undefined, groups); const t = envelope.data?.title; const d = envelope.data?.description; add("page", "title", typeof t === "string" && t ? t : undefined); if (hasHeaders) { const url = envelope.finalUrl ?? envelope.url; const host = typeof url === "string" && URL.canParse(url) ? new URL(url).hostname.replace(/^www\./iu, "") : undefined; add("page", "site", host); } add("page", "description", typeof d === "string" && d ? d : undefined); add("details", "url", envelope.url); if (envelope.finalUrl && envelope.finalUrl !== envelope.url) add("details", "final", envelope.finalUrl); add("details", "status", envelope.status ? String(envelope.status) : undefined); add("details", "mode", envelope.mode); add("details", "format", envelope.format); add("details", "size", formatBytes(envelope.downloadedBytes)); add("details", "duration", formatDuration(envelope.timing?.durationMs)); add("details", "type", envelope.contentType); add("details", "source", envelope.cache?.cached ? "cache hit" : "fresh fetch"); const chunks = envelope.data?.chunks as Chunk[] | undefined; if (chunks?.length) { add("chunks", "count", String(chunks.length)); add("chunks", "tokens", `${chunks.reduce((sum, chunk) => sum + chunk.tokenCount, 0)} total`); } if (envelope.error) { const code = envelope.error.code; add("error", "code", code ? (theme ? failure(code, theme) : code) : undefined); add("error", "phase", envelope.error.phase); add("error", "message", envelope.error.message); } if (hasHeaders) addHeaderSections(groups, envelope, headers); return buildToolResultTree(Array.from(groups.entries(), ([name, rows]) => ({ name, rows }))); } function addHeaderSections( groups: Map, envelope: Partial>>, headers: Record, ): void { const add = addScrapeRow.bind(undefined, groups); add("cache", "status", headers["cf-cache-status"]); if (headers["age"]) { const n = Number(headers["age"]); const sec = Number.isFinite(n) && n >= 0 ? n : undefined; add("cache", "age", sec !== undefined ? formatSeconds(sec) : headers["age"]); } const cc = parseCacheControl(headers["cache-control"]); const cdnCc = parseCacheControl(headers["cdn-cache-control"]); const fmtCc = (maxAge: number, swr: number | undefined) => `max-age ${formatSeconds(maxAge)}${swr ? ` +swr ${formatSeconds(swr)}` : ""}`; const primary = cdnCc ?? cc; if (primary) add("cache", "cdn", fmtCc(primary.maxAge, primary.swr)); if (cc?.maxAge !== undefined && (!cdnCc || cdnCc.maxAge !== cc.maxAge)) { const swr = cc.swr && (!cdnCc || cdnCc.swr !== cc.swr) ? cc.swr : undefined; add("cache", "browser", fmtCc(cc.maxAge, swr)); } add("server", "vendor", headers["server"]); if (headers["cf-ray"]) { const di = headers["cf-ray"].lastIndexOf("-"); const ray = di !== -1 ? headers["cf-ray"].slice(0, di) : headers["cf-ray"]; const loc = di !== -1 ? headers["cf-ray"].slice(di + 1) : ""; add("server", "ray", `${ray}${loc ? ` \u2192 ${loc}` : ""}`); } if (headers["date"]) add("time", "fetched", formatHttpTime(headers["date"])); if (headers["last-modified"]) { const now = new Date(headers["date"] ?? Date.now()).getTime(); const diffSec = Math.floor((now - new Date(headers["last-modified"]).getTime()) / 1000); add( "time", "modified", `${formatHttpTime(headers["last-modified"])}${diffSec > 0 ? ` (${formatSeconds(diffSec)} ago)` : ""}`, ); } const headerEntries = Object.entries(headers).filter(([, v]) => typeof v === "string"); for (const [k, v] of headerEntries) add("headers", `${k}:`, v.length > 120 ? `${v.slice(0, 120)}...` : v); const respId = envelope.responseId ?? ""; if (respId) add("trace", "response", respId.slice(0, 8)); if (respId || headerEntries.length > 0) add("trace", "headers", `${headerEntries.length} total`); } function addScrapeRow( groups: Map, group: string, key: string, value: string | undefined, ): void { if (value === undefined || value === "") return; const rows = groups.get(group) ?? []; rows.push([key, value]); groups.set(group, rows); } const fmtTwoUnit = (whole: number, big: string, rem: number, small: string) => rem > 0 ? `${whole}${big} ${rem}${small}` : `${whole}${big}`; function formatSeconds(s: number): string { if (s < 60) return `${s}s`; if (s < 3600) return fmtTwoUnit(Math.floor(s / 60), "m", s % 60, "s"); if (s < 86400) return fmtTwoUnit(Math.floor(s / 3600), "h", Math.floor((s % 3600) / 60), "m"); return fmtTwoUnit(Math.floor(s / 86400), "d", Math.floor((s % 86400) / 3600), "h"); } function parseCacheControl(value: string | undefined) { if (!value) return; let maxAge: number | undefined; let swr: number | undefined; for (const part of value.toLowerCase().split(",")) { const t = part.trim(); const eq = t.indexOf("="); if (eq === -1) continue; const key = t.slice(0, eq); const n = Number(t.slice(eq + 1)); if (!Number.isFinite(n)) continue; if (key === "max-age" || key === "s-maxage") maxAge = n; else if (key === "stale-while-revalidate") swr = n; } return maxAge !== undefined ? { maxAge, swr } : undefined; } function formatHttpTime(s: string): string { const d = new Date(s); return Number.isNaN(d.getTime()) ? s : `${d.toISOString().slice(11, 19)} GMT`; }