{"version":3,"file":"webfetch.d.ts","sourceRoot":"","sources":["../../../src/core/tools/webfetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAe,IAAI,EAAE,MAAM,SAAS,CAAC;AAG5C,OAAO,KAAK,EAAE,cAAc,EAA2B,MAAM,wBAAwB,CAAC;AAItF,OAAO,EAON,KAAK,qBAAqB,EAC1B,KAAK,cAAc,EACnB,aAAa,EACb,KAAK,iBAAiB,EACtB,MAAM,sBAAsB,CAAC;AAO9B,wFAAwF;AACxF,wBAAgB,cAAc,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAGzD;AAED,QAAA,MAAM,cAAc;;;;;;;EA+BlB,CAAC;AAIH,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,4EAA4E;IAC5E,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sEAAsE;IACtE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,uEAAuE;IACvE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kDAAkD;IAClD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4FAA4F;IAC5F,MAAM,CAAC,EAAE,qBAAqB,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAoB,SAAQ,iBAAiB;IAC7D,oDAAoD;IACpD,KAAK,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IACtC,qFAAqF;IACrF,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AA+FD,wBAAgB,4BAA4B,CAC3C,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE,mBAAmB,GAC3B,cAAc,CAAC,OAAO,cAAc,EAAE,mBAAmB,GAAG,SAAS,CAAC,CAiHxE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,SAAS,CAAC,OAAO,cAAc,CAAC,CAE/G","sourcesContent":["import type { AgentTool } from \"@kolisachint/hoocode-agent-core\";\nimport { Text } from \"@kolisachint/hoocode-tui\";\nimport { type Static, Type } from \"typebox\";\nimport { keyHint } from \"../../modes/interactive/components/keybinding-hints.js\";\nimport { theme as appTheme } from \"../../modes/interactive/theme/theme.js\";\nimport type { ToolDefinition, ToolRenderResultOptions } from \"../extensions/types.js\";\nimport { PEEK_LINES } from \"../tool-output-view.js\";\nimport { getTextOutput, invalidArgText, str } from \"./render-utils.js\";\nimport { wrapToolDefinition } from \"./tool-definition-wrapper.js\";\nimport {\n\tblockedHostForUrl,\n\tfetchStatusNote,\n\tisTruncatedContent,\n\tresolveWebtoolsTimeoutSecs,\n\tresolveWebtoolsTLSConfig,\n\trunWebtools,\n\ttype WebFetchContentStatus,\n\ttype WebFetchResult,\n\tWebToolsCache,\n\ttype WebtoolsTLSConfig,\n} from \"./webtools-shared.js\";\n\nconst DEFAULT_MAX_TOKENS = 4000;\n// Hard ceiling so a single fetch can never flood the context window, regardless\n// of what the model requests. The binary still applies its own soft cap.\nconst MAX_TOKENS_CAP = 25000;\n\n/** Clamp a requested token budget into `(0, MAX_TOKENS_CAP]`, defaulting when unset. */\nexport function clampMaxTokens(requested?: number): number {\n\tif (!requested || requested <= 0) return DEFAULT_MAX_TOKENS;\n\treturn Math.min(requested, MAX_TOKENS_CAP);\n}\n\nconst webfetchSchema = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch (http or https)\" }),\n\tmaxTokens: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription: `Soft cap on returned output size in estimated tokens (default: ${DEFAULT_MAX_TOKENS}, max: ${MAX_TOKENS_CAP})`,\n\t\t}),\n\t),\n\toutput: Type.Optional(\n\t\tType.Union([Type.Literal(\"text\"), Type.Literal(\"markdown\")], {\n\t\t\tdescription:\n\t\t\t\t\"Output format: 'text' (default, most token-efficient, links as [N] with a trailing reference block) or 'markdown' (inline links).\",\n\t\t}),\n\t),\n\toffset: Type.Optional(\n\t\tType.Number({\n\t\t\tdescription:\n\t\t\t\t\"Byte offset into the page's extracted text to read from, for continuing a long page. Use the offset the previous fetch reported; windows tile the document exactly, so nothing is skipped or repeated.\",\n\t\t}),\n\t),\n\toutline: Type.Optional(\n\t\tType.Boolean({\n\t\t\tdescription:\n\t\t\t\t\"Return the page's headings, each with the offset that reads its section and what that section costs, instead of the page body. Map a long page with this first, then fetch the one section you need at its offset.\",\n\t\t}),\n\t),\n\tgrep: Type.Optional(\n\t\tType.String({\n\t\t\tdescription:\n\t\t\t\t\"Return where the page matches this regular expression — offset, surrounding text, and the section it falls in — instead of the page body. Use it to find a mention on a page whose headings do not name it, then fetch the hit at its offset. Case-insensitive unless the pattern carries an uppercase letter.\",\n\t\t}),\n\t),\n});\n\ntype WebFetchToolInput = Static<typeof webfetchSchema>;\n\nexport interface WebFetchToolDetails {\n\tfinalUrl?: string;\n\ttitle?: string;\n\ttokenEstimate?: number;\n\t/** The page continued past the token budget: what came back is a prefix. */\n\ttruncated?: boolean;\n\t/** The budget the cut was made at, so the TUI can say what to raise. */\n\tmaxTokens?: number;\n\t/** Estimated tokens of the whole page, when the binary reports it. */\n\ttotalTokenEstimate?: number;\n\t/** Where to resume reading, when the binary reports paging offsets. */\n\tnextOffset?: number;\n\t/** How many sections an outline listed, for the TUI to show at a glance. */\n\tsectionCount?: number;\n\t/** How many places a search matched, likewise. */\n\tmatchCount?: number;\n\tcontentType?: string;\n\tmedia?: string;\n\t/** Non-\"ok\" means extraction produced nothing usable; see {@link WebFetchContentStatus}. */\n\tstatus?: WebFetchContentStatus;\n}\n\nexport interface WebFetchToolOptions extends WebtoolsTLSConfig {\n\t/** Override the result cache (mainly for tests). */\n\tcache?: WebToolsCache<WebFetchResult>;\n\t/** Effective per-request timeout (seconds); falls back to env/default when unset. */\n\ttimeoutSecs?: number;\n}\n\nfunction formatWebfetchCall(args: { url?: string; output?: string } | undefined): string {\n\tconst url = str(args?.url);\n\tconst urlDisplay = url === null ? invalidArgText(appTheme) : url ? url : appTheme.fg(\"toolOutput\", \"...\");\n\tconst format = args?.output === \"markdown\" ? appTheme.fg(\"muted\", \" (markdown)\") : \"\";\n\treturn appTheme.fg(\"toolTitle\", appTheme.bold(\"webfetch \")) + appTheme.fg(\"accent\", urlDisplay) + format;\n}\n\nfunction formatWebfetchResult(\n\tresult: { content: Array<{ type: string; text?: string }>; details?: WebFetchToolDetails },\n\toptions: ToolRenderResultOptions,\n\tshowImages: boolean,\n): string {\n\tconst output = getTextOutput(result as any, showImages).trim();\n\tlet text = \"\";\n\tif (output) {\n\t\tconst lines = output.split(\"\\n\");\n\t\tconst maxLines = options.expanded ? lines.length : PEEK_LINES;\n\t\tconst displayLines = lines.slice(0, maxLines);\n\t\tconst remaining = lines.length - maxLines;\n\t\ttext += `\\n${displayLines.map((line) => appTheme.fg(\"toolOutput\", line)).join(\"\\n\")}`;\n\t\tif (remaining > 0) {\n\t\t\ttext += `${appTheme.fg(\"muted\", `\\n... (${remaining} more lines,`)} ${keyHint(\"app.tools.expand\", \"to expand\")})`;\n\t\t}\n\t}\n\tconst tokenEstimate = result.details?.tokenEstimate;\n\tif (tokenEstimate !== undefined) {\n\t\t// A cut page and a complete one cost the same at the budget, so the number\n\t\t// alone reads as \"this is the page\". Mark the ones that are a prefix.\n\t\tconst cut = result.details?.truncated\n\t\t\t? appTheme.fg(\"warning\", ` (truncated at ${result.details.maxTokens ?? tokenEstimate})`)\n\t\t\t: \"\";\n\t\t// An outline's cost is its own, not the page's, so say which was read.\n\t\tconst sections = result.details?.sectionCount;\n\t\tconst hits = result.details?.matchCount;\n\t\tlet view = \"\";\n\t\tif (sections !== undefined) view = appTheme.fg(\"muted\", ` · outline, ${sections} sections`);\n\t\telse if (hits !== undefined) view = appTheme.fg(\"muted\", ` · ${hits} match${hits === 1 ? \"\" : \"es\"}`);\n\t\ttext += `\\n${appTheme.fg(\"muted\", `~${tokenEstimate} tokens`)}${cut}${view}`;\n\t}\n\treturn text;\n}\n\n/**\n * Turn an older binary's argument-parsing error into advice.\n *\n * `--offset` and `--outline` postdate binaries already in the wild, which\n * reject an unknown flag with a parser message naming it. That message says\n * nothing about what to do, and the fix is never the call site — it is the\n * binary — so name it here. Returns undefined for anything else, leaving the\n * original error to speak for itself.\n */\nfunction unsupportedFlagError(\n\terror: unknown,\n\twantsOutline: boolean,\n\twantsGrep: boolean,\n\toffset: number,\n): Error | undefined {\n\tconst message = error instanceof Error ? error.message : String(error);\n\tif (!/unexpected argument|unrecognized|unknown (option|argument)/i.test(message)) return undefined;\n\n\tconst flag =\n\t\twantsOutline && /outline/.test(message)\n\t\t\t? \"--outline\"\n\t\t\t: wantsGrep && /grep/.test(message)\n\t\t\t\t? \"--grep\"\n\t\t\t\t: offset > 0\n\t\t\t\t\t? \"--offset\"\n\t\t\t\t\t: undefined;\n\tif (!flag) return undefined;\n\treturn new Error(\n\t\t`the installed webtools binary does not support ${flag}; update it (or delete it from the hoocode bin directory to re-download). Original error: ${message}`,\n\t);\n}\n\n/**\n * What to tell the model when a page did not fit.\n *\n * With paging offsets it is a position to resume at, which is the whole point:\n * the rest of the document is one call away and costs another window, not\n * another copy of the page. Without them (an older binary) the only truthful\n * advice is a larger budget.\n */\nfunction continuationNote(result: WebFetchResult, offset: number, maxTokens: number): string {\n\tconst next = result.next_offset;\n\tif (next === undefined) {\n\t\treturn `output stopped at the ${maxTokens}-token budget; the page continues past this point. Re-fetch with a larger maxTokens (up to ${MAX_TOKENS_CAP}) for more, or fetch a more specific URL or #anchor.`;\n\t}\n\tconst total = result.total_token_estimate;\n\tconst progress =\n\t\ttotal !== undefined ? `~${result.token_estimate} of ~${total} tokens` : `${result.token_estimate} tokens`;\n\treturn `showing bytes ${result.offset ?? offset}-${next} of ${result.total_bytes ?? \"?\"} (${progress}); continue with offset=${next}`;\n}\n\nexport function createWebFetchToolDefinition(\n\tcwd: string,\n\toptions?: WebFetchToolOptions,\n): ToolDefinition<typeof webfetchSchema, WebFetchToolDetails | undefined> {\n\tconst cache = options?.cache ?? new WebToolsCache<WebFetchResult>();\n\t// Resolve CA/insecure plumbing and the request timeout once (settings\n\t// overrides, else env) and thread them into every spawn; not hardcoded.\n\tconst tlsConfig = resolveWebtoolsTLSConfig(options);\n\tconst timeoutSecs = resolveWebtoolsTimeoutSecs(options?.timeoutSecs);\n\treturn {\n\t\tname: \"webfetch\",\n\t\tlabel: \"webfetch\",\n\t\tdescription:\n\t\t\t\"Fetch a web page (or JSON/text resource) and return token-efficient, reference-style content. HTML is extracted to clean text; links become inline [N] markers with full URLs in a trailing reference block. Returns title, final URL (after redirects), and an estimated token count. Off by default; enabled with --enable-webtools.\",\n\t\tpromptSnippet: \"Fetch a URL and return clean, token-efficient page content\",\n\t\tpromptGuidelines: [\n\t\t\t\"Use webfetch to read a known URL instead of bash curl/wget; it returns clean extracted text with reference-style [N] links, not raw HTML.\",\n\t\t\t\"A fetch that reports it stopped at its token budget returned a prefix, not the page. When it names a continue offset, pass that as `offset` to read the next window; windows tile exactly, so nothing is skipped or repeated. Only keep going while the answer is genuinely further down — a more specific URL or #anchor is usually cheaper than paging a whole document.\",\n\t\t\t\"For a long page whose relevant part is unknown, fetch it once with `outline: true`: that costs a few dozen tokens and returns the headings with the offset and cost of each section. Then fetch the one section at its offset instead of paging the whole document.\",\n\t\t\t\"When the page has no headings, or none of them names what you are after, use `grep` instead: it returns each match with its offset and the text around it. Fetch the hit that looks right at its offset. Outline and grep are alternatives, not a pair — ask for one.\",\n\t\t],\n\t\tparameters: webfetchSchema,\n\t\tasync execute(\n\t\t\t_toolCallId,\n\t\t\t{ url, maxTokens, output, offset, outline, grep }: WebFetchToolInput,\n\t\t\tsignal?: AbortSignal,\n\t\t) {\n\t\t\tif (signal?.aborted) throw new Error(\"Operation aborted\");\n\n\t\t\t// Policy gate (.webtoolsignore). SSRF/private-address blocking lives in\n\t\t\t// the binary; this is host-level allow/deny policy only.\n\t\t\tconst blockedHost = blockedHostForUrl(cwd, url);\n\t\t\tif (blockedHost) {\n\t\t\t\tthrow new Error(`Blocked by .webtoolsignore policy: ${blockedHost}`);\n\t\t\t}\n\n\t\t\tconst effectiveMaxTokens = clampMaxTokens(maxTokens);\n\t\t\tconst format = output ?? \"text\";\n\t\t\tconst effectiveOffset = Number.isFinite(offset) && offset !== undefined ? Math.max(0, Math.floor(offset)) : 0;\n\t\t\tconst wantsOutline = outline === true;\n\t\t\tconst pattern = grep?.trim() ? grep : undefined;\n\t\t\t// Two views of one page, and the binary refuses both. Say so here rather\n\t\t\t// than spending a subprocess to be told.\n\t\t\tif (wantsOutline && pattern !== undefined) {\n\t\t\t\tthrow new Error(\n\t\t\t\t\t\"webfetch takes outline or grep, not both: an outline maps the page by heading, grep finds where it mentions something. Ask for one, then fetch what it points at.\",\n\t\t\t\t);\n\t\t\t}\n\t\t\tconst cacheKey = `${format}:${effectiveMaxTokens}:${effectiveOffset}:${wantsOutline}:${pattern ?? \"\"}:${url}`;\n\n\t\t\tconst args = [\"--url\", url, \"--max-tokens\", String(effectiveMaxTokens), \"--output\", format];\n\t\t\t// Both flags are sent only when asked for: an older binary rejects an\n\t\t\t// unknown argument, and neither is needed to read a page from the start.\n\t\t\tif (effectiveOffset > 0) args.push(\"--offset\", String(effectiveOffset));\n\t\t\tif (wantsOutline) args.push(\"--outline\");\n\t\t\tif (pattern !== undefined) args.push(\"--grep\", pattern);\n\t\t\tconst result = await cache\n\t\t\t\t.getOrCompute(cacheKey, signal, (sig) =>\n\t\t\t\t\trunWebtools<WebFetchResult>(\"fetch\", args, cwd, sig, timeoutSecs, tlsConfig),\n\t\t\t\t)\n\t\t\t\t.catch((error: unknown) => {\n\t\t\t\t\t// A binary predating these flags rejects them as unknown arguments,\n\t\t\t\t\t// which surfaces as an argument-parsing error naming the flag. Say\n\t\t\t\t\t// what to do about it rather than passing the raw parser message on.\n\t\t\t\t\tthrow unsupportedFlagError(error, wantsOutline, pattern !== undefined, effectiveOffset) ?? error;\n\t\t\t\t});\n\n\t\t\tconst header = result.title ? `${result.title}\\n${result.final_url}\\n\\n` : `${result.final_url}\\n\\n`;\n\t\t\t// An empty body and a JavaScript-rendered shell look identical in the\n\t\t\t// content alone. Say which it was, so the page is not read as \"nothing\n\t\t\t// to say\" when it simply needs a browser.\n\t\t\tconst note = fetchStatusNote(result.status);\n\t\t\tlet body = note ? `${result.content}\\n\\n[webtools: ${note}]`.trimStart() : result.content;\n\n\t\t\t// A cut page used to end in a bare elision marker: the model could see\n\t\t\t// that something was missing but had no way to act on it, so a long\n\t\t\t// document was a dead end rather than a first page. Say where the\n\t\t\t// window sits and how to continue past it.\n\t\t\t//\n\t\t\t// The binary's own flag is authoritative; the marker is the fallback\n\t\t\t// for binaries older than the paging fields, where the only honest\n\t\t\t// advice is a larger budget because there is no offset to resume at.\n\t\t\tconst truncated = result.truncated ?? isTruncatedContent(result.content);\n\t\t\tif (truncated) {\n\t\t\t\tbody += `\\n\\n[webtools: ${continuationNote(result, effectiveOffset, effectiveMaxTokens)}]`;\n\t\t\t}\n\n\t\t\treturn {\n\t\t\t\tcontent: [{ type: \"text\" as const, text: header + body }],\n\t\t\t\tdetails: {\n\t\t\t\t\tfinalUrl: result.final_url,\n\t\t\t\t\ttitle: result.title,\n\t\t\t\t\ttokenEstimate: result.token_estimate,\n\t\t\t\t\ttruncated,\n\t\t\t\t\tmaxTokens: effectiveMaxTokens,\n\t\t\t\t\tsectionCount: result.outline?.length,\n\t\t\t\t\tmatchCount: result.matches?.length,\n\t\t\t\t\ttotalTokenEstimate: result.total_token_estimate,\n\t\t\t\t\tnextOffset: result.next_offset,\n\t\t\t\t\tcontentType: result.content_type,\n\t\t\t\t\tmedia: result.media,\n\t\t\t\t\tstatus: result.status,\n\t\t\t\t},\n\t\t\t};\n\t\t},\n\t\trenderCall(args, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchCall(args));\n\t\t\treturn text;\n\t\t},\n\t\trenderResult(result, options, _theme, context) {\n\t\t\tconst text = (context.lastComponent as Text | undefined) ?? new Text(\"\", 0, 0);\n\t\t\ttext.setText(formatWebfetchResult(result as any, options, context.showImages));\n\t\t\treturn text;\n\t\t},\n\t};\n}\n\nexport function createWebFetchTool(cwd: string, options?: WebFetchToolOptions): AgentTool<typeof webfetchSchema> {\n\treturn wrapToolDefinition(createWebFetchToolDefinition(cwd, options));\n}\n"]}