/** * Shared plumbing for the `webfetch` and `websearch` tools. * * Both tools shell out to the `webtools` binary (fetch / search subcommands, * resolved/downloaded via {@link ensureTool}) and parse its `--json` output. * This module owns: * - the spawn-and-parse runner, * - the locked JSON result types, * - a short-lived in-process result cache, * - the `.webtoolsignore` policy matcher (gitignore semantics) used to block * hosts both before a fetch and when filtering search result links, and * - the read-only check for whether `websearch` has a keyed backend configured. */ import ignore from "ignore"; type IgnoreMatcher = ReturnType; /** * Whether the binary actually extracted content (`FetchResult.status`). * * Optional here because an older `webtools` on PATH predates the field; absent * is treated as `ok`. Without this, a JavaScript-rendered shell and a genuinely * blank page are both "empty content, exit 0" and the model reads either as * "this page has nothing to say". */ export type WebFetchContentStatus = "ok" | "empty" | "needs_js" | "too_complex"; /** Whether the search answered (`SearchOutput.status`). See {@link WebFetchContentStatus} on optionality. */ export type WebSearchStatus = "ok" | "empty" | "blocked"; /** * The elision marker the binary appends when `--max-tokens` cuts a body * (`compress::TRUNCATION_MARKER`). Both output formats hoocode asks for — text * and markdown — route their budgeting through `truncate_to_tokens`, so its * presence is what tells us a page continued past what we were handed. The * binary reports no structured flag today; when it grows one, prefer that and * keep this as the fallback for older binaries. */ export declare const WEBTOOLS_TRUNCATION_MARKER = "\u2026[truncated]"; /** * Whether a fetch came back cut off. Substring rather than suffix: the marker * lands at the end of the *body*, and the reference block is assembled after * it. */ export declare function isTruncatedContent(content: string | undefined): boolean; /** One-line explanation for a non-`ok` fetch status, mirroring the binary's own note. */ export declare function fetchStatusNote(status: WebFetchContentStatus | undefined): string | undefined; interface WebFetchReference { index: number; url: string; text?: string; } interface WebFetchMetadata { description?: string; author?: string; published?: string; lang?: string; site_name?: string; } /** One place a page matched a search pattern. */ export interface WebFetchMatch { /** Byte offset in the extracted text — the same space `offset` addresses. */ offset: number; snippet: string; /** The heading whose section contains the hit, when the page has one. */ section?: string; /** Further occurrences close enough that this snippet already covers them. */ nearby?: number; } /** One heading in a page's outline, and the span of text it opens. */ export interface WebFetchOutlineSection { level: number; title: string; /** Byte offset in the extracted text — the same space `offset` addresses. */ offset: number; bytes: number; token_estimate: number; } export interface WebFetchResult { title?: string; final_url: string; /** * Where this window sits in the extracted document. Absent on binaries * older than the paging fields, which is why every consumer falls back to * the elision marker (see {@link isTruncatedContent}) rather than treating * a missing `next_offset` as "the page ended here". */ offset?: number; /** Byte offset to resume at, absent when the document ended in this window. */ next_offset?: number; /** Size of the whole extracted body, the space offsets index into. */ total_bytes?: number; /** Estimated tokens of the whole extracted body, before budget and window. */ total_token_estimate?: number; /** The binary's own truncation flag; authoritative when present. */ truncated?: boolean; /** The page's headings, when an outline was requested. */ outline?: WebFetchOutlineSection[]; /** Where the page matched, when a search pattern was given. */ matches?: WebFetchMatch[]; content: string; content_type: string; media: string; token_estimate: number; /** Absent on binaries older than the status field; treated as "ok". */ status?: WebFetchContentStatus; references: WebFetchReference[]; metadata?: WebFetchMetadata; /** The URL that was requested, before any redirect (`final_url` is post-redirect). */ source: string; } export interface WebSearchResultItem { title: string; snippet: string; url: string; ref_index: number; } interface WebSearchReference { index: number; url: string; } export interface WebSearchOutput { query: string; results: WebSearchResultItem[]; references: WebSearchReference[]; token_estimate: number; result_count: number; /** Absent on binaries older than the status field; treated as "ok". */ status?: WebSearchStatus; /** Which backend answered, so a silent fallback to DuckDuckGo stays visible. */ provider?: string; } /** * TLS plumbing forwarded to the `webtools` binary for `webfetch`/`websearch`. * Kept separate from hoocode's own app-level TLS trust (utils/tls-ca.ts): the * binary has its own TLS stack, so it needs the CA / insecure flag passed in. */ export interface WebtoolsTLSConfig { /** Path to a PEM CA bundle forwarded as `--ca-cert ` (validated readable). */ caCertPath?: string; /** Forward `--insecure` (disables TLS verification in the binary). Strictly opt-in. */ insecure?: boolean; } /** * Resolve the webtools TLS config from explicit overrides (e.g. settings.json * passed down from the tool factories) falling back to the environment * (`HOOCODE_WEBTOOLS_CA_CERT`, `HOOCODE_WEBTOOLS_INSECURE`). Never hardcoded. */ export declare function resolveWebtoolsTLSConfig(overrides?: WebtoolsTLSConfig): WebtoolsTLSConfig; /** * Resolve the effective webtools request timeout (seconds) from an explicit * override (e.g. settings.json passed down from the tool factories) falling back * to the environment (`HOOCODE_WEBTOOLS_TIMEOUT`) and finally the default. Mirrors * {@link resolveWebtoolsTLSConfig}: resolve once, thread in, never hardcode. A * malformed or out-of-range env value falls back to the default; every result is * clamped to [1, 120]. */ export declare function resolveWebtoolsTimeoutSecs(override?: number): number; /** * The `webtools.search` block of `~/.hoocode/settings.json`. * * hoocode and the binary share that file: the binary reads its own `webtools` * key (snake_case, per its own schema) and ignores everything else, so these * keys are mirrored verbatim rather than camelCased. hoocode never writes them * — it only reads them to tell whether `websearch` has a keyed backend. */ export interface WebtoolsSearchSettings { /** Primary backend: "duckduckgo" | "brave" | "tavily" | "searxng". */ provider?: string; /** Backend tried when the primary fails; "none" disables the fallback. */ fallback?: string; providers?: { brave?: { api_key?: string; }; tavily?: { api_key?: string; }; searxng?: { base_url?: string; api_key?: string; }; }; } /** A search backend that answers over an API contract instead of scraped HTML. */ export type KeyedSearchProvider = "brave" | "tavily" | "searxng"; export interface WebSearchCredentialStatus { /** A keyed backend is reachable, so search does not depend on scraped DuckDuckGo. */ configured: boolean; /** Which backend the credential belongs to, when one is configured. */ provider?: KeyedSearchProvider; /** Where the credential came from — env wins over the settings file. */ source?: "env" | "settings"; /** The user explicitly asked for the keyless backend, so nothing is missing. */ explicitKeyless?: boolean; } /** * Whether `websearch` has a keyed backend configured, and where it came from. * * Mirrors the binary's own resolution order (env over settings file) for the * three keyed backends. This is a read-only check used to decide whether to * tell the user that search is running on keyless DuckDuckGo — it never * returns the credential itself, so a key cannot leak into the UI or a log. * * A provider pinned to `duckduckgo` (env or settings) is reported as * `explicitKeyless`: the user chose the scraped backend, so nothing is missing. */ export declare function resolveWebSearchCredentials(search?: WebtoolsSearchSettings): WebSearchCredentialStatus; /** * Run a `webtools` subcommand with `--json` and return parsed stdout. * * Throws on missing binary, non-zero exit (surfacing the binary's stderr, or the * status carried on stdout when stderr is empty), or unparseable output. Callers * convert thrown errors into tool error results. */ export declare function runWebtools(subcommand: "fetch" | "search", args: string[], cwd: string, signal?: AbortSignal, timeoutSecs?: number, tlsConfig?: WebtoolsTLSConfig): Promise; export declare class WebToolsCache { private readonly entries; private readonly inflight; get(key: string): T | undefined; set(key: string, value: T): void; /** * Return a cached value, join an identical in-flight computation, or start a * new one — collapsing concurrent duplicate fetch/search calls onto a single * subprocess. Successful results are cached; failures are not. * * Cancellation is shared safely: a caller whose own `signal` aborts rejects * promptly and releases its reference, but the underlying work keeps running * for the remaining callers and is only cancelled once none are left. */ getOrCompute(key: string, signal: AbortSignal | undefined, compute: (signal: AbortSignal) => Promise): Promise; } /** * Build an {@link Ignore} matcher from `.webtoolsignore` policy files. * * Precedence is project-after-user so a project file can re-allow (`!host`) * something the user blocked, matching gitignore layering. Returns undefined * when no policy files exist (the common case: everything allowed). * * Hosts are matched as single path components, so subdomains need an explicit * wildcard (`*.example.com`), exactly like gitignore directory matching. */ export declare function loadWebtoolsIgnore(cwd: string): IgnoreMatcher | undefined; /** Extract the lowercased hostname from a URL, or undefined if it cannot be parsed. */ export declare function hostnameOf(url: string): string | undefined; /** * Whether a host is blocked by policy. A matcher is required; with no policy * files present callers treat every host as allowed. */ export declare function isHostBlocked(matcher: IgnoreMatcher, host: string): boolean; /** * Convenience used by the permission gate: returns the blocked host for a URL, * or undefined when the URL is allowed (or there is no policy / unparseable URL). */ export declare function blockedHostForUrl(cwd: string, url: string): string | undefined; export {}; //# sourceMappingURL=webtools-shared.d.ts.map