/** * Search backend abstraction. * * `web_search` talks to a `SearchProvider` rather than a hardcoded API, so the * backend is swappable. **Kagi** (v1 default, v0 legacy) and **Wyna** ship today; * SearXNG/Brave/etc. can be added by implementing the interface and extending * `getSearchProvider`. * * Provider selection + endpoints are GLOBAL-config only (see config.ts) — a * project must never be able to redirect your queries to another server. */ import path from "node:path"; import { DEFAULTS, EXTENSION_DIR, resolveToken, resolveTokenV0, resolveWynaApiKey, wynaKeyFromModelsJson, type WebSearchConfig, } from "../config.ts"; import { kagiSearch, kagiSearchV1 } from "./kagi.ts"; import { wynaSearchGet } from "./wyna.ts"; /** A normalized search hit, provider-agnostic. */ export interface SearchResult { url: string; title: string; snippet?: string; published?: string; } export interface SearchProvider { readonly name: string; search(query: string, count: number, signal?: AbortSignal): Promise; } /** Assert the Wyna endpoint is safe to send the API key to (https only). */ function assertWynaEndpoint(endpoint: string): URL { let u: URL; try { u = new URL(endpoint); } catch { throw new Error(`invalid wynaEndpoint: ${endpoint}`); } if (u.protocol === "https:") return u; throw new Error(`wynaEndpoint must be https (got "${u.protocol}//${u.host}")`); } /** * Short user-facing hint (one line). Printed once at session start and again * when a search fails for lack of credentials; the details live in the * agent-facing setupBriefing. */ export const SETUP_HINT = "not configured — ask pi to set up web search"; /** True when ANY search credential (Kagi v1, Kagi v0, or Wyna) is available. */ export function hasAnySearchCredential(cfg: WebSearchConfig, agentDir?: string): boolean { return !!(resolveToken(cfg) || resolveTokenV0(cfg) || resolveWynaApiKey(cfg, agentDir)); } /** * Agent-facing setup briefing: which credential sources were checked (with real * paths) and how to finish setup. Returned as the web_search error when nothing * is configured, and injected into the tool guidelines at startup — the short * user-facing warning just says "ask pi"; THIS is what pi acts on. */ export function setupBriefing(cfg: WebSearchConfig, agentDir?: string): string { const envFile = path.join(EXTENSION_DIR, ".env"); const globalCfg = path.join(EXTENSION_DIR, "config.json"); const modelsPath = agentDir ? path.join(agentDir, "models.json") : "~/.pi/agent/models.json"; const status = (set: boolean) => (set ? "set" : "not set"); return ( "Web search is not configured — no search API key found. Credential sources checked:\n" + ` - KAGI_API_KEY (env, or ${envFile}): ${status(!!process.env.KAGI_API_KEY?.trim())}\n` + ` - kagiToken in ${globalCfg}: ${status(!!cfg.kagiToken)}\n` + ` - WYNA_API_KEY (env, or ${envFile}): ${status(!!process.env.WYNA_API_KEY?.trim())}\n` + ` - providers.wyna.apiKey in ${modelsPath}: ${status(!!wynaKeyFromModelsJson(agentDir))}\n` + "To finish setup, ask the user for ONE of these (never guess or invent a key):\n" + " - a Kagi Search API key (they can create one at https://kagi.com/api/keys), or\n" + " - their Wyna API key (the same key used for the Wyna LLM gateway).\n" + `Then store it for them: append KAGI_API_KEY= or WYNA_API_KEY= to ${envFile} ` + "(create the file if needed; it is git-ignored). The provider is auto-detected from whichever " + `key exists; to force one, set {"searchProvider": "kagi" | "wyna"} in ${globalCfg}. ` + "The user can verify with the /web-research-status command." ); } // Common second-level registries (bbc.co.uk → bbc.co.uk, not co.uk). const SECOND_LEVEL_LABELS = new Set(["co", "com", "net", "org", "ac", "gov", "edu"]); /** * Approximate registrable domain (publisher identity) of a URL: hostname minus * `www.`, reduced to eTLD+1 with a small second-level-registry heuristic. Used * to group results by publisher — not a full Public Suffix List lookup. */ export function registrableDomain(url: string): string { let host: string; try { host = new URL(url).hostname.toLowerCase(); } catch { return url; } host = host.replace(/^www\./, ""); const parts = host.split("."); if (parts.length <= 2 || parts.every((p) => /^\d+$/.test(p))) return host; const [sld, tld] = parts.slice(-2); if (tld.length === 2 && SECOND_LEVEL_LABELS.has(sld)) return parts.slice(-3).join("."); return parts.slice(-2).join("."); } /** * Reorder results so no publisher dominates: keep ranked order but defer * results beyond `maxPerDomain` per registrable domain to the end (used only * when there aren't enough diverse results to fill the requested count). A * content farm that floods the ranking gets at most `maxPerDomain` of the * pages actually read. */ export function diversifyByDomain(results: T[], maxPerDomain = 2): T[] { const counts = new Map(); const kept: T[] = []; const overflow: T[] = []; for (const r of results) { const dom = registrableDomain(r.url); const n = counts.get(dom) ?? 0; if (n < maxPerDomain) { counts.set(dom, n + 1); kept.push(r); } else { overflow.push(r); } } return [...kept, ...overflow]; } /** Thrown when the selected provider is misconfigured (missing creds, etc.). */ export class SearchConfigError extends Error { readonly code: string; constructor(message: string, code: string) { super(message); this.name = "SearchConfigError"; this.code = code; } } /** Wyna: GET + Bearer API key from models.json / WYNA_API_KEY / wynaApiKey. */ function makeWynaProvider(cfg: WebSearchConfig, agentDir?: string): SearchProvider { const token = resolveWynaApiKey(cfg, agentDir); if (!token) { throw new SearchConfigError( "Wyna API key not found. The key is read from ~/.pi/agent/models.json → " + 'providers.wyna.apiKey (the Pi LLM provider config), the WYNA_API_KEY environment variable, ' + "or wynaApiKey in global config.json.", "no-token", ); } const endpoint = cfg.wynaEndpoint ?? DEFAULTS.wynaEndpoint; assertWynaEndpoint(endpoint); // must be https — the token is in the Authorization header return { name: "wyna", search: (query, count, signal) => wynaSearchGet(query, count, token, endpoint, signal), }; } /** Kagi v1 (default): POST + Bearer API key from KAGI_API_KEY / kagiToken. */ function makeKagiV1Provider(cfg: WebSearchConfig): SearchProvider { const token = resolveToken(cfg); if (!token) { throw new SearchConfigError( "Kagi v1 API key not configured. Set the KAGI_API_KEY environment variable (a Bearer key from " + "kagi.com/api/keys), add it to .env, or set kagiToken in global config.json.", "no-token", ); } const endpoint = cfg.kagiEndpoint ?? DEFAULTS.kagiEndpoint; const allowPrivate = cfg.allowPrivateNetwork ?? DEFAULTS.allowPrivateNetwork; return { name: "kagi", search: (query, count, signal) => kagiSearchV1(query, count, token, endpoint, allowPrivate, signal), }; } /** Kagi v0 (legacy/deprecated): GET + Bot token from KAGI_API_KEY_V0 / kagiV0Token. */ function makeKagiV0Provider(cfg: WebSearchConfig): SearchProvider { const token = resolveTokenV0(cfg); if (!token) { throw new SearchConfigError( "Kagi v0 (legacy) token not configured. Set the KAGI_API_KEY_V0 environment variable (the deprecated " + '"Bot" token), or set kagiV0Token in global config.json.', "no-token", ); } const endpoint = cfg.kagiV0Endpoint ?? DEFAULTS.kagiV0Endpoint; const allowPrivate = cfg.allowPrivateNetwork ?? DEFAULTS.allowPrivateNetwork; return { name: "kagi-v0", search: (query, count, signal) => kagiSearch(query, count, token, endpoint, allowPrivate, signal), }; } /** * Resolve the effective provider name, auto-detecting Wyna when the default * (kagi) is active but only a Wyna key is available. * * Returns the provider name to use for `getSearchProvider()`. Never throws — * it may return "kagi" even when no key is present (the subsequent token * resolution will throw `SearchConfigError` with a helpful message). * * Order: * 1. Explicit `searchProvider` in config → honour it as-is. * 2. Default "kagi": if Kagi key exists → kagi. Else if Wyna key exists → * wyna (auto-detect). Else → kagi (will error with setup instructions). */ export function resolveEffectiveProviderName(cfg: WebSearchConfig, agentDir?: string): string { const name = ((cfg.searchProvider ?? DEFAULTS.searchProvider) || "kagi").toLowerCase(); // If the user explicitly chose a non-default provider, honour it. if (name !== "kagi" && name !== "kagi-v1") return name; // Default (kagi) — auto-detect based on available credentials. if (resolveToken(cfg)) return "kagi"; if (resolveWynaApiKey(cfg, agentDir)) return "wyna"; return "kagi"; // no key at all → will throw a clear setup error } /** * Resolve the configured search provider, validating its credentials. Throws a * SearchConfigError (with a `code`) the caller can map to a friendly message. * `"kagi"` (and the `"kagi-v1"` alias) select the v1 API (default — auto- * detects Wyna when only a Wyna key is present); `"kagi-v0"` (alias * `"kagi-legacy"`) selects the deprecated v0 API; `"wyna"` selects the Wyna * search API (uses the same API key as the Wyna LLM gateway). * * `agentDir` is only needed by the Wyna provider to read the Pi provider config. */ export function getSearchProvider(cfg: WebSearchConfig, agentDir?: string): SearchProvider { // Nothing configured at all: return the unified setup briefing so the agent // knows exactly what's missing and can finish setup with the user. (With an // explicit searchProvider, the provider-specific message below is more targeted.) if (!cfg.searchProvider?.trim() && !hasAnySearchCredential(cfg, agentDir)) { throw new SearchConfigError(setupBriefing(cfg, agentDir), "no-token"); } const name = resolveEffectiveProviderName(cfg, agentDir); switch (name) { case "wyna": return makeWynaProvider(cfg, agentDir); case "kagi": case "kagi-v1": return makeKagiV1Provider(cfg); case "kagi-v0": case "kagi-legacy": return makeKagiV0Provider(cfg); default: throw new SearchConfigError( `Unknown searchProvider "${name}" (supported: kagi (v1, default), kagi-v0, wyna).`, "unknown-provider", ); } }