import type { ExtensionContext } from "@earendil-works/pi-coding-agent"; import { CredentialResolutionError } from "../config/credentials.ts"; import { getConfig } from "../config/store.ts"; import { getSearchProviderDefinition, loadSearchProvider } from "./registry.ts"; import type { SearchOptions, SearchResponse, SearchResult } from "./types.ts"; export const RESOLVED_SEARCH_PROVIDERS = ["openai", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "exa", "serpdive", "kagi", "ollama", "anysearch", "xai", "brightdata", "serpbase", "serper", "valyu", "bocha", "xcrawl"] as const; export const SEARCH_PROVIDERS = ["auto", "all", ...RESOLVED_SEARCH_PROVIDERS] as const; export const ALL_SEARCH_PROVIDERS: ResolvedSearchProvider[] = ["searxng", "openai", "exa", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "ollama", "perplexity", "gemini", "bocha"]; const DEFAULT_AUTO_PROVIDERS: ResolvedSearchProvider[] = ["searxng", "exa", "openai", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "bocha", "ollama", "perplexity", "gemini"]; const VALID_ROUTING_KINDS = ["transient", "quota", "network", "invalid-response", "unsupported"] as const; export type ResolvedSearchProvider = typeof RESOLVED_SEARCH_PROVIDERS[number]; export type SearchProvider = typeof SEARCH_PROVIDERS[number]; export type SearchProviderSelection = SearchProvider | ResolvedSearchProvider[]; export type ProviderAvailability = { all: boolean } & Record; export type SearchProviderErrorKind = "transient" | "quota" | "network" | "credential" | "config" | "auth" | "invalid-request" | "invalid-response" | "unsupported" | "aborted" | "unknown"; export interface SearchRoutingConfig { providers: ResolvedSearchProvider[]; useCurrentModel?: boolean; fallbackOn: Array>; } export class SearchProviderError extends Error { readonly provider: ResolvedSearchProvider; readonly kind: SearchProviderErrorKind; readonly status: number | undefined; readonly causeError: unknown; constructor(provider: ResolvedSearchProvider, kind: SearchProviderErrorKind, message: string, status: number | undefined, causeError: unknown) { super(`${provider} search failed (${kind}): ${message}`); this.name = "SearchProviderError"; this.provider = provider; this.kind = kind; this.status = status; this.causeError = causeError; } } export interface ProviderSearchResponse extends SearchResponse { provider: ResolvedSearchProvider } export interface ProviderSearchFailure { provider: ResolvedSearchProvider; error: string } export interface AttributedSearchResponse extends SearchResponse { provider: ResolvedSearchProvider | "all"; providerResponses?: ProviderSearchResponse[]; providerErrors?: ProviderSearchFailure[]; } export interface FullSearchOptions extends SearchOptions { provider?: SearchProviderSelection; includeContent?: boolean; extensionContext?: ExtensionContext; } type SearchConfig = { searchProvider: SearchProviderSelection; searchProviderConfigured: boolean; searchRouting?: SearchRoutingConfig; }; function getSearchConfig(): SearchConfig { const root = getConfig(); const configured = Object.hasOwn(root, "searchProvider") || Object.hasOwn(root, "provider"); return { searchProvider: normalizeSearchProviderSelection(root.searchProvider ?? root.provider), searchProviderConfigured: configured, ...(Object.hasOwn(root, "searchRouting") ? { searchRouting: normalizeSearchRouting(root.searchRouting) } : {}), }; } function normalizeSearchRouting(value: unknown): SearchRoutingConfig { if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("searchRouting must be an object"); const raw = value as Record; const providers = normalizeResolvedProviderList(raw.providers, "searchRouting.providers"); if (raw.useCurrentModel !== undefined && typeof raw.useCurrentModel !== "boolean") throw new Error("searchRouting.useCurrentModel must be a boolean"); if (!Array.isArray(raw.fallbackOn) || raw.fallbackOn.length === 0) throw new Error("searchRouting.fallbackOn must be a non-empty array"); const fallbackOn: SearchRoutingConfig["fallbackOn"] = []; for (const value of raw.fallbackOn) { if (typeof value !== "string" || !VALID_ROUTING_KINDS.includes(value as typeof VALID_ROUTING_KINDS[number])) { throw new Error("searchRouting.fallbackOn may only contain transient, quota, network, invalid-response, or unsupported"); } if (!fallbackOn.includes(value as SearchRoutingConfig["fallbackOn"][number])) fallbackOn.push(value as SearchRoutingConfig["fallbackOn"][number]); } return { providers, ...(raw.useCurrentModel !== undefined ? { useCurrentModel: raw.useCurrentModel as boolean } : {}), fallbackOn }; } export function getConfiguredSearchRouting(): SearchRoutingConfig | undefined { const config = getSearchConfig(); return config.searchProviderConfigured ? undefined : config.searchRouting; } function normalizeResolvedProviderList(value: unknown, label: string): ResolvedSearchProvider[] { if (!Array.isArray(value) || value.length === 0) throw new Error(`${label} must be a non-empty array`); const providers: ResolvedSearchProvider[] = []; for (const item of value) { const provider = typeof item === "string" ? item.trim().toLowerCase() : ""; if (!RESOLVED_SEARCH_PROVIDERS.includes(provider as ResolvedSearchProvider)) throw new Error(`${label} contains an invalid provider: ${String(item)}`); if (providers.includes(provider as ResolvedSearchProvider)) throw new Error(`${label} must not contain duplicates: ${provider}`); providers.push(provider as ResolvedSearchProvider); } return providers; } export function normalizeSearchProviderSelection(value: unknown, label = "provider"): SearchProviderSelection { if (Array.isArray(value)) return normalizeResolvedProviderList(value, label); const provider = typeof value === "string" ? value.trim().toLowerCase() : ""; return SEARCH_PROVIDERS.includes(provider as SearchProvider) ? provider as SearchProvider : "auto"; } function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } function isAbortError(error: unknown): boolean { return errorMessage(error).toLowerCase().includes("abort"); } function providerErrorStatus(message: string): number | undefined { const match = message.match(/\b(?:error|status|http)\s+(\d{3})\b/i); return match ? Number(match[1]) : undefined; } function classifyProviderError(provider: ResolvedSearchProvider, error: unknown): SearchProviderError { if (error instanceof SearchProviderError) return error; const message = errorMessage(error); const lower = message.toLowerCase(); const status = providerErrorStatus(message); let kind: SearchProviderErrorKind = "unknown"; const unsupportedTool = /(?:web[_ -]?search|web[_ -]?search_preview|(?:the )?tool)\b.*\b(?:unsupported|not supported|does not support|doesn't support|unknown|unrecognized|unavailable|not found)|\b(?:unsupported|not supported|does not support|doesn't support|unknown|unrecognized|unavailable|not found)\b.*\b(?:web[_ -]?search|web[_ -]?search_preview|(?:the )?tool)/i.test(lower); if (error instanceof CredentialResolutionError || /(?:api )?key (?:not found|missing)|credential resolution/.test(lower)) kind = "credential"; else if (isAbortError(error)) kind = "aborted"; else if (provider === "xai" && status === 403 && /spending[- ]limit|(?:no|out of) credits?|insufficient quota|quota (?:exceeded|exhausted)|credits? (?:exhausted|depleted|used up)/.test(lower)) kind = "quota"; else if (status === 401 || status === 403) kind = "auth"; else if (provider === "openai" && (status === 400 || status === 422) && unsupportedTool) kind = "unsupported"; else if (status === 400 || status === 422) kind = "invalid-request"; else if (status === 402 || status === 429) kind = "quota"; else if (status !== undefined && (status === 408 || status === 425 || status >= 500)) kind = "transient"; else if (/rate limit|quota|too many requests/.test(lower)) kind = "quota"; else if (/unauthorized|forbidden|permission denied/.test(lower)) kind = "auth"; else if (/bad request|invalid request/.test(lower)) kind = "invalid-request"; else if (/invalid json|no parseable response|no parseable results|invalid response|returned empty response|no web_search_call/.test(lower)) kind = "invalid-response"; else if (/temporar|service unavailable|server error/.test(lower)) kind = "transient"; else if (error instanceof TypeError || /fetch failed|network|econnreset|econnrefused|enotfound|etimedout|timed out|socket/.test(lower)) kind = "network"; else if (/invalid or missing|invalid config|failed to parse|must be an? |configuration/.test(lower)) kind = "config"; return new SearchProviderError(provider, kind, message, status, error); } async function searchWithResolvedProvider(provider: ResolvedSearchProvider, query: string, options: FullSearchOptions, useCurrentModel = false): Promise { const response = await (await loadSearchProvider(provider)).search(query, options, useCurrentModel); return { ...response, provider }; } async function isResolvedProviderAvailable(provider: ResolvedSearchProvider, options: FullSearchOptions, useCurrentModel = false, forAll = false): Promise { const adapter = await loadSearchProvider(provider); return forAll && adapter.isAvailableForAll ? adapter.isAvailableForAll(options) : adapter.isAvailable(options, useCurrentModel); } function providerLabel(provider: ResolvedSearchProvider): string { return getSearchProviderDefinition(provider).label; } async function searchWithProviders(query: string, options: FullSearchOptions, selected?: ResolvedSearchProvider[]): Promise { const providers = selected ?? (await Promise.all(ALL_SEARCH_PROVIDERS.map(async provider => ({ provider, available: await isResolvedProviderAvailable(provider, options, false, true) })))).filter(item => item.available).map(item => item.provider); if (providers.length === 0) throw new Error("No configured search provider available for provider \"all\"."); const settled = await Promise.allSettled(providers.map(async provider => { const adapter = await loadSearchProvider(provider); const response = selected || !adapter.searchForAll ? await adapter.search(query, options) : await adapter.searchForAll(query, options); return { ...response, provider } as ProviderSearchResponse; })); if (options.signal?.aborted) throw new Error("Aborted"); const successes: ProviderSearchResponse[] = []; const failures: ProviderSearchFailure[] = []; settled.forEach((result, index) => result.status === "fulfilled" ? successes.push(result.value) : failures.push({ provider: providers[index], error: errorMessage(result.reason) })); if (successes.length === 0) throw new Error(`${selected ? "Selected-provider" : "All-provider"} search failed:\n - ${failures.map(item => `${providerLabel(item.provider)}: ${item.error}`).join("\n - ")}`); const results: SearchResult[] = []; const inlineContent: NonNullable = []; const seenUrls = new Set(); const seenContent = new Set(); for (const response of successes) { for (const result of response.results) if (!seenUrls.has(result.url)) { seenUrls.add(result.url); results.push(result); } for (const content of response.inlineContent ?? []) if (!seenContent.has(content.url)) { seenContent.add(content.url); inlineContent.push(content); } } const sections = successes.map(response => `## ${providerLabel(response.provider)}\n\n${response.answer || "(No answer text returned.)"}`); if (failures.length) sections.push(`## Provider errors\n\n${failures.map(item => `- **${providerLabel(item.provider)}:** ${item.error}`).join("\n")}`); return { provider: "all", answer: sections.join("\n\n"), results, providerResponses: successes, ...(failures.length ? { providerErrors: failures } : {}), ...(inlineContent.length ? { inlineContent } : {}), }; } async function searchWithConfiguredRouting(query: string, options: FullSearchOptions, routing: SearchRoutingConfig): Promise { const diagnostics: string[] = []; for (const provider of routing.providers) { const current = provider === "openai" && routing.useCurrentModel === true; if (!(await isResolvedProviderAvailable(provider, options, current))) { diagnostics.push(`${provider}: unavailable`); continue; } try { return await searchWithResolvedProvider(provider, query, options, current); } catch (error) { const classified = classifyProviderError(provider, error); diagnostics.push(`${provider} [${classified.kind}]: ${errorMessage(error)}`); if (!routing.fallbackOn.includes(classified.kind as SearchRoutingConfig["fallbackOn"][number])) throw classified; } } throw new Error(`Configured search routing exhausted:\n - ${diagnostics.join("\n - ")}`); } function supportsOpenAIDefault(options: SearchOptions): boolean { return !options.recencyFilter && (options.numResults === undefined || (Number.isFinite(options.numResults) && Math.floor(options.numResults) === 5)); } function automaticProviders(options: FullSearchOptions): ResolvedSearchProvider[] { if (!supportsOpenAIDefault(options)) return DEFAULT_AUTO_PROVIDERS.filter(provider => provider !== "openai"); if (!options.extensionContext || options.extensionContext.model?.provider === "openai-codex") return ["searxng", "openai", ...DEFAULT_AUTO_PROVIDERS.filter(provider => provider !== "searxng" && provider !== "openai")]; return DEFAULT_AUTO_PROVIDERS; } async function searchAutomatically(query: string, options: FullSearchOptions): Promise { const failures: string[] = []; for (const provider of automaticProviders(options)) { if (!(await isResolvedProviderAvailable(provider, options))) continue; try { return await searchWithResolvedProvider(provider, query, options); } catch (error) { if (isAbortError(error) || error instanceof CredentialResolutionError) throw error; failures.push(`${providerLabel(provider)}: ${errorMessage(error)}`); } } if (failures.length) throw new Error(`Auto provider search failed:\n - ${failures.join("\n - ")}`); throw new Error("No search provider available. Configure a Provider in web-search.json or select one explicitly."); } export async function search(query: string, options: FullSearchOptions = {}): Promise { const config = getSearchConfig(); const provider = options.provider === undefined || options.provider === "auto" ? config.searchProvider : options.provider; if (Array.isArray(provider)) return searchWithProviders(query, options, normalizeResolvedProviderList(provider, "provider")); if (provider === "all") return searchWithProviders(query, options); if (provider !== "auto") return searchWithResolvedProvider(provider, query, options); if (!config.searchProviderConfigured && config.searchRouting) return searchWithConfiguredRouting(query, options, config.searchRouting); return searchAutomatically(query, options); }