import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { activityMonitor } from "./activity.js"; import type { ExtractedContent } from "./extract.js"; import type { SearchOptions, SearchResponse } from "./perplexity.js"; const EXA_ANSWER_URL = "https://api.exa.ai/answer"; const EXA_SEARCH_URL = "https://api.exa.ai/search"; const EXA_MCP_URL = "https://mcp.exa.ai/mcp"; const CONFIG_PATH = join(homedir(), ".pi", "web-search.json"); const USAGE_PATH = join(homedir(), ".pi", "exa-usage.json"); const MONTHLY_LIMIT = 1000; const WARNING_THRESHOLD = 800; const MCP_TOOLS_CACHE_TTL_MS = 5 * 60 * 1000; const CODE_SEARCH_FETCH_URL_LIMIT = 4; interface WebSearchConfig { exaApiKey?: unknown; } interface ExaUsage { month: string; count: number; } interface ExaAnswerResponse { answer?: string; citations?: Array<{ url?: string; title?: string; text?: string; publishedDate?: string }>; } interface ExaSearchResponse { results?: Array<{ title?: string; url?: string; publishedDate?: string; author?: string; text?: string; highlights?: unknown; highlightScores?: number[]; }>; } interface ExaMcpToolDefinition { name?: string; inputSchema?: { properties?: Record; }; [key: string]: unknown; } interface ExaMcpRpcResult { content?: Array<{ type?: string; text?: string }>; isError?: boolean; tools?: ExaMcpToolDefinition[]; [key: string]: unknown; } interface ExaMcpRpcResponse { result?: ExaMcpRpcResult; error?: { code?: number; message?: string; }; } export type ExaSearchResult = SearchResponse | { exhausted: true } | null; export interface ExaSearchOptions extends SearchOptions { includeContent?: boolean; } type McpParsedResult = { title: string; url: string; content: string }; let cachedConfig: WebSearchConfig | null = null; let warnedMonth: string | null = null; let cachedMcpTools: { fetchedAt: number; tools: ExaMcpToolDefinition[] } | null = null; function loadConfig(): WebSearchConfig { if (cachedConfig) return cachedConfig; if (!existsSync(CONFIG_PATH)) { cachedConfig = {}; return cachedConfig; } const raw = readFileSync(CONFIG_PATH, "utf-8"); try { cachedConfig = JSON.parse(raw) as WebSearchConfig; return cachedConfig; } catch (err) { const message = err instanceof Error ? err.message : String(err); throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`); } } function normalizeApiKey(value: unknown): string | null { if (typeof value !== "string") return null; const normalized = value.trim(); return normalized.length > 0 ? normalized : null; } function getApiKey(): string | null { return normalizeApiKey(process.env.EXA_API_KEY) ?? normalizeApiKey(loadConfig().exaApiKey); } function getCurrentMonth(): string { return new Date().toISOString().slice(0, 7); } function normalizeUsage(raw: unknown): ExaUsage { const month = getCurrentMonth(); if (!raw || typeof raw !== "object") return { month, count: 0 }; const data = raw as { month?: unknown; count?: unknown }; const parsedMonth = typeof data.month === "string" ? data.month : month; const parsedCount = typeof data.count === "number" && Number.isFinite(data.count) ? data.count : 0; if (parsedMonth !== month) return { month, count: 0 }; return { month: parsedMonth, count: Math.max(0, Math.floor(parsedCount)) }; } function readUsage(): ExaUsage { if (!existsSync(USAGE_PATH)) return { month: getCurrentMonth(), count: 0 }; const raw = readFileSync(USAGE_PATH, "utf-8"); try { return normalizeUsage(JSON.parse(raw)); } catch (err) { const message = err instanceof Error ? err.message : String(err); throw new Error(`Failed to parse ${USAGE_PATH}: ${message}`); } } function writeUsage(usage: ExaUsage): void { const dir = join(homedir(), ".pi"); if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); writeFileSync(USAGE_PATH, JSON.stringify(usage, null, 2) + "\n"); } function reserveRequestBudget(): { exhausted: true } | null { const usage = readUsage(); if (usage.count >= MONTHLY_LIMIT) { return { exhausted: true }; } const nextCount = usage.count + 1; if (nextCount >= WARNING_THRESHOLD && warnedMonth !== usage.month) { warnedMonth = usage.month; console.error(`Exa usage warning: ${nextCount}/${MONTHLY_LIMIT} monthly requests used.`); } writeUsage({ month: usage.month, count: nextCount }); return null; } function requestSignal(signal?: AbortSignal): AbortSignal { const timeout = AbortSignal.timeout(60000); return signal ? AbortSignal.any([signal, timeout]) : timeout; } function recencyToStartDate(filter: string): string { const now = new Date(); const offsets: Record = { day: 1, week: 7, month: 30, year: 365, }; const days = offsets[filter] ?? 0; return new Date(now.getTime() - days * 86400000).toISOString(); } function mapDomainFilter(domainFilter: string[] | undefined): { includeDomains?: string[]; excludeDomains?: string[] } { if (!domainFilter?.length) return {}; const includeDomains = domainFilter .filter(d => !d.startsWith("-") && d.trim().length > 0) .map(d => d.trim()); const excludeDomains = domainFilter .filter(d => d.startsWith("-")) .map(d => d.slice(1).trim()) .filter(Boolean); return { ...(includeDomains.length ? { includeDomains } : {}), ...(excludeDomains.length ? { excludeDomains } : {}), }; } function normalizeHighlights(value: unknown): string[] { if (!Array.isArray(value)) return []; return value.filter((item): item is string => typeof item === "string" && item.trim().length > 0); } function buildAnswerFromSearchResults(results: ExaSearchResponse["results"]): string { if (!results?.length) return ""; const parts: string[] = []; for (let i = 0; i < results.length; i++) { const item = results[i]; if (!item?.url) continue; const highlights = normalizeHighlights(item.highlights); const content = highlights.length > 0 ? highlights.join(" ") : typeof item.text === "string" ? item.text.trim().slice(0, 1000) : ""; if (!content) continue; const sourceTitle = item.title || `Source ${i + 1}`; parts.push(`${content}\nSource: ${sourceTitle} (${item.url})`); } return parts.join("\n\n"); } function mapResults(results: ExaSearchResponse["results"] | ExaAnswerResponse["citations"]): SearchResponse["results"] { if (!Array.isArray(results)) return []; const mapped: SearchResponse["results"] = []; for (let i = 0; i < results.length; i++) { const item = results[i]; if (!item?.url) continue; mapped.push({ title: item.title || `Source ${i + 1}`, url: item.url, snippet: "", }); } return mapped; } function mapInlineContent(results: ExaSearchResponse["results"]): ExtractedContent[] { if (!results?.length) return []; return results .filter((r): r is NonNullable[number] & { url: string; text: string } => !!r?.url && typeof r.text === "string" && r.text.length > 0) .map(r => ({ url: r.url, title: r.title || "", content: r.text, error: null, })); } function parseExaMcpResponse(body: string): ExaMcpRpcResponse | null { const dataLines = body.split("\n").filter(line => line.startsWith("data:")); for (const line of dataLines) { const payload = line.slice(5).trim(); if (!payload) continue; try { const candidate = JSON.parse(payload) as ExaMcpRpcResponse; if (candidate?.result || candidate?.error) { return candidate; } } catch { } } try { const candidate = JSON.parse(body) as ExaMcpRpcResponse; if (candidate?.result || candidate?.error) { return candidate; } } catch { } return null; } async function exaMcpRpc(method: string, params: Record, signal?: AbortSignal): Promise { const response = await fetch(EXA_MCP_URL, { method: "POST", headers: { "Content-Type": "application/json", "Accept": "application/json, text/event-stream", }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params, }), signal: requestSignal(signal), }); if (!response.ok) { const errorText = await response.text(); throw new Error(`Exa MCP error ${response.status}: ${errorText.slice(0, 300)}`); } const body = await response.text(); const parsed = parseExaMcpResponse(body); if (!parsed) { throw new Error("Exa MCP returned an empty response"); } if (parsed.error) { const code = typeof parsed.error.code === "number" ? ` ${parsed.error.code}` : ""; const message = parsed.error.message || "Unknown error"; throw new Error(`Exa MCP error${code}: ${message}`); } return parsed; } async function listExaMcpToolDefinitions(signal?: AbortSignal): Promise { if (cachedMcpTools && Date.now() - cachedMcpTools.fetchedAt < MCP_TOOLS_CACHE_TTL_MS) { return cachedMcpTools.tools; } const parsed = await exaMcpRpc("tools/list", {}, signal); const tools = parsed.result?.tools?.filter((tool): tool is ExaMcpToolDefinition => !!tool && typeof tool === "object") ?? []; cachedMcpTools = { fetchedAt: Date.now(), tools }; return tools; } export async function listExaMcpTools(signal?: AbortSignal): Promise { const tools = await listExaMcpToolDefinitions(signal); return tools .map(tool => typeof tool.name === "string" ? tool.name.trim() : "") .filter((name): name is string => name.length > 0); } function formatAvailableExaMcpTools(tools: string[]): string { return tools.length > 0 ? tools.join(", ") : "unknown"; } function isMissingExaMcpToolError(message: string): boolean { return /tool\s+.+\s+(?:not found|unavailable)/i.test(message); } function buildCodeSearchFallbackQuery(query: string): string { return `Programming documentation, API references, GitHub issues, Stack Overflow answers, and concrete code examples for: ${query}`; } function truncateText(text: string, maxChars: number): string { if (text.length <= maxChars) return text; return text.slice(0, Math.max(0, maxChars - 16)).trimEnd() + "\n\n[truncated]"; } function parseMcpFetchedContent(text: string): ExtractedContent[] { const blocks = text.split(/(?=^# .+\nURL: )/m).filter(block => block.trim().length > 0); return blocks.map(block => { const title = block.match(/^# (.+)/m)?.[1]?.trim() ?? ""; const url = block.match(/^URL: (.+)/m)?.[1]?.trim() ?? ""; return { url, title, content: block.trim(), error: null, }; }).filter(item => item.url.length > 0 && item.content.length > 0); } async function buildCodeSearchFallback( query: string, maxTokens: number, knownTools: string[] | null, signal?: AbortSignal, ): Promise { const tools = knownTools ?? await listExaMcpTools(signal).catch(() => []); if (tools.length > 0 && !tools.includes("web_search_exa")) { throw new Error(`Exa MCP code_search fallback unavailable. Available tools: ${tools.join(", ")}`); } const searchText = await callExaMcp( "web_search_exa", { query: buildCodeSearchFallbackQuery(query), numResults: 6, }, signal, ); const parsedResults = parseMcpResults(searchText) ?? []; const urls = Array.from(new Set(parsedResults.map(result => result.url).filter(Boolean))).slice(0, CODE_SEARCH_FETCH_URL_LIMIT); const targetChars = Math.min(Math.max(maxTokens * 4, 4000), 24000); const sections = [ "Exa MCP fallback: assembled code context via web_search_exa/web_fetch_exa because get_code_context_exa is unavailable on the current Exa MCP server.", `Query: ${query}`, "", "Search results:", searchText.trim(), ]; if (urls.length > 0 && (tools.length === 0 || tools.includes("web_fetch_exa"))) { try { const fetchText = await callExaMcp( "web_fetch_exa", { urls, maxCharacters: Math.max(1500, Math.floor(targetChars / urls.length)), }, signal, ); if (fetchText.trim()) { sections.push("", "Fetched source content:", fetchText.trim()); } } catch (err) { const message = err instanceof Error ? err.message : String(err); sections.push("", `Note: web_fetch_exa fallback failed: ${message}`); } } return truncateText(sections.join("\n"), targetChars); } export async function executeExaCodeSearch( query: string, maxTokens: number, signal?: AbortSignal, ): Promise<{ text: string; strategy: "get_code_context_exa" | "web_search_exa_fallback" }> { let knownTools: string[] | null = null; try { knownTools = await listExaMcpTools(signal); } catch { } if (!knownTools || knownTools.includes("get_code_context_exa")) { try { const text = await callExaMcp( "get_code_context_exa", { query, tokensNum: maxTokens, }, signal, ); return { text, strategy: "get_code_context_exa" }; } catch (err) { const message = err instanceof Error ? err.message : String(err); if (!isMissingExaMcpToolError(message)) { throw err; } cachedMcpTools = null; knownTools = await listExaMcpTools(signal).catch(() => null); } } const text = await buildCodeSearchFallback(query, maxTokens, knownTools, signal); return { text, strategy: "web_search_exa_fallback" }; } export async function callExaMcp( toolName: string, args: Record, signal?: AbortSignal, ): Promise { const knownTools = await listExaMcpTools(signal).catch(() => []); if (knownTools.length > 0 && !knownTools.includes(toolName)) { throw new Error(`Exa MCP tool unavailable: ${toolName}. Available tools: ${formatAvailableExaMcpTools(knownTools)}`); } const parsed = await exaMcpRpc("tools/call", { name: toolName, arguments: args, }, signal); if (parsed.result?.isError) { const message = parsed.result.content ?.find(item => item.type === "text" && typeof item.text === "string") ?.text?.trim(); throw new Error(message || "Exa MCP returned an error"); } const text = parsed.result?.content ?.find(item => item.type === "text" && typeof item.text === "string" && item.text.trim().length > 0) ?.text; if (!text) { throw new Error("Exa MCP returned empty content"); } return text; } function parseMcpResults(text: string): McpParsedResult[] | null { const blocks = text.split(/(?=^Title: )/m).filter(block => block.trim().length > 0); const parsed = blocks.map(block => { const title = block.match(/^Title: (.+)/m)?.[1]?.trim() ?? ""; const url = block.match(/^URL: (.+)/m)?.[1]?.trim() ?? ""; let content = ""; const textStart = block.indexOf("\nText: "); if (textStart >= 0) { content = block.slice(textStart + 7).trim(); } else { const hlMatch = block.match(/\nHighlights:\s*\n/); if (hlMatch?.index != null) { content = block.slice(hlMatch.index + hlMatch[0].length).trim(); } } content = content.replace(/\n---\s*$/, "").trim(); return { title, url, content }; }).filter(result => result.url.length > 0); return parsed.length > 0 ? parsed : null; } function buildAnswerFromMcpResults(results: McpParsedResult[]): string { if (results.length === 0) return ""; const parts: string[] = []; for (let i = 0; i < results.length; i++) { const result = results[i]; const snippet = result.content.replace(/\s+/g, " ").trim().slice(0, 500); if (!snippet) continue; const sourceTitle = result.title || `Source ${i + 1}`; parts.push(`${snippet}\nSource: ${sourceTitle} (${result.url})`); } return parts.join("\n\n"); } function mapMcpInlineContent(results: McpParsedResult[]): ExtractedContent[] { return results .filter(result => result.content.length > 0) .map(result => ({ url: result.url, title: result.title, content: result.content, error: null, })); } function buildMcpQuery(query: string, options: ExaSearchOptions): string { const parts = [query]; if (options.domainFilter?.length) { for (const d of options.domainFilter) { parts.push(d.startsWith("-") ? `-site:${d.slice(1)}` : `site:${d}`); } } if (options.recencyFilter) { const now = new Date(); switch (options.recencyFilter) { case "day": parts.push("past 24 hours"); break; case "week": parts.push("past week"); break; case "month": parts.push(`${now.toLocaleString("en", { month: "long" })} ${now.getFullYear()}`); break; case "year": parts.push(String(now.getFullYear())); break; } } return parts.join(" "); } async function searchWithExaMcp(query: string, options: ExaSearchOptions = {}): Promise { const enrichedQuery = buildMcpQuery(query, options); const activityId = activityMonitor.logStart({ type: "api", query: enrichedQuery }); try { const tools = await listExaMcpTools(options.signal).catch(() => []); if (tools.length > 0 && !tools.includes("web_search_exa")) { throw new Error(`Exa MCP tool unavailable: web_search_exa. Available tools: ${formatAvailableExaMcpTools(tools)}`); } const text = await callExaMcp( "web_search_exa", { query: enrichedQuery, numResults: options.numResults ?? 5, }, options.signal, ); const parsedResults = parseMcpResults(text); activityMonitor.logComplete(activityId, 200); if (!parsedResults) return null; const response: SearchResponse = { answer: buildAnswerFromMcpResults(parsedResults), results: parsedResults.map((result, index) => ({ title: result.title || `Source ${index + 1}`, url: result.url, snippet: "", })), }; if (options.includeContent) { const urls = Array.from(new Set(parsedResults.map(result => result.url).filter(Boolean))).slice(0, options.numResults ?? 5); if (urls.length > 0 && (tools.length === 0 || tools.includes("web_fetch_exa"))) { try { const fetchText = await callExaMcp( "web_fetch_exa", { urls, maxCharacters: 50000, }, options.signal, ); const inlineContent = parseMcpFetchedContent(fetchText); if (inlineContent.length > 0) { response.inlineContent = inlineContent; } else { const partialInlineContent = mapMcpInlineContent(parsedResults); if (partialInlineContent.length > 0) response.inlineContent = partialInlineContent; } } catch { const partialInlineContent = mapMcpInlineContent(parsedResults); if (partialInlineContent.length > 0) response.inlineContent = partialInlineContent; } } else { const inlineContent = mapMcpInlineContent(parsedResults); if (inlineContent.length > 0) response.inlineContent = inlineContent; } } return response; } catch (err) { const message = err instanceof Error ? err.message : String(err); if (message.toLowerCase().includes("abort")) { activityMonitor.logComplete(activityId, 0); } else { activityMonitor.logError(activityId, message); } throw err; } } export function isExaAvailable(): boolean { if (getApiKey()) { const usage = readUsage(); return usage.count < MONTHLY_LIMIT; } return true; } export function hasExaApiKey(): boolean { return !!getApiKey(); } export async function searchWithExa(query: string, options: ExaSearchOptions = {}): Promise { const apiKey = getApiKey(); if (!apiKey) { return searchWithExaMcp(query, options); } const budget = reserveRequestBudget(); if (budget) return budget; const useSearch = options.includeContent || !!options.recencyFilter || !!options.domainFilter?.length || !!(options.numResults && options.numResults !== 5); const activityId = activityMonitor.logStart({ type: "api", query }); try { if (!useSearch) { const response = await fetch(EXA_ANSWER_URL, { method: "POST", headers: { "x-api-key": apiKey, "Content-Type": "application/json", }, body: JSON.stringify({ query, text: true, }), signal: requestSignal(options.signal), }); if (!response.ok) { const errorText = await response.text(); throw new Error(`Exa API error ${response.status}: ${errorText.slice(0, 300)}`); } const data = await response.json() as ExaAnswerResponse; activityMonitor.logComplete(activityId, response.status); return { answer: data.answer || "", results: mapResults(data.citations), }; } const startDate = options.recencyFilter ? recencyToStartDate(options.recencyFilter) : null; const domainFilters = mapDomainFilter(options.domainFilter); const response = await fetch(EXA_SEARCH_URL, { method: "POST", headers: { "x-api-key": apiKey, "Content-Type": "application/json", }, body: JSON.stringify({ query, type: "auto", numResults: options.numResults ?? 5, ...domainFilters, ...(startDate ? { startPublishedDate: startDate } : {}), contents: { text: options.includeContent ? true : { maxCharacters: 3000 }, highlights: true, }, }), signal: requestSignal(options.signal), }); if (!response.ok) { const errorText = await response.text(); throw new Error(`Exa API error ${response.status}: ${errorText.slice(0, 300)}`); } const data = await response.json() as ExaSearchResponse; activityMonitor.logComplete(activityId, response.status); const mapped: SearchResponse = { answer: buildAnswerFromSearchResults(data.results), results: mapResults(data.results), }; if (options.includeContent) { const inlineContent = mapInlineContent(data.results); if (inlineContent.length > 0) mapped.inlineContent = inlineContent; } return mapped; } catch (err) { const message = err instanceof Error ? err.message : String(err); if (message.toLowerCase().includes("abort")) { activityMonitor.logComplete(activityId, 0); } else { activityMonitor.logError(activityId, message); } throw err; } }