/** * LLM provider abstraction for the Lucidly research agent. * * The `ResearchAgent` (researchAgent.ts) drives a tool-use loop. It needs: * 1. A normalized message format it can append to as the conversation unfolds. * 2. Some `LLMProvider` that knows how to (a) translate that normalized history into the * provider's native wire format, (b) call the API (streaming), and (c) translate the * response back. * * Four providers ship out of the box: * • `AnthropicProvider` — Messages API. Tool-use first-class, normalized blocks map 1:1. * • `GroqProvider` — OpenAI-compatible on Groq Cloud (Llama 3.x). Free tier available. * • `DeepSeekProvider` — OpenAI-compatible on api.deepseek.com (deepseek-chat / -reasoner). * • `OpenAIProvider` — OpenAI-compatible on api.openai.com (gpt-4.1, gpt-5, Codex models). * * The three OpenAI-compatible providers share a common base (`OpenAICompatProvider`) since their * wire format is identical — they only differ in default model + base URL. Adding a fifth * OpenAI-compat provider (Mistral, Together, Bedrock-via-OpenAI-proxy) = ~10 lines of subclass. * * Streaming + resilience: every provider streams SSE, supports `AbortSignal` for user-stop, * auto-retries on HTTP 429 (parses `Retry-After` + "try again in Xs" bodies), and auto-retries * on network-blip mid-stream (up to 2x, ONLY if no content has been emitted — retrying after * partial text would duplicate it in the consumer's UI). * * Why we ship our own adapters instead of LangChain/LiteLLM: 350-line abstraction with tight * control over the wire format. No transitive deps. */ import type { ToolDescriptor } from "./researchTools.js"; // ───────────────────────────────────────────────────────────────────────────── // Normalized agent message types // ───────────────────────────────────────────────────────────────────────────── export type AgentBlock = | { type: "text"; text: string } | { type: "tool_call"; id: string; name: string; input: Record } | { type: "tool_result"; toolCallId: string; content: string; isError?: boolean }; export interface AgentMessage { role: "user" | "assistant"; content: AgentBlock[]; } export type AgentStopReason = | "end_turn" | "tool_use" | "max_tokens" | "stop_sequence" | null; export interface LLMCompleteParams { system: string; tools: ToolDescriptor[]; messages: AgentMessage[]; maxTokens: number; /** Called once per streamed text fragment from the model. Optional — non-streaming consumers omit. */ onTextDelta?: (chunk: string) => void; /** Called when the provider returned 429 and we're about to back off. Optional. */ onRateLimit?: (info: { waitSec: number; attempt: number; maxRetries: number }) => void; /** AbortSignal — when aborted, in-flight fetch is cancelled and a retry loop bails immediately. */ abortSignal?: AbortSignal; } export interface LLMResponse { content: AgentBlock[]; stopReason: AgentStopReason; usage: { input: number; output: number }; } export interface LLMProvider { readonly name: string; readonly model: string; complete(params: LLMCompleteParams): Promise; } // ───────────────────────────────────────────────────────────────────────────── // Internal helpers shared by all providers // ───────────────────────────────────────────────────────────────────────────── /** Sleep that resolves early (reject with AbortError) if the signal aborts. */ async function abortableSleep(ms: number, signal?: AbortSignal): Promise { return new Promise((resolve, reject) => { if (signal?.aborted) return reject(new DOMException("Aborted", "AbortError")); const t = setTimeout(resolve, ms); signal?.addEventListener( "abort", () => { clearTimeout(t); reject(new DOMException("Aborted", "AbortError")); }, { once: true }, ); }); } interface RateLimitOpts { abortSignal?: AbortSignal; onRateLimit?: LLMCompleteParams["onRateLimit"]; maxRetries?: number; } /** * fetch + 429 auto-retry. Parses `Retry-After` header first, falls back to "try again in Xs" * regex in the body (Anthropic, Groq, DeepSeek, OpenAI all use one or the other). Honors abort. */ async function fetchWithRateLimitRetry( url: string, init: RequestInit, { abortSignal, onRateLimit, maxRetries = 3 }: RateLimitOpts = {}, ): Promise { let lastBody = ""; for (let attempt = 0; attempt <= maxRetries; attempt++) { const res = await fetch(url, init); if (res.status !== 429) return res; lastBody = await res.text().catch(() => ""); if (attempt === maxRetries) throw new Error(`${res.status}: ${lastBody}`); let waitSec = 10; const ra = res.headers.get("retry-after"); if (ra && !isNaN(+ra)) waitSec = Math.max(+ra, 0.5); else { const m = lastBody.match(/try again in ([\d.]+)\s*(ms|s)?/i); if (m) { const n = parseFloat(m[1]); waitSec = m[2] && m[2].toLowerCase() === "ms" ? n / 1000 : n; } } waitSec = Math.min(Math.max(waitSec + 0.3, 1), 60); onRateLimit?.({ waitSec, attempt: attempt + 1, maxRetries }); await abortableSleep(waitSec * 1000, abortSignal); } throw new Error(`429: ${lastBody}`); } /** Async-iterate `data: …` SSE events from a streaming Response. Handles `[DONE]` sentinel. */ async function* readSSE(response: Response): AsyncGenerator<{ event: string; data: any }> { if (!response.body) return; const reader = response.body.getReader(); const dec = new TextDecoder(); let buf = ""; while (true) { const { done, value } = await reader.read(); if (done) break; buf += dec.decode(value, { stream: true }); let i: number; while ((i = buf.indexOf("\n\n")) >= 0) { const raw = buf.slice(0, i); buf = buf.slice(i + 2); let evName = ""; let dataStr = ""; for (const line of raw.split("\n")) { if (line.startsWith("data: ")) dataStr += (dataStr ? "\n" : "") + line.slice(6); else if (line.startsWith("event: ")) evName = line.slice(7).trim(); } if (!dataStr) continue; if (dataStr === "[DONE]") return; try { yield { event: evName, data: JSON.parse(dataStr) }; } catch { /* malformed event — skip */ } } } } /** * Run a streaming HTTP request with network-blip retry. The supplied `runOnce` callback performs * one full fetch + SSE-read and either returns the assembled LLMResponse or throws. If it throws * BEFORE the `receivedAnyContent` ref is flipped, we retry the whole thing. After first content, * we surface the error (retrying would duplicate text in the consumer's UI). */ async function withNetworkRetry( runOnce: (state: { receivedAnyContent: boolean }) => Promise, { abortSignal, providerName, maxRetries = 2 }: { abortSignal?: AbortSignal; providerName: string; maxRetries?: number }, ): Promise { let lastErr: unknown; for (let attempt = 0; attempt <= maxRetries; attempt++) { const state = { receivedAnyContent: false }; try { return await runOnce(state); } catch (e) { lastErr = e; if ((e as { name?: string })?.name === "AbortError") throw e; if (state.receivedAnyContent) throw e; if (attempt >= maxRetries) throw e; // eslint-disable-next-line no-console console.warn( `[${providerName}] stream failed (attempt ${attempt + 1}/${maxRetries + 1}), retrying:`, (e as Error)?.message, ); await abortableSleep(400 * Math.pow(2, attempt), abortSignal); } } throw lastErr; } // ───────────────────────────────────────────────────────────────────────────── // AnthropicProvider // ───────────────────────────────────────────────────────────────────────────── export interface AnthropicProviderConfig { apiKey: string; /** Default `claude-haiku-4-5-20251001`. Override with `claude-sonnet-4-6` for harder composition. */ model?: string; /** Override the API base URL. Use a proxy in production. Default: "https://api.anthropic.com". */ apiBaseUrl?: string; /** Set to true if you're calling this directly from a browser (key will be exposed). */ dangerousDirectBrowserAccess?: boolean; } export class AnthropicProvider implements LLMProvider { readonly name = "anthropic"; readonly model: string; private readonly apiKey: string; private readonly apiBaseUrl: string; private readonly directBrowser: boolean; constructor(cfg: AnthropicProviderConfig) { if (!cfg.apiKey) throw new Error("AnthropicProvider: apiKey required"); this.apiKey = cfg.apiKey; this.model = cfg.model ?? "claude-haiku-4-5-20251001"; this.apiBaseUrl = (cfg.apiBaseUrl ?? "https://api.anthropic.com").replace(/\/+$/, ""); this.directBrowser = cfg.dangerousDirectBrowserAccess === true; } async complete(p: LLMCompleteParams): Promise { const headers: Record = { "x-api-key": this.apiKey, "anthropic-version": "2023-06-01", "content-type": "application/json", }; if (this.directBrowser) headers["anthropic-dangerous-direct-browser-access"] = "true"; const init: RequestInit = { method: "POST", signal: p.abortSignal, headers, body: JSON.stringify({ model: this.model, max_tokens: p.maxTokens, stream: true, system: p.system, tools: p.tools, messages: p.messages.map(toAnthropicMessage), }), }; return withNetworkRetry( async (state) => { const res = await fetchWithRateLimitRetry(`${this.apiBaseUrl}/v1/messages`, init, { abortSignal: p.abortSignal, onRateLimit: p.onRateLimit, }); if (!res.ok) { const body = await res.text().catch(() => ""); throw new Error(`Anthropic ${res.status} ${res.statusText}: ${body}`); } const blocks: AgentBlock[] = []; const building: Record = {}; let stopReason: AgentStopReason = null; const usage = { input: 0, output: 0 }; for await (const { event, data } of readSSE(res)) { switch (event) { case "message_start": usage.input = data.message?.usage?.input_tokens ?? 0; usage.output = data.message?.usage?.output_tokens ?? 0; break; case "content_block_start": { const idx = data.index; if (data.content_block?.type === "text") { building[idx] = { type: "text", text: "" }; } else if (data.content_block?.type === "tool_use") { building[idx] = { type: "tool_call", id: data.content_block.id, name: data.content_block.name, _argJson: "", }; } break; } case "content_block_delta": { const cur = building[data.index]; if (!cur) break; if (data.delta?.type === "text_delta") { cur.text += data.delta.text; state.receivedAnyContent = true; p.onTextDelta?.(data.delta.text); } else if (data.delta?.type === "input_json_delta") { cur._argJson += data.delta.partial_json; state.receivedAnyContent = true; } break; } case "content_block_stop": { const cur = building[data.index]; if (!cur) break; if (cur.type === "tool_call") { try { cur.input = cur._argJson ? JSON.parse(cur._argJson) : {}; } catch { cur.input = { __raw_arguments: cur._argJson }; } delete cur._argJson; } blocks[data.index] = cur; break; } case "message_delta": if (data.delta?.stop_reason) stopReason = data.delta.stop_reason as AgentStopReason; if (data.usage?.output_tokens != null) usage.output = data.usage.output_tokens; break; case "message_stop": default: break; } } return { content: blocks.filter(Boolean), stopReason, usage }; }, { abortSignal: p.abortSignal, providerName: this.name }, ); } } function toAnthropicMessage(m: AgentMessage): { role: "user" | "assistant"; content: unknown[] } { return { role: m.role, content: m.content.map((b) => { if (b.type === "text") return { type: "text", text: b.text }; if (b.type === "tool_call") return { type: "tool_use", id: b.id, name: b.name, input: b.input }; return { type: "tool_result", tool_use_id: b.toolCallId, content: b.content, is_error: b.isError || undefined, }; }), }; } // ───────────────────────────────────────────────────────────────────────────── // OpenAI-compatible providers // ───────────────────────────────────────────────────────────────────────────── export interface OpenAICompatProviderConfig { name: string; apiKey: string; model: string; apiBaseUrl: string; } /** * Base class for any OpenAI-compatible chat-completions provider. Subclasses below (Groq, * DeepSeek, OpenAI) just set name + apiBaseUrl + default model — the wire format is identical * across all three. */ export class OpenAICompatProvider implements LLMProvider { readonly name: string; readonly model: string; protected readonly apiKey: string; protected readonly apiBaseUrl: string; constructor(cfg: OpenAICompatProviderConfig) { if (!cfg.apiKey) throw new Error(`${cfg.name}: apiKey required`); this.name = cfg.name; this.model = cfg.model; this.apiKey = cfg.apiKey; this.apiBaseUrl = cfg.apiBaseUrl.replace(/\/+$/, ""); } async complete(p: LLMCompleteParams): Promise { // ─── Tool-name sanitization ──────────────────────────────────────────── // Strict OpenAI-compat APIs (DeepSeek, some Codex / GPT-5 deploys) enforce the function // name regex ^[a-zA-Z0-9_-]+$. Our research toolkit uses dotted names like // "lucidly.policies" which DeepSeek rejects with HTTP 400. We sanitize names to the API // on the way out and un-sanitize names from the response on the way back so the rest of // the agent loop (dispatch, trace, history) still uses the canonical dotted names. // Anthropic and Groq accept dots, but sanitization is harmless for them too — we keep the // logic universal in OpenAICompatProvider rather than gating per-subclass. const sanitize = (n: string) => n.replace(/[^a-zA-Z0-9_-]/g, "_"); const nameMap: Record = {}; for (const t of p.tools) { const safe = sanitize(t.name); // Only register the FIRST tool that sanitizes to a given key. If two tools collide // (e.g. "foo.bar" and "foo_bar"), the second sanitization is a no-op for the second // one but the un-mapping would pick the first canonical. Practical collision risk is // ~zero given our tool naming conventions; if it happens, we'd surface it in tests. if (!(safe in nameMap)) nameMap[safe] = t.name; } const oaiMessages: unknown[] = [{ role: "system", content: p.system }]; for (const m of p.messages) { if (m.role === "user") { const texts = m.content.filter((b): b is Extract => b.type === "text"); const results = m.content.filter( (b): b is Extract => b.type === "tool_result", ); if (texts.length) oaiMessages.push({ role: "user", content: texts.map((b) => b.text).join("\n") }); for (const tr of results) oaiMessages.push({ role: "tool", tool_call_id: tr.toolCallId, content: tr.content }); } else { const texts = m.content.filter((b): b is Extract => b.type === "text"); const calls = m.content.filter((b): b is Extract => b.type === "tool_call"); const msg: Record = { role: "assistant", content: texts.length ? texts.map((b) => b.text).join("\n") : null, }; if (calls.length) { msg.tool_calls = calls.map((tc) => ({ id: tc.id, type: "function", // Sanitize history tool_call names so the model's view of past turns matches // the tools array currently in play (also sanitized). function: { name: sanitize(tc.name), arguments: JSON.stringify(tc.input) }, })); } oaiMessages.push(msg); } } const oaiTools = p.tools.map((t) => ({ type: "function", function: { name: sanitize(t.name), description: t.description, parameters: t.input_schema }, })); const init: RequestInit = { method: "POST", signal: p.abortSignal, headers: { authorization: `Bearer ${this.apiKey}`, "content-type": "application/json" }, body: JSON.stringify({ model: this.model, messages: oaiMessages, tools: oaiTools, max_tokens: p.maxTokens, stream: true, // Without this opt-in, OpenAI-compat providers don't emit usage in stream responses. stream_options: { include_usage: true }, }), }; return withNetworkRetry( async (state) => { const res = await fetchWithRateLimitRetry(`${this.apiBaseUrl}/v1/chat/completions`, init, { abortSignal: p.abortSignal, onRateLimit: p.onRateLimit, }); if (!res.ok) { const body = await res.text().catch(() => ""); throw new Error(`${this.name} ${res.status} ${res.statusText}: ${body}`); } let textContent = ""; const toolBuilders: Record = {}; let finishReason: string | null = null; const usage = { input: 0, output: 0 }; for await (const { data } of readSSE(res)) { if (data.usage) { usage.input = data.usage.prompt_tokens ?? usage.input; usage.output = data.usage.completion_tokens ?? usage.output; } const choice = data.choices?.[0]; if (!choice) continue; if (typeof choice.delta?.content === "string" && choice.delta.content.length) { textContent += choice.delta.content; state.receivedAnyContent = true; p.onTextDelta?.(choice.delta.content); } for (const tc of choice.delta?.tool_calls || []) { const idx = tc.index ?? 0; if (!toolBuilders[idx]) toolBuilders[idx] = { id: "", name: "", args: "" }; if (tc.id) toolBuilders[idx].id = tc.id; if (tc.function?.name) toolBuilders[idx].name = tc.function.name; if (tc.function?.arguments) toolBuilders[idx].args += tc.function.arguments; state.receivedAnyContent = true; } if (choice.finish_reason) finishReason = choice.finish_reason; } const blocks: AgentBlock[] = []; if (textContent) blocks.push({ type: "text", text: textContent }); for (const idx of Object.keys(toolBuilders) .map((k) => +k) .sort((a, b) => a - b)) { const b = toolBuilders[idx]; let input: Record = {}; try { input = b.args ? JSON.parse(b.args) : {}; } catch { input = { __raw_arguments: b.args }; } // Map sanitized name back to the canonical dotted name so dispatchLucidlyTool sees // the right key. Falls back to the raw name when the model emits something we don't // recognize — the dispatcher's "unknown tool" error then propagates correctly. const canonicalName = nameMap[b.name] ?? b.name; blocks.push({ type: "tool_call", id: b.id, name: canonicalName, input }); } const sr: AgentStopReason = ({ stop: "end_turn", tool_calls: "tool_use", length: "max_tokens", content_filter: "stop_sequence" } as const)[ finishReason as string ] ?? null; return { content: blocks, stopReason: sr, usage }; }, { abortSignal: p.abortSignal, providerName: this.name }, ); } } export interface GroqProviderConfig { apiKey: string; model?: string; apiBaseUrl?: string; } export class GroqProvider extends OpenAICompatProvider { constructor(cfg: GroqProviderConfig) { super({ name: "groq", apiKey: cfg.apiKey, model: cfg.model ?? "llama-3.3-70b-versatile", apiBaseUrl: cfg.apiBaseUrl ?? "https://api.groq.com/openai", }); } } export interface DeepSeekProviderConfig { apiKey: string; /** Default `deepseek-chat`. Set `deepseek-reasoner` for R1 (reasoning trace currently ignored). */ model?: string; apiBaseUrl?: string; } export class DeepSeekProvider extends OpenAICompatProvider { constructor(cfg: DeepSeekProviderConfig) { super({ name: "deepseek", apiKey: cfg.apiKey, model: cfg.model ?? "deepseek-chat", apiBaseUrl: cfg.apiBaseUrl ?? "https://api.deepseek.com", }); } } export interface OpenAIProviderConfig { apiKey: string; /** Default `gpt-4.1`. Set to `gpt-5`, `gpt-5-codex`, `o3`, `o4-mini` etc. if you have access. */ model?: string; apiBaseUrl?: string; } export class OpenAIProvider extends OpenAICompatProvider { constructor(cfg: OpenAIProviderConfig) { super({ name: "openai", apiKey: cfg.apiKey, model: cfg.model ?? "gpt-4.1", apiBaseUrl: cfg.apiBaseUrl ?? "https://api.openai.com", }); } }