/** * Multi-provider LLM router with circuit-breaker, total-budget, and * pluggable per-call ledger (v0.9). * * const breaker = new CircuitBreaker(); * const ledger = new AICallLedger({ db: env.DB }); * const router = new AIRouter({ * providers: { * groq_8b: () => new OpenAICompatProvider({ * name: 'groq_8b', * url: 'https://api.groq.com/openai/v1/chat/completions', * apiKey: env.GROQ_API_KEY, * model: 'llama-3.1-8b-instant', * }), * workers_ai: () => new WorkersAIProvider({ * name: 'workers_ai', * ai: env.AI, * model: '@cf/meta/llama-3.1-8b-instruct-fast', * }), * }, * resolveChain: (task) => ['groq_8b', 'workers_ai'], * breaker, * ledger, * }); * * const r = await router.route('classifier', { system: '...', user: 'hi' }); * if (r.ok) console.log(r.response); * * The router walks the chain in order, skipping providers whose breaker is * OPEN and providers that would exceed the total wall-clock budget. Each * attempt is logged via the ledger when one is configured. * * Cost estimation is the caller's responsibility — pass `estimateCost` on * construction (typically backed by `LLMCostCalculator` or a static price * table) to get per-call µUSD in the ledger rows. */ import type { LLMProvider, ProviderErrorKind } from './llm_provider.js'; import type { CircuitBreaker } from './circuit_breaker.js'; import type { AICallLedger, CallStatus } from './ai_call_log.js'; type ProviderFactory = LLMProvider | (() => LLMProvider); export interface AIRouterOptions { /** * Map of provider name → factory OR instance. Factories are invoked once * per route call (cheap; usually wraps env-bound constructors). Instances * are reused as-is. */ providers: Record; /** * Resolve a task label to an ordered provider chain. Return [] to * short-circuit (router returns `errorKind: 'config'`). * * Typical implementations read env vars (`env.AI_CHAIN_${task}`) and/or * a per-tenant override D1 table. */ resolveChain: (task: string) => Promise | readonly string[]; /** * Per-provider circuit breaker. Optional — when omitted, every provider * is tried regardless of past failures. */ breaker?: CircuitBreaker | null; /** * Persistent ledger of every attempt. Optional — when omitted, the router * only emits a structured stdout log per attempt. */ ledger?: AICallLedger | null; /** Per-call timeout passed to the provider's `run()`. Default 3000ms. */ timeoutMs?: number; /** * Total wall-clock budget across the chain. When exceeded, remaining * providers are skipped with status `skipped_budget`. Default 9000ms. */ totalBudgetMs?: number; /** * Optional cost estimator: `(provider, tokensIn, tokensOut) → micro-USD`. * The result is forwarded to the ledger's `estCostMicroUsd`. When absent, * the ledger column is NULL. */ estimateCost?: (provider: string, tokensIn: number | null, tokensOut: number | null) => number | null; /** `Date.now`-compatible clock (test override). */ now?: () => number; } export interface RouteArgs { system: string; user: string; maxTokens?: number; temperature?: number; /** Forwarded to ledger rows for per-tenant analytics. */ tenantId?: string | null; /** Forwarded to ledger rows for per-user debug. */ whatsapp?: string | null; /** * v0.16: derive extra ledger columns from the winning provider's raw * response. Fires on SUCCESS only. Return keys must be listed in the * ledger's `allowedExtraColumns` — the callback is a cheap way to * inline classifier category / intent / route tag into `ai_call_log` * without a follow-up UPDATE. Errors from the callback are swallowed * (never fail the route because logging failed). */ extraLogFields?: (response: string) => Record; } export interface ChainAttempt { provider: string; status: CallStatus; latencyMs?: number; } export interface RouteSuccess { ok: true; provider: string; response: string; model: string; tokensIn: number | null; tokensOut: number | null; latencyMs: number; chainTried: ChainAttempt[]; } export interface RouteFailure { ok: false; errorKind: ProviderErrorKind | 'config' | 'all_failed'; errorMessage: string; chainTried: ChainAttempt[]; } export type RouteResult = RouteSuccess | RouteFailure; export declare class AIRouter { readonly providers: Record; readonly resolveChain: AIRouterOptions['resolveChain']; readonly breaker: CircuitBreaker | null; readonly ledger: AICallLedger | null; readonly timeoutMs: number; readonly totalBudgetMs: number; readonly estimateCost: AIRouterOptions['estimateCost']; readonly now: () => number; constructor(opts: AIRouterOptions); route(task: string, args: RouteArgs): Promise; private logAttempt; } /** * Common pattern for `resolveChain`: read `env.AI_CHAIN_${task.toUpperCase()}` * as a comma-separated provider list. Returns an empty array when the env * var is missing or empty. */ export declare function envChainResolver(env: Record): (task: string) => string[]; /** * v0.17: D1-backed chain resolver with per-isolate cache. Reads an * `(task, chain)` override from a table (default `ai_provider_overrides`), * falling back to a secondary resolver (typically `envChainResolver(env)`) * when no row exists for the task. * * The cache lives per Cloudflare Workers isolate (module-scoped Map). Rows * fetched from D1 are memoized for `cacheMs` (default 60s) — hot-path calls * are one Map lookup, not a D1 query. Any live update to the overrides table * takes effect within `cacheMs` on every isolate. * * Table shape (create this migration in your consumer — the framework does * NOT ship it because column names / ownership vary): * * CREATE TABLE ai_provider_overrides ( * task TEXT PRIMARY KEY, * chain TEXT NOT NULL, -- comma-separated provider names * updated_at TEXT DEFAULT CURRENT_TIMESTAMP * ); * * Usage: * * const router = new AIRouter({ * providers, * resolveChain: createD1ChainResolver({ * db: env.DB, * fallback: envChainResolver(env), * cacheMs: 60_000, * }), * ... * }); */ export interface CreateD1ChainResolverOptions { /** * D1Database or Drizzle client. The resolver only needs `prepare().bind().first()`, * so a raw `D1Database` works fine. */ db: { prepare(query: string): { bind(...v: unknown[]): { first(): Promise; }; }; }; /** * Secondary resolver — used when no D1 row exists for a task. Typical: * `envChainResolver(env)`. Absent → the D1 miss yields `[]` and the * router returns `errorKind: 'config'`. */ fallback?: (task: string) => readonly string[] | Promise; /** Table name. Default `ai_provider_overrides`. */ tableName?: string; /** Column holding the task label. Default `task`. */ taskColumn?: string; /** Column holding the comma-separated chain. Default `chain`. */ chainColumn?: string; /** Cache TTL in ms. Default 60_000 (60s). Set 0 to disable cache. */ cacheMs?: number; /** Injectable clock for tests. */ now?: () => number; } export declare function createD1ChainResolver(opts: CreateD1ChainResolverOptions): (task: string) => Promise; export {}; //# sourceMappingURL=router.d.ts.map