import { MiddlewareTripContext } from "../../contracts/middleware/middleware-context.type.mjs"; import { AgentMiddleware } from "../../contracts/middleware/middleware.contract.mjs"; import { EmbedderContract } from "../../contracts/embedder.contract.mjs"; import { CacheDriver } from "@warlock.js/cache"; //#region ../ai/src/middleware/builtins/semantic-cache.d.ts /** * Isolation boundary for cache reads and writes. * * - `"session"` (default) — key every entry off the run's * `AgentExecuteOptions.sessionId`, so one session never receives a * response cached for another. Calls made WITHOUT a `sessionId` share * one unscoped pool (the pre-4.15.0 behavior); an unscoped read never * sees a session-scoped entry and vice versa. * - `"shared"` — one pool for every caller, regardless of session. The * explicit opt-in for genuinely public Q&A (docs bots, FAQ) where the * cross-user hit rate is the point and no response can carry one * caller's private context. * - a resolver — derive the key yourself, e.g. per tenant * (`ctx => ctx.options?.toolCtx?.tenantId`). Returning `undefined` * falls back to the unscoped pool, so return a constant sentinel (or * throw) if you need the call to fail closed instead. */ type SemanticCacheScope = "session" | "shared" | ((context: MiddlewareTripContext) => string | undefined); /** * Configuration for `semanticCache()`. */ type SemanticCacheOptions = { /** Embedder used to produce the query vector from the prompt text. */embedder: EmbedderContract; /** * Vector-capable cache driver from `@warlock.js/cache`. Production * deployments pick a driver with a real ANN index (`pg` with * pgvector, `redis` with RediSearch). Dev / test environments use * `new MemoryCacheDriver()` — zero config, correct, but O(N) per * query. Drivers without similarity support throw * `CacheUnsupportedError` from `set({ vector })` / `similar()`. * * Falls back to `ai.config({ defaultStore })` when omitted. When * neither is set, the factory throws at construction time — * semantic cache cannot operate without a store. */ store?: CacheDriver; /** * Minimum cosine similarity for a vector hit. Between 0 and 1 — * 0.95 is a solid default for question-answering caches. */ threshold: number; /** * Optional TTL in milliseconds. Entries whose `storedAt` is older * than this are treated as misses on read and overwritten on the * next write. Default: no expiry — entries live until the store * evicts them (per its own TTL/eviction policy). */ ttlMs?: number; /** * Namespace prefix applied to every key the cache writes. Lets * multiple agents share one driver without collision. Default * `"ai.cache"`. */ namespace?: string; /** * Per-caller isolation boundary. Default `"session"` — a cached * response is served back only to the session that produced it. * * A `semanticCache` is normally built once at app boot and shared by * every end user, and a hit is returned as the model's answer with no * LLM call in between; without a scope that pools every caller's Q&A * pairs into one namespace, which is both a disclosure path (user B's * near-enough prompt gets served user A's answer, personal context * included) and a poisoning path (an attacker seeds an entry near a * predictable future query). Set `"shared"` to opt back into pooling * where that is actually desirable. See {@link SemanticCacheScope}. */ scope?: SemanticCacheScope; /** * Middleware name — also the state-bag key prefix inside a single * execution. Default `"semantic-cache"`. */ name?: string; }; /** * Semantic-similarity response cache for an agent run. * * **Role.** Skips LLM round-trips when the current prompt is * semantically close to one the agent has already answered. For * FAQ / support-style traffic this often eliminates 60–80% of * model calls — the production win is massive for cost and * latency. * * **Delegation to `@warlock.js/cache`.** This middleware does NOT * implement similarity search itself. It delegates to the supplied * `CacheDriver`. Production deployments pick a driver with an ANN * index (`pg` + pgvector, `redis` + RediSearch). Dev / test * environments pass `new MemoryCacheDriver()` — zero config, correct, * but O(N) per query. Drivers without similarity support throw * `CacheUnsupportedError` from `set({ vector })` / `similar()`. * * **Two-tier lookup.** * 1. *Exact-match key* — a cheap FNV hash over the entire message * list. `store.get(hash)` returns the entry without an embedding * round trip when the prompt hasn't changed at all. * 2. *Vector-match* — on exact-match miss, embed the prompt and * call `store.similar(vector, { topK: 1, threshold })`. The * driver uses its native similarity index; anything clearing * `threshold` is returned as a hit. * * **Write-on-miss.** When both tiers miss, `trip.before` stashes * the prompt hash + vector in `ctx.state`; `trip.after` reads back * the pending entry and calls * `store.set(hash, entry, { vector })`. If an outer middleware * (guardrail) throws in `trip.after` before the cache's `trip.after` * runs, the pending entry is never written — bad responses stay out * of the cache **as long as the canonical install order is followed** * (cache outermost). * * **Synthetic-response on hit.** Returns a `ModelResponse` with * `usage: { input: 0, output: 0, total: 0 }` so budget / * observability correctly exclude the saved trip. * * **Per-session scoping (4.15.0).** One `semanticCache` instance * normally serves every end user, and a hit is returned as the answer * with no model call in between — so entries are keyed by the run's * `sessionId` (`scope`, default `"session"`) and a lookup only ever * sees entries written under the same key. Runs made without a * `sessionId` share one unscoped pool; pass `sessionId` on * `agent.execute()` (composites thread their own through automatically) * to get the isolation, or set `scope: "shared"` to pool deliberately. * Note the cost/benefit shift: scoping trades cross-user hit rate for * isolation, so public-FAQ deployments where no response can carry a * caller's private context should opt into `"shared"` explicitly. * * @example * import { semanticCache } from "@warlock.js/ai"; * import { MemoryCacheDriver } from "@warlock.js/cache"; * * const store = new MemoryCacheDriver(); * store.setOptions({}); * * const cache = semanticCache({ * embedder: openai.embedder({ name: "text-embedding-3-small" }), * store, * threshold: 0.95, * ttlMs: 60 * 60 * 1000, * }); * * const myAgent = agent({ model, middleware: [cache] }); */ declare function semanticCache(options: SemanticCacheOptions): AgentMiddleware; //#endregion export { SemanticCacheOptions, SemanticCacheScope, semanticCache }; //# sourceMappingURL=semantic-cache.d.mts.map