import type { CompletionRequest, ToolDefinition } from '../types/messages.js'; import type { NexusResponse } from '../types/response.js'; /** The model that plays the tools: the client, or anything with its `complete()`. */ export interface EmulatorClient { /** Runs one completion. */ complete(request: CompletionRequest): Promise; } /** One call to an emulated or passed-through tool, as `onCall` reports it. */ export interface EmulatedToolCall { /** The tool's name. */ tool: string; /** The arguments the agent called it with. */ args: Record; /** What it returned, when it did not fail. */ result?: unknown; /** The error it failed with, when it did. */ error?: string; /** Who answered: the emulating model, the store, replaying an earlier answer, or the real tool. */ source: 'model' | 'store' | 'real'; /** The call's key in the store. */ key: string; } /** An emulated answer, as a store keeps it. */ export interface EmulatedResult { /** The tool's name. */ tool: string; /** The arguments it was called with. */ args: Record; /** What it returned. */ result?: unknown; /** The error it failed with. */ error?: string; } /** Where emulated answers are kept, by `emulationKey()`, so a later run replays them. */ export interface EmulationStore { /** The answer kept for a key. */ get(key: string): Promise | EmulatedResult | undefined; /** Keeps an answer. */ set(key: string, value: EmulatedResult): Promise | void; } /** Options for `emulatedTools()`. */ export interface EmulatedToolsOptions { /** The model that plays each tool. */ client: EmulatorClient; /** The model to ask. Defaults to `auto`, the client's routing. */ model?: string; /** * The world the tools act on, as the emulator reads it, such as `Order 1042 was delivered on * 3 October and cost $59.` Part of each call's key, so one store holds answers for many scenarios. */ scenario?: string; /** Each tool's own notes, by name: what it returns, when it fails, and its rules. */ instructions?: Readonly>; /** Real calls to imitate, by tool: their arguments and what the real tool returned. */ examples?: Readonly; result: unknown; }>>>; /** JSON Schema of each tool's result, by name, shown to the emulator. */ results?: Readonly>>; /** * Checks each tool's emulated result, by name: `true`, or why not. A result that fails is asked * for again once, with the reason, and then fails the call with an `EmulationError`. */ validate?: Readonly true | string>>; /** Tools that run for real, by name: ones that read and change nothing. Defaults to none. */ passthrough?: readonly string[]; /** Where answers are kept and replayed from. */ store?: EmulationStore; /** * `auto`, the default with a store, replays a kept answer and asks the model for the rest, keeping * them; `replay` answers only from the store, failing a call it has no answer for; `record` asks the * model every time and keeps each answer; `emulate`, the default without a store, asks the model. */ mode?: 'auto' | 'replay' | 'record' | 'emulate'; /** * How many earlier calls of this set the emulator sees, so a refund it reported shows when the * order is looked up again. Defaults to 20; 0 shows none. */ history?: number; /** Sampling temperature for the emulator. Defaults to 0. */ temperature?: number; /** Longest answer, in tokens. */ maxTokens?: number; /** Called after each call, with who answered it. */ onCall?: (call: EmulatedToolCall) => void; /** * The world the tools act in, held across every run that shares it: each call made in it, and the * changes they made, so an order one example refunds is refunded for the next. The emulator is * told every change, and the latest `history` calls; its scenario is the tools' when they name * none. Share one world among runs made one at a time, in order; or start each run from a * `fork()`, or from a `snapshot()` read back. */ world?: EmulatedWorld; /** * Tools that change nothing, by name, beside those whose every capability reads: `filesystem:read`, * `mcp.read`, or `api.read`, or none at all. A world counts what any other tool did, when it did * not fail, as a change. */ reads?: readonly string[]; } /** One call made in an emulated world. */ export interface EmulatedWorldCall extends EmulatedResult { /** Whether it changed the world: it succeeded, and its tool does more than read. */ changed: boolean; } /** An emulated world as plain JSON, to write to a file and start runs from. */ export interface EmulatedWorldSnapshot { /** The world the tools act on, before any call. */ scenario?: string; /** Every call made in it, oldest first. */ calls: EmulatedWorldCall[]; } /** * A world emulated tools act in, kept across the runs of an evaluation: the scenario it started * from, and every call made in it, so what one run changes is there for the next, as on a staging * copy of a service. `snapshot()` writes it as JSON, `new EmulatedWorld(snapshot)` reads it back, * and `fork()` starts a world from this one's present whose changes stay its own. * * ```ts * const world = new EmulatedWorld({ scenario: 'Order 1042 is delivered and cost $59.' }); * await evaluate( * (inputs) => agentOver(emulatedTools(shop.tools, { client: ai, results: shop.results, world })).invoke(agentInput(inputs.request)), * dataset, * evaluators, * { concurrency: 1 }, * ); * ``` */ export declare class EmulatedWorld { /** The world the tools act on, before any call. */ readonly scenario?: string; private readonly log; constructor(start?: { scenario?: string; calls?: readonly EmulatedWorldCall[]; }); /** Every call made in it, oldest first. */ get calls(): readonly EmulatedWorldCall[]; /** The calls that changed it, oldest first. */ get changes(): readonly EmulatedWorldCall[]; /** Records a call made in it, and whether it changed it. */ record(call: EmulatedResult, changed: boolean): void; /** * A digest of its scenario and every change made in it, which the keys of its calls' answers carry: * two worlds that started alike and changed alike have the same digest, however much was read. */ digest(): string; /** How many calls with these arguments it has seen since its last change. */ ordinal(tool: string, args: Record): number; /** It as plain JSON, to write to a file and start runs from. */ snapshot(): EmulatedWorldSnapshot; /** A world starting from this one's present, whose changes stay its own. */ fork(): EmulatedWorld; /** The same as `snapshot()`, so `JSON.stringify(world)` writes it. */ toJSON(): EmulatedWorldSnapshot; } /** Raised when a call cannot be emulated: no kept answer in `replay`, or no valid answer from the model. */ export declare class EmulationError extends Error { /** `EMULATION_MISS` when the store has no answer, `EMULATION_INVALID` when the model gave none usable. */ readonly code: 'EMULATION_MISS' | 'EMULATION_INVALID'; /** The tool. */ readonly tool: string; /** The arguments. */ readonly args: Record; constructor( /** `EMULATION_MISS` when the store has no answer, `EMULATION_INVALID` when the model gave none usable. */ code: 'EMULATION_MISS' | 'EMULATION_INVALID', /** The tool. */ tool: string, /** The arguments. */ args: Record, message: string); } /** Emulated answers in memory, from an earlier run's `toJSON()` when given one. */ export declare class MemoryEmulationStore implements EmulationStore { private readonly entries; constructor(entries?: Readonly>); /** The answer kept for a key. */ get(key: string): EmulatedResult | undefined; /** Keeps an answer, replacing the one kept for its key. */ set(key: string, value: EmulatedResult): void; /** How many answers it keeps. */ get size(): number; /** Every answer, by key, to write to a file the next run reads back. */ toJSON(): Record; } /** * The key of a call: its tool, its arguments, its scenario, and which call with those arguments it * is, so an order looked up before and after a refund keeps both answers. In a world, also the * world's `digest()`, so the same call answers again only after the same changes. */ export declare function emulationKey(call: { tool: string; args: Record; scenario?: string; ordinal?: number; world?: string; }): string; /** * Tools that look the same to the agent, by name, description, parameters, and capabilities, but are * answered by a model playing each one, or by the answers a store kept, and never run: nothing they * would send is sent. Tools in `passthrough` run for real. A tool the emulator says fails throws its * error, so the agent sees a failure as it would from the real tool. * * ```ts * const tools = emulatedTools([chargeCard, lookupOrder], { client: ai, model: 'gpt-5-mini', scenario, store }); * const agent = createAgent({ client: ai, model: 'gpt-5', tools }); * ``` */ export declare function emulatedTools(tools: readonly ToolDefinition[], options: EmulatedToolsOptions): ToolDefinition[];