import type { BaseMessage } from "@langchain/core/messages"; import { ChatCompletionMessage, EvaluatorResult, SimpleEvaluator, Messages, MultiturnSimulationResult } from "../types.js"; export { MultiturnSimulationResult }; export type MultiturnSimulationParams = { app: (params: { inputs: ChatCompletionMessage; threadId: string; }) => ChatCompletionMessage | BaseMessage | Promise; user: ((params: { trajectory: ChatCompletionMessage[]; turnCounter: number; }) => ChatCompletionMessage | BaseMessage | Promise) | (string | Messages)[]; maxTurns?: number; trajectoryEvaluators?: SimpleEvaluator[]; stoppingCondition?: (params: { trajectory: ChatCompletionMessage[]; turnCounter: number; threadId: string; }) => boolean | Promise; referenceOutputs?: unknown; threadId?: string; }; /** * Run a simulation for multi-turn conversations between an application and a simulated user. * * This function runs a simulation between an app and * either a dynamic user simulator or a list of static user responses. The simulation supports * evaluation of conversation trajectories and customizable stopping conditions. * * Conversation trajectories are represented as lists of message objects with "role" and "content" keys. * The "app" param you provide will receive the next message in sequence as an input, and should * return a message. Internally, the simulation will dedupe these messages by id and merge them into * a complete trajectory. * * Once "maxTurns" is reached or a provided stopping condition is met, the final trajectory * will be passed to provided trajectory evaluators, which will receive the final trajectory * as an "outputs" param. * * When calling the created simulator, you may also provide a "referenceOutputs" param, * which will be passed directly through to the provided evaluators. * * @param {Object} params - Configuration parameters for the simulator * @param {(params: {inputs: ChatCompletionMessage, threadId: string}) => ChatCompletionMessage | Promise} params.app - Your application. Can be either a LangChain Runnable or a * callable that takes the current conversation trajectory and returns * a message. * @param {(params: {trajectory: ChatCompletionMessage[], turnCounter: number, threadId: string}) => ChatCompletionMessage | Promise | (string | Messages)[]} params.user - The simulated user. Can be: * - A function that takes the current conversation trajectory and returns a message. * - A list of strings or Messages representing static user responses * @param {number} [params.maxTurns] - Maximum number of conversation turns to simulate * @param {SimpleEvaluator[]} [params.trajectoryEvaluators] - Optional list of evaluator functions that assess the conversation * trajectory. Each evaluator will receive the final trajectory of the conversation as * a param named "outputs" and a param named "referenceOutputs" if provided. * @param {(params: {trajectory: ChatCompletionMessage[], turnCounter: number, threadId: string}) => boolean | Promise} [params.stoppingCondition] - Optional callable that determines if the simulation should end early. * Takes the current trajectory as input and returns a boolean. * @param {unknown} [params.referenceOutputs] - Optional reference outputs for evaluation * @param {string} [params.threadId] - Optional thread ID. If not provided, a random one will be generated. * * @returns Returns a Promise that resolves to a MultiturnSimulationResult containing: * - evaluatorResults: List of results from trajectory evaluators * - trajectory: The complete conversation trajectory * * @example * ```typescript * import { runMultiturnSimulation } from "openevals"; * * // Create a simulator with static user responses * const result = runMultiturnSimulation({ * app: myChatApp, * user: ["Hello!", "How are you?", "Goodbye"], * maxTurns: 3, * trajectoryEvaluators: [myEvaluator] * }); * ``` */ export declare const runMultiturnSimulation: (params: MultiturnSimulationParams) => Promise<{ trajectory: ChatCompletionMessage[]; evaluatorResults: EvaluatorResult[]; }>;