/**
 * Evaluation of whole conversations. A simulated user holds a conversation with an agent for each
 * goal in a dataset, and thread evaluators score the conversation: whether the goal was met, each
 * turn, and the turn where it went wrong. A multi-turn agent is then compared between versions as a
 * single-turn one is, with `evaluate()` and `compareExperiments()`.
 */
import type { LLMJudge } from '../evals/judge.js';
import type { Evaluator } from '../types/evaluate.js';
import type { CompletionRequest, Message } from '../types/messages.js';
import type { EvaluationTarget } from './run.js';
/** One turn of a conversation. */
export interface ThreadTurn {
    /** Who spoke. */
    role: 'user' | 'assistant';
    /** What they said. */
    content: string;
    /** For an assistant turn, how long the agent took, in milliseconds. */
    latencyMs?: number;
    /**
     * For an assistant turn, every message the agent added: its tool calls, their results, and the
     * answer. The agent is given them back on later turns, and an evaluator can read its trajectory.
     */
    messages?: Message[];
}
/** Why a conversation ended. */
export type ThreadEnding = 'goal' | 'gave-up' | 'max-turns' | 'error';
/** A finished conversation: what a conversation target returns, and what thread evaluators read. */
export interface Thread {
    /** Every turn, in order. */
    turns: ThreadTurn[];
    /**
     * `goal` when the user said its goal was met, `gave-up` when it said it never would be, `max-turns`
     * when the turn limit ended it, and `error` when the agent failed.
     */
    ended: ThreadEnding;
    /** The agent's error, when it failed. */
    error?: string;
}
/** What a conversation is about: the user's goal, and optionally who they are and how they open. */
export interface ThreadInputs {
    /** What the user wants, in their words or the dataset's. */
    goal: string;
    /** Who the user is and how they talk, such as "an impatient customer who writes in short lines". */
    persona?: string;
    /** The user's first message, instead of letting the simulated user write it. */
    opening?: string;
}
/** What a simulated user sees when it writes its next message. */
export interface SimulatedUserContext {
    /** The goal. */
    goal: string;
    /** The persona, when the example gives one. */
    persona?: string;
    /** The conversation so far. Empty for the first message. */
    turns: readonly ThreadTurn[];
    /** Aborted when the evaluation is cancelled or times out. */
    signal?: AbortSignal;
}
/** A simulated user's move: its next message, or the end of the conversation and why. */
export type SimulatedUserReply = {
    message: string;
} | {
    done: 'goal' | 'gave-up';
};
/** Plays the user in a conversation. */
export interface SimulatedUser {
    /** The next move, given the conversation so far. */
    next(context: SimulatedUserContext): Promise<SimulatedUserReply> | SimulatedUserReply;
}
/** A model client, as the judges take one. */
export interface SimulatedUserClient {
    /** Runs one completion. */
    complete(request: CompletionRequest): Promise<{
        content: string;
    }>;
}
/** Options for `simulatedUser()`. */
export interface SimulatedUserOptions {
    /** The client the user's model is called through. */
    client: SimulatedUserClient;
    /** The model that plays the user. */
    model: string;
    /** A persona for every conversation, when the examples give none. */
    persona?: string;
    /** Sampling temperature. Defaults to 0, so a conversation repeats. */
    temperature?: number;
    /** Output token limit for each message. Defaults to 300. */
    maxTokens?: number;
    /** Replaces the instructions the model is given. `{goal}` and `{persona}` are filled in. */
    instructions?: string;
}
/**
 * A simulated user driven by a model: it is told the goal and the persona, sees the conversation
 * with the roles turned around, and writes the next message, or says it is done and why.
 */
export declare function simulatedUser(options: SimulatedUserOptions): SimulatedUser;
/**
 * A user that follows a script, for deterministic tests: its messages in order, then `done`. A
 * function decides each move itself.
 */
export declare function scriptedUser(script: {
    messages: readonly string[];
    done?: 'goal' | 'gave-up';
} | SimulatedUser['next']): SimulatedUser;
/** What a thread agent is told besides the conversation. */
export interface ThreadAgentContext<I extends ThreadInputs = ThreadInputs> {
    /** The example's inputs. */
    inputs: I;
    /** The turn, counted from 1. */
    turn: number;
    /** Names this conversation, the same on every turn and new for every conversation: a thread id. */
    conversationId: string;
    /** Aborted when the evaluation is cancelled or times out. */
    signal?: AbortSignal;
}
/**
 * What a thread agent answers: the reply, or the reply and every message the agent added on the way,
 * such as its tool calls and their results. Those are given back to it on later turns.
 */
export type ThreadAgentReply = string | {
    content: string;
    messages?: Message[];
};
/**
 * The agent a conversation target talks to. It is given the conversation so far, with its own earlier
 * turns as it recorded them, and answers the user's newest message.
 */
export type ThreadAgent<I extends ThreadInputs = ThreadInputs> = (messages: readonly Message[], context: ThreadAgentContext<I>) => Promise<ThreadAgentReply> | ThreadAgentReply;
/** Options for `conversationTarget()`. */
export interface ConversationTargetOptions<I extends ThreadInputs = ThreadInputs> {
    /** The user, or a user per example, such as one with the example's own persona. */
    user: SimulatedUser | ((inputs: I) => SimulatedUser);
    /** The most user messages a conversation may have. Defaults to 10. */
    maxTurns?: number;
}
/**
 * An `evaluate()` target that holds a conversation per example: the simulated user speaks, the agent
 * answers, until the user says it is done or the turn limit is reached. It returns the `Thread`.
 */
export declare function conversationTarget<I extends ThreadInputs = ThreadInputs>(agent: ThreadAgent<I>, options: ConversationTargetOptions<I>): EvaluationTarget<I>;
/** A graph a thread agent can drive: one that takes `{ messages }`, as `createAgent()` builds. */
export interface ThreadGraph {
    /** Runs one turn. */
    invoke(input: {
        messages: Message[];
    }, options?: {
        threadId?: string;
        signal?: AbortSignal;
    }): Promise<{
        state: unknown;
    }>;
}
/**
 * An agent from `createAgent()`, or any graph that takes `{ messages }` and ends with an `answer`, as
 * a thread agent. Each turn runs on the conversation's thread with the transcript so far, and the
 * messages the run added to its `messages` are recorded with the turn. The agent therefore sees its
 * own tool calls on later turns, as it does with a real user, without needing a checkpointer.
 */
export declare function graphThreadAgent(graph: ThreadGraph): ThreadAgent;
/** The conversation as text, for a judge or a person. */
export declare function formatThread(thread: Thread): string;
/**
 * Whether the user said its goal was met: 1 when the conversation ended on `goal`, otherwise 0, with
 * how it ended as the comment. Key `goal_reached`.
 */
export declare function goalReached(): Evaluator<ThreadInputs>;
/**
 * Whether the goal was met, in a judge's opinion of the whole conversation, not the user's. Key
 * `goal_completion`, with the judge's rationale as the comment.
 */
export declare function goalCompletion(judge: Pick<LLMJudge, 'evaluate'>): Evaluator<ThreadInputs>;
/** How many times the assistant spoke: fewer is better. Key `turns`; compare with `lowerIsBetter: ['turns']`. */
export declare function turnCount(): Evaluator<ThreadInputs>;
/** Options for `turnScores()`. */
export interface TurnScoresOptions {
    /** The score's key. Defaults to `turn_score`. */
    key?: string;
    /** A turn scoring below this went wrong. Defaults to 0.5. */
    threshold?: number;
}
/**
 * Scores every assistant turn with an evaluator of single answers, and the conversation by their
 * mean. The evaluator sees each turn as an example of its own: the user's message as the input, the
 * conversation before it in `metadata.history`, and the turn as the output. The score's metadata holds
 * each turn's score, and `wentWrongAt`, the first turn below the threshold, counted from 1.
 */
export declare function turnScores<I = unknown>(evaluator: Evaluator<I>, options?: TurnScoresOptions): Evaluator<ThreadInputs>;