import { z } from 'zod';
import { LanguageModel } from 'ai';

declare const SEQUENCE_BRAND: unique symbol;
interface SequenceMock extends ToolMockFn {
    [SEQUENCE_BRAND]: true;
    reset(): void;
}
declare function sequence<T>(values: T[]): SequenceMock;
declare class AgentestError extends Error {
    readonly context: {
        scenario: string;
        conversationId: string;
        turnIndex: number;
        toolName?: string;
    };
    constructor(message: string, context: {
        scenario: string;
        conversationId: string;
        turnIndex: number;
        toolName?: string;
    });
}
declare class MockResolver {
    private mocks;
    private unmockedBehavior;
    private scenarioName;
    private conversationId;
    private callCounts;
    /**
     * Every unmocked call seen this conversation, recorded even though `resolve()` also
     * throws. The throw alone is not dependable: for a `type: 'custom'` agent it surfaces
     * inside the consumer's own tool implementation, and agent frameworks catch tool errors
     * by design so the model can recover from a genuinely failing tool (LangGraph's ToolNode
     * does this by default). Our error is then handed to the model as an ordinary tool
     * result and the run continues, reporting a downstream symptom instead of the cause.
     *
     * The simulator reads this ledger after the handler returns and fails the conversation
     * on the record alone — no propagation required.
     */
    private unmocked;
    constructor(mocks: Record<string, ToolMockFn> | undefined, unmockedBehavior: 'error' | 'passthrough', scenarioName: string, conversationId: string);
    /** Unmocked calls recorded this conversation, in the order they happened. */
    get unmockedCalls(): ReadonlyArray<{
        toolName: string;
        turnIndex: number;
        message: string;
    }>;
    reset(): void;
    resolve(toolName: string, args: Record<string, unknown>, turnIndex: number, 
    /**
     * OPTIONAL calls come from the HARNESS, not the agent: a synthetic marker a consumer
     * offers so a scenario CAN assert on it, which most scenarios ignore. An unmocked one is
     * not a fixture gap, so it neither throws nor lands in the ledger — otherwise every
     * optional marker silently becomes a required mock.
     */
    opts?: {
        optional?: boolean;
    }): Promise<{
        result: unknown;
        mocked: boolean;
    }>;
}

type ToolMockFn = (args: Record<string, unknown>, ctx: {
    callIndex: number;
    conversationId: string;
    turnIndex: number;
}) => unknown | Promise<unknown>;
interface ToolCallAssertion {
    name: string;
    args?: Record<string, unknown>;
    argMatchMode?: 'exact' | 'partial' | 'ignore';
}
interface ForbiddenToolAssertion {
    name: string;
    args?: Record<string, unknown>;
    argMatchMode?: 'exact' | 'partial' | 'ignore';
}
interface TrajectoryAssertions {
    matchMode: 'strict' | 'unordered' | 'contains' | 'within';
    expected: ToolCallAssertion[];
    forbidden?: ForbiddenToolAssertion[];
}
interface KnowledgeItem {
    content: string;
}
interface ScriptedTurn {
    userMessage: string;
    assertions?: {
        toolCalls?: TrajectoryAssertions;
    };
}
interface ScenarioOptions {
    /** Named agent key from config.agents to use instead of the default agent. */
    agent?: string;
    profile?: string;
    goal?: string;
    knowledge?: KnowledgeItem[];
    userPromptTemplate?: string;
    conversationsPerScenario?: number;
    maxTurns?: number;
    /** Scripted multi-turn conversation. When provided, profile/goal are optional and SimulatedUser is skipped. */
    turns?: ScriptedTurn[];
    mocks?: {
        tools?: Record<string, ToolMockFn | ReturnType<typeof sequence>>;
    };
    assertions?: {
        toolCalls?: TrajectoryAssertions;
    };
}
declare function validateScenarioOptions(options: ScenarioOptions): void;
interface Scenario {
    name: string;
    options: ScenarioOptions;
}
interface ToolCallRecord {
    name: string;
    args: Record<string, unknown>;
    result: unknown;
    turnIndex: number;
}
interface CustomHandlerContext {
    /** Resolve a tool call through the scenario's mock system. Records the call for trajectory assertions. */
    /**
     * Resolve a tool call through the scenario's mock system. Records the call for trajectory
     * assertions.
     *
     * Pass `{ optional: true }` for a call the HARNESS makes rather than the agent — a marker
     * a scenario may opt into by mocking it. An unmocked optional call resolves to undefined
     * instead of failing the conversation.
     */
    resolveTool: (name: string, args: Record<string, unknown>, opts?: {
        optional?: boolean;
    }) => Promise<unknown>;
    turnIndex: number;
    conversationId: string;
    scenarioName: string;
}

declare const DEFAULT_MODEL = "claude-sonnet-4-20250514";
type CustomAgentHandler = (messages: Array<{
    role: 'system' | 'user' | 'assistant' | 'tool';
    content: string | null;
    tool_call_id?: string;
    tool_calls?: Array<{
        id: string;
        type: 'function';
        function: {
            name: string;
            arguments: string;
        };
    }>;
}>, ctx: CustomHandlerContext) => Promise<{
    role: 'assistant';
    content: string | null;
    tool_calls?: Array<{
        id: string;
        type: 'function';
        function: {
            name: string;
            arguments: string;
        };
    }>;
}>;
declare const configSchema: z.ZodObject<{
    agent: z.ZodEffects<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
        type: z.ZodLiteral<"chat_completions">;
        name: z.ZodString;
        endpoint: z.ZodString;
        headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodEffects<z.ZodString, string, string>>>;
        body: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
        streaming: z.ZodDefault<z.ZodBoolean>;
    }, "strip", z.ZodTypeAny, {
        type: "chat_completions";
        name: string;
        endpoint: string;
        streaming: boolean;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
    }, {
        type: "chat_completions";
        name: string;
        endpoint: string;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
        streaming?: boolean | undefined;
    }>, z.ZodObject<{
        type: z.ZodLiteral<"custom">;
        name: z.ZodString;
        handler: z.ZodFunction<z.ZodTuple<[z.ZodArray<z.ZodObject<{
            role: z.ZodEnum<["system", "user", "assistant", "tool"]>;
            content: z.ZodString;
            tool_call_id: z.ZodOptional<z.ZodString>;
            tool_calls: z.ZodOptional<z.ZodArray<z.ZodObject<{
                id: z.ZodString;
                type: z.ZodLiteral<"function">;
                function: z.ZodObject<{
                    name: z.ZodString;
                    arguments: z.ZodString;
                }, "strip", z.ZodTypeAny, {
                    name: string;
                    arguments: string;
                }, {
                    name: string;
                    arguments: string;
                }>;
            }, "strip", z.ZodTypeAny, {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }, {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }>, "many">>;
        }, "strip", z.ZodTypeAny, {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }, {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>, "many">], z.ZodUnknown>, z.ZodPromise<z.ZodObject<{
            role: z.ZodEnum<["system", "user", "assistant", "tool"]>;
            content: z.ZodString;
            tool_call_id: z.ZodOptional<z.ZodString>;
            tool_calls: z.ZodOptional<z.ZodArray<z.ZodObject<{
                id: z.ZodString;
                type: z.ZodLiteral<"function">;
                function: z.ZodObject<{
                    name: z.ZodString;
                    arguments: z.ZodString;
                }, "strip", z.ZodTypeAny, {
                    name: string;
                    arguments: string;
                }, {
                    name: string;
                    arguments: string;
                }>;
            }, "strip", z.ZodTypeAny, {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }, {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }>, "many">>;
        }, "strip", z.ZodTypeAny, {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }, {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>>>;
    }, "strip", z.ZodTypeAny, {
        type: "custom";
        name: string;
        handler: (args_0: {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }[], ...args: unknown[]) => Promise<{
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>;
    }, {
        type: "custom";
        name: string;
        handler: (args_0: {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }[], ...args: unknown[]) => Promise<{
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>;
    }>]>, {
        type: "chat_completions";
        name: string;
        endpoint: string;
        streaming: boolean;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
    } | {
        type: "custom";
        name: string;
        handler: (args_0: {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }[], ...args: unknown[]) => Promise<{
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>;
    }, unknown>;
    agents: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodEffects<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
        type: z.ZodLiteral<"chat_completions">;
        name: z.ZodString;
        endpoint: z.ZodString;
        headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodEffects<z.ZodString, string, string>>>;
        body: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
        streaming: z.ZodDefault<z.ZodBoolean>;
    }, "strip", z.ZodTypeAny, {
        type: "chat_completions";
        name: string;
        endpoint: string;
        streaming: boolean;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
    }, {
        type: "chat_completions";
        name: string;
        endpoint: string;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
        streaming?: boolean | undefined;
    }>, z.ZodObject<{
        type: z.ZodLiteral<"custom">;
        name: z.ZodString;
        handler: z.ZodFunction<z.ZodTuple<[z.ZodArray<z.ZodObject<{
            role: z.ZodEnum<["system", "user", "assistant", "tool"]>;
            content: z.ZodString;
            tool_call_id: z.ZodOptional<z.ZodString>;
            tool_calls: z.ZodOptional<z.ZodArray<z.ZodObject<{
                id: z.ZodString;
                type: z.ZodLiteral<"function">;
                function: z.ZodObject<{
                    name: z.ZodString;
                    arguments: z.ZodString;
                }, "strip", z.ZodTypeAny, {
                    name: string;
                    arguments: string;
                }, {
                    name: string;
                    arguments: string;
                }>;
            }, "strip", z.ZodTypeAny, {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }, {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }>, "many">>;
        }, "strip", z.ZodTypeAny, {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }, {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>, "many">], z.ZodUnknown>, z.ZodPromise<z.ZodObject<{
            role: z.ZodEnum<["system", "user", "assistant", "tool"]>;
            content: z.ZodString;
            tool_call_id: z.ZodOptional<z.ZodString>;
            tool_calls: z.ZodOptional<z.ZodArray<z.ZodObject<{
                id: z.ZodString;
                type: z.ZodLiteral<"function">;
                function: z.ZodObject<{
                    name: z.ZodString;
                    arguments: z.ZodString;
                }, "strip", z.ZodTypeAny, {
                    name: string;
                    arguments: string;
                }, {
                    name: string;
                    arguments: string;
                }>;
            }, "strip", z.ZodTypeAny, {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }, {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }>, "many">>;
        }, "strip", z.ZodTypeAny, {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }, {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>>>;
    }, "strip", z.ZodTypeAny, {
        type: "custom";
        name: string;
        handler: (args_0: {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }[], ...args: unknown[]) => Promise<{
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>;
    }, {
        type: "custom";
        name: string;
        handler: (args_0: {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }[], ...args: unknown[]) => Promise<{
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>;
    }>]>, {
        type: "chat_completions";
        name: string;
        endpoint: string;
        streaming: boolean;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
    } | {
        type: "custom";
        name: string;
        handler: (args_0: {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }[], ...args: unknown[]) => Promise<{
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>;
    }, unknown>>>;
    model: z.ZodDefault<z.ZodString>;
    provider: z.ZodDefault<z.ZodEnum<["openai", "anthropic", "google", "ollama", "openai-compatible"]>>;
    providerOptions: z.ZodOptional<z.ZodObject<{
        baseURL: z.ZodOptional<z.ZodString>;
        apiKey: z.ZodOptional<z.ZodString>;
        extraBody: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
    }, "strip", z.ZodTypeAny, {
        baseURL?: string | undefined;
        apiKey?: string | undefined;
        extraBody?: Record<string, unknown> | undefined;
    }, {
        baseURL?: string | undefined;
        apiKey?: string | undefined;
        extraBody?: Record<string, unknown> | undefined;
    }>>;
    conversationsPerScenario: z.ZodDefault<z.ZodNumber>;
    maxTurns: z.ZodDefault<z.ZodNumber>;
    requestTimeoutMs: z.ZodDefault<z.ZodNumber>;
    concurrency: z.ZodDefault<z.ZodNumber>;
    debounceMs: z.ZodDefault<z.ZodNumber>;
    /**
     * Re-run failed scenarios up to this many times AFTER the parallel pass,
     * serially — by then resource contention (e.g. a local LLM serving
     * `concurrency` scenarios at once) has subsided, which is the typical
     * flaky-failure profile. A scenario that passes on retry counts as passed
     * and carries `retried: <attempt>` in its result.
     */
    retries: z.ZodDefault<z.ZodNumber>;
    metrics: z.ZodOptional<z.ZodArray<z.ZodEnum<["helpfulness", "coherence", "relevance", "faithfulness", "verbosity", "goal_completion", "agent_behavior_failure", "tool_call_behavior_failure"]>, "many">>;
    thresholds: z.ZodOptional<z.ZodEffects<z.ZodEffects<z.ZodRecord<z.ZodString, z.ZodNumber>, Record<string, number>, Record<string, number>>, Record<string, number>, Record<string, number>>>;
    include: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
    exclude: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
    reporters: z.ZodDefault<z.ZodArray<z.ZodEnum<["console", "json", "github-actions"]>, "many">>;
    unmockedTools: z.ZodDefault<z.ZodEnum<["error", "passthrough"]>>;
    failOnErrorSeverity: z.ZodDefault<z.ZodEnum<["low", "medium", "high", "critical"]>>;
    customMetrics: z.ZodOptional<z.ZodArray<z.ZodObject<{
        name: z.ZodString;
        setLLM: z.ZodFunction<z.ZodTuple<[], z.ZodUnknown>, z.ZodUnknown>;
    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
        name: z.ZodString;
        setLLM: z.ZodFunction<z.ZodTuple<[], z.ZodUnknown>, z.ZodUnknown>;
    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
        name: z.ZodString;
        setLLM: z.ZodFunction<z.ZodTuple<[], z.ZodUnknown>, z.ZodUnknown>;
    }, z.ZodTypeAny, "passthrough">>, "many">>;
    compare: z.ZodOptional<z.ZodArray<z.ZodUnion<[z.ZodObject<{
        name: z.ZodString;
        endpoint: z.ZodOptional<z.ZodString>;
        headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
        body: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
        streaming: z.ZodOptional<z.ZodBoolean>;
    }, "strip", z.ZodTypeAny, {
        name: string;
        endpoint?: string | undefined;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
        streaming?: boolean | undefined;
    }, {
        name: string;
        endpoint?: string | undefined;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
        streaming?: boolean | undefined;
    }>, z.ZodObject<{
        type: z.ZodLiteral<"custom">;
        name: z.ZodString;
        handler: z.ZodFunction<z.ZodTuple<[z.ZodArray<z.ZodAny, "many">], z.ZodUnknown>, z.ZodAny>;
    }, "strip", z.ZodTypeAny, {
        type: "custom";
        name: string;
        handler: (args_0: any[], ...args: unknown[]) => any;
    }, {
        type: "custom";
        name: string;
        handler: (args_0: any[], ...args: unknown[]) => any;
    }>]>, "many">>;
}, "strip", z.ZodTypeAny, {
    agent: {
        type: "chat_completions";
        name: string;
        endpoint: string;
        streaming: boolean;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
    } | {
        type: "custom";
        name: string;
        handler: (args_0: {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }[], ...args: unknown[]) => Promise<{
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>;
    };
    model: string;
    provider: "openai" | "anthropic" | "google" | "ollama" | "openai-compatible";
    conversationsPerScenario: number;
    maxTurns: number;
    requestTimeoutMs: number;
    concurrency: number;
    debounceMs: number;
    retries: number;
    include: string[];
    exclude: string[];
    reporters: ("console" | "json" | "github-actions")[];
    unmockedTools: "error" | "passthrough";
    failOnErrorSeverity: "low" | "medium" | "high" | "critical";
    agents?: Record<string, {
        type: "chat_completions";
        name: string;
        endpoint: string;
        streaming: boolean;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
    } | {
        type: "custom";
        name: string;
        handler: (args_0: {
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }[], ...args: unknown[]) => Promise<{
            role: "system" | "user" | "assistant" | "tool";
            content: string;
            tool_call_id?: string | undefined;
            tool_calls?: {
                function: {
                    name: string;
                    arguments: string;
                };
                type: "function";
                id: string;
            }[] | undefined;
        }>;
    }> | undefined;
    providerOptions?: {
        baseURL?: string | undefined;
        apiKey?: string | undefined;
        extraBody?: Record<string, unknown> | undefined;
    } | undefined;
    metrics?: ("helpfulness" | "coherence" | "relevance" | "faithfulness" | "verbosity" | "goal_completion" | "agent_behavior_failure" | "tool_call_behavior_failure")[] | undefined;
    thresholds?: Record<string, number> | undefined;
    customMetrics?: z.objectOutputType<{
        name: z.ZodString;
        setLLM: z.ZodFunction<z.ZodTuple<[], z.ZodUnknown>, z.ZodUnknown>;
    }, z.ZodTypeAny, "passthrough">[] | undefined;
    compare?: ({
        name: string;
        endpoint?: string | undefined;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
        streaming?: boolean | undefined;
    } | {
        type: "custom";
        name: string;
        handler: (args_0: any[], ...args: unknown[]) => any;
    })[] | undefined;
}, {
    agent?: unknown;
    agents?: Record<string, unknown> | undefined;
    model?: string | undefined;
    provider?: "openai" | "anthropic" | "google" | "ollama" | "openai-compatible" | undefined;
    providerOptions?: {
        baseURL?: string | undefined;
        apiKey?: string | undefined;
        extraBody?: Record<string, unknown> | undefined;
    } | undefined;
    conversationsPerScenario?: number | undefined;
    maxTurns?: number | undefined;
    requestTimeoutMs?: number | undefined;
    concurrency?: number | undefined;
    debounceMs?: number | undefined;
    retries?: number | undefined;
    metrics?: ("helpfulness" | "coherence" | "relevance" | "faithfulness" | "verbosity" | "goal_completion" | "agent_behavior_failure" | "tool_call_behavior_failure")[] | undefined;
    thresholds?: Record<string, number> | undefined;
    include?: string[] | undefined;
    exclude?: string[] | undefined;
    reporters?: ("console" | "json" | "github-actions")[] | undefined;
    unmockedTools?: "error" | "passthrough" | undefined;
    failOnErrorSeverity?: "low" | "medium" | "high" | "critical" | undefined;
    customMetrics?: z.objectInputType<{
        name: z.ZodString;
        setLLM: z.ZodFunction<z.ZodTuple<[], z.ZodUnknown>, z.ZodUnknown>;
    }, z.ZodTypeAny, "passthrough">[] | undefined;
    compare?: ({
        name: string;
        endpoint?: string | undefined;
        headers?: Record<string, string> | undefined;
        body?: Record<string, unknown> | undefined;
        streaming?: boolean | undefined;
    } | {
        type: "custom";
        name: string;
        handler: (args_0: any[], ...args: unknown[]) => any;
    })[] | undefined;
}>;
type AgentestConfig = z.infer<typeof configSchema>;
type AgentestConfigInput = z.input<typeof configSchema>;

interface GenerateTextOptions {
    system?: string;
    messages: Array<{
        role: 'user' | 'assistant' | 'system';
        content: string;
    }>;
}
interface GenerateObjectOptions<T extends z.ZodType> {
    schema: T;
    system?: string;
    messages: Array<{
        role: 'user' | 'assistant' | 'system';
        content: string;
    }>;
}
interface LLMProvider {
    readonly model: LanguageModel;
    generateText(options: GenerateTextOptions): Promise<{
        text: string;
    }>;
    generateObject<T extends z.ZodType>(options: GenerateObjectOptions<T>): Promise<{
        object: z.infer<T>;
    }>;
}
type ProviderName = 'openai' | 'anthropic' | 'google' | 'ollama' | 'openai-compatible';
interface ProviderOptions {
    baseURL?: string;
    apiKey?: string;
    timeoutMs?: number;
    /**
     * Extra JSON fields merged into every request body. Currently honored by the
     * `openai-compatible` and `ollama` providers. Useful for vendor-specific params
     * like `{ chat_template_kwargs: { enable_thinking: false } }` (Qwen3).
     */
    extraBody?: Record<string, unknown>;
}
declare function createProvider(provider: ProviderName, modelId: string, options?: ProviderOptions): Promise<LLMProvider>;

interface TurnRecord {
    turnIndex: number;
    userMessage: string;
    agentMessage: string;
    toolCalls: ToolCallRecord[];
}
interface ConversationRecord {
    conversationId: string;
    turns: TurnRecord[];
    error?: string;
}
interface SimulationResult {
    scenario: Scenario;
    conversations: ConversationRecord[];
}
type SimulatorProgressCallback = (event: {
    scenario: string;
    conversationId: string;
    turn: number;
    maxTurns: number;
    phase: 'user-message' | 'agent-call' | 'tool-resolution';
}) => void;
declare class Simulator {
    private llmProvider;
    private agentClient;
    private config;
    onProgress?: SimulatorProgressCallback;
    constructor(config: AgentestConfig, llmProvider: LLMProvider);
    runScenario(scenario: Scenario): Promise<SimulationResult>;
    private runConversation;
    private resolveToolCalls;
}

interface ScoreInput {
    goal: string;
    profile: string;
    knowledge: KnowledgeItem[];
    userMessage: string;
    agentMessage: string;
    toolCalls: ToolCallRecord[];
    history: Array<{
        userMessage: string;
        agentMessage: string;
    }>;
    turnIndex: number;
}
interface ConversationScoreInput {
    goal: string;
    profile: string;
    knowledge: KnowledgeItem[];
    turns: Array<{
        userMessage: string;
        agentMessage: string;
        toolCalls: ToolCallRecord[];
    }>;
}
interface QuantResult {
    value: number;
    reason: string;
}
interface QualResult {
    value: string;
    reason: string;
}
declare abstract class BaseMetric {
    abstract readonly name: string;
    private _llm;
    protected get llm(): LLMProvider;
    setLLM(llm: LLMProvider): void;
}
declare abstract class QuantitativeMetric extends BaseMetric {
    abstract score(input: ScoreInput): Promise<QuantResult>;
}
declare abstract class QualitativeMetric extends BaseMetric {
    abstract evaluate(input: ScoreInput): Promise<QualResult>;
}
declare abstract class ConversationMetric extends BaseMetric {
    abstract score(input: ConversationScoreInput): Promise<QuantResult>;
}

type MetricName = 'helpfulness' | 'coherence' | 'relevance' | 'faithfulness' | 'verbosity' | 'goal_completion' | 'agent_behavior_failure' | 'tool_call_behavior_failure';
interface TurnEvaluation {
    turnIndex: number;
    metrics: Record<string, QuantResult | QualResult>;
}
interface ConversationEvaluation {
    turnEvaluations: TurnEvaluation[];
    goalCompletion: QuantResult | null;
}
declare class Evaluator {
    private llm;
    private turnMetrics;
    private goalCompletionMetric;
    private customMetrics;
    private limit;
    constructor(llm: LLMProvider, metricNames: MetricName[] | undefined, customMetrics: Array<{
        name: string;
        setLLM: (llm: LLMProvider) => void;
    }> | undefined, concurrency: number);
    evaluateConversation(turns: TurnRecord[], scenario: ScenarioOptions): Promise<ConversationEvaluation>;
    private evaluateTurn;
}

interface TrajectoryResult {
    matched: boolean;
    failureLabel?: string;
    reason: string;
    missingCalls: string[];
    extraCalls: string[];
    orderingIssues: string[];
    forbiddenCalls: string[];
}
declare class TrajectoryMatcher {
    match(actual: ToolCallRecord[], assertions: TrajectoryAssertions): TrajectoryResult;
    private matchStrict;
    private matchUnordered;
    private matchContains;
    private matchWithin;
    private checkForbidden;
    private matchArgs;
    private buildReason;
}

interface FailureTurn {
    id: string;
    scenarioName: string;
    conversationId: string;
    turnIndex: number;
    metricName: string;
    label: string;
    reason: string;
}
interface UniqueError {
    label: string;
    severity: 'low' | 'medium' | 'high' | 'critical';
    occurrences: string[];
    explanation: string;
}
declare class ErrorDetection {
    private llm;
    constructor(llm: LLMProvider);
    deduplicate(failures: FailureTurn[]): Promise<UniqueError[]>;
    collectFailures(scenarioName: string, conversationId: string, turnEvaluations: Array<{
        turnIndex: number;
        metrics: Record<string, {
            value: string | number;
            reason: string;
        }>;
    }>): FailureTurn[];
}

interface DiscoveryResult {
    file: string;
    scenarios: Scenario[];
}
declare function discoverScenarioFiles(config: AgentestConfig, cwd?: string): Promise<string[]>;
declare function loadScenarioFile(filePath: string, cwd?: string): Promise<Scenario[]>;
declare function discoverAndLoad(config: AgentestConfig, cwd?: string): Promise<DiscoveryResult[]>;

interface ProgressEvent {
    scenario: string;
    conversationId?: string;
    phase: 'simulating' | 'evaluating' | 'trajectories' | 'deduplicating';
    detail?: string;
}
interface Reporter {
    onRunStart?(scenarios: Scenario[], agents?: string[]): void | Promise<void>;
    onScenarioStart?(scenario: Scenario): void | Promise<void>;
    onScenarioEnd?(result: ScenarioResult): void | Promise<void>;
    onRunEnd?(result: RunResult): void | Promise<void>;
    onProgress?(event: ProgressEvent): void;
    onComparisonScenarioEnd?(result: ComparisonScenarioResult): void | Promise<void>;
    onComparisonRunEnd?(result: ComparisonRunResult): void | Promise<void>;
}

interface PerTurnTrajectoryResult {
    turnIndex: number;
    result: TrajectoryResult;
}
interface ScenarioResult {
    scenario: Scenario;
    simulation: SimulationResult;
    evaluations: Map<string, ConversationEvaluation>;
    trajectoryResults: Map<string, TrajectoryResult>;
    perTurnTrajectoryResults: Map<string, PerTurnTrajectoryResult[]>;
    errors: UniqueError[];
    passed: boolean;
    /** Set when the scenario failed the parallel pass and passed on retry N. */
    retried?: number;
}
interface RunResult {
    scenarioResults: ScenarioResult[];
    passed: boolean;
    totalScenarios: number;
    passedScenarios: number;
    failedScenarios: number;
}
interface ComparisonAgentResult {
    agentName: string;
    scenarioResult: ScenarioResult;
}
interface ComparisonScenarioResult {
    scenario: Scenario;
    agentResults: ComparisonAgentResult[];
}
interface ComparisonRunResult {
    agents: string[];
    scenarioComparisons: ComparisonScenarioResult[];
    perAgent: Map<string, RunResult>;
    passed: boolean;
}
declare class Runner {
    private config;
    private reporters;
    private limit;
    private lastScenarioStartTime;
    constructor(config: AgentestConfig, reporters: Reporter[]);
    private debounce;
    private createInfrastructure;
    run(discoveries: DiscoveryResult[]): Promise<RunResult>;
    runComparison(discoveries: DiscoveryResult[]): Promise<ComparisonRunResult>;
    private runScenario;
    private emitProgress;
    private static formatPhaseLabel;
    private determinePassFail;
}

export { discoverScenarioFiles as $, type AgentestConfigInput as A, type ScriptedTurn as B, type CustomHandlerContext as C, DEFAULT_MODEL as D, ErrorDetection as E, type FailureTurn as F, type GenerateObjectOptions as G, type SimulationResult as H, Simulator as I, type ToolCallRecord as J, type KnowledgeItem as K, type LLMProvider as L, type MetricName as M, type ToolMockFn as N, type TrajectoryAssertions as O, type ProgressEvent as P, QuantitativeMetric as Q, type Reporter as R, type Scenario as S, type ToolCallAssertion as T, TrajectoryMatcher as U, type TrajectoryResult as V, type TurnEvaluation as W, type TurnRecord as X, type UniqueError as Y, createProvider as Z, discoverAndLoad as _, type AgentestConfig as a, loadScenarioFile as a0, sequence as a1, validateScenarioOptions as a2, type ScenarioOptions as b, type ScoreInput as c, type QuantResult as d, ConversationMetric as e, type ConversationScoreInput as f, QualitativeMetric as g, type QualResult as h, type ConversationEvaluation as i, type ScenarioResult as j, type RunResult as k, type ComparisonScenarioResult as l, type ComparisonRunResult as m, AgentestError as n, type ComparisonAgentResult as o, type ConversationRecord as p, type CustomAgentHandler as q, type DiscoveryResult as r, Evaluator as s, type ForbiddenToolAssertion as t, type GenerateTextOptions as u, MockResolver as v, type PerTurnTrajectoryResult as w, type ProviderName as x, type ProviderOptions as y, Runner as z };