import * as vitest from 'vitest';
import { TestAPI } from 'vitest';
import { HarnessMetadata, Harness } from './harness.js';
export { CreateHarnessOptions, CreateHarnessRunArgs, CreateToolCallSpansOptions, EnsureRunTraceOptions, HarnessContext, HarnessResultLike, MaybePromise, SimpleHarnessResult, SimpleSpanEvent, SimpleSpanRecord, SimpleToolCallRecord, SimpleTraceRecord, attachHarnessRunToError, createFailedHarnessRun, createGenAiUsageAttributes, createHarness, createToolCallSpans, ensureRunTrace, getHarnessRunFromError, normalizeHarnessRun, normalizeSpanAttributes, normalizeSpanError, toJsonValue } from './harness.js';
import { JudgeContext, Judge, JudgeResult, JudgeAssessFn, JudgeAssessor, JudgeAssessWithAssessorFn } from './judges/types.js';
export { BoundJudgeAssessor, JudgeAssessorOptions, JudgeOptions } from './judges/types.js';
import { JudgeHarness } from './judges/judgeHarness.js';
export { CreateJudgeHarnessOptions, CreateJudgeHarnessRunOptions, JudgeHarnessInput, JudgeHarnessOutput, RunJudge, RunJudgeOptions, createJudgeHarness, runJudgeHarness } from './judges/judgeHarness.js';
export { wrapText } from './wrapText.js';
export { FactualityJudge, FactualityJudgeChoice, FactualityJudgeConfig, FactualityJudgeExpected, FactualityJudgeOptions, FactualityJudgePrompt, FactualityJudgeVerdict } from './judges/factualityJudge.js';
export { StructuredOutputJudge, StructuredOutputJudgeConfig, StructuredOutputJudgeExpected, StructuredOutputJudgeOptions } from './judges/structuredOutputJudge.js';
export { ToolCallJudge, ToolCallJudgeConfig, ToolCallJudgeExpectedTool, ToolCallJudgeOptions } from './judges/toolCallJudge.js';
export { BaseMatcherConfig, FuzzyMatchOptions, MatchStrategy } from './internal/matchers.js';
import { JsonValue, HarnessRun, NormalizedSession, ToolCallRecord } from '@vitest-evals/core';
export { GenAiOperationName, GenAiOutputType, GenAiProviderName, GenAiSemanticAttributeKey, GenAiSemanticAttributes, GenAiTokenType, GenAiToolType, HarnessRun, HarnessRunError, JsonPrimitive, JsonValue, NormalizedMessage, NormalizedSession, NormalizedSpan, NormalizedSpanAttributeKey, NormalizedSpanAttributes, NormalizedSpanEvent, NormalizedTrace, OpenTelemetrySemanticAttributeKey, OpenTelemetrySemanticAttributes, TimingSummary, ToolCallRecord, UsageSummary, assistantMessages, failedSpans, latestAssistantMessageContent, messagesByRole, spans, spansByKind, systemMessages, toolCalls, toolMessages, userMessages } from '@vitest-evals/core';
import './internal/structuredOutputScorer.js';
import './internal/scoring.js';
import './internal/toolCallScorer.js';

type EvalTaskMeta = {
    eval?: {
        scores: (JudgeResult & {
            name: string;
        })[];
        avgScore: number;
        output?: unknown;
        toolCalls?: ToolCallRecord[];
        thresholdFailed?: boolean;
    };
    harness?: {
        name: string;
        run: HarnessRun;
    };
};
type HarnessInput<THarness extends Harness<any, any, any>> = THarness extends Harness<infer TInput, any, any> ? TInput : unknown;
type HarnessMetadataFor<THarness extends Harness<any, any, any>> = THarness extends Harness<any, any, infer TMetadata> ? TMetadata : HarnessMetadata;
type HarnessOutput<THarness extends Harness<any, any, any>> = THarness extends Harness<any, infer TOutput, any> ? TOutput : JsonValue | undefined;
type CreateJudgeConfig<TOptions extends JudgeContext<any, any, any, any> = JudgeContext> = {
    name: string;
    judgeHarness?: JudgeHarness;
    assess: JudgeAssessFn<TOptions>;
};
declare const evalHarnessRunBrand: unique symbol;
/**
 * Harness run returned by the fixture-backed `run(...)` API.
 *
 * @example
 * ```ts
 * it("approves a refund", async ({ run }) => {
 *   const result = await run("Refund invoice inv_123");
 *
 *   expect(result.output.status).toBe("approved");
 * });
 * ```
 */
type EvalHarnessRun<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TOutput, TMetadata> = Harness<TInput, TOutput, TMetadata>> = HarnessRun<TOutput> & {
    readonly [evalHarnessRunBrand]: {
        readonly input: TInput;
        readonly metadata: TMetadata;
        readonly output: TOutput;
        readonly harness: THarness;
    };
};
/**
 * Per-run metadata forwarded to the harness alongside the test input.
 *
 * @example
 * ```ts
 * await run("Refund invoice inv_123", {
 *   metadata: {
 *     expected: { status: "approved" },
 *     expectedTools: ["lookupInvoice", "createRefund"],
 *   },
 * });
 * ```
 */
interface EvalRunOptions<TMetadata extends HarnessMetadata = HarnessMetadata> {
    /** Per-run expectations or configuration forwarded to harnesses and judges. */
    metadata?: TMetadata;
}
/**
 * Explicit harness execution primitive exposed to each eval test.
 *
 * @example
 * ```ts
 * const result = await run("Refund invoice inv_123", {
 *   metadata: { expected: { status: "approved" } },
 * });
 * ```
 */
type EvalRun<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TOutput, TMetadata> = Harness<TInput, TOutput, TMetadata>> = (input: TInput, options?: EvalRunOptions<TMetadata>) => Promise<EvalHarnessRun<TInput, TOutput, TMetadata, THarness>>;
/**
 * Fixture-backed Vitest context exposed inside `describeEval(...)` tests.
 *
 * @example
 * ```ts
 * type RefundOutput = { status: "approved" | "denied" };
 *
 * it("approves a refund", async ({ run }: EvalTestContext<string, RefundOutput>) => {
 *   const result = await run("Refund invoice inv_123");
 *
 *   expect(result.output.status).toBe("approved");
 * });
 * ```
 */
interface EvalTestContext<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TOutput, TMetadata> = Harness<TInput, TOutput, TMetadata>> {
    run: EvalRun<TInput, TOutput, TMetadata, THarness>;
}
/** Fixture-backed Vitest test API exposed inside `describeEval(...)`. */
type EvalTestAPI<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TOutput, TMetadata> = Harness<TInput, TOutput, TMetadata>> = TestAPI<EvalTestContext<TInput, TOutput, TMetadata, THarness>>;
/**
 * Suite-level configuration for a harness-backed eval block.
 *
 * @example
 * ```ts
 * const options: DescribeEvalOptions<
 *   string,
 *   { status: "approved" | "denied" },
 *   { expected: { status: "approved" | "denied" } }
 * > = {
 *   harness: refundHarness,
 *   judges: [ToolCallJudge(), StructuredOutputJudge()],
 *   judgeThreshold: 1,
 * };
 * ```
 */
interface DescribeEvalOptions<TInput = unknown, TOutput extends JsonValue | undefined = JsonValue | undefined, TMetadata extends HarnessMetadata = HarnessMetadata, THarness extends Harness<TInput, TOutput, TMetadata> = Harness<TInput, TOutput, TMetadata>> {
    /** Harness used for every explicit `run(...)` call in the suite. */
    harness: THarness;
    /** Automatic judges applied after each successful `run(...)`. */
    judges?: Array<Judge<JudgeContext<TInput, TOutput, TMetadata, THarness>>>;
    /** Optional judge-side harness used only by judges that call `ctx.runJudge(...)`. */
    judgeHarness?: JudgeHarness;
    /** Passing threshold for automatic suite-level judges. `null` disables fail-on-score. */
    judgeThreshold?: number | null;
    /** Skips the entire eval suite when the predicate returns true. */
    skipIf?: () => boolean;
}
type JudgeAssertionInput<TJudgeOptions extends JudgeContext<any, any, any, any>> = TJudgeOptions extends {
    input: infer TInput;
} ? TInput : unknown;
type JudgeAssertionOutput<TJudgeOptions extends JudgeContext<any, any, any, any>> = TJudgeOptions extends {
    output: infer TOutput;
} ? TOutput : JsonValue | undefined;
type JudgeAssertionMetadata<TJudgeOptions extends JudgeContext<any, any, any, any>> = TJudgeOptions extends {
    metadata: infer TMetadata;
} ? TMetadata : HarnessMetadata;
type JudgeAssertionHarness<TJudgeOptions extends JudgeContext<any, any, any, any>> = TJudgeOptions extends {
    harness: infer THarness;
} ? Exclude<THarness, undefined> : Harness<JudgeAssertionInput<TJudgeOptions>, JudgeAssertionOutput<TJudgeOptions>, JudgeAssertionMetadata<TJudgeOptions>>;
type JudgeAssertionReservedKey = keyof JudgeContext<any, any, any, any> | "judgeHarness" | "signal" | "threshold";
type JudgeAssertionParams<TJudgeOptions extends JudgeContext<any, any, any, any>> = Omit<TJudgeOptions, JudgeAssertionReservedKey>;
type RequiredKeys<T> = {
    [K in keyof T]-?: Record<string, never> extends Pick<T, K> ? never : K;
}[keyof T];
type JudgeAssertionArgs<TJudgeOptions extends JudgeContext<any, any, any, any>> = RequiredKeys<JudgeAssertionParams<TJudgeOptions>> extends never ? [options?: JudgeAssertionOptions<TJudgeOptions>] : [options: JudgeAssertionOptions<TJudgeOptions>];
type MatcherOutput<TReceived> = TReceived extends EvalHarnessRun<any, infer TOutput, any, any> ? TOutput : TReceived extends HarnessRun<infer TOutput> ? TOutput : TReceived extends NormalizedSession ? JsonValue | undefined : TReceived extends JsonValue ? TReceived : JsonValue | undefined;
type JudgeForReceived<TReceived, TJudgeOptions extends JudgeContext<any, any, any, any>> = MatcherOutput<TReceived> extends JudgeAssertionOutput<TJudgeOptions> ? Judge<TJudgeOptions> : never;
/**
 * Optional overrides passed to `expect(...).toSatisfyJudge(...)`.
 *
 * @example
 * ```ts
 * await expect(result).toSatisfyJudge(RefundStatusJudge, {
 *   threshold: null,
 * });
 * ```
 */
type JudgeAssertionOptions<TJudgeOptions extends JudgeContext<any, any, any, any> = JudgeContext> = JudgeAssertionParams<TJudgeOptions> & {
    /** Override or provide the original eval input for the judge. */
    input?: JudgeAssertionInput<TJudgeOptions>;
    /** Override or provide the app-facing output for the judge. */
    output?: JudgeAssertionOutput<TJudgeOptions>;
    /** Override or provide per-run judge metadata. */
    metadata?: JudgeAssertionMetadata<TJudgeOptions>;
    /** Override or provide flattened tool calls for the judge. */
    toolCalls?: ToolCallRecord[];
    /** Override or provide the complete normalized harness run. */
    run?: HarnessRun<JudgeAssertionOutput<TJudgeOptions>>;
    /** Override or provide the normalized session transcript. */
    session?: HarnessRun["session"];
    /** Override or provide the harness associated with the judge context. */
    harness?: JudgeAssertionHarness<TJudgeOptions>;
    /** Override or provide the judge harness for judges that call `ctx.runJudge(...)`. */
    judgeHarness?: JudgeHarness;
    /** Passing threshold for the explicit matcher. `null` records the score without failing. */
    threshold?: number | null;
};
/** Function type installed as the `toSatisfyJudge(...)` matcher. */
type ToSatisfyJudge<TReceived = unknown> = <TJudgeOptions extends JudgeContext<any, any, any, any> = JudgeContext>(judge: JudgeForReceived<TReceived, TJudgeOptions>, ...args: JudgeAssertionArgs<TJudgeOptions>) => Promise<TReceived>;
/**
 * Vitest matcher extension surface added by `vitest-evals`.
 *
 * @example
 * ```ts
 * await expect(result).toSatisfyJudge(RefundStatusJudge);
 * ```
 */
interface EvalMatchers<R = unknown> {
    toSatisfyJudge: ToSatisfyJudge<R>;
}
declare module "vitest" {
    interface Assertion<T = any> extends EvalMatchers<T> {
    }
    interface AsymmetricMatchersContaining extends EvalMatchers {
    }
    interface TaskMeta extends EvalTaskMeta {
    }
}
/**
 * Creates a harness-backed eval suite on top of a fixture-backed Vitest test API.
 *
 * @param name - Suite name shown by Vitest and reporters.
 * @param options - Harness, automatic judges, threshold, and suite skip settings.
 * @param define - Callback that receives the eval-aware `it` API.
 *
 * @example
 * ```ts
 * import { piAiHarness } from "@vitest-evals/harness-pi-ai";
 * import { getModel } from "@mariozechner/pi-ai";
 * import { piAiJudgeHarness } from "@vitest-evals/harness-pi-ai";
 * import { expect } from "vitest";
 * import {
 *   describeEval,
 *   FactualityJudge,
 *   ToolCallJudge,
 *   toolCalls,
 * } from "vitest-evals";
 * import { createRefundAgent } from "../src/refundAgent";
 *
 * const judgeHarness = piAiJudgeHarness({
 *   model: getModel("anthropic", "claude-sonnet-4-5"),
 *   temperature: 0,
 * });
 *
 * describeEval("refund agent", {
 *   harness: piAiHarness({
 *     agent: () => createRefundAgent(),
 *   }),
 *   judgeHarness,
 *   judges: [ToolCallJudge()],
 * }, (it) => {
 *   it("approves a refundable invoice", async ({ run }) => {
 *     const result = await run("Refund invoice inv_123", {
 *       metadata: {
 *         expected: "Invoice inv_123 should be refunded.",
 *       },
 *     });
 *
 *     expect(result.output).toMatchObject({ status: "approved" });
 *     expect(toolCalls(result.session)).toHaveLength(2);
 *     await expect(result).toSatisfyJudge(FactualityJudge(), {
 *       threshold: 0.6,
 *     });
 *   });
 * });
 * ```
 */
declare function describeEval<THarness extends Harness<any, any, any>>(name: string, options: DescribeEvalOptions<HarnessInput<THarness>, HarnessOutput<THarness>, HarnessMetadataFor<THarness>, THarness>, define: (it: EvalTestAPI<HarnessInput<THarness>, HarnessOutput<THarness>, HarnessMetadataFor<THarness>, THarness>) => void): vitest.SuiteCollector<object>;
/**
 * Formats judge results for reporter and assertion output.
 *
 * @param scores - Named judge results to sort and format.
 *
 * @example
 * ```ts
 * const message = formatScores([
 *   {
 *     name: "RefundStatusJudge",
 *     score: 0,
 *     metadata: { rationale: "Expected approved, got denied" },
 *   },
 * ]);
 * ```
 */
declare function formatScores(scores: (JudgeResult & {
    name: string;
})[]): string;
/**
 * Creates a named judge object from an assessment function.
 *
 * @param name - Stable judge name shown in assertion messages and reports.
 * @param assess - Function that scores one normalized judge context.
 *
 * @example
 * ```ts
 * import { createJudge, type JudgeContext } from "vitest-evals";
 *
 * type RefundOutput = { status: "approved" | "denied" };
 * type RefundMetadata = { expected: { status: RefundOutput["status"] } };
 *
 * export const RefundStatusJudge = createJudge(
 *   "RefundStatusJudge",
 *   async ({ output, metadata }: JudgeContext<string, RefundOutput, RefundMetadata>) => ({
 *     score: output.status === metadata.expected.status ? 1 : 0,
 *     metadata: {
 *       rationale: `Expected ${metadata.expected.status}, got ${output.status}`,
 *     },
 *   }),
 * );
 * ```
 *
 * For LLM-backed judges, prefer the object form with `ctx.runJudge(...)` so
 * provider-specific model configuration stays in the judge harness.
 */
declare function createJudge<TOptions extends JudgeContext<any, any, any, any>>(name: string, assess: JudgeAssessFn<TOptions>): Judge<TOptions>;
declare function createJudge<TOptions extends JudgeContext<any, any, any, any>>(config: CreateJudgeConfig<TOptions>): Judge<TOptions>;
/**
 * @deprecated Prefer `createJudge({ name, judgeHarness, assess })` and call
 * `ctx.runJudge(...)` from LLM-backed judges.
 */
declare function createJudge<TOptions extends JudgeContext<any, any, any, any>, TInput, TOutput>(name: string, assessor: JudgeAssessor<TInput, TOutput>, assess: JudgeAssessWithAssessorFn<TOptions, TInput, TOutput>): Judge<TOptions>;

export { type DescribeEvalOptions, type EvalHarnessRun, type EvalMatchers, type EvalRun, type EvalRunOptions, type EvalTestAPI, type EvalTestContext, Harness, HarnessMetadata, Judge, type JudgeAssertionOptions, JudgeAssessFn, JudgeAssessWithAssessorFn, JudgeAssessor, JudgeContext, JudgeHarness, JudgeResult, type ToSatisfyJudge, createJudge, describeEval, formatScores };