/**
 * `@gemstack/ai-sdk/eval` — built-in eval framework for #A5 Phase 1.
 *
 * Define a suite of input cases + assertions, run them against any
 * `Agent`, get a console report with pass/fail + cost + tokens. Same
 * `Agent` instances as your app code — one source of truth.
 *
 * @example
 * ```ts
 * // evals/support-agent.eval.ts
 * import { evalSuite, llmJudge, exactMatch, regex } from '@gemstack/ai-sdk/eval'
 * import { SupportAgent } from '../app/Agents/SupportAgent.js'
 *
 * export default evalSuite('SupportAgent', {
 *   agent: () => new SupportAgent(),
 *   cases: [
 *     { name: 'password reset', input: 'How do I reset my password?',
 *       assert: llmJudge('mentions a password reset link') },
 *     { name: 'price', input: 'How much?', assert: exactMatch('$99/month') },
 *     { name: 'support email', input: 'Contact?', assert: regex(/support@/) },
 *   ],
 * })
 * ```
 *
 * Run programmatically via `runSuite(suite)` from this entry, or via
 * `pnpm rudder ai:eval` once Phase 2 lands.
 *
 * Built-in metrics: `exactMatch`, `regex`, `llmJudge`, `jsonShape`,
 * `semanticMatch`, `tokenCost`. Compose multiple via `compose(...)`.
 * User-defined metrics work today — any `(response, ctx) =>
 * MetricResult` qualifies.
 */
import type { Agent } from '../agent.js';
import type { AgentResponse } from '../types.js';
import { z } from 'zod';
export { reportJson } from './json-reporter.js';
export type { SuiteJson, SuiteJsonCase } from './json-reporter.js';
export { stepsFromResponse, defaultFixturesDir, readFixture, writeFixture } from './fixtures.js';
export type { EvalFixture } from './fixtures.js';
export { reportHtml } from './html-reporter.js';
export type { HtmlReportOptions } from './html-reporter.js';
/**
 * Result of a single assertion. `pass` is the only required field;
 * `score` (0..1) and `reason` are surfaced in reports.
 */
export interface MetricResult {
    pass: boolean;
    score?: number;
    reason?: string;
}
/**
 * Assertion signature. Sync or async; the runner awaits both.
 *
 * `ctx` carries the case context so user metrics can opt into the
 * input/case-name (e.g. for logging). The built-ins ignore it.
 */
export type Metric = (response: AgentResponse, ctx: MetricContext) => MetricResult | Promise<MetricResult>;
export interface MetricContext {
    /** The case's input string (the same passed to `agent.prompt`). */
    input: string;
    /** Optional case `name` if set on the spec. */
    caseName: string;
}
/** A single eval case. */
export interface EvalCase {
    /** Stable identifier used in reports. Defaults to `case-<index>`. */
    name?: string;
    /** Input passed to `agent.prompt(input)`. */
    input: string;
    /** The assertion. Pass-fail + optional score/reason. */
    assert: Metric;
    /**
     * Per-case agent override. When set, replaces the suite-level
     * `agent` factory for this case (e.g. swap models for a stress
     * test).
     */
    agent?: () => Agent;
    /**
     * Per-case timeout in ms. Defaults to the suite-level timeout
     * (or no timeout if neither is set).
     */
    timeout?: number;
    /**
     * Skip this case. Pass `true` to silently skip, or a string for
     * a reason that surfaces in the report.
     */
    skip?: boolean | string;
}
export interface EvalSuiteSpec {
    /** Factory for the agent under test. Called once per case. */
    agent: () => Agent;
    /** The cases to run. */
    cases: EvalCase[];
    /**
     * Suite-wide timeout in ms applied to every case unless the case
     * overrides. Throws cause `pass: false` with the timeout message.
     */
    timeout?: number;
    /**
     * Optional ownership / context surfaced in the HTML report (#A5
     * Phase 5). Well-known keys (`owner`, `lastReviewed`, `ticket`)
     * get formatted headings; any extra string keys render as a
     * generic key/value row so teams can attach their own metadata.
     */
    metadata?: EvalMetadata;
}
export interface EvalMetadata {
    owner?: string;
    lastReviewed?: string;
    ticket?: string;
    [key: string]: string | undefined;
}
export interface EvalSuite {
    name: string;
    spec: EvalSuiteSpec;
}
/** Per-case run record collected by {@link runSuite}. */
export interface CaseResult {
    name: string;
    /** Final result; `'skipped'` skips assertion + cost. */
    status: 'passed' | 'failed' | 'skipped';
    metric?: MetricResult;
    /** Skip reason (when `status === 'skipped'`). */
    reason?: string;
    /** Wall-clock ms for the agent call + assertion. */
    duration: number;
    /**
     * Token usage from the agent's `prompt()` (zero on skip / failure
     * before the call). Includes BOTH the agent under test AND any
     * judge-model calls the assertion made.
     */
    tokens: number;
    /** USD estimate (see {@link estimateCost}; zero on skip). */
    cost: number;
    /**
     * The case's input string, copied through from `EvalCase.input`
     * for reporters that want to render the prompt alongside the
     * response (#A5 Phase 5 HTML report). Always present — runners
     * always know the input.
     */
    input: string;
    /**
     * The agent's final assistant text. Absent when the case skipped
     * or the agent threw before producing a response. The HTML
     * reporter renders `<no response>` in that case.
     */
    responseText?: string;
}
/** Full report returned by {@link runSuite}. */
export interface SuiteReport {
    suite: string;
    cases: CaseResult[];
    passed: number;
    failed: number;
    skipped: number;
    duration: number;
    cost: number;
    tokens: number;
    /** Suite-level metadata (#A5 Phase 5), copied through from the spec. */
    metadata?: EvalMetadata;
}
/**
 * Define an eval suite. Returns a frozen `EvalSuite` ready to pass
 * into {@link runSuite} or to default-export from an `evals/*.eval.ts`
 * file (Phase 2's CLI auto-discovers those).
 *
 * The shape is deliberately a function rather than a class — keeps the
 * file's default export trivially serializable (Phase 2 needs to load
 * suites via dynamic import) and avoids the "did you forget `new`?"
 * footgun.
 */
export declare function evalSuite(name: string, spec: EvalSuiteSpec): EvalSuite;
/** Exact string equality against `response.text`. */
export declare function exactMatch(expected: string): Metric;
/** Pattern match against `response.text`. */
export declare function regex(pattern: RegExp): Metric;
/**
 * LLM-as-judge: ask a small model whether the response satisfies a
 * natural-language criterion. Returns the judge's reasoning in
 * `reason` so failures are debuggable.
 *
 * Design: the judge runs as a one-shot anonymous agent (no recursion
 * concern — default `remembers()` is `false`). Output is shaped via
 * `Output.object({ schema })` for deterministic parsing. Failures
 * (network, parse, unhandled judge error) bubble as `pass: false`
 * with the error in `reason` — a broken judge is not a passing case.
 *
 * Pitfall: the judge model has the same biases as any LLM. Use it
 * for fuzzy "did the answer mention X?" assertions; for exact
 * structural checks prefer `jsonShape` (Phase 3) or `regex`.
 */
export declare function llmJudge(criterion: string, opts?: {
    model?: string;
}): Metric;
/**
 * Strict structural assertion: parse `response.text` as JSON
 * (stripping ```json fences) and run it through a zod schema.
 *
 * Pairs naturally with `Output.object({ schema })` on the agent —
 * if the agent declares the same schema, this metric verifies the
 * output actually conforms. Failures surface the zod issue path
 * (e.g. `customer.email`) so debugging doesn't require a separate
 * console log.
 */
export declare function jsonShape<T>(schema: z.ZodType<T>): Metric;
/**
 * Embedding-based fuzzy match. Embeds both `reference` and
 * `response.text` via `AI.embed()`, computes cosine similarity,
 * passes when >= `threshold` (default `0.85` — tighter than
 * `EmbeddingUserMemory`'s 0.5 retrieval-rank floor since this is
 * an assertion, not a ranking).
 *
 * Uses ≤ 2 embedding calls per case; embed tokens roll into the
 * case's cost rollup via the same side-channel `llmJudge` uses.
 *
 * Pitfall: requires a provider that implements `createEmbedding()`
 * (openai / google / mistral / cohere / jina). Failures (no
 * provider, network, etc.) surface as `pass: false` with the
 * error in `reason` — a broken embed is not a passing case.
 */
export declare function semanticMatch(reference: string, opts?: {
    threshold?: number;
    model?: string;
}): Metric;
/**
 * Token budget guard. Passes when `response.usage.totalTokens
 * <= threshold`. Pair with cost-conscious agents to detect prompt-
 * size regressions before they show up as a billing surprise.
 *
 * `response.usage` is the multi-step rollup, so it's meaningful
 * even when the agent runs tools across several provider calls.
 */
export declare function tokenCost(threshold: number): Metric;
/**
 * Compose multiple metrics into one assertion. Runs them in order
 * and short-circuits on the first failure — failure `reason` is
 * surfaced; success returns `{ pass: true, score: 1 }`.
 *
 * @example
 *   { input: '…',
 *     assert: compose(
 *       jsonShape(SummarySchema),
 *       tokenCost(800),
 *     ),
 *   }
 */
export declare function compose(...metrics: Metric[]): Metric;
/**
 * Run every case in the suite, in declaration order. Returns the
 * full report; never throws (assertion errors become `failed` cases,
 * not exceptions).
 *
 * Phase 1 runs serially. Parallel execution lands in a follow-up
 * once we understand the rate-limit shape of real-world judge
 * models — sequential is correct under any rate limit.
 */
export declare function runSuite(suite: EvalSuite): Promise<SuiteReport>;
export { estimateCost, ModelPricing } from '../budget/pricing.js';
export type { ModelPriceEntry } from '../budget/pricing.js';
/**
 * Default reporter — prints a colorless ANSI-aware table to a
 * caller-supplied `console`-like sink. Uses Unicode pass/fail glyphs
 * for visual scanning. JSON / HTML reporters land in Phase 2 / 5.
 *
 * Returns the report unchanged so chains compose: `await
 * reportConsole(await runSuite(suite))`.
 */
export declare function reportConsole(report: SuiteReport, sink?: {
    log: (s: string) => void;
}): SuiteReport;
//# sourceMappingURL=index.d.ts.map