/**
 * Grader — the fundamental unit of evaluation.
 *
 * Every check, from "is the name kebab-case?" to "did the LLM judge prefer
 * this output?", is a grader. Graders are tagged with taxonomy metadata so
 * eval authors can pick the right graders for each loop (inner/CI/outer).
 */
/**
 * Helpers for implementing {@link Grader.defaultName}. Re-exported here so
 * plugin graders can reach them through the published `@microsoft/vally/graders`
 * subpath instead of a deep `dist/` import.
 */
export { describeGraderValue, describeGraderList, graderNameFrom, COMPOSITE_VALUE_LENGTH, } from "./default-name.js";
/**
 * Shared read-root resolver used by the built-in file graders: prefer the run's
 * `artifactDir` for a path when present, else fall back to `workDir`, honoring
 * strict mode. Re-exported so plugin graders can reuse the same output-first
 * read semantics through the published `@microsoft/vally/graders` subpath.
 */
export { resolveGraderReadRoot } from "./static/workspace-guard.js";
export type Determinism = "static" | "complex-static" | "slm" | "llm";
export type ReferenceRequirement = "reference-free" | "reference-based";
export type TemporalScope = "point-in-time" | "trajectory-level" | "cross-trajectory";
export type CostProfile = "free" | "low" | "medium" | "high";
export type ScoreKind = "code" | "llm" | "human";
export type GraderResultStatus = "success" | "error";
/**
 * Behavioral contract — how the pipeline should treat this grader.
 *
 * Separated from taxonomy metadata so that descriptive fields (determinism,
 * cost, etc.) don't double as control-plane signals.
 *
 * Note: there is no static "comparative" flag. Whether a grader can compare
 * trajectories is a structural capability — it implements the optional
 * {@link Grader.compare} method — not a metadata declaration that could drift
 * out of sync with the implementation.
 */
export interface GraderBehavior {
    /** Whether the grader requires an LLM client to be provisioned. */
    requiresLlmClient?: boolean;
    /** Whether the grader needs filesystem access to the trajectory workspace. */
    requiresWorkspace?: boolean;
}
export interface GraderMetadata {
    name: string;
    description: string;
    behavior: GraderBehavior;
    determinism: Determinism;
    reference: ReferenceRequirement;
    temporalScope: TemporalScope;
    costProfile: CostProfile;
}
export interface GraderResult {
    name: string;
    /**
     * Whether the grader produced a valid verdict.
     *
     * Optional for compatibility with results written before this field existed
     * and with existing plugins; absence means "success".
     */
    status?: GraderResultStatus;
    /**
     * Registered grader type this result came from (e.g. `output-matches`).
     *
     * Shared by every instance of that type, so it is the key for "all
     * `output-matches` checks" rather than for one specific check. Prefer
     * `configuredName` to identify a single configured grader; fall back to this
     * when the author named nothing, since `name` may be derived from config and
     * changes whenever that config is edited.
     *
     * Optional because results loaded from older JSONL predate this field.
     */
    graderType?: string;
    /**
     * The `name` the eval author wrote on this grader's config, if any.
     *
     * The one identifier that is both stable and unique: `graderType` is shared
     * by every instance of a type, and `name` may be derived from config and so
     * changes when that config is edited. Validation constrains it to a slug,
     * unique within the eval file, so it is safe to use as a key. Note the file
     * scope: one run can span several eval files, so a consumer aggregating a
     * whole run should qualify this with the eval file.
     *
     * Absent when the author named nothing — a derived name is a display label
     * and deliberately does not land here.
     */
    configuredName?: string;
    kind: ScoreKind;
    passed: boolean;
    score: number;
    label?: "correct" | "hallucinated" | "incorrect" | (string & {});
    evidence: string;
    details?: GraderResult[];
    metadata?: Record<string, unknown>;
}
/** True when this result or any nested detail failed to produce a valid verdict. */
export declare function hasGraderError(result: GraderResult): boolean;
export interface GraderInput {
    trajectory?: Trajectory;
    stimulus?: Stimulus;
    config?: Record<string, unknown>;
    /** Base directory for resolving relative sidecar diff file paths (trajectory.diffPath). */
    diffBaseDir?: string;
    goldenPatch?: string;
}
/**
 * A labeled trajectory participating in a comparison.
 *
 * The `label` is a stable identifier for the participant — for experiment
 * comparisons it is the variant name (e.g. `"baseline"`, `"with-skill"`).
 */
export interface ComparisonEntry {
    label: string;
    trajectory: Trajectory;
}
/**
 * Comparison context handed to {@link Grader.compare}.
 *
 * Discriminated on `topology` so the contract can grow without breaking:
 * - `baseline-relative` (v1) — exactly one baseline vs one treatment. The
 *   judge sees a single pair, which keeps prompts unambiguous and makes
 *   position-swap debiasing straightforward.
 * - `n-way` (reserved) — a full set of entries ranked together. Not emitted
 *   yet; declared so adding it later is additive, not a breaking change.
 */
export type GraderComparison = {
    topology: "baseline-relative";
    baseline: ComparisonEntry;
    treatment: ComparisonEntry;
} | {
    topology: "n-way";
    entries: ComparisonEntry[];
    baselineLabel: string;
};
export interface GraderComparisonInput {
    comparison: GraderComparison;
    stimulus?: Stimulus;
    config?: Record<string, unknown>;
}
/** Which side of a baseline-relative comparison prevailed. */
export type ComparisonWinner = "baseline" | "treatment" | "tie";
/**
 * Signed magnitude of a comparison verdict, from the treatment's perspective.
 * `much-better` means the treatment is much better than the baseline.
 */
export type ComparisonMagnitude = "much-better" | "slightly-better" | "equal" | "slightly-worse" | "much-worse";
/**
 * Signed [-1, 1] score for each comparison magnitude, from the treatment's
 * perspective. Mirrors the prior-art skill-validator scale: the outer buckets
 * are ±1 and the inner buckets ±0.4, so a "slightly better" win can't swamp a
 * genuine "much better" one when averaged across criteria or trials.
 */
export declare const COMPARISON_MAGNITUDE_SCORES: Record<ComparisonMagnitude, number>;
export interface GraderComparisonResult {
    name: string;
    kind: ScoreKind;
    /** Overall winner relative to the baseline. */
    winner: ComparisonWinner;
    /** Signed magnitude (treatment-relative). */
    magnitude: ComparisonMagnitude;
    /**
     * Signed preference score in [-1, 1] from the treatment's perspective:
     * +1 = treatment much better, 0 = tie, -1 = baseline much better.
     */
    score: number;
    evidence: string;
    /** Per-criterion sub-verdicts for programmatic consumption. */
    details?: GraderComparisonResult[];
    metadata?: Record<string, unknown>;
}
export interface Grader {
    metadata: GraderMetadata;
    grade(input: GraderInput): Promise<GraderResult>;
    /**
     * Optional config-derived default name for one configured instance of this
     * grader, used when the eval author did not set an explicit `name`. Lets
     * several instances of the same type stay distinguishable by what they check
     * (`output-matches /Cosmos/`) instead of by position.
     *
     * MUST be a pure function of `config` — a name that varies between calls
     * (timestamps, random ids) fragments multi-trial aggregation, JUnit property
     * keys, and the analytics store. MUST NOT be derived from config that can
     * carry secrets (prefer `command` over `env`); names reach reports and the
     * analytics store. Resolution is centralized in `resolveDefaultGraderName`,
     * which sanitizes, truncates, and ignores a thrown or unusable value — but
     * cannot verify either MUST, so both are the implementor's responsibility.
     *
     * What the resolver does to the returned value:
     *
     * - A non-string return, or a string that normalizes to empty, is treated as
     *   "no default" and the grader falls back to its own `GraderResult.name`.
     *   Return `this.metadata.name` to opt out for a given config rather than
     *   returning `undefined`, which the declared type forbids.
     * - The implementation must be synchronous. An `async` one returns a Promise,
     *   which is rejected as a non-string.
     * - Control characters (`U+0000`–`U+001F`, `U+007F`) become spaces and
     *   whitespace runs collapse, since names render on one line.
     * - The result is truncated to at most 60 grapheme clusters, counting the
     *   ellipsis, so the cap is a true upper bound.
     * - A throw is caught and reported via `process.emitWarning`; it never fails
     *   the run.
     */
    defaultName?(config: Record<string, unknown>): string;
    /**
     * Optional: whether this configured instance's pass condition is the ABSENCE
     * of something — e.g. `tool-calls` with only `disallowed`, or the output
     * negation graders (`output-not-contains`, `output-not-matches`). MUST be a
     * pure function of `config`.
     *
     * Used by `vally oracle --no-golden-input` (the negative control). Withholding
     * the golden solution yields an empty baseline, so an absence-asserting grader
     * is expected to still pass — passing is CORRECT, not a broken grader. Oracle
     * reports such a grader as N/A for the negative control instead of flagging it
     * as trivially passing. Graders that assert PRESENCE (the default, no method)
     * are expected to fail on the baseline; a pass there is reported as a problem.
     *
     * Only implement this on graders that genuinely PASS on the empty baseline.
     * The `transcript-not-*` graders, for instance, deliberately FAIL on a
     * transcript with no assistant messages (they refuse to vacuously satisfy the
     * check), so they already yield the healthy `expected-fail` signal and must
     * NOT implement this method.
     */
    assertsAbsence?(config: Record<string, unknown>): boolean;
    /**
     * Optional comparison capability. A grader that implements this method can
     * judge trajectories head-to-head (e.g. an experiment's treatment vs its
     * baseline). The presence of this method *is* the "supports comparison"
     * signal — there is no separate metadata flag.
     *
     * In v1, `vally compare` uses the built-in `prompt` judge directly; this hook
     * exists so future callers can select a comparison-capable grader by
     * capability. There is no plugin-driven comparison-grader selection path yet.
     */
    compare?(input: GraderComparisonInput): Promise<GraderComparisonResult>;
}
/** Every grader plugin package exports a `registerGraders(registry)` function. */
export interface GraderPluginEntry {
    registerGraders: (registry: GraderRegistry) => void | Promise<void>;
}
export declare function isGraderPluginEntry(mod: unknown): mod is GraderPluginEntry;
import type { Stimulus } from "../eval/types.js";
import type { Trajectory } from "../trajectory/types.js";
import type { GraderRegistry } from "./registry.js";
//# sourceMappingURL=types.d.ts.map