/**
 * Experiment config types — defines the *.experiment.yaml format.
 *
 * An experiment config declares a controlled comparison of multiple
 * eval runs that differ along a declared axis (the `vary` fields).
 * Everything outside `vary` is validated to be identical across variants.
 */
import type { EvalDefaults, EnvironmentConfig, McpServerConfig } from "../eval/types.js";
/**
 * Top-level experiment configuration as parsed from YAML.
 *
 * Paths/globs in `evals` and inside variant overrides resolve relative
 * to the experiment file's directory.
 */
export interface ExperimentConfig {
    /** Human-readable experiment name (used in reports and provenance). */
    name: string;
    /** Eval spec files to include. Supports globs. Error on zero matches. */
    evals: string[];
    /**
     * Execution overrides that override eval-level defaults.
     * Precedence: CLI flags > experiment overrides > eval defaults.
     */
    overrides?: EvalDefaults;
    /** Runner settings (excluded from config hashing). */
    execution?: ExperimentExecution;
    /**
     * Stimulus tag filter applied to all variants consistently.
     * Same semantics as suite filters: AND across keys, OR within values.
     *
     * **Not yet wired into `vally experiment run`** — the field is
     * parsed and validated but currently has no effect at execution
     * time. Tracked as a follow-up.
     *
     * @example { priority: ["p0", "p1"], area: "auth" }
     */
    filter?: Record<string, string | string[]>;
    /**
     * Grader plugin specifiers (npm packages or local paths). Local
     * paths resolve relative to the experiment file directory.
     *
     * **Not yet wired into `vally experiment run`** — parsed and
     * validated, but the runner does not load these plugins today.
     * Tracked as a follow-up.
     */
    grader_plugins?: string[];
    /**
     * Executor plugin specifiers (npm packages or local paths). Local
     * paths resolve relative to the experiment file directory.
     *
     * **Not yet wired into `vally experiment run`** — parsed and
     * validated, but the runner does not load these plugins today.
     * Tracked as a follow-up.
     */
    executor_plugins?: string[];
    /**
     * Eval-provider plugin specifier (npm package or local path).
     * Only one eval-provider plugin may be active at a time.
     * Local paths resolve relative to the experiment file directory.
     *
     * **Not yet wired into `vally experiment run`** — parsed and
     * validated, but the runner does not load this plugin today.
     * Tracked as a follow-up.
     */
    eval_plugin?: string;
    /**
     * JSON Pointer paths that may differ between variants.
     * Subtree prefix semantics: `/environment/skills` authorizes any
     * change at or below that path. Differences outside `vary` paths
     * in the effective resolved config are hard errors.
     *
     * @example ["/environment/skills", "/defaults/model"]
     */
    vary: string[];
    /**
     * Name of the control variant. Comparison reports use this as the
     * baseline for metric deltas. Must be a key in `variants`.
     */
    baseline: string;
    /**
     * Map of variant name → override object. Each variant's overrides
     * are merged on top of the eval spec as a final overlay after
     * stimulus resolution.
     *
     * Values may contain interpolation variables like `${eval.name}`,
     * `${eval.parent}`, `${eval.grandparent}`, etc.
     */
    variants: Record<string, VariantOverride>;
}
/**
 * Fields a variant may override on the eval spec.
 *
 * v1 scope: only `overrides` and `environment` can be overridden.
 * This covers the primary use cases: varying skills, MCP servers,
 * models, git refs, files, and commands. Scoring, grader, and
 * stimulus-level overrides may be added in future versions.
 *
 * Merge semantics (different from eval environment merging):
 * - Scalars: replace
 * - Arrays: replace entirely
 * - Maps (e.g. mcpServers): deep-merge by key
 * - `null` on a map entry: delete that entry from the inherited config
 */
export interface VariantOverride {
    /** Override eval-level defaults (model, runs, timeout, etc.). */
    overrides?: Partial<EvalDefaults>;
    /**
     * Override environment fields (skills, mcpServers, git, files, commands, commandTimeout, env).
     * Set to `null` to remove the entire inherited environment.
     * Set individual map entries to `null` to delete them (e.g. mcpServers.foo: null, env.FOO: null).
     */
    environment?: VariantEnvironmentOverride | null;
}
/**
 * Environment override with deletion support for map entries.
 * Top-level arrays (skills, files, commands) replace entirely.
 * mcpServers and env are deep-merged by key; a per-key `null` deletes that
 * entry. A whole-field `null` clears all inherited entries for that field.
 */
export type VariantEnvironmentOverride = {
    [K in keyof EnvironmentConfig]?: K extends "mcpServers" ? Record<string, McpServerConfig | null> | null : K extends "env" ? Record<string, string | null> | null : EnvironmentConfig[K] | null;
};
/** Runner mechanics — not part of the eval scenario. */
export interface ExperimentExecution {
    /** Max concurrent stimuli within an eval (default: 5). */
    workers?: number;
}
/**
 * Full run summary emitted once per variant + eval combination.
 * Provides complete diagnosability of what ran.
 */
export interface ExperimentRunSummary {
    /** Record type discriminator for JSONL output. */
    type: "experiment-run-summary";
    /** Experiment config name. */
    experiment: string;
    /** Unique ID for this experiment execution. */
    runId: string;
    /** Variant ID. */
    variant: string;
    /** Eval file this summary describes. */
    evalFile: string;
    /** Content hash of the eval file. */
    evalHash: string;
    /** Hash of the effective resolved config. */
    configHash: string;
    /** Full effective resolved defaults (eval defaults merged with experiment + variant). */
    resolvedDefaults: EvalDefaults;
    /** Full effective resolved environment config. */
    resolvedEnvironment?: EnvironmentConfig;
    /** Path to the experiment config file. */
    experimentFile: string;
    /** Content hash of the experiment config file. */
    experimentHash: string;
    /** Vally version. */
    vallyVersion: string;
    /** ISO 8601 timestamp. */
    timestamp: string;
}
/**
 * Variables available for interpolation in variant override values.
 * Derived from the eval file being processed.
 *
 * All paths are relative to the experiment file's directory.
 *
 * Given `tests/dotnet-maui/maui-theming/eval.vally.yaml`:
 * - name: "maui-theming" (from eval spec `name` field)
 * - path: "tests/dotnet-maui/maui-theming/eval.vally.yaml"
 * - dir: "tests/dotnet-maui/maui-theming"
 * - basename: "eval.vally.yaml"
 * - parent: "maui-theming"
 * - grandparent: "dotnet-maui"
 */
export interface EvalInterpolationVars {
    name: string;
    path: string;
    dir: string;
    basename: string;
    parent: string;
    grandparent: string;
}
//# sourceMappingURL=types.d.ts.map