/**
 * Eval schema types — defines the eval.yaml format.
 *
 * Follows the conventions from PR #19: multi-skill environments,
 * two-level environment merging, graders via StimulusGraderConfig.
 */
export type EvalType = "capability" | "regression";
import type { Duration } from "../utils/duration.js";
export type { Duration } from "../utils/duration.js";
import type { ReasoningEffort } from "../graders/llm/types.js";
export type { ReasoningEffort } from "../graders/llm/types.js";
import type { AgentScope } from "../trajectory/turn.js";
import type { AtifTrajectory } from "../trajectory/atif-types.js";
import type { JudgeProviderSpec } from "../provider/copilot-provider.js";
export type { CopilotSdkProviderConfig, JudgeProviderSpec } from "../provider/copilot-provider.js";
export interface EvalSchema {
    name: string;
    description?: string;
    version?: string;
    type?: EvalType;
    tags?: Record<string, string | string[]>;
    defaults: EvalDefaults;
    stimuli: Stimulus[];
    scoring?: ScoringConfig;
    /**
     * Root-level grading environment (merged into every stimulus). Peer of the
     * raw eval's root agent environment ({@link RawEvalSchema} `environment` /
     * `agent_environment`) — see {@link GradingEnvironmentConfig}.
     */
    grading_environment?: GradingEnvironmentConfig;
}
/**
 * The grading environment — a peer of the agent `environment`, staged for
 * graders only. Files listed here are overlaid into the workspace **only for
 * the grading step** (after the agent finishes and after the workspace diff is
 * captured) and removed again before any workspace is preserved or exported, so
 * the agent under test never sees them and they don't appear in the recorded
 * diff or a preserved/exported workspace.
 *
 * This is the home for answer keys, golden data, hidden test suites, and other
 * fixtures graders need but that would give the agent an unfair advantage. It is
 * a separate block (not a field on {@link EnvironmentConfig}) so grader-only
 * inputs are structurally distinct from agent-visible ones: you can't leak a
 * secret into the run by forgetting a flag.
 *
 * In eval YAML this is authored as the top-level `grading_environment` key,
 * peer to `agent_environment` (whose deprecated alias is `environment`).
 */
export interface GradingEnvironmentConfig {
    /**
     * Grader-only files. Same `{ src, dest }` shape as {@link EnvironmentConfig}
     * `files` (paths relative to the eval file); `dest` is where the file lands in
     * the workspace for graders to read.
     */
    files?: Array<{
        src: string;
        dest: string;
    }>;
}
export interface EvalDefaults {
    runs?: number;
    timeout?: Duration;
    model?: string;
    reasoning_effort?: ReasoningEffort;
    /**
     * Executor for this eval: a bare registered name, or an `{ name, config }`
     * object pairing the name with executor-specific config.
     *
     * Note: this widened from `string` to {@link ExecutorSelection}. Consumers
     * that previously read `defaults.executor` as a `string` should narrow with
     * the exported {@link resolveExecutorName} / {@link resolveExecutorConfig}
     * helpers rather than assuming a string.
     */
    executor?: ExecutorSelection;
    judge_model?: string;
    judge_reasoning_effort?: ReasoningEffort;
    /**
     * Bring-your-own-key (BYOK) model provider for the LLM *judge* graders
     * (`prompt`, `panel`). Points the judge at a custom OpenAI-compatible endpoint
     * instead of the default GitHub/Copilot auth chain — the judge counterpart to
     * `defaults.executor.config.provider` for the agent. Applies to every LLM
     * judge call in the eval (`judge_model` picks *which* model; this picks
     * *where* the request is sent). May instead be the `{ source: "copilot-env" }`
     * selector to resolve the provider from ambient `COPILOT_PROVIDER_*` variables.
     */
    judge_provider?: JudgeProviderSpec;
}
/** Registered executor name. String literals are hints; any registered name is valid. */
export type ExecutorName = "copilot-sdk" | "mock" | (string & {});
/**
 * How an eval selects its executor: either a bare executor name, or an object
 * pairing the name with executor-specific `config`.
 *
 * The `config` payload is opaque to vally core — the selected executor owns
 * parsing and validation of it (see `Executor.validateConfig`). This keeps
 * backend-specific options (e.g. the copilot-sdk BYOK provider block) out of
 * the shared pipeline types. Core fails closed when `config` is supplied for an
 * executor that declares no `validateConfig` hook.
 */
export type ExecutorSelection = ExecutorName | ExecutorSpec;
export interface ExecutorSpec {
    /** Registered executor name (e.g. `"copilot-sdk"`, `"mock"`). */
    name: ExecutorName;
    /**
     * Executor-specific configuration, interpreted by the named executor. For the
     * `copilot-sdk` executor this is a `CopilotSdkExecutorConfig` (BYOK provider).
     */
    config?: unknown;
}
export interface Stimulus {
    name: string;
    /**
     * The prompt sent to the agent. For multi-turn stimuli, this is synthesized
     * by joining `turns` — used for reporting and LLM judges, not execution.
     */
    prompt: string;
    /**
     * Ordered prompts for a multi-turn conversation. Each entry is sent
     * sequentially to the same agent session, preserving conversation context.
     */
    turns?: string[];
    /**
     * File paths (images or native documents) attached to the agent's initial prompt.
     * In raw eval YAML, paths are relative to the eval file (absolute paths allowed); during planning they are resolved to absolute paths.
     * Only executors advertising `supportsAttachments` accept them.
     */
    attachments?: string[];
    tags?: Record<string, string | string[]>;
    environment?: EnvironmentConfig;
    /**
     * Stimulus-level grading environment (merged with the root
     * {@link EvalSchema} `grading_environment`). See {@link GradingEnvironmentConfig}.
     */
    grading_environment?: GradingEnvironmentConfig;
    artifacts?: ArtifactsConfig;
    graders?: StimulusGraderConfig[];
    rubric?: string[];
    constraints?: StimulusConstraints;
    /**
     * Optional allow-list of executor IDs (registry names, as passed to
     * `--executor` / `defaults.executor`) that this stimulus is compatible
     * with. When set, the stimulus is skipped (counted as skipped, not
     * failed) on any run whose active executor is not in the list. When
     * absent, the stimulus runs under every executor (default behavior).
     * An empty array is rejected at load time.
     */
    supported_executors?: string[];
    /**
     * Reference-solution diff for oracle / golden-patch grading. Applied to the
     * stimulus's environment before grading; also exposed to `vally oracle`.
     */
    golden_patch?: GoldenPatch;
    /**
     * Reference-solution trajectory for oracle grading. When set, `vally oracle`
     * grades against its events and metrics instead of a synthesized empty one, so
     * trajectory-, metric-, and transcript-scoped graders can be validated offline
     * without running an agent.
     */
    golden_trajectory?: GoldenTrajectory;
    /**
     * Reference-solution custom metrics for oracle grading. When set, `vally oracle`
     * materializes this JSON into the workspace so `custom-metrics` graders can be
     * validated offline without an agent run.
     */
    golden_custom_metrics?: GoldenCustomMetrics;
    /**
     * Tool-call simulation: canned responses for specific first-party tools so the
     * agent runs against a deterministic, mocked toolset instead of the real
     * environment. Only executors that advertise `supportsSimulation` accept this;
     * `runEval()` rejects it before invoking an executor without the flag.
     */
    simulation?: SimulationConfig;
}
/**
 * A reference-solution diff for a stimulus. Exactly one of `inline` (embedded
 * text) or `path` (file path relative to the eval file) must be set — the
 * discriminated union encodes that exclusivity at the type level.
 */
export type GoldenPatch = {
    inline: string;
    path?: never;
} | {
    inline?: never;
    path: string;
};
/**
 * Reference-solution trajectory for a stimulus, used by `vally oracle` to
 * validate trajectory-/metric-/transcript-scoped graders without running an
 * agent. Exactly one of `inline` (an ATIF document) or `path` (an ATIF JSON file
 * relative to the eval file) must be set. `inline` is typed as an
 * {@link AtifTrajectory} for consumers, but untyped (YAML/JSON) inputs are still
 * fully validated as ATIF at oracle time, not load time.
 */
export type GoldenTrajectory = {
    inline: AtifTrajectory;
    path?: never;
} | {
    inline?: never;
    path: string;
};
/**
 * Reference-solution custom-metrics document for a stimulus, used by
 * `vally oracle` to validate `custom-metrics` graders without running an agent.
 * Exactly one of `inline` (a metrics object) or `path` (a JSON file relative to
 * the eval file) must be set. Oracle writes the resolved metrics into the
 * workspace where the grader reads them.
 */
export type GoldenCustomMetrics = {
    inline: Record<string, unknown>;
    path?: never;
} | {
    inline?: never;
    path: string;
};
export interface ArtifactsConfig {
    include: string[];
    exclude?: string[];
}
/** Returns `stimulus.turns` for multi-turn stimuli, or `[stimulus.prompt]` for single-turn. */
export declare function stimulusPrompts(stimulus: Pick<Stimulus, "prompt" | "turns">): string[];
/**
 * Backs `environment.git`: materializes the evaluation workspace from a git
 * repo. Either a local worktree (`type: "worktree"`) or a remote clone
 * (`type: "clone"`), discriminated on `type`.
 */
export type GitConfig = GitWorktreeConfig | GitCloneConfig;
/**
 * Materialize the workspace as a detached git worktree of a *local* source repo.
 */
export interface GitWorktreeConfig {
    type: "worktree";
    /**
     * A commit-ish value (ie: a tag, a commit, a branch)
     * Ref: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-commit-ishalsocommittish
     */
    ref: string;
    /** The path to the local repo we'll use as the source of the worktree. */
    source: string;
    /**
     * Timeout for the local git steps that materialize the worktree
     * (`git worktree add`, and teardown via `git worktree remove`), e.g. `"5m"`.
     * Raise it for very large repos where checkout takes longer than the default.
     * Must be a positive duration; `0` is rejected (it would disable the timeout
     * and let setup hang). Defaults to `5m` when unset.
     */
    timeout?: Duration;
}
/**
 * Materialize the workspace by cloning a *remote* repository at a given ref.
 */
export interface GitCloneConfig {
    type: "clone";
    /** Remote repository URL to clone: http(s), ssh, git, file://, or scp-style git@host:path. */
    url: string;
    /**
     * A commit-ish value (tag, commit SHA, or branch) to check out after cloning.
     * When omitted, the remote's default branch is checked out.
     * Ref: https://git-scm.com/docs/gitglossary#Documentation/gitglossary.txt-commit-ishalsocommittish
     */
    ref?: string;
    /**
     * Shallow-clone the history to reduce transfer for data-only fixtures.
     * `true` fetches a single commit (depth 1); a number sets an explicit depth.
     * Omit (or `false`) to fetch full history.
     */
    shallow?: boolean | number;
    /**
     * Sparse-checkout patterns (cone mode). When set, only these directories/paths
     * are materialized in the working tree, leaving the rest of the repo out.
     */
    sparse?: string[];
    /**
     * Timeout for each local git step of the clone (`init`, `remote add`,
     * `sparse-checkout`, `checkout`), e.g. `"5m"`. Raise it for very large repos
     * where checkout takes longer than the default. Does *not* apply to the
     * network `fetch`, which keeps its own longer network budget. Must be a
     * positive duration; `0` is rejected. Defaults to `5m` when unset.
     */
    timeout?: Duration;
}
/**
 * Shared fields for all MCP server configurations.
 */
export interface McpServerConfigBase {
    /** Timeout for connecting to / invoking the server (e.g. 30s, 5000ms). */
    timeout?: Duration;
}
/**
 * MCP server launched as a child process (stdio transport).
 */
export interface McpLocalServerConfig extends McpServerConfigBase {
    type: "stdio";
    /** Executable to run. */
    command: string;
    /** Arguments passed to the command. */
    args?: string[];
    /** Extra environment variables for the child process. */
    env?: Record<string, string>;
    /** Working directory for the child process. */
    cwd?: string;
}
/**
 * MCP server accessed over HTTP / SSE.
 */
export interface McpRemoteServerConfig extends McpServerConfigBase {
    type: "http" | "sse";
    /** Server endpoint URL. */
    url: string;
    /** Extra HTTP headers (e.g. auth tokens). */
    headers?: Record<string, string>;
}
/**
 * Discriminated union of MCP server configurations.
 */
export type McpServerConfig = McpLocalServerConfig | McpRemoteServerConfig;
export interface EnvironmentConfig {
    skills?: string[];
    git?: GitConfig;
    /**
     * Files copied into the run environment before the agent starts.
     *
     * `dest_root` selects where `dest` is rooted:
     * - `"workdir"` (default) — `dest` is joined onto the graded workspace, so the
     *   file is visible to the agent and captured in the diff baseline (today's
     *   behavior).
     * - `"assets"` — `dest` is joined onto a managed, per-trial assets directory
     *   that lives *outside* the graded workspace. Such files never contaminate
     *   `workDir`; they are referenceable from `environment.env` via the
     *   `${EVALUATE_ASSETS}` token and by graders via `trajectory.assetsDir`
     *   (program graders receive it as `EVALUATE_ASSETS`).
     */
    files?: Array<{
        src: string;
        dest: string;
        dest_root?: "workdir" | "assets";
    }>;
    commands?: string[];
    /** Per-command timeout for `commands` setup steps (e.g. 2m, 90s). Defaults to 60s when unset. */
    commandTimeout?: Duration;
    mcpServers?: Record<string, McpServerConfig>;
    /**
     * Environment variables for the agent's spawned process. Distinct from
     * `McpLocalServerConfig.env` (which configures an MCP child).
     *
     * Not for secrets: values can surface in raw trajectory output, and error
     * redaction is only best-effort. (Reports and provenance emit names only.)
     */
    env?: Record<string, string>;
}
export interface StimulusGraderConfig {
    type: string;
    /**
     * Stable identifier for this grader instance, used as a key wherever results
     * are consumed programmatically (JUnit property keys, the analytics store).
     * Constrained to a slug and unique within the eval file. When omitted,
     * reports fall back to a label derived from this grader's config, which is
     * for humans only and changes whenever that config does.
     */
    name?: string;
    /** Scope this grader to a specific conversation turn (0-based). The pipeline slices the trajectory to that turn before grading. */
    turn?: number;
    /**
     * Scope this grader to the parent agent or a specific subagent. The pipeline
     * slices the trajectory to the selected agent's events before grading, and it
     * composes with `turn`. `"parent"` selects the root agent only, `"subagent"`
     * any subagent, and `{ agent: "<name/id>" }` a specific subagent.
     */
    scope?: AgentScope;
    config?: Record<string, unknown>;
}
export interface StimulusConstraints {
    max_turns?: number;
    max_tokens?: number;
    /** Hard wall-clock cap on the agent's in-flight run — workspace setup and teardown/disconnect are bounded separately, not by this cap. Exceeding it aborts and **fails** the run. */
    max_duration?: Duration;
    /** Agent working-time limit. Agent gets this long to work, then is stopped cleanly and the partial run **passes**. */
    max_agent_duration?: Duration;
}
/**
 * Tool-call simulation configuration for deterministic, side-effect-free
 * behavioral evals, applied via the Copilot SDK's in-process `onPreToolUse` hook.
 *
 * Scope: first-party tools only. MCP server tools are dispatched through a
 * separate SDK hook and are intentionally not simulated here.
 */
export interface SimulationConfig {
    /**
     * Per-tool overrides keyed by tool name (e.g. `bash`, `web_fetch`, `view`).
     *
     * Shell-like tools (`bash`/`shell`/`powershell`, resolved as aliases of one
     * another) can return **successful** or **error** output — the command is
     * rewritten so execution produces the canned result. All other first-party
     * tools can only simulate **failures** (the tool is denied and the model sees
     * the canned text as an error); a non-shell override that resolves to a
     * successful result is rejected at validation time.
     *
     * A value may be:
     * - a plain string — shorthand for `{ output: <str>, is_error: false }`
     *   (a successful stdout);
     * - a {@link ToolOverride} object (`output` + optional `is_error`);
     * - a {@link BashPatternOverride} for input-dependent shell tools.
     */
    tool_overrides?: Record<string, string | ToolOverride | BashPatternOverride>;
    /**
     * Maximum number of model iterations (assistant turns) before the run is
     * stopped cleanly. This is a simulation-only backstop for loop tests, distinct
     * from `constraints.max_turns` (a conversation-turn count) and
     * `constraints.max_duration`/`max_agent_duration` (wall-clock caps). When the
     * cap is hit the run ends cleanly with `endReason: "simulation_cap"`. This is
     * not a run error — graders still run on the collected (partial) trajectory,
     * which may pass or fail.
     */
    max_iterations?: number;
}
/** A single fixed tool override. */
export interface ToolOverride {
    /** The output text returned for every invocation of this tool. */
    output: string;
    /**
     * Whether the output represents an error. Default: `false` (successful
     * stdout). For non-shell tools only `true` is supported (see
     * {@link SimulationConfig.tool_overrides}).
     */
    is_error?: boolean;
}
/**
 * Pattern-based override for input-dependent shell tools: the command string is
 * tested against each entry's `match` (case-insensitive regex) in order and the
 * first match wins. Non-matching commands fall back to `default`, or — when no
 * default is given — pass through to real execution.
 */
export interface BashPatternOverride {
    patterns: BashPattern[];
    /** Response when no pattern matches. Omit to pass through to real execution. */
    default?: string | ToolOverride;
}
/** A single pattern-match rule for shell overrides. */
export interface BashPattern {
    /** Case-insensitive regex tested against the shell command string. */
    match: string;
    /** The output text returned when this pattern matches. */
    output: string;
    /** Whether the output represents an error (non-zero exit). Default: `false`. */
    is_error?: boolean;
}
export interface ScoringConfig {
    weights?: Record<string, number>;
    threshold?: number;
}
/**
 * The agent-environment keys on a raw (unresolved) spec. `agent_environment` is
 * the preferred name; `environment` is its deprecated alias. Set at most one —
 * the loader rejects both — and `resolveStimulus()` normalizes to the canonical
 * `environment` field on the resolved {@link Stimulus}/{@link EvalSchema}.
 */
export type RawAgentEnvironment = {
    environment?: string | EnvironmentConfig;
    agent_environment?: string | EnvironmentConfig;
};
/**
 * Raw stimulus as parsed from YAML — environment may be a string ref to a named environment.
 * Use `resolveStimulus()` to convert to the resolved `Stimulus` form.
 */
export type RawStimulus = Omit<Stimulus, "environment" | "prompt"> & RawAgentEnvironment & {
    /**
     * Optional in raw YAML. When `turns` is provided the loader always synthesizes
     * `prompt` from the joined turns, replacing any explicit value so execution and
     * reporting/LLM graders can't diverge.
     */
    prompt?: string;
};
/**
 * Raw eval schema as parsed from YAML — environments may be string refs.
 * Use `resolveStimulus()` to resolve individual stimuli.
 */
export type RawEvalSchema = Omit<EvalSchema, "stimuli"> & RawAgentEnvironment & {
    stimuli: RawStimulus[];
    /**
     * Optional source path set by an eval provider to give each returned schema
     * a distinct identity. When present the CLI uses this value as the `filePath`
     * key (used for run bucketing and diagnostics) instead of `args.path`.
     */
    filePath?: string;
};
//# sourceMappingURL=types.d.ts.map