/**
 * Eval-run planner — turns a set of discovered eval specs into a flat list
 * of work items plus a summarizer that aggregates their results back into
 * per-eval and run-level verdicts.
 *
 * Mental model: a run of N evals × M models × K stimuli × R trials is
 * just `N×M×K×R` independent work items. The CLI's pool admits up to
 * `--workers` of them concurrently; reporters consume per-trial events
 * tagged with full context.
 *
 * Boundary:
 *   - Core decides WHAT to run (filter, validate, resolve env, enumerate
 *     trials, plan-time collision detection) and HOW to orchestrate one trial
 *     at the host boundary (retry, session-log lifecycle, reporting shape).
 *   - The backend decides WHERE and HOW the attempt runs, including any
 *     prepared-workspace cache.
 *   - The CLI (or any other consumer) decides WHEN each trial runs (pool
 *     admission, rate limiting), where logs/artifacts go, and how events
 *     are rendered.
 *
 * Design constraints baked into the contracts here:
 *   - `runTrialItem(item, backend, ctx)` MUST NOT reject. Runtime errors surface as
 *     `TrialResult { status: "error" }`. The CLI relies on this
 *     when it uses `Promise.all` over admitted closures — a rejection
 *     would discard already-completed sibling results.
 *   - Prepared-workspace details stay behind the backend boundary so planning
 *     remains backend-agnostic.
 *   - Reporters MUST NOT mutate shared `TrialResult` instances. The
 *     CLI owns trajectory memory lifecycle: after `summarize()`
 *     completes it MAY null out trajectory references on its retained
 *     result objects. `summarize()` reads `trajectory.id`,
 *     `trajectory.metrics`, and optionally passes through the full
 *     trajectory to reporters.
 */
import type { ProjectContext } from "../config/context.js";
import type { Diagnostic } from "../eval/diagnostics.js";
import type { SuiteFilterInput } from "../eval/suite-filter.js";
import type { Stimulus, StimulusGraderConfig, ReasoningEffort, JudgeProviderSpec } from "../eval/types.js";
import type { Executor, ExecutorSessionLogOptions } from "../executor/types.js";
import type { GraderRegistry } from "../graders/registry.js";
import type { Skill } from "../skill/types.js";
import type { Trajectory } from "../trajectory/types.js";
import type { SerializedRateLimiter } from "../utils/rate-limiter.js";
import type { RetryInfo } from "../utils/retry.js";
import { type DiscoveredEvalSpec } from "./discover-filter.js";
import type { StimulusGradeResult } from "./grading.js";
import type { MultiTrialResult } from "./types.js";
/**
 * One named variant in a {@link planRun} call. Variants are an
 * independent dimension parallel to `model`: each variant produces its
 * own `(eval × model × variant)` sub-plan.
 *
 * `planRun` always has at least one variant. Callers with no
 * conceptual variants should pass a single variant such as `"main"` or
 * `"default"`; the name appears in work-item IDs and workspace paths,
 * so pick a stable one.
 */
export interface VariantInput {
    /** Variant name. Unique within the `variants` array; must not
     *  contain `"::"` or `"\0"`, and must not be empty/whitespace-only. */
    name: string;
    /** Eval specs to run under this variant. Avoid sharing mutable
     *  `DiscoveredEvalSpec` references between variants — `planRun`
     *  currently does not mutate input specs, but callers should not
     *  rely on that. */
    specs: DiscoveredEvalSpec[];
}
/** Arguments to {@link planRun}. */
export interface PlanRunInput {
    /** Variants to run. Must be non-empty. Even single-variant callers
     *  pass `[{ name: "main", specs: [...] }]` — there is no
     *  anonymous-variant or specs-only mode. */
    variants: readonly [VariantInput, ...VariantInput[]];
    /** Project context for environment resolution. */
    projectCtx: ProjectContext;
    /**
     * Callback the CLI provides to materialize an executor for each
     * (eval × model × variant) combination. Core does not instantiate
     * executors — the CLI owns lifecycle (creation, disposal, plugin
     * loading).
     *
     * @remarks Callers SHOULD memoize by some stable key derived from
     * the arguments (typically `(variant, spec.defaults.executor, model)`).
     * `planRun` calls this once per `(eval × model × variant)` and
     * stores the reference for all trials of that combination.
     */
    getExecutor: (spec: import("../eval/types.js").RawEvalSchema, model: string | undefined, ctx: {
        variant: string;
    }) => Executor;
    /** Models to run each eval against. `[undefined]` = use eval's
     *  configured model (or the executor default). */
    models: (string | undefined)[];
    /** Override: number of trials per stimulus. */
    runs?: number;
    /** Override: pass/fail score threshold. */
    threshold?: number;
    /** Override: default model for LLM judge graders. */
    judgeModel?: string;
    /** Default reasoning effort for the LLM judge model. */
    judgeReasoningEffort?: ReasoningEffort;
    /** Reasoning effort for the agent under test (executor). Forwarded to
     *  `ExecutorOptions.reasoningEffort`. */
    reasoningEffort?: ReasoningEffort;
    /** Stimulus-level tag filter. Accepts a {@link SuiteFilter} or the legacy
     *  include-only tag map. */
    tagFilter?: SuiteFilterInput;
    /** Skip validation. */
    skipValidate?: boolean;
    /** Override: executor name (`--executor`), superseding each eval's
     *  `defaults.executor`. Forwarded to validation. */
    executor?: string;
    /** Skip grading. */
    skipGrade?: boolean;
    /** Root directory for preserved per-trial workspaces (`--workspace`). */
    workspaceRoot?: string;
    /** Custom grader registry (e.g. with LLM graders pre-registered). */
    graderRegistry?: GraderRegistry;
}
/** A materialized plan returned by {@link planRun}. */
export interface RunPlan {
    /** Per-eval planning metadata in input order, including plan-time failures.
     *  Reporters consume this in `onRunStart` to size their outputs. */
    evals: EvalPlanMetadata[];
    /** Total planned work items. Equal to `items.length`. */
    totalItems: number;
    /** Planning-time diagnostics (filter warnings, dropped stimuli, plan
     *  collisions). Each diagnostic carries an optional `scope` so per-eval
     *  warnings reach the correct markdown/JUnit section. */
    diagnostics: ScopedDiagnostic[];
    /**
     * Per planned work item: pure data describing one execution trial per
     * stimulus × trial. The CLI admits items through its pool and runs each via
     * `runTrialItem(item, backend, ctx)` — work items no longer carry an
     * `execute()` closure or a backend.
     *
     * Items are sorted lexicographically by `id` after planning, so the
     * array order is a deterministic function of the inputs (independent
     * of how the caller ordered variants, specs, or models). The order is
     * not a stable interface across vally releases.
     */
    items: TrialWorkItem[];
    /**
     * Pure aggregator over trial results. Depends only on `results` and
     * the metadata captured at plan time. Idempotent.
     *
     * When `opts.variant` is set, the returned {@link RunSummary}
     * includes only evals and results for the named variant. This is the
     * idiomatic way to produce per-variant summaries from a single
     * multi-variant plan (e.g., for per-variant output files in an
     * experiment runner). Pass an `undefined` variant filter (or omit
     * `opts`) to summarize the whole plan.
     *
     * @throws {Error} If `opts.variant` is set to a name that doesn't
     * appear in the plan's variants. Returning an empty summary would be
     * a footgun — a caller looping over typo'd variant names would print
     * "0/0 passed" as a vacuous pass.
     */
    summarize(results: TrialResult[], opts?: {
        variant?: string;
    }): RunSummary;
}
/**
 * A diagnostic with optional eval/model/variant scope. Per-eval
 * warnings (e.g. `stimuli-filtered`, `validation-failed`) carry
 * `scope` so reporters can route them to the right markdown/JUnit
 * section. Run-level diagnostics omit `scope`.
 */
export interface ScopedDiagnostic extends Diagnostic {
    scope?: {
        evalName: string;
        evalFilePath: string;
        variant: string;
        model?: string;
    };
}
/** Metadata about a single (eval × model × variant) combination,
 *  exposed at plan time so reporters can pre-allocate per-eval
 *  sections and the `onRunStart` consumer can size its outputs. */
export interface EvalPlanMetadata {
    evalName: string;
    evalFilePath: string;
    variant: string;
    model: string | undefined;
    /** Number of stimuli this combination will actually run (after tag
     *  filtering + validation). Equal to `plannedStimulusNames.length`.
     *  0 when the entire eval has a plan-time failure. */
    plannedStimulusCount: number;
    /** Total stimuli in the eval spec before tag filtering. Useful for
     *  reporting "ran X of Y stimuli". */
    inputStimulusCount: number;
    /** Names of the stimuli this combination will run, in their source
     *  order from the eval spec (after tag filtering). This is NOT
     *  the execution order — that's `RunPlan.items`, which is sorted
     *  by item id. */
    plannedStimulusNames: string[];
    /** Number of stimuli skipped because the active executor is not in
     *  their `supported_executors` allow-list. These contribute one
     *  no-op skipped work item each (not `runs` items). */
    skippedStimulusCount: number;
    /** Names of the stimuli skipped due to executor filtering, in source
     *  order. */
    skippedStimulusNames: string[];
    /** Effective trials per stimulus. Always >= 1 for valid planned evals. */
    runs: number;
    /** Effective threshold (eval scoring config or override). Undefined
     *  when scoring isn't applied. */
    threshold?: number;
    /** Present when this combination failed at plan time. */
    failure?: string;
    /** Eval-level description (from spec). Used by markdown/JUnit
     *  section headers. */
    evalDescription?: string;
    /** Effective judge model for LLM graders (after CLI override / env /
     *  spec resolution). */
    judgeModel?: string;
    /** Resolved eval-level environment block. Reporters use this to
     *  render the per-section "Environment" line. */
    environment?: import("../eval/types.js").EnvironmentConfig;
    /** Display name of the executor used for this `(eval × model)` pair
     *  (e.g. "copilot-sdk", "external"). Reporters use this in section
     *  footers; the CLI session log manager uses this for log filenames. */
    executorName?: string;
}
/**
 * A single unit of planned work: one execution trial of one stimulus
 * against one model in one eval. Each work item produces exactly one
 * {@link TrialResult}.
 */
export interface TrialWorkItem {
    /** Globally unique within a `RunPlan`, across `(eval × model ×
     *  variant × stimulus × trial)`. Reporters use this to correlate
     *  `onTrialStart` / `onTrialPhase` / `onTrialResult` events. */
    id: string;
    evalName: string;
    evalFilePath: string;
    variant: string;
    model: string | undefined;
    stimulus: Stimulus;
    /** undefined when this is the sole trial for a single-run stimulus;
     *  the 0-based trial index for multi-trial stimuli. */
    trialIndex: number | undefined;
    /** Number of execution trials this stimulus runs. 1 for single-run,
     *  K for multi-trial. Always >= 1. Reporters that bucket per stimulus
     *  flush when they have observed this many results for the bucket. */
    totalTrials: number;
    /** Whether the CLI should treat this trial as eligible for
     *  retry-on-rate-limit. Set at plan time as `totalTrials === 1`.
     *  Carried as an explicit field rather than inferred from
     *  `totalTrials` so future trial-count semantic shifts don't silently
     *  change retry behavior. */
    retryEligible: boolean;
    /**
     * Plan-computed execution inputs. The caller passes this work item (with a
     * resolved {@link import("../backend/types.js").Backend} and a
     * {@link TrialExecuteContext}) to
     * `runTrialItem`, which owns the host orchestration and calls
     * `backend.runTrial()`. The planner does not execute — it only describes.
     */
    run: TrialRunSpec;
}
/**
 * The plan-computed inputs `runTrialItem` needs to execute one trial — the data
 * the per-trial `execute()` closure used to capture from plan scope. Holds live
 * host objects (executor, grader registry); it is host-side and not serialized
 * (a relocating backend extracts its own serializable subset inside `runTrial`).
 */
export interface TrialRunSpec {
    /** Executor that runs the agent for this trial. */
    executor: Executor;
    /** Executor-specific config (opaque), forwarded to the executor's `executorConfig` option. */
    executorConfig?: unknown;
    /** The trial's base working directory (also the executor `workDir`). */
    baseDir: string;
    /** The `--workspace` destination for this trial, or undefined when unset. */
    trialWorkspace: string | undefined;
    /** Grader configs for this stimulus. */
    graderConfigs: StimulusGraderConfig[];
    /** Grader registry used to resolve grader types. */
    graderRegistry?: GraderRegistry;
    /** Eval-level default judge model; fallback when `ctx.judgeModel` is unset. */
    judgeModel?: string;
    /** Eval-level default reasoning effort for the judge model. */
    judgeReasoningEffort?: ReasoningEffort;
    /** Eval-level BYOK judge provider (`defaults.judge_provider`). */
    judgeProvider?: JudgeProviderSpec;
    /** Reasoning effort for the agent under test, forwarded to the executor. */
    reasoningEffort?: ReasoningEffort;
    /**
     * Hard wall-clock cap in ms; fallback when `ctx.timeout` is unset. Resolved
     * from the stimulus `max_duration` constraint, else the eval-level default.
     */
    timeout?: number;
    /** Agent working-time limit in ms, from the stimulus `max_agent_duration` constraint. */
    maxAgentDurationMs?: number;
    /** Per-grader scoring weights. */
    scoringWeights?: Record<string, number>;
    /** Orchestration flag: skip grading entirely (distinct from empty graders). */
    skipGrade: boolean;
    /**
     * When set, this trial is a no-op skip: {@link runTrialItem} returns a
     * `status: "skipped"` result with this reason without invoking the
     * executor or graders. Set for stimuli whose active executor is not in
     * their `supported_executors` allow-list.
     */
    skip?: {
        reason: string;
    };
}
/**
 * Inputs to the two host-destination seams
 * ({@link TrialExecuteContext.resolveArtifactsDir} and
 * {@link TrialExecuteContext.resolveExecutorArtifactMaterializeDir}). Both seams
 * ONLY compute a path from this; {@link runTrialItem} drives any copy/export.
 */
export interface ArtifactResolveInfo {
    stimulus: Stimulus;
    trajectoryId: string;
    trialIndex: number | undefined;
    /** Directory of the successful attempt's session log, when one was allocated.
     *  Lets the CLI co-locate outputs with the session log. */
    sessionDir?: string;
}
/** Read-only context the CLI passes to {@link runTrialItem} as its `ctx` argument. */
export interface TrialExecuteContext {
    /** Skills to pass to the executor. */
    skills?: Skill[];
    /** Override: per-stimulus timeout in ms. */
    timeout?: number;
    /** Override: judge model for LLM graders. */
    judgeModel?: string;
    /** Override: reasoning effort for the judge model. */
    judgeReasoningEffort?: ReasoningEffort;
    /**
     * Per-trial diagnostics flow here. The CLI fans them to the reporter
     * with the trial's scope already attached.
     */
    onDiagnostic?(d: Diagnostic): void | Promise<void>;
    /**
     * Phase callback for live progress UI. Phase is a stable enum;
     * reporters that own UI (e.g. the console spinner) format display
     * strings from the enum + structured detail. Awaited so reporters
     * can serialize redraws without races.
     */
    onPhase?(phase: TrialPhase, detail?: TrialPhaseDetail): void | Promise<void>;
    /**
     * Optional raw-event sink forwarded to the executor. Per-trial sinks
     * (separate file streams etc.) belong here so concurrent trials don't
     * share an output stream.
     */
    onRawEvent?(event: unknown): void;
    /**
     * Per-attempt session-log allocator. The CLI provides this when
     * preserving session logs; core invokes it once per attempt
     * (including retry attempts) and lifecycles the session around the
     * inner runEval call. Pass `undefined` (or omit) to disable session
     * logs.
     *
     * `attemptIndex` is the 0-based attempt counter for retry-eligible
     * trials, or `undefined` for trials that don't retry. The session-log
     * implementation uses this to disambiguate retry artifacts on disk.
     */
    acquireSessionLog?(attemptIndex: number | undefined): Promise<TrialSessionLog | undefined>;
    /**
     * Rate limiter, gated before each attempt's inner executor call.
     * Provided by the CLI when retries are enabled.
     */
    rateLimiter?: SerializedRateLimiter;
    /**
     * Maximum retries for retry-eligible trials. 0 (default) disables
     * the retry loop; the closure makes exactly one attempt and surfaces
     * failures as `TrialResult { status: "error" }`.
     */
    maxRetries?: number;
    /**
     * Called before each retry's delay. Used by the CLI to emit a
     * `stimulus-retry` diagnostic. The plan's closure passes this through
     * to {@link withStimulusRetry}.
     */
    onRetry?(info: RetryInfo): void | Promise<void>;
    /**
     * Resolve the host destination directory for a stimulus's configured
     * artifacts. The CLI owns this layout. Return `undefined` to skip
     * artifact export. This seam ONLY computes the path; it MUST NOT copy
     * files or emit diagnostics — {@link runTrialItem} drives the export and
     * surfaces copy diagnostics via {@link TrialExecuteContext.onDiagnostic}.
     *
     * Contrast {@link resolveExecutorArtifactMaterializeDir}: this one is the
     * destination for stimulus-declared **output artifacts** selected out of the
     * workspace (`{kind:"artifacts"}`); that one is the destination for
     * materializing a relocating backend's whole **executor artifact directory**
     * (the source of `trajectory.artifactDir`). Same signature, different targets.
     */
    resolveArtifactsDir?(info: ArtifactResolveInfo): string | undefined;
    /**
     * Resolve the host-readable destination for a relocating backend's executor
     * artifact directory (the far-side source of
     * {@link import("../trajectory/types.js").Trajectory.artifactDir}). The CLI
     * owns this layout — typically the session's durable `executorArtifactsDir`.
     * Return `undefined` when no durable location is available (materialization
     * is then skipped). Like {@link resolveArtifactsDir} this seam ONLY computes
     * the path; {@link runTrialItem} drives the materialization and rewrites
     * `trajectory.artifactDir` to the placed path. See that sibling for the
     * artifacts-vs-artifact-dir distinction (easy to confuse — same signature).
     */
    resolveExecutorArtifactMaterializeDir?(info: ArtifactResolveInfo): string | undefined;
    /**
     * Resolve the host path for `workspace.patch`. The *presence* of this
     * function (not its return value) enables capture in `runEval`. Returning
     * `undefined` skips the write but not the capture.
     */
    resolveWorkspacePatchPath?(info: ArtifactResolveInfo): string | undefined;
}
/**
 * Per-attempt session log handle. The CLI's
 * {@link TrialExecuteContext.acquireSessionLog} returns one of these
 * per attempt; the plan's execute closure threads
 * {@link TrialSessionLog.executorOptions} into the inner `runEval`
 * call and finalizes via `complete()` / `fail()`.
 */
export interface TrialSessionLog {
    /** Absolute path of the directory this attempt's session log was written
     *  to, if the provider materializes one. Lets cleanup-time consumers
     *  (artifact copy) co-locate output with the session log. */
    directory?: string;
    /** Forwarded to `runEval` as `sessionLog`, then to the executor. */
    executorOptions?: ExecutorSessionLogOptions;
    /** Per-event raw fallback when the executor doesn't write a native
     *  events.jsonl. */
    onRawEventFallback?: (event: unknown) => void;
    /** Import backend-collected log files before finalizing the session.
     *  Optional: only relocating backends populate `TrialOutcome.logs`,
     *  and consumers that don't relocate work need not implement it. */
    importLogs?(logs: {
        rawEventsPath?: string;
        sessionLogPath?: string;
    }): Promise<void>;
    /** Finalize on success. The optional update lets the closure
     *  enrich the session metadata with trajectory/executor session ids
     *  observed during the run. */
    complete(update?: {
        trajectoryId?: string;
        executorSessionId?: string;
    }): Promise<void>;
    /** Finalize on failure. */
    fail(error: unknown): Promise<void>;
}
/**
 * Phase of a single work item. Stable enum — reporters that render
 * progress (CLI spinner) format display strings from this +
 * {@link TrialPhaseDetail}.
 *
 * There is no `"queued"` value: an item is queued iff the reporter has
 * not yet received `onTrialStart` for it; the absence of the start
 * event is the signal. Adding a `"queued"` enum would require an
 * emitter at admission time which the CLI's pool doesn't have.
 */
export type TrialPhase = "preparing" | "running-prompt" | "running-graders" | "completed" | "errored";
export interface TrialPhaseDetail {
    /** Zero-based index of the grader currently running, for the `running-graders` phase. */
    graderIndex?: number;
    /** Display name of the grader currently running, for the `running-graders` phase. */
    graderName?: string;
    /** Number of graders about to run for the `running-graders` phase. */
    graderTotal?: number;
}
/**
 * The result of one {@link runTrialItem} call. Carried into reporters via
 * `onTrialResult` and into `summarize()`.
 *
 * Reporters MUST NOT mutate the shared instance. The CLI/composite owns
 * trajectory memory lifecycle: after `summarize()` completes, the CLI
 * MAY null out its retained `trajectory` reference. `summarize()` reads
 * `trajectory.id`, `trajectory.metrics`, and optionally passes through
 * the trajectory.
 */
export interface TrialResult {
    /** Matches the originating `TrialWorkItem.id`. */
    itemId: string;
    /** Wall-clock duration of one {@link runTrialItem} call, up to (but not
     *  including) post-run workspace teardown. */
    durationMs: number;
    /** Outcome status. Maps to today's `EvalOutcome.status`. */
    status: "success" | "error" | "skipped";
    /** Captured trajectory. `null` when the executor errored before
     *  producing one. */
    trajectory: Trajectory | null;
    /** Top-level grading result with per-grader breakdown. `null` when
     *  `--skip-grade` was set or no graders were configured. */
    grade: StimulusGradeResult | null;
    /** Error message when `status === "error"`. Absent for success. */
    error?: string;
    /** Human-readable reason when `status === "skipped"` (e.g. the active
     *  executor is not in the stimulus's `supported_executors` allow-list). */
    skipReason?: string;
    /** When set, the per-trial workspace was preserved at this absolute
     *  path. Reporters print "Workspace preserved: <path>" off this field
     *  rather than receiving a separate event. */
    workspacePath?: string;
}
/** Aggregated output of `RunPlan.summarize(results)`. */
export interface RunSummary {
    /** Per-eval verdicts in input order, including plan-time failures. */
    evals: EvalSummary[];
    /** True iff every eval verdict passed. Callers may opt into gating on this verdict. */
    passed: boolean;
    /** True when any trial reported `status === "error"`.
     *  Disambiguates "all evals passed but execution had issues" from a
     *  clean run. */
    hadExecutionErrors: boolean;
}
/** Per-eval verdict produced by `summarize()`. */
export interface EvalSummary {
    evalName: string;
    evalFilePath: string;
    variant: string;
    model: string | undefined;
    passed: boolean;
    /** True when any trial in this (eval × model × variant) errored. Forces
     *  the eval to fail regardless of score; reporters surface it as the
     *  failure reason. */
    hadExecutionErrors?: boolean;
    /** Set when this eval failed at plan time (validation, environment,
     *  threshold, workspace collision). Takes precedence over `stimuli`. */
    error?: string;
    /** Per-stimulus aggregates. Empty when `error` is set. */
    stimuli: StimulusSummary[];
    /** Threshold-based aggregate score (0..1). Present only when scoring
     *  was applied. */
    overallScore?: number;
    /** Effective threshold. Present only when scoring was applied. */
    threshold?: number;
    /** True when threshold-based scoring was applied (the eval has a
     *  scoring.threshold config and at least one stimulus produced
     *  scores). */
    scoringApplied: boolean;
    /** Sum of trial durations in this (eval × model × variant). */
    durationMs: number;
    /** Number of stimuli that produced trial results (excludes
     *  filtered + plan-time failures). */
    stimuliRun: number;
    /** Total stimuli in the eval spec (before any filter). */
    stimuliTotal: number;
    /** Number of stimuli skipped because the active executor was not in
     *  their `supported_executors` allow-list. Skipped ≠ failed. */
    stimuliSkipped: number;
    /** Per-stimulus skip details (name + reason) for reporters. Empty
     *  when nothing was skipped. */
    skippedStimuli: SkippedStimulusSummary[];
}
/** A stimulus skipped because the active executor was not compatible. */
export interface SkippedStimulusSummary {
    stimulusName: string;
    reason: string;
}
/**
 * Per-stimulus summary inside an `EvalSummary`. Carries the raw
 * work-item results for the bucket plus convenience projections.
 *
 * - `results`: complete work-item stream for the stimulus bucket, in
 *   plan order. This is the canonical view; reporters that don't need
 *   variant-specific helpers should consume this directly.
 * - `multiTrial`: synthetic aggregate over trial results for reporters
 *   that consume the multi-trial shape directly.
 */
export interface StimulusSummary {
    stimulusName: string;
    results: TrialResult[];
    multiTrial: MultiTrialResult;
}
/**
 * Materialize a {@link RunPlan} for the supplied input. Filters
 * stimuli by tag, runs validation, resolves environments, decides on
 * prepared-workspace caching per stimulus, performs cross-eval
 * workspace-collision detection, and enumerates the trial work items.
 *
 * Async because planning may load and resolve eval/environment state.
 *
 * The order of the returned `RunPlan.items` is stable: semantically
 * identical `input` always yields the same order.
 *
 * @throws {Error} If any model name in `input.models` is invalid, or if
 * multiple model entries resolve to the same effective model for an eval.
 * All other planning failures surface as `EvalPlanMetadata.failure`
 * (so `summarize` can include them in the grid) or as `diagnostics`
 * entries (for warnings).
 */
export declare function planRun(input: PlanRunInput): Promise<RunPlan>;
/**
 * Build the key for the per-stimulus grader-config map consumed by
 * {@link summarizeOutcomes}. Identical to the key {@link summarizePlan} looks up
 * internally, exposed at module scope so the artifact-reconstruction helper
 * (`experiment/report-from-artifacts.ts`) can assemble a compatible map. Keep
 * the `evalFilePath` convention consistent with the {@link EvalPlanMetadata}
 * passed alongside — relative for artifact-reconstructed runs.
 */
export declare function stimulusGraderMapKey(variant: string, evalFilePath: string, model: string | undefined, stimulusName: string): string;
/**
 * Fallback skip reason used when a `status: "skipped"` result reaches a
 * consumer without a specific `skipReason`. The planner always sets a
 * detailed reason; this guards display/pass-through paths against an
 * empty message.
 */
export declare const DEFAULT_EXECUTOR_SKIP_REASON = "executor not supported";
/**
 * Aggregate a flat set of {@link TrialResult}s into a {@link RunSummary} from
 * caller-supplied plan metadata, independent of a live {@link RunPlan}.
 *
 * This is the stable entry point for callers that reconstruct a run from
 * persisted artifacts rather than executing it — chiefly `vally experiment
 * merge`, which rebuilds the report from a `plan-snapshot.json` plus the
 * concatenated per-trial JSONL of several shards. {@link RunPlan.summarize} is
 * the in-process equivalent; both delegate to the same pure aggregator.
 *
 * Identity is matched purely by string keys: each result's `itemId` and each
 * `EvalPlanMetadata.evalFilePath` must agree on the *same* path convention. The
 * persisted artifacts are keyed on the experiment-relative eval path (the
 * `shardKey`), so reconstructing callers set `itemId = shardKey` and
 * `evalFilePath = <relative eval file>`; the aggregator then buckets correctly
 * without ever resolving a filesystem path, which is what makes a merged report
 * machine-independent. The `shardKey` format intentionally mirrors the internal
 * item-id format so this substitution is exact.
 *
 * Pure and idempotent — no I/O, no `RunPlan` lifecycle. When `opts.variant` is
 * set, only that variant's evals and results are summarized.
 */
export declare function summarizeOutcomes(results: TrialResult[], evals: EvalPlanMetadata[], stimulusGraders: Map<string, StimulusGraderConfig[]>, opts?: {
    skipGrade?: boolean;
    variant?: string;
}): RunSummary;
//# sourceMappingURL=plan.d.ts.map