/**
 * Backend plugin type — a first-class abstraction orthogonal to executors.
 *
 * Executors answer "which AI agent?"; backends answer "where does execution
 * happen?" A backend is applied by the runner: {@link import("../pipeline/trial-runner.js").runTrialItem}
 * owns the backend-agnostic host orchestration and calls {@link Backend.runTrial}
 * for the per-trial core, while planning, scoring, and reporting stay in core.
 */
import type { Stimulus, StimulusGraderConfig } from "../eval/types.js";
import type { Diagnostic } from "../eval/diagnostics.js";
import type { EvalRunOptions } from "../pipeline/run.js";
import type { GradeTrajectoryOptions, StimulusGradeResult } from "../pipeline/grading.js";
import type { TrialPhase, TrialPhaseDetail } from "../pipeline/plan.js";
import type { Trajectory } from "../trajectory/types.js";
import type { RegisterFn } from "../utils/plugin-loader.js";
/**
 * Per-trial identity passed to {@link Backend.runTrial}.
 *
 * These are the planner's per-trial identity fields read-only — for resource
 * naming and log correlation — listed explicitly (rather than
 * `Omit<TrialWorkItem, "execute">`) so a future
 * {@link import("../pipeline/plan.js").TrialWorkItem} field
 * can't silently cross the backend boundary; in particular a non-serializable
 * one would break a relocating backend.
 */
export interface TrialIdentity {
    readonly id: string;
    readonly evalName: string;
    readonly evalFilePath: string;
    readonly variant: string;
    readonly model: string | undefined;
    readonly stimulus: Stimulus;
    readonly trialIndex: number | undefined;
    readonly totalTrials: number;
    readonly retryEligible: boolean;
}
/**
 * The outbound, JSON-serializable result of {@link Backend.runTrial}. Across a
 * process boundary it is moved with `JSON.stringify` / `JSON.parse`; its
 * `Date`-typed fields arrive as ISO-8601 strings and are NOT revived to
 * `Date` — the same convention the `vally eval` -> `vally grade` /
 * analytics-server boundary uses. Consumers validate structurally and must not
 * call `Date` methods on a deserialized outcome. See
 * {@link import("../reporting/jsonl-record.js").TrialResultRecord} and
 * {@link import("../pipeline/types.js").EvalOutcome}.
 *
 * Expected successes, terminal failures, and retryable failures are all
 * structured outcomes so the runner does not infer backend-specific workspace or
 * retry behavior.
 */
export type TrialOutcome = TrialSuccess | TrialFailure;
export interface TrialSuccess {
    status: "success";
    trajectory: Trajectory;
    workDir?: string;
    gradeResult?: StimulusGradeResult;
    logs?: TrialLogs;
}
export interface TrialFailure {
    status: "error";
    error: TrialError;
    trajectory?: Trajectory;
    workDir?: string;
    gradeResult?: StimulusGradeResult;
    logs?: TrialLogs;
}
export interface TrialError {
    message: string;
    stack?: string;
    /**
     * Requests a host-level retry when the trial is retry-eligible and the caller
     * configured a retry budget. The retry is a fresh {@link Backend.runTrial}
     * invocation; no backend attempt state is threaded back into the next call.
     *
     * Backends with durable internal attempts (for example, queue jobs that can be
     * resumed after worker eviction) should absorb that retry/resume loop
     * themselves and reserve this flag for failures before any durable attempt
     * exists, or for cases where starting a fresh backend attempt is intentional.
     */
    retryable: boolean;
}
export interface TrialLogs {
    rawEventsPath?: string;
    sessionLogPath?: string;
}
/**
 * Per-attempt progress channel the host wraps so a callback failure can't break
 * its never-rejects contract (a throw is swallowed and surfaced as a
 * `phase-notification-failed` diagnostic).
 *
 * **Phase ownership.** The backend SHOULD emit `running-prompt` via `onPhase`
 * when it begins running the trial, and MAY emit `preparing` before
 * any expensive workspace setup. The host owns `running-graders` (wired through
 * {@link GradeTrajectoryOptions.onGraderStart}) and the terminal
 * `completed`/`errored` phases.
 */
export interface TrialContext {
    onPhase?(phase: TrialPhase, detail?: TrialPhaseDetail): Promise<void>;
    onDiagnostic?(diagnostic: Diagnostic): Promise<void>;
}
/**
 * What a {@link DisposableTrialResult.exportWorkspace} call should egress.
 *
 * The kinds are different operations, not "copy a lot" vs "copy a little":
 * - `"workspace"` reproduces the complete final tree at `targetDir` (the
 *   `--workspace` flag).
 * - `"artifacts"` extracts a matched subset of the workspace for inspection
 *   (the Stimulus's `artifacts` block).
 * - `"artifact-dir"` reproduces the executor's per-trial artifact directory —
 *   the far-side source of {@link import("../trajectory/types.js").Trajectory.artifactDir} —
 *   at `targetDir`. Used by the host to materialize a relocating backend's
 *   artifact dir into a host-readable location so graders (and offline
 *   re-grade) can read it. Same completeness/`recovery` contract as
 *   `"workspace"`: a successful export MAY return `files: []`, and a
 *   failed/incomplete export sets `recovery` to where the source survives.
 */
export type WorkspaceSelection = {
    kind: "workspace";
} | {
    kind: "artifacts";
    include: string[];
    exclude?: string[];
} | {
    kind: "artifact-dir";
};
/**
 * The outcome of a {@link DisposableTrialResult.exportWorkspace} call.
 */
export interface ExportManifest {
    /**
     * Relative POSIX paths actually written under `targetDir`. For
     * `{kind:"artifacts"}` this is the matched files copied. For
     * `{kind:"workspace"}` completeness is represented by the successful mirror at
     * `targetDir`, so a successful workspace export MAY return `files: []` (the
     * host does not read `files` for that kind).
     */
    files: string[];
    /** Total bytes written (best-effort; omit if not tracked). */
    bytes?: number;
    /** True if the export could not be completed in full. */
    incomplete?: boolean;
    failures?: Array<{
        path?: string;
        message: string;
    }>;
    /**
     * Set when a `{kind:"workspace"}` or `{kind:"artifact-dir"}` export
     * failed/was incomplete: an opaque locator (a host path for a local backend, a
     * backend-specific URI for a remote one) telling the user where the source
     * still lives. The host surfaces it and suppresses cleanup so it survives.
     */
    recovery?: {
        kind: "path";
        path: string;
    } | {
        kind: "uri";
        uri: string;
    };
}
/**
 * {@link Backend.runTrial}'s return: the JSON-serializable {@link TrialOutcome}
 * (see its note on `Date` -> ISO-8601 across a boundary) plus per-attempt host
 * capabilities. Egress lives here because it depends on resources captured
 * for this attempt and must run before cleanup.
 */
export interface DisposableTrialResult {
    result: TrialOutcome;
    /**
     * Egress files from the trial workspace into a host-provided `targetDir`.
     * Callable **after** {@link Backend.runTrial} resolves and **before**
     * {@link DisposableTrialResult.cleanup}; not invoked concurrently for one
     * disposable, but sequential calls are supported.
     *
     * A successful `{kind:"workspace"}` export may consume/move the source
     * workspace; callers that need artifacts afterward should read from the
     * placed workspace path or export artifacts before workspace export.
     *
     * `manifest.files` lists **only** successfully written relative POSIX paths (a
     * successful `{kind:"workspace"}` export may report `files: []`).
     *
     * - `{kind:"workspace"}` success ⇒ `targetDir` holds the complete final tree.
     *   Failure ⇒ the source is preserved, `recovery` is set to the real surviving
     *   location, and the host suppresses cleanup and doesn't report a `workspacePath`.
     * - `{kind:"artifacts"}` skips any `.git` path segment, writes only safe
     *   relative paths under `targetDir`, and may set `incomplete: true`.
     * - `{kind:"artifact-dir"}` reproduces the executor's per-trial artifact
     *   directory at `targetDir` with the same success/`recovery` contract as
     *   `{kind:"workspace"}`, and like it MAY consume/move the source. When the
     *   artifact dir lives under `workDir`, egress it before a consuming
     *   `{kind:"workspace"}` export (or copy what each needs first), since moving
     *   the workspace removes the artifact source the later call expects. Only
     *   relocating backends whose artifact dir is not already host-readable need
     *   implement it, and they advertise it via
     *   {@link DisposableTrialResult.supportedExportKinds}.
     *
     * **Never fails the trial:** a thrown/rejected export surfaces as a
     * diagnostic, except that a `{kind:"workspace"}` failure (thrown OR an
     * unsuccessful manifest) must still trigger the host's preserve-and-report
     * path.
     */
    exportWorkspace?(selection: WorkspaceSelection, targetDir: string): Promise<ExportManifest>;
    /**
     * Selection kinds the runner may dispatch to {@link exportWorkspace} for
     * host-side materialization. Absent ⇒ a legacy backend handling only
     * `"workspace"` / `"artifacts"`. A relocating backend whose
     * `trajectory.artifactDir` is not host-readable lists `"artifact-dir"` here so
     * the runner materializes it; a backend that omits it is left untouched (its
     * artifact dir is assumed already host-readable) rather than being sent a kind
     * it would misroute. A backend MAY still handle additional kinds for direct /
     * conformance callers without advertising them here.
     */
    supportedExportKinds?: ReadonlyArray<WorkspaceSelection["kind"]>;
    cleanup: () => Promise<void>;
}
/**
 * Per-trial inputs passed to {@link Backend.runTrial} alongside the trial
 * {@link TrialIdentity}.
 */
export interface TrialOptions {
    /**
     * Eval inputs for this trial. `workDir`/`baseDir` point at the eval *source*
     * directory, not a prepared agent workspace. The backend owns environment
     * setup because it owns the workspace. `LocalBackend` does this via
     * {@link import("../pipeline/run.js").runEval}.
     *
     * Note `runEval` materializes only `environment.files` (agent-visible), not
     * `grading_environment.files` (grader-only) — the backend stages the latter
     * into a disposable grading copy around its grading step. See
     * {@link Backend.runTrial}.
     */
    runOptions: EvalRunOptions;
    graderConfigs: StimulusGraderConfig[];
    gradeOptions: GradeTrajectoryOptions;
    skipGrade: boolean;
    context: TrialContext;
}
/**
 * Per-call inputs for {@link Backend.gradeTrial}. Grade-only: the trajectory is
 * already produced (no executor step) and carries its own `workDir` and
 * `stimulus`. For turn-scoped diff grading, callers must provide
 * `gradeOptions.stimulus` from the authoritative eval spec rather than copying
 * a trajectory's possibly collapsed persisted stimulus.
 *
 * A relocating backend must make the grade workspace available in its environment
 * (e.g. via `trajectory.workDir`). For diff graders, `trajectory.diffPath` and every
 * `trajectory.turnDiffs[].diffPath` are resolved relative to `gradeOptions.diffBaseDir`
 * (when provided) or the grading process `cwd`; the host can instead inline
 * `trajectory.diff` and `trajectory.turnDiffs[].diff` before dispatch.
 */
export interface GradeTrialOptions {
    /** The already-produced trajectory to grade (carries `workDir` + `stimulus`). */
    trajectory: Trajectory;
    graderConfigs: StimulusGraderConfig[];
    gradeOptions: GradeTrajectoryOptions;
    /**
     * Progress/diagnostic channel for the grade. Backends report grade-lifecycle
     * signals here. Per-grader progress stays on {@link GradeTrajectoryOptions.onGraderStart},
     * which the host owns.
     */
    context: TrialContext;
}
/**
 * Backend-specific configuration from `--backend-args`. A plain args bag.
 * Values are strings for simple options, arrays for repeatable options
 * (e.g., mount, compose-file). CLI parsing collects repeated keys into arrays.
 */
export type BackendConfig = Record<string, string | string[]>;
/**
 * Host-controlled run context injected at {@link Backend.prepare} — kept
 * separate from the user's {@link BackendConfig} so injected metadata can't
 * collide with `--backend-args` keys. `runId`/`experimentName` are present only
 * for `vally experiment run`.
 *
 * `plugins` carries the run-scoped executor/grader plugin specifiers the user
 * selected (`--executor-plugin` / `--grader-plugin`) plus the `cwd` to resolve
 * them from. The host already loads these into its own registries; they're
 * surfaced here so a *relocating* backend (Docker, remote) can reconstruct the
 * same registries on the far side without re-deriving CLI state. The default
 * `LocalBackend` ignores them (it holds the live registries directly).
 */
export interface BackendRunContext {
    runId?: string;
    experimentName?: string;
    plugins?: {
        /** `--executor-plugin` specifiers (repeatable). */
        executor?: string[];
        /** `--grader-plugin` specifiers (repeatable). */
        grader?: string[];
        /** Directory to resolve the above specifiers from. */
        cwd?: string;
    };
}
/**
 * Migration fallback for backends that cannot yet produce a structured
 * retryable failure. New expected failures should resolve with
 * {@link TrialFailure} so cleanup and workspace disposition travel with
 * the outcome.
 */
export declare class RetryableBackendError extends Error {
    /** Authoritative retry signal read by the host's retry loop. */
    readonly retryable = true;
    constructor(message: string, options?: {
        cause?: unknown;
    });
}
/**
 * Backend — controls WHERE a trial runs (this host, a container, a remote VM).
 *
 * The default `vally eval` / `vally experiment run` path resolves a
 * `LocalBackend` for the run; injecting a different backend relocates the
 * per-trial core instead.
 *
 * **Concurrency.** The host shares a single backend instance across the run and
 * may call {@link Backend.runTrial} concurrently for multiple trials (bounded by
 * the `--workers` pool). Implementations MUST be safe under concurrent
 * invocation — keep per-trial state local to each `runTrial` call, and guard any
 * shared/instance state (e.g. caches, connection pools). `prepare` runs once
 * before any `runTrial`; `shutdown` runs once after all `runTrial` calls settle.
 */
export interface Backend {
    name: string;
    /**
     * One-time setup (e.g., pull/verify images, check prerequisites). Called
     * once before any trials run. `config` is the user's `--backend-args` (and
     * ONLY that); run-scoped metadata the CLI injects (experiment runId/name and
     * the selected executor/grader plugin specifiers) is the separate `context`
     * arg so the two never collide.
     */
    prepare?(config: BackendConfig, context: BackendRunContext): Promise<void>;
    /**
     * Run a single trial: env setup → executor → grading.
     *
     * `item` is the originating work item's identity (no `execute` closure); all
     * other per-trial inputs are passed via {@link TrialOptions}.
     *
     * Resolve with a {@link DisposableTrialResult} for success, terminal failure,
     * and retryable failure. A retryable failure asks the host to clean up the
     * returned disposable and re-enter `runTrial` as a new call; the contract does
     * not carry an opaque resume token or any prior attempt state. Durable remote
     * backends that can resume an existing job should generally do so internally,
     * surfacing `retryable: true` only before a durable attempt exists or when a
     * fresh backend attempt is desired.
     *
     * Throws are reserved for backend bugs or situations where no attempt envelope
     * can be produced; the runner catches them but will not report a workspace
     * path.
     *
     * Progress phases flow through `options.context` (see {@link TrialContext} for
     * the ownership contract).
     *
     * **Environment files vs. grader files.** The stimulus separates two file
     * sets across two blocks, and a backend MUST treat them differently:
     * - `environment.files` (the agent environment) are materialized **before**
     *   the executor runs — the agent under test works with them. `runEval` (used
     *   by {@link LocalBackend}) applies these during setup.
     * - `grading_environment.files` are grader-only fixtures (answer keys, golden
     *   data, hidden tests) that the agent must **never** see. They are staged
     *   into a **disposable copy** of the produced workspace **only for grading**
     *   — after the executor finishes and after the workspace diff is captured —
     *   and the copy is destroyed afterward. The preserved/exported workspace is
     *   the untouched original, so grader files never reach the agent, the
     *   recorded diff, or a preserved/exported tree, and a mutating grader can't
     *   contaminate the evaluated result. `runEval` does NOT stage these; the
     *   backend owns this around its grading step.
     *
     * Backends that reimplement this method (rather than delegating to
     * `runEval`/`LocalBackend`) should grade via
     * {@link import("../pipeline/run.js").gradeInDisposableWorkspace}, which owns
     * the risky parts (copy the produced workspace, neutralize a linked-worktree
     * `.git`, stage grader files into the copy, grade a trajectory clone, destroy
     * the copy). A relocating backend calls it on the host after downloading the
     * produced workspace. Skipping grader staging silently drops the feature: a
     * workspace grader that reads a grader file passes on {@link LocalBackend} but
     * fails on the backend that forgot to stage them.
     */
    runTrial(item: TrialIdentity, options: TrialOptions): Promise<DisposableTrialResult>;
    /**
     * Grade an already-produced trajectory in the backend's compute environment,
     * without running an executor. Used by {@link Backend.runTrial} implementations
     * for their grading step, and available for grade-only runs (grading a
     * trajectory a prior run produced).
     *
     * Grades `options.graderConfigs`. If the list is empty, grading returns a passing
     * "No graders configured" result; callers may also choose to skip invoking
     * `gradeTrial` when there is nothing to grade.
     *
     * Grading-logic failures should resolve with `status: "error"` carrying the raw
     * message (instead of throwing), so the returned {@link DisposableTrialResult.cleanup}
     * always runs and a relocating backend never strands resources.
     *
     * This method grades whatever is on disk at `options.trajectory.workDir`; it
     * does NOT stage `grading_environment.files`. Within `runTrial` the backend
     * wraps this in {@link import("../pipeline/run.js").gradeInDisposableWorkspace}
     * (which points the trajectory clone at the disposable copy that has the grader
     * files staged). A standalone grade-only caller that needs grader fixtures
     * present is responsible for staging them into a disposable workspace itself.
     */
    gradeTrial(options: GradeTrialOptions): Promise<DisposableTrialResult>;
    /**
     * Release backend resources. Must be idempotent and safe to call before or after
     * prepare().
     */
    shutdown(): Promise<void>;
}
/**
 * Names a backend and constructs a fresh, run-scoped instance on demand.
 *
 * `create()` MUST return a new instance each call (no shared singletons) so a
 * prior run's `shutdown()` can't disable a later run and per-run cache state
 * can't leak. Keep it cheap/synchronous — heavyweight or async setup belongs in
 * {@link Backend.prepare}. By convention `name` matches the created backend's
 * `name`.
 */
export interface BackendFactory {
    readonly name: string;
    create(): Backend;
}
export interface BackendRegistry {
    register(factory: BackendFactory): void;
    get(name: string): BackendFactory | undefined;
    getAll(): BackendFactory[];
    names(): string[];
}
export declare function createBackendRegistry(): BackendRegistry;
/**
 * Every backend plugin package exports a `registerBackends(registry)` function
 * that registers one or more {@link BackendFactory} descriptors.
 */
export interface BackendPluginEntry {
    registerBackends: RegisterFn<BackendRegistry>;
}
//# sourceMappingURL=types.d.ts.map