/**
 * Reporter interface for the planRun-based pipeline.
 *
 * Reporters MAY implement any subset of the optional methods. The CLI
 * fans every event through a composite reporter which serializes calls
 * per-reporter (each reporter receives callbacks sequentially, not
 * concurrently). Reporter implementations therefore do not need their
 * own internal locks.
 *
 * **Memory-safety contract.** Reporters MUST treat {@link TrialResult}
 * (including the `trajectory` field) as immutable. The CLI owns
 * lifecycle for trajectory memory: after `summarize()` completes it MAY
 * null out trajectory references on its retained result objects.
 * Reporters that need trajectory data MUST copy what they require during
 * their `onTrialResult` call; later access is undefined.
 *
 * **Lifecycle & ordering contract.** The CLI guarantees the following
 * for every reporter (built-in or plugin-loaded), except where a clause
 * is explicitly scoped to plugin-loaded reporters:
 *
 * 1. `onRunStart` fires exactly once, and strictly BEFORE any
 *    `onTrialResult`.
 * 2. `onTrialResult` fires AT MOST once per planned trial. Ordering
 *    ACROSS concurrent trials is NOT guaranteed — trials complete in
 *    pool-admission/finish order, not plan order. The stronger guarantee
 *    — exactly once per planned trial — holds only when the run reaches
 *    `onRunComplete` with `status === "completed"`; on a failed or
 *    cancelled run, trials that never reached a terminal result before
 *    shutdown are simply never delivered. However, each individual
 *    `onTrialResult` delivers a COMPLETE, immutable trajectory + grade
 *    for that one trial; reporters never observe a partially-populated
 *    trial.
 * 3. `onRunComplete` fires strictly AFTER the last `onTrialResult`. On a
 *    clean run it fires exactly once for every reporter. On a failed or
 *    cancelled run, only reporters loaded via `--reporter-plugin` are
 *    GUARANTEED a terminal `onRunComplete` (carrying the abort `status`
 *    and a possibly-partial summary). The built-in FILE reporters
 *    (markdown, and JUnit when `--junit` is set) are ALSO finalized with
 *    a direct terminal `onRunComplete` on abort so their report files are
 *    still written, then immediately sealed from the composite so no
 *    later event reaches them. The built-in CONSOLE reporter is the sole
 *    exception: it is intentionally skipped on abort, to avoid rendering a
 *    partial summary.
 *
 * Within a single reporter, the {@link CompositeEvalReporter}
 * serializes callbacks (each reporter advances down its own queue), so
 * a reporter observes its own events in the order above without needing
 * an internal lock. The optional callbacks (`onTrialStart`,
 * `onTrialPhase`, `onEvalComplete`, `onDiagnostics`) interleave with
 * `onTrialResult` and always fire AFTER `onRunStart` (they never precede
 * it). Every reporter that receives a terminal `onRunComplete` — every
 * reporter on a completed run, and on a failed/cancelled run both the
 * plugin-loaded reporters AND the finalized built-in file reporters (see
 * clause 3) — sees those optional callbacks strictly BEFORE it, closing
 * the `onRunStart … onRunComplete` envelope: the file reporters are
 * sealed the instant they are finalized, so the late `onDiagnostics` the
 * CLI MAY emit afterward (e.g. an OTel trace-location notice) never
 * reaches them. The only reporter that observes such late `onDiagnostics`
 * with no closing `onRunComplete` is the console reporter, which is not
 * finalized on abort.
 *
 * **Stable identifiers (idempotency / at-least-once delivery).** Each
 * trial exposes two stable identifiers that retry-safe consumers can use
 * as idempotency keys:
 *
 * - `event.item.id` — the {@link TrialWorkItem} id, globally unique
 *   within a run across `(eval × model × variant × stimulus × trial)`.
 * - `event.result.trajectory?.id` and
 *   `event.result.trajectory?.metadata.sessionID` — the executor-level
 *   trajectory/session id. `trajectory` is `null` only when the executor
 *   errored before producing one (`result.status === "error"`); use
 *   `item.id` as the fallback key in that case.
 *
 * Event ordering WITHIN a single trajectory (`trajectory.events`) is
 * deterministic — events appear in the order the executor emitted them.
 *
 * **Async + error isolation.** All lifecycle methods MAY be `async`. A
 * reporter that throws or returns a rejected promise is logged to stderr
 * by the composite and skipped for that event; it never fails the run or
 * blocks sibling reporters. To keep those swallowed errors visible, each
 * throw is logged with the reporter's stable name (for plugin reporters,
 * the `--reporter-plugin` specifier — plain-object reporters have no useful
 * `constructor.name`), and the CLI emits a single end-of-run summary warning
 * counting how many reporter callback errors occurred; the exit code is
 * unaffected. This runtime isolation is distinct from a MALFORMED plugin: a
 * reporter missing a required method (`onTrialResult` / `onRunComplete`)
 * fails fast at LOAD, not per-event during the run.
 */
import type { EvalPlanMetadata, EvalSummary, RunSummary, ScopedDiagnostic, TrialPhase, TrialPhaseDetail, TrialResult, TrialWorkItem } from "../pipeline/plan.js";
/** Run-level output paths surfaced through `PlanReporter.onRunComplete`.
 *  The console reporter prints the grouped "saved to" block from these
 *  fields. */
export interface RunArtifacts {
    /** Path to the JSONL file when `--jsonl <file>` (or the per-eval JSONL
     *  under `--output-dir`). Absent when JSONL streamed to stdout. */
    jsonl?: string;
    /** Path to the markdown report (when `--output-dir` is set). */
    markdown?: string;
    /** Path to the JUnit XML file (when `--junit` is set). */
    junit?: string;
    /** Run output directory under which executor session logs are written
     *  in `[<variant>/]<eval>/<stimulus>/<model>/<trial>/` subdirectories
     *  (the `<variant>/` segment is present only for experiment runs). This
     *  directory also contains other run artifacts such as results.jsonl,
     *  the markdown report, and otel spans. */
    sessionLogsDir?: string;
    /** Path to the merged run-level OpenTelemetry span JSONL file. */
    otelTracesFile?: string;
}
/** Plan-time snapshot passed to {@link PlanReporter.onRunStart}. */
export interface PlanRunStartContext {
    /** Per-eval planning metadata (in input order, including plan-time failures).
     *  Carries eval names, models, executor names, planned stimulus counts +
     *  names, and per-stimulus trial counts (`runs`) — i.e. far more than bare
     *  totals. A reporter opening a run-level container reads eval names /
     *  models / counts from here. */
    evals: EvalPlanMetadata[];
    /** Total planned work items. Equal to `RunPlan.items.length`;
     *  equivalent to the sum over `(eval × model × variant)` of
     *  `plannedStimulusCount * runs`. */
    totalItems: number;
    /** Pool size — the maximum number of concurrent in-flight trials.
     *  Reporters that render in-flight slots (the console reporter)
     *  use this to size their UI. */
    workers: number;
    /** Producer identity (tool name + version) for run-level provenance. */
    source?: {
        name: string;
        version: string;
    };
    /** Sorted union of grader names available for this run. Lets a reporter
     *  pre-declare the grader columns/dimensions of its container. */
    graderNames?: string[];
    /** Active include tag filter scoping the run (`key -> allowed values`).
     *  An empty map means a suite/`--tag` filter was applied with no include
     *  conditions (e.g. exclude-only). Absent when the run had no tag or suite
     *  filter at all. */
    tagFilter?: Record<string, string[]>;
    /** Active exclude tag filter scoping the run (`key -> rejected values`).
     *  Absent when the run had no exclude filter. */
    tagExclude?: Record<string, string[]>;
}
/** Eval reporter — consumed by the CLI's pool loop. */
export interface PlanReporter {
    /**
     * Called once at run start with plan-time + runtime info. Reporters
     * use this to pre-allocate per-eval sections (markdown, JUnit) or to
     * size their live UI (console).
     */
    onRunStart?(ctx: PlanRunStartContext): Promise<void>;
    /**
     * Per-trial admission notification. Reporters that show live progress
     * (CLI spinner) use this to mark items "starting". A trial is queued
     * iff this event has not yet fired for it; the absence of the event
     * is the queue signal.
     */
    onTrialStart?(item: TrialWorkItem): Promise<void>;
    /**
     * Per-trial completion. Carries full context — no implicit per-eval
     * or per-stimulus state.
     *
     * Reporters that render per-stimulus aggregates (multi-trial
     * pass-rate boxes) bucket by `(item.evalName, item.evalFilePath,
     * item.variant, item.model, item.stimulus.name)` and flush when
     * they have observed `item.totalTrials` results for the bucket. The
     * plan guarantees `id` uniqueness across the run.
     */
    onTrialResult(event: {
        item: TrialWorkItem;
        result: TrialResult;
    }): Promise<void>;
    /**
     * Phase callback for live progress UI. The phase is a stable enum;
     * reporters that own UI format display strings from the enum +
     * structured detail. Phases are conditional and may fire zero or
     * more times per item. Awaitable so reporters can serialize redraws
     * without races against `onTrialResult`.
     */
    onTrialPhase?(itemId: string, phase: TrialPhase, detail?: TrialPhaseDetail): Promise<void>;
    /**
     * Diagnostics emitted outside the `onTrialResult` stream. Each
     * diagnostic carries an optional `scope: { evalName, evalFilePath,
     * variant, model? }` so per-eval warnings (validation-failed,
     * stimuli-filtered) reach the right markdown/JUnit section; absent
     * scope = run-level.
     *
     * Batches may contain plan-time diagnostics, run-level diagnostics,
     * or per-trial runtime diagnostics emitted via
     * {@link TrialExecuteContext.onDiagnostic}. Per-trial diagnostics
     * arrive here with their `scope` populated to the originating
     * trial's `(evalName, evalFilePath, variant, model)`.
     */
    onDiagnostics?(diagnostics: ScopedDiagnostic[]): Promise<void>;
    /**
     * Called once when an entire `(eval × model × variant)` is complete:
     * after the last planned item has produced a result, immediately for
     * a plan-time failure, or immediately when filtering leaves no
     * planned items. Reporters that emit per-eval sections (markdown,
     * JUnit) use this signal to flush.
     */
    onEvalComplete?(summary: EvalSummary): Promise<void>;
    /**
     * Final wrap-up — write summary files, drain buffers. Fires exactly
     * once, strictly after the last `onTrialResult`. `artifacts` carries
     * run-level output paths (jsonl/markdown/junit/session-logs); the
     * console reporter prints the grouped "saved to" block from these.
     *
     * `status` is the run's terminal lifecycle state, independent of the
     * pass/fail verdict (which lives in `summary.passed` /
     * `summary.hadExecutionErrors`):
     *
     * - `"completed"` — the run executed every planned trial to its
     *   natural end. Read the verdict from `summary`.
     * - `"failed"` — the run aborted on an unrecoverable error before all
     *   trials finished. `summary` may be partial.
     * - `"cancelled"` — the user interrupted the run (Ctrl+C / SIGTERM).
     *   `summary` may be partial.
     *
     * The parameter is optional and trailing so existing reporters that
     * implement `onRunComplete(summary, artifacts)` remain valid; treat an
     * absent `status` as `"completed"`.
     */
    onRunComplete(summary: RunSummary, artifacts: RunArtifacts, status?: RunCompletionStatus): Promise<void>;
}
/** Terminal lifecycle state of a run, surfaced to
 *  {@link PlanReporter.onRunComplete}. Distinct from the pass/fail
 *  verdict carried by {@link RunSummary}. */
export type RunCompletionStatus = "completed" | "failed" | "cancelled";
/**
 * Build a minimal, well-formed {@link RunSummary} for terminal
 * `onRunComplete` delivery on the failure/cancel path, used only when a
 * run aborts BEFORE its real summary has been computed. Every required
 * field is present with safe zero/empty values (`evals: []`,
 * `passed: false`) so a plugin reporter that reads required `RunSummary`
 * fields on the terminal event cannot crash on a missing field. The real
 * summary is always preferred when it has been computed.
 *
 * The abort-vs-complete distinction is NOT carried here — it is conveyed by
 * the `status` argument (`"completed" | "failed" | "cancelled"`) of
 * `onRunComplete`. `hadExecutionErrors` reflects ONLY real per-trial error
 * status (a trial reported `status === "error"`), matching the
 * {@link RunSummary} contract. It therefore defaults to `false` when the
 * real signal is unknown (e.g. the run aborted before any trial ran or
 * errored), so plugin reporters never read an unknown as a known error
 * signal. Callers thread through the real execution-error signal when they
 * have one.
 */
export declare function createPartialRunSummary(hadExecutionErrors?: boolean): RunSummary;
//# sourceMappingURL=plan-reporter.d.ts.map