/** * Console reporter for the planRun-based pipeline. * * Owns its own progress UI (spinner + status table) and serialized * stdout writes. Internally aggregates per-stimulus trial blocks for * multi-trial runs so the user sees one tidy block per stimulus rather * than interleaved partial results from concurrent workers. * * Layout per event: * onRunStart → header lines, start spinner with phase table * onTrialStart → mark slot `queued → starting` * onTrialPhase → update slot phase * onTrialResult → for single-trial: flush per-stimulus block * for multi-trial: append to stimulus bucket; * when bucket has totalTrials, flush block * onEvalComplete → per-eval verdict line * onRunComplete → stop spinner, cross-model summary, * artifacts list */ import type { PlanReporter, PlanRunStartContext, RunArtifacts } from "./plan-reporter.js"; import type { EvalSummary, RunSummary, ScopedDiagnostic, TrialPhase, TrialPhaseDetail, TrialResult, TrialWorkItem } from "../pipeline/plan.js"; import { sanitizeForDisplay } from "./sanitize.js"; export { sanitizeForDisplay }; export interface EvalConsoleReporterOptions { verbose?: boolean; /** Where rendered blocks land. Defaults to `process.stdout`. */ stream?: NodeJS.WritableStream; /** Where the spinner renders. Defaults to `process.stderr`. */ progressStream?: NodeJS.WriteStream; /** Disable spinner explicitly (useful for tests / non-TTY). */ disableSpinner?: boolean; } export declare class EvalConsoleReporter implements PlanReporter { private readonly verbose; private readonly stream; private readonly mutex; private readonly spinner; private readonly slotsById; private nextSlotIndex; private completed; private totalItems; private workers; private readonly buckets; /** Effective threshold per (eval × variant × model), captured at * onRunStart. Drives the threshold-adjusted per-trial pass in * multi-trial blocks so glyphs/counts agree with pass@k and the * eval verdict. undefined when scoring isn't applied. */ private readonly thresholdByEval; /** Variant names present in the plan, captured at onRunStart so * per-trial headers can include the variant only when more than one * is in play (single-variant runs would otherwise show every header * with a redundant "main" label). */ private readonly variantNames; private summaryHeadingShown; constructor(options?: EvalConsoleReporterOptions); onRunStart(ctx: PlanRunStartContext): Promise; onTrialStart(item: TrialWorkItem): Promise; onTrialPhase(itemId: string, phase: TrialPhase, detail?: TrialPhaseDetail): Promise; onTrialResult(event: { item: TrialWorkItem; result: TrialResult; }): Promise; onDiagnostics(diagnostics: ScopedDiagnostic[]): Promise; onEvalComplete(summary: EvalSummary): Promise; /** * Stop the live progress spinner without rendering the run summary. Safe to * call multiple times and when no spinner is active; used on the error path so * the terminal isn't left spinning when a run aborts before completion. */ stopProgress(): void; onRunComplete(summary: RunSummary, artifacts: RunArtifacts): Promise; /** * Format the `━━━ name · stim [variant] [model] ━━━` header used for * both single-trial and multi-trial blocks. The variant tag only * appears when the plan has more than one variant — single-variant * runs (which is every `vally eval` invocation) would otherwise * decorate every header with a redundant default name. */ private formatTrialHeader; private renderSingleTrialBlock; private renderMultiTrialBlock; private renderMetricsBlock; private renderGradedBlock; /** * Render one `icon label evidence` row per grader, plus indented rows for * the failing sub-checks of a failing grader. Graders that share a name get * a `#n` suffix so each row stays distinguishable. * * Grader names and evidence originate in user-controlled YAML and * model-generated text, so every interpolated string is sanitized. */ private renderGraderRows; private renderAgentOutputBlock; private flushBlock; private write; private spinnerText; private refreshSpinnerStatus; private refreshSpinnerTable; } //# sourceMappingURL=eval-console.d.ts.map