import type { EventEnvelope } from "../statelog/wireTypes.js"; import { type Annotation } from "./annotations.js"; import type { RunDirectorySnapshot } from "./runDir.js"; /** * The canonical rows one run is made of, wherever they are stored: the * trace's events in order and every annotation row about it. Statelog hands * its database rows to `summarizeEvalRun` in this shape; a run directory is * adapted by `summarizeRunDirectory`. A run that died before its first event * has no events and still summarizes: its harness `run` row is the evidence. */ export type EvalRunInput = { traceId: string; events: readonly EventEnvelope[]; annotations: readonly Annotation[]; /** Diagnostic identity only (a file path, a database id); never read. */ source: string; notes?: string | null; }; /** `ok`: the harness saw a clean finish. `killed`: the harness cut the run * short (timeout, cost cap, signal) after it had produced events. `failed`: * every other harness outcome, including a run with no events at all. * `trace`: no harness row, so only the trace's own ending is known. * `partial` is never produced for one run; aggregates over several runs * (the runs explorer) use it. */ export type RunStatus = "ok" | "partial" | "failed" | "killed" | "trace"; /** One row per run, for `agency runs list`, the viewer's overview, the runs * explorer, batch statistics, and statelog's per-trace summary table. */ export type RunSummary = { traceId: string; /** The test id from the harness's `run` row; null for an ad-hoc trace. */ testId: string | null; input: string | null; /** The trace's own `agentName` event, when it emitted one. */ agentName: string | null; /** The harness's agent label from the `run` row (`flags.agent`: an agent * file target or a command line), unchanged; null for an ad-hoc trace. */ agentLabel: string | null; startedAt: string | null; startedAtMs: number | null; /** When the harness recorded the run (its `run` row's `createdAt`), else * the last event's timestamp; null for an empty ad-hoc trace. */ endedAt: string | null; durationMs: number; costUsd: number; llmCalls: number; toolCalls: number; eventCount: number; models: string[]; /** The harness's verdict when it recorded one, else read from the trace. */ ended: string; status: RunStatus; /** The weighted mean of the effective scores on record; null with none. */ latestScore: number | null; /** What this run counts as in a mean, by grading's rule (`gradeRun.ts`): * `latestScore` when the run ended ok, 0 when it did not finish (a crash * earns no points, graded or not), null when it ended ok but has no score. */ score: number | null; /** Complete grading passes on this trace; `latestScore` is from the last. */ gradingPasses: number; /** False when an effective must-pass score failed; null with no scores. */ gatesPassed: boolean | null; /** `notes.md` exists and is not blank. */ hasNotes: boolean; labeled: boolean; codeHash: string | null; /** The suite invocation this run belongs to; null before batches existed. */ batch: string | null; /** The test's 1-based repetition within the batch; null before trials existed. */ trial: number | null; suiteSource: string | null; suiteSha: string | null; }; /** Summarize one run from its canonical rows. Every event and annotation * must belong to `input.traceId`: a mismatch is a caller bug, not data. */ export declare function summarizeEvalRun(input: EvalRunInput): RunSummary; /** The run a directory holds: its trace, or, when the run never wrote one, * the run its harness row records. Null for a directory with neither. */ export declare function summarizeRunDirectory(snapshot: RunDirectorySnapshot): RunSummary | null; export declare function summarizeRuns(snapshot: RunDirectorySnapshot): RunSummary[]; /** One short line about a trace's annotations for a listing or the viewer's * trace row: "notes · score 0.70 (3 passes) · labeled". Empty when there are * none; the pass count appears only after a re-grade. */ export declare function annotationSummaryText(summary: RunSummary): string; /** traceId → `annotationSummaryText`, for every trace that has one. */ export declare function annotationSummaries(snapshot: RunDirectorySnapshot): Record; /** What a listing shows for "which agent": the trace's own name when it * emitted one, else the harness label unchanged (a command line is not * shortened: its basename could be any argument), else null. */ export declare function displayAgent(summary: RunSummary): string | null; /** What `runs list` renders for a set of run directories. */ export type RunsListing = { /** One row per run found. */ summaries: RunSummary[]; /** Runs whose statelog holds no event: the harness recorded a run that died before its first event. */ silentRunCount: number; /** summaries.length. */ runCount: number; /** Rows with a persisted score, and their mean; null when none. */ gradedCount: number; meanScore: number | null; }; export declare function buildRunsListing(snapshots: RunDirectorySnapshot[]): RunsListing;