import type { AssistantMessageScenario, PiIntegrationTestResult, PiIntegrationTestThinkingLevel, ToolSelection, TraceEvent } from "pi-coding-agent-test"; /** One benchmark task with a stable identifier. */ export interface EvalTask { readonly id: string; } /** Tasks and defaults resolved from one named benchmark preset. */ export interface EvalBenchmarkPreset { /** Tasks selected by this preset. */ readonly tasks: readonly Task[]; /** Seed used for deterministic AB/BA ordering. */ readonly seed: number; /** Default number of paired attempts per task. */ readonly attempts: number; } /** Input for {@link createEvalSchedule}; agent profiles are compared in balanced order. */ export interface EvalScheduleInput { /** Tasks to repeat for every agent profile. */ readonly tasks: readonly Task[]; /** Positive number of attempts per task. */ readonly attempts: number; /** Stable seed used to derive agent profile order. */ readonly seed: number; /** Benchmark preset name included in the ordering hash. */ readonly benchmarkPreset: string; /** Agent profile IDs to schedule and compare. */ readonly agentProfiles: readonly string[]; } /** Generic scalar score plus suite-owned diagnostics. */ export interface EvalScore { /** Normalized primary score, conventionally from zero to one. */ readonly reward: number; /** Whether the submission passed the suite acceptance rule. */ readonly passed: boolean; /** Named scalar dimensions preserved in artifacts. */ readonly dimensions?: Readonly>; /** Additional JSON-compatible validation details. */ readonly details?: unknown; } /** * One Pi setup participating in an evaluation. * * Profiles are caller-owned. The evaluator compares only the agent profiles supplied * in `RunEvaluationOptions.agentProfiles`; it does not add external profiles. Profile * settings inherit the matching run-level setting when omitted. */ export interface EvalAgentProfile { /** Stable agent profile id used in schedules and reports. */ readonly id: string; /** Optional model override for this agent profile. */ readonly model?: string; /** Optional reasoning level override for this agent profile. */ readonly thinking?: PiIntegrationTestThinkingLevel; /** Skill files or directories loaded for this agent profile. */ readonly skills?: readonly string[]; /** Replaces the global system prompt for this agent profile. */ readonly systemPrompt?: string; /** Appends to the global system prompt for this agent profile. */ readonly appendSystemPrompt?: readonly string[]; /** Extension entrypoints loaded for this agent profile. */ readonly extensions?: readonly string[]; /** Active tools, or a task-aware resolver. Omit to expose all registered tools. */ readonly tools?: ToolSelection | ((task: EvalTask, benchmarkPreset: string) => ToolSelection); /** Scripted assistant responses for deterministic runs. Omit to use the user's real provider. */ readonly conversation?: readonly AssistantMessageScenario[] | ((task: EvalTask, benchmarkPreset: string) => readonly AssistantMessageScenario[]); } /** Stable identity and order of one scheduled trial. */ export interface EvalTrial { /** Task executed by this trial. */ readonly task: Task; /** One-based repeat number. */ readonly attempt: number; /** Stable identifier shared by all agent profiles. */ readonly pairId: string; /** One-based position inside the balanced agent profile block. */ readonly position: number; /** Agent profile used for this trial. */ readonly agentProfile: string; } /** Workspace and suite state prepared before Pi starts. */ export interface PreparedEvalTrial { /** Isolated working directory passed to Pi. */ readonly cwd: string; /** Suite-owned state passed to validation and cleanup. */ readonly state: State; /** Safe environment facts written to the trial artifact. */ readonly environment?: Readonly>; } /** Adapter for one benchmark dataset and its validator. */ export interface EvalSuite { /** Stable suite name used by the CLI. */ readonly id: string; /** Resolve a named benchmark preset into tasks and scheduling defaults. */ loadBenchmarkPreset(benchmarkPreset: string): Promise>; /** Prepare an isolated workspace before starting Pi. */ prepareTrial(input: EvalPrepareInput): Promise>; /** Build the user prompt for a prepared task. */ prompt(input: EvalPromptInput): Promise | string; /** Validate the settled Pi run and return a normalized score. */ validate(input: EvalValidateInput): Promise; /** Remove suite-owned temporary state after validation or failure. */ cleanupTrial?(input: EvalCleanupInput): Promise; /** Return safe task metadata for schedule artifacts. */ taskMetadata?(task: Task): Readonly>; } /** Inputs provided while a suite prepares one trial. */ export interface EvalPrepareInput { /** Scheduled trial. */ readonly trial: EvalTrial; /** Root directory for this evaluation run. */ readonly runDirectory: string; /** Directory reserved for this trial artifacts. */ readonly trialDirectory: string; /** Workspace path reserved for this trial. */ readonly workspaceDirectory: string; } /** Inputs provided while a suite builds a prompt. */ export interface EvalPromptInput { /** Scheduled trial. */ readonly trial: EvalTrial; /** Prepared trial state. */ readonly prepared: PreparedEvalTrial; } /** Inputs provided while a suite validates one settled run. */ export interface EvalValidateInput { /** Scheduled trial. */ readonly trial: EvalTrial; /** Prepared trial state. */ readonly prepared: PreparedEvalTrial; /** Result returned by the real Pi process. */ readonly piResult: PiIntegrationTestResult; /** Directory reserved for suite verifier artifacts. */ readonly verifierDirectory: string; } /** Inputs provided while a suite cleans up one trial. */ export interface EvalCleanupInput { /** Scheduled trial. */ readonly trial: EvalTrial; /** Prepared trial state. */ readonly prepared: PreparedEvalTrial; /** Whether the caller requested the workspace to be preserved. */ readonly keepWorkspace: boolean; } /** * Options for one evaluation run. * * Required inputs are the suite, benchmark preset name, artifact roots, and * the explicit list of agent profiles. All other fields are optional * overrides or extension points. */ export interface RunEvaluationOptions { /** Suite adapter to execute. */ readonly suite: EvalSuite; /** Named benchmark preset. */ readonly benchmarkPreset: string; /** Root directory that receives the run directory. */ readonly resultsDirectory: string; /** Root directory that receives isolated trial workspaces. */ readonly workspacesDirectory: string; /** Pi model selector used by every profile unless a profile overrides it. */ readonly model?: string; /** Pi reasoning level used by every profile unless a profile overrides it. */ readonly thinking?: PiIntegrationTestThinkingLevel; /** Skills loaded for every profile unless a profile provides its own list. */ readonly skills?: readonly string[]; /** System prompt used unless a profile provides its own prompt. */ readonly systemPrompt?: string; /** Text appended to the global system prompt unless a profile provides its own list. */ readonly appendSystemPrompt?: readonly string[]; /** Agent profiles evaluated on the same benchmark trials. */ readonly agentProfiles: readonly EvalAgentProfile[]; /** Run only one task from the benchmark preset. */ readonly taskId?: string; /** Override the benchmark preset repeat count. */ readonly attempts?: number; /** Use case-specific tools or expose all registered tools. Defaults to case. */ readonly tools?: "case" | "all"; /** Stable caller-provided run directory name. */ readonly runId?: string; /** Display the native Pi terminal while each trial runs. */ readonly live?: boolean; /** Preserve suite workspaces after each trial. */ readonly keepWorkspace?: boolean; /** Maximum time for one Pi run. */ readonly timeoutMs?: number; /** Extensions loaded identically for every agent profile. */ readonly commonExtensions?: readonly string[]; /** Additional per-trial metrics calculated and persisted by the evaluator. */ readonly metricCalculators?: readonly EvalMetricCalculator[]; /** Consumer-owned report presentation. Omit to use the exhaustive default renderer. */ readonly reportRenderer?: EvalReportRenderer; } /** Top-level configuration loaded by the `pi-eval` CLI. */ export interface PiEvalConfig { /** Root directory that receives evaluation runs. */ readonly resultsDirectory: string; /** Root directory outside the evaluator repository for isolated trial workspaces. */ readonly workspacesDirectory: string; /** Suites addressable by CLI name. */ readonly suites: Readonly>; /** Agent profiles addressable by CLI id. The built-in vanilla agent profile is always available. */ readonly agentProfiles?: Readonly>; /** Resolve CLI extension names into package specifiers or absolute entrypoint paths. */ readonly resolveExtensions?: (extensions: readonly string[]) => Promise | readonly string[]; /** Additional metrics used by CLI-driven evaluations. */ readonly metricCalculators?: readonly EvalMetricCalculator[]; /** Report presentation used by CLI-driven evaluations. */ readonly reportRenderer?: EvalReportRenderer; } /** One ordered tool call captured from the real Pi trace. */ export interface ToolCallMetrics { /** One-based call position. */ readonly index: number; /** Stable tool-call identifier. */ readonly id: string; /** Registered tool name. */ readonly name: string; /** Tool arguments in their emitted field order. */ readonly arguments: unknown; /** Whether Pi finalized the execution as an error. */ readonly error: boolean; /** Error text for a failed call, when available. */ readonly errorMessage?: string; } /** One mutation decision located in the agent trajectory. */ export interface MutationPointMetrics { /** One-based tool-call position. */ readonly callIndex: number; /** Mutating tool name. */ readonly tool: string; /** Seconds from agent start until the mutation was requested. */ readonly decisionSeconds: number | null; } /** Model-visible source gathered before the first successful mutation. */ export interface SourceExposureMetrics { /** Distinct paths read directly through the read tool. */ readonly uniqueReadFiles: number; /** Distinct source line numbers returned by direct reads. */ readonly uniqueReadLines: number; /** UTF-8 bytes rendered by read results. */ readonly renderedReadBytes: number; /** UTF-8 bytes rendered by search results. */ readonly searchResultBytes: number; /** UTF-8 bytes rendered by bash results, whose source coverage is opaque. */ readonly bashResultBytes: number; /** Successful bash calls before mutation. */ readonly opaqueBashCalls: number; } /** Localization and reasoning telemetry before the first successful mutation. */ export interface PreMutationMetrics { /** Whether a mutation succeeded, was only attempted, or was never requested. */ readonly outcome: "mutated" | "attempted" | "none"; /** First mutating call, including failed attempts. */ readonly firstAttemptedMutation: MutationPointMetrics | null; /** First successfully completed mutating call. */ readonly firstSuccessfulMutation: (MutationPointMetrics & { /** Seconds from agent start until successful tool completion. */ readonly appliedSeconds: number | null; }) | null; /** Tool calls strictly before the first successful mutation, or every call when none succeeded. */ readonly callsBeforeMutation: number; /** Failed tool calls completed before the first successful mutation. */ readonly failedCallsBeforeMutation: number; /** Provider usage through the response that requested the first successful mutation. */ readonly usageToMutation: UsageMetrics; /** Provider-reported cost through that response. */ readonly costToMutationUsd: number; /** Source and tool output exposed before successful mutation. */ readonly sourceExposure: SourceExposureMetrics; } /** Stable size counters for the system prompt. */ export interface SystemPromptMetrics { /** UTF-8 byte count. */ readonly bytes: number; /** Unicode code-point count. */ readonly characters: number; /** Number of text lines. */ readonly lines: number; } /** Aggregated agent telemetry derived from a trace. */ export interface AgentMetrics { /** Seconds from agent start until settlement, when both events exist. */ readonly agentWallSeconds: number | null; /** Total completed tool calls. */ readonly toolCalls: number; /** Tool calls completed with an error. */ readonly failedToolCalls: number; /** Completed tool calls grouped by name. */ readonly toolCallsByName: Readonly>; /** Ordered tool calls with arguments and outcomes. */ readonly toolCallDetails: readonly ToolCallMetrics[]; /** Localization and reasoning telemetry before the first successful mutation. */ readonly preMutation: PreMutationMetrics; /** Size of the captured system prompt, when available. */ readonly systemPrompt: SystemPromptMetrics | null; /** Provider token usage summed across assistant messages. */ readonly usage: UsageMetrics; /** Provider-reported total cost in US dollars. */ readonly cost: { readonly total: number; }; } /** Scalar value produced by a consumer-defined trial metric. */ export type EvalMetricValue = number | boolean | null; /** Inputs available while calculating a consumer-defined metric. */ export interface EvalMetricInput { /** Scheduled task/agent profile identity for this trial. */ readonly trial: EvalTrial; /** Prepared workspace and suite state for this trial. */ readonly prepared: PreparedEvalTrial; /** Settled result returned by pi-coding-agent-test. */ readonly piResult: PiIntegrationTestResult; /** Suite validation score for this trial. */ readonly score: EvalScore; /** Evaluator-derived telemetry for the settled trace. */ readonly agentMetrics: AgentMetrics; /** Wall-clock duration from trial start through validation. */ readonly elapsedSeconds: number; } /** Consumer-defined metric calculated and persisted by the evaluator. */ export interface EvalMetricCalculator { /** Stable metric ID; it becomes `custom.` in calculated reports. */ readonly id: string; /** Calculate one scalar value for a completed trial. */ calculate(input: EvalMetricInput): Promise | EvalMetricValue; } /** A numeric distribution embedded in calculated report data. */ export interface EvalDistribution { /** Number of finite observations included in the distribution. */ readonly count: number; /** Smallest included value, or `null` when there are no observations. */ readonly min: number | null; /** Linearly interpolated 25th percentile. */ readonly p25: number | null; /** Median (50th percentile). */ readonly p50: number | null; /** Linearly interpolated 75th percentile. */ readonly p75: number | null; /** Largest included value, or `null` when there are no observations. */ readonly max: number | null; /** Arithmetic mean, or `null` when there are no observations. */ readonly mean: number | null; } /** One completed trial exposed to report renderers. */ export interface EvalTrialReport { /** Stable task/attempt pair shared by all agent profiles. */ readonly pairId: string; /** Task identifier. */ readonly taskId: string; /** Agent profile identifier. */ readonly agentProfile: string; /** Total elapsed time for the trial in seconds. */ readonly elapsedSeconds: number; /** Suite validation result. */ readonly score: EvalScore; /** Evaluator-derived agent telemetry. */ readonly agentMetrics: AgentMetrics; /** Consumer-defined values before aggregation. */ readonly customMetrics?: Readonly>; /** Flattened built-in and custom scalar values calculated by the evaluator. */ readonly metrics: Readonly>; /** Additional artifact fields preserved for renderer consumers. */ readonly [key: string]: unknown; } /** Fully calculated data for one agent profile. */ export interface EvalAgentProfileReport { /** Number of completed trials in this agent profile. */ readonly trials: number; /** Arithmetic mean for every calculated numeric metric. */ readonly means: Readonly>; /** Sum for every calculated numeric metric. */ readonly totals: Readonly>; /** Distribution for every calculated numeric metric. */ readonly distributions: Readonly>; } /** Fully calculated right-minus-left comparison. */ export interface EvalComparisonReport { /** Left-hand agent profile in the comparison. */ readonly left: string; /** Right-hand agent profile; all deltas are `right - left`. */ readonly right: string; /** Number of pairs containing both agent profiles. */ readonly completePairs: number; /** Mean right-minus-left delta for every metric. */ readonly means: Readonly>; /** Distribution of right-minus-left deltas for every metric. */ readonly distributions: Readonly>; /** Per-pair deltas, including task and pair identifiers. */ readonly pairs: readonly Readonly>[]; } /** Complete presentation-independent report model. */ export interface EvalReportData { /** Version of the calculated report data contract. */ readonly schemaVersion: 4; /** Completed trial records with flattened metric values. */ readonly trials: readonly EvalTrialReport[]; /** Aggregated values keyed by agent profile ID. */ readonly agentProfiles: Readonly>; /** One comparison for every ordered agent profile pair. */ readonly comparisons: readonly EvalComparisonReport[]; } /** Files emitted by a report renderer. */ export interface EvalRenderedReport { /** Optional concise terminal representation returned to the runner. */ readonly terminal?: string; /** Optional Markdown written to `/summary.md`. */ readonly markdown?: string; /** Additional files written relative to the run directory. */ readonly files?: Readonly>; } /** * Render calculated report data into terminal text, Markdown, and/or extra files. * * The evaluator calculates all built-in and consumer-defined metrics before calling * the renderer. Extra file names are relative to the run directory; `summary.json` * is reserved for the calculated data. */ export type EvalReportRenderer = (data: EvalReportData) => Promise | EvalRenderedReport; /** Provider token counters used by reports. */ export interface UsageMetrics { /** Uncached input tokens. */ readonly input: number; /** Output tokens. */ readonly output: number; /** Cache-read input tokens. */ readonly cacheRead: number; /** Cache-write input tokens. */ readonly cacheWrite: number; /** Reasoning tokens. */ readonly reasoning: number; /** Provider-reported total tokens. */ readonly totalTokens: number; } /** Convert one Pi trace into stable evaluation telemetry. */ export type TraceSummarizer = (events: readonly TraceEvent[]) => AgentMetrics;