/** * A workspace diff scoped to one 0-based configured conversation turn. * * Exactly one payload is permitted. In particular, an error must never * coexist with an empty `diff`, because that could make a negative diff grader * pass against evidence whose capture failed. */ export type TrajectoryTurnDiff = { turn: number; diff: string; diffPath?: never; error?: never; } | { turn: number; diffPath: string; diff?: never; error?: never; } | { turn: number; error: string; diff?: never; diffPath?: never; }; /** * Trajectory — the internal behavioral representation of a single run. * * Intentionally simple: a flat event array that's easy for graders to * construct, query, and assert against. External trace formats * (OpenTelemetry/OpenInference, agent dumps) are supported via adapters * in trajectory/adapters/, not as the native representation. */ export interface Trajectory { id: string; stimulus: import("../eval/types.js").Stimulus; events: TrajectoryEvent[]; metrics: TrajectoryMetrics; output: string; workDir: string; metadata: TrajectoryMetadata; /** * Optional directory where the executor records run artifacts *outside* the * graded {@link Trajectory.workDir}. Some graders (e.g. `custom-metrics`) look * here before falling back to {@link Trajectory.workDir}. */ artifactDir?: string; /** * Optional managed directory holding setup **inputs** staged *outside* the * graded {@link Trajectory.workDir} (environment files with * `dest_root: "assets"`). Distinct from {@link artifactDir}, which holds run * *outputs*. Exposed to graders (program graders receive it as the reserved * `EVALUATE_ASSETS` env var) so validation fixtures can live off the graded * tree. A host path — may be absent on a different grading host (offline * re-grade); graders must treat a missing value gracefully. */ assetsDir?: string; /** * When true, {@link artifactDir} is an *authoritative* location and graders * that read artifacts from it must NOT fall back to {@link workDir}: a missing * file surfaces as a failure against the artifact dir rather than silently * grading a stale workspace copy. Set by the CLI `grade --artifact-dir` flag * for offline re-grade (where the workspace is typically absent or stale), and * by the trial runner when a relocating backend's artifact dir could not be * materialized to a host-readable path (so re-grade fails clearly instead of * grading a stale copy). Left unset when the executor artifact dir is already * host-readable or was materialized successfully, keeping the lenient * artifact-dir-first, workspace-fallback lookup. */ artifactDirStrict?: boolean; /** How the run ended. Absent on legacy trajectories. */ endReason?: TrajectoryEndReason; /** * Path to a throwaway git dir in the system temp directory used by diff * graders. Created via `mkdtemp` so it is always unique and vally-owned. * Absent when no diff grader is configured. Paired with {@link baselineRef}. */ baselineGitDir?: string; /** Baseline commit SHA in {@link baselineGitDir}. */ baselineRef?: string; /** * Unified workspace diff (baseline → final state). May be up to 64 MiB and * contains raw file content — treat as potentially sensitive. Absent when no * diff grader is configured, or diff computation failed. */ diff?: string; /** * Path to the sidecar diff artifact written by the JSONL reporter. */ diffPath?: string; /** * Workspace diffs produced by individual configured conversation turns. * * These are additive evidence for turn-scoped static diff graders. The * cumulative {@link diff} / {@link diffPath} remains the source for * unscoped graders. */ turnDiffs?: TrajectoryTurnDiff[]; /** * Workspace diff (baseline → final state). May be up to 64 MiB and contains * raw file content — treat as potentially sensitive. Where it's persisted to * disk (e.g. as `workspace.patch`) is a CLI/caller concern, not guaranteed * by core. Empty string = unchanged workspace. `undefined` = capture not * requested, or capture failed — see {@link workspacePatchCaptureError}. * Stripped from JSONL output via {@link stripWorkspacePatchForJsonl}. */ workspacePatch?: string; /** * Which strategy produced {@link workspacePatch}: `"fast-path"` diffs the * pre-execution HEAD against the final workspace using its own git repo; * `"snapshot"` diffs a throwaway baseline commit; `"reused-diff"` reuses * {@link diff} already computed for a diff grader. Absent when * `workspacePatch` is absent. */ workspacePatchSource?: "fast-path" | "snapshot" | "reused-diff"; /** * Set when workspace patch capture was attempted but failed (git error, * buffer overflow, etc.). Absent for baseline capture failures, which are * reported in {@link baselineCaptureError}. */ workspacePatchCaptureError?: string; /** * Error message from baseline capture, when it failed. */ baselineCaptureError?: string; /** * Escape hatch: the original, unconverted source document this trajectory * was built from (e.g. the raw ATIF object). * * **Prefer the normalized {@link events}.** Reading `raw` opts out of Vally's * portability contract — it is untyped (`unknown`) and its shape is dictated * by the producer, so a grader keyed on it couples to a foreign schema and * breaks when the input comes from a different executor/format. Reach for it * only when the flat event model genuinely can't express what you need, and * treat each such use as a signal that the missing field should be promoted * into a first-class {@link TrajectoryEvent}. * * Populated only by offline adapters like `fromAtif`; absent for live * executor runs. Stripped from JSONL output — it stays in-process only, so it * reaches graders during the same run but is not persisted (see the JSONL * reporter's `serializeTrajectory`). * * Because it is not persisted, a grader that reads `raw` works only on the * live pass: re-grading from JSONL (`vally grade `) will see * `undefined`. Don't write a grader whose result depends on `raw` if it must * survive re-grade. * * Treat as read-only. Like the rest of the trajectory, it is shared across * graders (and, for offline adapters, held by reference to the caller's * source object) — mutating it can corrupt inputs observed by subsequent * graders. Graders must not modify it. */ raw?: unknown; } export type TrajectoryEndReason = "completed" | "agent_timeout" | "simulation_cap"; interface BaseEvent { timestamp?: Date; turn?: number; /** * Identifier of the agent that produced this event; absent means the * parent/root agent. Derived from each source's own identity (e.g. the ATIF * adapter uses the subagent's name), so not a guaranteed-unique instance id. */ agentId?: string; } /** * Tool-call arguments use producer-defined root shapes. Values loaded from * persisted trajectories are JSON-compatible; nested values remain `unknown` * so consumers can narrow without claiming recursive validation. */ export type ToolCallArguments = Record | unknown[] | string | number | boolean | null; /** * The start of a tool call. The result will be delivered in {@link ToolResultEvent}. */ export interface ToolCallEvent extends BaseEvent { type: "tool_call"; data: { toolName: string; toolCallId: string; /** * Identity of the model-response batch (turn) this call belongs to. Matches * the surrounding {@link TurnStartEvent} when present, but may also be set * without any surrounding boundaries (e.g. subagent-inlined calls). Graders * batch `parallel` calls by this when present, and fall back to positional * turn counting when absent. Only needs to be unique within a single agent: * consumers scope batches by the event's `agentId`, so a subagent and its * parent may reuse the same value without merging. */ turnId?: string; /** * Agent identities launched by this call, when the source trajectory * explicitly links the call to subagent trajectories. */ launchedAgentIds?: string[]; /** * True when this tool call was intercepted and simulated * rather than executed against the real environment. Absent for real calls. */ simulated?: boolean; /** * Commonly an object, but scalar, array, and `null` roots are also valid. * Keyed consumers should narrow before reading properties. */ arguments?: ToolCallArguments; }; } /** * The result of a tool call, first seen in {@link ToolCallEvent}. */ export interface ToolResultEvent extends BaseEvent { type: "tool_result"; data: { toolName: string; toolCallId: string; success: boolean; result: unknown; }; } interface TokenUsageEvent extends BaseEvent { type: "token_usage"; data: TokenUsageData; } interface TurnEndEvent extends BaseEvent { type: "turn_end"; data: { turnId: string; }; } interface TurnStartEvent extends BaseEvent { type: "turn_start"; data: { turnId: string; }; } interface AssistantMessageEvent extends BaseEvent { type: "assistant_message"; data: { content: string; }; } interface UserMessageEvent extends BaseEvent { type: "user_message"; data: { content: string; agent_mode?: string; }; } interface ErrorEvent extends BaseEvent { type: "error"; data: { message: string; type?: string; url?: string; code?: number; }; } interface SkillActivationEvent extends BaseEvent { type: "skill_activation"; data: { name: string; path: string; content?: string; pluginName?: string; pluginVersion?: string; allowedTools?: string[]; }; } /** Used for reasoning or thinking events from the agent */ interface ReasoningEvent extends BaseEvent { type: "reasoning"; data: { content: string; }; } /** * A system-level event such as context compaction. Carries a typed * {@link SystemEvent.data.eventType} (e.g. `"compaction"`) mapped from the * source's system-event kind, so graders can match on the semantics rather * than parsing free-text. */ interface SystemEvent extends BaseEvent { type: "system"; data: { /** * System event kind, e.g. `"compaction"`. Derived from the source's typed * kind (ATIF `context_management.type`). System steps without a typed kind * are not emitted as `system` events by the ATIF adapter. */ eventType: string; /** The system step's message text, when present. */ message?: string; /** Flattened observation content emitted alongside the step, when present. */ observation?: string; /** Structured fields for this `eventType`; names are stable only per `eventType`, not across all system events. */ details?: Record; }; } interface CustomEvent extends BaseEvent { type: "custom"; data: Record; } export type TrajectoryEvent = AssistantMessageEvent | CostUnavailableEvent | CustomEvent | ErrorEvent | ReasoningEvent | SkillActivationEvent | SystemEvent | TokenUsageEvent | ToolCallEvent | ToolResultEvent | TurnEndEvent | TurnStartEvent | UserMessageEvent; /** * Token usage from a single LLM call. * * Executors emit token_usage events with this shape in event.data. * Values come from API responses (response.usage), not local estimation. */ export type ProviderCost = { provider: "github-copilot"; unit: "nano-aiu"; amount: number; }; export type ProviderCostIdentity = Omit; export interface TokenUsageData { inputTokens: number; outputTokens: number; model: string; cacheReadTokens?: number; cacheWriteTokens?: number; /** Provider-labelled usage cost for this request. */ cost?: ProviderCost; } interface CostUnavailableEvent extends BaseEvent { type: "cost_unavailable"; data: ProviderCostIdentity & { model: string; reason: "malformed_usage"; }; } /** Aggregated token usage across all LLM calls in a trajectory. */ export interface TokenMetrics { inputTokens: number; outputTokens: number; totalTokens: number; cacheReadTokens: number; cacheWriteTokens: number; callCount: number; /** * Provider-labelled usage cost across every LLM call. Omitted when the source * did not report a complete cost with a consistent provider and unit. */ cost?: ProviderCost; byModel: Record; } export interface TrajectoryMetrics { tokenUsage: TokenMetrics; toolCallCount: number; toolCallBreakdown: Record; /** * Number of tool calls that were simulated rather than * executed for real. Always `<= toolCallCount`. Zero when simulation is unused. */ simulatedToolCallCount: number; skillActivationCount: number; skillActivationBreakdown: Record; turnCount: number; wallTimeMs: number; errorCount: number; } export interface TrajectoryMetadata { model: string; /** * Names of skills that were loaded for this run. */ skillsLoaded: string[]; startedAt?: Date; completedAt?: Date; executor: string; sessionID: string; } export {}; //# sourceMappingURL=types.d.ts.map