/** * loop-outcome grader — asserts whether an agent looped, matching an expected * outcome. * * Like `max-repeat`, this is a post-hoc, trajectory-level detector that walks * {@link Trajectory.events} after the run without intervening. Unlike * `max-repeat` (which only *fails* on loops), it can also assert a loop *was* * expected (`should_loop: true`) — useful for probing recovery tooling. * * Two independent signals count as "looping": * 1. The agent re-issued the same action or short action pattern more than * `max_acceptable_retries` times (N complete occurrences are N-1 retries). * `match` (default `name-args`) defines each action and `tools` scopes * watched tools — both mirror `max-repeat`. Parallel calls in one turn * count once. Works on any trajectory. * 2. The run ended with `endReason: "simulation_cap"` — the agent exhausted * the simulation's `max_iterations` budget (see {@link SimulationConfig}). * Always counts as looping, regardless of `match`/`tools`. * * The verdict compares that to `should_loop`: * - `false` (default) — pass when the agent did **not** loop. * - `true` — pass when the agent **did** loop. * * ```yaml * graders: * - type: loop-outcome * config: * should_loop: false # the agent should recover, not spin * max_acceptable_retries: 3 # >3 retries = a loop * max_cycle_period: 2 # optionally detect A B A B cycles * match: name-args # "name-args" (default) | "name" | "observation" * tools: ["^bash$"] # optional regex patterns; default watches all tools * ``` */ import type { Grader, GraderInput, GraderMetadata, GraderResult } from "../types.js"; import { type MatchMode } from "../helpers.js"; /** Expected loop outcome for a run. */ export interface LoopOutcomeConfig { /** * Whether looping is the expected behavior. Defaults to `false` — the agent * should recognize the dead end and stop. */ should_loop?: boolean; /** * Maximum retries after the first complete occurrence still considered * acceptable; more counts as looping. A run of N identical calls is N-1 * retries. Required, integer `>= 0` — authors must opt in (mirrors * `max-repeat`); use `0` for zero tolerance. * * Only governs the retry signal — a run that hits the simulation cap * (`endReason: "simulation_cap"`) always counts as looping. Since a * non-capped run has at most `max_iterations - 1` retries, a threshold that * high or higher can never trip the retry signal. */ max_acceptable_retries: number; /** * Longest repeating pattern period to detect. Defaults to `1`, preserving * consecutive-only retries. Integer 1–10 (inclusive). */ max_cycle_period?: number; /** * What counts as "the same" repeated action. Defaults to `"name-args"` * - `name-args`: tool name plus arguments (deep equal). * - `name`: tool name only. * - `observation`: the tool's result/output. */ match?: MatchMode; /** * Unanchored regex patterns selecting which tools the retry signal watches * (all when omitted; use `^bash$` for exact). Doesn't affect the * simulation-cap signal. */ tools?: string[]; } export declare class LoopOutcomeGrader implements Grader { metadata: GraderMetadata; /** `loop-outcome retries 2` — the retry budget discriminates instances. */ defaultName(config: Record): string; /** * `should_loop: false` (the default) asserts the agent did NOT loop. An empty * trajectory has no loop, so the check passes — under the oracle baseline it is * reported N/A. `should_loop: true` expects a loop instead and correctly fails * on the empty baseline, so it is not an absence assertion. */ assertsAbsence(config: Record): boolean; grade(input: GraderInput): Promise; } //# sourceMappingURL=loop-outcome-grader.d.ts.map