import type { AgencyConfig } from "../../config/config.js";
import type { Test } from "../../eval/runTypes.js";
import { type RunDirectorySnapshot } from "../../runDirectory/runDir.js";
import type { AgencyRunner } from "./agencyRunner.js";
import type { BaseGrader } from "./baseGrader.js";
import { Scorecard } from "./scorecard.js";
/** Where each test's graders come from. Graders are test-side, like goal and
 *  expected: a test's own `graders.ts` and its harness pairs. The question
 *  is which copy.
 *  - "snapshot": the copy the run directory stored when the agent ran (the
 *    default; a run grades the same days later, wherever it is read).
 *  - "suite": the test's CURRENT graders in this loaded suite, matched by
 *    test id (`eval grade --suite`: improve a grader, re-score old runs
 *    without re-running the agent). A run whose test is not in the suite is
 *    an error, not a silent fallback.
 *  - "override": one set for every test, ignoring what the tests carry. The
 *    optimizer's objective; never a CLI option.
 *  A test with no graders of its own is scored by the bundled goal judge
 *  against its `goal`. */
export type GraderSource = {
    kind: "snapshot";
} | {
    kind: "suite";
    tests: Test[];
} | {
    kind: "override";
    graders: BaseGrader[];
};
/** What grading needs besides the run directory itself. */
export type GradingContext = {
    graders: GraderSource;
    /** Capability to execute a judge .agency file. Built from an AgencyConfig. */
    runAgency: AgencyRunner;
    /** For loading per-test grading modules. */
    config: AgencyConfig;
    /** `eval grade --goal`: the goal for every test that recorded none of its
     *  own (an ad-hoc trace, or a suite that never had goals). A test's own
     *  goal always wins. */
    defaultGoal?: string;
};
/**
 * Score every trace in a run directory. The run directory is grading's ONLY
 * input: run data is never handed over in memory, so grading right after
 * `eval run` reads the same directory `eval grade` reads days later. (See
 * docs/dev/evals/eval-grading.md.)
 */
export declare function gradeRun(runDir: string, ctx: GradingContext): Promise<Scorecard>;
/** The same, from a snapshot the caller already read. */
export declare function gradeSnapshot(snapshot: RunDirectorySnapshot, ctx: GradingContext): Promise<Scorecard>;
/**
 * Fail fast on a misconfigured grader, checked against one test before any agent
 * runs. Match-based graders override `validateInput` to reject an unresolved
 * `matchOn`; without this the error surfaces per test, after the whole suite has
 * already been run and paid for. Mirrors what `BaseOptimizer` does in its preamble.
 */
export declare function validateGraders(graders: BaseGrader[], test: Test | undefined): void;
/** One load per graders path per call, however many tests share a
 *  `graders.ts` or `graders.json`. Exported so the optimizer's pre-run
 *  validation loads through the same path grading does. */
export declare function makeGraderModuleCache(_config: AgencyConfig): (modulePath: string) => Promise<BaseGrader[]>;