import { EvalCache } from "./evalCache.js"; import { AgencyRunner } from "./grading/agencyRunner.js"; import type { BaseGrader } from "./grading/baseGrader.js"; import { Scorecard } from "./grading/scorecard.js"; import type { AgentRun, Input } from "./grading/types.js"; import type { BaseOptimizerConfig, OptimizeTarget } from "./optimizer.js"; import { type PointwiseReporter } from "./reporter.js"; import type { OptimizeMutationDiagnostic, OptimizeMutationOperation, OptimizeMutationPreview } from "./sourceMutator.js"; import { type OptimizeTargetSet } from "./targets.js"; import type { MutationProposal, OptimizeDecision, OptimizeResult } from "./types.js"; import { WorkspaceManager, type Workspace } from "./workspace.js"; /** Result of proposing a mutation: a clean preview, or the reason it couldn't be produced. */ export type MutationOutcome = { ok: true; preview: OptimizeMutationPreview; rationale: string; } | { ok: false; rationale: string; diagnostics: OptimizeMutationDiagnostic[]; }; /** A function that runs the agent for one input in a workspace and returns its run. * Receives the candidate's `source` (`baseDir`/`entryFile` live here) and `files` * (the candidate's complete file map, used as the workdir overlay). */ export type RunInput = (ws: Workspace, source: OptimizeTargetSet, files: Record, input: Input, id: string) => Promise; export type BaseOptimizerDeps = { agencyRunner?: AgencyRunner; cache?: EvalCache; /** Override how the agent under test runs (tests inject a fake; default uses the eval-run path). */ runInput?: RunInput; /** Override the progress reporter (tests inject one that captures lines). */ reporter?: PointwiseReporter; /** Override target discovery (tests inject a fixed target set; default parses the agent file). */ discover?: (agentFile: string) => OptimizeTargetSet; }; export declare abstract class BaseOptimizer { protected readonly config: BaseOptimizerConfig; protected readonly workspace: WorkspaceManager; protected readonly agencyRunner: AgencyRunner; protected readonly cache: EvalCache; protected readonly reporter: PointwiseReporter; private readonly runInput; private readonly discover; private runCounter; /** Held-out validation inputs (empty when none); set in optimize(). */ protected validationInputs: Input[]; constructor(config: BaseOptimizerConfig, deps?: BaseOptimizerDeps); abstract readonly name: string; /** * Resolve the agent file and discover its optimize targets once, then hand the * target set to the subclass. Every optimizer needs this preamble, so it lives * here — subclasses implement {@link optimizeTargets} and never touch discovery. */ optimize(target: OptimizeTarget): Promise; /** Print the resolved grading setup and fail fast on a misconfigured grader, * checked against the first input before any agent run. */ private echoAndValidateGrading; /** Run the search over already-discovered targets. The one method an optimizer must implement. */ protected abstract optimizeTargets(source: OptimizeTargetSet, inputs: Input[]): Promise; /** * A scorecard at the maximum objective can't be improved, so optimizers stop * early (or skip the loop when the baseline is already there). Assumes graders * are normalized to [0, 1]; binary-only setups score 0 and never trip this. */ protected isMaxObjective(scorecard: Scorecard): boolean; /** * Propose a mutation and validate it, with bounded retries. Two failure modes * are handled here so a single bad LLM response never aborts the run: * - the proposer throws (malformed/unparseable response) — caught and retried; * - the proposal is well-formed but fails validation (e.g. dropped an * interpolation) — the diagnostics are fed back into the next `propose` * call so the model can correct itself. * Returns the first clean preview, or `{ ok: false }` with the last failure's * reason after `maxAttempts`. Optimizers turn that into a failed iteration. */ protected proposeValidMutation(propose: (priorDiagnostics: OptimizeMutationDiagnostic[]) => Promise, preview: (operations: OptimizeMutationOperation[]) => OptimizeMutationPreview, maxAttempts?: number): Promise; protected fork(): Workspace; /** Allocate a fresh cache-partition workspace and grade `files` on `inputs`. * The canonical fresh-scoring primitive (used for validation). */ protected scoreFiles(source: OptimizeTargetSet, files: Record, inputs: Input[]): Promise; /** Choose the writeback champion among candidates: the one with the best * validation objective when a validation set exists, else the given train * champion. Scoring (the "how") is separated from the max selection (the * "what"); shared by the pointwise optimizers so validation selection lives * in one place. */ protected pickValidationChampion; scorecard: Scorecard; }>(source: OptimizeTargetSet, candidates: C[], trainChampion: C): Promise<{ champion: C; validationObjective?: number; }>; /** The shared tail every pointwise optimizer runs: pick the writeback champion * (by validation when configured), write it back, build the result with its * train/validation objectives + grade breakdown, and report completion. An * optimizer's job is just to produce the candidates and per-iteration attempts; * this turns them into the final OptimizeResult. */ protected finishPointwise; scorecard: Scorecard; targetSet: OptimizeTargetSet; }>(source: OptimizeTargetSet, candidates: C[], trainChampion: C, attempts: { iter: number; decision: OptimizeDecision; detail?: string; }[], startedAt: number): Promise; /** Run the agent once per input (cached by (workspace, input)), grade each, return a Scorecard. * The candidate's `files` map is the overlay applied inside each per-input workdir. */ protected evaluate(ws: Workspace, source: OptimizeTargetSet, files: Record, inputs: Input[]): Promise; private gradeInput; /** Default runInput: run the agent for one input via the eval-run subprocess path. * Passes `seed` (used verbatim — no closure recomputation, no silent divergence * from `source.baseDir`) and `overlayFiles` (the candidate's complete file map) * to `evalRunLoadedInputs`. The per-input `working_dir` field is forbidden * here (the caller-supplied seed would conflict with it). */ private runInputViaEval; protected eachIteration(step: (iter: number) => Promise): Promise; protected get graders(): BaseGrader[]; /** Refuse to optimize a program whose baseline already fails a must-pass grader. */ protected requireBaselineGatesPass(scorecard: Scorecard): void; /** * Build the pointwise OptimizeResult shared by greedy and GEPA. `winsA`/`winsB`/`ties` * are pairwise-judge artifacts that pointwise optimizers leave at 0. */ protected buildPointwiseResult(args: { championIter: number | "baseline"; championFiles: Record; attempts: { iter: number; decision: OptimizeDecision; detail?: string; }[]; }): OptimizeResult; } /** * The last graded output value, or a clear error when there is none — the agent * returned nothing AND didn't call evalOutput(), so there is nothing to grade. */ export declare function gradedOutput(evalOutputs: { value: unknown; }[], inputLabel: string): unknown;