import { type JudgeSuiteArgs } from "../eval/judge/suite.js"; import type { PairwiseVerdict, SuiteVerdict } from "../eval/judge/types.js"; import { type EvalRunState, type PreparedInput } from "../eval/runArtifacts.js"; import type { EvalRunResult, Input, EvalRunInputResult } from "../eval/runTypes.js"; import type { EvalRecord } from "../eval/types.js"; import { optimizeLoop } from "../optimize/loop.js"; import type { OptimizeLoopConfig, OptimizeResult } from "../optimize/types.js"; /** * State carried by the Agency-side `evalRun` loop. Note: this is just the * underlying `EvalRunState` plus the parsed input list — we deliberately do * not stash a mutable `results: EvalRunInputResult[]` here. The Agency loop * accumulates results in its own local array so the data flow is visible * at the call site instead of hidden as a side effect. */ export type AgencyEvalRunState = EvalRunState & { inputs: Input[]; }; /** * Result of the "prepare" step exposed to Agency. Discriminated so the * Agency loop can branch on `.ok` instead of relying on `try` to catch * thrown errors from the TS helper. */ export type AgencyPrepareResult = { ok: true; prepared: PreparedInput; } | { ok: false; result: EvalRunInputResult; }; export declare function _initEvalRun(compiled: { moduleId: string; }, inputs: Input[], node: string, runsDir: string, runId: string, continueOnError: boolean): AgencyEvalRunState; /** * Prepare an input's artifacts. Never throws — validation errors are returned * as a `{ ok: false, result }` so the Agency loop has a single branching * pattern (`if (prep.ok)`) instead of an extra `try` per iteration. */ export declare function _prepareInput(state: AgencyEvalRunState, input: Input): AgencyPrepareResult; /** * Finalize a prepared input after `std::agency.run` returned. `runError` is * empty on success, otherwise a flattened error message from the Agency * `try`/`isFailure` branch. Extracts the eval record when a statelog was * written. Never throws — any extract failure is turned into an error * result so the Agency loop never has to wrap this call in `try`. */ export declare function _finalizeInput(prepared: PreparedInput, runError: string): Promise; export declare function _finishEvalRun(state: AgencyEvalRunState, results: EvalRunInputResult[]): EvalRunResult; /** * Flatten an Agency `try` failure value into a single string message. The * shape can vary because failures come from a mix of structured limit * payloads, JS Error wrappers, and plain strings. */ export declare function _formatEvalRunFailure(value: any): string; /** * Stdlib binding for `eval extract`. Reads a JSONL statelog at * `statelogPath` and returns the structured EvalRecord that * `agency eval extract` would write. Composes the existing extractor * pipeline; no separate logic here. */ export declare function _evalExtract(statelogPath: string): Promise; /** * Stdlib binding for `eval judge`. Pairwise-judges two eval records against * a goal and returns the structured PairwiseVerdict. Delegates to the * existing `judgePairwise` so CLI and stdlib paths share judge behavior * (including the subprocess judge invocation through `runAgencyJudge`). */ export declare function _evalJudge(goal: string, recordPathA: string, recordPathB: string): Promise; /** * Stdlib binding for suite-aware eval judging. Compares two eval run * directories by input id and returns the suite verdict produced by the core * judgeSuite helper. */ export declare function _evalJudgeSuite(runA: string, runB: string, inputs: Input[], samples: number, confidenceThreshold: number, marginThreshold: number, positionBias: string, judge?: (args: JudgeSuiteArgs) => Promise): Promise; /** * Stdlib binding for `agency.eval.optimize`. This deliberately does not * install any approval handler; callers decide which handlers are in scope. * * Mirrors the `agency eval optimize` CLI: exactly one of `inputs` or `goal` * selects the suite, a goal desugars through `inputFromGoal()`, and a * candidate is accepted iff the judge suite returns winner `B`. */ export declare function _optimize(config: AgencyConfigLike, entryFile: string, workingDir: string, node: string, inputs: Input[], goal: string, iterations: number, samples: number, confidenceThreshold: number, marginThreshold: number, runsDir: string, runId: string, mutatorModel: string, writeback: boolean, verbosity: string, loop?: typeof optimizeLoop): Promise; type AgencyConfigLike = OptimizeLoopConfig["runtime"]["config"]; export {};