/** * `dryRunFlow`: `runFlow` itself, with every agent turn, every check, every * commit and every question answered by a script. * * The same walk, the same `spawn`, the same events. Only the session under * each subagent and the `check` port are scripted, so a typed answer goes * through the real `submit` tool, a verdict through the real `verdict` tool * and its ledger, and a failure takes its real path, retries and `fail-fast` * included. Nothing of the world is touched: it takes a `CheckedFlow`, no * working directory is given, no script is read, no model is reached and git * is never run. `diff` reads as an empty text, since no agent wrote anything, * and a `copies: true` block makes no copy: each of its branches reads as * landed, and its copy is a journal fact alone. Given `from`, it resumes that * journal through the same `resumePoint` a real resume takes. */ import { expiry } from "../../deadline.ts"; import { spawn } from "../../subagent.ts"; import type { Usage } from "../../usage.ts"; import type { SpawnFn } from "../../workflows/options.ts"; import type { ScriptOutcome } from "../../verify.ts"; import type { CheckedAskNode, CheckedCallNode, CheckedCheckNode, CheckedCommitNode, CheckedFlow } from "../checked.ts"; import type { Heard } from "./ask.ts"; import type { Attempt } from "./agent.ts"; import { Script, type AnswerFault, type Answers } from "./answers.ts"; import type { CommitOutcome } from "./commit.ts"; import { checkInput, walkFlow, type FlowResult, type RunFlowOptions } from "./flow.ts"; import type { JournalEntry } from "./journal.ts"; import { resumePoint } from "./resume-point.ts"; import { scriptedSession, type ScriptedSession } from "./scripted.ts"; /** What a dry run varies: what a run does, short of reaching the world. */ export type DryRunOptions = Pick & { /** A journal to resume, as `resumeFlow` resumes a run directory's: its facts come first in the one handed back. */ readonly from?: readonly JournalEntry[]; }; /** * How a dry run ended: as the flow did, or at the first visit its script did * not answer, which is no outcome of the flow's; or refused before the start, * with every fault of the script, or a journal `from` a resume would refuse, * with why. The journal holds every fact a run would * write, in order, the run's end last. Its tokens and cost are zero, since a * script spends nothing; the time is measured, like any run's. */ export type DryRun = | (FlowResult & { readonly journal: readonly JournalEntry[] }) | { readonly ok: false; readonly unscripted: string; readonly journal: readonly JournalEntry[]; readonly usage: Usage } | { readonly ok: false; readonly faults: readonly AnswerFault[] } | { readonly ok: false; readonly refused: string }; /** Runs `checked` on `input`, each agent turn, check, commit and question answered from `answers`. */ export async function dryRunFlow(checked: CheckedFlow, input: unknown, answers: Answers, options: DryRunOptions = {}): Promise { const checkedScript = Script.check(checked, answers); if (!checkedScript.ok) return { ok: false, faults: checkedScript.faults }; const { script } = checkedScript; checkInput(checked, input); const point = options.from && resumePoint(checked, options.from); if (point !== undefined && !point.ok) return point; const journal: JournalEntry[] = [...(options.from ?? [])]; // A hole in the script stops the run the way a person would, so nothing // in the flow can absorb it, and it is reported apart from the flow's end. const halt = new AbortController(); const signal = options.signal === undefined ? halt.signal : AbortSignal.any([options.signal, halt.signal]); let unscripted: string | undefined; const sessions = new Map(); const scripted: SpawnFn = async (agent, spawnOptions) => { let session: ScriptedSession | undefined; // A scripted answer stands for the whole turn, reading included: a text // is a turn that did its work, and `{ fail: "no-tool" }` one that did not. const subagent = await spawn({ ...agent, mustCallTool: undefined }, { ...spawnOptions, createSession: async (_agent, sessionOptions) => (session = scriptedSession(sessionOptions)) }); sessions.set(subagent.id, session as ScriptedSession); return subagent; }; // No clock: each attempt's deadline is a switch the scripted session // throws when its answer is a timeout, with the reason a clock would give. const deadline = ({ path, at, subagent, ms }: Attempt) => { const controller = new AbortController(); const turn = script.next(path, at); if (turn === undefined) hole(path); else sessions.get(subagent)?.stage(turn, () => controller.abort(expiry(ms))); return controller.signal; }; const ran = async (node: CheckedCheckNode | CheckedCommitNode, path: string): Promise => script.ran(path, node.at) ?? hole(path); const hole = (path: string): T => { unscripted ??= path; halt.abort(); return { ok: false, kind: "stopped", message: "unscripted" } as T; }; // A hole takes the card down the way a stop does: the visit reads the stop, not this. const ask = async (node: CheckedAskNode, path: string): Promise => script.heard(path, node.at) ?? (hole(path), { declined: true }); const whole = (node: CheckedCallNode, path: string) => script.whole(path, node.at); const diff = async () => ({ ok: true as const, value: "" }); const world = { deadline, check: ran, commit: ran, diff, ask, whole, journal: { append: (entry: JournalEntry) => journal.push(entry) }, replay: point?.replay }; const { onEvent, model, timeoutMs } = options; const result = await walkFlow(checked, input, { onEvent, model, timeoutMs, signal, spawn: scripted }, world); if (unscripted !== undefined) return { ok: false, unscripted, journal, usage: result.usage }; return { ...result, journal }; }