import { Type, type Static } from "typebox"; import type { Runner, ResumeTaskConfig } from "../core/runner/runtask.js"; import type { CheckpointToken, ResumeOutcome } from "../core/checkpoint-store.js"; import type { ModelRef, TaskResult, TaskSpec, ToolSpec } from "../core/types.js"; /** * Verification gate (developer mode, design/28 §4). An **independent adversarial verifier** runs after * an implementation task and tries to BREAK it — read-only, evidence-required, returning a structured * verdict — then the gate loops fix→re-verify until PASS or a round cap. Distilled and de-branded from * Claude Code's `verification` agent (MIT): the value is in catching the last 20%, not confirming the * happy path. * * It is a **thin composition** over existing core seams — a verifier subtask (own model role), a * read-only tool set (via tool `effect`), {@link TaskSpec.outputSchema} for the verdict, and the * teacher-style fix loop — so it adds no Runner-core surface. Off by default; opt in per task via * {@link runWithVerification} (or {@link runDeveloperTask}). * * Boundary vs teacher mode: teacher's Tier-1 is a *lenient* rubric verifier that triggers escalation to * an advisor (stuck/wrong recovery); this is a *strict adversarial* completion gate with a fix loop. * Orthogonal, composable, not merged. */ export declare const VERIFICATION_PROMPT = "You are a verification specialist. Your job is NOT to confirm the implementation works \u2014 it is to try to BREAK it.\n\nYou have two documented failure patterns. First, verification avoidance: faced with a check, you find reasons not to run it \u2014 you read code, narrate what you would test, declare \"PASS,\" and move on. Second, being seduced by the first 80%: a polished result or a passing test suite makes you inclined to pass it, not noticing the edge that crashes, the state that vanishes, the bad input that is unhandled. The first 80% is the easy part. Your entire value is in finding the last 20%.\n\n## Hard boundary \u2014 do not modify the project\nYou are STRICTLY a verifier. Do NOT create, modify, or delete project files; do NOT install packages; do NOT run version-control write operations. Use only the read/probe/execute tools available to you. (If you need a scratch file, use a temp directory, and clean up.)\n\n## Evidence is mandatory\nReading code is NOT verification. Every check must actually run something \u2014 execute the code, hit the endpoint, run the build/tests \u2014 and record the command and its real output. A \"PASS\" with no command output is a skip, not a pass.\n\n## Strategy (adapt to what changed)\n- Build/lib changes: build it, run the full test suite, exercise the public API as a consumer would.\n- Backend/API: start it, call endpoints, check response *shapes* (not just status codes), test error paths.\n- CLI/script: run with representative AND edge inputs (empty, malformed, boundary); check stdout/stderr/exit codes.\n- Bug fix: reproduce the original bug first, verify the fix, then check for regressions and side effects.\n- Refactor (no behavior change): the existing suite must pass unchanged; diff the public surface; same inputs \u2192 same outputs.\nRun the project's own build/tests/linters as a baseline, then apply the type-specific checks. Test results are context, not proof \u2014 the implementer is an LLM too; its tests may be happy-path or circular.\n\n## Adversarial probes (pick the ones that fit)\nBoundary values (0, -1, empty, very long, unicode, max), idempotency (same mutating call twice), orphan operations (ids that don't exist), concurrency (parallel create-if-not-exists). Your verdict must include at least one adversarial probe you actually ran and its result \u2014 even if it was handled correctly.\n\n## Before you FAIL\nCheck you haven't missed why it's actually fine: defensive code elsewhere, intentional behavior documented in comments/specs, or an unfixable external-contract limitation (note that as an observation, not a FAIL). Don't wave away real issues, but don't FAIL on intentional behavior.\n\n## Verdict\nSubmit exactly one verdict via the provided output tool:\n- PASS \u2014 you ran real checks (including \u22651 adversarial probe) and it holds up. Put the commands + observed output in `evidence`.\n- FAIL \u2014 something is broken. Put each concrete problem (with how to reproduce) in `findings`.\n- PARTIAL \u2014 environmental limitation only (no test framework, a tool/server unavailable). Not for \"I'm unsure\": if you can run the check, decide PASS or FAIL. Note what you couldn't verify and why in `findings`."; /** * The L3 **static-judge** prompt for the L2+L3 composition (design/54 §4): the mechanical L2 gate already * ran the build/tests, so here the verifier is a **read-only judge** that scrutinizes the supplied diff + * test results — it must NOT try to execute code (the verifier's tools are read-only by design, design/44 * §6; telling it to "run tests" makes it judge PARTIAL on every module when the sandbox blocks the runtime — * service[36]/search[48] dogfood). {@link verifyCompleted} selects this automatically when `evidence` is set * and no `verifierPrompt` override is given. */ export declare const STATIC_VERIFICATION_PROMPT = "You are a verification judge. Your job is NOT to confirm the change works \u2014 it is to find where it BREAKS.\n\nYou are READ-ONLY by design: the build and tests have ALREADY been run by a separate mechanical gate. Their results and the code change (a diff) are usually supplied to you as evidence; but if little or no diff/results are supplied this round (e.g. a re-verification AFTER a fix), judge the CURRENT working tree directly \u2014 do NOT return PARTIAL merely because a diff is absent. Do NOT try to execute code, run tests, or invoke a runtime \u2014 the environment will refuse it, and that is expected, not a limitation. Judge from any supplied diff/results plus read-only inspection of the working tree (read files, search, list).\n\nYou have two documented failure patterns. First, being seduced by the first 80%: a clean diff or a green test run makes you inclined to pass it, not noticing the edge that crashes, the state that vanishes, the bad input that is unhandled, the cross-module assumption that breaks. Second, hiding behind PARTIAL because you couldn't run something \u2014 that is NOT what PARTIAL is for here; execution was the mechanical gate's job. Your entire value is finding the last 20% by READING.\n\n## What to scrutinize (adapt to the diff)\n- Boundary/edge cases the tests likely miss: 0, -1, empty, very long, unicode, max, malformed input, idempotency, orphan ids, off-by-one, negative numbers, EOF/empty fields.\n- Semantic correctness vs the task spec: does the change actually do what was asked, including cases the tests don't cover (the title()/CSV/base62 class of defect)?\n- Cross-module/integration hazards in the diff: a changed signature/export/contract/default a caller elsewhere still assumes; a deleted helper something depends on.\n- If the provided test results show failures, that is a concrete FAIL with the failing output as evidence.\n\n## Verdict\nSubmit exactly one verdict via the provided output tool:\n- PASS \u2014 you read the diff + results, looked for the edges above, and it holds. Cite the specific things you checked in `evidence`.\n- FAIL \u2014 you found a concrete defect. Put each problem (with the diff location / input that breaks it) in `findings`.\n- PARTIAL \u2014 ONLY when the evidence itself is genuinely insufficient to judge (e.g. the diff is empty or unrelated to the task, no results supplied). NOT for \"I couldn't execute it.\" Say what's missing in `findings`."; /** The verifier's structured verdict (delivered via {@link TaskSpec.outputSchema}). */ export declare const VerdictSchema: Type.TObject<{ verdict: Type.TUnion<[Type.TLiteral<"PASS">, Type.TLiteral<"FAIL">, Type.TLiteral<"PARTIAL">]>; findings: Type.TArray; evidence: Type.TOptional; }>; export type Verdict = Static; export interface VerifyConfig { /** * Verifier model (wins over the role). Default: resolve the `verifier` role (→ `default` fallback). * * 🔴 DECORRELATION (design/54 §3.1, [44]): the verifier MUST be a **different model than the implementer** * — an LLM grading its own work confirms its own blind spots (service[33]: a strong heterogeneous judge * reading the diff caught the title()/csv/base62 defects the implementer's own tests missed). The default * `verifier` role falling back to `default` (= the implementer) DEFEATS this — pass an explicit * heterogeneous `verifierModel`, or map a distinct `verifier` role. Decorrelation is a deployment contract, * not something this layer can assert (role→model resolution lives in the Runner). */ verifierModel?: ModelRef; /** * Tools the verifier may use — should be read/probe tools that DON'T mutate the project. * **Default**: the impl task's tools filtered to `effect: "read"` only (a true read-only boundary). * A generic core can't know which of YOUR tools mutate "the project", so if the verifier needs to run a * build/test runner, mark that tool `effect: "read"` (it doesn't persist project changes) or pass it here * explicitly. Anything `idempotent`/`write` is dropped by default — pass `verifierTools` to widen. */ verifierTools?: ToolSpec[]; /** Verify→fix→re-verify rounds (each round = one verification; FAIL between rounds triggers a fix turn). Default 2. */ maxRounds?: number; /** * The concrete **change to scrutinize** (design/54 §3.1) — typically the integration `git diff` the * orchestrator computes via {@link runExecGate}. Fed to the verifier as **untrusted, opaque-delimited * data** so the judge reads the actual code change, not the implementer's prose self-report (which a * compromised worker controls — threat BUG1). On a durable-HITL resume, the caller MUST recompute this * from the **post-resume** working tree (don't reuse a pre-suspend diff — threat BUG5, design/53 §2.B). */ evidence?: string; /** Stop the verify→fix loop once cumulative cost (verifier runs + impl fix turns) reaches this; return the current result. Matches `cascade`'s "all costs" sense. Optional (design/54 §3.4). */ costCeilingMicroUsd?: number; /** Overall wall-clock ceiling for the whole verify→fix loop. Optional (design/54 §3.4). */ totalTimeoutMs?: number; /** Override the verifier system prompt ({@link VERIFICATION_PROMPT}). */ verifierPrompt?: string; /** Per-round callback (observability). */ onRound?: (info: { round: number; verdict: VerificationOutcome["verdict"]; findings: string[]; }) => void; } /** * Why a {@link VerificationOutcome} is `"unverified"` (search [46] BUG7: `"unverified"` was overloaded across * three distinct situations a consumer must tell apart to gate correctly): * - `"suspended"` — the impl (or resumed impl) suspended on a durable HITL gate, so it isn't done yet; the * result is failed-with-token and the caller should resume (NOT treat as a verification failure). * - `"no_verdict"` — the verifier ran but never produced a structured verdict (a broken/flaky verifier); the * work IS done but could not be gated — treat as a gate failure, not a pass. * - `"could_not_verify"` — the verifier returned PARTIAL: it ran but could NOT verify due to an environmental * limit (e.g. no test framework). The work IS done but was NOT gated — a careless `verdict !== "FAIL"` * consumer must treat this as a non-verification, not a pass; `findings` carry what couldn't be checked. * - `"needs_review"` — the impl/verifier paused on a non-durable human review and isn't gated yet. * - `"impl_incomplete"` — the implementation itself didn't complete (failed/blocked/timeout) before verification. * - `"opted_out"` — verification was explicitly disabled (`runDeveloperTask({ verify: false })`); not gated by design. */ export type UnverifiedReason = "suspended" | "needs_review" | "no_verdict" | "opted_out" | "impl_incomplete" | "could_not_verify"; export interface VerificationOutcome { /** Final verdict. `"unverified"` = the work was not gated; see {@link unverifiedReason} for WHY (they differ). */ verdict: "PASS" | "FAIL" | "PARTIAL" | "unverified"; /** Set iff `verdict === "unverified"` — disambiguates the three unverified situations (search [46] BUG7). */ unverifiedReason?: UnverifiedReason; /** How many verification rounds ran. */ rounds: number; /** Findings from the FINAL verification (problems on FAIL, caveats on PARTIAL). */ findings: string[]; /** Evidence (commands + output) from the final verification, if the verifier supplied it. */ evidence?: string; /** * Total cost (micro-USD) of the VERIFIER run(s) across all rounds — the verification OVERHEAD, separate from * the implementation's own cost (which is the returned `TaskResult.stats`, as the verifier runs in its own * session). Mirrors `runWithTeacher`'s `teacherStats` work-vs-overhead split. Σ of each verifier run's * cost+nested. `result.stats.costMicroUsd + verification.verifierCost` is the EXACT operation total for the * common single-pass case; in the rarer multi-round fix case `result.stats` is the FINAL impl attempt's cost * (the returned `...current`) so an earlier failed attempt's impl cost is not separately surfaced. Omitted * (undefined) when no verifier ran (e.g. an impl that suspended/was opted out before verification). */ verifierCost?: number; } export interface VerificationResult extends TaskResult { /** The verification outcome. The task `result`/`status` is the implementation's; consult `verdict` for quality. */ verification: VerificationOutcome; } /** * Verify an already-**completed** implementation `result` behind the independent adversarial verifier, * looping fix→re-verify until PASS (or a round cap) — **without re-running the implementation**. This is the * L3 entry for the L2+L3 composition (design/54 §4): in a fan-out, a worker's module has already been * produced, so the orchestrator runs the mechanical L2 gate ({@link runExecGate}), computes the diff, then * calls this with `config.evidence = diff (+ L2 results)` to judge the finished work. The sibling of * {@link runWithVerification} (fresh run + verify) and {@link resumeWithVerification} (resume + verify) for * the "I already have the result, just verify it" case — all three share this gate so a task is judged * identically however it reached completion (design/45 §11 Q6 + design/51 P1-b). * * `specBase` carries the impl's tools/roles/model/limits/signal/keys (everything but `objective`/`sessionId`) * so the verifier inherits the right role map + read-only tool boundary; `objective` is the original task * objective the verifier needs as context. When `config.evidence` is set and no `config.verifierPrompt` * override is given, the verifier uses {@link STATIC_VERIFICATION_PROMPT} (read the diff/results; don't try * to execute — that was L2's job), avoiding the "PARTIAL on everything" failure in a read-only sandbox. */ export declare function verifyCompleted(runner: Runner, result: TaskResult, specBase: ResumeTaskConfig, objective: string, config: VerifyConfig): Promise; /** * Run an implementation task, then gate it behind an independent adversarial verifier, looping * fix→re-verify until PASS (or a round cap). Returns the implementation result plus the * {@link VerificationOutcome}. The caller decides WHEN to use this (explicit opt-in) — it always * verifies once invoked. If the impl **suspends on a durable HITL gate**, it is surfaced as * failed-with-token (verdict `unverified`); the caller approves and calls {@link resumeWithVerification} * to resume AND verify (design/51 P1-b: the durable + HITL + verify integration). */ export declare function runWithVerification(runner: Runner, implSpec: TaskSpec, config?: VerifyConfig): Promise; /** * Resume a durable-suspended implementation task (design/45 F4) **and** verify it on completion — the * durable + HITL + verify integration (design/51 P1-b). The mirror of {@link runWithVerification} for the * resume path: `runWithVerification` surfaces a HITL suspend as failed-with-token; once the human * adjudicates, the caller calls this with the `token` + `outcome`, and it resumes the implementation and — * **if it COMPLETES** — runs the identical adversarial verifier + fix loop. If the resumed run suspends * AGAIN (a later durable gate), it is surfaced as failed-with-token (`unverified`) for the caller to resume * once more. `objective` is the ORIGINAL task objective (the resume carries none of its own) — the verifier * needs it as context; pass the same objective the original `runWithVerification` ran with. * * 🔴 Freshness (threat BUG5, design/53 §2.B): if you pass `config.evidence` (a diff), recompute it from the * **post-resume** working tree — a worker can clean-report → suspend at a gated tool → plant a backdoor * after approval, so a pre-suspend diff would grade stale code. The verifier already verifies the current working * tree (verifierObjective), so dropping the stale diff closes the timing window. */ export declare function resumeWithVerification(runner: Runner, token: CheckpointToken, outcome: ResumeOutcome, taskConfig: ResumeTaskConfig, objective: string, config?: VerifyConfig): Promise; export interface DeveloperTaskConfig extends VerifyConfig { /** Run the verification gate. Default `true` (that's the point of developer mode). Set `false` for prompt-only. */ verify?: boolean; } /** * One-stop developer-mode convenience: applies {@link CODE_AGENT_PROMPT} (unless the task supplies its * own `systemPrompt`) and, by default, runs the {@link runWithVerification} gate. Equivalent to wiring * the building blocks by hand — use the building blocks directly when you want full control. * * Pair with a role map that gives a strong implementation model and cheaper helper/verifier models for * "auto model selection" (design/28 §3.2): `roles: { default: strong, subagent: cheap, verifier: strong }`. */ export declare function runDeveloperTask(runner: Runner, spec: TaskSpec, config?: DeveloperTaskConfig): Promise; //# sourceMappingURL=verify.d.ts.map