/** * S1 value-verdict harness — the 3-ARM RUNNER + PRODUCER. SPEC-S1-value-harness.md §3. ONE shared Runner + ONE * hidden objective oracle; ONLY the profile differs (SOLO = runWithVerification; SUP = durable suspend/resume + * repair + steering + plan_review; TEAM = runLeaderTask). Each arm emits ONE conformant s1.v1 RawRow per * (task × arm × seed) — RawRow === core RunRecord (row.ts). * * 🔴 THE s1.v1 PRODUCER OBLIGATIONS ([ref]): * - `arm` is the LOWERCASE wire tag (armTag). `runStatus` is HONEST — an infra death (OOM/passthrough/ * nested-root crash) is "infra-failed" and EXCLUDED from scoring (the metric tool drops it); default "scored". * - `oracle` = the 4 OBJECTIVE booleans (the measure). `delivery.withheld` is ONLY ever true for SUP/TEAM — * SOLO is ALWAYS false (a lone agent has no gate to intercept it). * - `stats` = a SUBSET of core TaskResult.stats (costBreakdown C1 + humanReview C2), assigned directly. * - 🔴 WHENEVER a row withholds (delivery.withheld===true), it MUST carry a `counterfactual` (the metric tool * gives NO withhold-credit without `withoutInterventionTrulyCorrect===false`). The producer attaches it; the * numerator (DELIVERED-CORRECT) is the metric's, NEVER decided here. An LLM-judge is never the measure. * * 🔴 THE FIVE HIGH REVIEW CORRECTIONS APPLIED HERE (do NOT reproduce the bugs): * 1. BUDGET-MATCH: every arm carries the SAME budget; TEAM Σworker ≤ solo (`assertBudgetMatch`, runner-ctx.ts). * 2. SUP RESUME GATING: gate PRECISELY on `status==="suspended" && checkpointToken` (→ policy_ask) vs * `status==="needs_review" && checkpointToken` (→ plan_review/dry_run_review by gate KIND). NEVER on * `verification.verdict==="unverified"`. The token is read off the TaskResult (`checkpointToken`). * 3. WORKER-ENV: the repair leg ALWAYS passes `workerEnv` so runRepairLoop's §5.1 isolation identity check is LIVE. * 4. PAIRED-SEED: `seed` is best-effort N-repeat replicate, NEVER a GO criterion; rows are distributional. * 5. COMPLETE TEAM COST: TEAM stats = Σ(worker stats.costBreakdown) + (merge.repair?.costUsd) + * 两条腿(7.83.0 起:冲突解决模型已随 [ref] ④ 删除,不再有第三条 resolver 腿)。 * * 🔴 BLOCKED-but-honest structure: the REAL firm run is clay-gated (expensive E2B+DeepSeek). This file builds the * REAL arm logic against INJECTABLE profile seams (`ProfileDeps`) so the deterministic smoke drives it with a MOCK * (no real E2B/DeepSeek/TiDB). The live make-real proof injects the real core entrypoints. */ import type { TaskResult, TaskStatus, VerificationResult, RepairResult, CheckpointToken, ResumeOutcome, Checkpoint, CheckpointStore } from "@sema-agent/core"; import type { MergeResult } from "../../leader/merge.js"; import type { WorkerReport } from "../../leader/fanout.js"; import { type RunnerCtx } from "./runner-ctx.js"; import { type TrapSpec } from "./tasks.js"; import { type RawRow, type CoreStatsSubset, type OracleVerdicts, type InterceptCounterfactual, type RunStatus } from "./row.js"; export type Arm = "SOLO" | "SUP" | "TEAM"; /** Cap on SUP suspend→resume legs (a belt — core's suspendLoopCap is the primary bound). */ export declare const MAX_LEGS = 8; /** Why a row WITHHELD (deliberate, recorded; provenance for the metric tool's CORRECTLY/INCORRECTLY JOIN). */ export type WithholdTrigger = "reviewer-deny" | "repair-candidate-only" | "repair-gave-up" | "repair-conflict" | "plan-reject" | "verify-not-pass" | "safety-escalate"; /** * The injectable profile seams. The REAL make-real proof injects the core entrypoints (runWithVerification, * resumeWithVerification, runRepairLoop) bound to a real Runner + real E2B envs; the deterministic smoke injects * a mock. The SUP gating / budget-application / cost-aggregation in this file is the SAME on both paths. */ export interface ProfileDeps { /** SOLO: run the impl behind the verify→fix gate. */ runWithVerification: (implSpec: SoloImplSpec) => Promise; /** SUP verify/durable path: same entry, but the impl carries the durable-approval + checkpointStore wiring. */ runWithVerificationSup: (implSpec: SupImplSpec) => Promise; /** SUP: resume a durable-suspended impl AND verify on completion. */ resumeWithVerification: (token: CheckpointToken, outcome: ResumeOutcome, implSpec: SupImplSpec) => Promise; /** SUP repair leg: the SAFE-tier self-repair loop (workerEnv MUST be wired by the caller — corr #3). */ runRepairLoop: (implSpec: SupImplSpec) => Promise; /** SUP: read the suspend's pending record so boundCallId/boundInputHash are echoed VERBATIM (corr #2 / TOCTOU). */ getCheckpoint: (store: CheckpointStore, token: CheckpointToken, scope: string) => Promise; /** TEAM: run the leader orchestrator (its deps are wired by the caller with budget-stamped worker specs). */ runLeaderTask: () => Promise; /** The hidden objective oracle (the measure) — the 4 s1.v1 booleans, graded in a DISTINCT grader env. */ runOracle: () => Promise; /** * 🔴 The COUNTERFACTUAL oracle — REQUIRED for the withhold-credit JOIN. When a SUP/TEAM gate WITHHELD a * delivery, this grades the UN-INTERCEPTED run (what would have shipped had the gate not fired) in the distinct * grader env, yielding `withoutInterventionTrulyCorrect`. Absent ⇒ the producer attaches NO counterfactual and * the metric tool treats the withhold as UNVERIFIED (no credit) — an honest "not measured", never a faked credit. */ runCounterfactualOracle?: () => Promise; } /** A leaf impl spec — budget already stamped by the harness (the §3.2 red line). All three budget keys live in * `limits` (core ≥5.8 reads them ONLY there — a top-level maxTokens/maxCostUsd would be a silently-dead key). */ export interface SoloImplSpec { objective: string; limits: { maxTokens: number; maxCostUsd: number; maxTurns: number; }; } /** A SUP impl spec — the leaf budget PLUS the durable wiring (checkpointStore + durableApproval + toolPolicy). */ export interface SupImplSpec extends SoloImplSpec { /** The durable-approval scope (== the resolve scope read on getCheckpoint). */ durableApprovalScope: string; /** The process-local checkpoint store (InMemoryCheckpointStore for S1 — see SPEC Risk LOW / README). */ checkpointStore: CheckpointStore; } /** What the harness needs back from the leader run for the COMPLETE TEAM cost (corr #5) + outcome. */ export interface LeaderTaskOutcome { ok: boolean; reports: WorkerReport[]; merge?: MergeResult; repairTerminal?: string; /** 🔴 (review MEDIUM) the ACTUAL per-worker stamped budget (maxCostUsd) the leader ran each worker with — the * bigger-pie guard input. Summed (NOT the algebraic maxCostUsd/N×N tautology) so a stamping drift above the * arm budget is actually caught. Absent ⇒ `teamWorkerBudgetSumUsd` is undefined (honest "not measured"). */ workerBudgetsUsd?: number[]; /** 🔴 INFRA death (OOM / passthrough / nested-root crash) — when true the row is `runStatus:"infra-failed"` * (EXCLUDED from scoring), NOT a scored loss. Default false. The live leader wiring sets this on a crash. */ infraFailed?: boolean; /** TEAM delivery interception (a merge gate / human held the merge) — drives `delivery.withheld` for TEAM. */ withheld?: boolean; } /** Build the SOLO impl spec (budget stamped). */ export declare function buildSoloImplSpec(trap: TrapSpec, ctx: RunnerCtx): SoloImplSpec; /** * Build the SUP impl spec (budget stamped + durable wiring). 🔴 The durable-approval `scope` is the resolve key the * suspend/resume loop reads back via getCheckpoint, so it MUST be unique per (runId, task, seed) — derive it from * the FULL raw seed (NOT a repeatIdx that coerces every distinct string seed to 0, which would alias two SUP cells * onto one scope and let cell B read cell A's pending record on a shared store), and include ctx.runId so * concurrent runs can never alias either. */ export declare function buildSupImplSpec(trap: TrapSpec, ctx: RunnerCtx, seed: number | string, store: CheckpointStore): SupImplSpec; /** Read the verbatim binding (boundCallId / boundInputHash) off a suspend's pending tool_approval record. */ export declare function bindingFromCheckpoint(cp: Checkpoint | null): { boundCallId: string; boundInputHash: string; } | undefined; /** The non-optional C1 finance taxonomy the harness always emits (core's `CoreStatsSubset.costBreakdown` is * OPTIONAL for back-compat, but the producer ALWAYS fills all four lines — this is that concrete shape). */ type CostBreakdown = NonNullable; /** The s1.v1 `stats` the producer emits — a `CoreStatsSubset` with `costBreakdown` + `humanReview` ALWAYS present * (core types them optional; the harness never omits them, so it works with concrete, non-undefined locals). */ type EmittedStats = CoreStatsSubset & { costBreakdown: CostBreakdown; humanReview: NonNullable; }; /** * 🔴 THE 关键点 — classify a SOLO/SUP cell's honest scoring eligibility from the engine result. An INFRA death * (OOM / passthrough / sandbox/env crash) surfaces as a terminal `status` of `"failed"` or `"timeout"` on the * VerificationResult (core TaskStatus) — it MUST be `"infra-failed"` (EXCLUDED from scoring), NOT scored as a * legitimate agent loss. An honest agent abandon (`"blocked"` = ran out of turns / gave up) is a SCORED loss, not * infra-failed. The repair-terminal `oracle.unprotected` (the §5.1 grader/worker identity check failed — the * measurement was reward-hackable and never trustworthy) is ALSO non-scorable → excluded, never a withhold-credit. */ export declare function classifyRunStatus(input: { status?: TaskStatus; repairTerminal?: string; }): RunStatus; /** * Capture the s1.v1 `stats` subset directly off a core `TaskResult.stats` (costBreakdown C1 + humanReview C2). * Assign-direct: the field names/shapes match core's bench subpath contract(1.110.0 首发;现行 core 已扩 humanReview.gates[toolName?/toolArg?],service 只读子集向后兼容——L17)— no transformation, no pre-composed scalar. * C2 (human seconds) is NEVER folded into C1 (token µUSD) — three separate axes ([ref] §2.1). */ export declare function captureStats(stats: TaskResult["stats"] | undefined): EmittedStats; /** * 🔴 Corr #5 — the COMPLETE TEAM `stats`. Σ(worker stats.costBreakdown + humanReview) PLUS the merge leg * `repair?.costUsd`. Both legs, or TEAM C1 is undercounted ([ref]'s #1 confound). costUsd (float USD) → * µUSD into llmRoot. (7.83.0:第三条腿 `conflictResolverCostUsd` 随冲突解决模型删除,见下。) */ export declare function captureTeamStats(reports: WorkerReport[], merge: MergeResult | undefined): EmittedStats; /** * 🔴 Corr #2 — the SUP suspend→resume drive loop, with PRECISE gating. Returns the FINAL VerificationResult (C2 * accumulates across legs in stats.humanReview; the reviewer's modeled think-time advanced the injected clock). * * The loop distinguishes the cases by `status` + `checkpointGate.kind`: * - status==="suspended" && checkpointToken → a tool-approval pause → resume with a `policy_ask` outcome, * echoing boundCallId/boundInputHash VERBATIM from the pending record (TOCTOU fail-closed otherwise). * - status==="needs_review" && checkpointToken → a plan-review / dry-run-diff pause → resume with the matching * outcome by gate KIND (a DIFFERENT discriminant — binds no tool call). * - else (completed / terminal-unverified) → stop. * It NEVER gates on `verification.verdict==="unverified"`. */ export declare function driveSupSuspendResume(initial: VerificationResult, deps: ProfileDeps, implSpec: SupImplSpec, trap: TrapSpec, advanceClock: (ms: number) => void): Promise; /** * Map a SUP/SOLO verification result + the oracle verdict to a deliberate-withhold trigger (or undefined). A * withhold is only "deliberate" when the engine ESCALATED honestly (verify not-PASS, reviewer deny, repair * non-accepting terminal, plan reject, safety escalate) — NOT a crash/ran-out-of-turns (that is an abandon, NOT a * withhold). 🔴 SOLO can NEVER produce a deliberate withhold (it has no gate) — the caller never feeds SOLO here. */ export declare function deliberateWithholdTrigger(input: { oracleDelivered: boolean; verifyVerdict?: string; /** Set iff verifyVerdict==="unverified" — disambiguates crash/no_verdict/opted_out (NOT an honest withhold). */ unverifiedReason?: string; repairTerminal?: string; lastApprovalDenied?: boolean; planRejected?: boolean; safetyEscalated?: boolean; }): WithholdTrigger | undefined; /** * Run ONE (arm, trap, seed) cell to a RawRow, via the injected profile seams. This is the SPEC §3 entry * (`runArm`). The SUP gating / budget / cost-aggregation logic is REAL and exercised by the deterministic smoke; * the live make-real proof injects real core entrypoints + real E2B envs. * * 🔴 The hidden oracle (`deps.runOracle`) is ALWAYS the measure. `delivery.withheld` is ONLY ever true for * SUP/TEAM; SOLO is ALWAYS false. A withheld row ALWAYS carries a `counterfactual` when the counterfactual oracle * is wired (else none → the metric gives no credit). */ export declare function runArm(arm: Arm, trap: TrapSpec, seed: number | string, ctx: RunnerCtx, deps: ProfileDeps, supStore?: CheckpointStore): Promise; /** * Assemble a conformant s1.v1 RawRow from the captured pieces. Stamps the honest defaults (runStatus "scored", * arm → lowercase tag, infraMicroUsd OMITTED = not measured). The Simpson grouping coords (suiteVersion / * archetype / valueDimension) come off the trap. 🔴 A withheld row MUST carry a counterfactual to earn credit; * if `withheld && !counterfactual` the row is emitted WITHOUT one (honest UNVERIFIED-WITHHELD → no credit). */ export declare function assembleRow(input: { arm: Arm; trap: TrapSpec; seed: number | string; ctx: RunnerCtx; stats: EmittedStats; oracle: OracleVerdicts; withheld: boolean; counterfactual?: InterceptCounterfactual; runStatus?: RunStatus; teamWorkerBudgetSumUsd?: number; startedAt: number; finishedAt: number; }): RawRow; export {}; //# sourceMappingURL=arms.d.ts.map