{ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "name": { "default": "", "description": "scenario identity; defaults to the filename (sans extension) if omitted — an explicit value overrides that and keys the run dir + cassette", "type": "string" }, "baseline": { "default": "latest", "description": "platform baseline to run against (auto-synced via `cowork-harness sync`)", "type": "string" }, "session": { "default": "(inline)", "description": "hand-authored session setup file (pre-prompt: model, mounts, discovery); defaults to an all-defaults inline session", "type": "string" }, "fidelity": { "default": "container", "description": "isolation tier: protocol (L0, no sandbox) | container/microvm (force a VM-loop tier) | hostloop (force host-loop) | cowork (auto-pick host-loop vs. container via Cowork's own gate logic). DEPRECATION: omitting this key is deprecated and the field becomes REQUIRED in the next major. The `container` default models the VM loop, while production runs the host loop by default (gate 1143815894), so an omitted key likely measures the scenario against a lane your users are not on — a bare relative path lands elsewhere, the shell starts elsewhere, and the offered tool set differs. Name a tier: hostloop to match production, cowork to auto-pick the way Cowork does, or container to keep the current behaviour deliberately.", "type": "string", "enum": [ "protocol", "container", "microvm", "hostloop", "cowork" ] }, "execution": { "default": "local", "description": "execution location axis, ORTHOGONAL to fidelity (a local privilege tier): local (default — run the agent locally) | cloud-describe (RESERVED — describe/annotate a cloud-run scenario without executing it; no runner exists yet, authoring it is a load-time error, not a silent no-op)", "type": "string", "enum": [ "local", "cloud-describe" ] }, "lane": { "default": "local", "description": "which Cowork lane's DELIVERY CONTRACT to hold the run to, orthogonal to fidelity (isolation tier) and execution (where the run happens): local (default) — a file under a user-visible root is delivered by LOCATION, and present_files is served | remote — location delivers NOTHING (verified: a remote container has no auto-delivering outputs dir), so only an explicit delivery counts, and present_files is NOT served because a local MCP server cannot reach a remote session. Scoped to delivery semantics: the remote device bridge (device_bash/device_commit_files) is deliberately unmodeled — see docs/fidelity-gaps.md", "type": "string", "enum": [ "local", "remote" ] }, "prompt": { "type": "string", "description": "the user turn sent to the agent" }, "timeout_ms": { "description": "wall-clock budget for the agent run; on expiry the harness kills the agent and the run ends result:error / errorSource:timeout. Omitted = no timeout (the agent runs to its own completion). Distinct from the max_turns assertion and agent_max_turns (turn budget).", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "answers": { "default": [], "description": "scripted answers, matched in order: AskUserQuestion gates by question-text regex, tool-permission gates by tool name (`when_tool`)", "type": "array", "items": { "type": "object", "properties": { "when_question": { "type": "string" }, "choose": { "anyOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ] }, "answer": { "type": "string" }, "when_tool": { "type": "string" }, "decide": { "type": "string", "enum": [ "allow", "deny" ] }, "allow_if": { "type": "string" }, "else": { "type": "string", "enum": [ "allow", "deny" ] }, "grant": { "type": "string", "enum": [ "once", "domain" ] } }, "additionalProperties": false } }, "on_unanswered": { "description": "policy when a gate arrives with no matching `answers:` rule — `fail` (default for `run`, deterministic) | `first` (pick the first offered option) | `llm` (delegate to a decider LLM) | `prompt` (interactive; valid only via `skill --on-unanswered prompt` or the adaptive-TTY default — rejected in scenario YAML under `run`)", "type": "string", "enum": [ "fail", "prompt", "llm", "first" ] }, "expect_denied": { "default": [], "description": "shorthand for asserting egress to these hosts was DENIED — expands to one egress_denied assertion per host", "type": "array", "items": { "type": "string" } }, "assert": { "default": [], "description": "post-run assertions; see each key's own description for what it checks", "type": "array", "items": { "type": "object", "properties": { "transcript_contains": { "description": "the transcript contains this literal substring. Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so text the agent emitted inside a tool call (an AskUserQuestion gate question, an option label or description) can never match at any phrasing", "type": "string", "minLength": 1 }, "transcript_not_contains": { "description": "the transcript does NOT contain this literal substring. Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so text the agent emitted inside a tool call (an AskUserQuestion gate question, an option label or description) can never match at any phrasing", "type": "string", "minLength": 1 }, "transcript_matches": { "description": "regex (case-insensitive) over the transcript — fuzzy content for stochastic prose. Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so text the agent emitted inside a tool call (an AskUserQuestion gate question, an option label or description) can never match at any phrasing", "type": "string" }, "transcript_not_matches": { "description": "regex (case-insensitive) that must NOT match the transcript. Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so text the agent emitted inside a tool call (an AskUserQuestion gate question, an option label or description) can never match at any phrasing", "type": "string" }, "tool_result_contains": { "description": "at least one tool result contains this literal substring (per-result match, not concatenated; 10 KB cap per result)", "type": "string", "minLength": 1 }, "tool_result_not_contains": { "description": "no tool result contains this literal substring (per-result match, not concatenated; 10 KB cap per result)", "type": "string", "minLength": 1 }, "tool_result_matches": { "description": "regex (case-insensitive) — at least one tool result matches (per-result, 10 KB cap). The regex sibling of tool_result_contains; use for an error-signature FAMILY a script may print even when its exit code was swallowed by its wrapper", "type": "string", "minLength": 1 }, "tool_result_not_matches": { "description": "regex (case-insensitive) that must NOT match any tool result (per-result, 10 KB cap). The regex sibling of tool_result_not_contains", "type": "string", "minLength": 1 }, "file_exists": { "description": "a file exists at this path under the agent's work root", "type": "string", "minLength": 1 }, "user_visible_artifact": { "description": "a file exists AND is under a user-visible prefix. Write the path workRoot-relative (e.g. `outputs/x.md`), NOT with an `mnt/` prefix: the accepted prefixes are `outputs/`, each connected-folder mount (`/`), or the legacy `.projects` fallback (pre-1.14271.0). (At fidelity tiers the workRoot is the `mnt/` mount, so the file lands at `mnt/outputs/…` on disk, but the assertion value is the relative form.)", "type": "string" }, "tool_called": { "description": "a called tool matched this glob (* = any run, ? = one char; exact when literal; anchored, case-sensitive) — e.g. mcp__workspace__*. Legacy tool spellings the agent binary canonicalizes (Task/Agent, KillShell/TaskStop, ...) match either way", "type": "string", "minLength": 1 }, "tool_not_called": { "description": "NO called tool matched this glob (* / ?; exact when literal; anchored, case-sensitive; legacy spellings match as in tool_called). A LITERAL naming a tool the tier does not serve (Bash/WebFetch/NotebookEdit at hostloop; mcp__workspace__bash at container/microvm) is refused at load — it could never be violated", "type": "string", "minLength": 1 }, "reference_read": { "description": "a skill references/ or scripts/ file matching this regex was ACCESSED (main agent or sub-agents) via Read, Grep, or a Bash command naming it — regex is unanchored + case-insensitive; fails when the run recorded no observable tool stream", "type": "string", "minLength": 1 }, "no_observed_reference_access": { "description": "NO observed access to a skill references/ or scripts/ file matching this regex. Named 'observed' because detection under-approximates (a cd then a bare relative cat, a heredoc, or a $VAR-built path is invisible) — it is not proof of absence; fails when no observable tool stream was recorded", "type": "string", "minLength": 1 }, "subagent_tool_used": { "description": "a sub-agent used a tool matching this glob (* / ?; exact when literal; anchored, case-sensitive)", "type": "string", "minLength": 1 }, "subagent_tool_absent": { "description": "NO sub-agent used a tool matching this glob (* / ?; exact when literal; anchored, case-sensitive)", "type": "string", "minLength": 1 }, "no_vm_path_file_op": { "description": "hostloop-only: NO gated file tool (Read/Write/Edit/Glob/Grep/MultiEdit) attempted a path that is exactly /sessions or /sessions/-prefixed — the production VM-path boundary. Only `true` is valid — omit to not require it. Any other tier FAILS (cannot verify: /sessions/... is valid there).", "type": "boolean", "const": true }, "vm_path_denied": { "description": "hostloop-only: at least one recorded path denial targeted a /sessions VM path (any source). Only `true` is valid. Needs controlOut on replay (else skipped-and-surfaced).", "type": "boolean", "const": true }, "path_denied": { "description": "hostloop-only: a path denial matching ALL given matchers was recorded", "type": "object", "properties": { "tool": { "description": "glob over the denied tool name", "type": "string", "minLength": 1 }, "path_matches": { "description": "regex over the denied path", "type": "string" }, "source": { "type": "string", "enum": [ "pretooluse", "can_use_tool", "permission_denied" ] }, "agent_scope": { "description": "subagent = the binary's agent_id attribution present; main = absent; default any", "type": "string", "enum": [ "main", "subagent", "any" ] } }, "additionalProperties": false }, "no_path_denied": { "description": "hostloop-only: NO path denial was recorded (the channel is path-scoped already, unlike no_hook_blocked). Only `true` is valid.", "type": "boolean", "const": true }, "subagent_file_write": { "description": "a SUB-AGENT-origin write attempt whose raw path EQUALS `path` (or ends with `path_suffix`) has a PAIRED non-error tool_result — the causal half of a delivery probe (pair with artifact_json for content). Prefer `path` (exact) so a foo/artifacts/probe.json can't satisfy an artifacts/probe.json suffix. Tier-agnostic.", "type": "object", "properties": { "path": { "description": "EXACT raw path the sub-agent's write must have sent (strongest — proves the exact path, not just a suffix)", "type": "string", "minLength": 1 }, "path_suffix": { "description": "the target path's suffix (weaker than `path`; e.g. artifacts/probe.json)", "type": "string", "minLength": 1 }, "tool": { "description": "glob over the writing tool; default matches Write/Edit/MultiEdit", "type": "string", "minLength": 1 } }, "additionalProperties": false }, "subagent_dispatch_healthy": { "description": "hostloop-only composite: ties ONE dispatch's resolved type to ITS OWN delivered write and path-cleanliness via parentToolUseId — the per-dispatch correlation `subagent_file_write` (which matches ANY sub-agent write) lacks. `type` matches the dispatch's resolvedAgentType OR dispatchAgentType OR description (a regex can narrow to one dispatch in a same-agent-type fleet). Content-class (fileToolAttempts + toolResults), replay-checkable without controlOut.", "type": "object", "properties": { "type": { "description": "regex over dispatchAgentType OR resolvedAgentType OR description, selecting the dispatch(es) to check — same matching as subagent_dispatched; omit to require EVERY dispatch to be healthy", "type": "string" }, "delivered": { "description": "default true: the selected dispatch's OWN sub-agent-origin write (matched by parentToolUseId, not any sub-agent's write) has a paired non-error tool_result", "type": "boolean" }, "path": { "description": "EXACT raw path the delivered write must have sent (strongest); narrows `delivered`", "type": "string", "minLength": 1 }, "path_suffix": { "description": "the delivered write's path suffix (weaker than `path`)", "type": "string", "minLength": 1 }, "no_vm_paths": { "description": "default true: the selected dispatch attempted NO `/sessions` VM path (its own parentToolUseId only)", "type": "boolean" } }, "additionalProperties": false }, "subagent_dispatched": { "description": "a sub-agent matching this regex (by dispatch or resolved agent type, or description) was dispatched", "type": "string" }, "subagent_declared_but_unused": { "description": "a sub-agent declared this tool but never used it (the fabrication proxy). Fires only on a dispatch that declares a tools/allowedTools list; the Agent tool carries neither, so declaredTools is [] and the key passes — a green means not-applicable, not absence of fabrication", "type": "string" }, "subagent_output_contains": { "description": "a dispatched sub-agent's own output contained this substring (optionally narrowed to dispatches matching `match`)", "type": "object", "properties": { "match": { "description": "regex over dispatchAgentType or description, narrowing to specific dispatch(es); omit to check all", "type": "string" }, "contains": { "type": "string", "description": "substring that must appear in the matched dispatch(es)' output" } }, "required": [ "contains" ], "additionalProperties": false }, "dispatch_count_max": { "description": "total sub-agent dispatches ≤ N — an author-chosen budget under Cowork's agent-side fan-out cap, not a reproduction of it (see SPEC §10)", "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "skill_triggered": { "description": "a skill matching this regex (by its invoked skill id, e.g. \"plugin:skill\") was invoked via the Skill tool", "type": "string" }, "no_skill_triggered": { "description": "no invoked skill id matched this regex — the negative-control / description-collision catcher", "type": "string" }, "skill_available": { "description": "a staged skill's id matched this regex (offered, not necessarily invoked — see skill_triggered for invocation)", "type": "string" }, "connector_available": { "description": "an MCP server/connector's name matched this regex (available, not necessarily used)", "type": "string" }, "tool_available": { "description": "a tool in the init manifest matched this regex (available, not necessarily called — see tool_called for invocation)", "type": "string" }, "max_cost_usd": { "description": "the run's SDK-reported total_cost_usd is ≤ N — live lane only; on replay this asserts the frozen recording's cost, not fresh spend", "type": "number", "exclusiveMinimum": 0 }, "max_tokens": { "description": "usage.input_tokens + usage.output_tokens ≤ N (cache-read/creation tokens excluded — priced separately) — live lane only; on replay this asserts the frozen recording's usage, not fresh spend", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "tool_calls_max": { "description": "total top-level tool calls (sum of toolCounts, sub-agent tools excluded) ≤ N", "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "tool_no_error": { "description": "no tool whose name matches this regex recorded any error (RunResult.toolErrors[name].errors === 0 for every match) — REQUIRES at least one matching tool call (fails if the regex matched nothing, so a typo can't silently pass; use tool_no_error_if_called for the presence-free variant)", "type": "string" }, "tool_no_error_if_called": { "description": "like tool_no_error, but PASSES VACUOUSLY when no tool matches the regex — the lenient, presence-free variant for a tool that may legitimately not run", "type": "string" }, "max_tool_errors": { "description": "total tool errors across all tools (sum of RunResult.toolErrors[*].errors) ≤ N", "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "max_redundant_tool_calls": { "description": "total WASTED repeated tool calls (sum of (count-1) across every redundant {name,args} group in RunResult.redundantToolCalls) ≤ N — not the raw count of redundant groups", "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "skill_tool_used": { "description": "a tool matching `tool` ran inside a skill-activation window whose skillId matches `skill` — heuristic for inline skills (a sticky, sequential window faithfully matching the real agent's activeSkill scope, not an exact per-tool boundary; see RunResult.skillActivity's doc comment). SCOPE: the window's tool counts INCLUDE calls made by any sub-agent dispatched during it, so this key cannot distinguish a main-agent call from a sub-agent one (use subagent_tool_used for a sub-agent-only claim), and it matches tool NAMES only — never the path/arguments a tool was called with", "type": "object", "properties": { "skill": { "type": "string", "description": "regex matched against a skill-activation window's skillId" }, "tool": { "type": "string", "description": "regex matched against a tool name in that window's toolCounts" } }, "required": [ "skill", "tool" ], "additionalProperties": false }, "all_tasks_completed": { "description": "every task in RunResult.tasks[] reached status \"completed\" — REQUIRES at least one task (a run with zero tasks fails: it cannot have \"completed them all\"); only `true` is valid", "type": "boolean", "const": true }, "task_count_min": { "description": "at least N tasks were created (RunResult.tasks.length >= N) — the presence companion for task assertions", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "task_status": { "description": "a task whose subject or id matches `match` reached `status`", "type": "object", "properties": { "match": { "type": "string", "description": "regex matched against a task's subject OR id" }, "status": { "type": "string", "description": "the status the matching task must have reached" } }, "required": [ "match", "status" ], "additionalProperties": false }, "max_turns": { "description": "the SDK-reported (or fallback-counted) turn count ≤ N — replay-checkable (the re-drive recounts turns deterministically)", "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "max_peak_rss_bytes": { "description": "peak sampled RSS of the agent sandbox <= N bytes — live-only (container/hostloop/microvm); evidence-unavailable on replay/protocol or when sampling captured no RSS", "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 }, "compaction_occurred": { "description": "a context-compaction boundary occurred during the run (a `compact_boundary` system event was recorded); only `true` is valid — omit to not require it", "type": "boolean", "const": true }, "no_mcp_error": { "description": "no MCP round-trip failed (RunResult.mcpErrors is empty) — live-only (excluded on replay); only `true` is valid", "type": "boolean", "const": true }, "hook_blocked": { "description": "a PreToolUse hook blocked a tool whose name matches this regex (RunResult.hookEvents) — replay needs controlOut", "type": "string" }, "no_hook_blocked": { "description": "no tool was hook-blocked during the run (distinguishes a real tool crash from an intentional block) — replay needs controlOut; only `true` is valid", "type": "boolean", "const": true }, "no_scratchpad_leak": { "description": "every file presented via present_files that was in the scratchpad was successfully promoted to outputs (none left behind); vacuous pass if nothing was presented — pair with a presence check to require a delivery; content-class (re-derived from the tool_use/tool_result stream, so checkable on replay too); CONTAINER TIER ONLY — present_files itself is now served at BOTH container and hostloop, but this key's promotion/leak semantics apply only at container: production's own host-loop branch validates a path and passes it through WITHOUT promoting (the agent's cwd there already IS the outputs dir), so there is no scratch→outputs copy to leak, and at hostloop this key is cannot-verify rather than a claim the tool is absent (use fidelity: container for present_files-based delivery; microvm still doesn't serve the tool at all). present_files is the DESKTOP-LOCAL lane's tool; remote Cowork delivers via the agent-native SendUserFile instead, so a skill should describe the delivery outcome rather than naming either tool (docs/fidelity-gaps.md, 'File delivery'); only `true` is valid", "type": "boolean", "const": true }, "present_files_called": { "description": "at least one file was actually delivered via the present_files tool (at least one call carried a well-formed file_path). Presence is read from the INVOCATION count, not from the classified presentedFiles list, so it is unaffected by a redaction policy that rewrites host paths; a run that called the tool but whose every call carried an unusable path reports cannot-verify, never 'the tool was never called'. The presence companion to no_scratchpad_leak (which passes vacuously when nothing was presented, and stays container-only) — pair them to require a delivery AND require it not to leak; CONTAINER + HOSTLOOP TIERS — the harness serves present_files at both, mirroring real Cowork advertising the tool in both its VM and host-loop modes; every other tier is still a harness coverage gap (see docs/fidelity-gaps.md, 'File delivery'). present_files is the DESKTOP-LOCAL lane's tool name; remote Cowork uses the agent-native SendUserFile (docs/fidelity-gaps.md, 'File delivery') — this key asserts the harness-side delivery record either way; only `true` is valid", "type": "boolean", "const": true }, "egress_denied": { "description": "egress to this host was denied", "type": "string" }, "egress_allowed": { "description": "egress to this host was allowed", "type": "string" }, "no_delete_in_outputs": { "description": "fails if a delete touching mnt/outputs is DETECTED (post-run bash-command scan, not mount-level enforcement — a green means none was detected); only `true` is valid (writing `false` is a rejected footgun). Omitting the key does NOT allow deletes — a detected delete fails via the outputs_delete signal; use allow_outputs_delete to accept one", "type": "boolean", "const": true }, "no_unexpected_files": { "description": "fails if the run CREATED a file under a user-visible root whose workRoot-relative path (e.g. outputs/x.md) matches none of these globs (** = whole path segment for any depth, * within a segment, ? one char); [] = no new files allowed; new-files-only — overwriting a pre-existing file in place is invisible (use content-level producer stamping); needs a pre-run manifest (harness ≥0.24 recordings) — absence fails loud on live/verify-run; captured on every live sandbox tier including microvm (its outputs are snapshotted from the VM into the run dir), except a --resume run (no fresh manifest ⇒ fails loud)", "type": "array", "items": { "type": "string", "minLength": 1 } }, "file_absent": { "description": "the named path does NOT exist under the work root after the run — the negative-existence check no other key expresses (no_unexpected_files is new-files-only and needs a pre-run manifest, so it cannot say 'X must not exist'). LIVE/verify-run only: absence is provable only where the walk was authoritative, and a cassette records no walk health. Fails evidence-unavailable on `lane: remote` and on a pre-run origin of `remote-unavailable` — a filesystem that is not locally observable makes a missing snapshot indistinguishable from absence", "type": "string", "minLength": 1 }, "artifact_text": { "description": "assert over a delivered artifact's TEXT body — the companion to artifact_json for non-JSON deliverables, and the only way to check that an internal path/name did not leak into a file a user receives. At least one matcher is required. A body captured body-less (uploaded input, read-only folder input, over the size cap) or recorded as a symlink fails evidence-unavailable, and for the NEGATIVE matchers a body that is not lossless UTF-8 does too — a binary body read as text would 'pass' against bytes it never saw", "type": "object", "properties": { "artifact": { "type": "string", "minLength": 1, "description": "relative path to an artifact under the work root (e.g. outputs/report.json) — a literal path, not a glob" }, "contains": { "description": "every listed substring appears in the body", "minItems": 1, "type": "array", "items": { "type": "string", "minLength": 1 } }, "not_contains": { "description": "no listed substring appears in the body", "minItems": 1, "type": "array", "items": { "type": "string", "minLength": 1 } }, "matches": { "description": "the body matches this regex", "type": "string", "minLength": 1 }, "not_matches": { "description": "the body does not match this regex", "type": "string", "minLength": 1 } }, "required": [ "artifact" ], "additionalProperties": false }, "input_unmodified": { "description": "a single glob OR an array of globs; every pre-existing file whose workRoot-relative path matches has an unchanged content hash after the run (in-place mutation detector)", "anyOf": [ { "type": "string", "minLength": 1 }, { "minItems": 1, "type": "array", "items": { "type": "string", "minLength": 1 } } ] }, "self_heal_ran": { "description": "skill resolved scripts via /sessions (plugin-root self-heal)", "type": "boolean" }, "no_lost_write_back": { "description": "fails if the run authored an interactive HTML artifact (or a .py/.js generator of one) whose relative Submit/POST write-back is lost under Cowork — runs the static Tier A analyzer over the files the run authored; a lost write-back on an ADDED agent-authored source fails, a pre-existing file the skill merely modified on a read-write mount is advisory; could-not-verify (fail-closed) on a --resume scratchpad or an unanalyzable candidate; runs on every live sandbox tier including microvm (outputs snapshotted from the VM); only `true` is valid (omit to skip). LIVE/verify-run only — skipped on replay", "type": "boolean", "const": true }, "transcript_no_host_path": { "description": "fails if a host path (/Users, /opt) leaked into model-visible text (post-run scan); only `true` is valid (writing `false` is a rejected footgun — omit to allow or use allow_stall)", "type": "boolean", "const": true }, "computer_links_resolve": { "description": "fails if any computer:// link in the model-visible transcript does not resolve to an artifact that exists in the run's collected outputs/mounts — REQUIRES at least one link (zero links FAILS: use computer_links_resolve_if_present for the presence-free variant); only `true` is valid (writing `false` is a rejected footgun — omit to skip). Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so a computer:// link that appeared only inside a tool call or its result is invisible here", "type": "boolean", "const": true }, "computer_links_resolve_if_present": { "description": "like computer_links_resolve, but PASSES VACUOUSLY when the transcript has zero computer:// links — the lenient, presence-free variant; only `true` is valid. Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so a computer:// link that appeared only inside a tool call or its result is invisible here", "type": "boolean", "const": true }, "question_asked": { "description": "a question matching this regex was asked. This text is model-composed and is reworded run to run — pin a producer-authored constant, not model prose", "type": "string" }, "question_options": { "description": "assert the option SET and ORDER a gate offered the user, by LABEL (question_asked matches question text only; option DESCRIPTIONS are not compared here — use question_context for those). Exactly one of equals|contains is required. Evidence is captured at ask time, so it covers a gate that was shown and then denied/stalled/unanswered; a run whose gate evidence is absent fails evidence-unavailable, never vacuously. This text is model-composed and is reworded run to run — pin a producer-authored constant, not model prose", "type": "object", "properties": { "when_question": { "description": "regex selecting the sub-question by its label (`question`, falling back to `header` — the same string question_asked matches); omit only when the run fired exactly one sub-question", "type": "string" }, "equals": { "description": "the offered option labels, as a complete set — in this exact ORDER unless `order: any`", "type": "array", "items": { "type": "string" } }, "contains": { "description": "these option labels are present (others may be too); in this relative order unless `order: any`", "type": "array", "items": { "type": "string" } }, "order": { "description": "`exact` (default) compares order as well as membership — an option list re-ordered by the model is the defect this key exists for; `any` compares membership only", "type": "string", "enum": [ "exact", "any" ] } }, "additionalProperties": false }, "question_context": { "description": "a regex matched against everything a gate put in front of the user: the question label, every option LABEL, and every option DESCRIPTION. Use this when the skill's own wording may land in any of those fields — question_asked sees only the question text and question_options compares only labels, so a sentence delivered in an option's `description` is invisible to both. Evidence is the ask-time AskUserQuestion payload (never a producer's tool_result, which would grade true whether or not the model surfaced anything). Zero gates recorded FAILS; a lane that cannot read the gate payload fails evidence-unavailable, never vacuously. This text is model-composed and is reworded run to run — pin a producer-authored constant, not model prose", "type": "object", "properties": { "when_question": { "description": "regex narrowing to sub-questions whose label matches (the same string question_asked matches); omit to search EVERY gate — unlike question_options, omitting it is not ambiguous here, because this key asks whether the text was shown at all, not which gate offered which set", "type": "string" }, "matches": { "type": "string", "minLength": 1, "description": "regex that must match somewhere in the selected gate(s)' founder-visible payload (case-insensitive). NON-EMPTY: an empty pattern compiles to //i and matches every field of every gate, so it would green any run that fired one — a required field whose empty value asserts nothing is worse than an absent one" } }, "required": [ "matches" ], "additionalProperties": false }, "questions_count_max": { "description": "at most N sub-questions asked — a bundled AskUserQuestion with K sub-questions counts as K, not 1", "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "gate_answers_delivered": { "description": "accepts a boolean. `true`: every answered AskUserQuestion gate's tool_result was non-error (the answer reached the model); zero gates fired passes vacuously — pair with gate_answer_count_min to also require a gate. `false` is the inverse — it asserts a CONFIRMED delivery failure (at least one gate whose delivered === false), for negative-path scenarios; an unobserved (delivered === null) delivery satisfies neither true nor false", "type": "boolean" }, "gate_answer_count_min": { "description": "at least N AskUserQuestion gates fired AND were delivered non-error (presence companion to gate_answers_delivered)", "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "result": { "description": "the run's final result was success | error", "type": "string", "enum": [ "success", "error" ] }, "allow_permissive_auto_allow": { "description": "(verdict modifier) suppress the default-fail when the run recorded a cowork-parity permissive auto-allow — for tests that deliberately assert Cowork's permissive behavior", "type": "boolean", "const": true }, "allow_outputs_delete": { "description": "(verdict modifier) accept a detected outputs delete for this scenario instead of failing the run — for a skill whose deletion is intended. WAIVES the harness's post-hoc detection; it does NOT model production's allow_cowork_file_delete approval handshake, so a skill relying on the live EPERM still behaves differently here. Mutually exclusive with no_delete_in_outputs", "type": "boolean", "const": true }, "no_delete_in_mounts": { "description": "fails if a delete is DETECTED in any delete-denied mount (outputs + every `rw` connected folder) that is not waived by allow_delete_in — post-run bash-command scan, not mount-level enforcement, so a green means none was detected; only `true` is valid. Production denies unlink/rmdir on every such mount until per-mount approval", "type": "boolean", "const": true }, "allow_delete_in": { "description": "(verdict modifier) accept detected deletes in these mounts by NAME (e.g. [\"reports\"]) instead of failing/warning — the per-mount analogue of allow_outputs_delete, mirroring production's per-mount fileDeleteApprovedMounts. WAIVES the harness's post-hoc detection (which still runs and is still recorded); it does NOT model the live approval handshake. Listing \"outputs\" conflicts with no_delete_in_outputs", "minItems": 1, "type": "array", "items": { "type": "string", "minLength": 1 } }, "allow_l0_plugin_divergence": { "description": "(verdict modifier) suppress the default-fail when L0 (protocol) runs with plugins that load via --settings/managed config instead of --plugin-dir — for tests that deliberately test at L0 with plugins", "type": "boolean", "const": true }, "allow_missing_capability": { "description": "(verdict modifier) suppress the default-fail when the (partial 'core') agent image omits a capability the skill used but real Cowork ships — assert this only when the skill's fallback is genuinely equivalent (otherwise rebuild full parity, --build-arg COWORK_FULL_PARITY=1)", "type": "boolean", "const": true }, "allow_stall": { "description": "(verdict modifier) suppress the default-fail when a run ends on a question having done no productive tool work after its last gate (the agent asked for input and stopped — incl. re-asking in plain text after answering an AskUserQuestion) — assert this only when ending on a question is the intended terminal state; otherwise script the answer (answer:/--answer/decider)", "type": "boolean", "const": true }, "allow_undelivered_deliverables": { "description": "(verdict modifier) suppress the `undelivered_deliverables` WARN for this scenario — assert it when the skill legitimately leaves working files behind that were never meant to reach the user (intermediates, caches, downloaded inputs). The signal is warn-only and never fails a run on its own; this exists so a scenario whose scratch activity is intentional can say so instead of carrying permanent noise. ALSO suppresses the sibling `delivery_unobservable` WARN on `lane: remote` (where delivery cannot be measured at all because no remote delivery tool is modeled) — on that lane this key means 'I know delivery is unverifiable here and accept it', NOT 'the files were delivered'", "type": "boolean", "const": true }, "replay_protocol_fidelity": { "description": "(replay-only, NOT authorable) serializeDecision output matched the frozen recording — the token-free re-serialization guard; synthesized by the replay lane and rejected if written in a scenario — listed here only so schema-driven editors can display it in read-only contexts; authoring it in a scenario is a load-time error, see src/run/execute.ts", "type": "boolean" }, "artifact_json": { "description": "assert over a JSON artifact's contents (dotted path + equals|in|gt|exists|absent|is_null)", "type": "object", "properties": { "artifact": { "type": "string", "minLength": 1, "description": "relative path to a JSON artifact under the work root (e.g. outputs/cap_state.json)" }, "path": { "description": "dotted path into the JSON (e.g. me.run_id); omit to target the whole document (an explicit empty string is rejected)", "type": "string", "minLength": 1 }, "equals": { "description": "the resolved value deep-equals this" }, "in": { "description": "the resolved value deep-equals one of these (stable for stochastic/LLM-extracted values where equals churns)", "type": "array", "items": {} }, "gt": { "description": "the resolved value is a number greater than this", "type": "number" }, "exists": { "description": "the path resolves to a present (non-absent) value", "type": "boolean" }, "absent": { "description": "the final key is absent from its (resolved) parent — the anti-hallucination negative", "type": "boolean" }, "is_null": { "description": "the resolved value is JSON null (distinct from absent)", "type": "boolean" } }, "required": [ "artifact" ], "additionalProperties": false }, "semantic_matches": { "description": "LIVE-ONLY: a pinned LLM judge grades the rubric against the run's answer; skipped-loud on replay (like egress_*). The judged document is finalMessage + transcript + authored files. NOTE the transcript is TOP-LEVEL assistant_text ONLY — it excludes every tool_use/tool_result, and no sub-agent text (even fork-scoped) unless include_subagent_text is set. A rubric claim about whether a TOOL was called can therefore never grade true; use tool_called/present_files_called/subagent_dispatched for that", "type": "object", "properties": { "rubric": { "minItems": 1, "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "fixed, authored checkable claims — a pinned judge grades each; results align by INDEX (not re-extracted per rep)" }, "min_pass": { "description": "how many rubric claims must pass for the assert to pass (default: all; do NOT rely on all for a gating scenario)", "anyOf": [ { "type": "string", "const": "all" }, { "type": "integer", "exclusiveMinimum": 0, "maximum": 9007199254740991 } ] }, "judge_model": { "description": "override the run-level pinned judge model for this assert", "type": "string" }, "include_subagent_text": { "description": "default false: also send each sub-agent's TEXT turns (RunResult.subagents[].reasoning, kind:'text' only) to the judge. Opt-in because it enlarges the judged document, which can re-grade an existing rubric. Use for a fan-out skill whose real work happens in sub-agents — their text is otherwise invisible to the judge. Sub-agent THINKING is excluded: it arrives empty with redacted:true, so including it would pad the document with blanks", "type": "boolean" } }, "required": [ "rubric" ], "additionalProperties": false } }, "additionalProperties": false } }, "skills": { "default": [], "description": "named `skills/` dirs this scenario exercises, scoping the cassette-staleness hash to just those (+ shared plugin roots); omitted/empty hashes the whole mounted tree", "type": "array", "items": { "type": "string" } }, "requires_capabilities": { "default": [], "description": "capability families (e.g. office_convert, ocr, pdf_tables) this scenario's core path needs; the run hard-fails if the tier omits or can't verify one, unless `allow_missing_capability: true` is set on the relevant assertion", "type": "array", "items": { "type": "string" } }, "allow_host_writes": { "description": "required consent for `fidelity: hostloop` with a writable connected folder (mode rw/rwd) — the agent gets real, software-checked-only host filesystem access there, no container sandbox; read-only/folder-less hostloop runs need no opt-in", "type": "boolean" } }, "required": [ "prompt" ], "additionalProperties": false, "not": { "anyOf": [ { "allOf": [ { "type": "object", "required": [ "assert" ], "properties": { "assert": { "type": "array", "contains": { "type": "object", "required": [ "no_delete_in_outputs" ], "properties": { "no_delete_in_outputs": {} } } } } }, { "type": "object", "required": [ "assert" ], "properties": { "assert": { "type": "array", "contains": { "type": "object", "required": [ "allow_outputs_delete" ], "properties": { "allow_outputs_delete": {} } } } } } ] }, { "allOf": [ { "type": "object", "required": [ "assert" ], "properties": { "assert": { "type": "array", "contains": { "type": "object", "required": [ "no_delete_in_outputs" ], "properties": { "no_delete_in_outputs": {} } } } } }, { "type": "object", "required": [ "assert" ], "properties": { "assert": { "type": "array", "contains": { "type": "object", "required": [ "allow_delete_in" ], "properties": { "allow_delete_in": { "type": "array", "contains": { "const": "outputs" } } } } } } } ] } ] }, "description": "cowork-harness scenario YAML — prompt + scripted answers + assert:. STRUCTURAL validation only, plus the mutually-exclusive delete-assertion rules mirrored as a top-level `not`. The loader and the runner enforce cross-field rules this schema cannot express, so a file that validates here can still be refused: an `answers:` entry with no matcher is rejected at load, and delivery-shaped assertion keys are rejected on `lane: remote`. Neither tool alone covers those — run `cowork-harness lint ` for the authoring checks AND `cowork-harness record --dry-run` for the real loader. See docs/scenario.md." }