{ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "critique-report.json", "title": "cowork-harness critique --output-format json report (and critique-report.json run-dir artifact)", "description": "EXPERIMENTAL, descriptive schema — NOT a SPEC §12-frozen surface: additive field changes are expected between minor versions (unlike doctor.json). It exists so automation consumers (budget pacers, harvesters) have authoritative field names/shapes instead of prose. Exactly one of infraFailure / evaluatorError / evaluatorModel-with-findings describes the outcome; items is [] on both failure branches.", "type": "object", "required": ["skillFolder", "prompt", "sessionId", "outDir", "fidelity", "selfReportStatus", "verdictProvenance", "items"], "additionalProperties": false, "properties": { "skillFolder": { "type": "string", "description": "the positional folder BOTH turns mounted (a plain skill, or a plugin root)" }, "prompt": { "type": "string" }, "sessionId": { "type": "string" }, "outDir": { "type": "string", "description": "the kept run dir (turns/1 = graded, turns/2 = reflection; critique-* artifacts at its root)" }, "fidelity": { "type": "string", "description": "the tier critique pinned for both turns; always concrete (--fidelity cowork is resolved at parse time, never forwarded as-is)", "enum": ["container", "hostloop"] }, "requestedFidelity": { "type": "string", "description": "what the caller asked for when it differs from what ran — present only for --fidelity cowork, naming the tier it resolved to in `fidelity`", "enum": ["cowork"] }, "gradedEffectiveFidelity": { "type": "string", "description": "best-effort: the tier the graded turn's own result.json RECORDS (should equal fidelity; both surfaced so a mismatch is visible)" }, "gradedBaseline": { "type": "string", "description": "best-effort: the graded turn's fingerprint.baseline (Desktop appVersion)" }, "costUsd": { "type": "object", "description": "per-critique cost across the FOUR model workloads. GATE ON `complete` BEFORE TRUSTING totalUsd: false means one or more workloads were unpriced and the total UNDERCOUNTS true spend.", "required": ["totalUsd", "complete"], "additionalProperties": false, "properties": { "taskTurnUsd": { "type": "number" }, "reflectionTurnUsd": { "type": "number" }, "evaluatorPass1Usd": { "type": "number" }, "evaluatorPass2Usd": { "type": "number" }, "totalUsd": { "type": "number" }, "complete": { "type": "boolean" }, "evaluatorPass1Tokens": { "type": "object", "description": "token split for one evaluator pass. Answers 'is this pass's money evidence or thinking?' — the question that decides whether sending more evidence is cheap. cacheRead is separate because it prices at a tenth of fresh input", "properties": { "input": { "type": "number" }, "output": { "type": "number" }, "cacheRead": { "type": "number" } }, "required": ["input", "output", "cacheRead"], "additionalProperties": false }, "evaluatorPass2Tokens": { "type": "object", "description": "token split for one evaluator pass. Answers 'is this pass's money evidence or thinking?' — the question that decides whether sending more evidence is cheap. cacheRead is separate because it prices at a tenth of fresh input", "properties": { "input": { "type": "number" }, "output": { "type": "number" }, "cacheRead": { "type": "number" } }, "required": ["input", "output", "cacheRead"], "additionalProperties": false } } }, "gradedSkill": { "type": "string", "description": "the resolved skills/ the packager graded (--skill or single-skill auto-selection); ABSENT for a plain skill folder. Multi-skill-plugin pairing key: skillHash keys the whole mounted plugin, so pair by (gradedSkillHash, gradedSkill), never skillHash alone" }, "skillInvocationObserved": { "type": "boolean", "description": "advisory: whether the graded run's own skillActivity mentions gradedSkill; false = the critique may be grading a run that never invoked the selected skill" }, "gateAnswers": { "type": "array", "description": "the graded run's resolved gate answers, for a deterministic follow-up run", "items": { "type": "object", "required": ["question", "answer", "answeredBy"], "additionalProperties": false, "properties": { "question": { "type": "string" }, "answer": { "type": "string" }, "answeredBy": { "type": "string" } } } }, "taskResult": { "type": "string", "description": "the graded turn's own result — error is a GRADEABLE outcome, not an instrument failure", "enum": ["success", "error"] }, "gradedOutcome": { "type": "string", "description": "the graded turn's result.json outcome (e.g. delivered_clean)" }, "gradedSkillHash": { "type": "string", "description": "content-exact hash of the MOUNTED folder — the cross-FIX pairing key (per-plugin under --skill)" }, "selfReportStatus": { "type": "string", "description": "unavailable = pass 2 was skipped; findings are pass-1-only", "enum": ["captured", "unavailable"] }, "evaluatorIntegrity": { "type": "object", "description": "mechanical canary check per pass; false = that pass ignored a trusted instruction — an empty critique may be adversarial silencing, not a clean skill", "required": ["pass1Canary"], "additionalProperties": false, "properties": { "pass1Canary": { "type": "boolean" }, "pass2Canary": { "type": "boolean" } } }, "droppedEvaluatorItems": { "type": "object", "description": "malformed items the per-item-tolerant parse dropped; non-zero means the surviving findings under-represent the full reply", "required": ["pass1"], "additionalProperties": false, "properties": { "pass1": { "type": "number" }, "pass2": { "type": "number" } } }, "turn1ResultDegraded": { "type": "boolean", "description": "the graded turn's canonical result was corrupted/never archived — result-derived sections are empty DEFAULTS, treat as unknown" }, "turn1SliceDegraded": { "type": "boolean", "description": "the turn-1 transcript slice's boundary could not be trusted — treat gaps as unknown" }, "skillMdStatus": { "type": "string", "description": "readability of the packaged SKILL.md; missing/unreadable force the mechanical already-covered -> not-adjudicable downgrade. untracked = present on the host but NOT delivered by staging (git-tracked files only), so it was never in the agent's mount and is not valid evidence", "enum": ["readable", "missing", "unreadable", "untracked"] }, "evidenceBudget": { "type": "object", "description": "what the evaluator was actually shown. Skill-authored content (SKILL.md + references + agents md) ships WHOLE; these fields exist so a consumer never has to read compiled source to learn the budgets — the previous per-file caps were discoverable only by inspecting dist/, and let 11 of 13 reference files go permanently ungraded with no signal", "properties": { "corpusBytes": { "type": "number", "description": "total bytes of skill-authored content found, BEFORE any ceiling cut" }, "corpusCeiling": { "type": "number", "description": "the sanity ceiling governing SKILL.md + references + agents md COMBINED" }, "corpusCuts": { "type": "array", "description": "per-file record of what the ceiling cut. EMPTY on every real skill. omitted:true means the file's share fell below the minimum useful slice — a different instruction to the author (split this file) than a partial cut (the corpus as a whole is too big)", "items": { "type": "object", "properties": { "name": { "type": "string" }, "keptBytes": { "type": "number" }, "totalBytes": { "type": "number" }, "omitted": { "type": "boolean" } }, "required": ["name", "keptBytes", "totalBytes", "omitted"], "additionalProperties": false } }, "corpusExcluded": { "type": "array", "description": "skill files present on the HOST but excluded because staging would not deliver them (untracked, git-mode on). They were never in the agent's mount, so grading against them would manufacture false already-covered verdicts — but the author must be told, or the grade silently covers less than they believe", "items": { "type": "string" } }, "trimRecord": { "type": "array", "description": "which sections the overall belt-and-suspenders trim shaved and by how much. Replaces a bare boolean that made transcript loss undetectable after the fact", "items": { "type": "object", "properties": { "section": { "type": "string" }, "droppedBytes": { "type": "number" } }, "required": ["section", "droppedBytes"], "additionalProperties": false } }, "packageTruncated": { "type": "boolean", "description": "ANY section was cut — most often the transcript's 128 KiB head+tail elision, the cut that actually happens on long runs. This is what adds the evaluator's truncation caveat, so a consumer reading DROPPED findings or a rise in not-adjudicable needs it; corpusCuts is empty in that case and would otherwise imply nothing was cut" } }, "required": ["corpusBytes", "corpusCeiling", "corpusCuts", "corpusExcluded", "trimRecord", "packageTruncated"], "additionalProperties": false }, "verdictProvenance": { "type": "object", "description": "advisory scoping: the verdict is a self-run discovery lead, never an independent attestation", "required": ["kind", "advisory", "caveat"], "additionalProperties": false, "properties": { "kind": { "type": "string", "enum": ["self-run"] }, "advisory": { "type": "boolean" }, "caveat": { "type": "string" } } }, "infraFailure": { "type": "string", "description": "instrument failure (task turn killed / reflection protocol broke) — no critique was produced; items is []; process exits 2" }, "evaluatorError": { "type": "string", "description": "the evaluator threw (embeds the raw reply) — no critique was produced; items is []; process exits 2; see critique-salvage.json" }, "evaluatorModel": { "type": "string", "description": "the transport-RESOLVED evaluator model (never the requested alias); present only when the evaluator completed" }, "items": { "type": "array", "description": "the citation-validated findings from both passes. citationResolved:false = the cited excerpt did not resolve verbatim against the evidence package (DROPPED section — transparency only, do not act on as-is)", "items": { "type": "object", "required": ["source", "idea", "classification", "evidence", "recommendedAction"], "additionalProperties": false, "properties": { "source": { "type": "string", "enum": ["evaluator", "self-report"] }, "idea": { "type": "string" }, "classification": { "type": "string", "enum": ["grounded-and-actionable", "grounded-but-not-worth-it", "confabulated", "already-covered", "not-adjudicable"] }, "evidence": { "type": "string", "description": "the model's cited excerpt — verbatim-checked against the evidence package" }, "recommendedAction": { "type": "string" }, "citationResolved": { "type": "boolean" }, "findingFingerprint": { "type": "string", "description": "sha256/16 over normalized idea+classification+recommendedAction (evidence excluded). Cross-INPUT clustering key — HIGH-PRECISION, LOW-RECALL: idea is model-authored free text, so a MATCH proves the same finding recurred; a MISMATCH does NOT prove non-reproduction (the same finding reworded fingerprints differently)" } } } }, "noSkillFilesRead": { "type": "boolean", "description": "the graded turn ACCESSED no references/ or scripts/ file at all — neither the main agent nor any sub-agent, through any observed tool channel (Read, Grep, or a Bash command naming the path). OBSERVATIONAL: the predicate matches references/+scripts/ only (never assets/, never SKILL.md), and detection UNDER-APPROXIMATES — a `cd` into the skill dir then a bare relative `cat`, a heredoc body, and a $VAR-built path are invisible — so `true` is weak evidence of non-use, never proof the content went unread. Absent when the turn-1 result was degraded, when the skill ships nothing to read, or when the run recorded no observable tool stream (unknown, not zero)." }, "gradedModels": { "type": "array", "items": { "type": "string" }, "description": "the model ids the GRADED (task) turn actually ran on, read back from its own result.json `models` and filtered of the agent's `<…>`-wrapped locally-fabricated entries. The turns are a SUBPROCESS and inherit no model from the caller, so an omitted --model grades whatever the spawned agent defaults to — this is the only field that records which model produced the graded behaviour. Absent when no result.json was readable or it recorded no live id." }, "infraFailurePhase": { "type": "string", "enum": ["task turn", "reflection turn"], "description": "WHICH turn produced `infraFailure` — `task turn` (the graded run) or `reflection turn` (critique's own protocol turn). Required alongside infraFailure (see the `dependencies` keyword): without it the text report falls back to a default phase and can name the wrong turn." }, "infraFailureKind": { "type": "string", "description": "the failed turn's own classification of WHY it failed. Either a harness ErrCategory (unanswered | usage | boundary | runtime | internal) when it printed a `fail()` error envelope, or a RunResult.resultErrorKind (usage_limit | transport | agent) when it RAN and reported an errored result (exit 1, top-level error null). Absent = killed, or exited with no envelope at all. DO NOT read a present kind as \"the instrument is fine\": cli.ts's top-level catch funnels every unexpected throw (Docker down, container start failure, missing staged agent, a harness bug) into `internal`, and LegacyRunDirError into `runtime`. Ordinary/actionable = unanswered, usage, boundary, usage_limit, transport; every other value — including one added after this text — renders as an instrument failure, which is how the text header treats it." }, "gradedErrorReason": { "type": "string", "description": "why a graded turn that ended in result:\"error\" errored — its `resultErrorKind` (usage_limit / transport / agent) plus the finer termination source. An errored task turn is a GRADEABLE outcome and the critique still runs, so this is what separates a skill defect from an exhausted quota or a dropped connection. Absent when the run did not error or recorded no classification." } }, "dependencies": { "infraFailure": ["infraFailurePhase"] } }