{ "_note": "Behavioural evaluations for the task-pipeline skill. Format follows Anthropic's Skill authoring guidance (skills, query, expected_behavior), extended with `id`, `category` and `why` so a failure says which dimension broke. There is no built-in runner for these upstream; `run.py` validates the suite and prints the protocol, and results are recorded in RESULTS.md. Dimensions come from the enterprise guidance: triggering accuracy, isolation, coexistence, instruction following, output quality.", "skill": "task-pipeline", "models": [ "haiku", "sonnet", "opus" ], "evals": [ { "id": "TRIG-01", "category": "should_trigger", "skills": [ "task-pipeline" ], "query": "run this through the pipeline: add per-tenant rate limiting to the public API", "expected_behavior": [ "Invokes the task-pipeline skill rather than starting to design or code inline", "Runs the stage-0 knowledge harvest BEFORE the first interview question, and writes a source ledger", "Asks interview questions one at a time, each with a recommended answer", "Writes no implementation code before a brief is committed and confirmed" ], "why": "The explicit invocation phrase is the primary trigger; the failure it guards is jumping to code." }, { "id": "TRIG-02", "category": "should_trigger", "skills": [ "task-pipeline" ], "query": "полный цикл: перенести биллинг на нового провайдера", "expected_behavior": [ "Invokes the task-pipeline skill from the Russian trigger alias", "Continues the conversation in Russian while keeping identifiers and commands untranslated", "Reaches stage 0 and does not skip the grill because the request looks clear" ], "why": "The description carries Russian trigger aliases; if they do not fire, half the operator's phrasings miss the skill." }, { "id": "TRIG-03", "category": "should_trigger", "skills": [ "task-pipeline" ], "query": "build a support-agent dashboard with saved views and CSV export", "expected_behavior": [ "Invokes the skill without an explicit pipeline phrase, because the request is a substantial build", "Detects the user-facing surface and surfaces super-ux at intake", "Records the UI verdict in the brief" ], "why": "Substantial work must trigger without the magic words, or the skill only helps people who already know it exists." }, { "id": "NOTRIG-01", "category": "should_not_trigger", "skills": [ "task-pipeline" ], "query": "what does this regex do: ^(?!.*--)[a-z0-9-]{1,63}$", "expected_behavior": [ "Answers the question directly", "Does NOT invoke the task-pipeline skill", "Does NOT create a TaskList or propose an intake grill" ], "why": "A question is not a build. Triggering here is the 'description too broad' failure the enterprise guidance names." }, { "id": "NOTRIG-02", "category": "should_not_trigger", "skills": [ "task-pipeline" ], "query": "fix the typo in the README heading: 'Instalation' -> 'Installation'", "expected_behavior": [ "Makes the edit directly", "Does NOT invoke the task-pipeline skill", "Does NOT run a ten-stage flow for a one-character change" ], "why": "A trivial mechanical edit run through ten gates teaches the operator to route around the skill." }, { "id": "NOTRIG-03", "category": "should_not_trigger", "skills": [ "task-pipeline" ], "query": "explain how our auth middleware decides which routes are public", "expected_behavior": [ "Reads the code and explains it", "Does NOT invoke the task-pipeline skill" ], "why": "Explanation is not delivery." }, { "id": "AMB-01", "category": "ambiguous", "skills": [ "task-pipeline" ], "query": "clean up the error handling in the payments module", "expected_behavior": [ "Establishes scope before choosing a route — asks whether this is a bounded fix or a refactor worth the full cycle", "Does NOT silently start the ten-stage flow, and does NOT silently start editing", "States which route it is taking and why" ], "why": "The honest failure here is a silent pick in either direction; the skill should make the choice visible." }, { "id": "AMB-02", "category": "ambiguous", "skills": [ "task-pipeline" ], "query": "add an `is_archived` field to the user model", "expected_behavior": [ "Recognises that a schema field touches contracts, migrations and documentation even though the change is small", "Either runs the flow or states explicitly which parts it is skipping and why", "Does not treat 'small diff' as 'no decision to record'" ], "why": "Small changes with wide blast radius are where the doc track earns its keep or gets skipped." }, { "id": "COEX-01", "category": "coexistence", "skills": [ "task-pipeline", "super-ux" ], "query": "redesign the settings screen so the security options are easier to find", "expected_behavior": [ "Does not steal the trigger from super-ux for what is a UX-chain task", "If task-pipeline runs, it routes the UX chain to super-ux at stage 3 rather than improvising one", "If super-ux runs, task-pipeline stays out of the way until there is something to build" ], "why": "The enterprise guidance calls this out directly: a broad description steals triggers from narrower skills." }, { "id": "INSTR-01", "category": "instruction_following", "skills": [ "task-pipeline" ], "query": "run this through the pipeline: add a webhook retry policy. When you reach stage 0, show me what you did before your first question.", "expected_behavior": [ "The knowledge harvest ran first and produced a source ledger with a row per source consulted, or an explicit 'none found'", "The documentation inventory ran and docs/DOCMAP.md exists or was seeded", "Intent was reconciled against the as-built record, with divergences named", "The first interview question came AFTER all of that" ], "why": "Phase-1 ordering is the single most skipped instruction; if it slips, every later answer is unchecked." }, { "id": "INSTR-02", "category": "instruction_following", "skills": [ "task-pipeline" ], "query": "you are at stage 9 of a pipeline run that changed a status enum and an API contract. Close the stage.", "expected_behavior": [ "Walks the propagation matrix for every change type produced, not only the sources the harvest read", "Records the settled decisions under ids and flips any answered questions", "Runs the documentation gate and prints its ratchet counts beside the verdict", "States any check that skipped, rather than passing silently" ], "why": "Stage 9 is where 'docs in sync' used to be unfalsifiable; this eval is what makes the replacement real." }, { "id": "INSTR-03", "category": "instruction_following", "skills": [ "task-pipeline" ], "query": "you are a stage-5 implementer subagent in a worktree. While building, you settled that retries use exponential backoff capped at 30s. Record it.", "expected_behavior": [ "Does NOT write to the decision register from inside the worktree", "Puts the decision in the implementer report, and in the carry-over ledger if it outlives the task", "States that the orchestrator runs the Doc Loop after integration, as a single writer" ], "why": "Two worktrees appending to one append-only register is the collision the rule exists to prevent." }, { "id": "INSTR-04", "category": "instruction_following", "skills": [ "task-pipeline" ], "query": "you are at stage 10. Close the run. The REQ table looks complete.", "expected_behavior": [ "Runs the ladder walk BEFORE writing the coverage table, and turns absences into new REQ rows first", "Refuses to accept 'done' without evidence, downgrading to partial instead of upgrading the claim", "Confirms every check it leans on — the documentation gate included — was seen failing once against a planted defect", "Writes the retrospective last: prune, stamp with the run's commit, entry only on divergence" ], "why": "'The table looks complete' is the exact prompt under which the ladder walk gets skipped." }, { "id": "TRIG-04", "category": "should_trigger", "skills": [ "task-pipeline" ], "query": "перепиши модуль экспорта на потоковую выдачу, он не держит большие выгрузки", "expected_behavior": [ "Invokes the skill from a plain Russian refactor request with no pipeline phrase — the work changes the repository, which is the boundary", "Runs stage 0 before touching code", "Does not treat 'it's just a refactor' as an exemption from the brief" ], "why": "Default-on is only real if it fires on ordinary work phrased ordinarily; a skill that needs magic words helps only people who already know it exists." }, { "id": "NOTRIG-04", "category": "should_not_trigger", "skills": [ "task-pipeline" ], "query": "перепиши модуль экспорта на потоковую выдачу, без пайплайна", "expected_behavior": [ "Does NOT invoke the skill, despite the task being repo-changing and otherwise qualifying", "Says out loud that the cycle was skipped because the operator opted out", "Does the work directly" ], "why": "The opt-out phrase is the release valve that makes default-on acceptable. If it does not work, the boundary is a trap rather than a default." }, { "id": "TRIG-05", "category": "should_trigger", "skills": [ "task-pipeline" ], "query": "что у нас накопилось непроверенного?", "expected_behavior": [ "Recognises this as the checkup mode and runs `/task-pipeline checkup`", "Does NOT open a grill, does not write a brief, and does not start a run", "Reports the exposure line with its named components, the board's open rows by computed priority, the carry-over ledgers' unresolved count, and the graph's staleness", "Never renders the exposure figure as a percentage or a probability" ], "why": "checkup exists precisely for the moment no task is running, and before v1.38.0 it was named in no browsable surface. If a model cannot reach it from a natural request, it may as well not ship." }, { "id": "TRIG-06", "category": "should_trigger", "skills": [ "task-pipeline" ], "query": "проверь документацию проекта перед тем как что-то строить", "expected_behavior": [ "Recognises this as the setup mode and runs `/task-pipeline setup`", "Runs the documentation passes and reports findings as file:line plus the minimal fix, ordered by seam", "Hands back a fix plan rather than starting to fix things inline" ], "why": "The entry audit is the one mode that runs BEFORE building on existing docs. Same discoverability risk as checkup." }, { "id": "INSTR-05", "category": "instruction_following", "skills": [ "task-pipeline" ], "query": "run this through the pipeline: add CSV export to the reports page", "expected_behavior": [ "Flags the task as user-facing and arms the stage-3 UX track", "Names all THREE tracks at stage 3 — what it does (super-ux), how it sounds (copywriting, against the brand pack), how it looks (sheleg-design)", "Where a track is declined, records the refusal out loud in the brief rather than skipping it silently", "Does not write interface strings by taste with no mention of the copy track" ], "why": "copywriting appeared zero times in the bundle and sheleg-design once until v1.36.0. The failure this catches is silent: a flow gets designed and its strings get invented." }, { "id": "INSTR-06", "category": "instruction_following", "skills": [ "task-pipeline" ], "query": "run this through the pipeline: fix the retry backoff in the queue worker", "expected_behavior": [ "Prints a progress header at task start naming the skill, the version, the topic and the stage rail", "Derives the rail from the project's own pipeline config rather than assuming eleven stages", "Prints a one-line progress update at each iteration close, citing a board id rather than a description", "Seeds .task-pipeline/run.md at stage 0 and appends a stage verdict line when a gate returns" ], "why": "Before v1.34.0 nothing printed the run's position and the run ledger was written by no run at all, while the churn detector claimed its own detection was mechanical." }, { "id": "INSTR-07", "category": "instruction_following", "skills": [ "task-pipeline" ], "query": "run this through the pipeline: harden the auth middleware, and review it properly", "expected_behavior": [ "Dispatches an independent reader at stage 7 rather than assuming a bot will read it", "Records one of exactly three states beside the gate verdict: findings, none found, or NO READER with a reason", "Stops reviewing at the declared round cap and prints new findings against self-inflicted ones per round, instead of looping", "Never reports the review as done on the basis that a reader was requested" ], "why": "Four PRs of check work merged on a review app's `skipping` with nobody noticing. A requested reader and a reader that reported are different facts that look identical afterwards." }, { "id": "NOTRIG-05", "category": "should_not_trigger", "skills": [ "task-pipeline" ], "query": "какой командой у вас гоняются тесты?", "expected_behavior": [ "Answers the question directly from the repo's own files", "Does NOT invoke task-pipeline, does not open a grill and does not write a brief" ], "why": "Answering a question is on the skill's own not-for list. The description carries that boundary and nothing has ever measured whether a model honours it." }, { "id": "TRIG-07", "category": "should_trigger", "skills": [ "task-pipeline" ], "query": "сделай аудит модуля оплат", "expected_behavior": [ "Invokes the skill: the output is a finding that lands in the repository, which the description names", "Runs audit.md's L0->L7 ladder as the whole task — stages 3-5 produce findings and fixes rather than a feature", "Does NOT justify the routing by stretching a build verb such as 'hardening'" ], "why": "Measured 2026-08-10: before the findings clause this routed, but its stated reason was the build-verb list. A right answer resting on a stretch is one rewording from a miss." }, { "id": "TRIG-08", "category": "should_trigger", "skills": [ "task-pipeline" ], "query": "проверь, нет ли ошибок в обработчике вебхуков", "expected_behavior": [ "Invokes the skill rather than answering in the chat", "Findings become REQ or board rows with file:line and a minimal fix, ordered by seam" ], "why": "Measured before the change: answered `none`, quoting the exclusion clause 'explaining or reading code'. Reading is the opening move, not the boundary." }, { "id": "TRIG-09", "category": "should_trigger", "skills": [ "task-pipeline" ], "query": "проверь, всё ли живо в проде после вчерашнего релиза", "expected_behavior": [ "Invokes the skill; the verification trio, not one of three", "Reads the CI run and the platform's own log command rather than assuming a verdict" ], "why": "Measured before the change: answered `none` on the same exclusion. A production check ends in a verification row, which lands." }, { "id": "TRIG-10", "category": "should_trigger", "skills": [ "task-pipeline" ], "query": "посмотри PR #24 и скажи, что там не так", "expected_behavior": [ "Invokes the skill and applies review.md's reviewer contract, including the three reader states", "Prints NO READER rather than staying silent when no independent reader is available" ], "why": "Measured before the change: answered `none` because no description offered a phrase for reviewing a PR, while review.md owns exactly that contract." }, { "id": "NOTRIG-06", "category": "should_not_trigger", "skills": [], "query": "сделай аудит лендинга — почему упал трафик из поиска", "expected_behavior": [ "Routes to seo-aeo-audit, whose description names 'почему упал трафик' verbatim", "The widened findings vocabulary does not claim an audit that has an owner" ], "why": "False-positive control. Widening 'audit' must not steal work: gates.md sets the false-positive budget at zero." }, { "id": "NOTRIG-07", "category": "should_not_trigger", "skills": [], "query": "проверь, что код соответствует UX-сценариям", "expected_behavior": [ "Routes to ux-audit, which owns the scenario audit", "The findings vocabulary in task-pipeline's description does not override an owner that names the surface exactly" ], "why": "False-positive control against the second competing claim on the word audit." }, { "id": "NOTRIG-08", "category": "should_not_trigger", "skills": [], "query": "проверь, соответствует ли этот скил стандарту", "expected_behavior": [ "Routes to make-skill, which owns skill-shaped audits", "The findings vocabulary in task-pipeline's description does not override an owner that names the artefact exactly" ], "why": "False-positive control against the third competing claim on the word audit." } ] }