{
  "$schema": "https://anthropic.com/schemas/agent-routing-evals.v1.json",
  "version": "0.2.0",
  "description": "Routing-accuracy corpus for the adia-ui-forge in-repo agent roster (6 cards + README). For each user phrase, declares the expected agent that should activate (or null for phrases that route nowhere / off-topic). Scored by `scripts/skills/run-agent-evals.mjs --agents-dir packages/plugins/adia-ui-forge/agents`. Boris falsification test T1 (agent routing). First eval pass 2026-06-08.",
  "scope": "Forge agents: tech-lead, author, a2ui-maintenance-agent, package-release-agent, repo-steward, spec-architect, verifier.",
  "scoring_notes": "Same TF-IDF scorer as scripts/skills/run-skill-evals.mjs. Treat misroutes as signal to tighten the card's description field. MIN_SCORE_THRESHOLD 2.0 + MIN_MATCH_COUNT 2 required to activate.",
  "phrases": [
    {
      "id": "tech-lead-01",
      "phrase": "decompose this feature into layer-slices and assign them",
      "expected": null,
      "rationale": "Feature decomposition + assignment, tech-lead's orchestration role."
    },
    {
      "id": "tech-lead-02",
      "phrase": "gate the release on a green verify pass",
      "expected": null,
      "rationale": "Verify-gate sequencing, tech-lead's integration closure."
    },
    {
      "id": "tech-lead-03",
      "phrase": "sequence the cross-layer handoff for this release window",
      "expected": null,
      "rationale": "Multi-agent handoff sequencing is tech-lead's primary role."
    },
    {
      "id": "tech-lead-04",
      "phrase": "orchestrate the v0.8.0 release window across all layers",
      "expected": null,
      "rationale": "Orchestrate release window, tech-lead."
    },
    {
      "id": "tech-lead-05",
      "phrase": "I have 6 items for this sprint, who should own what?",
      "expected": null,
      "rationale": "Work decomposition + claim assignment, tech-lead."
    },
    {
      "id": "author-01",
      "phrase": "add a new web-component primitive for timeline-ui",
      "expected": "primitive-authoring-agent",
      "rationale": "In-tree primitive authoring, author's four-layer scope."
    },
    {
      "id": "author-02",
      "phrase": "modify the card-ui yaml to add a compact attribute",
      "expected": "primitive-authoring-agent",
      "rationale": "In-tree yaml edit, author."
    },
    {
      "id": "author-03",
      "phrase": "fix the four-axis contract violation on this component",
      "expected": "primitive-authoring-agent",
      "rationale": "Four-axis contract, author's authoring discipline."
    },
    {
      "id": "author-04",
      "phrase": "add a new token to the styles/ foundation bundle",
      "expected": "primitive-authoring-agent",
      "rationale": "Foundation token authoring, author."
    },
    {
      "id": "author-05",
      "phrase": "wire up a .contents.js behavior for the chat surface",
      "expected": "primitive-authoring-agent",
      "rationale": "Functional wiring (.contents.js) is author's scope."
    },
    {
      "id": "author-06",
      "phrase": "update the docs site sitemap for the new component page",
      "expected": "primitive-authoring-agent",
      "rationale": "Docs site authoring, author's fourth layer."
    },
    {
      "id": "author-07",
      "phrase": "promote inline CSS to a proper @layer module file",
      "expected": "primitive-authoring-agent",
      "rationale": "Styles refactor, author."
    },
    {
      "id": "a2ui-01",
      "phrase": "diagnose why zettel composition coverage dropped",
      "expected": "a2ui-maintenance-agent",
      "rationale": "Zettel engine + eval gap diagnosis, a2ui-maintenance-agent."
    },
    {
      "id": "a2ui-02",
      "phrase": "harvest a new training chunk for the corpus",
      "expected": "a2ui-maintenance-agent",
      "rationale": "Corpus authoring, a2ui-maintenance-agent."
    },
    {
      "id": "a2ui-03",
      "phrase": "tune the STRONG_MATCH threshold for free-form composition",
      "expected": "a2ui-maintenance-agent",
      "rationale": "Strategy calibration, a2ui-maintenance-agent."
    },
    {
      "id": "a2ui-04",
      "phrase": "add a new MCP tool to the a2ui-mcp server",
      "expected": "a2ui-maintenance-agent",
      "rationale": "MCP server authoring, a2ui-maintenance-agent."
    },
    {
      "id": "a2ui-05",
      "phrase": "the eval regression shows MRR dropped, investigate",
      "expected": "a2ui-maintenance-agent",
      "rationale": "Eval regression diagnosis, a2ui-maintenance-agent."
    },
    {
      "id": "a2ui-06",
      "phrase": "extract a fragment from this monolithic HTML pattern for the corpus",
      "expected": "a2ui-maintenance-agent",
      "rationale": "Fragment extraction into corpus, a2ui-maintenance-agent."
    },
    {
      "id": "release-01",
      "phrase": "cut a v0.7.17 lockstep release of all 9 packages",
      "expected": "package-release-agent",
      "rationale": "Lockstep cut, package-release-agent’s primary task. (The phrase’s \"all 9 packages\" is historically accurate for v0.7.17; the lockstep is 11 since 2026-07-15.)"
    },
    {
      "id": "release-02",
      "phrase": "publish the npm packages after the tag",
      "expected": "package-release-agent",
      "rationale": "npm publish, package-release-agent."
    },
    {
      "id": "release-03",
      "phrase": "batch push the accumulated v0.7.x releases",
      "expected": "package-release-agent",
      "rationale": "Batch push, package-release-agent."
    },
    {
      "id": "release-04",
      "phrase": "we shipped a version skip, need to recover",
      "expected": "package-release-agent",
      "rationale": "Version-skip recovery, package-release-agent."
    },
    {
      "id": "release-05",
      "phrase": "write the GitHub release notes for v0.7.16",
      "expected": "package-release-agent",
      "rationale": "Release notes authoring, package-release-agent."
    },
    {
      "id": "release-06",
      "phrase": "ship the adia-ui-factory plugin as a standalone version cut",
      "expected": "package-release-agent",
      "rationale": "Independent-version plugin ship, also package-release-agent."
    },
    {
      "id": "steward-01",
      "phrase": "audit the .brain folder for stale content",
      "expected": null,
      "rationale": "Repo-brain / memory-layer audit, repo-steward."
    },
    {
      "id": "steward-02",
      "phrase": "update the PLAN for the v0.8 milestone",
      "expected": null,
      "rationale": "PLAN/ROADMAP currency, repo-steward."
    },
    {
      "id": "steward-03",
      "phrase": "write the migration guide for the v0.7 → v0.8 breaking changes",
      "expected": null,
      "rationale": "Migration guide authoring, repo-steward (folds in migration-agent)."
    },
    {
      "id": "steward-04",
      "phrase": "harvest a memory entry from today's session",
      "expected": null,
      "rationale": "Memory-layer curation, repo-steward."
    },
    {
      "id": "steward-05",
      "phrase": "curate the AGENTS.md stewardship section after the rename",
      "expected": null,
      "rationale": "AGENTS.md / skill stewardship, repo-steward."
    },
    {
      "id": "steward-06",
      "phrase": "write a postmortem for the 2026-06-02 agent collision incident",
      "expected": null,
      "rationale": "Postmortems, repo-steward."
    },
    {
      "id": "architect-01",
      "phrase": "turn this PRD into a locked spec before authoring starts",
      "expected": null,
      "rationale": "PRD → spec, spec-architect's primary task."
    },
    {
      "id": "architect-02",
      "phrase": "author an ADR for the light-DOM substrate decision",
      "expected": null,
      "rationale": "ADR authoring, spec-architect."
    },
    {
      "id": "architect-03",
      "phrase": "write a design note with guardrails before we build this feature",
      "expected": null,
      "rationale": "Design note + guardrails, spec-architect."
    },
    {
      "id": "architect-04",
      "phrase": "validate this premise before any code is written",
      "expected": null,
      "rationale": "Premise validation (plan-before-execute), spec-architect."
    },
    {
      "id": "architect-05",
      "phrase": "I need a spec for the new canvas feature",
      "expected": null,
      "rationale": "Spec authoring, spec-architect."
    },
    {
      "id": "verifier-01",
      "phrase": "run the visual regression probes on the dogfood app",
      "expected": "demo-audit-agent",
      "rationale": "Visual + structural review, demo-audit-agent (gh#1733 split of the retired verifier/framework-reviewer role)."
    },
    {
      "id": "verifier-02",
      "phrase": "run a11y probes on the new admin-shell surface",
      "expected": "demo-audit-agent",
      "rationale": "A11y probing, demo-audit-agent (gh#1733 split of the retired verifier/framework-reviewer role)."
    },
    {
      "id": "verifier-03",
      "phrase": "set up a new gate script to enforce the card-ui body contract",
      "expected": "demo-audit-agent",
      "rationale": "Test substrate / gate scripts, demo-audit-agent (gh#1733 split of the retired verifier/framework-reviewer role)."
    },
    {
      "id": "verifier-04",
      "phrase": "the gen-UI review flagged visual drift, investigate",
      "expected": "gen-ui-review-agent",
      "rationale": "Gen-UI review + visual coherence, gen-ui-review-agent (gh#1733 split of the retired verifier/framework-reviewer role)."
    },
    {
      "id": "verifier-05",
      "phrase": "run the full pre-merge verification pass",
      "expected": "demo-audit-agent",
      "rationale": "Pre-merge verify pass, demo-audit-agent's general dogfood sweep (gh#1733 split of the retired verifier/framework-reviewer role)."
    },
    {
      "id": "null-01",
      "phrase": "what's the weather in Helsinki today?",
      "expected": null,
      "note": "Off-topic."
    },
    {
      "id": "null-02",
      "phrase": "explain how React useEffect hooks work",
      "expected": null,
      "note": "Generic programming, no forge agent applies."
    },
    {
      "id": "null-03",
      "phrase": "compose a settings page for a consumer app",
      "expected": null,
      "note": "Consumer composition, factory/screen-composition-agent, not a forge card."
    },
    {
      "id": "null-04",
      "phrase": "scaffold a new SPA host for the consumer's app",
      "expected": null,
      "note": "Consumer scaffolding, factory, not forge."
    },
    {
      "id": "ambig-01",
      "phrase": "verify the release before cutting",
      "expected": "demo-audit-agent",
      "ambiguous_between": [
        "demo-audit-agent",
        "package-release-agent"
      ],
      "note": "Both cards mention 'verify'. demo-audit-agent (gh#1733 split of the retired verifier/framework-reviewer role) owns the independent structural pass; package-release-agent runs pre-flight gates. Corpus labels demo-audit-agent for the structural-proof framing; package-release-agent for 'pre-flight' framing. This phrasing skews demo-audit-agent."
    },
    {
      "id": "ambig-02",
      "phrase": "write up what happened in the last cycle",
      "expected": "package-release-agent",
      "ambiguous_between": [
        "package-release-agent",
        "repo-steward"
      ],
      "note": "Post-cycle writeup defaults to release ledger (package-release-agent). If framed as an incident → repo-steward postmortem. Context determines the split."
    }
  ],
  "expected_baseline": {
    "as_of": "2026-06-08",
    "scorer_version": "0.1.0",
    "scorer_min_score_threshold": 2.0,
    "scorer_min_match_count": 2,
    "accuracy": 0.935,
    "per_agent_f1": {
      "a2ui-maintenance-agent": 1.0,
      "author": 0.93,
      "package-release-agent": 0.92,
      "repo-steward": 1.0,
      "spec-architect": 1.0,
      "tech-lead": 0.8,
      "verifier": 0.91,
      "__null__": 0.89
    },
    "notes": "First measured baseline 2026-06-08 (post-triggers vocabulary pass). Two iterations: (1) 56.5% pre-triggers; (2) 93.5% after adding triggers: fields to all 7 cards. Remaining 3 misroutes are INHERENT-SCORER-LIMITS documented below. null-03/null-04 consumer phrases correctly fire null (forge cards don't match consumer-domain vocabulary).",
    "strict_floor_accuracy": 0.9,
    "named_weakpoints": [
      "tech-lead-04 'orchestrate the v0.8.0 release window across all layers' → author (7.159) over tech-lead (6.753), margin 0.4 points. Author's description contains @layer which contributes 'layer' token; tech-lead's trigger block has 'release window' + 'orchestrate' but fewer matching tokens in this phrase. Near-tie inherent-scorer-limit; monitor in next pass.",
      "ambig-01 'verify the release before cutting' → tech-lead (3.962) ties verifier (3.962) exactly; alphabetic tiebreak selects tech-lead ('t' > 'v' alphabetically... wait, alphabetic ascending: tech-lead < verifier). Labeled verifier; inherent alphabetic-tie ambiguity.",
      "ambig-02 'write up what happened in the last cycle' → null (max score 1.981 < 2.0 threshold). Only content tokens are 'write' + 'happened'; 'happened' doesn't appear in any card's vocabulary. Inherent: vague retrospective phrasing. A real orchestrator would ask 'release ledger or incident postmortem?'."
    ]
  },
  "notes": "Re-keyed to the v2 4-agent roster 2026-07-11 (tech-lead/spec-architect/repo-steward retired, their phrases now expect null: the host loop owns orchestration/specs/hygiene)."
}