{ "schema_version": "spec-rule-miner-trigger-evals/v1", "skill": "spec-rule-miner", "source_refs": [ "skills/spec-rule-miner/SKILL.md", "skills/spec-rule-miner/references/pattern-categories.md", "skills/spec-rule-miner/references/write-targets.md" ], "source_ref_authority": "source", "coverage_tags": [ "trigger", "boundary", "near-neighbor" ], "cases": [ { "id": "mine-project-rules-agents", "case_id": "SPEC-RULE-MINER-TRIGGER-001", "case_type": "should-trigger", "input": "分析这个仓库的编码风格，生成 AGENTS.md，让后续 AI 写代码像这个项目。", "prompt": "分析这个仓库的编码风格，生成 AGENTS.md，让后续 AI 写代码像这个项目。", "coverage_tags": ["trigger"], "expected_outcome": "should-trigger", "expected_result": "should-trigger", "expected_mode": "mine-rules", "reason_code": "project-rule-mining" }, { "id": "generate-cursor-inline-rules-from-code", "case_id": "SPEC-RULE-MINER-TRIGGER-002", "case_type": "should-trigger", "input": "Generate Cursor inline rules in .cursor/rules/project-rules.mdc from the existing code conventions in this repo.", "prompt": "Generate Cursor inline rules in .cursor/rules/project-rules.mdc from the existing code conventions in this repo.", "coverage_tags": ["trigger", "inline-target"], "expected_outcome": "should-trigger", "expected_result": "should-trigger", "expected_mode": "mine-rules", "reason_code": "tool-rule-file-request" }, { "id": "team-standards-governance-neighbor", "case_id": "SPEC-RULE-MINER-NEAR-001", "case_type": "near-neighbor", "input": "把这条团队规范提升为 confirmed standard，并更新 docs/standards/index.md。", "prompt": "把这条团队规范提升为 confirmed standard，并更新 docs/standards/index.md。", "coverage_tags": ["near-neighbor", "boundary"], "expected_outcome": "near-neighbor", "expected_result": "near-neighbor", "expected_mode": "spec-team-standards-governance", "boundary_note": "Team standards promotion is governance work, not project rule mining.", "reason_code": "standards-governance-not-rule-mining" }, { "id": "code-review-neighbor", "case_id": "SPEC-RULE-MINER-NEAR-002", "case_type": "near-neighbor", "input": "审查这个 PR 是否符合项目代码风格，并指出具体问题。", "prompt": "审查这个 PR 是否符合项目代码风格，并指出具体问题。", "coverage_tags": ["near-neighbor", "boundary"], "expected_outcome": "near-neighbor", "expected_result": "near-neighbor", "expected_mode": "spec-code-review", "boundary_note": "Diff review with findings belongs to code review; spec-rule-miner writes future-facing project rules.", "reason_code": "review-not-rule-mining" }, { "id": "generic-best-practices-negative", "case_id": "SPEC-RULE-MINER-NEG-001", "case_type": "should-not-trigger", "input": "给我一份通用 TypeScript 最佳实践清单。", "prompt": "给我一份通用 TypeScript 最佳实践清单。", "coverage_tags": ["boundary"], "expected_outcome": "should-not-trigger", "expected_result": "should-not-trigger", "expected_mode": "direct-answer-or-doc-task", "boundary_note": "Generic best practices are not mined from current repo evidence.", "reason_code": "generic-best-practices" }, { "id": "unsupported-tool-rule-file-negative", "case_id": "SPEC-RULE-MINER-NEG-002", "case_type": "should-not-trigger", "input": "Generate GitHub Copilot instructions from this repo's coding conventions.", "prompt": "Generate GitHub Copilot instructions from this repo's coding conventions.", "coverage_tags": ["boundary", "unsupported-tool"], "expected_outcome": "should-not-trigger", "expected_result": "should-not-trigger", "expected_mode": "unsupported-tool-target", "boundary_note": "spec-rule-miner only writes rule targets for programming tools currently supported by spec-first.", "forbidden_signals": ["write .github/copilot-instructions.md", "treat unsupported tool as built-in target"], "reason_code": "unsupported-tool-rule-file" }, { "id": "unsupported-legacy-cursorrules-negative", "case_id": "SPEC-RULE-MINER-NEG-003", "case_type": "should-not-trigger", "input": "Generate .cursorrules from this repo's coding conventions.", "prompt": "Generate .cursorrules from this repo's coding conventions.", "coverage_tags": ["boundary", "unsupported-tool"], "expected_outcome": "should-not-trigger", "expected_result": "should-not-trigger", "expected_mode": "unsupported-tool-target", "boundary_note": "Legacy .cursorrules is not a current spec-first supported rule target.", "forbidden_signals": ["write .cursorrules", "treat legacy .cursorrules as built-in target"], "reason_code": "unsupported-legacy-cursorrules" }, { "id": "unsupported-kiro-steering-negative", "case_id": "SPEC-RULE-MINER-NEG-004", "case_type": "should-not-trigger", "input": "Write the mined rules to .kiro/steering/project-rules.md with inclusion: always.", "prompt": "Write the mined rules to .kiro/steering/project-rules.md with inclusion: always.", "coverage_tags": ["boundary", "unsupported-tool", "kiro-steering"], "expected_outcome": "should-not-trigger", "expected_result": "should-not-trigger", "expected_mode": "unsupported-tool-target", "boundary_note": "Current Kiro support consumes root AGENTS.md and does not generate .kiro/steering rules.", "forbidden_signals": ["write .kiro/steering", "set inclusion: always"], "reason_code": "unsupported-kiro-steering" }, { "id": "multi-package-scope-boundary", "case_id": "SPEC-RULE-MINER-EXPECTED-001", "case_type": "expected", "input": "Mine project rules for a workspace with 75 package.json files: Vue2 apps, a few Vue3 apps, Nuxt SSR packages, Koa services, route registry files in some apps, old constantRouterMap pages, and a few legacy pages importing axios directly.", "prompt": "Mine project rules for a workspace with 75 package.json files: Vue2 apps, a few Vue3 apps, Nuxt SSR packages, Koa services, route registry files in some apps, old constantRouterMap pages, and a few legacy pages importing axios directly.", "coverage_tags": ["expected", "scope", "multi-package", "output-quality"], "expected_outcome": "scoped-rules-with-limitations", "expected_result": "scoped-rules-with-limitations", "expected_mode": "write-scoped-rules-with-limitations", "boundary_note": "Large mixed workspaces require package-level scope, sampled limitations, and historical-exception wording instead of workspace-wide absolute rules.", "required_signals": [ "package-level scope", "follow the current package structure first", "do not expand historical exceptions", "preview limitations include sampled packages" ], "forbidden_signals": [ "write workspace-wide absolute rule", "claim every page only uses @/api", "omit package-level scope", "turn legacy axios pages into a global ban" ], "reason_code": "multi-package-scope-boundary" }, { "id": "runtime-mirror-boundary", "case_id": "SPEC-RULE-MINER-BOUNDARY-001", "case_type": "boundary", "input": "把挖出来的规则直接写进 .agents/skills/using-spec-first/SKILL.md，让 Codex runtime 生效。", "prompt": "把挖出来的规则直接写进 .agents/skills/using-spec-first/SKILL.md，让 Codex runtime 生效。", "coverage_tags": ["boundary", "generated-runtime"], "expected_outcome": "boundary", "expected_result": "boundary", "expected_mode": "refuse-runtime-mirror-patch", "boundary_note": "Generated runtime mirrors are not spec-rule-miner targets.", "forbidden_signals": ["target .agents/skills", "treat generated runtime mirror as rule target"], "reason_code": "generated-runtime-not-rule-target" }, { "id": "headless-default-write-boundary", "case_id": "SPEC-RULE-MINER-WRITE-001", "case_type": "boundary", "input": "我暂时不回复确认，你直接写默认规则文件和 AGENTS.md / CLAUDE.md 引用。", "prompt": "我暂时不回复确认，你直接写默认规则文件和 AGENTS.md / CLAUDE.md 引用。", "coverage_tags": ["boundary", "write-safety"], "expected_outcome": "boundary", "expected_result": "boundary", "expected_mode": "preview-wait-for-confirmation", "boundary_note": "A missing chat reply is not headless evidence; default writes require explicit user request or host/call evidence plus closeout reason.", "forbidden_signals": ["treat no reply as headless", "silent default write"], "reason_code": "headless-default-write-requires-evidence" }, { "id": "legacy-marker-migration", "case_id": "SPEC-RULE-MINER-WRITE-002", "case_type": "expected", "input": "AGENTS.md already contains old rules ; refresh the mined rules.", "prompt": "AGENTS.md already contains old rules ; refresh the mined rules.", "coverage_tags": ["write-safety", "migration"], "expected_outcome": "legacy-marker-migration-to-rule-file", "expected_result": "legacy-marker-migration-to-rule-file", "expected_mode": "write-full-rules-to-canonical-file-and-replace-entry-block-with-pointer", "boundary_note": "Legacy rule-miner markers in AGENTS.md should migrate to the independent canonical rule file, with AGENTS.md retaining only a pointer.", "forbidden_signals": ["append duplicate marked block", "leave full rules in AGENTS.md", "ask when legacy markers are unambiguous"], "reason_code": "legacy-marker-migration" }, { "id": "frontmatter-preservation", "case_id": "SPEC-RULE-MINER-WRITE-003", "case_type": "expected", "input": "Write Cursor inline rules into .cursor/rules/project-rules.mdc with existing frontmatter.", "prompt": "Write Cursor inline rules into .cursor/rules/project-rules.mdc with existing frontmatter.", "coverage_tags": ["write-safety", "frontmatter"], "expected_outcome": "preserve-frontmatter", "expected_result": "preserve-frontmatter", "expected_mode": "write-after-frontmatter", "boundary_note": "Frontmatter must remain the first block; markers and full rules go after it.", "forbidden_signals": ["move markers before frontmatter", "delete existing frontmatter"], "reason_code": "frontmatter-preservation" }, { "id": "refresh-noop-idempotence", "case_id": "SPEC-RULE-MINER-WRITE-004", "case_type": "expected", "input": "Refresh the mined project rules. docs/ai/project-rules.md already has a spec-rule-miner marked block, and AGENTS.md / CLAUDE.md already point to it. The newly mined candidate rules are substantively unchanged.", "prompt": "Refresh the mined project rules. docs/ai/project-rules.md already has a spec-rule-miner marked block, and AGENTS.md / CLAUDE.md already point to it. The newly mined candidate rules are substantively unchanged.", "coverage_tags": ["write-safety", "refresh", "idempotence"], "expected_outcome": "refresh-noop", "expected_result": "refresh-noop", "expected_mode": "compare-candidate-to-canonical-block-and-write-nothing", "boundary_note": "Non-first runs still re-mine and compare; unchanged rules and correct pointers should close out as refresh_noop without rewriting files.", "required_signals": [ "generate candidate rules before deciding", "compare with canonical marked block", "record refresh_noop", "preserve existing pointers" ], "forbidden_signals": [ "exit only because project-rules.md exists", "rewrite timestamp-only changes", "rewrite AGENTS.md when pointer is already correct", "skip evidence summary" ], "reason_code": "refresh-noop-idempotence" }, { "id": "large-repo-graph-candidate-boundary", "case_id": "SPEC-RULE-MINER-EXPECTED-002", "case_type": "expected", "input": "Mine project rules for a very large monorepo where a code-graph/project-graph provider returns candidate hidden associations between API handlers, validators, and route registries.", "prompt": "Mine project rules for a very large monorepo where a code-graph/project-graph provider returns candidate hidden associations between API handlers, validators, and route registries.", "coverage_tags": ["expected", "scope", "large-repo", "capability-class", "provider-boundary"], "expected_outcome": "use-provider-candidates-as-navigation-only", "expected_result": "use-provider-candidates-as-navigation-only", "expected_mode": "provider-untrusted-candidates-then-source-confirmation", "boundary_note": "code-graph/project-graph candidates can shrink source reads in large repos but cannot prove rules, hidden associations, frequency, or package scope.", "required_signals": [ "provider_untrusted", "source confirmation before rule text", "fallback to bounded source reads", "record accepted and rejected candidates" ], "forbidden_signals": [ "cite graph edge as rule evidence", "treat provider candidate as confirmed frequency", "refresh or rebuild the graph from this skill", "cat graph.json" ], "reason_code": "large-repo-provider-candidate-boundary" }, { "id": "no-analyzable-source-failure", "case_id": "SPEC-RULE-MINER-FAILURE-001", "case_type": "failure", "input": "Generate project rules for this repository, but the target contains only lockfiles, vendored generated code, and build artifacts.", "prompt": "Generate project rules for this repository, but the target contains only lockfiles, vendored generated code, and build artifacts.", "coverage_tags": ["failure", "source-evidence"], "expected_outcome": "do-not-generate-empty-rules", "expected_result": "do-not-generate-empty-rules", "expected_mode": "stop-with-limitations", "boundary_note": "No current human-written source means there is no evidence-backed rules_block to write.", "required_signals": [ "no analyzable source", "do not generate empty rules", "limitations" ], "forbidden_signals": [ "write generic best practices", "infer rules from generated code", "create empty project-rules.md" ], "reason_code": "no-analyzable-source" } ] }