{
  "skill_name": "debugging-and-error-recovery",
  "evals": [
    {
      "id": 1,
      "name": "intermittent-test-failure",
      "prompt": "A test fails 1 in 20 runs with a timeout error. The test calls an external API. How do you debug it?",
      "expected_output": "A systematic debugging approach: reproduce, isolate variables, add logging, check for race conditions, and verify the fix.",
      "assertions": [
        "The output suggests increasing reproducibility first",
        "The output mentions logging or tracing",
        "The output considers race conditions or timing issues",
        "The output suggests mocking the external dependency",
        "The output does not jump to conclusions about the fix"
      ]
    },
    {
      "id": 2,
      "name": "production-outage",
      "prompt": "The production API is returning 500 errors for 10% of requests. Logs show 'Connection timeout to database'. Recent deploy was 2 hours ago.",
      "expected_output": "A structured incident response: rollback assessment, connection pool check, query performance analysis, and communication plan.",
      "assertions": [
        "The output suggests checking the recent deploy first",
        "The output mentions database connection pool limits",
        "The output considers query performance degradation",
        "The output includes a rollback or mitigation step",
        "The output distinguishes symptom from root cause"
      ]
    },
    {
      "id": 3,
      "name": "resists-guessing",
      "prompt": "The app crashes sometimes when users click the save button. Fix it.",
      "expected_output": "The skill insists on reproduction and evidence before proposing fixes.",
      "assertions": [
        "The output asks for reproduction steps",
        "The output asks for error logs or stack traces",
        "The output does not guess at a fix without evidence",
        "The output suggests adding logging if none exists"
      ]
    }
  ]
}