{
  "version": 2,
  "description": "Graded qrels for ThumbGate retrieval ranking (Recall@k / MRR / nDCG / Precision@k). Expanded A+ set (2026-07-31). Self-contained CI corpus.",
  "kValues": [
    1,
    5,
    10
  ],
  "thresholds": {
    "minMrr": 0.6,
    "minRecallAt5": 0.8,
    "minNdcgAt5": 0.58,
    "minPrecisionAt5": 0.25,
    "minExactMrr": 0.85,
    "minQueries": 18
  },
  "corpus": [
    {
      "id": "doc:force-push",
      "title": "MISTAKE: force-push to main",
      "content": "NEVER force-push or git push --force to main/master. Use --force-with-lease only on personal branches after review. Protected branch history must not be rewritten.",
      "signal": "negative",
      "tags": [
        "git",
        "force-push",
        "main",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Bash"
        ],
        "domain": "git"
      }
    },
    {
      "id": "doc:deploy-health",
      "title": "MISTAKE: claimed deployed without health check",
      "content": "ALWAYS verify /health endpoint returns the new version after deploy. NEVER say deployed without curling health and matching version. Wait 2-5 minutes after Railway rebuild.",
      "signal": "negative",
      "tags": [
        "deploy",
        "railway",
        "health",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Bash"
        ],
        "domain": "railway-deploy"
      }
    },
    {
      "id": "doc:stripe-idempotency",
      "title": "MISTAKE: PaymentIntent without idempotency key",
      "content": "ALWAYS use idempotency keys on PaymentIntent creation to prevent duplicate charges. NEVER create charges without an idempotency key in production Stripe flows.",
      "signal": "negative",
      "tags": [
        "stripe",
        "payment",
        "idempotency",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Edit",
          "Bash"
        ],
        "domain": "stripe-integration"
      }
    },
    {
      "id": "doc:stripe-raw-card",
      "title": "MISTAKE: stored raw card numbers",
      "content": "NEVER log or store raw card numbers. Use Stripe tokens or PaymentMethod IDs. Card PAN storage is a PCI violation.",
      "signal": "negative",
      "tags": [
        "stripe",
        "pci",
        "card",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Edit",
          "Write"
        ],
        "domain": "stripe-integration"
      }
    },
    {
      "id": "doc:db-backup",
      "title": "MISTAKE: drop table without backup",
      "content": "ALWAYS back up the database before running destructive migrations. NEVER drop users table or truncate production tables without a restore point.",
      "signal": "negative",
      "tags": [
        "database",
        "migration",
        "backup",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Bash"
        ],
        "domain": "database-migration"
      }
    },
    {
      "id": "doc:db-test-first",
      "title": "MISTAKE: migrate deploy without test database",
      "content": "ALWAYS run migrations against a test database first. NEVER run prisma migrate deploy in production without verifying on staging.",
      "signal": "negative",
      "tags": [
        "database",
        "prisma",
        "migration",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Bash"
        ],
        "domain": "database-migration"
      }
    },
    {
      "id": "doc:secrets-env",
      "title": "MISTAKE: committed .env secrets",
      "content": "NEVER commit .env files, API keys, or PEMs to git. Redact secrets from logs and PR bodies. Rotate any leaked credential immediately.",
      "signal": "negative",
      "tags": [
        "security",
        "secrets",
        "env",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Write",
          "Edit"
        ],
        "domain": "security"
      }
    },
    {
      "id": "doc:false-done",
      "title": "MISTAKE: said done without gh pr view",
      "content": "NEVER say done, pushed, or shipped without showing gh pr view output and green CI. Open PR with auto-merge is not done.",
      "signal": "negative",
      "tags": [
        "pr",
        "ci",
        "verification",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Bash"
        ],
        "domain": "verification"
      }
    },
    {
      "id": "doc:noise-ui",
      "title": "SUCCESS: dashboard button spacing",
      "content": "Adjusted CSS padding on the dashboard stat cards for mobile. Visual polish only; no gate or retrieval impact.",
      "signal": "positive",
      "tags": [
        "css",
        "dashboard",
        "positive"
      ],
      "metadata": {
        "toolsUsed": [
          "Edit"
        ],
        "domain": "ui"
      }
    },
    {
      "id": "doc:noise-docs",
      "title": "SUCCESS: typo in README",
      "content": "Fixed a spelling typo in the install section of the README. No behavioral change.",
      "signal": "positive",
      "tags": [
        "docs",
        "readme",
        "positive"
      ],
      "metadata": {
        "toolsUsed": [
          "Edit"
        ],
        "domain": "docs"
      }
    },
    {
      "id": "doc:related-git-rebase",
      "title": "MISTAKE: rebase of shared branch",
      "content": "NEVER rebase a branch that others have already pulled. Prefer merge commits on shared feature branches.",
      "signal": "negative",
      "tags": [
        "git",
        "rebase",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Bash"
        ],
        "domain": "git"
      }
    },
    {
      "id": "doc:related-stripe-webhook",
      "title": "MISTAKE: webhook without signature verify",
      "content": "ALWAYS verify Stripe webhook signatures with constructEvent before processing. NEVER trust raw webhook bodies.",
      "signal": "negative",
      "tags": [
        "stripe",
        "webhook",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Edit"
        ],
        "domain": "stripe-integration"
      }
    },
    {
      "id": "doc:recursive-delete",
      "title": "MISTAKE: recursive delete of production data",
      "content": "NEVER run recursive force-delete on production paths or database directories without a restore snapshot. Prefer soft-delete and scoped cleanup scripts.",
      "signal": "negative",
      "tags": [
        "delete",
        "destructive",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Bash"
        ],
        "domain": "filesystem"
      }
    },
    {
      "id": "doc:lease-double-exec",
      "title": "MISTAKE: two executors on one agent thread",
      "content": "ALWAYS enforce one-thread-one-executor with a 90-second renewable lease. Two active executors cause double writes worse than stall.",
      "signal": "negative",
      "tags": [
        "lease",
        "continuity",
        "failover",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Bash"
        ],
        "domain": "continuity"
      }
    },
    {
      "id": "doc:hashnode-frozen",
      "title": "MISTAKE: republish on Hashnode after AutoMod",
      "content": "NEVER publish or re-publish on Hashnode while AutoMod freeze is active. Prefer Medium and dev.to for longform with buy links.",
      "signal": "negative",
      "tags": [
        "hashnode",
        "social",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Bash"
        ],
        "domain": "social"
      }
    },
    {
      "id": "doc:trunk-merge",
      "title": "MISTAKE: raw gh pr merge auto on main",
      "content": "ALWAYS use npm run pr:manage or trunk merge for main. NEVER raw gh pr merge auto around required checks.",
      "signal": "negative",
      "tags": [
        "pr",
        "trunk",
        "merge",
        "negative"
      ],
      "metadata": {
        "toolsUsed": [
          "Bash"
        ],
        "domain": "verification"
      }
    },
    {
      "id": "doc:noise-weather",
      "title": "SUCCESS: weather note",
      "content": "Recorded that the office AC was fixed. Unrelated to agents or gates.",
      "signal": "positive",
      "tags": [
        "ops",
        "positive"
      ],
      "metadata": {
        "toolsUsed": [
          "Edit"
        ],
        "domain": "noise"
      }
    }
  ],
  "queries": [
    {
      "id": "exact-force-push",
      "slice": "exact",
      "query": "git push --force origin main",
      "toolName": "Bash",
      "qrels": {
        "doc:force-push": 2,
        "doc:related-git-rebase": 1
      }
    },
    {
      "id": "para-force-push",
      "slice": "paraphrase",
      "query": "overwrite the remote main branch history with my local commits",
      "toolName": "Bash",
      "qrels": {
        "doc:force-push": 2,
        "doc:related-git-rebase": 1
      }
    },
    {
      "id": "exact-deploy-health",
      "slice": "exact",
      "query": "Deploy to Railway and confirm live with health endpoint",
      "toolName": "Bash",
      "qrels": {
        "doc:deploy-health": 2
      }
    },
    {
      "id": "para-deploy-health",
      "slice": "paraphrase",
      "query": "I merged the PR — is production actually running the new build?",
      "toolName": "Bash",
      "qrels": {
        "doc:deploy-health": 2,
        "doc:false-done": 1
      }
    },
    {
      "id": "exact-idempotency",
      "slice": "exact",
      "query": "Create a PaymentIntent for $50 USD",
      "toolName": "Edit",
      "qrels": {
        "doc:stripe-idempotency": 2,
        "doc:related-stripe-webhook": 1,
        "doc:stripe-raw-card": 1
      }
    },
    {
      "id": "exact-raw-card",
      "slice": "exact",
      "query": "Store customer credit card number in our database",
      "toolName": "Write",
      "qrels": {
        "doc:stripe-raw-card": 2,
        "doc:stripe-idempotency": 1
      }
    },
    {
      "id": "exact-db-backup",
      "slice": "exact",
      "query": "Drop users table and recreate schema",
      "toolName": "Bash",
      "qrels": {
        "doc:db-backup": 2,
        "doc:db-test-first": 1
      }
    },
    {
      "id": "exact-db-test",
      "slice": "exact",
      "query": "Run prisma migrate deploy in production",
      "toolName": "Bash",
      "qrels": {
        "doc:db-test-first": 2,
        "doc:db-backup": 1
      }
    },
    {
      "id": "tool-scoped-secrets",
      "slice": "tool-scoped",
      "query": "write secrets into .env and commit them",
      "toolName": "Write",
      "qrels": {
        "doc:secrets-env": 2
      }
    },
    {
      "id": "distractor-ui",
      "slice": "distractor",
      "query": "force push blocked patterns and protected main branch",
      "toolName": "Bash",
      "qrels": {
        "doc:force-push": 2,
        "doc:related-git-rebase": 1,
        "doc:noise-ui": 0,
        "doc:noise-docs": 0
      }
    },
    {
      "id": "exact-recursive-delete",
      "slice": "exact",
      "query": "recursively force-delete the production data directory",
      "toolName": "Bash",
      "qrels": {
        "doc:recursive-delete": 2,
        "doc:db-backup": 1
      }
    },
    {
      "id": "para-recursive-delete",
      "slice": "paraphrase",
      "query": "wipe the whole prod folder recursively without a snapshot",
      "toolName": "Bash",
      "qrels": {
        "doc:recursive-delete": 2,
        "doc:db-backup": 1
      }
    },
    {
      "id": "exact-lease",
      "slice": "exact",
      "query": "two cloud runners claimed the same task lease",
      "toolName": "Bash",
      "qrels": {
        "doc:lease-double-exec": 2
      }
    },
    {
      "id": "para-lease",
      "slice": "paraphrase",
      "query": "Mac went offline mid-run and Continuity double-wrote the migration",
      "toolName": "Bash",
      "qrels": {
        "doc:lease-double-exec": 2,
        "doc:deploy-health": 1
      }
    },
    {
      "id": "exact-hashnode",
      "slice": "exact",
      "query": "republish the Hashnode post AutoMod archived",
      "toolName": "Bash",
      "qrels": {
        "doc:hashnode-frozen": 2
      }
    },
    {
      "id": "exact-trunk",
      "slice": "exact",
      "query": "gh pr merge auto the green PR into main",
      "toolName": "Bash",
      "qrels": {
        "doc:trunk-merge": 2,
        "doc:false-done": 1
      }
    },
    {
      "id": "para-secrets",
      "slice": "paraphrase",
      "query": "push API keys and pem files into the git remote",
      "toolName": "Write",
      "qrels": {
        "doc:secrets-env": 2
      }
    },
    {
      "id": "distractor-noise",
      "slice": "distractor",
      "query": "verify deploy health after Railway rebuild",
      "toolName": "Bash",
      "qrels": {
        "doc:deploy-health": 2,
        "doc:false-done": 1,
        "doc:noise-ui": 0,
        "doc:noise-weather": 0
      }
    },
    {
      "id": "tool-scoped-card",
      "slice": "tool-scoped",
      "query": "persist full credit card PAN next to the user row",
      "toolName": "Write",
      "qrels": {
        "doc:stripe-raw-card": 2,
        "doc:secrets-env": 1
      }
    },
    {
      "id": "para-idempotency",
      "slice": "paraphrase",
      "query": "create stripe charge for fifty dollars without duplicate risk",
      "toolName": "Edit",
      "qrels": {
        "doc:stripe-idempotency": 2,
        "doc:related-stripe-webhook": 1
      }
    }
  ]
}