{
  "$schema": "./corpus.schema.note",
  "description": "Labeled evaluation corpus for otito retrieval, risk classification, fixture-backed harness command execution, and gate effectiveness. Gate cases initialize an isolated Git repository from a committed base, apply one committed change-set, and assert both the verdict and deterministic check reason. See docs/EVALS.md for how to add a case and how thresholds work.",
  "k": 5,
  "thresholds": {
    "comment": "Floors set to current-baseline-minus-slack so the suite is green today but a regression trips it. Baseline (2026-09-27): retrieval p@5=0.873 r@5=1.0 mrr=0.976 (22/22); risk accuracy=1.0 (22/22). Aggregate metrics must be >= these floors for exit code 0. The risk floor is 0.96 so that a single failing case (21/22 = 0.955) still trips the gate. Re-record both baseline and floors in docs/EVALS.md after any intentional behavior change.",
    "retrieval": {
      "precisionAtK": 0.85,
      "recallAtK": 0.9,
      "mrr": 0.9
    },
    "risk": {
      "accuracy": 0.96
    }
  },
  "fixtureRoots": {
    "comment": "Logical fixture name -> directory relative to repo root under evals/fixtures/.",
    "sample-api": "evals/fixtures/sample-api",
    "sample-web": "evals/fixtures/sample-web",
    "sample-node-api": "evals/fixtures/sample-node-api",
    "shop-api": "evals/fixtures/shop-api",
    "polyglot-svc": "evals/fixtures/polyglot-svc",
    "web-client": "evals/fixtures/web-client",
    "harness-node": "evals/fixtures/harness-node",
    "auth-signup-api": "evals/fixtures/auth-signup-api",
    "checkin-api": "evals/fixtures/checkin-api",
    "rsvp-configuration-web": "evals/fixtures/rsvp-configuration-web",
    "mobile-host-rsvp": "evals/fixtures/mobile-host-rsvp",
    "campaign-email": "evals/fixtures/campaign-email",
    "gate-node": "evals/fixtures/gate-node",
    "multi-locale-web": "evals/fixtures/multi-locale-web"
  },
  "retrieval": [
    {
      "name": "api-add-rsvp-endpoint",
      "query": "add rsvp endpoint to events",
      "repoFixture": "sample-api",
      "expectedPrimary": ["src/events/events.controller.ts"]
    },
    {
      "name": "api-events-controller-noun",
      "query": "events controller",
      "repoFixture": "sample-api",
      "expectedPrimary": ["src/events/events.controller.ts"]
    },
    {
      "name": "web-fix-rsvp-button",
      "query": "fix the rsvp button",
      "repoFixture": "sample-web",
      "expectedPrimary": ["src/components/EventRsvpButton.tsx"],
      "expectedAnyOf": ["src/api/events.ts"]
    },
    {
      "name": "web-api-client-by-symbol",
      "query": "submitEventRsvp api client",
      "repoFixture": "sample-web",
      "expectedPrimary": ["src/api/events.ts"]
    },
    {
      "name": "node-route-shaped-query",
      "query": "where are the routes defined",
      "repoFixture": "sample-node-api",
      "expectedPrimary": ["src/server.js"]
    },
    {
      "name": "shop-refund-payment",
      "query": "refund a payment",
      "repoFixture": "shop-api",
      "expectedPrimary": ["src/payment/checkout.service.ts"],
      "expectedAnyOf": ["src/payment/stripe.webhook.ts"]
    },
    {
      "name": "shop-stripe-webhook",
      "query": "stripe webhook",
      "repoFixture": "shop-api",
      "expectedPrimary": ["src/payment/stripe.webhook.ts"]
    },
    {
      "name": "shop-roles-guard",
      "query": "role based access guard",
      "repoFixture": "shop-api",
      "expectedPrimary": ["src/auth/roles.guard.ts"]
    },
    {
      "name": "shop-create-order-route",
      "query": "create an order endpoint",
      "repoFixture": "shop-api",
      "expectedPrimary": ["src/orders/orders.controller.ts"]
    },
    {
      "name": "shop-error-shaped-fallback",
      "comment": "Error-message-shaped query matches no indexed keywords; must fall back gracefully to a non-empty primary set rather than returning nothing. expectedAnyOf accepts any indexed source file the fallback ranker surfaces.",
      "query": "ENOENT no such file or directory at readFileSync",
      "repoFixture": "shop-api",
      "expectedPrimary": [],
      "expectedAnyOf": ["src/auth/roles.guard.ts", "src/orders/orders.controller.ts", "src/payment/checkout.service.ts", "src/payment/stripe.webhook.ts"]
    },
    {
      "name": "poly-plural-users-service",
      "comment": "Plural query token 'users' must fold to singular and rank the Python service.",
      "query": "update the users service",
      "repoFixture": "polyglot-svc",
      "expectedPrimary": ["src/users_service.py"]
    },
    {
      "name": "poly-singular-user",
      "query": "user lookup",
      "repoFixture": "polyglot-svc",
      "expectedPrimary": ["src/users_service.py"]
    },
    {
      "name": "poly-go-inventory",
      "comment": "Cross-language: Go file ranked from an English query.",
      "query": "inventory reservation",
      "repoFixture": "polyglot-svc",
      "expectedPrimary": ["src/inventory_handler.go"]
    },
    {
      "name": "poly-ts-notification",
      "query": "send a notification",
      "repoFixture": "polyglot-svc",
      "expectedPrimary": ["src/notifications.ts"]
    },
    {
      "name": "web-checkout-button",
      "query": "checkout button component",
      "repoFixture": "web-client",
      "expectedPrimary": ["src/components/CheckoutButton.tsx"],
      "expectedAnyOf": ["src/api/orders.ts"]
    },
    {
      "name": "multi-repo-order-pairing",
      "comment": "Multi-repo route<->client pairing: backend controller and frontend client should both surface as primary for 'order creation'. Paths are namespaced as <fixture>/<repo-relative-path>.",
      "query": "order creation",
      "repoFixtures": ["shop-api", "web-client"],
      "expectedPrimary": ["shop-api/src/orders/orders.controller.ts", "web-client/src/api/orders.ts"]
    },
    {
      "name": "auth-email-verification-during-signup",
      "comment": "A plain-English question about email verification during signup must prioritize the registration and OTP verification flow over generic email operations.",
      "query": "where is email verification implemented during signup?",
      "repoFixture": "auth-signup-api",
      "expectedPrimary": ["src/authentication/auth.controller.ts", "src/authentication/auth.service.ts"]
    },
    {
      "name": "api-qr-checkin",
      "comment": "Plain-language QR check-in should find the ticket controller rather than generic booking helpers.",
      "query": "where is QR booking check-in handled for event staff?",
      "repoFixture": "checkin-api",
      "expectedPrimary": ["src/booking/ticket.controller.ts"]
    },
    {
      "name": "web-rsvp-privacy-configuration",
      "comment": "An organiser configuration question should prefer the form control, not the public RSVP reader.",
      "query": "where do organisers configure the RSVP page to keep venue details private?",
      "repoFixture": "rsvp-configuration-web",
      "expectedPrimary": ["components/conversational/questions/RsvpStudioQuestion.tsx"]
    },
    {
      "name": "mobile-host-bio-rsvp",
      "comment": "A mobile attendee question must bring the event screen and RSVP screen ahead of a dense type barrel.",
      "query": "where does the mobile app show an event host bio and let an attendee RSVP?",
      "repoFixture": "mobile-host-rsvp",
      "expectedPrimary": ["app/(tabs)/explore/[id].tsx", "app/rsvp/[eventId].tsx"]
    },
    {
      "name": "campaign-email-handlebars-template",
      "comment": "A campaign-email request must surface the Handlebars message template rather than only generic email services.",
      "query": "where is the branded campaign email Handlebars template rendered?",
      "repoFixture": "campaign-email",
      "expectedPrimary": [
        "src/email/template/campaign-message-responsive.hbs",
        "src/audience/campaign.service.ts",
        "src/email/template/layouts/responsive-base.hbs"
      ]
    },
    {
      "name": "multi-locale-named-files-and-symbol",
      "comment": "The bashbop-event-web overnight-event request (PR #588). It names two files and a symbol; the labels are the five source files that PR changed. Before named-file pinning, catalog demotion and extension stripping, five locale catalogs and every .ts/.tsx file competed for these slots.",
      "query": "fix overnight event bug: combineDateAndTime does not roll over to the next day; event-service.ts only compares getHours; EditableDate.tsx blocks overnight ranges",
      "repoFixture": "multi-locale-web",
      "expectedPrimary": [
        "services/event-service.ts",
        "components/inline-edit/EditableDate.tsx",
        "utils/create-event.ts",
        "utils/combineDateAndTime.ts",
        "components/conversational/utils/aiEventSchema.ts"
      ]
    }
  ],
  "risk": [
    {
      "name": "query-payload-not-money",
      "comment": "Regression guard: 'pay' substring in 'payload' must NOT imply money flow.",
      "mode": "query",
      "query": "fix payload parsing",
      "expectedConcepts": [],
      "notExpectedConcepts": ["money flow"]
    },
    {
      "name": "query-apple-signin-is-auth",
      "mode": "query",
      "query": "add apple sign in",
      "expectedConcepts": ["auth/security"],
      "notExpectedConcepts": []
    },
    {
      "name": "query-apple-emoji-not-auth",
      "comment": "'apple' alone (no sign-in verb) must NOT imply auth.",
      "mode": "query",
      "query": "add an apple emoji to the header",
      "expectedConcepts": [],
      "notExpectedConcepts": ["auth/security"]
    },
    {
      "name": "query-refund-is-money",
      "mode": "query",
      "query": "refund a payment",
      "expectedConcepts": ["money flow"],
      "notExpectedConcepts": []
    },
    {
      "name": "query-migration-is-data-model",
      "mode": "query",
      "query": "update the database migration",
      "expectedConcepts": ["data model"],
      "notExpectedConcepts": []
    },
    {
      "name": "query-order-route-is-request-surface",
      "mode": "query",
      "query": "add a new order route",
      "expectedConcepts": ["request surface"],
      "notExpectedConcepts": ["money flow"]
    },
    {
      "name": "path-roles-guard-is-auth",
      "comment": "Regression guard: roles.guard.ts must classify as auth/security.",
      "mode": "path",
      "path": "src/auth/roles.guard.ts",
      "expectedConcepts": ["auth/security"],
      "notExpectedConcepts": ["money flow"]
    },
    {
      "name": "path-checkout-service-is-money",
      "mode": "path",
      "path": "src/payment/checkout.service.ts",
      "expectedConcepts": ["money flow"],
      "notExpectedConcepts": ["auth/security"]
    },
    {
      "name": "path-tokens-util-not-auth",
      "comment": "Regression guard: a generic tokens.js utility must NOT fold into auth/security.",
      "mode": "path",
      "path": "src/lib/tokens.js",
      "expectedConcepts": [],
      "notExpectedConcepts": ["auth/security"]
    },
    {
      "name": "path-sessions-service-is-auth",
      "mode": "path",
      "path": "src/users/sessions.service.ts",
      "expectedConcepts": ["auth/security"],
      "notExpectedConcepts": []
    },
    {
      "name": "path-prisma-schema-is-data-model",
      "mode": "path",
      "path": "prisma/schema.prisma",
      "expectedConcepts": ["data model"],
      "notExpectedConcepts": []
    },
    {
      "name": "path-config-module-not-configuration",
      "comment": "Regression guard: a source module named config.js is code, not configuration — the bare 'config' word token used to match it.",
      "mode": "path",
      "path": "src/lib/config.js",
      "expectedConcepts": [],
      "notExpectedConcepts": ["configuration"]
    },
    {
      "name": "path-fixture-manifest-not-configuration",
      "comment": "Regression guard: a manifest inside a fixture corpus is test input, not this repository's configuration — 'package.json' used to match as a raw substring anywhere in the path.",
      "mode": "path",
      "path": "evals/fixtures/shop-api/package.json",
      "expectedConcepts": [],
      "notExpectedConcepts": ["configuration"]
    },
    {
      "name": "path-root-manifest-is-dependency-not-configuration",
      "comment": "A line-overlap backtest put manifest-only commits at 0.39x the repair base rate and real-config commits at 2.25x, so manifests carry their own flag and score zero.",
      "mode": "path",
      "path": "package.json",
      "expectedConcepts": ["dependency"],
      "notExpectedConcepts": ["configuration"]
    },
    {
      "name": "path-lockfile-is-dependency",
      "comment": "Lockfiles are dependency churn, not configuration.",
      "mode": "path",
      "path": "package-lock.json",
      "expectedConcepts": ["dependency"],
      "notExpectedConcepts": ["configuration"]
    },
    {
      "name": "path-environment-config-is-configuration",
      "comment": "A config/ directory holding environment JSON is real configuration and must keep flagging.",
      "mode": "path",
      "path": "src/feature-flags/config/environments/production.json",
      "expectedConcepts": ["configuration"],
      "notExpectedConcepts": []
    },
    {
      "name": "gate-checkout-spec-no-gate",
      "comment": "Regression guard: tests/checkout.spec.ts must NOT trip the merge gate even though it mentions checkout.",
      "mode": "gate",
      "path": "tests/checkout.spec.ts",
      "expectedConcepts": [],
      "notExpectedConcepts": ["gate"]
    },
    {
      "name": "gate-dependency-bump-no-gate",
      "comment": "A lockfile bump carries the zero-weight `dependency` flag; it must not demand maintainer sign-off on a risk-sensitive scope. Supply chain is covered by the separate dependency audit check.",
      "mode": "gate",
      "path": "package-lock.json",
      "expectedConcepts": [],
      "notExpectedConcepts": ["gate"]
    },
    {
      "name": "gate-payment-service-gates",
      "mode": "gate",
      "path": "src/payment/checkout.service.ts",
      "expectedConcepts": ["gate"],
      "notExpectedConcepts": []
    },
    {
      "name": "secret-dev-environments-not-secret",
      "comment": "Regression guard: dev.environments.ts must NOT be treated as a secret file via the '.env' substring.",
      "mode": "secret",
      "path": "config/dev.environments.ts",
      "expectedConcepts": [],
      "notExpectedConcepts": ["secret"]
    },
    {
      "name": "secret-env-production-is-secret",
      "mode": "secret",
      "path": ".env.production",
      "expectedConcepts": ["secret"],
      "notExpectedConcepts": []
    },
    {
      "name": "secret-docs-not-secret",
      "comment": "Regression guard: docs/secrets-management.md merely documents secrets; must NOT be a secret file.",
      "mode": "secret",
      "path": "docs/secrets-management.md",
      "expectedConcepts": [],
      "notExpectedConcepts": ["secret"]
    }
  ],
  "harnessExecution": [
    {
      "name": "node-install-test-typecheck-build",
      "repoFixture": "harness-node",
      "commands": [
        {
          "kind": "install",
          "group": "setup",
          "command": "npm install"
        },
        {
          "kind": "test",
          "group": "validate",
          "command": "npm test",
          "script": "test"
        },
        {
          "kind": "typecheck",
          "group": "validate",
          "command": "npm run typecheck",
          "script": "typecheck"
        },
        {
          "kind": "build",
          "group": "validate",
          "command": "npm run build",
          "script": "build"
        }
      ]
    }
  ],
  "gateEffectiveness": [
    {
      "name": "valid-control-passes",
      "repoFixture": "gate-node",
      "changeSet": "valid",
      "expectedVerdict": "PASS",
      "expectedChecks": [
        {
          "name": "Changed files",
          "status": "PASS",
          "summaryIncludes": "1 changed file"
        },
        {
          "name": "Validation commands",
          "status": "PASS"
        },
        {
          "name": "Review state",
          "status": "PASS"
        }
      ]
    },
    {
      "name": "aligned-change-to-source-kind-owner-converges",
      "repoFixture": "gate-node",
      "changeSet": "valid",
      "expectedVerdict": "PASS",
      "options": {
        "request": "update the greeting message",
        "minConvergence": 80
      },
      "expectedChecks": [
        {
          "name": "Convergence",
          "status": "PASS",
          "summaryIncludes": "satisfy the convergence requirement"
        }
      ]
    },
    {
      "name": "secret-file-is-blocked",
      "repoFixture": "gate-node",
      "changeSet": "secret",
      "expectedVerdict": "FAIL",
      "expectedChecks": [
        {
          "name": "Secret safety",
          "status": "FAIL",
          "summaryIncludes": "Potential secret or environment file changed",
          "detailsInclude": ".env.production"
        }
      ]
    },
    {
      "name": "secret-value-in-source-is-blocked",
      "repoFixture": "gate-node",
      "changeSet": "secret-content",
      "expectedVerdict": "FAIL",
      "expectedChecks": [
        {
          "name": "Secret safety",
          "status": "FAIL",
          "summaryIncludes": "credential value found in changed content",
          "detailsInclude": "src/greeting.js"
        }
      ]
    },
    {
      "name": "high-risk-change-needs-remote-controls",
      "repoFixture": "gate-node",
      "changeSet": "high-risk",
      "expectedVerdict": "FAIL",
      "options": {
        "policy": "high-risk",
        "governance": "team"
      },
      "expectedChecks": [
        {
          "name": "Risk review",
          "status": "WARN",
          "detailsInclude": "prisma/schema.prisma"
        },
        {
          "name": "Policy profile",
          "status": "FAIL",
          "summaryIncludes": "High-risk policy requires company controls",
          "detailsInclude": "GitHub PR mode evidence"
        }
      ]
    },
    {
      "name": "incomplete-release-is-blocked",
      "repoFixture": "gate-node",
      "changeSet": "release-incomplete",
      "expectedVerdict": "FAIL",
      "options": {
        "governance": "team"
      },
      "expectedChecks": [
        {
          "name": "Release discipline",
          "status": "FAIL",
          "summaryIncludes": "without a changelog update"
        }
      ]
    },
    {
      "name": "missing-validation-policy-is-blocked",
      "repoFixture": "gate-node",
      "changeSet": "validation-missing",
      "expectedVerdict": "FAIL",
      "options": {
        "runValidation": true
      },
      "expectedChecks": [
        {
          "name": "Validation execution",
          "status": "FAIL",
          "summaryIncludes": "versioned validation policy"
        }
      ]
    },
    {
      "name": "ownership-cannot-be-self-certified-locally",
      "repoFixture": "gate-node",
      "changeSet": "ownership-unverified",
      "expectedVerdict": "FAIL",
      "options": {
        "policy": "company",
        "governance": "team"
      },
      "expectedChecks": [
        {
          "name": "Policy profile",
          "status": "FAIL",
          "summaryIncludes": "verified GitHub PR controls",
          "detailsInclude": "CODEOWNERS"
        }
      ]
    },
    {
      "name": "scope-drift-fails-convergence",
      "repoFixture": "gate-node",
      "changeSet": "scope-drift",
      "expectedVerdict": "FAIL",
      "options": {
        "request": "update the greeting function",
        "minConvergence": 80
      },
      "expectedChecks": [
        {
          "name": "Convergence",
          "status": "FAIL",
          "summaryIncludes": "below the required minimum"
        }
      ]
    }
  ]
}