{"version":3,"file":"eval-compare.d.ts","sourceRoot":"","sources":["../../../src/core/search/eval-compare.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAEjD,8CAA8C;AAC9C,MAAM,MAAM,gBAAgB,GAAG,WAAW,GAAG,WAAW,GAAG,YAAY,GAAG,YAAY,GAAG,KAAK,CAAC;AAE/F,MAAM,WAAW,gBAAgB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAgB;IAChC,MAAM,EAAE,gBAAgB,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,gBAAgB,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,aAAa;IAC7B,QAAQ,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,eAAe,EAAE,CAAA;KAAE,CAAC,CAAC;CAC3E;AAQD;mBACmB;AACnB,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAOpE;AAED;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,GAAG,aAAa,CAkBhH;AAED,wBAAgB,aAAa,CAC5B,GAAG,EAAE,aAAa,EAClB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,gBAAgB,GACtB,gBAAgB,CA0ClB;AAED,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,gBAAgB,GAAG,MAAM,CAcrE","sourcesContent":["/**\n * Paired comparison between two configs in one run record.\n *\n * Aggregate means are the wrong instrument for \"should we change this\n * default\". Both configs score the *same* queries, so the informative\n * question is per-query: how many did B improve, how many did it hurt, and\n * could that split have come from chance? A 3pp mean gap can be two queries\n * out of sixty moving, which is not a reason to change anything.\n *\n * Reports a two-sided sign test over the queries where the two configs\n * differ. Ties carry no information about direction and are excluded, which\n * is what makes the test paired.\n */\n\nimport type { EvalQueryResult } from \"./eval.js\";\n\n/** Per-query metrics that can be compared. */\nexport type ComparableMetric = \"recallAt1\" | \"recallAt5\" | \"recallAt10\" | \"recallAt50\" | \"mrr\";\n\nexport interface PairedQueryDelta {\n\tid: string;\n\tclass: string;\n\tbefore: number;\n\tafter: number;\n}\n\nexport interface PairedComparison {\n\tmetric: ComparableMetric;\n\tbefore: string;\n\tafter: string;\n\tbetter: number;\n\tworse: number;\n\ttied: number;\n\tmeanBefore: number;\n\tmeanAfter: number;\n\t/** Two-sided sign-test p-value over non-tied queries. 1 when all tie. */\n\tpValue: number;\n\tdeltas: PairedQueryDelta[];\n}\n\nexport interface ComparableRun {\n\tperQuery: Array<{ id: string; class: string; results: EvalQueryResult[] }>;\n}\n\nfunction binomial(n: number, k: number): number {\n\tlet result = 1;\n\tfor (let i = 0; i < k; i++) result = (result * (n - i)) / (i + 1);\n\treturn result;\n}\n\n/** Two-sided sign test: probability of a split at least this lopsided under\n *  a fair coin. */\nexport function signTestPValue(better: number, worse: number): number {\n\tconst n = better + worse;\n\tif (n === 0) return 1;\n\tconst tail = Math.min(better, worse);\n\tlet cumulative = 0;\n\tfor (let i = 0; i <= tail; i++) cumulative += binomial(n, i);\n\treturn Math.min(1, (cumulative / 2 ** n) * 2);\n}\n\n/**\n * Pair the *same* config across two run records, so \"did my change help?\" is\n * answerable at all. `comparePaired` compares two configs inside one run;\n * this compares one config before and after a retrieval change.\n *\n * Only sound when both runs scored the same corpus and the same gold set —\n * otherwise the delta includes corpus drift. The caller checks provenance.\n */\nexport function mergeRunsForComparison(before: ComparableRun, after: ComparableRun, label: string): ComparableRun {\n\tconst afterById = new Map(after.perQuery.map((q) => [q.id, q]));\n\tconst perQuery: ComparableRun[\"perQuery\"] = [];\n\tfor (const q of before.perQuery) {\n\t\tconst other = afterById.get(q.id);\n\t\tconst a = q.results.find((r) => r.label === label);\n\t\tconst b = other?.results.find((r) => r.label === label);\n\t\tif (!a || !b) continue;\n\t\tperQuery.push({\n\t\t\tid: q.id,\n\t\t\tclass: q.class,\n\t\t\tresults: [\n\t\t\t\t{ ...a, label: \"before\" },\n\t\t\t\t{ ...b, label: \"after\" },\n\t\t\t],\n\t\t});\n\t}\n\treturn { perQuery };\n}\n\nexport function comparePaired(\n\trun: ComparableRun,\n\tbefore: string,\n\tafter: string,\n\tmetric: ComparableMetric,\n): PairedComparison {\n\tconst pick = (results: readonly EvalQueryResult[], label: string): EvalQueryResult | undefined =>\n\t\tresults.find((r) => r.label === label);\n\n\tlet better = 0;\n\tlet worse = 0;\n\tlet tied = 0;\n\tlet sumBefore = 0;\n\tlet sumAfter = 0;\n\tlet n = 0;\n\tconst deltas: PairedQueryDelta[] = [];\n\n\tfor (const query of run.perQuery) {\n\t\tconst a = pick(query.results, before);\n\t\tconst b = pick(query.results, after);\n\t\tif (!a || !b) continue;\n\t\tn++;\n\t\tsumBefore += a[metric];\n\t\tsumAfter += b[metric];\n\t\tif (b[metric] > a[metric]) {\n\t\t\tbetter++;\n\t\t\tdeltas.push({ id: query.id, class: query.class, before: a[metric], after: b[metric] });\n\t\t} else if (b[metric] < a[metric]) {\n\t\t\tworse++;\n\t\t\tdeltas.push({ id: query.id, class: query.class, before: a[metric], after: b[metric] });\n\t\t} else {\n\t\t\ttied++;\n\t\t}\n\t}\n\n\treturn {\n\t\tmetric,\n\t\tbefore,\n\t\tafter,\n\t\tbetter,\n\t\tworse,\n\t\ttied,\n\t\tmeanBefore: n > 0 ? sumBefore / n : 0,\n\t\tmeanAfter: n > 0 ? sumAfter / n : 0,\n\t\tpValue: signTestPValue(better, worse),\n\t\tdeltas,\n\t};\n}\n\nexport function formatComparison(comparison: PairedComparison): string {\n\tconst { metric, before, after, better, worse, tied, pValue } = comparison;\n\tconst delta = comparison.meanAfter - comparison.meanBefore;\n\tconst verdict =\n\t\tpValue <= 0.05\n\t\t\t? \"significant at p<=0.05\"\n\t\t\t: `NOT significant (p=${pValue.toFixed(3)}) — mean gap is not evidence of a real difference`;\n\treturn (\n\t\t`${metric}: \"${after}\" vs \"${before}\"\\n` +\n\t\t`  mean ${comparison.meanBefore.toFixed(3)} -> ${comparison.meanAfter.toFixed(3)} ` +\n\t\t`(${delta >= 0 ? \"+\" : \"\"}${delta.toFixed(3)})\\n` +\n\t\t`  ${better} better, ${worse} worse, ${tied} tied\\n` +\n\t\t`  ${verdict}`\n\t);\n}\n"]}