{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../../../src/core/evaluation/statistics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEtG,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,wBAAwB,EAAE,GAAG,qBAAqB,CA4C/F;AAED,wBAAgB,mBAAmB,CAClC,MAAM,EAAE,MAAM,EAAE,EAChB,IAAI,SAAK,EACT,OAAO,SAAO,GACZ;IAAE,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAgB5G;AAED,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,kBAAkB,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAMhG","sourcesContent":["import type { EvaluationArtifact, PairwiseEvaluationResult, StatisticalComparison } from \"./types.js\";\n\nexport function comparePairedResults(results: PairwiseEvaluationResult[]): StatisticalComparison {\n\tconst valid = results.filter(\n\t\t(result) => result.deterministicComparison.baselinePassed || result.deterministicComparison.candidatePassed,\n\t);\n\tif (valid.length === 0)\n\t\treturn {\n\t\t\tsampleSize: 0,\n\t\t\tconfidenceLevel: 0.95,\n\t\t\tmethod: \"paired_win_tie_loss\",\n\t\t\twinRate: undefined,\n\t\t\ttieRate: undefined,\n\t\t\tlossRate: undefined,\n\t\t\tadequateSample: false,\n\t\t};\n\tlet wins = 0;\n\tlet losses = 0;\n\tlet ties = 0;\n\tfor (const result of valid) {\n\t\tif (\n\t\t\tresult.semanticPreference === \"candidate\" ||\n\t\t\t(!result.semanticPreference &&\n\t\t\t\tresult.deterministicComparison.candidatePassed &&\n\t\t\t\t!result.deterministicComparison.baselinePassed)\n\t\t)\n\t\t\twins++;\n\t\telse if (\n\t\t\tresult.semanticPreference === \"baseline\" ||\n\t\t\t(!result.semanticPreference &&\n\t\t\t\tresult.deterministicComparison.baselinePassed &&\n\t\t\t\t!result.deterministicComparison.candidatePassed)\n\t\t)\n\t\t\tlosses++;\n\t\telse ties++;\n\t}\n\treturn {\n\t\tsampleSize: valid.length,\n\t\tconfidenceLevel: 0.95,\n\t\tmethod: \"paired_win_tie_loss\",\n\t\twinRate: wins / valid.length,\n\t\ttieRate: ties / valid.length,\n\t\tlossRate: losses / valid.length,\n\t\tconfidenceInterval: wilsonInterval(wins, valid.length, 0.95),\n\t\tadequateSample: valid.length >= 10,\n\t};\n}\n\nexport function bootstrapPercentile(\n\tvalues: number[],\n\tseed = 17,\n\tsamples = 1000,\n): { estimate: number | undefined; lower: number | undefined; upper: number | undefined; sampleSize: number } {\n\tif (values.length === 0) return { estimate: undefined, lower: undefined, upper: undefined, sampleSize: 0 };\n\tconst random = seededRandom(seed);\n\tconst means: number[] = [];\n\tfor (let sample = 0; sample < samples; sample++) {\n\t\tlet total = 0;\n\t\tfor (let index = 0; index < values.length; index++) total += values[Math.floor(random() * values.length)]!;\n\t\tmeans.push(total / values.length);\n\t}\n\tmeans.sort((left, right) => left - right);\n\treturn {\n\t\testimate: mean(values),\n\t\tlower: means[Math.floor(samples * 0.025)],\n\t\tupper: means[Math.floor(samples * 0.975)],\n\t\tsampleSize: values.length,\n\t};\n}\n\nexport function artifactMetricValues(artifacts: EvaluationArtifact[], metricId: string): number[] {\n\treturn artifacts.flatMap((artifact) =>\n\t\tartifact.metrics\n\t\t\t.filter((metric) => metric.metricId === metricId && metric.value !== undefined)\n\t\t\t.map((metric) => metric.value!),\n\t);\n}\n\nfunction mean(values: number[]): number {\n\treturn values.reduce((total, value) => total + value, 0) / values.length;\n}\n\nfunction wilsonInterval(successes: number, total: number, confidence: number): { lower: number; upper: number } {\n\tconst z = confidence === 0.95 ? 1.96 : 1.645;\n\tconst p = successes / total;\n\tconst denominator = 1 + (z * z) / total;\n\tconst center = (p + (z * z) / (2 * total)) / denominator;\n\tconst margin = (z / denominator) * Math.sqrt((p * (1 - p)) / total + (z * z) / (4 * total * total));\n\treturn { lower: Math.max(0, center - margin), upper: Math.min(1, center + margin) };\n}\n\nfunction seededRandom(seed: number): () => number {\n\tlet state = seed >>> 0;\n\treturn () => {\n\t\tstate = (1664525 * state + 1013904223) >>> 0;\n\t\treturn state / 0x100000000;\n\t};\n}\n"]}