// src/report/scoring.ts — Scoring utilities ported from dashboard // Pure arithmetic, no browser dependencies // --------------------------------------------------------------------------- // Thresholds // --------------------------------------------------------------------------- export const SCORE_THRESHOLDS = { SUCCESS: 0.70, WARNING: 0.50, } as const; // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- export interface ScoreMetrics { criteriaRate: number | null; // 0-1, null when no data rubricRate: number | null; // 0-1, null when no rubric data criteriaPassed: number; criteriaTotal: number; rubricRulesPassed: number; rubricRulesTotal: number; } export type ScoreColor = 'success' | 'warning' | 'error'; export interface ScoreColorClasses { text: string; bg: string; bar: string; hex: string; } // Minimal types for scoring functions export interface ScoringRun { status?: string; passed?: number; completed?: number; criteria_passed?: number; criteria_total?: number; rubric_rules_passed?: number; rubric_rules_total?: number; component_scores?: Record; } export interface ScoringResult { criteria_scores?: Array<{ passed: boolean }>; rubric_scores?: Array<{ passed: boolean }>; } // V1 matrix types export type Severity = 'High' | 'Medium' | 'Low'; export interface EvaluatorInfo { key: string; name: string; severity: Severity; prompt?: string; model?: string; } export interface MatrixCell { score: number; comment: string | null; passed: boolean; } export interface MatrixRow { resultId: string; testCaseId: string; question: string; response: string; cells: Record; avgScore: number; } export interface MatrixData { evaluators: EvaluatorInfo[]; rows: MatrixRow[]; columnAverages: Record; overallAverage: number; } export interface WeightConfig { High: number; Medium: number; Low: number; } export const DEFAULT_WEIGHTS: WeightConfig = { High: 2, Medium: 1, Low: 0.5, }; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- function sumComponentScores(cs: Record): { passed: number; total: number } { const scopes = ['prompt', 'knowledge_base', 'function', 'general'] as const; let passed = 0; let total = 0; for (const scope of scopes) { passed += cs[scope]?.passed ?? 0; total += cs[scope]?.total ?? 0; } return { passed, total }; } // --------------------------------------------------------------------------- // Run-level scoring // --------------------------------------------------------------------------- export function getRunScoreMetrics(run: ScoringRun): ScoreMetrics { let criteriaPassed: number; let criteriaTotal: number; let rubricRulesPassed: number; let rubricRulesTotal: number; if (run.criteria_total !== undefined && run.criteria_total > 0) { criteriaPassed = run.criteria_passed ?? 0; criteriaTotal = run.criteria_total; } else { criteriaPassed = run.passed ?? 0; criteriaTotal = run.completed ?? 0; } const hasNewRubricFields = run.rubric_rules_total !== undefined; if (hasNewRubricFields) { rubricRulesPassed = run.rubric_rules_passed ?? 0; rubricRulesTotal = run.rubric_rules_total ?? 0; } else { const cs = run.component_scores; if (cs) { const sums = sumComponentScores(cs); rubricRulesPassed = sums.passed; rubricRulesTotal = sums.total; } else { rubricRulesPassed = 0; rubricRulesTotal = 0; } } const criteriaRate = criteriaTotal > 0 ? criteriaPassed / criteriaTotal : null; const rubricRate = rubricRulesTotal > 0 ? rubricRulesPassed / rubricRulesTotal : null; return { criteriaRate, rubricRate, criteriaPassed, criteriaTotal, rubricRulesPassed, rubricRulesTotal }; } export function getRunCriteriaScore(run: ScoringRun): number | null { const { criteriaRate } = getRunScoreMetrics(run); return criteriaRate !== null ? Math.round(criteriaRate * 100) : null; } export function getRunRubricScore(run: ScoringRun): number | null { const { rubricRate } = getRunScoreMetrics(run); return rubricRate !== null ? Math.round(rubricRate * 100) : null; } // --------------------------------------------------------------------------- // Result-level scoring // --------------------------------------------------------------------------- export function getResultsScoreMetrics(results: ScoringResult[]): ScoreMetrics { if (results.length === 0) { return { criteriaRate: null, rubricRate: null, criteriaPassed: 0, criteriaTotal: 0, rubricRulesPassed: 0, rubricRulesTotal: 0 }; } const allCriteria = results.flatMap((r) => r.criteria_scores || []); const criteriaTotal = allCriteria.length; const criteriaPassed = allCriteria.filter((c) => c.passed).length; const criteriaRate = criteriaTotal > 0 ? criteriaPassed / criteriaTotal : null; const allRubricScores = results.flatMap((r) => r.rubric_scores || []); const rubricRulesTotal = allRubricScores.length; const rubricRulesPassed = allRubricScores.filter((r) => r.passed).length; const rubricRate = rubricRulesTotal > 0 ? rubricRulesPassed / rubricRulesTotal : null; return { criteriaRate, rubricRate, criteriaPassed, criteriaTotal, rubricRulesPassed, rubricRulesTotal }; } // --------------------------------------------------------------------------- // Aggregate pass rate // --------------------------------------------------------------------------- export function getAggregatePassRate(runs: ScoringRun[]): number | null { const completed = runs.filter((r) => r.status === 'completed'); if (completed.length === 0) return null; let totalPassed = 0; let totalChecks = 0; for (const run of completed) { if (run.criteria_total !== undefined && run.criteria_total > 0) { totalPassed += run.criteria_passed ?? 0; totalChecks += run.criteria_total; } else { totalPassed += run.passed ?? 0; totalChecks += run.completed ?? 0; } } return totalChecks > 0 ? totalPassed / totalChecks : null; } // --------------------------------------------------------------------------- // Color utilities // --------------------------------------------------------------------------- export function getScoreColor(rate: number | null | undefined): ScoreColor { if (rate === null || rate === undefined) return 'error'; if (rate >= SCORE_THRESHOLDS.SUCCESS) return 'success'; if (rate >= SCORE_THRESHOLDS.WARNING) return 'warning'; return 'error'; } export function getScoreColorClasses(rate: number | null | undefined): ScoreColorClasses { const color = getScoreColor(rate); switch (color) { case 'success': return { text: 'text-success', bg: 'bg-success/10', bar: 'bg-success', hex: '#22c55e' }; case 'warning': return { text: 'text-warning', bg: 'bg-warning/10', bar: 'bg-warning', hex: '#f59e0b' }; case 'error': return { text: 'text-error', bg: 'bg-error/10', bar: 'bg-error', hex: '#ef4444' }; } } export function getScoreHexColor(rate: number | null | undefined): string { return getScoreColorClasses(rate).hex; } // --------------------------------------------------------------------------- // Weighted score calculation (V1 reports) // --------------------------------------------------------------------------- export function calculateWeightedScores( matrixData: MatrixData, weights: WeightConfig, ): { overall: number; bySeverity: Record; } { const bySeverity: Record = { High: { total: 0, weight: 0, count: 0 }, Medium: { total: 0, weight: 0, count: 0 }, Low: { total: 0, weight: 0, count: 0 }, }; for (const evaluator of matrixData.evaluators) { const avg = matrixData.columnAverages[evaluator.key] || 0; const weight = weights[evaluator.severity]; bySeverity[evaluator.severity].total += avg * weight; bySeverity[evaluator.severity].weight += weight; bySeverity[evaluator.severity].count++; } let overallTotal = 0; let overallWeight = 0; const result: Record = { High: { score: 0, count: 0 }, Medium: { score: 0, count: 0 }, Low: { score: 0, count: 0 }, }; for (const severity of ['High', 'Medium', 'Low'] as Severity[]) { const data = bySeverity[severity]; result[severity] = { score: data.weight > 0 ? data.total / data.weight : 0, count: data.count, }; overallTotal += data.total; overallWeight += data.weight; } return { overall: overallWeight > 0 ? overallTotal / overallWeight : 0, bySeverity: result, }; }