/** * ComparisonTable — Statistical comparison of benchmark variants (Story 6.3) * * Rows = metrics, Columns = variants + diff + p-value + confidence * Color coding: green = winner, red = loser, gray = not significant * Confidence: ★★★ (p<0.001), ★★ (p<0.01), ★ (p<0.05), — (not sig) */ import React from 'react'; import type { BenchmarkMetricResult } from '../../api/client'; // ─── Types ────────────────────────────────────────────────────────── export interface ComparisonTableProps { metrics: BenchmarkMetricResult[]; variantNames: { id: string; name: string }[]; } // ─── Helpers ──────────────────────────────────────────────────────── const METRIC_LABELS: Record = { cost_per_session: 'Cost/Session', avg_latency: 'Avg Latency', error_rate: 'Error Rate', tool_call_count: 'Tool Calls', tokens_per_session: 'Tokens/Session', session_duration: 'Duration', task_completion: 'Task Completion', user_satisfaction: 'User Satisfaction', }; /** Metrics where lower is better */ const LOWER_IS_BETTER = new Set([ 'cost_per_session', 'avg_latency', 'error_rate', 'tokens_per_session', 'session_duration', ]); function metricLabel(metric: string): string { return METRIC_LABELS[metric] || metric.replace(/_/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()); } function confidenceStars(pValue?: number): string { if (pValue == null) return '—'; if (pValue < 0.001) return '★★★'; if (pValue < 0.01) return '★★'; if (pValue < 0.05) return '★'; return '—'; } function confidenceTooltip(pValue?: number): string { if (pValue == null) return 'No comparison available'; if (pValue < 0.001) return `p=${pValue.toFixed(4)} — Very high confidence`; if (pValue < 0.01) return `p=${pValue.toFixed(4)} — High confidence`; if (pValue < 0.05) return `p=${pValue.toFixed(4)} — Moderate confidence`; return `p=${pValue.toFixed(4)} — Not significant`; } function formatValue(val: number): string { if (Math.abs(val) >= 1_000_000) return (val / 1_000_000).toFixed(2) + 'M'; if (Math.abs(val) >= 1_000) return (val / 1_000).toFixed(2) + 'k'; if (Number.isInteger(val)) return String(val); return val.toFixed(3); } function diffPercent(a: number, b: number): string { if (a === 0 && b === 0) return '0%'; if (a === 0) return '—'; const pct = ((b - a) / Math.abs(a)) * 100; const sign = pct > 0 ? '+' : ''; return `${sign}${pct.toFixed(1)}%`; } // ─── Component ────────────────────────────────────────────────────── export function ComparisonTable({ metrics, variantNames, }: ComparisonTableProps): React.ReactElement { if (metrics.length === 0) { return (
No results available yet. Start the benchmark and collect sessions to see comparisons.
); } return (
{variantNames.map((v) => ( ))} {variantNames.length >= 2 && ( <> )} {metrics.map((m) => { const lowerBetter = LOWER_IS_BETTER.has(m.metric); const isSignificant = m.significant ?? (m.pValue != null && m.pValue < 0.05); // Determine winner among variants let winnerId: string | null = m.winnerId ?? null; if (!winnerId && m.variantResults.length >= 2 && isSignificant) { // Auto-determine: for lower-is-better, lowest mean wins const sorted = [...m.variantResults].sort((a, b) => lowerBetter ? a.mean - b.mean : b.mean - a.mean, ); winnerId = sorted[0].variantId; } // Diff between first two variants const v0 = m.variantResults[0]; const v1 = m.variantResults[1]; const diff = v0 && v1 ? diffPercent(v0.mean, v1.mean) : '—'; return ( {m.variantResults.map((vr) => { let colorClass = 'text-gray-600'; if (isSignificant && winnerId) { if (vr.variantId === winnerId) { colorClass = 'text-green-700 font-semibold'; } else { colorClass = 'text-red-600'; } } return ( ); })} {variantNames.length >= 2 && ( <> )} ); })}
Metric {v.name} Diff p-value Confidence
{metricLabel(m.metric)} {formatValue(vr.mean)} (n={vr.sampleSize}) {diff} {m.pValue != null ? m.pValue.toFixed(4) : '—'} {confidenceStars(m.pValue)}
); }