/** * ComparisonTable — Statistical comparison of benchmark variants (Story 6.3) * * Rows = metrics, Columns = variants + diff + p-value + confidence * Color coding: green = winner, red = loser, gray = not significant * Confidence: ★★★ (p<0.001), ★★ (p<0.01), ★ (p<0.05), — (not sig) */ import React from 'react'; import type { BenchmarkMetricResult } from '../../api/client'; // ─── Types ────────────────────────────────────────────────────────── export interface ComparisonTableProps { metrics: BenchmarkMetricResult[]; variantNames: { id: string; name: string }[]; } // ─── Helpers ──────────────────────────────────────────────────────── const METRIC_LABELS: Record = { cost_per_session: 'Cost/Session', avg_latency: 'Avg Latency', error_rate: 'Error Rate', tool_call_count: 'Tool Calls', tokens_per_session: 'Tokens/Session', session_duration: 'Duration', task_completion: 'Task Completion', user_satisfaction: 'User Satisfaction', }; /** Metrics where lower is better */ const LOWER_IS_BETTER = new Set([ 'cost_per_session', 'avg_latency', 'error_rate', 'tokens_per_session', 'session_duration', ]); function metricLabel(metric: string): string { return METRIC_LABELS[metric] || metric.replace(/_/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()); } function confidenceStars(pValue?: number): string { if (pValue == null) return '—'; if (pValue < 0.001) return '★★★'; if (pValue < 0.01) return '★★'; if (pValue < 0.05) return '★'; return '—'; } function confidenceTooltip(pValue?: number): string { if (pValue == null) return 'No comparison available'; if (pValue < 0.001) return `p=${pValue.toFixed(4)} — Very high confidence`; if (pValue < 0.01) return `p=${pValue.toFixed(4)} — High confidence`; if (pValue < 0.05) return `p=${pValue.toFixed(4)} — Moderate confidence`; return `p=${pValue.toFixed(4)} — Not significant`; } function formatValue(val: number): string { if (Math.abs(val) >= 1_000_000) return (val / 1_000_000).toFixed(2) + 'M'; if (Math.abs(val) >= 1_000) return (val / 1_000).toFixed(2) + 'k'; if (Number.isInteger(val)) return String(val); return val.toFixed(3); } function diffPercent(a: number, b: number): string { if (a === 0 && b === 0) return '0%'; if (a === 0) return '—'; const pct = ((b - a) / Math.abs(a)) * 100; const sign = pct > 0 ? '+' : ''; return `${sign}${pct.toFixed(1)}%`; } // ─── Component ────────────────────────────────────────────────────── export function ComparisonTable({ metrics, variantNames, }: ComparisonTableProps): React.ReactElement { if (metrics.length === 0) { return (
No results available yet. Start the benchmark and collect sessions to see comparisons.
); } return (
{variantNames.map((v) => ( ))} {variantNames.length >= 2 && ( <> )} {metrics.map((m) => { const lowerBetter = LOWER_IS_BETTER.has(m.metric); const isSignificant = m.significant ?? (m.pValue != null && m.pValue < 0.05); // Determine winner among variants let winnerId: string | null = m.winnerId ?? null; if (!winnerId && m.variantResults.length >= 2 && isSignificant) { // Auto-determine: for lower-is-better, lowest mean wins const sorted = [...m.variantResults].sort((a, b) => lowerBetter ? a.mean - b.mean : b.mean - a.mean, ); winnerId = sorted[0].variantId; } // Diff between first two variants const v0 = m.variantResults[0]; const v1 = m.variantResults[1]; const diff = v0 && v1 ? diffPercent(v0.mean, v1.mean) : '—'; return ( {/* Drive body cells from variantNames (the header order) and look up each variant's result by id, so values always land under the right column even if variantResults is short or reordered. */} {variantNames.map((v) => { const vr = m.variantResults.find((r) => r.variantId === v.id); if (!vr) { return ( ); } let colorClass = 'text-gray-600'; if (isSignificant && winnerId) { colorClass = vr.variantId === winnerId ? 'text-green-700 font-semibold' : 'text-red-600'; } return ( ); })} {variantNames.length >= 2 && ( <> )} ); })}
Metric {v.name} Diff p-value Confidence
{metricLabel(m.metric)} {formatValue(vr.mean)} (n={vr.sampleSize}) {diff} {m.pValue != null ? m.pValue.toFixed(4) : '—'} {confidenceStars(m.pValue)}
); }