/** * Baseline comparison utilities * * Compare current benchmark results against stored baselines * to detect performance regressions. */ import type { ScenarioMetrics, ComparisonResult, ComparisonDetail, ComparisonThresholds, } from '../types'; import { DEFAULT_THRESHOLDS } from '../types'; /** * Compare a single metric value against baseline */ function compareMetric( name: string, current: number, baseline: number, isHigherBetter: boolean, warningDelta: number, failDelta: number, ): ComparisonDetail { const diff = current - baseline; const percentChange = baseline !== 0 ? (diff / baseline) * 100 : 0; // For FPS, higher is better; for memory/render time, lower is better const effectiveDiff = isHigherBetter ? -diff : diff; const effectiveWarning = Math.abs(warningDelta); const effectiveFail = Math.abs(failDelta); let status: 'pass' | 'warning' | 'fail' = 'pass'; let message = ''; if (effectiveDiff >= effectiveFail) { status = 'fail'; message = `${name} regressed by ${Math.abs(diff).toFixed(2)} (${Math.abs(percentChange).toFixed(1)}%)`; } else if (effectiveDiff >= effectiveWarning) { status = 'warning'; message = `${name} degraded by ${Math.abs(diff).toFixed(2)} (${Math.abs(percentChange).toFixed(1)}%)`; } else if (effectiveDiff < 0) { message = `${name} improved by ${Math.abs(diff).toFixed(2)} (${Math.abs(percentChange).toFixed(1)}%)`; } else { message = `${name} within tolerance (${diff >= 0 ? '+' : ''}${diff.toFixed(2)})`; } return { metric: name, current, baseline, diff, percentChange: Math.round(percentChange * 10) / 10, status, message, }; } /** * Compare current metrics against baseline * * @param current - Current benchmark results * @param baseline - Baseline results to compare against * @param thresholds - Optional custom thresholds (defaults to DEFAULT_THRESHOLDS) * @returns Comparison result with pass/fail status and detailed breakdown */ export function compareWithBaseline( current: ScenarioMetrics, baseline: ScenarioMetrics, thresholds: ComparisonThresholds = DEFAULT_THRESHOLDS, ): ComparisonResult { const details: ComparisonDetail[] = []; // Compare UI FPS (higher is better) details.push( compareMetric( 'UI FPS (avg)', current.uiFPS.avg, baseline.uiFPS.avg, true, // higher is better thresholds.fps.warningDelta, thresholds.fps.failDelta, ), ); details.push( compareMetric( 'UI FPS (min)', current.uiFPS.min, baseline.uiFPS.min, true, thresholds.fps.warningDelta, thresholds.fps.failDelta, ), ); // Compare jank percentage (lower is better) details.push( compareMetric( 'Jank %', current.uiFPS.jankPercentage, baseline.uiFPS.jankPercentage, false, // lower is better thresholds.jank.warningPercent, thresholds.jank.failPercent, ), ); // Compare memory (lower is better) details.push( compareMetric( 'Peak Memory (MB)', current.memory.peakMB, baseline.memory.peakMB, false, thresholds.memory.warningDeltaMB, thresholds.memory.failDeltaMB, ), ); details.push( compareMetric( 'Memory Delta (MB)', current.memory.deltaMB, baseline.memory.deltaMB, false, thresholds.memory.warningDeltaMB / 2, // Stricter for delta thresholds.memory.failDeltaMB / 2, ), ); // Compare render times (lower is better) details.push( compareMetric( 'Avg Render Time (ms)', current.render.avgRenderTime, baseline.render.avgRenderTime, false, thresholds.renderTime.warningAvgMs, thresholds.renderTime.failAvgMs, ), ); details.push( compareMetric( 'Max Render Time (ms)', current.render.maxRenderTime, baseline.render.maxRenderTime, false, thresholds.renderTime.warningAvgMs * 2, thresholds.renderTime.failAvgMs * 2, ), ); // Compare JS FPS if available if (current.jsFPS && baseline.jsFPS) { details.push( compareMetric( 'JS FPS (avg)', current.jsFPS.avg, baseline.jsFPS.avg, true, thresholds.fps.warningDelta, thresholds.fps.failDelta, ), ); } // Determine overall status const hasFailure = details.some((d) => d.status === 'fail'); const hasWarning = details.some((d) => d.status === 'warning'); const passed = !hasFailure; // Generate summary const failCount = details.filter((d) => d.status === 'fail').length; const warnCount = details.filter((d) => d.status === 'warning').length; const passCount = details.filter((d) => d.status === 'pass').length; let summary: string; if (hasFailure) { summary = `FAILED: ${failCount} metric(s) exceeded threshold. ${warnCount} warning(s), ${passCount} passed.`; } else if (hasWarning) { summary = `WARNING: ${warnCount} metric(s) degraded. ${passCount} passed.`; } else { summary = `PASSED: All ${passCount} metrics within tolerance.`; } return { passed, summary, details, current, baseline, }; } /** * Format comparison result as markdown for PR comments */ export function formatComparisonAsMarkdown( result: ComparisonResult, scenario: string, itemCount: number, ): string { const statusEmoji = result.passed ? '✅' : '❌'; const lines: string[] = [ `## ${statusEmoji} Performance Comparison: ${scenario} (${itemCount} items)`, '', result.summary, '', '| Metric | Current | Baseline | Diff | Status |', '|--------|---------|----------|------|--------|', ]; for (const detail of result.details) { const statusSymbol = detail.status === 'pass' ? '✅' : detail.status === 'warning' ? '⚠️' : '❌'; const diffStr = detail.diff >= 0 ? `+${detail.diff.toFixed(2)}` : detail.diff.toFixed(2); lines.push( `| ${detail.metric} | ${detail.current.toFixed(2)} | ${detail.baseline.toFixed(2)} | ${diffStr} (${detail.percentChange}%) | ${statusSymbol} |`, ); } return lines.join('\n'); } /** * Compare AnimatedFlashList vs vanilla FlashList */ export function compareVsVanilla( animated: ScenarioMetrics, vanilla: ScenarioMetrics, ): ComparisonResult { // Use relaxed thresholds for vanilla comparison since some overhead is expected const relaxedThresholds: ComparisonThresholds = { fps: { warningDelta: -3, // Allow 3 FPS drop failDelta: -8, // Fail at 8 FPS drop }, memory: { warningDeltaMB: 30, failDeltaMB: 75, }, jank: { warningPercent: 3, failPercent: 8, }, renderTime: { warningAvgMs: 5, failAvgMs: 12, }, }; const result = compareWithBaseline(animated, vanilla, relaxedThresholds); // Override summary for vanilla comparison context if (result.passed) { result.summary = `AnimatedFlashList overhead is within acceptable limits vs vanilla FlashList.`; } else { result.summary = `AnimatedFlashList overhead exceeds acceptable limits vs vanilla FlashList.`; } return result; }