/** * Performance Testing Utilities for postgres.do * * This module provides infrastructure for performance baseline tests and * regression detection. It includes: * * - Timing measurement utilities with statistical analysis * - Custom vitest matchers for performance assertions * - Baseline management for tracking expected performance * - Regression detection with configurable thresholds * * @module @dotdo/postgres-shared/perf-test-utils * * @example * ```typescript * import { * measureTiming, * PerformanceBaseline, * setupPerfMatchers, * } from '@dotdo/postgres-shared/perf-test-utils' * * // Setup custom matchers in vitest * setupPerfMatchers() * * describe('performance tests', () => { * it('completes query within baseline', async () => { * const timing = await measureTiming(async () => { * await db.query('SELECT 1') * }) * expect(timing).toBeWithinBaseline('simple-query', { p95: 10 }) * }) * }) * ``` */ // ============================================================================ // TYPES // ============================================================================ /** * Statistical summary of timing measurements */ export interface TimingStats { /** Number of samples collected */ count: number /** Minimum execution time in milliseconds */ min: number /** Maximum execution time in milliseconds */ max: number /** Mean (average) execution time in milliseconds */ mean: number /** Median (p50) execution time in milliseconds */ median: number /** 95th percentile execution time in milliseconds */ p95: number /** 99th percentile execution time in milliseconds */ p99: number /** Standard deviation in milliseconds */ stdDev: number /** Coefficient of variation (stdDev/mean) as a ratio */ cv: number /** All individual timing samples in milliseconds */ samples: number[] } /** * Options for measuring timing */ export interface MeasureTimingOptions { /** Number of iterations to run (default: 10) */ iterations?: number /** Number of warmup iterations to discard (default: 2) */ warmupIterations?: number /** Optional setup function to run before each iteration */ setup?: () => void | Promise /** Optional teardown function to run after each iteration */ teardown?: () => void | Promise /** Whether to use high-resolution timer (default: true) */ highResolution?: boolean } /** * Performance baseline definition */ export interface PerformanceBaselineDefinition { /** Operation name for identification */ name: string /** Description of what is being measured */ description?: string /** Expected p50 (median) latency in milliseconds */ p50?: number /** Expected p95 latency in milliseconds */ p95?: number /** Expected p99 latency in milliseconds */ p99?: number /** Expected maximum latency in milliseconds */ maxLatency?: number /** Expected minimum throughput (operations per second) */ minThroughput?: number /** Tolerance factor for regression detection (default: 1.2 = 20% tolerance) */ toleranceFactor?: number /** Environment-specific overrides */ environments?: Record> } /** * Result of a baseline comparison */ export interface BaselineComparisonResult { /** Whether the measured values are within the baseline */ withinBaseline: boolean /** Detailed results for each metric */ metrics: { name: string expected: number actual: number tolerance: number passed: boolean deviation: number deviationPercent: number }[] /** Summary message */ message: string } /** * Throughput measurement result */ export interface ThroughputResult { /** Operations per second */ opsPerSecond: number /** Total operations completed */ totalOps: number /** Total duration in milliseconds */ durationMs: number /** Average operation time in milliseconds */ avgOpTimeMs: number } /** * Options for throughput measurement */ export interface MeasureThroughputOptions { /** Duration to measure in milliseconds (default: 1000) */ durationMs?: number /** Number of warmup operations to run first (default: 10) */ warmupOps?: number /** Optional setup function */ setup?: () => void | Promise /** Optional teardown function */ teardown?: () => void | Promise } // ============================================================================ // TIMING MEASUREMENT // ============================================================================ /** * Gets current high-resolution time in milliseconds. * Uses performance.now() when available, falls back to Date.now(). */ export function getHighResTime(): number { if (typeof performance !== 'undefined' && typeof performance.now === 'function') { return performance.now() } return Date.now() } /** * Calculates statistical summary from an array of timing samples. * * @param samples - Array of timing measurements in milliseconds * @returns Statistical summary of the samples */ export function calculateStats(samples: number[]): TimingStats { if (samples.length === 0) { return { count: 0, min: 0, max: 0, mean: 0, median: 0, p95: 0, p99: 0, stdDev: 0, cv: 0, samples: [], } } const sorted = [...samples].sort((a, b) => a - b) const count = sorted.length const min = sorted[0]! const max = sorted[count - 1]! const sum = sorted.reduce((a, b) => a + b, 0) const mean = sum / count // Calculate standard deviation const squaredDiffs = sorted.map(x => Math.pow(x - mean, 2)) const avgSquaredDiff = squaredDiffs.reduce((a, b) => a + b, 0) / count const stdDev = Math.sqrt(avgSquaredDiff) // Coefficient of variation const cv = mean > 0 ? stdDev / mean : 0 // Percentile calculation const percentile = (p: number): number => { const index = Math.ceil((p / 100) * count) - 1 return sorted[Math.max(0, Math.min(index, count - 1))]! } return { count, min, max, mean, median: percentile(50), p95: percentile(95), p99: percentile(99), stdDev, cv, samples: sorted, } } /** * Measures execution time of an async function with statistical analysis. * * @param fn - Function to measure * @param options - Measurement options * @returns Statistical summary of timing measurements * * @example * ```typescript * const stats = await measureTiming(async () => { * await db.query('SELECT * FROM users LIMIT 100') * }, { iterations: 20 }) * * console.log(`Mean latency: ${stats.mean}ms, p95: ${stats.p95}ms`) * ``` */ export async function measureTiming( fn: () => void | Promise, options: MeasureTimingOptions = {} ): Promise { const { iterations = 10, warmupIterations = 2, setup, teardown, highResolution = true, } = options const getTime = highResolution ? getHighResTime : () => Date.now() const samples: number[] = [] // Warmup iterations (discarded) for (let i = 0; i < warmupIterations; i++) { if (setup) await setup() await fn() if (teardown) await teardown() } // Actual measurement iterations for (let i = 0; i < iterations; i++) { if (setup) await setup() const start = getTime() await fn() const end = getTime() samples.push(end - start) if (teardown) await teardown() } return calculateStats(samples) } /** * Measures execution time of a synchronous function with statistical analysis. * * @param fn - Function to measure * @param options - Measurement options (without async setup/teardown) * @returns Statistical summary of timing measurements */ export function measureTimingSync( fn: () => void, options: Omit & { setup?: () => void teardown?: () => void } = {} ): TimingStats { const { iterations = 10, warmupIterations = 2, setup, teardown, highResolution = true, } = options const getTime = highResolution ? getHighResTime : () => Date.now() const samples: number[] = [] // Warmup iterations (discarded) for (let i = 0; i < warmupIterations; i++) { if (setup) setup() fn() if (teardown) teardown() } // Actual measurement iterations for (let i = 0; i < iterations; i++) { if (setup) setup() const start = getTime() fn() const end = getTime() samples.push(end - start) if (teardown) teardown() } return calculateStats(samples) } // ============================================================================ // THROUGHPUT MEASUREMENT // ============================================================================ /** * Measures throughput (operations per second) for an async operation. * * @param fn - Function to measure * @param options - Measurement options * @returns Throughput measurement result * * @example * ```typescript * const throughput = await measureThroughput(async () => { * await cache.set('key', 'value') * }, { durationMs: 5000 }) * * console.log(`Throughput: ${throughput.opsPerSecond} ops/sec`) * ``` */ export async function measureThroughput( fn: () => void | Promise, options: MeasureThroughputOptions = {} ): Promise { const { durationMs = 1000, warmupOps = 10, setup, teardown } = options if (setup) await setup() // Warmup for (let i = 0; i < warmupOps; i++) { await fn() } // Measure throughput let totalOps = 0 const startTime = getHighResTime() const endTime = startTime + durationMs while (getHighResTime() < endTime) { await fn() totalOps++ } const actualDuration = getHighResTime() - startTime const opsPerSecond = (totalOps / actualDuration) * 1000 if (teardown) await teardown() return { opsPerSecond, totalOps, durationMs: actualDuration, avgOpTimeMs: actualDuration / totalOps, } } /** * Measures throughput for a synchronous operation. * * @param fn - Function to measure * @param options - Measurement options * @returns Throughput measurement result */ export function measureThroughputSync( fn: () => void, options: Omit & { setup?: () => void teardown?: () => void } = {} ): ThroughputResult { const { durationMs = 1000, warmupOps = 10, setup, teardown } = options if (setup) setup() // Warmup for (let i = 0; i < warmupOps; i++) { fn() } // Measure throughput let totalOps = 0 const startTime = getHighResTime() const endTime = startTime + durationMs while (getHighResTime() < endTime) { fn() totalOps++ } const actualDuration = getHighResTime() - startTime const opsPerSecond = (totalOps / actualDuration) * 1000 if (teardown) teardown() return { opsPerSecond, totalOps, durationMs: actualDuration, avgOpTimeMs: actualDuration / totalOps, } } // ============================================================================ // BASELINE MANAGEMENT // ============================================================================ /** * Registry of performance baselines for regression detection. */ export class PerformanceBaseline { private baselines: Map = new Map() private environment: string constructor(environment: string = 'test') { this.environment = environment } /** * Registers a performance baseline. * * @param baseline - Baseline definition */ register(baseline: PerformanceBaselineDefinition): void { this.baselines.set(baseline.name, baseline) } /** * Registers multiple baselines at once. * * @param baselines - Array of baseline definitions */ registerAll(baselines: PerformanceBaselineDefinition[]): void { for (const baseline of baselines) { this.register(baseline) } } /** * Gets a baseline definition by name. * * @param name - Baseline name * @returns Baseline definition or undefined */ get(name: string): PerformanceBaselineDefinition | undefined { return this.baselines.get(name) } /** * Gets all registered baselines. * * @returns Map of all baselines */ getAll(): Map { return new Map(this.baselines) } /** * Gets the effective baseline values for the current environment. * * @param name - Baseline name * @returns Merged baseline with environment overrides applied */ getEffective(name: string): PerformanceBaselineDefinition | undefined { const baseline = this.baselines.get(name) if (!baseline) return undefined const envOverrides = baseline.environments?.[this.environment] if (!envOverrides) return baseline return { ...baseline, ...envOverrides, name: baseline.name, // Preserve original name } } /** * Compares timing stats against a baseline. * * @param name - Baseline name * @param stats - Measured timing statistics * @returns Comparison result */ compare(name: string, stats: TimingStats): BaselineComparisonResult { const baseline = this.getEffective(name) if (!baseline) { return { withinBaseline: false, metrics: [], message: `Unknown baseline: ${name}`, } } const toleranceFactor = baseline.toleranceFactor ?? 1.2 const metrics: BaselineComparisonResult['metrics'] = [] // Check p50 (median) if (baseline.p50 !== undefined) { const expected = baseline.p50 const tolerance = expected * toleranceFactor const deviation = stats.median - expected const deviationPercent = (deviation / expected) * 100 metrics.push({ name: 'p50', expected, actual: stats.median, tolerance, passed: stats.median <= tolerance, deviation, deviationPercent, }) } // Check p95 if (baseline.p95 !== undefined) { const expected = baseline.p95 const tolerance = expected * toleranceFactor const deviation = stats.p95 - expected const deviationPercent = (deviation / expected) * 100 metrics.push({ name: 'p95', expected, actual: stats.p95, tolerance, passed: stats.p95 <= tolerance, deviation, deviationPercent, }) } // Check p99 if (baseline.p99 !== undefined) { const expected = baseline.p99 const tolerance = expected * toleranceFactor const deviation = stats.p99 - expected const deviationPercent = (deviation / expected) * 100 metrics.push({ name: 'p99', expected, actual: stats.p99, tolerance, passed: stats.p99 <= tolerance, deviation, deviationPercent, }) } // Check max latency if (baseline.maxLatency !== undefined) { const expected = baseline.maxLatency const tolerance = expected * toleranceFactor const deviation = stats.max - expected const deviationPercent = (deviation / expected) * 100 metrics.push({ name: 'maxLatency', expected, actual: stats.max, tolerance, passed: stats.max <= tolerance, deviation, deviationPercent, }) } const allPassed = metrics.every(m => m.passed) const failedMetrics = metrics.filter(m => !m.passed) let message: string if (allPassed) { message = `All ${metrics.length} metrics within baseline for "${name}"` } else { const failureDetails = failedMetrics .map(m => `${m.name}: ${m.actual.toFixed(2)}ms (expected <=${m.tolerance.toFixed(2)}ms, +${m.deviationPercent.toFixed(1)}%)`) .join(', ') message = `${failedMetrics.length}/${metrics.length} metrics exceeded baseline for "${name}": ${failureDetails}` } return { withinBaseline: allPassed, metrics, message, } } /** * Compares throughput against a baseline. * * @param name - Baseline name * @param throughput - Measured throughput * @returns Comparison result */ compareThroughput(name: string, throughput: ThroughputResult): BaselineComparisonResult { const baseline = this.getEffective(name) if (!baseline) { return { withinBaseline: false, metrics: [], message: `Unknown baseline: ${name}`, } } const metrics: BaselineComparisonResult['metrics'] = [] if (baseline.minThroughput !== undefined) { const expected = baseline.minThroughput // For throughput, we use inverse tolerance (must be >= expected/toleranceFactor) const toleranceFactor = baseline.toleranceFactor ?? 1.2 const tolerance = expected / toleranceFactor const deviation = throughput.opsPerSecond - expected const deviationPercent = (deviation / expected) * 100 metrics.push({ name: 'throughput', expected, actual: throughput.opsPerSecond, tolerance, passed: throughput.opsPerSecond >= tolerance, deviation, deviationPercent, }) } const allPassed = metrics.every(m => m.passed) let message: string if (allPassed) { message = `Throughput within baseline for "${name}": ${throughput.opsPerSecond.toFixed(0)} ops/sec` } else { const failedMetric = metrics[0]! message = `Throughput below baseline for "${name}": ${throughput.opsPerSecond.toFixed(0)} ops/sec (expected >=${failedMetric.tolerance.toFixed(0)} ops/sec)` } return { withinBaseline: allPassed, metrics, message, } } /** * Sets the current environment for baseline lookups. * * @param environment - Environment name */ setEnvironment(environment: string): void { this.environment = environment } /** * Clears all registered baselines. */ clear(): void { this.baselines.clear() } } // ============================================================================ // DEFAULT BASELINES // ============================================================================ /** * Default performance baselines for common postgres.do operations. * These are conservative baselines that should pass in most environments. */ export const DEFAULT_BASELINES: PerformanceBaselineDefinition[] = [ { name: 'simple-select', description: 'Simple SELECT query without table access', p50: 1, p95: 5, p99: 10, maxLatency: 50, toleranceFactor: 2.0, // More tolerance for simple ops environments: { ci: { p50: 5, p95: 20, p99: 50 }, // CI is slower }, }, { name: 'single-row-insert', description: 'Insert a single row into a table', p50: 2, p95: 10, p99: 25, maxLatency: 100, toleranceFactor: 1.5, environments: { ci: { p50: 10, p95: 50, p99: 100 }, }, }, { name: 'batch-insert-100', description: 'Insert 100 rows in a batch', p50: 20, p95: 50, p99: 100, maxLatency: 200, toleranceFactor: 1.5, environments: { ci: { p50: 100, p95: 250, p99: 500 }, }, }, { name: 'indexed-lookup', description: 'Lookup by indexed column', p50: 1, p95: 5, p99: 10, maxLatency: 25, toleranceFactor: 1.5, environments: { ci: { p50: 5, p95: 20, p99: 50 }, }, }, { name: 'full-table-scan-1k', description: 'Full table scan of 1000 rows', p50: 10, p95: 25, p99: 50, maxLatency: 100, toleranceFactor: 1.5, environments: { ci: { p50: 50, p95: 125, p99: 250 }, }, }, { name: 'json-operation', description: 'JSON/JSONB query operation', p50: 2, p95: 10, p99: 20, maxLatency: 50, toleranceFactor: 1.5, environments: { ci: { p50: 10, p95: 50, p99: 100 }, }, }, { name: 'transaction-simple', description: 'Simple transaction with few operations', p50: 5, p95: 15, p99: 30, maxLatency: 75, toleranceFactor: 1.5, environments: { ci: { p50: 25, p95: 75, p99: 150 }, }, }, { name: 'cache-hit', description: 'Cache layer hit operation', p50: 0.1, p95: 0.5, p99: 1, maxLatency: 5, toleranceFactor: 2.0, environments: { ci: { p50: 0.5, p95: 2, p99: 5 }, }, }, { name: 'memory-operation', description: 'In-memory data structure operation', p50: 0.01, p95: 0.1, p99: 0.5, maxLatency: 1, toleranceFactor: 3.0, // High variance expected environments: { ci: { p50: 0.1, p95: 0.5, p99: 2 }, }, }, ] /** * Global baseline registry instance. */ export const globalBaselines = new PerformanceBaseline() // Register default baselines globalBaselines.registerAll(DEFAULT_BASELINES) // ============================================================================ // VITEST CUSTOM MATCHERS // ============================================================================ /** * Custom matcher result type for vitest compatibility. */ interface MatcherResult { pass: boolean message: () => string } /** * Custom matchers for performance testing in vitest. */ export const perfMatchers = { /** * Asserts that timing stats are within a registered baseline. * * @example * expect(stats).toBeWithinBaseline('simple-select') */ toBeWithinBaseline( received: TimingStats, baselineName: string, customBaseline?: Partial ): MatcherResult { let baseline: PerformanceBaseline if (customBaseline) { baseline = new PerformanceBaseline() baseline.register({ name: baselineName, ...customBaseline, }) } else { baseline = globalBaselines } const result = baseline.compare(baselineName, received) return { pass: result.withinBaseline, message: () => result.message, } }, /** * Asserts that timing is faster than a specified threshold. * * @example * expect(stats).toBeFasterThan(10) // p95 < 10ms */ toBeFasterThan(received: TimingStats, maxP95Ms: number): MatcherResult { const pass = received.p95 <= maxP95Ms return { pass, message: () => pass ? `Expected p95 to exceed ${maxP95Ms}ms but was ${received.p95.toFixed(2)}ms` : `Expected p95 to be <= ${maxP95Ms}ms but was ${received.p95.toFixed(2)}ms`, } }, /** * Asserts that timing has low variance (deterministic). * * @example * expect(stats).toHaveLowVariance(0.5) // CV < 0.5 */ toHaveLowVariance(received: TimingStats, maxCV: number = 0.5): MatcherResult { const pass = received.cv <= maxCV return { pass, message: () => pass ? `Expected coefficient of variation to exceed ${maxCV} but was ${received.cv.toFixed(3)}` : `Expected coefficient of variation to be <= ${maxCV} but was ${received.cv.toFixed(3)} (high variance detected)`, } }, /** * Asserts that throughput meets minimum requirements. * * @example * expect(throughput).toMeetThroughput(1000) // >= 1000 ops/sec */ toMeetThroughput(received: ThroughputResult, minOpsPerSec: number): MatcherResult { const pass = received.opsPerSecond >= minOpsPerSec return { pass, message: () => pass ? `Expected throughput to be below ${minOpsPerSec} ops/sec but was ${received.opsPerSecond.toFixed(0)} ops/sec` : `Expected throughput to be >= ${minOpsPerSec} ops/sec but was ${received.opsPerSecond.toFixed(0)} ops/sec`, } }, /** * Asserts that a single timing value is within range. * * @example * expect(latency).toBeWithinRange(1, 10) // 1ms <= latency <= 10ms */ toBeWithinRange(received: number, minMs: number, maxMs: number): MatcherResult { const pass = received >= minMs && received <= maxMs return { pass, message: () => pass ? `Expected ${received.toFixed(2)}ms to be outside range [${minMs}, ${maxMs}]ms` : `Expected ${received.toFixed(2)}ms to be within range [${minMs}, ${maxMs}]ms`, } }, } /** * Type declarations for custom matchers. * This extends vitest's expect interface. */ declare module 'vitest' { // eslint-disable-next-line @typescript-eslint/no-unused-vars interface Assertion { toBeWithinBaseline( baselineName: string, customBaseline?: Partial ): this toBeFasterThan(maxP95Ms: number): this toHaveLowVariance(maxCV?: number): this toMeetThroughput(minOpsPerSec: number): this toBeWithinRange(minMs: number, maxMs: number): this } interface AsymmetricMatchersContaining { toBeWithinBaseline( baselineName: string, customBaseline?: Partial ): unknown toBeFasterThan(maxP95Ms: number): unknown toHaveLowVariance(maxCV?: number): unknown toMeetThroughput(minOpsPerSec: number): unknown toBeWithinRange(minMs: number, maxMs: number): unknown } } /** * Sets up performance matchers in vitest. * Call this in your test setup file or at the beginning of performance tests. * * @param expect - The vitest expect function * * @example * ```typescript * import { expect } from 'vitest' * import { setupPerfMatchers } from '@dotdo/postgres-shared/perf-test-utils' * * // In setup file or before tests * setupPerfMatchers(expect) * ``` */ export function setupPerfMatchers( // eslint-disable-next-line @typescript-eslint/no-explicit-any expect: { extend: (matchers: Record MatcherResult>) => void } ): void { expect.extend(perfMatchers) } // ============================================================================ // UTILITY FUNCTIONS // ============================================================================ /** * Creates a simple delay for testing purposes. * * @param ms - Milliseconds to delay */ export function delay(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)) } /** * Formats timing stats as a human-readable string. * * @param stats - Timing statistics to format * @returns Formatted string */ export function formatStats(stats: TimingStats): string { return [ `Samples: ${stats.count}`, `Min: ${stats.min.toFixed(3)}ms`, `Max: ${stats.max.toFixed(3)}ms`, `Mean: ${stats.mean.toFixed(3)}ms`, `Median (p50): ${stats.median.toFixed(3)}ms`, `p95: ${stats.p95.toFixed(3)}ms`, `p99: ${stats.p99.toFixed(3)}ms`, `StdDev: ${stats.stdDev.toFixed(3)}ms`, `CV: ${stats.cv.toFixed(3)}`, ].join('\n') } /** * Formats throughput result as a human-readable string. * * @param result - Throughput result to format * @returns Formatted string */ export function formatThroughput(result: ThroughputResult): string { return [ `Throughput: ${result.opsPerSecond.toFixed(2)} ops/sec`, `Total Operations: ${result.totalOps}`, `Duration: ${result.durationMs.toFixed(2)}ms`, `Avg Op Time: ${result.avgOpTimeMs.toFixed(3)}ms`, ].join('\n') } /** * Runs a performance test with automatic baseline comparison. * * @param name - Test name (used as baseline key) * @param fn - Function to measure * @param options - Measurement options * @returns Object with stats and comparison result */ export async function runPerfTest( name: string, fn: () => void | Promise, options: MeasureTimingOptions = {} ): Promise<{ stats: TimingStats; comparison: BaselineComparisonResult }> { const stats = await measureTiming(fn, options) const comparison = globalBaselines.compare(name, stats) return { stats, comparison } } /** * Asserts performance is within baseline, throwing if not. * Useful for non-vitest environments or simple assertions. * * @param name - Baseline name * @param stats - Measured statistics * @throws Error if performance exceeds baseline */ export function assertWithinBaseline(name: string, stats: TimingStats): void { const comparison = globalBaselines.compare(name, stats) if (!comparison.withinBaseline) { throw new Error(`Performance regression detected: ${comparison.message}`) } } // ============================================================================ // REGRESSION DETECTION // ============================================================================ /** * Configuration for regression detection. */ export interface RegressionDetectionConfig { /** Minimum number of samples required for valid comparison */ minSamples: number /** Tolerance factor for accepting variance */ toleranceFactor: number /** Whether to fail on high variance (flaky tests) */ failOnHighVariance: boolean /** Maximum acceptable coefficient of variation */ maxCV: number } /** * Default regression detection configuration. */ export const DEFAULT_REGRESSION_CONFIG: RegressionDetectionConfig = { minSamples: 10, toleranceFactor: 1.2, failOnHighVariance: true, maxCV: 0.5, } /** * Detects performance regression between two timing stats. * * @param baseline - Baseline (historical) timing stats * @param current - Current timing stats * @param config - Detection configuration * @returns Regression detection result */ export function detectRegression( baseline: TimingStats, current: TimingStats, config: Partial = {} ): { regressed: boolean improved: boolean stable: boolean details: { p50Change: number p95Change: number p99Change: number varianceIssue: boolean } message: string } { const effectiveConfig = { ...DEFAULT_REGRESSION_CONFIG, ...config } // Check for sufficient samples if (baseline.count < effectiveConfig.minSamples || current.count < effectiveConfig.minSamples) { return { regressed: false, improved: false, stable: false, details: { p50Change: 0, p95Change: 0, p99Change: 0, varianceIssue: false, }, message: `Insufficient samples: baseline=${baseline.count}, current=${current.count}, required=${effectiveConfig.minSamples}`, } } // Calculate changes const p50Change = ((current.median - baseline.median) / baseline.median) * 100 const p95Change = ((current.p95 - baseline.p95) / baseline.p95) * 100 const p99Change = ((current.p99 - baseline.p99) / baseline.p99) * 100 // Check variance const varianceIssue = effectiveConfig.failOnHighVariance && (current.cv > effectiveConfig.maxCV || baseline.cv > effectiveConfig.maxCV) // Determine regression threshold const regressionThreshold = (effectiveConfig.toleranceFactor - 1) * 100 // e.g., 20% for factor 1.2 const improvementThreshold = -10 // 10% improvement is significant const regressed = p95Change > regressionThreshold const improved = p95Change < improvementThreshold const stable = !regressed && !improved && !varianceIssue let message: string if (varianceIssue) { message = `High variance detected: baseline CV=${baseline.cv.toFixed(3)}, current CV=${current.cv.toFixed(3)}` } else if (regressed) { message = `Performance regression: p95 increased by ${p95Change.toFixed(1)}% (threshold: ${regressionThreshold.toFixed(1)}%)` } else if (improved) { message = `Performance improved: p95 decreased by ${Math.abs(p95Change).toFixed(1)}%` } else { message = `Performance stable: p95 change ${p95Change.toFixed(1)}%` } return { regressed, improved, stable, details: { p50Change, p95Change, p99Change, varianceIssue, }, message, } }