/** * Benchmarking library. * * @example * ```ts * import {Benchmark} from '@fuzdev/fuz_util/benchmark.js'; * * const bench = new Benchmark({ * duration_ms: 5000, * warmup_iterations: 5, * }); * * bench * .add('slugify', () => slugify(title)) * .add('slugify_slower', () => slugify_slower(title)); * * const results = await bench.run(); * console.log(bench.table()); * ``` * * @module */ import {is_promise, wait} from './async.js'; import {BenchmarkStats} from './benchmark_stats.js'; import {timer_default, time_unit_detect_best, time_format} from './time.js'; import { benchmark_format_table, benchmark_format_table_grouped, benchmark_format_markdown, benchmark_format_markdown_grouped, benchmark_format_json, benchmark_format_number, type BenchmarkFormatJsonOptions, } from './benchmark_format.js'; import type { BenchmarkConfig, BenchmarkTask, BenchmarkResult, BenchmarkFormatTableOptions, } from './benchmark_types.js'; // Default configuration values const DEFAULT_DURATION_MS = 1000; const DEFAULT_WARMUP_ITERATIONS = 10; const DEFAULT_COOLDOWN_MS = 100; // 30 (not 10) so Welch's-t DOF approximation is stable on the floor case // — a slow function that hits `min_iterations` exactly on both baseline and // current sides. At n=10 the DOF is noisy and a single tail outlier can swing // the p-value across the significance threshold; the system would then report // "regression" with confidence the math doesn't support. 30 is the standard // "central limit theorem starts behaving" sample size for this kind of test. const DEFAULT_MIN_ITERATIONS = 30; const DEFAULT_MAX_ITERATIONS = 100_000; /** * Validate benchmark configuration. * @throws Error if configuration values are out of range or `min_iterations` exceeds `max_iterations` */ const validate_config = (config: BenchmarkConfig): void => { if (config.duration_ms !== undefined && config.duration_ms <= 0) { throw new Error(`duration_ms must be positive, got ${config.duration_ms}`); } if (config.warmup_iterations !== undefined && config.warmup_iterations < 0) { throw new Error(`warmup_iterations must be non-negative, got ${config.warmup_iterations}`); } if (config.cooldown_ms !== undefined && config.cooldown_ms < 0) { throw new Error(`cooldown_ms must be non-negative, got ${config.cooldown_ms}`); } if (config.min_iterations !== undefined && config.min_iterations < 1) { throw new Error(`min_iterations must be at least 1, got ${config.min_iterations}`); } if (config.max_iterations !== undefined && config.max_iterations < 1) { throw new Error(`max_iterations must be at least 1, got ${config.max_iterations}`); } if ( config.min_iterations !== undefined && config.max_iterations !== undefined && config.min_iterations > config.max_iterations ) { throw new Error( `min_iterations (${config.min_iterations}) cannot exceed max_iterations (${config.max_iterations})`, ); } }; /** * Validate a task's per-task time-budget overrides against the already-resolved * suite config. Cross-field min/max is checked using effective values (task * field if set, otherwise suite default) so that e.g. a task raising * `min_iterations` past the suite's `max_iterations` errors at `add()` time. * * @throws Error if any override is out of range or effective min exceeds effective max */ const validate_task = ( task: BenchmarkTask, suite: {min_iterations: number; max_iterations: number}, ): void => { if (task.duration_ms !== undefined && task.duration_ms <= 0) { throw new Error(`task "${task.name}" duration_ms must be positive, got ${task.duration_ms}`); } if (task.warmup_iterations !== undefined && task.warmup_iterations < 0) { throw new Error( `task "${task.name}" warmup_iterations must be non-negative, got ${task.warmup_iterations}`, ); } if (task.min_iterations !== undefined && task.min_iterations < 1) { throw new Error( `task "${task.name}" min_iterations must be at least 1, got ${task.min_iterations}`, ); } if (task.max_iterations !== undefined && task.max_iterations < 1) { throw new Error( `task "${task.name}" max_iterations must be at least 1, got ${task.max_iterations}`, ); } if (task.min_iterations !== undefined || task.max_iterations !== undefined) { const effective_min = task.min_iterations ?? suite.min_iterations; const effective_max = task.max_iterations ?? suite.max_iterations; if (effective_min > effective_max) { throw new Error( `task "${task.name}" effective min_iterations (${effective_min}) cannot exceed effective max_iterations (${effective_max})`, ); } } }; /** * Warmup function by running it multiple times. * Detects whether the function is async based on return value. * * When no `async_hint` is provided, at least one detection iteration runs even * if `iterations` is 0 — otherwise async detection would be impossible and * async functions would have their returned promises leaked as unhandled. * * @param fn - function to warmup (sync or async) * @param async_hint - if provided, use this instead of detecting * @returns whether the function is async * * @example * ```ts * const is_async = await benchmark_warmup(() => expensive_operation(), 10); * ``` */ export const benchmark_warmup = async ( fn: () => unknown, iterations: number, async_hint?: boolean, ): Promise => { // If we have an explicit hint, use it if (async_hint !== undefined) { // Still run warmup iterations for JIT for (let i = 0; i < iterations; i++) { const result = fn(); if (async_hint && is_promise(result)) { await result; } } return async_hint; } // Detect on first iteration. Force at least one call even if iterations=0, // otherwise async fns are silently misclassified as sync (timings would // measure only the synchronous prelude, and the returned promise would // leak as unhandled). const total = iterations < 1 ? 1 : iterations; let detected_async = false; for (let i = 0; i < total; i++) { const result = fn(); if (i === 0) { detected_async = is_promise(result); } if (detected_async && is_promise(result)) { await result; } } return detected_async; }; /** * Benchmark class for measuring and comparing function performance. */ export class Benchmark { readonly #config: Required> & Pick; readonly #tasks: Array = []; #results: Array = []; constructor(config: BenchmarkConfig = {}) { validate_config(config); this.#config = { duration_ms: config.duration_ms ?? DEFAULT_DURATION_MS, warmup_iterations: config.warmup_iterations ?? DEFAULT_WARMUP_ITERATIONS, cooldown_ms: config.cooldown_ms ?? DEFAULT_COOLDOWN_MS, min_iterations: config.min_iterations ?? DEFAULT_MIN_ITERATIONS, max_iterations: config.max_iterations ?? DEFAULT_MAX_ITERATIONS, timer: config.timer ?? timer_default, on_iteration: config.on_iteration, on_task_complete: config.on_task_complete, }; } /** * Add a benchmark task. * @param name - task name or full task object * @param fn - Function to benchmark (if name is string). Return values are ignored. * @returns this `Benchmark` instance for chaining * @throws Error if a task with the same name already exists, or if `fn` is missing when `name` is a string * * @example * ```ts * bench.add('simple', () => fn()); * * // Or with setup/teardown: * bench.add({ * name: 'with setup', * fn: () => process(data), * setup: () => { data = load() }, * teardown: () => { cleanup() }, * }); * ``` */ add(name: string, fn: () => unknown): this; add(task: BenchmarkTask): this; add(name_or_task: string | BenchmarkTask, fn?: () => unknown): this { const task_name = typeof name_or_task === 'string' ? name_or_task : name_or_task.name; // Validate unique task names if (this.#tasks.some((t) => t.name === task_name)) { throw new Error(`Task "${task_name}" already exists`); } let task: BenchmarkTask; if (typeof name_or_task === 'string') { if (!fn) throw new Error('Function required when name is string'); task = {name: name_or_task, fn}; } else { task = name_or_task; } validate_task(task, this.#config); this.#tasks.push(task); return this; } /** * Remove a benchmark task by name. * @returns this `Benchmark` instance for chaining * @throws Error if task with given name doesn't exist * * @example * ```ts * bench.add('task1', () => fn1()); * bench.add('task2', () => fn2()); * bench.remove('task1'); * // Only task2 remains * ``` */ remove(name: string): this { const index = this.#tasks.findIndex((t) => t.name === name); if (index === -1) { throw new Error(`Task "${name}" not found`); } this.#tasks.splice(index, 1); return this; } /** * Run all benchmark tasks. * * Tasks execute in `add()` order. The first task runs against a colder * runtime than subsequent ones (uncompiled JS, cold caches) — a * property of in-process benchmarking that matters more when an early * task has aggressive overrides like low `warmup_iterations` or * `min_iterations`. If first-position bias is a concern, put a * throwaway warm-up task first, or call `run()` twice and use the * second result set. */ async run(): Promise> { this.#results = []; // Determine which tasks to run const has_only = this.#tasks.some((t) => t.only); const tasks_to_run = this.#tasks.filter((t) => { if (t.skip) return false; if (has_only) return t.only; return true; }); for (let i = 0; i < tasks_to_run.length; i++) { const task = tasks_to_run[i]!; const result = await this.#run_task(task); this.#results.push(result); // Call on_task_complete callback this.#config.on_task_complete?.(result, i, tasks_to_run.length); // Cooldown between tasks (skip after last task) if (this.#config.cooldown_ms > 0 && i < tasks_to_run.length - 1) { await wait(this.#config.cooldown_ms); } } return this.#results; } /** * Run a single benchmark task. * @throws Error if the task fails during setup, warmup, or measurement */ async #run_task(task: BenchmarkTask): Promise { // Per-task overrides shadow the suite config. Validation at `add()` time // guarantees effective min_iterations <= effective max_iterations. const duration_ms = task.duration_ms ?? this.#config.duration_ms; const warmup_iterations = task.warmup_iterations ?? this.#config.warmup_iterations; const min_iterations = task.min_iterations ?? this.#config.min_iterations; const max_iterations = task.max_iterations ?? this.#config.max_iterations; // Locals to avoid per-iteration property lookups in the un-optimized path. // `timer` and `on_iteration` come from the readonly `#config`, so they're // stable for the suite's lifetime. `fn` and `name` are captured below // after `setup()` runs, so setup is still allowed to mutate them. const {timer, on_iteration} = this.#config; const suite_start_ns = timer.now(); // Pre-allocate array to avoid GC pressure during measurement const timings_ns: Array = new Array(max_iterations); let timing_count = 0; // Resolved during warmup inside the try block, surfaced into // `result.budget` after the loop. Declared out here so it survives the // `try/finally` scope — the value is part of the result, not just // loop-local state. let async_resolved = false; try { // Setup if (task.setup) { await task.setup(); } // Capture after setup so setup can still mutate task.fn / task.name. const {fn, name} = task; // Warmup and detect async. The resolved boolean (not `task.async`) is // what the loop actually used; we persist it into `result.budget` // below so baseline comparison can detect an async-classification // flip between runs as methodology drift. async_resolved = await benchmark_warmup(fn, warmup_iterations, task.async); // Measurement phase const target_time_ns = duration_ms * 1_000_000; // Convert ms to ns let aborted = false as boolean; // widen — closure-only assignment doesn't widen for CFA const abort = (): void => { aborted = true; }; const measurement_start_ns = timer.now(); // Use separate code paths for sync vs async for better performance if (async_resolved) { // Async code path - await each iteration // eslint-disable-next-line no-unmodified-loop-condition while (timing_count < max_iterations && !aborted) { const iter_start_ns = timer.now(); await fn(); const iter_end_ns = timer.now(); timings_ns[timing_count++] = iter_end_ns - iter_start_ns; on_iteration?.(name, timing_count, abort); const total_elapsed_ns = iter_end_ns - measurement_start_ns; if (timing_count >= min_iterations && total_elapsed_ns >= target_time_ns) { break; } } } else { // Sync code path - no promise checking overhead // eslint-disable-next-line no-unmodified-loop-condition while (timing_count < max_iterations && !aborted) { const iter_start_ns = timer.now(); fn(); const iter_end_ns = timer.now(); timings_ns[timing_count++] = iter_end_ns - iter_start_ns; on_iteration?.(name, timing_count, abort); const total_elapsed_ns = iter_end_ns - measurement_start_ns; if (timing_count >= min_iterations && total_elapsed_ns >= target_time_ns) { break; } } } } finally { // Always run teardown if (task.teardown) { await task.teardown(); } } // Right-size: `length =` doesn't reliably shrink V8's backing store, so // we copy into a packed array and let the over-allocated original GC. const trimmed_timings = timings_ns.slice(0, timing_count); const suite_end_ns = timer.now(); const total_time_ms = (suite_end_ns - suite_start_ns) / 1_000_000; // Convert back to ms for display // Analyze results const stats = new BenchmarkStats(trimmed_timings); return { name: task.name, stats, iterations: timing_count, total_time_ms, timings_ns: trimmed_timings, budget: { duration_ms, warmup_iterations, min_iterations, max_iterations, async_resolved, }, }; } /** * Format results as an ASCII table with percentiles, min/max, and relative performance. * * @example * ```ts * // Standard table * console.log(bench.table()); * * // Grouped by category * console.log(bench.table({ * groups: [ * { name: 'FAST PATHS', filter: (r) => r.name.includes('fast') }, * { name: 'SLOW PATHS', filter: (r) => r.name.includes('slow') }, * ] * })); * ``` */ table(options?: BenchmarkFormatTableOptions): string { return options?.groups ? benchmark_format_table_grouped(this.#results, options.groups) : benchmark_format_table(this.#results); } /** * Format results as a Markdown table. * @param options - formatting options (groups for organized output with optional baselines) * @returns formatted markdown string * * @example * ```ts * // Standard table * console.log(bench.markdown()); * * // Grouped by category with custom baseline * console.log(bench.markdown({ * groups: [ * { name: 'Format', filter: (r) => r.name.startsWith('format/'), baseline: 'format/prettier' }, * { name: 'Parse', filter: (r) => r.name.startsWith('parse/') }, * ] * })); * ``` */ markdown(options?: BenchmarkFormatTableOptions): string { return options?.groups ? benchmark_format_markdown_grouped(this.#results, options.groups) : benchmark_format_markdown(this.#results); } /** * Format results as JSON. * @param options - formatting options (pretty, include_timings) * @returns JSON string */ json(options?: BenchmarkFormatJsonOptions): string { return benchmark_format_json(this.#results, options); } /** * Get the benchmark results. * @returns shallow copy of the results array (prevents external mutation) */ results(): Array { return [...this.#results]; } /** * Check if the benchmark has been run and has results. * @returns true if results are available * * @example * ```ts * if (bench.has_results) { * console.log(bench.table()); * } * ``` */ get has_results(): boolean { return this.#results.length > 0; } /** * Get results as a map for convenient lookup by task name. * @returns fresh `Map` of task name to benchmark result (prevents external mutation) * * @example * ```ts * const results_map = bench.results_by_name(); * const slugify_result = results_map.get('slugify'); * if (slugify_result) { * console.log(`slugify: ${slugify_result.stats.ops_per_second} ops/sec`); * } * ``` */ results_by_name(): Map { return new Map(this.#results.map((r) => [r.name, r])); } /** * Reset the benchmark results. * Keeps tasks intact so benchmarks can be rerun. * @returns this `Benchmark` instance for chaining */ reset(): this { this.#results = []; return this; } /** * Clear everything (results and tasks). * @returns this `Benchmark` instance for chaining */ clear(): this { this.#results = []; this.#tasks.length = 0; return this; } /** * Get a quick text summary of the fastest task. * @returns human-readable summary string * * @example * ```ts * console.log(bench.summary()); * // "Fastest: slugify_v2 (1,285,515.00 ops/sec, 786.52ns per op)" * // "Slowest: slugify (252,955.00 ops/sec, 3.95μs per op)" * // "Speed difference: 5.08x" * ``` */ summary(): string { if (this.#results.length === 0) return 'No results'; const fastest = this.#results.reduce((a, b) => a.stats.ops_per_second > b.stats.ops_per_second ? a : b, ); const slowest = this.#results.reduce((a, b) => a.stats.ops_per_second < b.stats.ops_per_second ? a : b, ); const ratio = fastest.stats.ops_per_second / slowest.stats.ops_per_second; // Detect best unit for consistent display const mean_times = this.#results.map((r) => r.stats.mean_ns); const unit = time_unit_detect_best(mean_times); const lines: Array = []; lines.push( `Fastest: ${fastest.name} (${benchmark_format_number(fastest.stats.ops_per_second)} ops/sec, ${time_format(fastest.stats.mean_ns, unit)} per op)`, ); if (this.#results.length > 1) { lines.push( `Slowest: ${slowest.name} (${benchmark_format_number(slowest.stats.ops_per_second)} ops/sec, ${time_format(slowest.stats.mean_ns, unit)} per op)`, ); lines.push(`Speed difference: ${ratio.toFixed(2)}x`); } return lines.join('\n'); } }