import fs from "node:fs"; import path from "node:path"; import {ForkName} from "@lodestar/params"; import {describeDirectorySpecTest} from "@lodestar/spec-test-util"; import {RunnerType, TestRunner} from "./types.js"; const ARTIFACT_FILENAMES = new Set([ // MacOS artifacts "._.DS_Store", ".DS_Store", // File included by spec tests downloader "version.txt", ]); export interface SkipOpts { skippedPrefixes?: string[]; skippedForks?: string[]; skippedRunners?: string[]; skippedHandlers?: string[]; } /** * This helper ensures that strictly all tests are run. There's no hardcoded value beyond "config". * Any additional unknown fork, testRunner, testHandler, or testSuite will result in an error. * * File path structure: * ``` * tests/ * / [general, mainnet, minimal] * / [phase0, altair, bellatrix] * / [bls, ssz_static, fork] * / ... * / * / * ``` * * Examples * ``` * / config / fork / test runner / test handler / test suite / test case * * tests / general / phase0 / bls / aggregate / small / aggregate_na_signatures/data.yaml * tests / general / phase0 / ssz_generic / basic_vector / valid / vec_bool_1_max/meta.yaml * tests / mainnet / altair / ssz_static / Validator / ssz_random / case_0/roots.yaml * tests / mainnet / altair / fork / fork / pyspec_tests / altair_fork_random_0/meta.yaml * ``` * Ref: https://github.com/ethereum/consensus-specs/tree/dev/tests/formats#test-structure */ export function specTestIterator( configDirpath: string, testRunners: Record, opts?: SkipOpts ): void { for (const forkStr of readdirSyncSpec(configDirpath)) { if (opts?.skippedForks?.includes(forkStr)) { continue; } const fork = ForkName[forkStr as ForkName]; const forkDirpath = path.join(configDirpath, forkStr); for (const testRunnerName of readdirSyncSpec(forkDirpath)) { if (opts?.skippedRunners?.includes(testRunnerName)) { continue; } const testRunnerDirpath = path.join(forkDirpath, testRunnerName); const testRunner = testRunners[testRunnerName]; for (const testHandler of readdirSyncSpec(testRunnerDirpath)) { if (opts?.skippedHandlers?.includes(testHandler)) { continue; } const testHandlerDirpath = path.join(testRunnerDirpath, testHandler); for (const testSuite of readdirSyncSpec(testHandlerDirpath)) { const testId = `${fork}/${testRunnerName}/${testHandler}/${testSuite}`; if (opts?.skippedPrefixes?.some((skippedPrefix) => testId.startsWith(skippedPrefix))) { displaySkipTest(testId); } else if (fork === undefined) { displayFailTest(testId, `Unknown fork ${forkStr}`); } else if (testRunner === undefined) { displayFailTest(testId, `No test runner for ${testRunnerName}`); } else { const testSuiteDirpath = path.join(testHandlerDirpath, testSuite); // Specific logic for ssz_static since it has one extra level of directories if (testRunner.type === RunnerType.custom) { describe(testId, () => { testRunner.fn(fork, testHandler, testSuite, testSuiteDirpath); }); } // Generic testRunner else { const {testFunction, options} = testRunner.fn(fork, testHandler, testSuite); describeDirectorySpecTest(testId, testSuiteDirpath, testFunction, options); } } } } } } } function displayFailTest(testId: string, msg: string): void { describe(testId, () => { it(testId, () => { throw Error(msg); }); }); } function displaySkipTest(testId: string): void { describe(testId, () => { it.skip(testId, () => { // }); }); } export function readdirSyncSpec(dirpath: string): string[] { const files = fs.readdirSync(dirpath); return files.filter((file) => !ARTIFACT_FILENAMES.has(file)); }