import { fileURLToPath } from 'url'; import path from 'path'; import semver from 'semver'; import fs from 'fs'; import { CoverageData, INSTRUMENTER_CONSTANTS, MutantCoverage, StrykerOptions, } from '@stryker-mutator/api/core'; import { Logger } from '@stryker-mutator/api/logging'; import { commonTokens, Injector, PluginContext, tokens, } from '@stryker-mutator/api/plugin'; import { TestRunner, DryRunResult, MutantRunOptions, MutantRunResult, TestRunnerCapabilities, DryRunStatus, toMutantRunResult, determineHitLimitReached, TestStatus, DryRunOptions, } from '@stryker-mutator/api/test-runner'; import { errorToString, escapeRegExp, normalizeFileName, notEmpty, testFilesProvided, } from '@stryker-mutator/util'; import { vitestWrapper, Vitest } from './vitest-wrapper.js'; import { VITEST_MODERN_TEST_NAME_SEPARATOR, VITEST_LEGACY_TEST_NAME_SEPARATOR, } from './test-helpers.js'; import { convertTestToTestResult, fromTestId, collectTestsFromSuite, normalizeCoverage, isErrorCodeError, VITEST_ERROR_CODES, } from './vitest-helpers.js'; import { VitestRunnerOptionsWithStrykerOptions } from './vitest-runner-options-with-stryker-options.js'; type StrykerNamespace = '__stryker__' | '__stryker2__'; const STRYKER_SETUP = fileURLToPath( new URL('./stryker-setup.js', import.meta.url), ); interface RunFilter { /** * Run only tests with the specified IDs */ testIds?: string[]; /** * Run only tests that cover a list of source files * @see https://vitest.dev/guide/cli.html#vitest-related */ relatedFiles?: string[]; /** * Run only tests from the specified test files (absolute paths) */ testFiles?: string[]; } export class VitestTestRunner implements TestRunner { public static inject = [ commonTokens.options, commonTokens.logger, 'globalNamespace', ] as const; private ctx?: Vitest; private readonly options: VitestRunnerOptionsWithStrykerOptions; private localSetupFile = path.resolve( `./stryker-setup-${process.env.STRYKER_MUTATOR_WORKER ?? 0}.js`, ); constructor( options: StrykerOptions, private readonly log: Logger, private globalNamespace: StrykerNamespace, ) { this.options = options as VitestRunnerOptionsWithStrykerOptions; } get #testNameSeparator(): string { return semver.gte(vitestWrapper.version, '5.0.0-0') ? VITEST_MODERN_TEST_NAME_SEPARATOR : VITEST_LEGACY_TEST_NAME_SEPARATOR; } public capabilities(): TestRunnerCapabilities { return { reloadEnvironment: true }; } #getVitestPoolConfig(version: string) { if (semver.lt(version, '1.0.0')) { switch (this.options.vitest.pool) { case 'threads': return { threads: true, maxThreads: 1, minThreads: 1, }; case 'forks': return { threads: false, singleThread: true, }; case 'vmThreads': return { experimentalVmThreads: true, maxThreads: 1, minThreads: 1, }; case 'vmForks': throw new Error( 'The vmForks pool is not supported by Vitest versions below 1.0.0', ); } } if (semver.lt(version, '4.0.0')) { switch (this.options.vitest.pool) { case 'forks': return { pool: 'forks', poolOptions: { forks: { maxForks: 1, minForks: 1, }, }, }; case 'vmForks': return { pool: 'vmForks', poolOptions: { vmForks: { maxForks: 1, minForks: 1, }, }, }; case 'threads': return { pool: 'threads', poolOptions: { threads: { maxThreads: 1, minThreads: 1, }, }, }; case 'vmThreads': return { pool: 'vmThreads', poolOptions: { vmThreads: { maxThreads: 1, minThreads: 1, }, }, }; } } return { pool: this.options.vitest.pool, maxWorkers: 1, }; } /** * Write the Stryker Setup file without sourceMappingURL * since we don't need a sourcemap for it, but we do want Vite * to handle sourcemaps for other files. * Removing the sourceMappingURL will prevent Vite from warning * that the setup file has no sourceMappingURL and prevents * it from performing unnecessary work. */ async #writeStrykerSetupFile() { const setupFileContents = await fs.promises.readFile(STRYKER_SETUP, 'utf8'); const lines = setupFileContents.split('\n'); // The last non-empty line is always a sourceMappingURL comment, remove it const lastLineIndex = lines.at(-1) === '' ? lines.length - 2 : lines.length - 1; lines.splice(lastLineIndex, 1); await fs.promises.writeFile(this.localSetupFile, lines.join('\n')); } public async init(): Promise { this.setEnv(); await this.#writeStrykerSetupFile(); this.ctx = await vitestWrapper.createVitest('test', { config: this.options.vitest?.configFile, ...this.#getVitestPoolConfig(vitestWrapper.version), coverage: { enabled: false }, // Incremental mode diffs tests by location. includeTaskLocation: true, maxConcurrency: 1, watch: false, dir: this.options.vitest.dir, bail: this.options.disableBail ? 0 : 1, onConsoleLog: () => false, }); this.ctx.provide('globalNamespace', this.globalNamespace); this.ctx.provide( 'isGreaterThanVitest4Point1', semver.gte(vitestWrapper.version, '4.1.0'), ); this.ctx.provide('testNameSeparator', this.#testNameSeparator); this.ctx.config.browser.screenshotFailures = false; this.ctx.projects.forEach((project) => { project.config.setupFiles = [ this.localSetupFile, ...project.config.setupFiles, ]; project.config.browser.screenshotFailures = false; }); if (this.log.isDebugEnabled()) { this.log.debug( `vitest final config: ${JSON.stringify(this.ctx.config, null, 2)}`, ); } } public async dryRun(options: DryRunOptions): Promise { this.ctx!.provide('mode', 'dry-run'); // If testFilter is provided, use those files directly instead of relying on related files // We still need to pass relatedFiles for vitest to properly resolve the test files const testResult = testFilesProvided(options) ? await this.run({ testFiles: options.testFiles, relatedFiles: options.files, }) : await this.run({ relatedFiles: options.files }); if ( testResult.status === DryRunStatus.Complete && testResult.tests.length === 0 && this.options.vitest.related && !options.testFiles ) { this.log.warn( 'Vitest failed to find test files related to mutated files. Either disable `vitest.related` or import your source files directly from your test files. See https://stryker-mutator.io/docs/stryker-js/troubleshooting/#vitest-failed-to-find-test-files-related-to-mutated-files', ); } const mutantCoverage = this.readMutantCoverage(); if (testResult.status === DryRunStatus.Complete) { return { status: testResult.status, tests: testResult.tests, mutantCoverage, }; } return testResult; } public async mutantRun(options: MutantRunOptions): Promise { this.ctx!.provide('mode', 'mutant'); this.ctx!.provide('hitLimit', options.hitLimit); this.ctx!.provide('mutantActivation', options.mutantActivation); this.ctx!.provide('activeMutant', options.activeMutant.id); const dryRunResult = await this.run({ testIds: options.testFilter, relatedFiles: [options.sandboxFileName], }); const hitCount = this.readHitCount(); const timeOut = determineHitLimitReached(hitCount, options.hitLimit); return toMutantRunResult(timeOut ?? dryRunResult); } private async run({ testIds = [], relatedFiles, testFiles: explicitTestFiles, }: RunFilter = {}): Promise { this.resetContext(); this.ctx!.config.related = this.options.vitest.related && relatedFiles ? relatedFiles.map(normalizeFileName) : undefined; let testFilesToRun: string[] | undefined = explicitTestFiles; if (testIds.length > 0) { const parsedTests = testIds.map(fromTestId); // Anchor the end of the pattern: it is matched against the full test name, so without it // a filter for "calc add" would also select "calc add negative". The start can't be anchored, // as vitest prefixes the name. Allow trailing whitespace: test ids have it trimmed, the full name doesn't. const regexTestNameFilter = parsedTests .map(({ test: name }) => escapeRegExp(name)) .join('|'); const regex = new RegExp(`(?:${regexTestNameFilter})\\s*$`); testFilesToRun = parsedTests.map(({ file }) => file); this.ctx!.projects.forEach((project) => { project.config.testNamePattern = regex; }); } else { this.ctx!.projects.forEach((project) => { project.config.testNamePattern = undefined; }); } try { await this.ctx!.start(testFilesToRun); } catch (error) { if ( // No tests found, this isn't a problem, we can continue !isErrorCodeError(error) || VITEST_ERROR_CODES.FILES_NOT_FOUND !== error.code ) { throw error; } } const tests = this.ctx!.state.getFiles() .flatMap((file) => collectTestsFromSuite(file)) .filter((test) => test.result); // if no result: it was skipped because of bail let failure = false; const testResults = tests.map((test) => { const testResult = convertTestToTestResult(test, this.#testNameSeparator); failure ||= testResult.status === TestStatus.Failed; return testResult; }); // Collection errors are stored on the file, not in errorsSet, and may // leave no individual tests to report the failure. const fileErrors = this.ctx!.state.getFiles() .filter((file) => file.result?.state === 'fail') .map( (file) => `${file.filepath}: ${file.result?.errors?.map((error) => error.message).join('\n') || 'StrykerJS: Test file execution failed'}`, ); if ( !failure && (fileErrors.length > 0 || this.ctx!.state.errorsSet.size > 0) ) { const errorText = [ ...fileErrors, ...[...this.ctx!.state.errorsSet].map(errorToString), ].join('\n'); return { status: DryRunStatus.Error, errorMessage: `An error occurred outside of a test run: ${errorText}`, }; } return { tests: testResults, status: DryRunStatus.Complete }; } private setEnv() { // Set node environment for issues like these: https://github.com/stryker-mutator/stryker-js/issues/4289 process.env.NODE_ENV = 'test'; // Set vitest environment to signal that we are running in vitest // as some plugins only initiate when this is set: https://github.com/testing-library/svelte-testing-library/blob/6096f05e805cf55474f52f303562f4013785d25f/src/vite.js#L20 process.env.VITEST = '1'; } private resetContext() { // Clear the state from the previous run // Note that this is kind of a hack, see https://github.com/vitest-dev/vitest/discussions/3017#discussioncomment-5901751 this.ctx!.state.filesMap.clear(); } private readHitCount() { const hitCounters: number[] = this.ctx!.state.getFiles() .map((file) => (file.meta as { hitCount?: number }).hitCount) .filter(notEmpty); return hitCounters.reduce((acc, hitCount) => acc + hitCount, 0); } private readMutantCoverage(): MutantCoverage { // Read coverage from all projects const coverages: MutantCoverage[] = [ ...new Map( this.ctx!.state.getFiles().map( (file) => [`${file.projectName}-${file.name}`, file] as const, ), ).entries(), ] .map( ([, file]) => (file.meta as { mutantCoverage?: MutantCoverage }).mutantCoverage, ) .filter(notEmpty) .map(normalizeCoverage); if (coverages.length > 1) { return coverages.reduce((acc, projectCoverage) => { // perTest contains the coverage per test id Object.entries(projectCoverage.perTest).forEach( ([testId, testCoverage]) => { if (testId in acc.perTest) { // Keys are mutant ids, the numbers are the amount of times it was hit. mergeCoverage(acc.perTest[testId], testCoverage); } else { acc.perTest[testId] = testCoverage; } }, ); mergeCoverage(acc.static, projectCoverage.static); return acc; }); } return coverages[0]; function mergeCoverage(to: CoverageData, from: CoverageData) { Object.entries(from).forEach(([mutantId, hitCount]) => { if (mutantId in to) { to[mutantId] += hitCount; } else { to[mutantId] = hitCount; } }); } } public async dispose(): Promise { this.ctx?.onClose(() => { return fs.promises.rm(this.localSetupFile, { force: true }); }); await this.ctx?.close(); } } export const vitestTestRunnerFactory = createVitestTestRunnerFactory(); export function createVitestTestRunnerFactory( namespace: | typeof INSTRUMENTER_CONSTANTS.NAMESPACE | '__stryker2__' = INSTRUMENTER_CONSTANTS.NAMESPACE, ): { (injector: Injector): VitestTestRunner; inject: ['$injector']; } { createVitestTestRunner.inject = tokens(commonTokens.injector); function createVitestTestRunner(injector: Injector) { return injector .provideValue('globalNamespace', namespace) .injectClass(VitestTestRunner); } return createVitestTestRunner; }