/** * test-report / execute — the runner's accounting, with child_process mocked: * no `npm test` / `dotnet test` runs here. Asserted: the vitest JSON is read * into passed/failed/skipped and every failing test is named by its FULL name * with a non-empty reason and its file (QC 2026-09-24, D5), an unparsable * output is a FAILURE carrying its reason (never the fabricated `passed: 1` of * before — audit 2026-09-24, D-WARN-11), `coverage` is null (never a * measured-looking 0), and the module filter is forwarded verbatim. (Plain * holders, not vi.fn: a spy whose implementation throws gets its error * re-reported by the runner even though execute() catches it.) * * The vitest fixtures under ./fixtures/ are REAL `--reporter=json` output * (vitest 2.1.9, captured 2026-09-24) — never a hand-shaped mimic. The previous * fixture was shaped to the code (one `testResults[]` entry per TEST carrying * `name`/`message`/`file`) where the reporter emits one entry per FILE with * `assertionResults[]`, so this test stayed green while a real red suite * reported the file path as the test name and an empty reason (D5). * Capture: a scratch root holding JourFerieList.test.ts (one passing, one * failing, one `it.skip`), StatusForm.test.ts (two passing) and * AddressBroken.test.ts (throws at import), a `node_modules` junction to this * repo's, then from the repo, one file per run: * npx vitest run --root --reporter=json --outputFile * Only the machine-specific prefixes were replaced: the scratch root by `` * (substituted below with the test's tmp project path) and this repo's * node_modules URL inside the stack frames by `file:////node_modules`. */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' const calls: Array<{ cmd: string; cwd: string }> = [] let execSyncImpl: (cmd: string) => string = () => '' vi.mock('node:child_process', () => ({ execSync: (cmd: string, opts: { cwd: string }) => { calls.push({ cmd, cwd: opts.cwd }) return execSyncImpl(cmd) }, })) import { execute, reportVerdict } from '../execute.js' import { TestReportInputSchema, type TestReportInput } from '../types.js' let tmp: string beforeEach(() => { tmp = mkdtempSync(join(tmpdir(), 'ss-test-report-')) calls.length = 0 execSyncImpl = () => '' }) afterEach(() => { rmSync(tmp, { recursive: true, force: true }) }) const input = (over: Partial = {}): TestReportInput => TestReportInputSchema.parse({ projectPath: tmp, module: 'hrm', includeBackend: false, includeFrontend: true, ...over }) /** A failing execSync: the thrown error carries stdout/stderr like the real one. */ const throwing = (stdout: string, stderr = '') => (): string => { throw Object.assign(new Error('Command failed'), { stdout, stderr, status: 1 }) } /** A real vitest JSON report, its `` re-anchored on `root` (the project dir by default) — POSIX slashes, as the reporter emits. */ const fixture = (name: string, root?: string): string => readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf-8').replace(//g, (root ?? resolve(tmp)).replace(/\\/g, '/')) const RED = 'vitest-json-red.json' // JourFerieList: 1 passed, 1 failed, 1 skipped const GREEN = 'vitest-json-green.json' // StatusForm: 2 passed const IMPORT_CRASH = 'vitest-json-import-crash.json' // AddressBroken: throws at import, 0 tests ran const RED_FAILURE = { test: 'JourFerieList filters by Category', layer: 'frontend', error: "AssertionError: expected [ { category: 'Legal' } ] to have a length of 2 but got 1", file: 'src/pages/JourFerieList.test.ts', } describe('test-report / execute — frontend layer', () => { it('forwards the module filter as `npm test -- --reporter=json `, run in the project dir', () => { execSyncImpl = () => fixture(GREEN) execute(input({ module: 'hrm' })) expect(calls).toHaveLength(1) expect(calls[0].cmd).toBe('npm test -- --reporter=json hrm') expect(calls[0].cwd).toBe(resolve(tmp)) }) it('a RED report: passed/failed/SKIPPED from the counters, the failing test by its FULL name, reason and project-relative file', () => { // FIRES on the D5 parser: it read `testResults[]` as tests → test = the file path, error = '' (the // file-level message), skipped hard-coded to 0. The fixture is the reporter's own output. execSyncImpl = () => fixture(RED) const { report } = execute(input()) expect(report.byLayer['frontend']).toEqual({ total: 3, passed: 1, failed: 1, skipped: 1 }) expect(report.summary).toEqual({ total: 3, passed: 1, failed: 1, skipped: 1 }) expect(report.failures).toEqual([RED_FAILURE]) // the assertion message, not the ` at …` frames vitest appends (failureMessages[] = error.stack) expect(report.failures[0].error).not.toMatch(/^\s+at\s/m) expect(report.failures[0].error.length).toBeGreaterThan(0) expect(report.failures[0].test).not.toContain('/') }) it('an all-green report yields zero failures and the counts of the counters', () => { execSyncImpl = () => fixture(GREEN) const { report } = execute(input()) expect(report.summary).toEqual({ total: 2, passed: 2, failed: 0, skipped: 0 }) expect(report.failures).toEqual([]) }) it('a red suite (vitest exits 1 with its JSON on stdout) KEEPS its counts and names the failing test', () => { // vitest exits 1 whenever one test fails — the report is still on stdout; until 2026-09-24 // the runner landed in its catch and reported {total:1, failed:1} with an empty reason. execSyncImpl = throwing(fixture(RED), '') const { report } = execute(input()) expect(report.byLayer['frontend']).toEqual({ total: 3, passed: 1, failed: 1, skipped: 1 }) expect(report.summary).toEqual({ total: 3, passed: 1, failed: 1, skipped: 1 }) expect(report.failures).toEqual([RED_FAILURE]) }) it('a file that throws at import (0 tests ran, numFailedTests 0, success false) is ONE failure named by the file with the file-level message', () => { // FIRES on the D5 parser and on any per-assertion rewrite that trusts numFailedTests: the reporter // says {numTotalTests:0, numFailedTests:0, success:false} and the only signal is // testResults[].message — a clean {0,0,0,0} layer would ship the red run. execSyncImpl = throwing(fixture(IMPORT_CRASH), '') const { report } = execute(input()) expect(report.byLayer['frontend']).toEqual({ total: 1, passed: 0, failed: 1, skipped: 0 }) expect(report.summary.failed).toBe(1) expect(report.failures).toEqual([ { test: 'src/pages/AddressBroken.test.ts', layer: 'frontend', error: 'AddressBroken: cannot import ./Address (module not found)', file: 'src/pages/AddressBroken.test.ts', }, ]) }) it('a test file OUTSIDE the project keeps its absolute path (POSIX slashes) — never a `../..` chain', () => { const elsewhere = resolve(tmpdir(), 'ss-elsewhere-' + process.pid) execSyncImpl = throwing(fixture(RED, elsewhere), '') const { report } = execute(input()) expect(report.failures).toHaveLength(1) expect(report.failures[0].test).toBe(RED_FAILURE.test) expect(report.failures[0].file).toBe(elsewhere.replace(/\\/g, '/') + '/src/pages/JourFerieList.test.ts') expect(report.failures[0].file).not.toContain('..') }) it('an unparsable output is ONE failed test with the reason — never a fabricated passed: 1', () => { execSyncImpl = () => '\n> app@1.0.0 test\n> vitest run\n\n ✓ src/x.test.tsx (3 tests)\n' const { report } = execute(input()) expect(report.byLayer['frontend']).toEqual({ total: 1, passed: 0, failed: 1, skipped: 0 }) expect(report.summary.passed).toBe(0) expect(report.summary.failed).toBe(1) expect(report.failures).toHaveLength(1) expect(report.failures[0].test).toBe('frontend-suite') expect(report.failures[0].layer).toBe('frontend') expect(report.failures[0].error).toContain('vitest output is not JSON') expect(report.failures[0].error).toContain('npm test -- --reporter=json hrm') expect(report.failures[0].error).toContain('vitest run') }) it('JSON that is not a vitest report (no testResults[]) is ONE failed test with the reason, not an empty green layer', () => { execSyncImpl = () => JSON.stringify({ error: 'ENOENT: no such file or directory, open vitest.config.ts' }) const { report } = execute(input()) expect(report.byLayer['frontend']).toEqual({ total: 1, passed: 0, failed: 1, skipped: 0 }) expect(report.failures[0].test).toBe('frontend-suite') expect(report.failures[0].error).toContain('not a vitest report') expect(report.failures[0].error).toContain('npm test -- --reporter=json hrm') expect(report.failures[0].error).toContain('ENOENT') }) it('a non-zero exit is a failure carrying the child stderr', () => { execSyncImpl = throwing('', 'Error: Cannot find module vitest') const { report } = execute(input()) expect(report.byLayer['frontend']).toEqual({ total: 1, passed: 0, failed: 1, skipped: 0 }) expect(report.summary.passed).toBe(0) expect(report.failures[0]).toMatchObject({ test: 'frontend-suite', layer: 'frontend', error: 'Error: Cannot find module vitest', }) }) it('a non-zero exit with NOTHING on stderr and no JSON still carries a non-empty reason', () => { execSyncImpl = throwing('', '') const { report } = execute(input()) expect(report.byLayer['frontend']).toEqual({ total: 1, passed: 0, failed: 1, skipped: 0 }) expect(report.failures[0].error.length).toBeGreaterThan(0) expect(report.failures[0].error).toContain('npm test -- --reporter=json hrm') }) it('SYNTHETIC safety net: `success: false` with no failed test nor file is still one failure with a reason', () => { // The real reporter was not observed emitting this state (a failed hook marks its tests failed); // the branch exists so that vitest saying "red" can never become a clean layer. execSyncImpl = throwing(JSON.stringify({ numTotalTests: 2, numPassedTests: 2, numFailedTests: 0, success: false, testResults: [] }), '') const { report } = execute(input()) expect(report.byLayer['frontend']).toEqual({ total: 3, passed: 2, failed: 1, skipped: 0 }) expect(report.failures).toHaveLength(1) expect(report.failures[0].test).toBe('frontend-suite') expect(report.failures[0].error).toContain('success flag false') expect(report.failures[0].error).toContain('npm test -- --reporter=json hrm') }) }) describe('test-report / reportVerdict — a run that executed nothing is not a pass (QC 2026-09-24)', () => { it('SILENT — tests ran, none failed → ok', () => { expect(reportVerdict({ total: 12, failed: 0 })).toEqual({ ok: true }) }) it('FIRES — a failure is never ok and names the count', () => { const v = reportVerdict({ total: 12, failed: 2 }) expect(v.ok).toBe(false) expect(v.reason).toBe('2 test(s) failed') }) it('FIRES — 0 tests ran (filter matched nothing on any layer) → not ok, the reason names the filter', () => { const v = reportVerdict({ total: 0, failed: 0 }) expect(v.ok).toBe(false) expect(v.reason).toMatch(/0 tests ran/) expect(v.reason).toMatch(/cannot pass/) }) }) describe('test-report / execute — report contract', () => { it('coverage is null — no tool integrated, never a 0 presented as a measurement', () => { execSyncImpl = () => fixture(GREEN) const { report } = execute(input()) expect(report.coverage).toBeNull() expect(report.coverage).not.toEqual({ line: 0, branch: 0 }) }) it('writes the report under /.smartstack/reports/TEST--.json', () => { execSyncImpl = () => fixture(GREEN) const { report, reportPath } = execute(input({ module: 'hrm' })) expect(reportPath.replace(/\\/g, '/')).toMatch( /\/\.smartstack\/reports\/TEST-hrm-\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.json$/, ) expect(reportPath.replace(/\\/g, '/').startsWith(resolve(tmp).replace(/\\/g, '/'))).toBe(true) expect(existsSync(reportPath)).toBe(true) expect(JSON.parse(readFileSync(reportPath, 'utf-8'))).toEqual(report) expect(report.module).toBe('hrm') }) it('runs nothing when both layers are excluded and reports an empty summary', () => { const { report } = execute(input({ includeBackend: false, includeFrontend: false })) expect(calls).toEqual([]) expect(report.byLayer).toEqual({}) expect(report.summary).toEqual({ total: 0, passed: 0, failed: 0, skipped: 0 }) expect(report.failures).toEqual([]) }) }) describe('test-report / execute — backend layer', () => { it('runs `dotnet test` once per category and reads Passed/Failed/Skipped from the summary line', () => { execSyncImpl = (cmd) => cmd.includes('Category=Domain') ? 'Passed! - Failed: 0, Passed: 7, Skipped: 1, Total: 8' : 'Passed! - Failed: 0, Passed: 2, Skipped: 0, Total: 2' const { report } = execute(input({ includeBackend: true, includeFrontend: false })) expect(calls.map((c) => c.cmd)).toEqual([ 'dotnet test --no-build --filter Category=Seed --verbosity normal --logger "trx;LogFileName=Seed.trx"', 'dotnet test --no-build --filter Category=Domain --verbosity normal --logger "trx;LogFileName=Domain.trx"', 'dotnet test --no-build --filter Category=Business --verbosity normal --logger "trx;LogFileName=Business.trx"', 'dotnet test --no-build --filter Category=Integration --verbosity normal --logger "trx;LogFileName=Integration.trx"', ]) expect(calls.every((c) => c.cwd === resolve(tmp))).toBe(true) expect(Object.keys(report.byLayer)).toEqual(['seed', 'domain', 'business', 'integration']) expect(report.byLayer['domain']).toEqual({ total: 8, passed: 7, failed: 0, skipped: 1 }) expect(report.summary).toEqual({ total: 14, passed: 13, failed: 0, skipped: 1 }) }) it('a failing dotnet run (non-zero exit) keeps the counts from stdout and names the failed test', () => { execSyncImpl = (cmd) => cmd.includes('Category=Business') ? throwing( ' Failed Hrm.Tests.Business.EmployeeServiceTests.Create_rejects_duplicate [12 ms]\n' + ' Error Message:\n Assert.Equal() Failure\n' + 'Failed! - Failed: 1, Passed: 4, Skipped: 0, Total: 5', )() : 'Passed! - Failed: 0, Passed: 1, Skipped: 0, Total: 1' const { report } = execute(input({ includeBackend: true, includeFrontend: false })) expect(report.byLayer['business']).toEqual({ total: 5, passed: 4, failed: 1, skipped: 0 }) expect(report.summary).toEqual({ total: 8, passed: 7, failed: 1, skipped: 0 }) expect(report.failures).toHaveLength(1) expect(report.failures[0]).toMatchObject({ test: 'Hrm.Tests.Business.EmployeeServiceTests.Create_rejects_duplicate', layer: 'business', file: '', // the MESSAGE, not the `Error Message:` header line dotnet prints before it error: 'Assert.Equal() Failure', }) }) it('a dotnet crash with no summary line at all is still one failed test, not a clean layer', () => { execSyncImpl = throwing('', 'MSBUILD : error MSB1009: Project file does not exist.') const { report } = execute(input({ includeBackend: true, includeFrontend: false })) for (const layer of ['seed', 'domain', 'business', 'integration']) { expect(report.byLayer[layer], layer).toEqual({ total: 1, passed: 0, failed: 1, skipped: 0 }) } expect(report.summary.failed).toBe(4) expect(report.summary.passed).toBe(0) }) })