/** * cli:test-report — execute.ts * Runs test suites and produces a structured report. */ import { execSync } from 'node:child_process' import { writeFileSync, mkdirSync } from 'node:fs' import { resolve, join, relative, isAbsolute } from 'node:path' import type { TestReportInput, TestReportResult, LayerResult, TestFailure } from './types.js' const BACKEND_CATEGORIES = ['Seed', 'Domain', 'Business', 'Integration'] export function execute(input: TestReportInput): { report: TestReportResult; reportPath: string } { const projectPath = resolve(input.projectPath) const start = Date.now() const byLayer: Record = {} const failures: TestFailure[] = [] // Run backend tests by category if (input.includeBackend) { for (const category of BACKEND_CATEGORIES) { const result = runDotnetTests(projectPath, category) byLayer[category.toLowerCase()] = result.layerResult failures.push(...result.failures) } } // Run frontend tests if (input.includeFrontend) { const result = runFrontendTests(projectPath, input.module) byLayer['frontend'] = result.layerResult failures.push(...result.failures) } // Aggregate summary const summary = { total: Object.values(byLayer).reduce((s, l) => s + l.total, 0), passed: Object.values(byLayer).reduce((s, l) => s + l.passed, 0), failed: Object.values(byLayer).reduce((s, l) => s + l.failed, 0), skipped: Object.values(byLayer).reduce((s, l) => s + l.skipped, 0), } const report: TestReportResult = { module: input.module, timestamp: new Date().toISOString(), duration: Date.now() - start, summary, byLayer, failures, coverage: null, // no coverage tool integrated — null, never a fabricated 0 presented as a measurement } // Write report const reportsDir = join(projectPath, '.smartstack', 'reports') mkdirSync(reportsDir, { recursive: true }) const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19) const reportPath = join(reportsDir, `TEST-${input.module}-${ts}.json`) writeFileSync(reportPath, JSON.stringify(report, null, 2), 'utf-8') return { report, reportPath } } function runDotnetTests(projectPath: string, category: string): { layerResult: LayerResult; failures: TestFailure[] } { try { const output = execSync( `dotnet test --no-build --filter Category=${category} --verbosity normal --logger "trx;LogFileName=${category}.trx"`, { cwd: projectPath, encoding: 'utf-8', timeout: 300_000, stdio: ['pipe', 'pipe', 'pipe'] }, ) // Parse test counts from output const passed = parseInt(output.match(/Passed:\s+(\d+)/)?.[1] ?? '0') const failed = parseInt(output.match(/Failed:\s+(\d+)/)?.[1] ?? '0') const skipped = parseInt(output.match(/Skipped:\s+(\d+)/)?.[1] ?? '0') const total = passed + failed + skipped return { layerResult: { total, passed, failed, skipped }, failures: [], } } catch (err: any) { const output = (err.stdout ?? '') + (err.stderr ?? '') const passed = parseInt(output.match(/Passed:\s+(\d+)/)?.[1] ?? '0') const failed = parseInt(output.match(/Failed:\s+(\d+)/)?.[1] ?? '0') const skipped = parseInt(output.match(/Skipped:\s+(\d+)/)?.[1] ?? '0') const total = passed + failed + skipped // Extract failure details // `dotnet test` prints `Failed X [12 ms]` then an `Error Message:` header line, THEN the // message — skip the header (found by the lot 12 tests, 2026-09-24). const failureMatches = output.matchAll(/Failed\s+(\S+)\s+\[.*?\]\s*\n(?:\s*Error Message:\s*\n)?\s*(.+)/g) const failures: TestFailure[] = [] for (const match of failureMatches) { failures.push({ test: match[1], layer: category.toLowerCase(), error: match[2]?.trim() ?? 'Unknown error', file: '', }) } return { layerResult: { total: Math.max(total, 1), passed, failed: Math.max(failed, 1), skipped }, failures, } } } function runFrontendTests(projectPath: string, module: string): { layerResult: LayerResult; failures: TestFailure[] } { let output = '' let stderr = '' try { output = execSync( `npm test -- --reporter=json ${module}`, { cwd: projectPath, encoding: 'utf-8', timeout: 300_000, stdio: ['pipe', 'pipe', 'pipe'] }, ) } catch (err: any) { // vitest exits 1 whenever ONE test fails — the JSON report is still on stdout. A red // suite used to land here and lose every count and failing test name (lot 12, 2026-09-24). output = typeof err?.stdout === 'string' ? err.stdout : '' stderr = typeof err?.stderr === 'string' ? err.stderr : '' } // Unparsable vitest output is NOT a pass (audit 2026-09-24, D-WARN-11: a fabricated // {total:1, passed:1} hid every misconfiguration). Fail, with a reason that is never // empty: the runner's stderr when it said something, else the by-hand command. const unparsable = (what: string): { layerResult: LayerResult; failures: TestFailure[] } => { const reason = stderr.trim().length > 0 ? stderr.slice(0, 500) : `vitest output is ${what} (run \`npm test -- --reporter=json ${module}\` by hand to see why): ${output.slice(0, 300)}` return { layerResult: { total: 1, passed: 0, failed: 1, skipped: 0 }, failures: [{ test: 'frontend-suite', layer: 'frontend', error: reason, file: '' }], } } let json: unknown try { json = JSON.parse(output) } catch { return unparsable('not JSON') } if (!isRecord(json) || !Array.isArray(json.testResults)) { return unparsable('JSON but not a vitest report (no testResults[])') } return readVitestReport(json, projectPath, module) } /** * The vitest `--reporter=json` shape (vitest 2.x, `JsonReporter.logTasks`): top-level counters * (`numPassedTests`, `numFailedTests`, `numPendingTests` = skipped, `numTodoTests`), then ONE * `testResults[]` entry per FILE — `{ name: , status, message: , assertionResults: [{ fullName, title, status: 'passed'|'failed'|'skipped'| * 'pending'|'todo', failureMessages: string[] (message + stack frames), ancestorTitles, duration, * location }] }`. Until 2026-09-24 this parser read each entry as a TEST (`t.name`, `t.message`, * `t.file`): a red run named the file path as the test with an empty reason (QC 2026-09-24, D5). */ function readVitestReport( json: Record, projectPath: string, module: string, ): { layerResult: LayerResult; failures: TestFailure[] } { const passed = count(json.numPassedTests) let failed = count(json.numFailedTests) const skipped = count(json.numPendingTests) + count(json.numTodoTests) const failures: TestFailure[] = [] for (const fileResult of json.testResults as unknown[]) { if (!isRecord(fileResult)) continue const file = projectRelative(projectPath, fileResult.name) const fileMessage = nonEmpty(fileResult.message) ?? '' const assertions = Array.isArray(fileResult.assertionResults) ? fileResult.assertionResults.filter(isRecord) : [] let failedHere = 0 for (const assertion of assertions) { if (assertion.status !== 'failed') continue failedHere += 1 const firstMessage = Array.isArray(assertion.failureMessages) ? assertion.failureMessages.find((m): m is string => nonEmpty(m) !== undefined) : undefined failures.push({ test: nonEmpty(assertion.fullName) ?? nonEmpty(assertion.title) ?? (file || 'unknown'), layer: 'frontend', // never empty: the assertion message without its stack frames, else the file-level message error: (firstMessage !== undefined ? withoutStackFrames(firstMessage) : '') || fileMessage || 'Test failed', file, }) } if (fileResult.status === 'failed' && failedHere === 0) { // The FILE failed and no test did: it threw at import (missing module, syntax error) — // its tests never ran, `numFailedTests` is 0 and only the file-level message says why. failed += 1 failures.push({ test: file || 'unknown', layer: 'frontend', error: fileMessage || 'Test file failed before any test ran', file, }) } } if (json.success === false && failed === 0) { // vitest itself says the run is red yet neither a test nor a file is marked failed — // never let that through as a clean layer. failed = 1 failures.push({ test: 'frontend-suite', layer: 'frontend', error: `vitest reported a red run (success flag false) with no failed test or file (run \`npm test -- --reporter=json ${module}\` by hand to see why)`, file: '', }) } const total = Math.max(count(json.numTotalTests), passed + failed + skipped) return { layerResult: { total, passed, failed, skipped }, failures } } const isRecord = (value: unknown): value is Record => typeof value === 'object' && value !== null && !Array.isArray(value) const count = (value: unknown): number => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0 const nonEmpty = (value: unknown): string | undefined => typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined /** vitest's `failureMessages[]` are `error.stack`: the message lines, then ` at …` frames — keep the message. */ function withoutStackFrames(message: string): string { const lines = message.split(/\r?\n/) const firstFrame = lines.findIndex((line) => /^\s+at\s/.test(line)) return (firstFrame === -1 ? lines : lines.slice(0, firstFrame)).join('\n').trim() } /** * The ONE verdict of a report: green only when something ran AND nothing failed. * A run with 0 tests (the module filter matched no test file on any layer) is not * a pass — a gate that ran nothing cannot pass (QC 2026-09-24). */ export function reportVerdict(summary: { total: number; failed: number }): { ok: boolean; reason?: string } { if (summary.failed > 0) return { ok: false, reason: `${summary.failed} test(s) failed` } if (summary.total === 0) { return { ok: false, reason: '0 tests ran — the module filter matched no test file on any layer (backend `dotnet test --filter`, frontend vitest pattern); a gate that ran nothing cannot pass. Check the module name and that scaffold-tests / scaffold-tests-from-ac ran.' } } return { ok: true } } /** The reporter's `testResults[].name` is absolute — relative to the project when it lives inside it, POSIX slashes. */ function projectRelative(projectPath: string, name: unknown): string { if (typeof name !== 'string' || name.length === 0) return '' const posix = (p: string): string => p.replace(/\\/g, '/') if (!isAbsolute(name)) return posix(name) const rel = relative(projectPath, name) if (rel.length === 0 || rel.startsWith('..') || isAbsolute(rel)) return posix(name) return posix(rel) }