// ============================================================================ // Agent 6: Test Runner // Executes generated tests and collects results // Supports: Vitest, Jest, Node.js built-in test runner, Playwright // ============================================================================ import * as path from 'path'; import { execSync } from 'child_process'; import { AgentState, TestResult, TestError, CoverageInfo, TestRunner, } from '../types'; import { logger } from '../utils/logger'; export async function runnerAgent(state: AgentState): Promise> { const startTime = Date.now(); logger.agentStart('runner'); try { if (state.generatedTests.length === 0) { throw new Error('No tests available. Writer Agent must run first.'); } const results: TestResult[] = []; const testTypes = ['unit', 'integration', 'e2e', 'security', 'performance']; const projectPath = state.projectPath; const runner = state.config.testRunner; const e2eRunner = state.config.e2eRunner; // Check if the configured test runner is available const runnerAvailable = checkRunnerAvailable(runner, projectPath); if (!runnerAvailable) { const installHint = getInstallHint(runner); logger.warning(`${runner} not installed. Install with: ${installHint}`); logger.agent('runner', `Running dry run (${runner} not installed)...`); for (const test of state.generatedTests) { results.push(createDryRunResult(test.filePath, test.testCount)); } } else { logger.agent('runner', `Using test runner: ${runner}`); // Run tests by type for (const testType of testTypes) { const testsOfType = state.generatedTests.filter(t => t.testType === testType); if (testsOfType.length === 0) continue; // E2E tests with Playwright use a different runner if (testType === 'e2e' && e2eRunner === 'playwright') { logger.agent('runner', `Running E2E tests with Playwright... (${testsOfType.length} files)`); const playwrightAvailable = checkPlaywrightAvailable(projectPath); if (playwrightAvailable) { for (const test of testsOfType) { const result = await runPlaywrightTest(test.filePath, projectPath); results.push(result); logTestResult(result); } } else { logger.warning('Playwright not installed. Install with: npm install --save-dev @playwright/test && npx playwright install'); for (const test of testsOfType) { results.push(createDryRunResult(test.filePath, test.testCount)); } } continue; } logger.agent('runner', `Running ${testType} tests with ${runner}... (${testsOfType.length} files)`); for (const test of testsOfType) { const result = await runTestFile(test.filePath, projectPath, runner); results.push(result); logTestResult(result); } } } // Summary const totalTests = results.reduce((s, r) => s + r.totalTests, 0); const totalPassed = results.reduce((s, r) => s + r.passed, 0); const totalFailed = results.reduce((s, r) => s + r.failed, 0); const totalSkipped = results.reduce((s, r) => s + r.skipped, 0); const totalDuration = results.reduce((s, r) => s + r.duration, 0); logger.newline(); logger.table( ['Metric', 'Value'], [ ['Test Runner', runner], ['E2E Runner', e2eRunner], ['Total Tests', String(totalTests)], ['Passed', String(totalPassed)], ['Failed', String(totalFailed)], ['Skipped', String(totalSkipped)], ['Duration', `${(totalDuration / 1000).toFixed(1)}s`], ['Success Rate', `${totalTests > 0 ? ((totalPassed / totalTests) * 100).toFixed(1) : 0}%`], ] ); logger.agentComplete('runner', Date.now() - startTime); return { testResults: results, agentLog: [ ...state.agentLog, { agent: 'runner', timestamp: new Date().toISOString(), action: 'Tests executed', details: `${totalPassed}/${totalTests} passed (${totalFailed} failed) [${runner}]`, duration: Date.now() - startTime, status: 'complete', }, ], }; } catch (error) { const errMsg = error instanceof Error ? error.message : String(error); logger.agentError('runner', errMsg); return { errors: [...state.errors, `Runner: ${errMsg}`], agentLog: [ ...state.agentLog, { agent: 'runner', timestamp: new Date().toISOString(), action: 'Error', details: errMsg, status: 'error', }, ], }; } } // ============================================================================ // Runner Availability Checks // ============================================================================ function checkRunnerAvailable(runner: TestRunner, projectPath: string): boolean { switch (runner) { case 'vitest': return checkCommandAvailable('npx vitest --version', projectPath); case 'jest': return checkCommandAvailable('npx jest --version', projectPath); case 'node': // Node.js built-in test runner is always available on Node >= 18 return true; default: return false; } } function checkPlaywrightAvailable(projectPath: string): boolean { return checkCommandAvailable('npx playwright --version', projectPath); } function checkCommandAvailable(command: string, cwd: string): boolean { try { execSync(command, { cwd, stdio: 'pipe', encoding: 'utf-8', timeout: 15000 }); return true; } catch { return false; } } function getInstallHint(runner: TestRunner): string { switch (runner) { case 'vitest': return 'npm install --save-dev vitest @vitest/coverage-v8'; case 'jest': return 'npm install --save-dev jest ts-jest @types/jest'; case 'node': return 'Node.js >= 18 required (built-in test runner)'; default: return 'npm install --save-dev vitest'; } } // ============================================================================ // Test File Execution (by runner type) // ============================================================================ async function runTestFile(testFilePath: string, projectPath: string, runner: TestRunner): Promise { switch (runner) { case 'vitest': return runVitestFile(testFilePath, projectPath); case 'jest': return runJestFile(testFilePath, projectPath); case 'node': return runNodeTestFile(testFilePath, projectPath); default: return runVitestFile(testFilePath, projectPath); } } // ============================================================================ // Vitest Runner // ============================================================================ async function runVitestFile(testFilePath: string, projectPath: string): Promise { const startTime = Date.now(); try { const output = execSync( `npx vitest run "${testFilePath}" --reporter=json 2>/dev/null`, { cwd: projectPath, encoding: 'utf-8', timeout: 60000, stdio: 'pipe' } ); const duration = Date.now() - startTime; try { const vitestResult = JSON.parse(output); const errors: TestError[] = []; let passed = 0; let failed = 0; let skipped = 0; for (const suite of vitestResult.testResults || []) { for (const test of suite.assertionResults || []) { switch (test.status) { case 'passed': passed++; break; case 'failed': failed++; errors.push({ testName: test.fullName || test.title || test.ancestorTitles?.join(' > '), message: (test.failureMessages || []).join('\n').substring(0, 500), }); break; default: skipped++; break; } } } return { testFile: testFilePath, totalTests: passed + failed + skipped, passed, failed, skipped, duration, errors }; } catch { // JSON parse failed - try to extract from output } return { testFile: testFilePath, totalTests: 1, passed: 1, failed: 0, skipped: 0, duration, errors: [] }; } catch (execError: any) { const duration = Date.now() - startTime; return parseExecError(execError, testFilePath, duration); } } // ============================================================================ // Jest Runner // ============================================================================ async function runJestFile(testFilePath: string, projectPath: string): Promise { const startTime = Date.now(); try { const output = execSync( `npx jest "${testFilePath}" --json --no-coverage --forceExit 2>/dev/null`, { cwd: projectPath, encoding: 'utf-8', timeout: 60000, stdio: 'pipe' } ); const duration = Date.now() - startTime; try { const jestResult = JSON.parse(output); const suiteResult = jestResult.testResults?.[0]; if (suiteResult) { const errors: TestError[] = []; let passed = 0; let failed = 0; let skipped = 0; for (const test of suiteResult.testResults || []) { switch (test.status) { case 'passed': passed++; break; case 'failed': failed++; errors.push({ testName: test.fullName || test.title, message: test.failureMessages?.join('\n') || 'Unknown error', }); break; case 'pending': case 'skipped': skipped++; break; } } return { testFile: testFilePath, totalTests: passed + failed + skipped, passed, failed, skipped, duration, errors }; } } catch { // JSON parse failed } return { testFile: testFilePath, totalTests: 1, passed: 1, failed: 0, skipped: 0, duration, errors: [] }; } catch (execError: any) { const duration = Date.now() - startTime; return parseExecError(execError, testFilePath, duration); } } // ============================================================================ // Node.js Built-in Test Runner // ============================================================================ async function runNodeTestFile(testFilePath: string, projectPath: string): Promise { const startTime = Date.now(); try { const output = execSync( `node --test --test-reporter=spec "${testFilePath}" 2>&1`, { cwd: projectPath, encoding: 'utf-8', timeout: 60000, stdio: 'pipe' } ); const duration = Date.now() - startTime; // Parse node:test spec output const passedMatches = output.match(/# pass (\d+)/); const failedMatches = output.match(/# fail (\d+)/); const skippedMatches = output.match(/# skip (\d+)/); const passed = passedMatches ? parseInt(passedMatches[1], 10) : 0; const failed = failedMatches ? parseInt(failedMatches[1], 10) : 0; const skipped = skippedMatches ? parseInt(skippedMatches[1], 10) : 0; const errors: TestError[] = []; if (failed > 0) { // Extract failure details from output const failureLines = output.split('\n').filter(l => l.includes('not ok') || l.includes('Error:')); for (const line of failureLines.slice(0, 10)) { errors.push({ testName: line.trim().substring(0, 100), message: line.trim() }); } } return { testFile: testFilePath, totalTests: passed + failed + skipped || 1, passed: passed || (failed === 0 ? 1 : 0), failed, skipped, duration, errors, }; } catch (execError: any) { const duration = Date.now() - startTime; return parseExecError(execError, testFilePath, duration); } } // ============================================================================ // Playwright Runner // ============================================================================ async function runPlaywrightTest(testFilePath: string, projectPath: string): Promise { const startTime = Date.now(); try { const output = execSync( `npx playwright test "${testFilePath}" --reporter=json 2>/dev/null`, { cwd: projectPath, encoding: 'utf-8', timeout: 120000, stdio: 'pipe' } ); const duration = Date.now() - startTime; try { const pwResult = JSON.parse(output); const errors: TestError[] = []; let passed = 0; let failed = 0; let skipped = 0; for (const suite of pwResult.suites || []) { for (const spec of suite.specs || []) { for (const test of spec.tests || []) { for (const result of test.results || []) { switch (result.status) { case 'passed': passed++; break; case 'failed': case 'timedOut': failed++; errors.push({ testName: spec.title || test.title || 'unknown', message: (result.error?.message || 'Test failed').substring(0, 500), }); break; case 'skipped': skipped++; break; } } } } } return { testFile: testFilePath, totalTests: passed + failed + skipped, passed, failed, skipped, duration, errors }; } catch { // JSON parse failed } return { testFile: testFilePath, totalTests: 1, passed: 1, failed: 0, skipped: 0, duration, errors: [] }; } catch (execError: any) { const duration = Date.now() - startTime; return parseExecError(execError, testFilePath, duration); } } // ============================================================================ // Shared Helpers // ============================================================================ function parseExecError(execError: any, testFilePath: string, duration: number): TestResult { // Try to parse JSON output from stderr/stdout try { const output = execError.stdout || execError.stderr || ''; const jsonMatch = output.match(/\{[\s\S]*"testResults"[\s\S]*\}/); if (jsonMatch) { const result = JSON.parse(jsonMatch[0]); const suiteResult = result.testResults?.[0]; const errors: TestError[] = []; let passed = 0; let failed = 0; let skipped = 0; for (const test of suiteResult?.testResults || suiteResult?.assertionResults || []) { switch (test.status) { case 'passed': passed++; break; case 'failed': failed++; errors.push({ testName: test.fullName || test.title, message: (test.failureMessages || []).join('\n').substring(0, 500), }); break; default: skipped++; break; } } return { testFile: testFilePath, totalTests: passed + failed + skipped, passed, failed, skipped, duration, errors }; } } catch { // ignore parse errors } return { testFile: testFilePath, totalTests: 1, passed: 0, failed: 1, skipped: 0, duration, errors: [{ testName: path.basename(testFilePath), message: (execError.message || 'Execution failed').substring(0, 500), }], }; } function logTestResult(result: TestResult): void { if (result.failed > 0) { logger.testResult(path.basename(result.testFile), false); for (const err of result.errors) { logger.error(` ${err.testName}: ${err.message}`); } } else { logger.testResult(path.basename(result.testFile), true, result.duration); } } function createDryRunResult(testFilePath: string, testCount: number): TestResult { return { testFile: testFilePath, totalTests: testCount, passed: 0, failed: 0, skipped: testCount, duration: 0, errors: [], }; }