// ============================================================================ // Agent 5: Test Reviewer // Reviews generated tests for quality, coverage, and correctness // ============================================================================ import * as path from 'path'; import { AgentState, TestReview, ReviewIssue, QualityMetrics, GeneratedTest, } from '../types'; import { readFileContent } from '../utils/file-utils'; import { logger } from '../utils/logger'; export async function reviewerAgent(state: AgentState): Promise> { const startTime = Date.now(); logger.agentStart('reviewer'); try { if (state.generatedTests.length === 0) { throw new Error('No generated tests available. Writer Agent must run first.'); } logger.agent('reviewer', `Reviewing ${state.generatedTests.length} test files...`); const reviews: TestReview[] = []; for (let i = 0; i < state.generatedTests.length; i++) { const test = state.generatedTests[i]; logger.progress(i + 1, state.generatedTests.length, path.basename(test.filePath)); const review = reviewTestFile(test); reviews.push(review); } // Summary const passedCount = reviews.filter(r => r.passed).length; const avgScore = reviews.reduce((s, r) => s + r.score, 0) / reviews.length; const totalIssues = reviews.reduce((s, r) => s + r.issues.length, 0); const issuesByType = { errors: reviews.reduce((s, r) => s + r.issues.filter(i => i.severity === 'error').length, 0), warnings: reviews.reduce((s, r) => s + r.issues.filter(i => i.severity === 'warning').length, 0), infos: reviews.reduce((s, r) => s + r.issues.filter(i => i.severity === 'info').length, 0), }; logger.table( ['Metric', 'Value'], [ ['Reviewed', String(reviews.length)], ['Passed', `${passedCount}/${reviews.length}`], ['Avg. Score', `${avgScore.toFixed(1)}/100`], ['Errors', String(issuesByType.errors)], ['Warnings', String(issuesByType.warnings)], ['Infos', String(issuesByType.infos)], ] ); // Show failed reviews const failedReviews = reviews.filter(r => !r.passed); if (failedReviews.length > 0) { logger.warning(`${failedReviews.length} test files did not pass quality review:`); for (const fr of failedReviews) { logger.error(` ${path.basename(fr.testFile)} (Score: ${fr.score})`); for (const issue of fr.issues.filter(i => i.severity === 'error')) { logger.error(` - ${issue.message}`); } } } logger.agentComplete('reviewer', Date.now() - startTime); return { testReviews: reviews, agentLog: [ ...state.agentLog, { agent: 'reviewer', timestamp: new Date().toISOString(), action: 'Tests reviewed', details: `${passedCount}/${reviews.length} passed, Score: ${avgScore.toFixed(1)}`, duration: Date.now() - startTime, status: 'complete', }, ], }; } catch (error) { const errMsg = error instanceof Error ? error.message : String(error); logger.agentError('reviewer', errMsg); return { errors: [...state.errors, `Reviewer: ${errMsg}`], agentLog: [ ...state.agentLog, { agent: 'reviewer', timestamp: new Date().toISOString(), action: 'Error', details: errMsg, status: 'error', }, ], }; } } function reviewTestFile(test: GeneratedTest): TestReview { const content = test.content; const issues: ReviewIssue[] = []; const suggestions: string[] = []; const coverageGaps: string[] = []; // 1. Check structure checkStructure(content, issues, suggestions); // 2. Check assertions checkAssertions(content, issues, suggestions); // 3. Check test isolation checkIsolation(content, issues, suggestions); // 4. Check edge case coverage checkEdgeCases(content, test, issues, coverageGaps); // 5. Check error handling tests checkErrorHandling(content, issues, coverageGaps); // 6. Check mock usage checkMockUsage(content, test, issues, suggestions); // 7. Check async patterns checkAsyncPatterns(content, issues); // 8. Check security test completeness if (test.testType === 'security') { checkSecurityCompleteness(content, issues, coverageGaps); } // Calculate quality metrics const metrics = calculateQualityMetrics(content, issues); // Calculate score const score = calculateScore(metrics, issues); return { testFile: test.filePath, score, passed: score >= 60 && issues.filter(i => i.severity === 'error').length === 0, issues, suggestions, coverageGaps, qualityMetrics: metrics, }; } function checkStructure(content: string, issues: ReviewIssue[], suggestions: string[]): void { // Check for describe blocks if (!content.includes('describe(')) { issues.push({ severity: 'error', message: 'No describe() block found - tests must be grouped', }); } // Check for it/test blocks const testCount = (content.match(/\b(it|test)\s*\(/g) || []).length; if (testCount === 0) { issues.push({ severity: 'error', message: 'No test cases (it/test) found', }); } // Check for setup/teardown if (!content.includes('beforeEach') && !content.includes('beforeAll')) { suggestions.push('Consider adding beforeEach/beforeAll for test setup'); } if (!content.includes('afterEach') && !content.includes('afterAll')) { suggestions.push('Consider adding afterEach/afterAll for cleanup'); } // Check for proper imports if (!content.includes('import') && !content.includes('require')) { issues.push({ severity: 'warning', message: 'No imports found - tests should import modules', }); } } function checkAssertions(content: string, issues: ReviewIssue[], suggestions: string[]): void { // Use .+? (lazy any-char) instead of [^)]+ to handle nested parentheses // e.g. expect(response.ok()).toBeTruthy() — [^)]+ stops at ok()'s ) const assertionPatterns = [ /expect\(.+?\)\.toBe\(/g, /expect\(.+?\)\.toEqual\(/g, /expect\(.+?\)\.toBeDefined\(/g, /expect\(.+?\)\.toBeUndefined\(/g, /expect\(.+?\)\.toThrow\(/g, /expect\(.+?\)\.toHaveBeenCalled/g, /expect\(.+?\)\.toContain\(/g, /expect\(.+?\)\.toMatch\(/g, /expect\(.+?\)\.toBeNull\(/g, /expect\(.+?\)\.toBeTruthy\(/g, /expect\(.+?\)\.toBeFalsy\(/g, /expect\(.+?\)\.toHaveLength\(/g, /expect\(.+?\)\.toHaveProperty\(/g, /expect\(.+?\)\.toBeInstanceOf\(/g, /expect\(.+?\)\.rejects\./g, /expect\(.+?\)\.resolves\./g, /expect\(.+?\)\.not\./g, /expect\(.+?\)\.toBeLessThan\(/g, /expect\(.+?\)\.toBeGreaterThan\(/g, /expect\(.+?\)\.toBeGreaterThanOrEqual\(/g, /expect\(.+?\)\.toBeLessThanOrEqual\(/g, /expect\(.+?\)\.toStrictEqual\(/g, /expect\(.+?\)\.toMatchObject\(/g, ]; let totalAssertions = 0; for (const pattern of assertionPatterns) { const matches = content.match(pattern); if (matches) totalAssertions += matches.length; } const testCount = (content.match(/\b(it|test)\s*\(/g) || []).length; if (totalAssertions === 0) { issues.push({ severity: 'error', message: 'No assertions found - tests without assertions are worthless', }); } else if (testCount > 0 && totalAssertions / testCount < 1.5) { issues.push({ severity: 'warning', message: `Few assertions per test (${(totalAssertions / testCount).toFixed(1)}) - at least 2 per test recommended`, }); } // Check for toBe(true) anti-pattern (placeholder assertions) const placeholderCount = (content.match(/expect\(true\)\.toBe\(true\)/g) || []).length; if (placeholderCount > 0) { issues.push({ severity: 'warning', message: `${placeholderCount} placeholder assertions (expect(true).toBe(true)) found - must be implemented`, }); } } function checkIsolation(content: string, issues: ReviewIssue[], suggestions: string[]): void { // Check for mock clearing (support both Jest and Vitest patterns) const hasMocks = content.includes('jest.mock') || content.includes('vi.mock'); const hasMockReset = content.includes('clearAllMocks') || content.includes('resetAllMocks') || content.includes('restoreAllMocks'); if (hasMocks && !hasMockReset) { issues.push({ severity: 'warning', message: 'Mocks not being reset - can lead to test contamination', fix: 'Add clearAllMocks() in beforeEach', }); } // Check for shared mutable state const globalLetMatches = content.match(/^let\s+\w+\s*[=;]/gm) || []; if (globalLetMatches.length > 3) { suggestions.push('Many global let variables - consider improving test isolation'); } } function checkEdgeCases(content: string, test: GeneratedTest, issues: ReviewIssue[], gaps: string[]): void { const edgeCaseKeywords = ['null', 'undefined', 'empty', 'boundary', 'edge', 'invalid', 'negative', 'zero', 'overflow', 'NaN', 'Infinity']; let edgeCaseCoverage = 0; for (const keyword of edgeCaseKeywords) { if (content.toLowerCase().includes(keyword)) { edgeCaseCoverage++; } } if (test.testType === 'unit' && edgeCaseCoverage < 2) { gaps.push('Few edge cases covered - check null, undefined, empty values'); } } function checkErrorHandling(content: string, issues: ReviewIssue[], gaps: string[]): void { const hasErrorTests = content.includes('toThrow') || content.includes('rejects') || content.includes('error') || content.includes('Error'); if (!hasErrorTests) { gaps.push('No error handling tests - add toThrow/rejects tests'); } } function checkMockUsage(content: string, test: GeneratedTest, issues: ReviewIssue[], suggestions: string[]): void { if (test.mocks.length > 0) { for (const mock of test.mocks) { // Support both Jest and Vitest mock patterns const hasJestMock = content.includes(`jest.mock('${mock}')`) || content.includes(`jest.mock("${mock}")`); const hasViMock = content.includes(`vi.mock('${mock}')`) || content.includes(`vi.mock("${mock}")`); if (!hasJestMock && !hasViMock) { issues.push({ severity: 'info', message: `Mock for '${mock}' defined but not used in mock()`, }); } } } // Check for proper spy assertions (support both jest.spyOn and vi.spyOn) const spyCount = (content.match(/(?:jest|vi)\.spyOn/g) || []).length; const spyAssertions = (content.match(/toHaveBeenCalled|toHaveBeenCalledWith|toHaveBeenCalledTimes/g) || []).length; if (spyCount > 0 && spyAssertions === 0) { suggestions.push('Spies created but not verified - add toHaveBeenCalled* assertions'); } } function checkAsyncPatterns(content: string, issues: ReviewIssue[]): void { // Check for async tests without await const asyncTestRegex = /(?:it|test)\s*\([^,]+,\s*async\s*\(/g; const asyncTests = content.match(asyncTestRegex) || []; if (asyncTests.length > 0) { const awaitCount = (content.match(/\bawait\b/g) || []).length; if (awaitCount < asyncTests.length) { issues.push({ severity: 'warning', message: 'Async tests without await found - can lead to false positives', }); } } // Check for done callback misuse if (content.includes('done()') && content.includes('async')) { issues.push({ severity: 'warning', message: 'done() and async used together - use only async/await', }); } } function checkSecurityCompleteness(content: string, issues: ReviewIssue[], gaps: string[]): void { const securityChecks = [ { pattern: /sql.*inject|injection/i, name: 'SQL Injection' }, { pattern: /xss|cross.site.script/i, name: 'XSS' }, { pattern: /command.*inject|exec|spawn/i, name: 'Command Injection' }, { pattern: /path.*travers|directory.*travers/i, name: 'Path Traversal' }, { pattern: /prototype.*pollut/i, name: 'Prototype Pollution' }, { pattern: /csrf|cross.site.request/i, name: 'CSRF' }, { pattern: /ssrf|server.side.request/i, name: 'SSRF' }, { pattern: /rate.limit|throttl/i, name: 'Rate Limiting' }, ]; for (const check of securityChecks) { if (!check.pattern.test(content)) { gaps.push(`Missing security check: ${check.name}`); } } } function calculateQualityMetrics(content: string, issues: ReviewIssue[]): QualityMetrics { const assertionCount = (content.match(/expect\(/g) || []).length; const edgeCasePatterns = ['null', 'undefined', 'empty', 'boundary', 'edge', 'error', 'invalid']; const edgeCaseCount = edgeCasePatterns.filter(p => content.toLowerCase().includes(p)).length; const errorHandlingCount = (content.match(/toThrow|rejects|catch/g) || []).length; const mockCount = (content.match(/(?:jest|vi)\.mock|(?:jest|vi)\.spyOn|(?:jest|vi)\.fn/g) || []).length; // Readability: shorter tests, clear naming const lines = content.split('\n').length; const readability = Math.min(100, Math.max(0, 100 - (lines > 200 ? (lines - 200) / 10 : 0))); // Maintainability: proper structure const hasDescribe = content.includes('describe('); const hasSetup = content.includes('beforeEach') || content.includes('beforeAll'); const maintainability = (hasDescribe ? 40 : 0) + (hasSetup ? 30 : 0) + (mockCount > 0 ? 30 : 0); const overall = Math.round( (assertionCount * 5 + edgeCaseCount * 10 + errorHandlingCount * 15 + readability * 0.3 + maintainability * 0.3) / Math.max(1, assertionCount + edgeCaseCount + errorHandlingCount + 2) ); return { assertions: assertionCount, edgeCases: edgeCaseCount, errorHandling: errorHandlingCount, mockUsage: mockCount, readability: Math.round(readability), maintainability: Math.min(100, maintainability), overallScore: Math.min(100, Math.max(0, overall * 10)), }; } function calculateScore(metrics: QualityMetrics, issues: ReviewIssue[]): number { let score = 100; // Deductions for issues for (const issue of issues) { switch (issue.severity) { case 'error': score -= 20; break; case 'warning': score -= 5; break; case 'info': score -= 1; break; } } // Bonus for quality metrics if (metrics.assertions >= 3) score += 5; if (metrics.edgeCases >= 3) score += 5; if (metrics.errorHandling >= 1) score += 5; return Math.max(0, Math.min(100, score)); }