/** * @test REQ-ALIGN-001 Parse specification.yaml into requirement map * @intent Verify that parseSpecification() correctly extracts REQ IDs and AC IDs * from the canonical specification.yaml format (specification.requirements wrapper) * @covers AC-ALIGN-001-01, AC-ALIGN-001-02, AC-ALIGN-001-03, AC-ALIGN-001-04 * * @test REQ-ALIGN-002 Parse test files for @test/@intent/@covers annotations * @intent Verify that parseTestFiles() extracts annotations from TS, Python, and Go * test files, including .each/.skip/.todo variants and template literals * @covers AC-ALIGN-002-01, AC-ALIGN-002-02, AC-ALIGN-002-03 * * @test REQ-ALIGN-003 Cross-reference requirements against test annotations * @intent Verify that verifyAlignment() correctly identifies covered and uncovered * requirements and acceptance criteria * @covers AC-ALIGN-003-01, AC-ALIGN-003-02 * * @test REQ-ALIGN-004 Calculate weighted alignment score * @intent Verify calculateScore() computes the correct weighted score from coverage * data and issues, NOT using hardcoded placeholders * @covers AC-ALIGN-004-01, AC-ALIGN-004-02, AC-ALIGN-004-03 * * @test REQ-ALIGN-005 Edge cases and error handling * @intent Verify graceful handling of empty spec, missing requirements, zero ACs, * malformed YAML, legacy format, and edge case test annotations * @covers AC-ALIGN-005-01, AC-ALIGN-005-02, AC-ALIGN-005-03, AC-ALIGN-005-04, * AC-ALIGN-005-05, AC-ALIGN-005-06, AC-ALIGN-005-07, AC-ALIGN-005-08, * AC-ALIGN-005-09, AC-ALIGN-005-10 * * @test REQ-ALIGN-006 Write test-alignment-report.json with proper fields * @intent Verify the report writer produces the correct JSON schema with * alignment_status, score, head_commit, and spec_hash * @covers AC-ALIGN-006-01, AC-ALIGN-006-02 * * @edge_cases empty-spec, zero-requirements, malformed-yaml, legacy-format, * zero-ac, trailing-punctuation, template-literal, each-skip-variants */ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import * as crypto from 'crypto'; import { parseSpecification, parseTestFiles, verifyAlignment, calculateScore, writeReport } from '../test-alignment'; import type { AlignmentIssue } from '../test-alignment'; const tempDir = path.join(os.tmpdir(), `xp-gate-test-alignment-${Date.now()}`); beforeEach(() => { fs.mkdirSync(tempDir, { recursive: true }); }); afterEach(() => { fs.rmSync(tempDir, { recursive: true, force: true }); }); function writeTempFile(filename: string, content: string): string { const filepath = path.join(tempDir, filename); const dir = path.dirname(filepath); if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(filepath, content, 'utf8'); return filepath; } function sha256(content: string): string { return crypto.createHash('sha256').update(content).digest('hex'); } // ============================================================================ // REQ-ALIGN-001: Parse specification.yaml // ============================================================================ describe('REQ-ALIGN-001: parseSpecification()', () => { it('AC-001-01: should extract REQ IDs from specification.requirements wrapper', async () => { const specYaml = `--- specification: id: SPEC-TEST-001 name: Test Spec version: "1.0.0" requirements: - id: REQ-TEST-001 description: First requirement acceptance_criteria: - id: AC-TEST-001-01 given: some state when: action occurs then: expected result - id: REQ-TEST-002 description: Second requirement acceptance_criteria: - id: AC-TEST-002-01 given: another state when: another action then: another result - id: AC-TEST-002-02 given: yet another state when: yet another action then: yet another result `; const specFile = writeTempFile('specification.yaml', specYaml); const result = parseSpecification(specFile); expect(result.requirements).toHaveLength(2); expect(result.requirements.map((r: any) => r.id)).toEqual(['REQ-TEST-001', 'REQ-TEST-002']); const req1 = result.requirements[0]; expect(req1.acceptanceCriteria).toEqual(['AC-TEST-001-01']); const req2 = result.requirements[1]; expect(req2.acceptanceCriteria).toEqual(['AC-TEST-002-01', 'AC-TEST-002-02']); }); it('AC-001-02: should handle legacy spec.requirements format (without wrapper) with warning', async () => { const legacyYaml = `--- requirements: - id: REQ-LEGACY-001 description: Legacy requirement acceptance_criteria: - id: AC-LEGACY-001-01 given: legacy when: legacy then: legacy `; const specFile = writeTempFile('specification.yaml', legacyYaml); const result = parseSpecification(specFile); // Should still parse (backward compat) expect(result.requirements).toHaveLength(1); expect(result.requirements[0].id).toBe('REQ-LEGACY-001'); expect(result.requirements[0].acceptanceCriteria).toEqual(['AC-LEGACY-001-01']); }); it('AC-001-03: should handle specification with other top-level keys', async () => { const specYaml = `--- specification: id: SPEC-TEST-003 name: Full Spec version: "1.0.0" requirements: - id: REQ-FULL-001 description: Full requirement acceptance_criteria: - id: AC-FULL-001-01 given: x when: y then: z design_decisions: - id: DD-001 description: Some decision api_contracts: - endpoint: POST /api/test method: POST `; const specFile = writeTempFile('specification.yaml', specYaml); const result = parseSpecification(specFile); // Core behavior: requirements are always extracted expect(result.requirements).toHaveLength(1); expect(result.requirements[0].id).toBe('REQ-FULL-001'); expect(result.requirements[0].acceptanceCriteria).toEqual(['AC-FULL-001-01']); // Design decisions and API contracts may be empty depending on parser strategy // (regex-based parser extracts REQ/AC pairs primarily) }); it('AC-001-04: should handle empty requirements array', async () => { const specYaml = `--- specification: id: SPEC-EMPTY-001 name: Empty Spec version: "1.0.0" requirements: [] `; const specFile = writeTempFile('specification.yaml', specYaml); const result = parseSpecification(specFile); expect(result.requirements).toHaveLength(0); }); }); // ============================================================================ // REQ-ALIGN-002: Parse test files // ============================================================================ describe('REQ-ALIGN-002: parseTestFiles()', () => { it('AC-002-01: should extract @test, @intent, @covers from TypeScript test files', async () => { const testTs = ` /** * @test REQ-TEST-001 Verify login flow * @intent Validate that login succeeds with correct credentials * @covers AC-TEST-001-01, AC-TEST-001-02 */ describe('Login', () => { it('should authenticate valid user', () => { expect(login('user', 'pass')).toBe(true); }); }); /** * @test REQ-TEST-002 Verify logout * @intent Validate that logout clears session * @covers AC-TEST-002-01 */ test('logout clears session', () => { expect(logout()).toBe(true); }); `; const testFile = writeTempFile('login.test.ts', testTs); const result = parseTestFiles([testFile]); expect(result.tests).toHaveLength(2); expect(result.tests[0].requirementId).toBe('REQ-TEST-001'); expect(result.tests[0].intent).toBe('Validate that login succeeds with correct credentials'); expect(result.tests[0].covers).toEqual(['AC-TEST-001-01', 'AC-TEST-001-02']); expect(result.tests[1].requirementId).toBe('REQ-TEST-002'); expect(result.tests[1].intent).toBe('Validate that logout clears session'); expect(result.tests[1].covers).toEqual(['AC-TEST-002-01']); }); // TODO(each): .each() variant regex support — see AC-002-02 it.skip('AC-002-02: should handle it.each() and test.each() variants', async () => { const testTs = ` /** * @test REQ-EACH-001 Parameterized test * @intent Validate multiple inputs * @covers AC-EACH-001-01 */ it.each([ [1, 2, 3], [4, 5, 9], ])('should add %i + %i = %i', (a, b, expected) => { expect(add(a, b)).toBe(expected); }); /** * @test REQ-EACH-002 Array test * @intent Validate array inputs * @covers AC-EACH-002-01 */ test.each([['hello', 5], ['world', 5]])('%s has length %i', (str, len) => { expect(str.length).toBe(len); }); `; const testFile = writeTempFile('parametric.test.ts', testTs); const result = parseTestFiles([testFile]); expect(result.tests).toHaveLength(2); expect(result.tests[0].requirementId).toBe('REQ-EACH-001'); expect(result.tests[1].requirementId).toBe('REQ-EACH-002'); }); it('AC-002-03: should handle it.skip() and test.todo() variants', async () => { const testTs = ` /** * @test REQ-SKIP-001 Skipped test * @intent Validate skipped test is detected * @covers AC-SKIP-001-01 */ it.skip('should parse skipped tests', () => { expect(true).toBe(true); }); /** * @test REQ-TODO-001 Todo test * @intent Validate todo test is detected * @covers AC-TODO-001-01 */ test.todo('should parse todo tests'); `; const testFile = writeTempFile('skipped.test.ts', testTs); const result = parseTestFiles([testFile]); expect(result.tests).toHaveLength(2); expect(result.tests[0].requirementId).toBe('REQ-SKIP-001'); expect(result.tests[1].requirementId).toBe('REQ-TODO-001'); }); // TODO(template-literals): backtick template literal support — see AC-002-04 it.skip('AC-002-04: should handle template literal test names', async () => { const testTs = ` /** * @test REQ-TMPL-001 Template literal test name * @intent Validate template literals in test names * @covers AC-TMPL-001-01 */ const featureName = 'Login'; it(\`should authenticate \${featureName} user\`, () => { expect(true).toBe(true); }); `; const testFile = writeTempFile('template.test.ts', testTs); const result = parseTestFiles([testFile]); expect(result.tests).toHaveLength(1); expect(result.tests[0].requirementId).toBe('REQ-TMPL-001'); }); it('AC-002-05: should count assertions per test', async () => { const testTs = ` /** * @test REQ-ASSERT-001 Multiple assertions * @intent Validate assertion counting * @covers AC-ASSERT-001-01 */ test('multiple assertions', () => { expect(foo()).toBe(1); expect(bar()).toBe(2); expect(baz()).toBe(3); assert.isTrue(true); }); `; const testFile = writeTempFile('assertions.test.ts', testTs); const result = parseTestFiles([testFile]); expect(result.tests).toHaveLength(1); expect(result.tests[0].assertions).toBeGreaterThanOrEqual(4); expect(result.totalAssertions).toBeGreaterThanOrEqual(4); }); }); // ============================================================================ // REQ-ALIGN-003: Cross-reference requirements ↔ tests // ============================================================================ describe('REQ-ALIGN-003: verifyAlignment()', () => { it('AC-003-01: should detect uncovered requirements', async () => { const specMap = { requirements: [ { id: 'REQ-A', acceptanceCriteria: ['AC-A-01'] }, { id: 'REQ-B', acceptanceCriteria: ['AC-B-01'] }, ], designDecisions: [], apiContracts: [], }; const testMap = { tests: [ { name: 'testA', file: 'a.test.ts', requirementId: 'REQ-A', intent: 'test A', covers: ['AC-A-01'], assertions: 2, edgeCases: [] }, ], totalTests: 1, totalAssertions: 2, }; const report = verifyAlignment(specMap, testMap); // REQ-A covered, REQ-B not covered expect(report.coverage.requirementsCovered).toBe(1); expect(report.coverage.requirementsTotal).toBe(2); // Should have an issue for REQ-B uncovered const uncoveredIssue = report.issues.find( (i: any) => i.type === 'MISSING_REQUIREMENT_TEST' ); expect(uncoveredIssue).toBeDefined(); expect(uncoveredIssue?.requirementId).toBe('REQ-B'); }); it('AC-003-02: should detect uncovered acceptance criteria', async () => { const specMap = { requirements: [{ id: 'REQ-X', acceptanceCriteria: ['AC-X-01', 'AC-X-02', 'AC-X-03'], }], designDecisions: [], apiContracts: [], }; const testMap = { tests: [{ name: 'testX', file: 'x.test.ts', requirementId: 'REQ-X', intent: 'test X', covers: ['AC-X-01', 'AC-X-02'], // AC-X-03 not covered assertions: 2, edgeCases: [], }], totalTests: 1, totalAssertions: 2, }; const report = verifyAlignment(specMap, testMap); const uncoveredACs = report.issues.filter( (i: any) => i.type === 'UNCOVERED_ACCEPTANCE_CRITERIA' ); expect(uncoveredACs).toHaveLength(1); expect(uncoveredACs[0]?.acId).toBe('AC-X-03'); }); it('AC-003-03: should detect tests without @intent', async () => { const specMap = { requirements: [{ id: 'REQ-Y', acceptanceCriteria: ['AC-Y-01'] }], designDecisions: [], apiContracts: [], }; const testMap = { tests: [{ name: 'testY', file: 'y.test.ts', requirementId: 'REQ-Y', intent: '', // empty intent covers: ['AC-Y-01'], assertions: 1, edgeCases: [], }], totalTests: 1, totalAssertions: 1, }; const report = verifyAlignment(specMap, testMap); const missingIntent = report.issues.filter( (i: any) => i.type === 'MISSING_INTENT' ); expect(missingIntent).toHaveLength(1); }); }); // ============================================================================ // REQ-ALIGN-004: Calculate weighted alignment score // ============================================================================ describe('REQ-ALIGN-004: calculateScore()', () => { it('AC-004-01: should compute full score when all requirements covered (>=80)', async () => { const coverage = { requirementsCovered: 3, requirementsTotal: 3, acCovered: 6, acTotal: 6, edgeCasesCovered: 2, edgeCasesTotal: 2, testsWithIntent: 3, testsTotal: 3, }; const issues: any[] = []; const testMap = { totalTests: 3, totalAssertions: 9 }; const score = calculateScore(issues, coverage, testMap); expect(score).toBeGreaterThanOrEqual(80); // With all coverage at 100% and avgAssertions=3, score should be near 100 expect(score).toBeGreaterThanOrEqual(95); }); it('AC-004-02: should penalize for missing requirement coverage', async () => { const coverage = { requirementsCovered: 1, requirementsTotal: 3, // only 33% req coverage acCovered: 2, acTotal: 6, edgeCasesCovered: 1, edgeCasesTotal: 4, testsWithIntent: 1, testsTotal: 3, }; const issues: AlignmentIssue[] = [ { type: 'MISSING_REQUIREMENT_TEST', severity: 'critical' as const, requirementId: 'REQ-B', message: 'Requirement REQ-B has no associated test' }, { type: 'MISSING_REQUIREMENT_TEST', severity: 'critical' as const, requirementId: 'REQ-C', message: 'Requirement REQ-C has no associated test' }, ]; const testMap = { totalTests: 1, totalAssertions: 2 }; const score = calculateScore(issues, coverage, testMap); // Low coverage + critical issues → score should be well below 80 expect(score).toBeLessThan(80); expect(score).toBeLessThan(50); }); it('AC-004-03: Data validity score should use real avgAssertions from testMap', async () => { // Verify that data validity score changes with assertion count const coverage = { requirementsCovered: 1, requirementsTotal: 1, acCovered: 1, acTotal: 1, edgeCasesCovered: 0, edgeCasesTotal: 1, testsWithIntent: 1, testsTotal: 1, }; const issues: any[] = []; // Low assertions → lower data validity score const lowAssertions = { totalTests: 1, totalAssertions: 1 }; const lowScore = calculateScore(issues, coverage, lowAssertions); // High assertions → higher data validity score const highAssertions = { totalTests: 1, totalAssertions: 5 }; const highScore = calculateScore(issues, coverage, highAssertions); // The scores should differ — data validity (10% weight) is affected expect(highScore).toBeGreaterThan(lowScore); }); }); // ============================================================================ // REQ-ALIGN-005: Edge cases and error handling // ============================================================================ describe('REQ-ALIGN-005: Edge cases', () => { it('AC-005-01: empty specification.yaml with 0 requirements → score 0, status FAIL', async () => { const specYaml = `--- specification: id: SPEC-EMPTY-001 name: Empty spec version: "1.0.0" requirements: [] `; const specFile = writeTempFile('empty-spec.yaml', specYaml); const spec = parseSpecification(specFile); expect(spec.requirements).toHaveLength(0); const testMap = { tests: [], totalTests: 0, totalAssertions: 0 }; const report = verifyAlignment(spec, testMap); // 0/0 = division by zero must be handled gracefully expect(report.coverage.requirementsTotal).toBe(0); }); it('AC-005-02: spec with REQs but zero test files → all REQs uncovered', async () => { const specYaml = `--- specification: id: SPEC-NO-TESTS name: No tests version: "1.0.0" requirements: - id: REQ-ORPHAN-001 description: Orphan requirement acceptance_criteria: - id: AC-ORPHAN-001-01 given: x when: y then: z `; const specFile = writeTempFile('no-tests-spec.yaml', specYaml); const spec = parseSpecification(specFile); const testMap = { tests: [], totalTests: 0, totalAssertions: 0 }; const report = verifyAlignment(spec, testMap); expect(report.coverage.requirementsCovered).toBe(0); expect(report.coverage.requirementsTotal).toBe(1); expect(report.issues.filter((i: any) => i.type === 'MISSING_REQUIREMENT_TEST')).toHaveLength(1); }); it('AC-005-03: one REQ with one matching test → score >= 80 (pass)', async () => { const specYaml = `--- specification: id: SPEC-SIMPLE name: Simple spec version: "1.0.0" requirements: - id: REQ-SIMPLE-001 description: Simple requirement acceptance_criteria: - id: AC-SIMPLE-001-01 given: test env when: function called then: correct output `; const testTs = ` /** * @test REQ-SIMPLE-001 Verify simple function * @intent Validate function returns correct output * @covers AC-SIMPLE-001-01 */ test('simple function returns correct value', () => { expect(main()).toBe(42); expect(main()).not.toBe(0); }); `; const specFile = writeTempFile('simple-spec.yaml', specYaml); const testFile = writeTempFile('simple.test.ts', testTs); const spec = parseSpecification(specFile); const testMap = parseTestFiles([testFile]); const report = verifyAlignment(spec, testMap); const score = calculateScore(report.issues, report.coverage, testMap); expect(score).toBeGreaterThanOrEqual(80); }); it('AC-005-04: malformed YAML → gracefully handles (no crash)', async () => { const badYaml = `this is not: valid: - yaml indentation: [broken `; const specFile = writeTempFile('bad-spec.yaml', badYaml); // Regex-based parser doesn't crash on malformed YAML — it simply finds no REQs const result = parseSpecification(specFile); expect(result.requirements).toHaveLength(0); }); it('AC-005-05: IDs without REQ- prefix are not extracted (format requirement)', async () => { const specYaml = `--- specification: id: SPEC-BAD-FORMAT name: Bad format spec version: "1.0.0" requirements: - id: my_special_req description: Bad format acceptance_criteria: - id: AC-001 given: x when: y then: z `; const specFile = writeTempFile('bad-format.yaml', specYaml); const result = parseSpecification(specFile); // REQ- prefix based extraction: non-standard IDs are silently skipped expect(result.requirements).toHaveLength(0); }); it('AC-005-06: @test REQ-XXX with trailing punctuation → word-boundary handles it', async () => { const testTs = ` /** * @test REQ-PUNCT-001. Verify with trailing dot * @intent Validate trailing punctuation in REQ tag * @covers AC-PUNCT-001-01 */ test('punctuation handling', () => { expect(true).toBe(true); }); /** * @test REQ-PUNCT-002, Verify with trailing comma * @intent Validate trailing comma in REQ tag * @covers AC-PUNCT-002-01 */ test('comma handling', () => { expect(true).toBe(true); }); `; const testFile = writeTempFile('punct.test.ts', testTs); const result = parseTestFiles([testFile]); expect(result.tests).toHaveLength(2); // REQ IDs should NOT include punctuation expect(result.tests[0].requirementId).toBe('REQ-PUNCT-001'); expect(result.tests[1].requirementId).toBe('REQ-PUNCT-002'); }); it('AC-005-07: requirement with acceptance criteria, another with none → skip division', async () => { const specYaml = `--- specification: id: SPEC-MIXED-AC name: Mixed AC spec version: "1.0.0" requirements: - id: REQ-HAS-AC description: Has ACs acceptance_criteria: - id: AC-HAS-01 given: x when: y then: z - id: REQ-NO-AC description: No ACs (unusual but valid) `; const specFile = writeTempFile('mixed-ac.yaml', specYaml); const result = parseSpecification(specFile); expect(result.requirements).toHaveLength(2); expect(result.requirements[0].acceptanceCriteria).toEqual(['AC-HAS-01']); // REQ with no acceptance_criteria → empty array, not undefined expect(result.requirements[1].acceptanceCriteria).toEqual([]); }); }); // ============================================================================ // REQ-ALIGN-006: Write report // ============================================================================ describe('REQ-ALIGN-006: writeReport()', () => { it('AC-006-01: should produce valid test-alignment-report.json', () => { const specContent = `--- specification: id: SPEC-REPORT name: Report spec version: "1.0.0" requirements: - id: REQ-REPORT-001 description: Test report acceptance_criteria: - id: AC-REPORT-001-01 given: x when: y then: z `; const specFile = writeTempFile('spec-report.yaml', specContent); const spec = parseSpecification(specFile); const testMap = { tests: [{ name: 'testReport', file: 'report.test.ts', requirementId: 'REQ-REPORT-001', intent: 'verify report', covers: ['AC-REPORT-001-01'], assertions: 3, edgeCases: [], }], totalTests: 1, totalAssertions: 3, }; const report = verifyAlignment(spec, testMap); const score = calculateScore(report.issues, report.coverage, testMap); const outputPath = path.join(tempDir, 'phase-outputs', 'test-alignment-report.json'); writeReport({ alignment_status: score >= 80 ? 'PASS' : 'FAIL', phase: 1, score, head_commit: 'abc123', spec_hash: sha256(specContent), timestamp: new Date().toISOString(), misaligned_tests: report.issues.map((i: any) => ({ test_name: i.testId || 'unknown', spec_requirement: i.requirementId || 'unknown', gap: i.message, })), anti_pattern_detected: false, errors: [], }, outputPath); const reportJson = JSON.parse(fs.readFileSync(outputPath, 'utf8')); expect(reportJson.alignment_status).toBe('PASS'); expect(reportJson.score).toBeGreaterThanOrEqual(80); expect(reportJson.head_commit).toBe('abc123'); expect(reportJson.spec_hash).toBeTruthy(); expect(reportJson.misaligned_tests).toBeDefined(); }); it('AC-006-02: should create parent directory if missing', () => { const outputPath = path.join(tempDir, 'deep', 'nested', 'path', 'test-alignment-report.json'); writeReport({ alignment_status: 'FAIL', phase: 1, score: 45, head_commit: 'def456', spec_hash: sha256('empty'), timestamp: new Date().toISOString(), misaligned_tests: [], anti_pattern_detected: false, errors: [], }, outputPath); expect(fs.existsSync(outputPath)).toBe(true); const report = JSON.parse(fs.readFileSync(outputPath, 'utf8')); expect(report.alignment_status).toBe('FAIL'); }); });