/** * run-ui-test / execute — the run itself, with the dev-browser layer and the * global `fetch` stubbed: nothing is spawned, no socket is opened. Asserted: * a run of zero tests is refused ("no test matched"), the /health probe runs * BEFORE the first scenario and fails closed, and a passing set is counted. * Only the verdict + the testid contract were tested before (audit D-WARN-18). */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' /** Ordered trace of every external call — the ORDER is part of the contract under test. */ const log: string[] = [] const driverCalls: Array<{ id: string; role: string; email: string; ctx: Record }> = [] let driverImpl: (id: string) => { status: 'pass' | 'fail'; reason: string } = () => ({ status: 'pass', reason: 'ok' }) vi.mock('../lib/dev-browser-driver.js', () => ({ runDevBrowserTest: async ( test: { id: string; role: string }, user: { email: string }, ctx: Record, ) => { log.push(`driver:${test.id}`) driverCalls.push({ id: test.id, role: test.role, email: user.email, ctx }) const r = driverImpl(test.id) return { status: r.status, reason: r.reason, capture: { exitCode: 0 }, durationMs: 1 } }, })) import { execute, verdict } from '../execute.js' import { RunUiTestInputSchema, type RunUiTestInput } from '../types.js' const FRONT = 'http://localhost:3000' const API = 'http://localhost:5000' const HEALTH = `${API}/health` let fetchImpl: (url: string) => Promise = async () => new Response('', { status: 200 }) let tmp: string beforeEach(() => { tmp = mkdtempSync(join(tmpdir(), 'ss-run-ui-test-')) log.length = 0 driverCalls.length = 0 driverImpl = () => ({ status: 'pass', reason: 'ok' }) fetchImpl = async () => new Response('', { status: 200 }) vi.stubGlobal('fetch', async (url: string | URL) => { log.push(`fetch:${String(url)}`) return fetchImpl(String(url)) }) }) afterEach(() => { vi.unstubAllGlobals() rmSync(tmp, { recursive: true, force: true }) }) interface ManifestTest { id: string scenario: string page: string role: string expectations?: Record } function writeFixture(tests: ManifestTest[], roles: string[] = ['admin']): void { const dir = join(tmp, 'tests', 'ui-test') mkdirSync(dir, { recursive: true }) writeFileSync( join(dir, 'manifest.json'), JSON.stringify({ module: 'hrm', appCode: 'myapp', tests: tests.map((t) => ({ expectations: { httpStatus: [200] }, ...t })), }), 'utf-8', ) writeFileSync( join(dir, 'test-users.json'), JSON.stringify({ module: 'hrm', appCode: 'myapp', users: roles.map((role) => ({ role, email: `${role}@test.local`, password: 'Pw!12345', applicationCode: 'myapp', roleName: role, })), }), 'utf-8', ) } const T1: ManifestTest = { id: 'T-1', scenario: 'list', page: '/hrm/employees', role: 'admin' } const T2: ManifestTest = { id: 'T-2', scenario: 'detail', page: '/hrm/employees/1', role: 'admin' } const spec = (over: Partial = {}): RunUiTestInput => RunUiTestInputSchema.parse({ projectPath: tmp, frontendUrl: FRONT, apiUrl: API, ...over }) describe('run-ui-test / execute — a run of zero tests is not a pass', () => { it('--module not matching the manifest → totalTests 0, verdict "no test matched", nothing probed nor spawned', async () => { writeFixture([T1, T2]) const s = spec({ module: 'crm' }) const report = await execute(s) expect(report).toMatchObject({ module: 'hrm', appCode: 'myapp', totalTests: 0, passed: 0, failed: 0, results: [] }) const v = verdict(report, { module: s.module, testId: s.testId }) expect(v.success).toBe(false) expect(v.errors[0]).toContain('no test matched') expect(v.errors[0]).toContain('--module "crm"') expect(log).toEqual([]) }) it('--test-id matching nothing → totalTests 0 and a failing verdict naming the id; no scenario runs', async () => { writeFixture([T1, T2]) const s = spec({ testId: 'T-9' }) const report = await execute(s) expect(report.totalTests).toBe(0) const v = verdict(report, { module: s.module, testId: s.testId }) expect(v.success).toBe(false) expect(v.errors[0]).toContain('no test matched') expect(v.errors[0]).toContain('--test-id "T-9"') expect(driverCalls).toHaveLength(0) }) it('an empty manifest → verdict "declares no test"', async () => { writeFixture([]) const report = await execute(spec()) expect(report.totalTests).toBe(0) const v = verdict(report) expect(v.success).toBe(false) expect(v.errors[0]).toContain('declares no test') expect(driverCalls).toHaveLength(0) }) }) describe('run-ui-test / execute — the /health probe guards the scenarios', () => { it('probes API /health then the frontend BEFORE the first scenario', async () => { writeFixture([T1, T2]) await execute(spec()) expect(log).toEqual([`fetch:${HEALTH}`, `fetch:${FRONT}`, 'driver:T-1', 'driver:T-2']) }) it('an unreachable API fails closed with a clear error and no scenario is spawned', async () => { writeFixture([T1, T2]) fetchImpl = async () => { throw new Error('ECONNREFUSED') } await expect(execute(spec())).rejects.toThrow( /API not reachable at http:\/\/localhost:5000\/health \(ECONNREFUSED\).*ss dev up/, ) expect(driverCalls).toHaveLength(0) }) it('a frontend answering 5xx fails closed and no scenario is spawned', async () => { writeFixture([T1]) fetchImpl = async (url) => new Response('', { status: url === HEALTH ? 401 : 503 }) await expect(execute(spec())).rejects.toThrow(/frontend answered HTTP 503 at http:\/\/localhost:3000/) expect(driverCalls).toHaveLength(0) }) it('a secured API answering 401 on /health counts as up', async () => { writeFixture([T1]) fetchImpl = async (url) => new Response('', { status: url === HEALTH ? 401 : 200 }) const report = await execute(spec()) expect(report.totalTests).toBe(1) expect(driverCalls).toHaveLength(1) }) }) describe('run-ui-test / execute — counts and the driver contract', () => { it('a passing scenario set → every test pass, counts add up, verdict success', async () => { writeFixture([T1, T2]) const s = spec() const report = await execute(s) expect(report).toMatchObject({ module: 'hrm', appCode: 'myapp', totalTests: 2, passed: 2, failed: 0 }) expect(report.results.map((r) => [r.testId, r.status, r.reason])).toEqual([ ['T-1', 'pass', 'ok'], ['T-2', 'pass', 'ok'], ]) expect(verdict(report, { module: s.module, testId: s.testId })).toEqual({ success: true, errors: [] }) }) it('hands the driver the seeded user of the role and the run context', async () => { writeFixture([T1], ['admin']) const s = spec({ timeoutMs: 1234, captureDir: 'out/captures' }) await execute(s) expect(driverCalls[0]).toMatchObject({ id: 'T-1', role: 'admin', email: 'admin@test.local' }) expect(driverCalls[0].ctx).toEqual({ frontendUrl: FRONT, apiUrl: API, captureDir: 'out/captures', timeoutMs: 1234, cwd: tmp, }) }) it('a failing scenario is counted and its reason propagated; the verdict names the ratio', async () => { writeFixture([T1, T2]) driverImpl = (id) => id === 'T-2' ? { status: 'fail', reason: 'expected testid employees-detail not rendered' } : { status: 'pass', reason: 'ok' } const report = await execute(spec()) expect(report).toMatchObject({ totalTests: 2, passed: 1, failed: 1 }) expect(report.results[1]).toMatchObject({ testId: 'T-2', status: 'fail', reason: 'expected testid employees-detail not rendered', }) const v = verdict(report) expect(v.success).toBe(false) expect(v.errors[0]).toBe('1/2 test(s) failed') }) it('a role with no seeded user fails that test without spawning it; the others still run', async () => { writeFixture([T1, { ...T2, role: 'manager' }], ['admin']) const report = await execute(spec()) expect(report).toMatchObject({ totalTests: 2, passed: 1, failed: 1 }) expect(report.results[1]).toMatchObject({ testId: 'T-2', status: 'fail', reason: 'driver-error: no test user seeded for role "manager"', }) expect(driverCalls.map((c) => c.id)).toEqual(['T-1']) }) it('--test-id runs exactly that test', async () => { writeFixture([T1, T2]) const report = await execute(spec({ testId: 'T-2' })) expect(report.totalTests).toBe(1) expect(driverCalls.map((c) => c.id)).toEqual(['T-2']) }) })