/** * execute.test (uat-ui) — the runner's verdict/metrics logic against a scripted * FakeDriver: access assertions, console/network gates, INDETERMINATE paths, * login lifecycle, screenshot policy, and the ui-results.json artifact. * No real browser anywhere. */ import { describe, it, expect, afterEach } from 'vitest'; import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { PlanTestSchema } from '../../lib/plantest-schema.js'; import { UiRunFileSchema } from '../../lib/run-results.js'; import type { UatUsersFile } from '../../lib/users-file.js'; import { UatUiInputSchema } from '../types.js'; import type { UiRunContext } from '../validate.js'; import { executeUiRun } from '../execute.js'; import type { FlowOutcome, LoginOutcome, NavOutcome, Observation, UiDriver } from '../driver-types.js'; import type { JourneyStep } from '../walker.js'; const tmpDirs: string[] = []; const makeTmp = (): string => { const dir = mkdtempSync(join(tmpdir(), 'uat-ui-')); tmpDirs.push(dir); return dir; }; afterEach(() => { while (tmpDirs.length) rmSync(tmpDirs.pop()!, { recursive: true, force: true }); }); const PLAN = PlanTestSchema.parse({ schema_version: '1.0.0', meta: { application: 'administration', path: 'administration', source_signature: { nav_sha: 'sig' } }, roles: ['admin', 'anonymous'], execution: { screenshots: { out_dir: 'shots' }, per_role: { anonymous: { mode: 'goto', expect: 'redirect_login' } }, }, routes: [ { id: 'ADMINISTRATION_USERS', component_key: 'administration.users', route: '/administration/users', navigation_strategy: 'menu_click', permission: 'administration.users.read', access: { admin: 'allowed', anonymous: 'redirect_login' }, }, { id: 'ADMINISTRATION_USERS_DETAIL', component_key: 'administration.users.detail', route: '/administration/users/:id', navigation_strategy: 'click_row_in_parent', parent_route: '/administration/users', on_empty_list: 'INDETERMINATE', access: { admin: 'allowed', anonymous: 'redirect_login' }, }, { id: 'ADMINISTRATION_USERS_CREATE', component_key: 'administration.users.create', route: '/administration/users/create', navigation_strategy: 'goto', access: { admin: 'allowed', anonymous: 'redirect_login' }, }, ], endpoints: [], }); const USERS: UatUsersFile = { version: '1', application: 'administration', apiUrl: 'http://api', tenant: { name: 'UAT', slug: 'uat' }, createdAt: 'x', users: [{ role: 'admin', email: 'uat.admin@uat.local', password: 'A!1' }], }; function makeContext(projectRoot: string, over: Partial = {}): UiRunContext { return { spec: UatUiInputSchema.parse({ projectPath: projectRoot, runId: 'r1' }), projectRoot, application: 'administration', plan: PLAN, planRelPath: 'plan.yml', users: USERS, frontendUrl: 'http://front', frontendUrlSource: 'spec', roles: ['admin', 'anonymous'], runDirRel: '.application-test/uat/administration/runs/r1', runId: 'r1', readinessTimeoutMs: 12000, retryOnNotReady: 2, ...over, }; } const okObservation = (over: Partial = {}): Observation => ({ url: 'http://front/administration/users', accessState: 'allowed', perf: { navMs: 80, fullyReadyMs: 500 }, network: { requestCount: 4, transferredBytes: 12_000, failed: [] }, consoleErrors: [], ...over, }); /** Scripted driver: queues per concern, records calls. */ class FakeDriver implements UiDriver { started = false; sessions: string[] = []; screenshots: string[] = []; loginResults: LoginOutcome[] = []; navResults: NavOutcome[] = []; observations: Observation[] = []; flowResults: FlowOutcome[] = []; async start(): Promise { this.started = true; } async stop(): Promise { this.started = false; } async openRoleSession(role: string): Promise { this.sessions.push(role); } async login(): Promise { return this.loginResults.shift() ?? { ok: true }; } async navigate(_step: JourneyStep): Promise { return this.navResults.shift() ?? { used: 'goto', ok: true }; } async observe(): Promise { return this.observations.shift() ?? okObservation(); } async runCreateFlow(): Promise { return this.flowResults.shift() ?? { ok: true, status: 201, detail: 'created (201)' }; } async runEditFlow(): Promise { return this.flowResults.shift() ?? { ok: true, status: 200 }; } async runDeleteFlow(): Promise { return this.flowResults.shift() ?? { ok: true, status: 204 }; } async screenshot(absPath: string): Promise { this.screenshots.push(absPath); return true; } } describe('executeUiRun', () => { it('replays the journeys, asserts verdicts, runs the write flows, writes the artifact', async () => { const root = makeTmp(); const ctx = makeContext(root); const driver = new FakeDriver(); // admin journey: list ok, detail (click_row) ok, create page ok → flows; anonymous: 3 gotos. driver.observations = [ okObservation(), okObservation({ url: 'http://front/administration/users/123' }), okObservation({ url: 'http://front/administration/users/create' }), okObservation({ url: 'http://front/login', accessState: 'redirect_login' }), okObservation({ url: 'http://front/login', accessState: 'redirect_login' }), okObservation({ url: 'http://front/login', accessState: 'redirect_login' }), ]; const outcome = await executeUiRun(ctx, { driver, nowIso: () => '2026-06-12T00:00:00.000Z' }); expect(outcome.errors).toEqual([]); expect(outcome.success).toBe(true); expect(outcome.allPassed).toBe(true); expect(driver.sessions).toEqual(['admin', 'anonymous']); // 3 admin pages + create/delete flows (no edit view in plan → no edit_flow) + 3 anonymous pages. expect(outcome.results.map((r) => `${r.role}:${r.kind}:${r.routeId}`)).toEqual([ 'admin:page:ADMINISTRATION_USERS', 'admin:page:ADMINISTRATION_USERS_DETAIL', 'admin:page:ADMINISTRATION_USERS_CREATE', 'admin:create_flow:ADMINISTRATION_USERS_CREATE', 'admin:delete_flow:ADMINISTRATION_USERS_CREATE', 'anonymous:page:ADMINISTRATION_USERS', 'anonymous:page:ADMINISTRATION_USERS_DETAIL', 'anonymous:page:ADMINISTRATION_USERS_CREATE', ]); expect(outcome.results.every((r) => r.ok)).toBe(true); // Plan route permission travels onto every result shape it applies to. const adminList = outcome.results.find((r) => r.role === 'admin' && r.routeId === 'ADMINISTRATION_USERS')!; expect(adminList.permission).toBe('administration.users.read'); const adminDetail = outcome.results.find((r) => r.role === 'admin' && r.routeId === 'ADMINISTRATION_USERS_DETAIL')!; expect(adminDetail.permission).toBeUndefined(); const file = UiRunFileSchema.parse(JSON.parse(readFileSync(join(root, ctx.runDirRel, 'ui-results.json'), 'utf-8'))); expect(file.kind).toBe('uat-ui'); expect(file.meta).toMatchObject({ runId: 'r1', frontendUrl: 'http://front' }); expect(file.results).toHaveLength(8); }); it('fails an allowed page on access mismatch, console errors, or failed requests', async () => { const ctx = makeContext(makeTmp(), { roles: ['admin'] }); const driver = new FakeDriver(); driver.observations = [ okObservation({ accessState: 'denied' }), // list: expected allowed → FAIL okObservation({ consoleErrors: ['TypeError: boom'] }), // detail: console gate → FAIL okObservation({ network: { requestCount: 3, transferredBytes: 1, failed: [{ url: 'http://api/x', status: 500 }] } }), // create: network gate → FAIL ]; driver.flowResults = [ { ok: false, indeterminate: true, detail: 'validation rejected the synthetic payload', status: 400 }, { ok: false, indeterminate: true, detail: 'no rows' }, ]; const outcome = await executeUiRun(ctx, { driver }); const pages = outcome.results.filter((r) => r.kind === 'page'); expect(pages.map((r) => r.ok)).toEqual([false, false, false]); expect(pages[1].warnings.join(' ')).toContain('console errors'); expect(pages[2].warnings.join(' ')).toContain('failed requests'); // Indeterminate flows are not failures. const flows = outcome.results.filter((r) => r.kind !== 'page'); expect(flows.every((r) => r.actual === 'indeterminate')).toBe(true); expect(outcome.aggregate.failed).toBe(3); expect(outcome.aggregate.indeterminate).toBe(2); expect(outcome.allPassed).toBe(false); expect(outcome.success).toBe(true); // findings, not infra errors }); it('an empty parent list yields INDETERMINATE; a navigation failure yields error', async () => { const ctx = makeContext(makeTmp(), { roles: ['admin'], spec: UatUiInputSchema.parse({ projectPath: '.', writes: false, runId: 'r1' }) }); const driver = new FakeDriver(); driver.navResults = [ { used: 'menu_click', ok: true }, { used: 'click_row', ok: true, emptyParent: true }, { used: 'goto', ok: false, error: 'net::ERR_CONNECTION_REFUSED' }, ]; driver.observations = [okObservation()]; const outcome = await executeUiRun(ctx, { driver }); const [, detail, create] = outcome.results; expect(detail).toMatchObject({ actual: 'indeterminate', executed: true, reason: 'empty_parent_list' }); expect(create).toMatchObject({ actual: 'error', ok: false }); expect(create.error).toContain('REFUSED'); }); it('login failure keeps the journey as skips and surfaces an error; missing creds warn', async () => { const ctx = makeContext(makeTmp(), { roles: ['admin'] }); const driver = new FakeDriver(); driver.loginResults = [{ ok: false, error: 'bad credentials' }]; const outcome = await executeUiRun(ctx, { driver }); expect(outcome.success).toBe(false); expect(outcome.errors[0]).toContain('UI login failed'); expect(outcome.results.every((r) => !r.executed && r.reason === 'login_failed')).toBe(true); // Skips keep the permission too (the report's matrix stays diagnosable). expect(outcome.results.find((r) => r.routeId === 'ADMINISTRATION_USERS')!.permission).toBe('administration.users.read'); const noCreds = makeContext(makeTmp(), { roles: ['admin'], users: { ...USERS, users: [] } }); const driver2 = new FakeDriver(); const outcome2 = await executeUiRun(noCreds, { driver: driver2 }); expect(outcome2.results.every((r) => r.reason === 'no_credentials')).toBe(true); expect(outcome2.warnings.join(' ')).toContain('uat-users.json'); }); it('screenshot policy: failures-only shoots only failing steps; all shoots everything', async () => { const root = makeTmp(); const failuresCtx = makeContext(root, { roles: ['admin'], spec: UatUiInputSchema.parse({ projectPath: '.', screenshots: 'failures', writes: false, runId: 'r1' }) }); const driver = new FakeDriver(); driver.observations = [okObservation({ accessState: 'denied' }), okObservation(), okObservation()]; const outcome = await executeUiRun(failuresCtx, { driver }); expect(outcome.results.filter((r) => !r.ok)).toHaveLength(1); expect(driver.screenshots).toHaveLength(1); expect(outcome.results[0].screenshot).toBe('screenshots/admin/ADMINISTRATION_USERS.png'); const allCtx = makeContext(root, { roles: ['admin'], spec: UatUiInputSchema.parse({ projectPath: '.', screenshots: 'all', writes: false, runId: 'r1' }) }); const driver2 = new FakeDriver(); await executeUiRun(allCtx, { driver: driver2 }); expect(driver2.screenshots).toHaveLength(3); }); it('flags slow pages as warnings without failing them', async () => { const ctx = makeContext(makeTmp(), { roles: ['admin'], spec: UatUiInputSchema.parse({ projectPath: '.', writes: false, runId: 'r1' }) }); const driver = new FakeDriver(); driver.observations = [ okObservation({ perf: { navMs: 10, fullyReadyMs: 9000 } }), // ≥ slow_ms 8000 okObservation({ perf: { navMs: 10, fullyReadyMs: 3500 } }), // ≥ warn_ms 3000 okObservation(), ]; const outcome = await executeUiRun(ctx, { driver }); expect(outcome.results[0].ok).toBe(true); expect(outcome.results[0].warnings.join(' ')).toContain('SLOW'); expect(outcome.results[1].warnings.join(' ')).toContain('perf warning'); expect(outcome.aggregate.perfWarnings).toBe(2); }); });