/** * uat-ui/execute.ts — Replay the role journeys against a UiDriver and emit * `ui-results.json`. * * All verdict logic lives HERE (driver-agnostic, fully unit-tested with a fake * driver): expected-vs-actual access, the no-console-error and no-failed-request * gates on allowed pages, INDETERMINATE handling (empty parent lists, write flows * without affordances), perf warnings against the plan thresholds, screenshot * policy, and the per-role login lifecycle (a role that cannot log in keeps its * steps in the results as skips so the report stays complete). */ import { mkdirSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { UiRunFileSchema, aggregateUi, type UiAggregate, type UiRunFile, type UiRunResult, } from '../lib/run-results.js'; import { pathSlug } from '../lib/run-id.js'; import { buildJourneys, type JourneyStep, type RoleJourney } from './walker.js'; import type { FlowOutcome, UiDriver } from './driver-types.js'; import type { UiRunContext } from './validate.js'; export interface UiExecuteDeps { driver: UiDriver; nowIso?: () => string; } export interface UiExecuteOutcome { success: boolean; allPassed: boolean; aggregate: UiAggregate; resultsFileRel: string; results: UiRunResult[]; errors: string[]; warnings: string[]; } const skippedResult = (step: JourneyStep, role: string, reason: string): UiRunResult => ({ routeId: step.routeId, ...(step.componentKey ? { componentKey: step.componentKey } : {}), role, kind: step.kind, url: step.url, navigationStrategy: step.strategy, navigationUsed: 'none', expected: step.expected, actual: 'indeterminate', ok: false, executed: false, reason, perf: { navMs: 0, fullyReadyMs: 0 }, network: { requestCount: 0, transferredBytes: 0, failed: [] }, consoleErrors: [], warnings: [], ...(step.permission ? { permission: step.permission } : {}), }); const flowToResult = ( step: JourneyStep, role: string, outcome: FlowOutcome, ): UiRunResult => ({ routeId: step.routeId, ...(step.componentKey ? { componentKey: step.componentKey } : {}), role, kind: step.kind, url: step.url, navigationStrategy: step.strategy, navigationUsed: 'goto', expected: 'allowed', actual: outcome.ok ? 'allowed' : outcome.indeterminate ? 'indeterminate' : 'error', ok: outcome.ok, executed: true, perf: { navMs: 0, fullyReadyMs: 0 }, network: { requestCount: 0, transferredBytes: 0, failed: [] }, consoleErrors: [], ...(outcome.detail ? { flowDetail: outcome.detail } : {}), warnings: [], ...(step.permission ? { permission: step.permission } : {}), }); /** Screenshot file path pieces for a step (relative to the run dir). */ export function screenshotRelPath(role: string, step: JourneyStep): string { const suffix = step.kind === 'page' ? '' : `_${step.kind}`; return `screenshots/${pathSlug(role)}/${step.routeId}${suffix}.png`; } async function runJourney( ctx: UiRunContext, journey: RoleJourney, driver: UiDriver, warnings: string[], errors: string[], ): Promise { const results: UiRunResult[] = []; const role = journey.role; if (journey.truncated > 0) { warnings.push(`[${role}] ${journey.truncated} step(s) dropped by the actions_per_role cap.`); } await driver.openRoleSession(role); if (!journey.anonymous) { const cred = ctx.users?.users.find((u) => u.role === role); if (!cred) { warnings.push(`Role "${role}" has no entry in uat-users.json — its UI journey is skipped. Re-run /uat provision.`); return journey.steps.map((s) => skippedResult(s, role, 'no_credentials')); } const loggedIn = await driver.login(cred.email, cred.password); if (!loggedIn.ok) { errors.push(`[${role}] UI login failed (${cred.email}): ${loggedIn.error ?? 'unknown'}`); return journey.steps.map((s) => skippedResult(s, role, 'login_failed')); } } const slowMs = ctx.plan.execution.perf.slow_ms; const warnMs = ctx.plan.execution.perf.warn_ms; for (const step of journey.steps) { if (step.kind !== 'page') { const outcome = step.kind === 'create_flow' ? await driver.runCreateFlow(step) : step.kind === 'edit_flow' ? await driver.runEditFlow(step) : await driver.runDeleteFlow(step); const result = flowToResult(step, role, outcome); if (!result.ok && result.actual === 'error') { // A permitted write that the app refused is a finding, not an infra error. result.warnings.push(`write flow failed: ${outcome.detail ?? 'unknown'}`); } await maybeScreenshot(ctx, driver, step, role, result); results.push(result); continue; } const nav = await driver.navigate(step); if (nav.note || step.note) { const note = [step.note, nav.note].filter(Boolean).join('; '); warnings.push(`[${role}] ${step.routeId}: ${note}`); } if (nav.emptyParent) { const result: UiRunResult = { ...skippedResult(step, role, 'empty_parent_list'), executed: true, navigationUsed: nav.used, warnings: ['parent list empty — verdict INDETERMINATE (plan on_empty_list)'], }; results.push(result); continue; } if (!nav.ok) { const result: UiRunResult = { ...skippedResult(step, role, 'navigation_failed'), executed: true, actual: 'error', navigationUsed: nav.used, ...(nav.error ? { error: nav.error } : {}), }; await maybeScreenshot(ctx, driver, step, role, result); results.push(result); continue; } const obs = await driver.observe(); const stepWarnings: string[] = []; let ok = obs.accessState === step.expected; if (ok && step.expected === 'allowed') { if (step.noConsoleError && obs.consoleErrors.length > 0) { ok = false; stepWarnings.push(`console errors on an allowed page (${obs.consoleErrors.length})`); } const serverErrors = obs.network.failed.filter((f) => f.status >= 500); const clientErrors = obs.network.failed.filter((f) => f.status >= 400 && f.status < 500); if (serverErrors.length > 0 || clientErrors.length > 0) { ok = false; stepWarnings.push( `failed requests on an allowed page (${obs.network.failed.map((f) => `${f.status} ${f.url}`).join(' | ')})`, ); } } if (obs.notReady) stepWarnings.push(`page never settled within ${ctx.readinessTimeoutMs}ms`); if (obs.perf.fullyReadyMs >= slowMs) stepWarnings.push(`SLOW: fully ready in ${obs.perf.fullyReadyMs}ms (≥ ${slowMs}ms)`); else if (obs.perf.fullyReadyMs >= warnMs) stepWarnings.push(`perf warning: fully ready in ${obs.perf.fullyReadyMs}ms (≥ ${warnMs}ms)`); const result: UiRunResult = { routeId: step.routeId, ...(step.componentKey ? { componentKey: step.componentKey } : {}), role, kind: 'page', url: obs.url || step.url, navigationStrategy: step.strategy, navigationUsed: nav.used, expected: step.expected, actual: obs.accessState, ok, executed: true, perf: obs.perf, network: obs.network, consoleErrors: obs.consoleErrors, warnings: stepWarnings, ...(step.permission ? { permission: step.permission } : {}), }; await maybeScreenshot(ctx, driver, step, role, result); results.push(result); } return results; } async function maybeScreenshot( ctx: UiRunContext, driver: UiDriver, step: JourneyStep, role: string, result: UiRunResult, ): Promise { const policy = ctx.spec.screenshots; if (policy === 'none') return; if (policy === 'failures' && result.ok) return; const rel = screenshotRelPath(role, step); const abs = join(ctx.projectRoot, ctx.runDirRel, rel); mkdirSync(dirname(abs), { recursive: true }); if (await driver.screenshot(abs)) result.screenshot = rel; } export async function executeUiRun(ctx: UiRunContext, deps: UiExecuteDeps): Promise { const nowIso = deps.nowIso ?? ((): string => new Date().toISOString()); const warnings: string[] = []; const errors: string[] = []; const startedAt = nowIso(); const journeys = buildJourneys(ctx.plan, { roles: ctx.roles, writes: ctx.spec.writes, runTag: ctx.runId.replace(/[^0-9]/g, '').slice(-6) || 'run', maxSteps: ctx.plan.execution.caps.actions_per_role, }); const results: UiRunResult[] = []; await deps.driver.start(); try { for (const journey of journeys) { results.push(...(await runJourney(ctx, journey, deps.driver, warnings, errors))); } } finally { await deps.driver.stop(); } const aggregate = aggregateUi(results, ctx.plan.execution.perf.warn_ms); const file: UiRunFile = UiRunFileSchema.parse({ kind: 'uat-ui', meta: { runId: ctx.runId, application: ctx.application, planPath: ctx.planRelPath, planSignature: ctx.plan.meta.source_signature, frontendUrl: ctx.frontendUrl, roles: journeys.map((j) => j.role), startedAt, finishedAt: nowIso(), }, results, }); const resultsFileRel = `${ctx.runDirRel}/ui-results.json`; const abs = join(ctx.projectRoot, resultsFileRel); mkdirSync(dirname(abs), { recursive: true }); writeFileSync(abs, `${JSON.stringify(file, null, 2)}\n`, 'utf-8'); return { success: errors.length === 0, allPassed: aggregate.failed === 0 && errors.length === 0, aggregate, resultsFileRel, results, errors, warnings, }; }