/** * uat/cli/lib/run-results.ts — The run-artifact contract shared by the runners and * the report: `api-results.json` (uat-api) and `ui-results.json` (uat-ui), one * entry per (unit × role), each carrying the three core metrics — status/verdict, * duration, size — plus the aggregation helpers the HTML report renders. * * PURE: schemas + math only. Zod schemas stay in the v3∩v4 subset (full literal * `.default()`s — never a `{}` relying on inner defaults; the deployed skill * runtime is zod v4 which does not re-parse defaults). */ import { z } from 'zod'; // ── API axis ──────────────────────────────────────────────────────────────── /** * How the runner asserted the call: * - exact: actual must equal expected (GETs, and writes expected to be denied). * - authz_only: write the role IS allowed to perform, probed with an EMPTY body — * passes when the gate lets it through (anything but 401/403); the * expected 2xx is unreachable without a real payload, which only the * UI axis (real form submits) provides. */ export const ApiAssertModeSchema = z.enum(['exact', 'authz_only']); export type ApiAssertMode = z.infer; export const ApiRunResultSchema = z.object({ id: z.string().min(1), method: z.enum(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']), route: z.string().min(1), role: z.string().min(1), mode: ApiAssertModeSchema, expected: z.number().int(), /** HTTP status received (0 = network error; null = not executed). */ actual: z.number().int().nullable(), /** Assertion verdict (false when not executed — see `executed`). */ ok: z.boolean(), /** False when the call was skipped (see reason). */ executed: z.boolean(), /** Why the call was skipped: route_params | no_credentials | login_failed. */ reason: z.string().optional(), /** Extra context, e.g. 'authorized_validation_only' for authz_only passes. */ note: z.string().optional(), durationMs: z.number().int().default(0), sizeBytes: z.number().int().default(0), error: z.string().optional(), /** Permission the plan resolved for this endpoint (RBAC diagnosis in the report). */ permission: z.string().optional(), /** How the plan obtained it (declared / declared-unseeded / inferred). */ permissionSource: z.string().optional(), /** The endpoint carries NO permission gate ([RequirePermission] absent, * no [AllowAnonymous]) — its expected-200-for-every-role rows are the * DEPLOYED behaviour, flagged as a defect surface (DEV-API-033), never a * certification. */ ungated: z.boolean().optional(), /** Controller class the endpoint was discovered on. */ controller: z.string().optional(), /** * Bounded excerpt of the response body — present ONLY on executed failures * (ProblemDetails title/detail when the body is JSON, else raw text, capped). */ bodyExcerpt: z.string().optional(), }); export type ApiRunResult = z.infer; // ── UI axis ───────────────────────────────────────────────────────────────── export const UiAccessOutcomeSchema = z.enum(['allowed', 'denied', 'redirect_login', 'error', 'indeterminate']); export type UiAccessOutcome = z.infer; /** page = navigate+assert; *_flow = the real user write journeys on UAT-created data. */ export const UiStepKindSchema = z.enum(['page', 'create_flow', 'edit_flow', 'delete_flow']); export type UiStepKind = z.infer; export const UiPerfSchema = z.object({ /** Navigation action → load committed. */ navMs: z.number().int().default(0), /** First contentful paint when the browser exposes it (full navigations only). */ ttfpMs: z.number().int().optional(), /** Navigation action → fully ready (spinners gone + network idle). */ fullyReadyMs: z.number().int().default(0), }); export const UiNetworkSchema = z.object({ requestCount: z.number().int().default(0), transferredBytes: z.number().int().default(0), /** 4xx/5xx responses observed while the page settled. */ failed: z.array(z.object({ url: z.string(), status: z.number().int() })).default([]), }); export const UiRunResultSchema = z.object({ routeId: z.string().min(1), componentKey: z.string().optional(), role: z.string().min(1), kind: UiStepKindSchema.default('page'), url: z.string().min(1), navigationStrategy: z.string().default('goto'), /** How the page was actually reached (menu_click | click_row | goto | none). */ navigationUsed: z.string().default('none'), expected: UiAccessOutcomeSchema, actual: UiAccessOutcomeSchema, ok: z.boolean(), executed: z.boolean().default(true), reason: z.string().optional(), perf: UiPerfSchema.default({ navMs: 0, fullyReadyMs: 0 }), network: UiNetworkSchema.default({ requestCount: 0, transferredBytes: 0, failed: [] }), consoleErrors: z.array(z.string()).default([]), screenshot: z.string().optional(), /** Outcome detail for *_flow steps (HTTP status of the submit, validation notes…). */ flowDetail: z.string().optional(), error: z.string().optional(), warnings: z.array(z.string()).default([]), /** Permission the plan resolved for this route (RBAC diagnosis in the report). */ permission: z.string().optional(), }); export type UiRunResult = z.infer; // ── Run files ─────────────────────────────────────────────────────────────── export const RunMetaSchema = z.object({ runId: z.string().min(1), application: z.string().min(1), planPath: z.string().min(1), /** The plan's source signature at run time (drift evidence in the report). */ planSignature: z .object({ nav_sha: z.string().optional(), rbac_sha: z.string().optional(), registry_sha: z.string().optional(), }) .default({}), apiUrl: z.string().optional(), frontendUrl: z.string().optional(), roles: z.array(z.string()).default([]), startedAt: z.string().min(1), finishedAt: z.string().optional(), }); export type RunMeta = z.infer; export const ApiRunFileSchema = z.object({ kind: z.literal('uat-api'), meta: RunMetaSchema, results: z.array(ApiRunResultSchema).default([]), }); export type ApiRunFile = z.infer; export const UiRunFileSchema = z.object({ kind: z.literal('uat-ui'), meta: RunMetaSchema, results: z.array(UiRunResultSchema).default([]), }); export type UiRunFile = z.infer; // ── Aggregation (consumed by the report + envelopes) ──────────────────────── export type DurationBand = 'ok' | 'warn' | 'slow'; /** * The SINGLE source of the duration bands. The plan schema's `execution.perf` * defaults mirror the UI pair; `resolveThresholds` (report) falls back here. */ export const DEFAULT_THRESHOLDS = { apiWarnMs: 1000, apiSlowMs: 3000, uiWarnMs: 3000, uiSlowMs: 8000, } as const; /** Band a duration against the warn/slow thresholds (warn ≤ ms < slow → warn). */ export function classifyDuration(ms: number, warnMs: number, slowMs: number): DurationBand { if (ms >= slowMs) return 'slow'; if (ms >= warnMs) return 'warn'; return 'ok'; } /** p-th percentile (nearest-rank) of a list; 0 for an empty list. */ export function percentile(values: readonly number[], p: number): number { if (values.length === 0) return 0; const sorted = [...values].sort((a, b) => a - b); const rank = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)); return sorted[rank]; } export interface ApiAggregate { total: number; executed: number; passed: number; failed: number; skipped: number; avgDurationMs: number; p95DurationMs: number; totalBytes: number; } export function aggregateApi(results: readonly ApiRunResult[]): ApiAggregate { const executed = results.filter((r) => r.executed); const durations = executed.map((r) => r.durationMs); const passed = executed.filter((r) => r.ok).length; return { total: results.length, executed: executed.length, passed, failed: executed.length - passed, skipped: results.length - executed.length, avgDurationMs: durations.length ? Math.round(durations.reduce((a, b) => a + b, 0) / durations.length) : 0, p95DurationMs: percentile(durations, 95), totalBytes: executed.reduce((a, r) => a + r.sizeBytes, 0), }; } export interface UiAggregate { total: number; executed: number; passed: number; failed: number; skipped: number; indeterminate: number; /** Executed page steps whose fullyReady breached warn_ms (perf advisory, not a failure). */ perfWarnings: number; avgFullyReadyMs: number; p95FullyReadyMs: number; totalBytes: number; consoleErrorCount: number; } export function aggregateUi(results: readonly UiRunResult[], warnMs: number): UiAggregate { const executed = results.filter((r) => r.executed); const determinate = executed.filter((r) => r.actual !== 'indeterminate'); const passed = determinate.filter((r) => r.ok).length; const readies = executed.filter((r) => r.kind === 'page').map((r) => r.perf.fullyReadyMs); return { total: results.length, executed: executed.length, passed, failed: determinate.length - passed, skipped: results.length - executed.length, indeterminate: executed.length - determinate.length, perfWarnings: readies.filter((ms) => ms >= warnMs).length, avgFullyReadyMs: readies.length ? Math.round(readies.reduce((a, b) => a + b, 0) / readies.length) : 0, p95FullyReadyMs: percentile(readies, 95), totalBytes: executed.reduce((a, r) => a + r.network.transferredBytes, 0), consoleErrorCount: executed.reduce((a, r) => a + r.consoleErrors.length, 0), }; } /** Per-role rollup across both axes (report's role summary table). */ export interface RoleRollup { role: string; apiPassed: number; apiFailed: number; uiPassed: number; uiFailed: number; } export function rollupByRole( roles: readonly string[], api: readonly ApiRunResult[], ui: readonly UiRunResult[], ): RoleRollup[] { return roles.map((role) => { const a = api.filter((r) => r.role === role && r.executed); const u = ui.filter((r) => r.role === role && r.executed && r.actual !== 'indeterminate'); return { role, apiPassed: a.filter((r) => r.ok).length, apiFailed: a.filter((r) => !r.ok).length, uiPassed: u.filter((r) => r.ok).length, uiFailed: u.filter((r) => !r.ok).length, }; }); } /** Human-readable byte count (B / KB / MB, one decimal). */ export function humanBytes(bytes: number): string { if (bytes < 1024) return `${bytes} B`; if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; }