/** * execute.test (uat-api) — the API runner against a fake fetch: per-role login, * measured calls, verdicts, deterministic ordering, login-failure skips, and the * api-results.json artifact. */ import { describe, it, expect, afterEach } from 'vitest'; import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { PlanTestSchema } from '../../lib/plantest-schema.js'; import { ApiRunFileSchema } from '../../lib/run-results.js'; import type { FetchLike } from '../../lib/http.js'; import type { UatUsersFile } from '../../lib/users-file.js'; import { UatApiInputSchema } from '../types.js'; import type { ApiRunContext } from '../validate.js'; import { executeApiRun } from '../execute.js'; const tmpDirs: string[] = []; const makeTmp = (): string => { const dir = mkdtempSync(join(tmpdir(), 'uat-api-')); tmpDirs.push(dir); return dir; }; afterEach(() => { while (tmpDirs.length) rmSync(tmpDirs.pop()!, { recursive: true, force: true }); }); const PLAN = PlanTestSchema.parse({ schema_version: '1.0.0', meta: { application: 'administration', path: 'administration', source_signature: { nav_sha: 'abc' } }, roles: ['admin', 'viewer', 'anonymous'], execution: { screenshots: { out_dir: 'shots' } }, routes: [], endpoints: [ { id: 'get:administration/users', method: 'GET', route: '/api/administration/users', controller: 'UsersController', permission: 'administration.users.read', expected_by_role: { admin: 200, viewer: 403, anonymous: 401 }, }, { id: 'post:administration/users', method: 'POST', route: '/api/administration/users', expected_by_role: { admin: 201, viewer: 403, anonymous: 401 }, }, ], }); const USERS: UatUsersFile = { version: '1', application: 'administration', apiUrl: 'http://api', tenant: { name: 'UAT', slug: 'uat' }, createdAt: 'x', users: [ { role: 'admin', email: 'uat.admin@uat.local', password: 'A!1' }, { role: 'viewer', email: 'uat.viewer@uat.local', password: 'V!1' }, ], }; function makeContext(projectRoot: string, over: Partial = {}): ApiRunContext { return { // These tests exercise the write-probe path, so opt in explicitly (the safe default is off). spec: UatApiInputSchema.parse({ projectPath: projectRoot, runId: 'r1', includeWriteProbes: true }), projectRoot, application: 'administration', plan: PLAN, planRelPath: 'plan.yml', users: USERS, apiUrl: 'http://api', apiUrlSource: 'spec', runDirRel: '.application-test/uat/administration/runs/r1', runId: 'r1', ...over, }; } /** Fake API: logins for both users; statuses by (method × token). */ const fakeFetch: FetchLike = async (url, init) => { const json = (body: unknown, status = 200): Response => new Response(JSON.stringify(body), { status }); const headers = (init?.headers ?? {}) as Record; if (url.endsWith('/api/auth/login')) { const body = JSON.parse(String(init?.body)) as { email: string; password: string }; if (body.password === 'broken') return json({ message: 'bad' }, 401); return json({ token: `tok-${body.email}`, user: { id: 'u', email: body.email, roles: [], permissions: [], tenants: [] } }); } const auth = headers.Authorization ?? ''; const isAdmin = auth.includes('uat.admin'); if ((init?.method ?? 'GET') === 'GET') { if (!auth) return json({ message: 'anon' }, 401); return isAdmin ? json([{ id: 1 }, { id: 2 }]) : json({ message: 'denied' }, 403); } // POST if (!auth) return json({ message: 'anon' }, 401); return isAdmin ? json({ errors: { name: ['required'] } }, 400) : json({ message: 'denied' }, 403); }; describe('executeApiRun', () => { it('runs every call, asserts per mode, measures, and writes the artifact', async () => { const root = makeTmp(); const ctx = makeContext(root); const outcome = await executeApiRun(ctx, { fetchImpl: fakeFetch, nowIso: () => '2026-06-12T00:00:00.000Z' }); expect(outcome.errors).toEqual([]); expect(outcome.success).toBe(true); expect(outcome.aggregate).toMatchObject({ total: 6, executed: 6, skipped: 0, failed: 0, passed: 6 }); const byKey = (id: string, role: string) => outcome.results.find((r) => r.id === id && r.role === role)!; expect(byKey('get:administration/users', 'admin')).toMatchObject({ ok: true, actual: 200, permission: 'administration.users.read', controller: 'UsersController', }); // Passing rows never carry a body excerpt. expect(byKey('get:administration/users', 'admin').bodyExcerpt).toBeUndefined(); expect(byKey('get:administration/users', 'viewer')).toMatchObject({ ok: true, actual: 403 }); expect(byKey('get:administration/users', 'anonymous')).toMatchObject({ ok: true, actual: 401 }); // authz_only pass: admin POST got 400 validation, which proves authorization. expect(byKey('post:administration/users', 'admin')).toMatchObject({ ok: true, actual: 400, note: 'authorized_validation_only' }); expect(byKey('post:administration/users', 'admin').sizeBytes).toBeGreaterThan(0); // Artifact round-trips through the schema. const file = ApiRunFileSchema.parse( JSON.parse(readFileSync(join(root, ctx.runDirRel, 'api-results.json'), 'utf-8')), ); expect(file.kind).toBe('uat-api'); expect(file.meta).toMatchObject({ runId: 'r1', application: 'administration', planSignature: { nav_sha: 'abc' } }); expect(file.results).toHaveLength(6); }); it('flags an RBAC violation when a denied role gets through', async () => { const leakyFetch: FetchLike = async (url, init) => url.endsWith('/api/auth/login') ? new Response(JSON.stringify({ token: 't', user: {} }), { status: 200 }) : new Response('[]', { status: 200 }); // everything 200 — viewer should have been 403 const outcome = await executeApiRun(makeContext(makeTmp()), { fetchImpl: leakyFetch }); const viewerGet = outcome.results.find((r) => r.id === 'get:administration/users' && r.role === 'viewer')!; expect(viewerGet.ok).toBe(false); // A failed assertion carries the response body excerpt — the report's diagnostic. expect(viewerGet.bodyExcerpt).toBe('[]'); expect(outcome.allPassed).toBe(false); expect(outcome.success).toBe(true); // findings ≠ infra failure }); it('extracts a ProblemDetails excerpt on a failing 500', async () => { const crashingFetch: FetchLike = async (url) => url.endsWith('/api/auth/login') ? new Response(JSON.stringify({ token: 't', user: {} }), { status: 200 }) : new Response( JSON.stringify({ title: 'Internal Server Error', detail: 'NullReferenceException in OrdersService' }), { status: 500 }, ); const outcome = await executeApiRun(makeContext(makeTmp()), { fetchImpl: crashingFetch }); const adminGet = outcome.results.find((r) => r.id === 'get:administration/users' && r.role === 'admin')!; expect(adminGet).toMatchObject({ ok: false, actual: 500 }); expect(adminGet.bodyExcerpt).toBe('Internal Server Error — NullReferenceException in OrdersService'); // The artifact round-trips with the new diagnostic fields. const reparsed = ApiRunFileSchema.safeParse( JSON.parse(JSON.stringify({ kind: 'uat-api', meta: { runId: 'r1', application: 'a', planPath: 'p', startedAt: 's' }, results: outcome.results })), ); expect(reparsed.success).toBe(true); }); it('keeps a login-failed role in the results as skips and reports the error', async () => { const users: UatUsersFile = { ...USERS, users: [USERS.users[0], { role: 'viewer', email: 'v@x', password: 'broken' }] }; const outcome = await executeApiRun(makeContext(makeTmp(), { users }), { fetchImpl: fakeFetch }); expect(outcome.success).toBe(false); expect(outcome.errors.join(' ')).toContain('viewer'); const viewerResults = outcome.results.filter((r) => r.role === 'viewer'); expect(viewerResults.every((r) => !r.executed && r.reason === 'login_failed')).toBe(true); // Admin + anonymous still ran fully. expect(outcome.results.filter((r) => r.executed)).toHaveLength(4); }); it('skips allowed writes by default (safe) and flags a real 2xx write when probing is enabled', async () => { // Default (includeWriteProbes omitted → false): the allowed admin POST is a skip, never fired. const safeCtx = makeContext(makeTmp(), { spec: UatApiInputSchema.parse({ projectPath: makeTmp(), runId: 'r1' }) }); const safe = await executeApiRun(safeCtx, { fetchImpl: fakeFetch }); const safeAdminPost = safe.results.find((r) => r.id === 'post:administration/users' && r.role === 'admin')!; expect(safeAdminPost).toMatchObject({ executed: false, reason: 'write_probes_disabled' }); // Opted in, and the empty-body write actually succeeded (201) → flagged, never a silent pass. const mutatingFetch: FetchLike = async (url, init) => { if (url.endsWith('/api/auth/login')) return new Response(JSON.stringify({ token: 'tok-uat.admin@uat.local', user: {} }), { status: 200 }); if ((init?.method ?? 'GET') !== 'GET') { const auth = ((init?.headers ?? {}) as Record).Authorization ?? ''; return auth.includes('uat.admin') ? new Response('{"id":1}', { status: 201 }) : new Response('{}', { status: 403 }); } return new Response('[]', { status: 200 }); }; const probed = await executeApiRun(makeContext(makeTmp()), { fetchImpl: mutatingFetch }); const adminPost = probed.results.find((r) => r.id === 'post:administration/users' && r.role === 'admin')!; expect(adminPost).toMatchObject({ ok: true, actual: 201, note: 'real_write_executed' }); }); it('keeps deterministic result ordering (plan endpoint order × plan role order)', async () => { const outcome = await executeApiRun(makeContext(makeTmp()), { fetchImpl: fakeFetch }); expect(outcome.results.map((r) => `${r.id}:${r.role}`)).toEqual([ 'get:administration/users:admin', 'get:administration/users:viewer', 'get:administration/users:anonymous', 'post:administration/users:admin', 'post:administration/users:viewer', 'post:administration/users:anonymous', ]); }); });