/** * uat-run/execute.ts — The orchestrator's I/O: chain plan → health → provision → * api → ui → report under ONE run id, each phase reusing the sibling CLIs' * validate/execute functions IN-PROCESS (no re-spawn). Failure policy: a failed * phase marks the run failed and skips its dependents, but the report still * renders whatever artifacts exist — partial evidence beats none. */ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; import { spawnSync } from 'node:child_process'; import { findSmartStackStructure } from '../../../lib/detector.js'; import { probeApi } from '../lib/auth-client.js'; import { timedFetch } from '../lib/http.js'; import { resolveApiUrl, resolveFrontendUrl } from '../lib/app-config.js'; import { defaultPlanLocation, loadPlan } from '../lib/plan-io.js'; import { defaultRunId, runDirRelPath } from '../lib/run-id.js'; import { ApiRunFileSchema, UiRunFileSchema, type ApiRunFile, type UiRunFile } from '../lib/run-results.js'; import { validate as validatePlanSpec } from '../uat-plan/validate.js'; import { discover } from '../uat-plan/discover.js'; import { generatePlan } from '../uat-plan/generate.js'; import { validate as validateProvision } from '../uat-provision/validate.js'; import { executeProvision } from '../uat-provision/execute.js'; import { validate as validateApi } from '../uat-api/validate.js'; import { executeApiRun } from '../uat-api/execute.js'; import { validate as validateUi } from '../uat-ui/validate.js'; import { executeUiRun } from '../uat-ui/execute.js'; import { PlaywrightDriver, PlaywrightMissingError } from '../uat-ui/playwright-driver.js'; import { buildModel, resolveThresholds } from '../uat-report/build-model.js'; import { renderHtml } from '../uat-report/render-html.js'; import { collectScreenshotEmbeds } from '../uat-report/screenshot-embeds.js'; import { decidePlanAction, parseSsDevUp, planCarriesRoleCatalog, runSucceeded, signaturesMatch } from './orchestrate.js'; import type { PhaseOutcome, UatRunInput } from './types.js'; export interface RunOutcome { success: boolean; phases: PhaseOutcome[]; data: Record; errors: string[]; warnings: string[]; } interface RunState { spec: UatRunInput; projectRoot: string; apiDir: string; webDir: string; apiUrl: string; frontendUrl: string; runId: string; runDirRel: string; planRelPath: string; phases: PhaseOutcome[]; errors: string[]; warnings: string[]; } const phase = (state: RunState, name: PhaseOutcome['phase'], status: PhaseOutcome['status'], detail: string): void => { state.phases.push({ phase: name, status, detail }); if (status === 'failed') state.errors.push(`[${name}] ${detail}`); }; /** Ensure the plan exists and is drift-fresh per the policy. */ async function ensurePlan(state: RunState): Promise { const { spec, projectRoot } = state; const location = defaultPlanLocation(projectRoot, spec.path, spec.name); state.planRelPath = location.relPath; const planExists = existsSync(location.absPath); if (spec.refreshPlan === 'never') { const action = decidePlanAction('never', planExists, null); if (action === 'fail') { phase(state, 'plan', 'failed', `No plan at ${location.relPath} and refreshPlan=never.`); return false; } phase(state, 'plan', 'ok', `kept ${location.relPath} (refreshPlan=never, drift not checked)`); return true; } // Discovery (DB + sources) — needed for generation AND for the drift check. const v = await validatePlanSpec({ projectPath: projectRoot, path: spec.path, name: spec.name, includeApi: true }); if (!v.valid || !v.spec || !v.layout || !v.connection) { if (planExists) { state.warnings.push( `Drift check unavailable (${v.errors.join('; ') || 'plan discovery failed'}) — keeping the existing plan unverified.`, ); phase(state, 'plan', 'ok', `kept ${location.relPath} (drift NOT verified)`); return true; } phase(state, 'plan', 'failed', `Cannot generate the plan: ${v.errors.join('; ')}`); return false; } let outcome; try { outcome = await discover({ apiDir: v.layout.apiDir, webDir: v.layout.webDir, path: spec.path, connection: v.connection, includeApi: true, }); } catch (e) { if (planExists) { state.warnings.push(`Drift check failed (${(e as Error).message}) — keeping the existing plan unverified.`); phase(state, 'plan', 'ok', `kept ${location.relPath} (drift NOT verified)`); return true; } phase(state, 'plan', 'failed', `Discovery failed: ${(e as Error).message}`); return false; } state.warnings.push(...outcome.warnings); let signatureFresh: boolean | null = null; if (planExists) { const loaded = loadPlan(location.absPath); // A plan without a role_catalog (pre-1.1.0) is never fresh: provisioning joins // roles by catalog id, so the plan must be regenerated to carry the identities. signatureFresh = loaded.ok && planCarriesRoleCatalog(loaded.plan) && signaturesMatch(loaded.plan.meta.source_signature, outcome.result.signature); } const action = decidePlanAction(spec.refreshPlan, planExists, signatureFresh); if (action === 'keep') { phase(state, 'plan', 'ok', `kept ${location.relPath} (signature fresh)`); return true; } const gen = generatePlan(outcome.result, { name: v.layout.name, outDir: v.layout.outDir, includeApi: true, }); for (const file of gen.files) { const abs = resolve(join(projectRoot, file.path)); mkdirSync(dirname(abs), { recursive: true }); writeFileSync(abs, file.content, 'utf-8'); } state.warnings.push(...gen.violations.map((x) => `[inv ${x.invariant}] ${x.where}: ${x.message}`)); phase( state, 'plan', 'ok', action === 'regenerate' ? `regenerated ${location.relPath} (source signature drifted)` : `generated ${location.relPath}`, ); return true; } /** Probe both sides; boot once via `ss dev up --json` when allowed. */ async function ensureHealth(state: RunState): Promise { const probeBoth = async (): Promise<{ api: boolean; front: boolean; apiDetail?: string }> => { const api = await probeApi({ apiUrl: state.apiUrl, timeoutMs: 5000 }); const front = await timedFetch(state.frontendUrl, {}, { timeoutMs: 5000 }); // Carry the probe's diagnosis: an identity failure (a squatter answering non-200 // on /api/config/features) reads very differently from a dead port — booting // the app would not help, freeing the port would. return { api: api.up, front: front.ok, ...(api.error ? { apiDetail: api.error } : {}) }; }; let up = await probeBoth(); if (up.api && up.front) { phase(state, 'health', 'ok', `API ${state.apiUrl} + frontend ${state.frontendUrl} responded`); return true; } if (!state.spec.bootIfDown) { phase( state, 'health', 'failed', `down: ${up.api ? '' : `API ${state.apiUrl}${up.apiDetail ? ` — ${up.apiDetail}` : ''} `}${up.front ? '' : `frontend ${state.frontendUrl}`} (bootIfDown=false — start the app with \`ss dev up\`)`, ); return false; } const boot = spawnSync('ss', ['dev', 'up', '--json'], { shell: true, encoding: 'utf-8', cwd: state.projectRoot, timeout: 420000, }); const parsed = parseSsDevUp(`${boot.stdout ?? ''}`); if (parsed.status !== 'ok') { phase( state, 'health', 'failed', `\`ss dev up\` did not reach ok (${parsed.status}${parsed.message ? `: ${parsed.message}` : ''}) — run it manually, then re-run /uat run.`, ); return false; } up = await probeBoth(); if (up.api && up.front) { phase(state, 'health', 'ok', 'booted via `ss dev up`'); return true; } phase( state, 'health', 'failed', `still down after boot: ${up.api ? '' : `API ${state.apiUrl}${up.apiDetail ? ` — ${up.apiDetail}` : ''} `}${up.front ? '' : `frontend ${state.frontendUrl}`}`, ); return false; } async function runProvision(state: RunState): Promise { const v = await validateProvision({ projectPath: state.projectRoot, planFile: state.planRelPath, apiUrl: state.apiUrl, adminEmail: state.spec.adminEmail, adminPassword: state.spec.adminPassword, tenantName: state.spec.tenantName, tenantSlug: state.spec.tenantSlug, emailDomain: state.spec.emailDomain, timeoutMs: state.spec.timeoutMs, }); if (!v.valid || !v.context) { phase(state, 'provision', 'failed', v.errors.join('; ')); return false; } state.warnings.push(...v.warnings); const outcome = await executeProvision(v.context); state.warnings.push(...outcome.warnings); if (!outcome.success) { phase(state, 'provision', 'failed', outcome.errors.join('; ')); return false; } const kept = outcome.report.users.filter((u) => u.status === 'kept').length; const created = outcome.report.users.filter((u) => u.status === 'created').length; phase(state, 'provision', 'ok', `tenant ${outcome.report.tenant.outcome}; users: ${created} created, ${kept} verified`); return true; } async function runApiAxis(state: RunState): Promise<{ ok: boolean; allPassed: boolean | null; summary?: string }> { const v = await validateApi({ projectPath: state.projectRoot, planFile: state.planRelPath, apiUrl: state.apiUrl, roles: state.spec.roles, runId: state.runId, includeWriteProbes: state.spec.includeWriteProbes, timeoutMs: state.spec.timeoutMs, }); if (!v.valid || !v.context) { phase(state, 'api', 'failed', v.errors.join('; ')); return { ok: false, allPassed: null }; } state.warnings.push(...v.warnings); const outcome = await executeApiRun(v.context); state.warnings.push(...outcome.warnings); const summary = `${outcome.aggregate.passed}/${outcome.aggregate.executed} passed, ${outcome.aggregate.failed} failed, ${outcome.aggregate.skipped} skipped`; if (!outcome.success) { phase(state, 'api', 'failed', `${outcome.errors.join('; ')} (${summary})`); return { ok: false, allPassed: outcome.allPassed, summary }; } phase(state, 'api', 'ok', summary); return { ok: true, allPassed: outcome.allPassed, summary }; } async function runUiAxis(state: RunState): Promise<{ ok: boolean; allPassed: boolean | null; summary?: string }> { const v = await validateUi({ projectPath: state.projectRoot, planFile: state.planRelPath, frontendUrl: state.frontendUrl, roles: state.spec.roles, runId: state.runId, headless: state.spec.headless, screenshots: state.spec.screenshots, writes: state.spec.writes, }); if (!v.valid || !v.context) { phase(state, 'ui', 'failed', v.errors.join('; ')); return { ok: false, allPassed: null }; } state.warnings.push(...v.warnings); const driver = new PlaywrightDriver({ frontendUrl: v.context.frontendUrl, headless: state.spec.headless, slowMo: 0, readinessTimeoutMs: v.context.readinessTimeoutMs, retryOnNotReady: v.context.retryOnNotReady, }); let outcome; try { outcome = await executeUiRun(v.context, { driver }); } catch (e) { if (e instanceof PlaywrightMissingError) { phase(state, 'ui', 'failed', e.message); return { ok: false, allPassed: null }; } throw e; } state.warnings.push(...outcome.warnings); const summary = `${outcome.aggregate.passed}/${outcome.aggregate.executed - outcome.aggregate.indeterminate} passed, ${outcome.aggregate.failed} failed, ${outcome.aggregate.indeterminate} indeterminate, ${outcome.aggregate.skipped} skipped`; if (!outcome.success) { phase(state, 'ui', 'failed', `${outcome.errors.join('; ')} (${summary})`); return { ok: false, allPassed: outcome.allPassed, summary }; } phase(state, 'ui', 'ok', summary); return { ok: true, allPassed: outcome.allPassed, summary }; } function runReport(state: RunState, generatedAt?: string): string | null { const runDirAbs = join(state.projectRoot, state.runDirRel); const read = (file: string, schema: { safeParse: (v: unknown) => { success: boolean; data?: T } }): T | null => { try { const abs = join(runDirAbs, file); if (!existsSync(abs)) return null; const parsed = schema.safeParse(JSON.parse(readFileSync(abs, 'utf-8'))); return parsed.success ? (parsed.data as T) : null; } catch { return null; } }; const apiFile = read('api-results.json', ApiRunFileSchema); const uiFile = read('ui-results.json', UiRunFileSchema); if (!apiFile && !uiFile) { phase(state, 'report', 'skipped', 'no run artifacts to report on'); return null; } // Band UI durations against the SAME thresholds the run used for perfWarnings // (the plan's execution.perf) so the report's colors and the warning count agree. const planAbs = join(state.projectRoot, state.planRelPath); const loaded = state.planRelPath && existsSync(planAbs) ? loadPlan(planAbs) : null; const perf = loaded?.ok ? loaded.plan.execution.perf : null; const model = buildModel({ apiFile, uiFile, title: state.spec.title, generatedAt: generatedAt ?? new Date().toISOString(), thresholds: resolveThresholds({}, perf), runWarnings: [ ...state.phases.map((p) => `[${p.phase}] ${p.status}: ${p.detail}`), ...state.warnings, ], screenshots: collectScreenshotEmbeds(runDirAbs, uiFile), }); const outAbs = join(runDirAbs, 'report.html'); mkdirSync(dirname(outAbs), { recursive: true }); writeFileSync(outAbs, renderHtml(model), 'utf-8'); phase(state, 'report', 'ok', outAbs); return outAbs; } export async function executeRun(spec: UatRunInput): Promise { const projectRoot = resolve(spec.projectPath); const structure = await findSmartStackStructure(projectRoot); const apiDir = structure.api ?? structure.apiExtensions ?? structure.apiCore ?? projectRoot; const webDir = structure.web ?? projectRoot; const application = spec.path.split('/').map((s) => s.trim()).filter(Boolean)[0] ?? spec.path; const runId = spec.runId ?? defaultRunId(); const state: RunState = { spec, projectRoot, apiDir, webDir, apiUrl: resolveApiUrl(apiDir, spec.apiUrl).value, frontendUrl: resolveFrontendUrl(webDir, spec.frontendUrl).value, runId, runDirRel: runDirRelPath(application, runId), planRelPath: '', phases: [], errors: [], warnings: [], }; let healthy = false; let credsReady = false; let apiAxis: { allPassed: boolean | null; summary?: string } = { allPassed: null }; let uiAxis: { allPassed: boolean | null; summary?: string } = { allPassed: null }; healthy = await ensureHealth(state); // Plan generation/drift needs the DB which exists independently of the app — // but a freshly booted app guarantees the connection config is materialized. const planOk = await ensurePlan(state); if (planOk && healthy && !spec.skipProvision) { credsReady = await runProvision(state); } else if (spec.skipProvision) { phase(state, 'provision', 'skipped', 'skipProvision=true (existing uat-users.json will be used)'); credsReady = true; } else { phase(state, 'provision', 'skipped', 'blocked by a failed phase upstream'); } if (planOk && healthy && credsReady && !spec.skipApi) { const r = await runApiAxis(state); apiAxis = r; } else if (spec.skipApi) { phase(state, 'api', 'skipped', 'skipApi=true'); } else { phase(state, 'api', 'skipped', 'blocked by a failed phase upstream'); } if (planOk && healthy && credsReady && !spec.skipUi) { const r = await runUiAxis(state); uiAxis = r; } else if (spec.skipUi) { phase(state, 'ui', 'skipped', 'skipUi=true'); } else { phase(state, 'ui', 'skipped', 'blocked by a failed phase upstream'); } let reportPath: string | null = null; if (!spec.skipReport) { reportPath = runReport(state); } else { phase(state, 'report', 'skipped', 'skipReport=true'); } return { success: runSucceeded(state.phases), phases: state.phases, data: { runId, runDir: state.runDirRel, plan: state.planRelPath, apiUrl: state.apiUrl, frontendUrl: state.frontendUrl, report: reportPath, apiAllPassed: apiAxis.allPassed, uiAllPassed: uiAxis.allPassed, apiSummary: apiAxis.summary ?? null, uiSummary: uiAxis.summary ?? null, phases: state.phases, }, errors: state.errors, warnings: state.warnings, }; }