import { chromium, firefox, webkit, type Page, type Locator } from 'playwright'; import type { ExecutionStep } from '@detiq/agents'; import { assertCrawlUrlSafe } from '@detiq/core'; import { analyzeFailure } from './failure-analysis-agent.js'; export interface StepResult { stepNumber: number; description: string; status: 'PASSED' | 'FAILED' | 'SKIPPED'; duration: number; error?: string; screenshotPath?: string; healed?: boolean; /** 'mechanical' = same selector text, different Playwright locator strategy. * 'ai' = AI Self-Healing Agent matched a semantically different element * (the app's selector text itself changed, e.g. "Sign In" -> "Log In"). */ healedVia?: 'mechanical' | 'ai'; healReasoning?: string; healConfidence?: number; elementBoundingRect?: { x: number; y: number; width: number; height: number } | null; elementSelector?: string; } export interface ExecutionResult { stepResults: StepResult[]; status: 'PASSED' | 'FAILED' | 'ERROR' | 'FLAKY'; errorStep?: number; totalDuration: number; finalScreenshotPath?: string; retryCount?: number; consoleLogs?: Array<{ type: string; text: string; timestamp: number }>; networkErrors?: Array<{ url: string; status: number; method: string; timestamp: number }>; rootCauseSummary?: string; networkTimings?: Array<{ url: string; method: string; status: number; duration: number }>; } const LOCATOR_STRATEGIES: ExecutionStep['selectorStrategy'][] = ['text', 'label', 'role', 'css']; function buildLocator(page: Page, selector: string, strategy: ExecutionStep['selectorStrategy'] = 'text'): Locator { switch (strategy) { case 'css': return page.locator(selector); case 'text': return page.getByText(selector, { exact: false }); case 'role': return page.getByRole(selector as any); case 'label': return page.getByLabel(selector); default: return page.locator(selector); } } /** Scans the live page for visible interactive elements — the "current page elements" * fed to the AI Self-Healing Agent when mechanical locator strategies fail. */ async function extractCurrentPageElements(page: Page): Promise> { return page.evaluate(() => { const els = Array.from(document.querySelectorAll( 'a, button, input, select, textarea, [role="button"], [role="link"], [role="tab"], [role="menuitem"], [aria-label]' )); return els .filter((el) => { const rect = el.getBoundingClientRect(); return rect.width > 0 && rect.height > 0; }) .slice(0, 200) .map((el) => { const ariaLabel = el.getAttribute('aria-label') ?? undefined; const textContent = (el.textContent ?? '').replace(/\s+/g, ' ').trim().slice(0, 100) || undefined; const role = el.getAttribute('role') ?? el.tagName.toLowerCase(); return { meaning: ariaLabel || textContent || role, role, ariaLabel, textContent }; }); }); } /** * Resolves a step's target element in three tiers: * 1. Exact selector + strategy as generated. * 2. Mechanical: same selector text, other Playwright locator strategies * (handles a strategy mismatch, not an actual UI change). * 3. AI: when tenantId/projectId are supplied, asks the Self-Healing Agent to find * the current page's best semantic match for the failed step description — the * tier that actually survives the app's selector text changing. */ async function resolveLocator( page: Page, step: ExecutionStep, healCtx?: { tenantId: string; projectId: string }, ): Promise<{ loc: Locator; healed: boolean; healedVia?: 'mechanical' | 'ai'; healReasoning?: string; healConfidence?: number } | null> { const { selector, selectorStrategy = 'text' } = step; if (!selector) return null; const primary = buildLocator(page, selector, selectorStrategy); const attached = await primary.first().isVisible({ timeout: parseInt(process.env.TEST_WAIT_TIMEOUT_MS ?? '1500', 10) }).catch(() => false); if (attached) return { loc: primary, healed: false }; // Tier 2 — mechanical: try other strategies before giving up for (const strategy of LOCATOR_STRATEGIES) { if (strategy === selectorStrategy) continue; const loc = buildLocator(page, selector, strategy); const visible = await loc.first().isVisible({ timeout: parseInt(process.env.TEST_SHORT_TIMEOUT_MS ?? '800', 10) }).catch(() => false); if (visible) return { loc, healed: true, healedVia: 'mechanical' }; } // Tier 3 — AI: the selector text itself may no longer match anything on the page. if (healCtx) { try { const { healElement } = await import('@detiq/agents'); const currentPageElements = await extractCurrentPageElements(page); const result = await healElement({ tenantId: healCtx.tenantId, projectId: healCtx.projectId, failedElementMeaning: step.description || selector, currentPageElements, }); if (result.healed && result.updatedTest) { const healedLoc = page.getByText(result.updatedTest.elementMeaning, { exact: false }); const visible = await healedLoc.first().isVisible({ timeout: parseInt(process.env.TEST_WAIT_TIMEOUT_MS ?? '1500', 10) }).catch(() => false); if (visible) { return { loc: healedLoc, healed: true, healedVia: 'ai', healReasoning: result.reasoning, healConfidence: result.confidence, }; } } } catch { // AI healing is a best-effort last resort — any failure here (LLM error, bad // JSON, etc.) just falls through to the caller's normal Playwright error below. } } // Return primary — caller will fail with a meaningful Playwright error return { loc: primary, healed: false }; } function applyHealInfo(result: StepResult, r: { healed: boolean; healedVia?: 'mechanical' | 'ai'; healReasoning?: string; healConfidence?: number }): void { result.healed = r.healed; result.healedVia = r.healedVia; result.healReasoning = r.healReasoning; result.healConfidence = r.healConfidence; } export async function executeExecutionStep( page: Page, step: ExecutionStep, options: { healCtx?: { tenantId: string; projectId: string } } = {}, ): Promise> { const result: Pick = {}; const timeout = step.timeout ?? parseInt(process.env.TEST_STEP_TIMEOUT_MS ?? '8000', 10); switch (step.action) { case 'navigate': if (!step.url) throw new Error('Navigate step missing url'); assertCrawlUrlSafe(step.url); await page.goto(step.url, { waitUntil: 'domcontentloaded', timeout }); break; case 'click': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; await r.loc.first().click({ timeout }); applyHealInfo(result as StepResult, r); break; } case 'fill': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; await r.loc.first().fill(step.value ?? '', { timeout }); applyHealInfo(result as StepResult, r); break; } case 'select': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; await r.loc.first().selectOption(step.value ?? '', { timeout }); applyHealInfo(result as StepResult, r); break; } case 'hover': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; await r.loc.first().hover({ timeout }); applyHealInfo(result as StepResult, r); break; } case 'press_key': await page.keyboard.press(step.value ?? 'Enter'); break; case 'assert_visible': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); await r.loc.first().waitFor({ state: 'visible', timeout }); result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; applyHealInfo(result as StepResult, r); break; } case 'assert_text': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); const text = await r.loc.first().textContent({ timeout }); if (!text?.includes(step.value ?? '')) { throw new Error(`Expected text "${step.value}" not found. Got: "${text?.slice(0, 100)}"`); } result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; applyHealInfo(result as StepResult, r); break; } case 'assert_url': { const url = page.url(); if (!url.includes(step.value ?? '')) { throw new Error(`Expected URL to contain "${step.value}". Got: "${url}"`); } break; } case 'wait': await page.waitForTimeout(step.timeout ?? parseInt(process.env.TEST_STEP_DELAY_MS ?? '2000', 10)); break; case 'screenshot': break; case 'script': case 'setVariable': if (step.value) { await page.evaluate(step.value).catch(err => { throw new Error(`Script execution failed: ${err.message}`); }); } break; } return result; } async function runPlan( page: Page, plan: ExecutionStep[], options: { screenshotDir: string; testRunId: string; onStepComplete?: (result: StepResult) => void; healCtx?: { tenantId: string; projectId: string } }, consoleLogs: Array<{ type: string; text: string; timestamp: number }>, networkErrors: Array<{ url: string; status: number; method: string; timestamp: number }>, ): Promise<{ stepResults: StepResult[]; errorStep?: number; finalScreenshotPath?: string; consoleLogs: typeof consoleLogs; networkErrors: typeof networkErrors }> { const stepResults: StepResult[] = []; let errorStep: number | undefined; let finalScreenshotPath: string | undefined; for (const step of plan) { const stepStart = Date.now(); const result: StepResult = { stepNumber: step.stepNumber, description: step.description, status: 'FAILED', duration: 0, }; try { const timeout = step.timeout ?? parseInt(process.env.TEST_STEP_TIMEOUT_MS ?? '8000', 10); switch (step.action) { case 'navigate': if (!step.url) throw new Error('Navigate step missing url'); assertCrawlUrlSafe(step.url); await page.goto(step.url, { waitUntil: 'domcontentloaded', timeout }); break; case 'click': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; await r.loc.first().click({ timeout }); applyHealInfo(result, r); break; } case 'fill': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; await r.loc.first().fill(step.value ?? '', { timeout }); applyHealInfo(result, r); break; } case 'select': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; await r.loc.first().selectOption(step.value ?? '', { timeout }); applyHealInfo(result, r); break; } case 'hover': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; await r.loc.first().hover({ timeout }); applyHealInfo(result, r); break; } case 'press_key': await page.keyboard.press(step.value ?? 'Enter'); break; case 'assert_visible': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); await r.loc.first().waitFor({ state: 'visible', timeout }); result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; applyHealInfo(result, r); break; } case 'assert_text': { const r = await resolveLocator(page, step, options.healCtx); if (!r) throw new Error('Could not resolve locator'); const text = await r.loc.first().textContent({ timeout }); if (!text?.includes(step.value ?? '')) { throw new Error(`Expected text "${step.value}" not found. Got: "${text?.slice(0, 100)}"`); } result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null); result.elementSelector = step.selector; applyHealInfo(result, r); break; } case 'assert_url': { const url = page.url(); if (!url.includes(step.value ?? '')) { throw new Error(`Expected URL to contain "${step.value}". Got: "${url}"`); } break; } case 'wait': await page.waitForTimeout(step.timeout ?? parseInt(process.env.TEST_STEP_DELAY_MS ?? '2000', 10)); break; case 'screenshot': { const ssPath = `${options.screenshotDir}/${options.testRunId}-step-${step.stepNumber}.png`; await page.screenshot({ path: ssPath, fullPage: true }); result.screenshotPath = ssPath; break; } case 'script': case 'setVariable': { if (step.value) { // Execute arbitrary JS string. Often used by manual test builder extension for custom variables // or assertions. await page.evaluate(step.value).catch(err => { throw new Error(`Script execution failed: ${err.message}`); }); } break; } } result.status = 'PASSED'; } catch (err: any) { result.status = step.optional ? 'SKIPPED' : 'FAILED'; result.error = err.message; if (!step.optional) { errorStep = step.stepNumber; } } try { await page.waitForLoadState('networkidle', { timeout: parseInt(process.env.TEST_NETWORK_IDLE_TIMEOUT_MS ?? '1500', 10) }).catch(() => {}); if (!result.screenshotPath) { const ssPath = `${options.screenshotDir}/${options.testRunId}-step-${step.stepNumber}.png`; await page.screenshot({ path: ssPath, fullPage: true }); result.screenshotPath = ssPath; } } catch { /* Screenshot evidence is best-effort and should not fail the run. */ } result.duration = Date.now() - stepStart; stepResults.push(result); options.onStepComplete?.(result); if (result.status === 'FAILED') break; } try { const finalPath = `${options.screenshotDir}/${options.testRunId}-final.png`; await page.screenshot({ path: finalPath, fullPage: true }); finalScreenshotPath = finalPath; } catch { /* non-fatal */ } return { stepResults, errorStep, finalScreenshotPath, consoleLogs, networkErrors }; } function interpolate(template: string | undefined, data: Record): string | undefined { if (!template) return template; return template.replace(/\{\{(\w+)\}\}/g, (_, key) => data[key] ?? `{{${key}}}`); } export async function executeTestPlan( plan: ExecutionStep[], options: { screenshotDir: string; testRunId: string; viewport?: { width: number; height: number }; onStepComplete?: (result: StepResult) => void; /** Run the test up to this many times to detect flakiness (default 1). */ maxRuns?: number; browser?: 'chromium' | 'firefox' | 'webkit'; /** Data row for parameterized execution: substitutes {{varname}} in step fields. */ dataRow?: Record; /** Enables the AI Self-Healing Agent tier when a step's selector can't be resolved * even after trying alternate locator strategies. Omit to keep mechanical-only * healing (no LLM calls during execution). */ healCtx?: { tenantId: string; projectId: string }; /** Decrypted Playwright storageState JSON — restores a logged-in session so * steps don't have to re-authenticate. Without this, any test case behind a * login wall fails at the first authenticated step. */ storageState?: string; }, ): Promise { if (options.dataRow && Object.keys(options.dataRow).length > 0) { const data = options.dataRow; plan = plan.map((step) => ({ ...step, url: interpolate(step.url, data), selector: interpolate(step.selector, data), value: interpolate(step.value, data), description: interpolate(step.description, data) ?? step.description, })); } const maxRuns = options.maxRuns ?? 1; const browserEngine = { chromium, firefox, webkit }[options.browser ?? 'chromium'] ?? chromium; const start = Date.now(); const outcomes: ('PASSED' | 'FAILED')[] = []; let lastResult: Awaited> | null = null; let lastNetworkTimings: Array<{ url: string; method: string; status: number; duration: number }> = []; for (let run = 0; run < maxRuns; run++) { const browser = await browserEngine.launch({ headless: true, // Optional override for environments with a non-standard/pre-provisioned // browser install (e.g. a custom base image with only system Chromium). ...(process.env.PLAYWRIGHT_EXECUTABLE_PATH ? { executablePath: process.env.PLAYWRIGHT_EXECUTABLE_PATH } : {}), }); let parsedStorageState: any; try { parsedStorageState = options.storageState ? JSON.parse(options.storageState) : undefined; } catch { parsedStorageState = undefined; } const context = await browser.newContext({ viewport: options.viewport ?? { width: 1440, height: 900 }, ...(parsedStorageState ? { storageState: parsedStorageState } : {}), }); const page = await context.newPage(); await page .addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; transition-duration: 0s !important; }' }) .catch(() => {}); const consoleLogs: Array<{ type: string; text: string; timestamp: number }> = []; const networkErrors: Array<{ url: string; status: number; method: string; timestamp: number }> = []; const networkTimings: Array<{ url: string; method: string; status: number; duration: number }> = []; const requestStartTimes = new Map(); page.on('console', msg => { if (consoleLogs.length < 100) { consoleLogs.push({ type: msg.type(), text: msg.text().slice(0, 500), timestamp: Date.now() }); } }); page.on('request', request => { const url = request.url(); if (url.includes('/api/') || url.match(/\/(graphql|rest|v\d)\//)) { requestStartTimes.set(request.url(), Date.now()); } }); page.on('response', response => { const url = response.url(); const start = requestStartTimes.get(url); if (start && networkTimings.length < 200) { networkTimings.push({ url: url.slice(0, 300), method: response.request().method(), status: response.status(), duration: Date.now() - start }); requestStartTimes.delete(url); } if (!response.ok() && !url.includes('favicon') && networkErrors.length < 50) { networkErrors.push({ url: url.slice(0, 300), status: response.status(), method: response.request().method(), timestamp: Date.now(), }); } }); const TEST_EXECUTION_TIMEOUT_MS = 5 * 60_000; lastResult = await Promise.race([ runPlan(page, plan, options, consoleLogs, networkErrors), new Promise((_, reject) => setTimeout(() => reject(new Error('Test execution timed out')), TEST_EXECUTION_TIMEOUT_MS) ), ]); lastNetworkTimings = networkTimings; await browser.close(); const passed = !lastResult.stepResults.some((r) => r.status === 'FAILED'); outcomes.push(passed ? 'PASSED' : 'FAILED'); // If all subsequent runs are consistent with first, no need to keep going if (run > 0 && outcomes.every((o) => o === outcomes[0])) break; } const passCount = outcomes.filter((o) => o === 'PASSED').length; const failCount = outcomes.filter((o) => o === 'FAILED').length; let status: ExecutionResult['status']; if (maxRuns > 1 && passCount > 0 && failCount > 0) { status = 'FLAKY'; } else if (passCount === outcomes.length) { status = 'PASSED'; } else { status = 'FAILED'; } let rootCauseSummary: string | undefined; if (status === 'FAILED' || status === 'FLAKY') { const failedStep = lastResult!.stepResults.find(s => s.status === 'FAILED'); rootCauseSummary = await analyzeFailure({ tenantId: options.healCtx?.tenantId, error: failedStep?.error ?? 'Unknown error', stepDescription: failedStep?.description, consoleLogs: lastResult!.consoleLogs, networkErrors: lastResult!.networkErrors, screenshotPath: lastResult!.finalScreenshotPath, }).catch(() => undefined) ?? undefined; } return { stepResults: lastResult!.stepResults, status, errorStep: lastResult!.errorStep, totalDuration: Date.now() - start, finalScreenshotPath: lastResult!.finalScreenshotPath, retryCount: outcomes.length - 1, consoleLogs: lastResult!.consoleLogs, networkErrors: lastResult!.networkErrors, rootCauseSummary, networkTimings: lastNetworkTimings, }; }