/** * Interactive Probe — Phase 2 Dynamic Observation * * The static crawl captures DOM structure at rest. This module runs AFTER the static crawl * to observe dynamic behaviour by actually interacting with the UI: * * 1. Empty submit → captures required-field validation error messages * 2. Invalid format → captures format validation errors (bad email, short password) * 3. Button clicks → captures modal/dialog contents * 4. Select changes → captures conditional reveals (show/hide logic) * 5. Success paths → captures toast/banner messages (where input is safe) * * Results are stored as KnowledgeChunks (sourceType='probe') and update the * ERROR_CATALOGUE + INTERACTION_PATTERNS skill files with real observed data * instead of LLM inference. * * Call signature: probeScreen(page, screenId, elements, opts) * Called from crawlProject() after static element extraction when opts.interactive=true. */ import type { Page } from 'playwright'; import { AppBrain } from '@detiq/app-brain'; export interface ProbeElementFact { meaning: string; role: string; expectedData?: string; } export interface ValidationError { field: string; trigger: 'empty_submit' | 'invalid_format' | 'boundary'; errorText: string; } export interface SuccessMessage { type: 'toast' | 'inline_success' | 'redirect'; content: string; } export interface ModalObservation { triggerElement: string; title: string; content: string; fields: string[]; } export interface ConditionalReveal { triggerElement: string; triggerValue: string; revealedText: string; } export interface TooltipCapture { triggerLabel: string; tooltipText: string; } export interface TabPanelCapture { tabLabel: string; panelContent: string; panelHeading?: string; } export interface CarouselSlide { slideIndex: number; heading?: string; content: string; } export interface WizardStep { stepIndex: number; stepLabel?: string; content: string; fields: string[]; } export interface FocusTrapResult { modalTrigger: string; focusTrapped: boolean; sampledFocused: string[]; } export interface ContextMenuCapture { trigger: string; items: string[]; } export interface ProbeResult { screenId: string; url: string; validationErrors: ValidationError[]; successMessages: SuccessMessage[]; modals: ModalObservation[]; conditionals: ConditionalReveal[]; loadingPatterns: string[]; tooltips?: TooltipCapture[]; tabPanels?: TabPanelCapture[]; carouselSlides?: CarouselSlide[]; wizardSteps?: WizardStep[]; focusTraps?: FocusTrapResult[]; skipNavIssues?: string[]; contextMenus?: ContextMenuCapture[]; } // ── CSS selectors that reliably find error messages across frameworks ──────── const ERROR_SELECTORS = [ '[role="alert"]', '[aria-live="assertive"]', '[aria-live="polite"]', '.invalid-feedback', '.field-error', '.error-message', '.form-error', '.validation-error', '[class*="error" i]', '[class*="invalid" i]', '[class*="Error"]', 'p.text-red-500', 'p.text-red-600', 'p.text-destructive', 'p.text-danger', '[data-error]', '[data-invalid]', '.ant-form-item-explain-error', // Ant Design '.el-form-item__error', // Element UI '.chakra-form__error-message', // Chakra UI '.mantine-InputWrapper-error', // Mantine ].join(', '); // ── Toast / notification selectors ─────────────────────────────────────────── const TOAST_SELECTORS = [ '[role="status"]', '[data-sonner-toast]', '[data-toast]', '[data-hot-toast]', '.Toastify__toast', '.toast-message', '.notification-body', '.notistack-snackbar', '[class*="toast" i]', '[class*="Toast"]', '[class*="snackbar" i]', '[class*="notification" i]', ].join(', '); /** Wait up to `maxMs` for any selector to appear, then return its text content. */ async function waitForAny(page: Page, selector: string, maxMs = 2500): Promise { try { await page.waitForSelector(selector, { timeout: maxMs, state: 'visible' }); } catch { return []; } return page.$$eval(selector, (els) => els.map((el) => (el as HTMLElement).innerText?.replace(/\s+/g, ' ').trim()).filter(Boolean) ); } /** Collect all currently-visible error messages. */ async function collectErrors(page: Page): Promise { try { return page.$$eval(ERROR_SELECTORS, (els) => els .filter((el) => { const style = window.getComputedStyle(el); return style.display !== 'none' && style.visibility !== 'hidden' && (el as HTMLElement).innerText?.trim().length > 0; }) .map((el) => (el as HTMLElement).innerText?.replace(/\s+/g, ' ').trim()) .filter(Boolean) ); } catch { return []; } } /** Collect any visible toast / notification content. */ async function collectToasts(page: Page): Promise { try { return page.$$eval(TOAST_SELECTORS, (els) => els .filter((el) => { const style = window.getComputedStyle(el); return style.display !== 'none' && (el as HTMLElement).innerText?.trim().length > 0; }) .map((el) => (el as HTMLElement).innerText?.replace(/\s+/g, ' ').trim()) .filter(Boolean) ); } catch { return []; } } /** Find the primary submit button on the page. */ async function findSubmitButton(page: Page): Promise { return page.evaluate(() => { const SUBMIT_KW = /submit|send|save|create|confirm|continue|proceed|register|sign.?in|log.?in|apply|verify|search|next|finish|complete|order|buy|checkout/i; const candidates = Array.from(document.querySelectorAll('button, [role="button"], input[type="submit"]')) as HTMLElement[]; const primary = candidates.find((el) => { const style = window.getComputedStyle(el); if (style.display === 'none' || style.visibility === 'hidden') return false; const text = el.innerText || (el as HTMLInputElement).value || el.getAttribute('aria-label') || ''; return SUBMIT_KW.test(text); }); if (!primary) return null; return primary.getAttribute('data-testid') || primary.getAttribute('aria-label') || primary.innerText?.trim().slice(0, 60); }); } /** Find all modal-trigger buttons (non-submit buttons). */ async function findModalTriggers(page: Page): Promise> { return page.evaluate(() => { const SKIP = /submit|send|save|create|log.?in|sign.?in|register|continue|checkout|buy|order|search|close|cancel|dismiss/i; const NAV_PARENTS = new Set(['NAV', 'HEADER', 'FOOTER']); return Array.from(document.querySelectorAll('button, [role="button"]')) .filter((el) => { const style = window.getComputedStyle(el as HTMLElement); if (style.display === 'none' || style.visibility === 'hidden') return false; const label = (el as HTMLElement).innerText?.trim() || el.getAttribute('aria-label') || ''; if (!label || SKIP.test(label)) return false; // Skip nav chrome let p = el.parentElement; while (p) { if (NAV_PARENTS.has(p.tagName)) return false; p = p.parentElement; } return true; }) .slice(0, 20) // probe at most 20 buttons per screen .map((el) => { const label = (el as HTMLElement).innerText?.trim() || el.getAttribute('aria-label') || 'button'; return { selector: `[aria-label="${label}"]`, label: label.slice(0, 60) }; }); }); } /** Try to interact with select elements and observe conditional reveals. */ async function probeConditionals(page: Page): Promise { const results: ConditionalReveal[] = []; try { const selects = await page.$$('select'); for (const sel of selects.slice(0, 5)) { const label = await sel.getAttribute('aria-label') || await sel.getAttribute('name') || 'select'; const options = await sel.$$eval('option', (opts) => opts.slice(1, 3).map((o) => ({ value: o.value, text: o.textContent?.trim() }))); for (const opt of options) { if (!opt.value) continue; const domBefore = await page.evaluate(() => document.body.innerText.length); await sel.selectOption(opt.value); await page.waitForTimeout(500); const domAfter = await page.evaluate(() => document.body.innerText.length); if (Math.abs(domAfter - domBefore) > 50) { const revealed = await page.evaluate(() => { // Grab text from any newly-visible sections return Array.from(document.querySelectorAll('[data-conditional],[data-show],[class*="conditional"]')) .filter((el) => window.getComputedStyle(el).display !== 'none') .map((el) => (el as HTMLElement).innerText?.trim()) .filter((t) => t && t.length > 10) .slice(0, 3) .join(' | '); }); if (revealed) { results.push({ triggerElement: label, triggerValue: opt.text || opt.value, revealedText: revealed.slice(0, 200) }); } } } } } catch { /* non-fatal */ } return results; } /** * Run interactive probing on a single screen page. * The page must already be navigated to the screen URL. * `elements` is the output of the static extractElements pass. */ export async function probeScreen( page: Page, screenId: string, url: string, elements: ProbeElementFact[], ): Promise { const result: ProbeResult = { screenId, url, validationErrors: [], successMessages: [], modals: [], conditionals: [], loadingPatterns: [], }; const formInputs = elements.filter((e) => /input|text|email|password|tel|search|textarea|select/i.test(e.role) ); const hasForm = formInputs.length > 0; // ── Probe 1: Empty submit → capture required-field errors ──────────────────── if (hasForm) { try { const submitLabel = await findSubmitButton(page); if (submitLabel) { // Click submit with no data filled in const submitBtn = page.locator(`button:has-text("${submitLabel}")`).first(); await submitBtn.click({ timeout: 2000 }); await page.waitForTimeout(1500); const errors = await collectErrors(page); // Prefer DOM-proximity mapping via aria-invalid + aria-describedby const ariaFieldErrors = await page.evaluate(() => Array.from(document.querySelectorAll('[aria-invalid="true"]')).map((el) => { const describedBy = el.getAttribute('aria-describedby'); const errorEl = describedBy ? document.getElementById(describedBy) : null; const label = el.getAttribute('aria-label') || el.getAttribute('placeholder') || (el as HTMLInputElement).name || ''; return { label, errorText: errorEl?.textContent?.trim() || '' }; }).filter((f) => f.errorText) ); const ariaMatched = new Set(); for (const af of ariaFieldErrors) { ariaMatched.add(af.errorText); result.validationErrors.push({ field: af.label || 'unknown', trigger: 'empty_submit', errorText: af.errorText.slice(0, 150) }); } // Fallback: text-proximity match for errors not covered by aria-invalid for (const errorText of errors.filter((e) => !ariaMatched.has(e))) { const matchedField = formInputs.find((el) => errorText.toLowerCase().includes(el.meaning.toLowerCase().split(' ')[0]) || el.meaning.toLowerCase().includes(errorText.toLowerCase().split(' ')[0]) ); result.validationErrors.push({ field: matchedField?.meaning ?? 'unknown', trigger: 'empty_submit', errorText: errorText.slice(0, 150), }); } // Check for loading state text during submit const loadingText = await page.evaluate(() => { const loading = Array.from(document.querySelectorAll('button, [aria-busy="true"], [class*="loading" i], [class*="spinner" i]')) .map((el) => (el as HTMLElement).innerText?.trim()) .filter((t) => t && /loading|saving|submitting|sending|processing/i.test(t)); return loading.slice(0, 3); }); result.loadingPatterns.push(...loadingText); } } catch { /* non-fatal — form probe failed, continue */ } } // ── Probe 2: Invalid format inputs ────────────────────────────────────────── if (hasForm) { try { const emailField = formInputs.find((e) => e.expectedData === 'valid_email' || /email/i.test(e.meaning)); const passwordField = formInputs.find((e) => e.expectedData === 'password' || /password/i.test(e.meaning)); if (emailField) { const emailLocator = page.locator('input[type="email"], input[name*="email"], input[placeholder*="email" i]').first(); try { await emailLocator.fill('notanemail', { timeout: 1000 }); await emailLocator.blur(); await page.waitForTimeout(600); const errors = await collectErrors(page); for (const errorText of errors) { result.validationErrors.push({ field: emailField.meaning, trigger: 'invalid_format', errorText: errorText.slice(0, 150) }); } } catch { /* locator not found */ } } if (passwordField) { const pwLocator = page.locator('input[type="password"]').first(); try { await pwLocator.fill('123', { timeout: 1000 }); await pwLocator.blur(); await page.waitForTimeout(600); const errors = await collectErrors(page); for (const errorText of errors) { if (!result.validationErrors.some((e) => e.errorText === errorText)) { result.validationErrors.push({ field: passwordField.meaning, trigger: 'invalid_format', errorText: errorText.slice(0, 150) }); } } } catch { /* locator not found */ } } } catch { /* non-fatal */ } } // ── Probe 3: Button clicks → discover modal/dialog contents ───────────────── try { const triggers = await findModalTriggers(page); for (const trigger of triggers) { try { const btn = page.locator(`button:has-text("${trigger.label}")`).first(); const domBefore = await page.evaluate(() => document.body.innerText.length); await btn.click({ timeout: 1500 }); await page.waitForTimeout(1000); // Check if a dialog/modal appeared const dialogText = await page.evaluate(() => { const DIALOG_SEL = '[role="dialog"], [role="alertdialog"], .modal, .Modal, [class*="modal" i], [class*="dialog" i], [class*="Dialog"]'; const dialogs = Array.from(document.querySelectorAll(DIALOG_SEL)) .filter((el) => window.getComputedStyle(el).display !== 'none'); if (dialogs.length === 0) return null; const d = dialogs[0] as HTMLElement; const title = d.querySelector('[role="heading"], h1, h2, h3')?.textContent?.trim() || ''; const content = d.innerText?.replace(/\s+/g, ' ').trim().slice(0, 400); const fields = Array.from(d.querySelectorAll('input, select, textarea, button')) .map((f) => (f as HTMLElement).getAttribute('aria-label') || (f as HTMLElement).getAttribute('placeholder') || (f as HTMLInputElement).value || '') .filter((t) => t.trim().length > 0); return { title, content, fields }; }); if (dialogText && dialogText.content.length > 20) { result.modals.push({ triggerElement: trigger.label, title: dialogText.title, content: dialogText.content, fields: dialogText.fields.slice(0, 10), }); } else { // No modal appeared — check for accordion/disclosure panel reveals // Clicks on accordion triggers change aria-expanded but don't open a dialog. const accordionText = await page.evaluate(() => { const panels = Array.from(document.querySelectorAll( '[aria-expanded="true"] + *, details[open] > :not(summary), [data-state="open"]:not([role="dialog"]):not([class*="modal" i]), [data-headlessui-state="open"] > *' )).filter((el) => { const s = window.getComputedStyle(el as HTMLElement); return s.display !== 'none' && s.visibility !== 'hidden'; }); if (panels.length === 0) return null; const panel = panels[0] as HTMLElement; const heading = panel.querySelector('h2,h3,h4')?.textContent?.replace(/\s+/g, ' ').trim() || ''; const content = panel.innerText?.replace(/\s+/g, ' ').trim().slice(0, 400) || ''; return content.length > 15 ? { heading, content } : null; }); if (accordionText) { result.modals.push({ triggerElement: trigger.label, title: accordionText.heading || `${trigger.label} (revealed)`, content: accordionText.content, fields: [], }); } } // Dismiss: Escape key first, then close button await page.keyboard.press('Escape'); await page.waitForTimeout(400); // Check if toast appeared after the click const toasts = await collectToasts(page); for (const t of toasts) { if (t.length > 5) result.successMessages.push({ type: 'toast', content: t.slice(0, 200) }); } } catch { /* this button's probe failed, try next */ } } } catch { /* non-fatal */ } // ── Probe 4: Select/radio → conditional reveals ────────────────────────────── try { const conditionals = await probeConditionals(page); result.conditionals.push(...conditionals); } catch { /* non-fatal */ } // ── Probe 5: Tab panels — click each tab, capture revealed content ──────────── try { result.tabPanels = await probeTabPanels(page); } catch { /* non-fatal */ } // ── Probe 6: Tooltips — hover/title/describedby capture ───────────────────── try { result.tooltips = await probeTooltips(page); } catch { /* non-fatal */ } // ── Probe 7: Carousel/slider slides ────────────────────────────────────────── try { result.carouselSlides = await probeCarousels(page); } catch { /* non-fatal */ } // ── Probe 8: Wizard/multi-step form steps ──────────────────────────────────── try { result.wizardSteps = await probeWizardSteps(page); } catch { /* non-fatal */ } // ── Probe 9: Focus-trap detection on any open dialogs ──────────────────────── try { result.focusTraps = await detectFocusTraps(page); } catch { /* non-fatal */ } // ── Probe 10: Skip navigation compliance (WCAG 2.4.1) ──────────────────────── try { result.skipNavIssues = await probeSkipNavigation(page); } catch { /* non-fatal */ } // ── Probe 11: Context menu capture on canvas/chart elements ────────────────── try { result.contextMenus = await probeContextMenus(page); } catch { /* non-fatal */ } return result; } /** Phase 5: Convert ProbeResult to typed ProbeObservationFact rows for direct DB storage. */ export function probeResultToObservations(result: ProbeResult): Array<{ elementMeaning: string; observationType: string; observedText: string; triggeredBy: string; }> { const obs: Array<{ elementMeaning: string; observationType: string; observedText: string; triggeredBy: string }> = []; for (const ve of result.validationErrors) { obs.push({ elementMeaning: ve.field, observationType: 'VALIDATION_ERROR', observedText: ve.errorText, triggeredBy: ve.trigger }); } for (const sm of result.successMessages) { obs.push({ elementMeaning: '(page)', observationType: 'SUCCESS_TOAST', observedText: sm.content, triggeredBy: sm.type }); } for (const m of result.modals) { obs.push({ elementMeaning: m.triggerElement, observationType: 'MODAL', observedText: [m.title, m.content].filter(Boolean).join(' — ').slice(0, 500), triggeredBy: `button_click:${m.triggerElement}` }); } for (const c of result.conditionals) { obs.push({ elementMeaning: c.triggerElement, observationType: 'CONDITIONAL_REVEAL', observedText: c.revealedText, triggeredBy: `select_change:${c.triggerElement}:${c.triggerValue}` }); } for (const lp of result.loadingPatterns) { obs.push({ elementMeaning: '(submit button)', observationType: 'LOADING_STATE', observedText: lp, triggeredBy: 'form_submit' }); } for (const tp of result.tabPanels ?? []) { obs.push({ elementMeaning: tp.tabLabel, observationType: 'TAB_PANEL', observedText: tp.panelContent, triggeredBy: `tab_click:${tp.tabLabel}` }); } for (const tt of result.tooltips ?? []) { obs.push({ elementMeaning: tt.triggerLabel, observationType: 'TOOLTIP', observedText: tt.tooltipText, triggeredBy: 'hover_or_title' }); } for (const cs of result.carouselSlides ?? []) { obs.push({ elementMeaning: `Slide ${cs.slideIndex + 1}`, observationType: 'CAROUSEL_SLIDE', observedText: cs.content, triggeredBy: `carousel_next:${cs.slideIndex}` }); } for (const ws of result.wizardSteps ?? []) { obs.push({ elementMeaning: ws.stepLabel ?? `Step ${ws.stepIndex + 1}`, observationType: 'WIZARD_STEP', observedText: ws.content, triggeredBy: `wizard_next:${ws.stepIndex}` }); } for (const ft of result.focusTraps ?? []) { obs.push({ elementMeaning: ft.modalTrigger, observationType: 'FOCUS_TRAP', observedText: ft.focusTrapped ? 'Focus correctly trapped inside dialog' : 'FAIL: focus escaped dialog — WCAG 2.1.2 violation', triggeredBy: 'tab_key_sequence' }); } for (const issue of result.skipNavIssues ?? []) { obs.push({ elementMeaning: '(page)', observationType: 'SKIP_NAV_ISSUE', observedText: issue, triggeredBy: 'skip_nav_audit' }); } for (const cm of result.contextMenus ?? []) { obs.push({ elementMeaning: cm.trigger, observationType: 'CONTEXT_MENU', observedText: cm.items.join(' | '), triggeredBy: 'right_click' }); } return obs; } /** * Convert a ProbeResult into markdown to store as KnowledgeChunks * and update ERROR_CATALOGUE + INTERACTION_PATTERNS skill files. */ export function probeResultToMarkdown(result: ProbeResult): { errorCatalogueChunk: string; interactionPatternsChunk: string; } { const lines: string[] = [`## Probe Observations — ${result.url}\n`]; if (result.validationErrors.length > 0) { lines.push('### Validation Errors (observed)'); for (const ve of result.validationErrors) { lines.push(`- **${ve.field}** (${ve.trigger}): "${ve.errorText}"`); } } if (result.successMessages.length > 0) { lines.push('\n### Success / Feedback Messages (observed)'); for (const sm of result.successMessages) { lines.push(`- ${sm.type}: "${sm.content}"`); } } if (result.modals.length > 0) { lines.push('\n### Modal / Dialog Contents (observed)'); for (const m of result.modals) { lines.push(`- **Trigger:** ${m.triggerElement}`); if (m.title) lines.push(` - Title: ${m.title}`); lines.push(` - Content: ${m.content.slice(0, 200)}`); if (m.fields.length > 0) lines.push(` - Fields: ${m.fields.join(', ')}`); } } if ((result.tabPanels ?? []).length > 0) { lines.push('\n### Tab Panel Contents (observed)'); for (const tp of result.tabPanels!) { lines.push(`- **Tab: ${tp.tabLabel}**${tp.panelHeading ? ` — ${tp.panelHeading}` : ''}: ${tp.panelContent.slice(0, 200)}`); } } if ((result.tooltips ?? []).length > 0) { lines.push('\n### Tooltips (observed)'); for (const tt of result.tooltips!) { lines.push(`- **${tt.triggerLabel}**: ${tt.tooltipText}`); } } if ((result.carouselSlides ?? []).length > 0) { lines.push('\n### Carousel Slides (observed)'); for (const cs of result.carouselSlides!) { lines.push(`- **Slide ${cs.slideIndex + 1}**${cs.heading ? ` — ${cs.heading}` : ''}: ${cs.content.slice(0, 200)}`); } } if ((result.wizardSteps ?? []).length > 0) { lines.push('\n### Wizard / Multi-Step Form (observed)'); for (const ws of result.wizardSteps!) { const label = ws.stepLabel ? ` — ${ws.stepLabel}` : ''; lines.push(`- **Step ${ws.stepIndex + 1}${label}**: ${ws.content.slice(0, 200)}`); if (ws.fields.length > 0) lines.push(` - Fields: ${ws.fields.join(', ')}`); } } if ((result.focusTraps ?? []).length > 0) { lines.push('\n### Focus-Trap Audit (observed)'); for (const ft of result.focusTraps!) { const verdict = ft.focusTrapped ? '✅ PASS' : '❌ FAIL (WCAG 2.1.2)'; lines.push(`- **${ft.modalTrigger}**: ${verdict} — focused: [${ft.sampledFocused.slice(0, 4).join(', ')}]`); } } if (result.conditionals.length > 0) { lines.push('\n### Conditional UI (observed)'); for (const c of result.conditionals) { lines.push(`- **${c.triggerElement}** = "${c.triggerValue}" reveals: ${c.revealedText}`); } } if (result.loadingPatterns.length > 0) { lines.push('\n### Loading State Text (observed)'); for (const lp of result.loadingPatterns) lines.push(`- "${lp}"`); } const full = lines.join('\n'); const errorSection = result.validationErrors.length > 0 ? `## Real Validation Errors — ${result.url}\n` + result.validationErrors.map((e) => `- **${e.field}**: "${e.errorText}"`).join('\n') : ''; const interactionSection = [ result.successMessages.length > 0 ? `## Success Messages — ${result.url}\n` + result.successMessages.map((s) => `- ${s.type}: "${s.content}"`).join('\n') : '', result.loadingPatterns.length > 0 ? `## Loading Patterns — ${result.url}\n` + result.loadingPatterns.map((lp) => `- "${lp}"`).join('\n') : '', result.modals.length > 0 ? `## Modal Contents — ${result.url}\n` + result.modals.map((m) => `### ${m.triggerElement}\n${m.content}`).join('\n') : '', ].filter(Boolean).join('\n\n'); return { errorCatalogueChunk: errorSection, interactionPatternsChunk: interactionSection, }; } /** * Run interactive probing for all screens discovered in a crawl session. * Called from crawlProject() when opts.interactive = true. */ export async function probeAllScreens( page: any, tenantId: string, projectId: string, discovered: Array<{ url: string; screenId?: string; elements?: any[] }>, onProgress?: (pct: number, step: string) => Promise, ): Promise { const probeable = discovered.filter((d) => d.screenId && d.url && !d.url.includes('#')); if (probeable.length === 0) return; const errorCatalogueChunks: string[] = []; const interactionChunks: string[] = []; for (let i = 0; i < probeable.length; i++) { const screen = probeable[i]; const pct = 88 + Math.round((i / probeable.length) * 10); await onProgress?.(pct, `Probing screen ${i + 1}/${probeable.length}: ${screen.url}`); try { // Navigate to screen if (page.url() !== screen.url) { await page.goto(screen.url, { waitUntil: 'domcontentloaded' }); await page.waitForTimeout(1500); // SPA hydration } const result = await probeScreen(page, screen.screenId!, screen.url, screen.elements ?? []); // Phase 5: save structured probe observations (replaces per-re-probe) const observations = probeResultToObservations(result); if (observations.length > 0) { await AppBrain.saveProbeObservations(tenantId, projectId, screen.screenId!, observations).catch(console.warn); } // Store raw probe result as KnowledgeChunk (kept for BM25 / oversized projects) const md = probeResultToMarkdown(result); if (md.errorCatalogueChunk) errorCatalogueChunks.push(md.errorCatalogueChunk); if (md.interactionPatternsChunk) interactionChunks.push(md.interactionPatternsChunk); const probeMarkdown = [md.errorCatalogueChunk, md.interactionPatternsChunk].filter(Boolean).join('\n\n'); if (probeMarkdown.length > 50) { await AppBrain.saveKnowledgeChunk(tenantId, projectId, 'probe', screen.screenId!, probeMarkdown, `Probe — ${screen.url}`).catch(console.warn); } if (result.validationErrors.length > 0) { console.log(`[probe] ${screen.url}: ${result.validationErrors.length} validation errors, ${result.modals.length} modals`); } } catch (err) { console.warn(`[probe] ✗ ${screen.url}:`, err instanceof Error ? err.message : err); } } // Update ERROR_CATALOGUE skill file with all real observed errors if (errorCatalogueChunks.length > 0) { const combined = `# Error Catalogue (Real Observed — Interactive Probe)\n\n${errorCatalogueChunks.join('\n\n')}`; await AppBrain.upsertSkillFile(tenantId, projectId, 'ERROR_CATALOGUE', combined, { summary: `Real validation error messages observed via interactive probe across ${errorCatalogueChunks.length} screens`, scope: 'PROJECT', }).catch(console.warn); } // Update INTERACTION_PATTERNS skill file with real observed patterns if (interactionChunks.length > 0) { const combined = `# Interaction Patterns (Real Observed — Interactive Probe)\n\n${interactionChunks.join('\n\n')}`; await AppBrain.upsertSkillFile(tenantId, projectId, 'INTERACTION_PATTERNS', combined, { summary: `Real loading states, success messages, and modal contents observed via interactive probe`, scope: 'PROJECT', }).catch(console.warn); } await onProgress?.(98, 'Interactive probe complete'); console.log(`[probe] Probed ${probeable.length} screens. Errors: ${errorCatalogueChunks.length}, Interactions: ${interactionChunks.length}`); } // ── Keyboard Navigation Probe ──────────────────────────────────────────────── export interface FocusableElement { tag: string; role: string | null; ariaLabel: string | null; text: string | null; hasFocusIndicator: boolean; tabIndex: number | null; } export interface KeyboardNavResult { focusableCount: number; elementsWithNoFocusIndicator: number; sequence: FocusableElement[]; issues: string[]; } /** Verify skip-navigation links exist and their targets are present in the DOM. */ async function probeSkipNavigation(page: Page): Promise { const issues: string[] = []; try { const skipLinks = await page.evaluate(() => Array.from(document.querySelectorAll('a[href^="#"]')) .filter((el) => /skip|jump to|main content|content|nav/i.test((el as HTMLElement).innerText?.toLowerCase().trim() || '')) .map((el) => ({ text: (el as HTMLElement).innerText?.trim() || '', href: el.getAttribute('href') || '', hasTarget: !!document.querySelector(el.getAttribute('href') || '#__invalid__'), })) ); if (skipLinks.length === 0) { issues.push('No skip navigation links found (WCAG 2.4.1 — keyboard users cannot bypass nav blocks)'); } for (const link of skipLinks) { if (!link.hasTarget) issues.push(`Skip link "${link.text}" → "${link.href}" target not found in DOM`); } } catch { /* non-fatal */ } return issues; } /** Right-click on canvas, chart, and context-menu elements to capture context menu items. */ async function probeContextMenus(page: Page): Promise { const results: ContextMenuCapture[] = []; try { const targets = await page.$$('canvas, [data-contextmenu], [oncontextmenu], [class*="chart" i]:not(script)'); for (const t of targets.slice(0, 4)) { try { await t.click({ button: 'right', timeout: 1000 }); await page.waitForTimeout(350); const items = await page.evaluate(() => Array.from(document.querySelectorAll('[role="menu"] [role="menuitem"], .context-menu-item, [class*="contextmenu" i] li')) .map((el) => (el as HTMLElement).innerText?.trim()) .filter(Boolean).slice(0, 12) ); if (items.length > 0) { const trigger = await t.evaluate((el) => (el as HTMLElement).getAttribute('aria-label') || (el as HTMLElement).className.slice(0, 40) || el.tagName).catch(() => ''); results.push({ trigger, items }); } await page.keyboard.press('Escape').catch(() => { }); } catch { /* skip this target */ } } } catch { /* non-fatal */ } return results; } /** Click carousel next buttons up to 8 slides and record content of each revealed slide. */ async function probeCarousels(page: Page): Promise { const results: CarouselSlide[] = []; try { const CAROUSEL_SEL = '[class*="carousel" i], [class*="slider" i], [class*="swiper" i], [role="region"][aria-roledescription*="carousel" i]'; const hasCarousel = await page.evaluate((sel) => !!document.querySelector(sel), CAROUSEL_SEL); if (!hasCarousel) return results; const NEXT_SEL = '[aria-label*="next" i], [aria-label*="Next"], [class*="next" i]:not([class*="footnext"]):not(a), button[class*="arrow-right" i], button[class*="chevron-right" i]'; // Capture slide 0 (current state) const slide0 = await page.evaluate((carSel) => { const car = document.querySelector(carSel) as HTMLElement | null; if (!car) return null; const active = car.querySelector('[aria-hidden="false"], .active, [class*="active" i], [class*="current" i]') as HTMLElement | null; const target = active ?? car; return { heading: target.querySelector('h2,h3,h4')?.textContent?.trim() ?? '', content: target.innerText?.replace(/\s+/g, ' ').trim().slice(0, 300) ?? '' }; }, CAROUSEL_SEL); if (slide0?.content && slide0.content.length > 15) results.push({ slideIndex: 0, heading: slide0.heading || undefined, content: slide0.content }); const seen = new Set(slide0?.content ? [slide0.content] : []); for (let i = 1; i < 8; i++) { try { const nextBtn = page.locator(NEXT_SEL).first(); const visible = await nextBtn.isVisible().catch(() => false); if (!visible) break; await nextBtn.click({ timeout: 1000 }); await page.waitForTimeout(600); const slide = await page.evaluate((carSel) => { const car = document.querySelector(carSel) as HTMLElement | null; if (!car) return null; const active = car.querySelector('[aria-hidden="false"], .active, [class*="active" i], [class*="current" i]') as HTMLElement | null; const target = active ?? car; return { heading: target.querySelector('h2,h3,h4')?.textContent?.trim() ?? '', content: target.innerText?.replace(/\s+/g, ' ').trim().slice(0, 300) ?? '' }; }, CAROUSEL_SEL); if (!slide?.content || slide.content.length < 15 || seen.has(slide.content)) break; seen.add(slide.content); results.push({ slideIndex: i, heading: slide.heading || undefined, content: slide.content }); } catch { break; } } } catch { /* non-fatal */ } return results; } /** Follow Next/Continue buttons to capture each wizard step's fields and content. */ async function probeWizardSteps(page: Page): Promise { const results: WizardStep[] = []; try { const NEXT_LABELS = /^(next|continue|proceed|step \d|›|→|forward)/i; const BACK_LABELS = /^(back|previous|prev|‹|←)/i; const WIZARD_INDICATORS = '[role="progressbar"], [class*="stepper" i], [class*="wizard" i], [class*="progress-step" i], [aria-label*="step" i]'; const isWizard = await page.evaluate((sel) => !!document.querySelector(sel), WIZARD_INDICATORS); if (!isWizard) return results; const captureStep = async (stepIndex: number): Promise => { const data = await page.evaluate(() => { const content = (document.querySelector('main, [role="main"], form') as HTMLElement | null)?.innerText?.replace(/\s+/g, ' ').trim().slice(0, 400) ?? ''; const stepLabel = document.querySelector('[aria-current="step"], [class*="step-label" i], [class*="stepper-label" i]')?.textContent?.trim() ?? ''; const fields = Array.from(document.querySelectorAll('input:not([type="hidden"]), select, textarea')) .map((el) => (el as HTMLElement).getAttribute('aria-label') || (el as HTMLInputElement).placeholder || (el as HTMLInputElement).name || '') .filter(Boolean).slice(0, 10); return { content, stepLabel, fields }; }); return { stepIndex, stepLabel: data.stepLabel || undefined, content: data.content, fields: data.fields }; }; results.push(await captureStep(0)); for (let i = 1; i < 6; i++) { try { const nextLabel = await page.evaluate((nextRe) => { const btns = Array.from(document.querySelectorAll('button, [role="button"]')); const btn = btns.find((b) => new RegExp(nextRe).test((b as HTMLElement).innerText?.trim() || '')); return (btn as HTMLElement)?.innerText?.trim() || null; }, NEXT_LABELS.source); if (!nextLabel) break; const btn = page.locator(`button:has-text("${nextLabel}")`).first(); await btn.click({ timeout: 1500 }); await page.waitForTimeout(800); results.push(await captureStep(i)); } catch { break; } } } catch { /* non-fatal */ } return results; } /** For each open modal, Tab 8× and verify focus stays inside the dialog (focus-trap check). */ async function detectFocusTraps(page: Page): Promise { const results: FocusTrapResult[] = []; try { const openDialogs = await page.evaluate(() => { return Array.from(document.querySelectorAll('[role="dialog"], [role="alertdialog"]')) .filter((el) => window.getComputedStyle(el).display !== 'none') .map((el) => el.getAttribute('aria-label') || el.querySelector('[role="heading"]')?.textContent?.trim() || 'dialog'); }); for (const dialogLabel of openDialogs.slice(0, 3)) { const sampled: string[] = []; let allInside = true; for (let i = 0; i < 8; i++) { await page.keyboard.press('Tab'); await page.waitForTimeout(80); const focused = await page.evaluate(() => { const el = document.activeElement as HTMLElement | null; const inDialog = !!el?.closest('[role="dialog"], [role="alertdialog"]'); return { label: el?.getAttribute('aria-label') || el?.textContent?.trim().slice(0, 40) || el?.tagName || '', inDialog }; }); sampled.push(focused.label); if (!focused.inDialog) { allInside = false; break; } } results.push({ modalTrigger: dialogLabel, focusTrapped: allInside, sampledFocused: sampled }); } } catch { /* non-fatal */ } return results; } /** Click each visible tab and record the panel content revealed. */ async function probeTabPanels(page: Page): Promise { const results: TabPanelCapture[] = []; try { const tabs = await page.$$('[role="tab"]:not([aria-disabled="true"])'); for (const tab of tabs.slice(0, 10)) { try { const tabLabel = await tab.evaluate((el) => (el as HTMLElement).innerText?.trim() || el.getAttribute('aria-label') || ''); if (!tabLabel) continue; await tab.click({ timeout: 1000 }); await page.waitForTimeout(400); const panelContent = await page.evaluate(() => { const panel = document.querySelector('[role="tabpanel"]:not([hidden])') as HTMLElement | null; if (!panel) return null; const style = window.getComputedStyle(panel); if (style.display === 'none' || style.visibility === 'hidden') return null; return { heading: panel.querySelector('h2,h3,h4')?.textContent?.trim() ?? '', content: panel.innerText?.replace(/\s+/g, ' ').trim().slice(0, 400) ?? '', }; }); if (panelContent?.content && panelContent.content.length > 15) { results.push({ tabLabel: tabLabel.slice(0, 60), panelContent: panelContent.content, panelHeading: panelContent.heading || undefined }); } } catch { /* this tab failed, try next */ } } } catch { /* non-fatal */ } return results; } /** Hover elements with title/aria-describedby to capture tooltip text. */ async function probeTooltips(page: Page): Promise { const results: TooltipCapture[] = []; try { const candidates = await page.evaluate(() => { return Array.from(document.querySelectorAll('[title], [aria-describedby], [data-tooltip], [data-tip], [class*="tooltip" i]')) .filter((el) => { const style = window.getComputedStyle(el as HTMLElement); return style.display !== 'none' && style.visibility !== 'hidden'; }) .slice(0, 8) .map((el) => ({ label: (el as HTMLElement).innerText?.trim().slice(0, 60) || el.getAttribute('aria-label') || '', title: el.getAttribute('title') || '', describedBy: el.getAttribute('aria-describedby') || '', selector: el.tagName.toLowerCase() + (el.id ? `#${el.id}` : ''), })); }); for (const c of candidates) { try { // Prefer title attribute directly — no need to hover for static tooltips if (c.title && c.title.length > 5) { results.push({ triggerLabel: c.label || c.selector, tooltipText: c.title.slice(0, 200) }); continue; } if (c.describedBy) { const tooltipText = await page.evaluate((id) => document.getElementById(id)?.textContent?.trim() ?? '', c.describedBy); if (tooltipText && tooltipText.length > 5) { results.push({ triggerLabel: c.label || c.selector, tooltipText: tooltipText.slice(0, 200) }); } } } catch { /* skip this element */ } } } catch { /* non-fatal */ } return results; } /** * Probe keyboard navigation accessibility by pressing Tab up to 50 times and * recording each focused element's properties and focus-indicator visibility. * Stops early if focus wraps back to the start or hits the same element twice. */ export async function probeKeyboardNav(page: Page): Promise { const sequence: FocusableElement[] = []; const issues: string[] = []; const seenKeys = new Set(); let firstKey: string | null = null; for (let i = 0; i < 50; i++) { await page.keyboard.press('Tab'); const el = await page.evaluate(() => { const active = document.activeElement as HTMLElement | null; if (!active || active === document.body) return null; const outline = window.getComputedStyle(active).outline; return { tag: active.tagName.toLowerCase(), role: active.getAttribute('role'), ariaLabel: active.getAttribute('aria-label'), text: (active.innerText ?? '').trim().slice(0, 60) || null, outline, tabIndex: active.tabIndex ?? null, }; }); if (!el) break; const hasFocusIndicator = el.outline !== 'none' && el.outline !== '' && !el.outline.startsWith('0px') && !el.outline.startsWith('rgba(0, 0, 0, 0)') && !el.outline.startsWith('rgba(0,0,0,0)'); const focusable: FocusableElement = { tag: el.tag, role: el.role, ariaLabel: el.ariaLabel, text: el.text, hasFocusIndicator, tabIndex: el.tabIndex, }; const key = `${el.tag}|${el.text ?? ''}|${el.role ?? ''}`; if (i === 0) { firstKey = key; } else if (key === firstKey || seenKeys.has(key)) { // Focus wrapped back to start or hit a duplicate — stop probing break; } seenKeys.add(key); sequence.push(focusable); if (!hasFocusIndicator) { issues.push(`Element #${i + 1} (${el.tag}) has no visible focus indicator`); } } return { focusableCount: sequence.length, elementsWithNoFocusIndicator: issues.length, sequence, issues, }; }