/**
 * ZeTa-AI Crawler — Stagehand hybrid architecture
 *
 * Three-layer stack:
 *   Layer 1 — Playwright (deterministic): navigation, screenshots, DOM fallbacks
 *   Layer 2 — Stagehand act/extract (AI): login, element understanding, smart link discovery
 *   Layer 3 — Skyvern (vision): fallback for CAPTCHAs, legacy apps, visual-only flows
 *
 * enableCaching: true → repeat crawls of the same site cost ~zero AI tokens after first run.
 */

import { Stagehand } from '@browserbasehq/stagehand';
// Option 2: rebrowser-playwright-core patches the CDP Runtime.enable leak at protocol level —
// the deepest possible stealth, defeats detection methods that JS init scripts cannot touch.
// @ts-ignore
import { chromium } from 'rebrowser-playwright-core';
import { chromium as stealthChromium } from 'playwright-extra';
// @ts-ignore
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
// Option 3: Camoufox — Firefox-based browser with built-in fingerprint randomisation.
// Loaded dynamically (not at startup) to avoid ESM/CJS conflict when API imports crawler.
import { spawn, execSync, type ChildProcess } from 'node:child_process';
import { existsSync as existsSyncFs, readFileSync } from 'node:fs';
import { createRequire } from 'node:module';

stealthChromium.use(StealthPlugin());
import { z } from 'zod';
import fs from 'node:fs/promises';
import path from 'node:path';
import { AppBrain, recordLLMCall } from '@detiq/app-brain';
import { aiVisualCompare, askTracked, deterministicSeed, estimateCost, getTaskAgentLLMConfig } from '@detiq/agents';
import { config } from '@detiq/core';
import { pHashVerdict, mergeAiVisualVerdict } from './visual-diff-verdict.js';
import type { Credentials } from './auth.js';
import { skyvernFallback, skyvernNavigate, type SkyvernConfig } from './skyvern.js';
import { loginWithStagehand } from './ai-login.js';
import { readScreen } from './screen-reader.js';
import { probeAllScreens, probeKeyboardNav } from './interactive-probe.js';
import { computeElementHash } from './element-hash.js';
import { logger } from './logger.js';
import { solveCaptcha } from './captcha-solver.js';
import { waitForEmailOTP } from './email-otp.js';
import { getSsoToken } from './sso-token.js';
import { discoverSitemapUrls, sameSite } from './sitemap.js';
import { extractMarkdown } from './markdown-extract.js';
import { CrawlFrontier, type TraversalStrategy } from './frontier.js';
import { RobotsCache } from './robots-txt.js';
import { classifyDomainProfile } from './profiles/domain-profile.js';
import { chooseCrawlEngines } from './engine-router.js';
import { assessActionSafety } from './policies/crawl-safety.js';
import { attachNetworkObserver, type ApiEndpointObservationInput } from './network-observer.js';
import { scrubPii } from './pii-scrubber.js';
import { computeContentHash } from './incremental-diff.js';
import { isPdfUrl, extractPdfFromUrl } from './pdf-extractor.js';
import { attachWebSocketDetector } from './websocket-detector.js';
import { ContentDeduplicator } from './content-dedup.js';
import { guessGraphQLEndpoints, introspectGraphQL } from './graphql-detector.js';
import { acceptCookieConsent } from './cookie-consent.js';
import { detectJsRoutes } from './js-router-detector.js';
import { runGapAnalysis } from './gap-pass.js';
import { buildCrawlArtifactEnvelope, buildFitMarkdown, sha256, summarizeStructuredElements } from './extractors/artifact-envelope.js';
import { pickFreeProxy, TieredProxyManager } from './proxy.js';
import type { ProxyTierConfig } from './proxy.js';
import { infiniteScrollToBottom, extractIframeContent, pierceShadowDom, injectFingerprintOverrides } from './stealth/human-behavior.js';
import { checkSeo } from './seo-checker.js';
import { auditColorContrast } from './color-contrast.js';
import { detectFeatureFlags } from './feature-flag-detector.js';
import { detectLanguage } from './language-detector.js';
import { detectForms } from './form-detector.js';
import { auditAccessibility } from './accessibility-auditor.js';
import { RedirectRegistry } from './redirect-tracker.js';
import { DeadLinkTracker } from './dead-link-detector.js';
import { screensToSitemapUrls, writeSitemap } from './sitemap-writer.js';
import { generateOpenApiSpec, specToYaml } from './openapi-generator.js';
import { WebhookNotifier } from './webhook-notifier.js';
import { generatePlaywrightTest } from './playwright-recorder.js';
import { enumerateDropdowns } from './dropdown-enumerator.js';
import { probeCrudFlows } from './crud-interaction.js';
import { crawlFiltersAndPagination } from './filter-pagination-crawler.js';
import { probeMultiContext } from './multi-context-handler.js';
import { probeAdvancedUI } from './advanced-ui-probes.js';
import { ScreenTransitionGraph } from './state-graph.js';
import { crawlNestedContexts } from './nested-window-crawler.js';
import { crawlShadowDom } from './shadow-dom-crawler.js';
import { crawlAccessibilityTree } from './accessibility-tree-crawler.js';
import { collectPerfMetrics } from './performance-metrics.js';
import { inspectServiceWorker } from './service-worker-interceptor.js';

async function getCrawlTuning() {
  const { getSystemConfig } = await import('@detiq/core');
  const cfg = await getSystemConfig('crawl_tuning');
  return {
    navTimeoutMs:     parseInt(String(cfg?.navTimeoutMs     ?? '15000'), 10),
    defaultTimeoutMs: parseInt(String(cfg?.defaultTimeoutMs ?? '2000'),  10),
    userAgent:        String(cfg?.userAgent ?? 'ZeTa-Crawler/1.0 (+https://zeta.taodigitalsolutions.com/crawler)'),
  };
}

// Module-level cache for per-crawl LLM config — avoids redundant DB round-trips inside per-page loops.
// Keyed by tenantId so multi-tenant workers don't share configs across tenants.
const _crawlLLMCache = new Map<string, any>();
async function getCachedCrawlLLM(tenantId: string): Promise<any> {
  if (_crawlLLMCache.has(tenantId)) return _crawlLLMCache.get(tenantId);
  const cfg = await getTaskAgentLLMConfig(tenantId, 'crawl').catch(() => null);
  if (cfg) _crawlLLMCache.set(tenantId, cfg);
  return cfg;
}

function getGitInfo(): { branch: string; commit: string } {
  try {
    const branch = execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf8', timeout: 3000 }).trim();
    const commit = execSync('git rev-parse HEAD', { encoding: 'utf8', timeout: 3000 }).trim().slice(0, 12);
    return { branch: branch || 'main', commit };
  } catch {
    return { branch: 'main', commit: '' };
  }
}

interface PerformanceMetric {
  url: string;
  screenName: string;
  lcp?: number;
  fcp?: number;
  ttfb?: number;
  cls?: number;
}

const ALLOWED_NUMERIC_PATHS = new Set(['/2fa', '/404', '/500', '/403', '/401']);
const UUID_PATH_SEGMENT_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
const CUID_PATH_SEGMENT_RE = /^c[a-z0-9]{20,}$/i;
// Match 4+ digit pure numeric IDs only — aligns with canonicalize()'s /\d{4,}/ threshold.
// 1-3 digit segments (page/2/, /tag/books/page/3/) are valid pagination — do NOT skip them.
const NUMERIC_ID_PATH_SEGMENT_RE = /^\d{4,}$/;
const COMPOUND_NUMERIC_ID_PATH_SEGMENT_RE = /^\d+-\d+$/;

function normalizePathForNumericGuard(pathname: string): string {
  const normalized = pathname.replace(/\/+$/, '') || '/';
  return normalized;
}

function isGeneratedPathSegment(segment: string): boolean {
  return UUID_PATH_SEGMENT_RE.test(segment)
    || CUID_PATH_SEGMENT_RE.test(segment)
    || COMPOUND_NUMERIC_ID_PATH_SEGMENT_RE.test(segment)
    || NUMERIC_ID_PATH_SEGMENT_RE.test(segment);
}

function isGeneratedDynamicPath(pathname: string): boolean {
  const normalized = normalizePathForNumericGuard(pathname);
  if (ALLOWED_NUMERIC_PATHS.has(normalized)) return false;

  // Parent-route agnostic: any generated ID-like segment can create hundreds of
  // duplicate detail/fallback screens under different sections of the app.
  return normalized.split('/').filter(Boolean).some(isGeneratedPathSegment);
}

// File extensions that trigger browser downloads rather than page renders.
// URLs ending with these are skipped from the BFS queue entirely.
const DOWNLOAD_EXT_RE = /\.(pdf|zip|gz|tar|rar|7z|exe|dmg|pkg|deb|rpm|msi|apk|ipa|doc|docx|xls|xlsx|ppt|pptx|csv|json|xml|yaml|yml|txt|log|sql|db|sqlite|mp4|mp3|avi|mov|mkv|webm|wav|ogg|flac|jpg|jpeg|png|gif|bmp|svg|webp|ico|woff|woff2|ttf|eot|otf|bin|iso|img|vhd|vmdk)(\?.*)?$/i;

// API endpoint path patterns — these return data (JSON/CSV/binary), not rendered pages.
// Crawling them triggers file downloads and wastes BFS slots.
const API_PATH_RE = /^\/api\//i;

// Export/download path segments that appear in apps — these return file data even without
// a file extension in the URL (e.g. /admin/exports/report, /download/invoice/123).
const EXPORT_PATH_RE = /\/(exports?|downloads?|export-csv|export-pdf|export-excel|download-report|generate-report|report-download)(\/|$|\?)/i;

function isUsefulNavigationUrl(href: string, siteOrigin: string): boolean {
  try {
    const u = new URL(href);
    if (!sameSite(u.toString(), siteOrigin)) return false;
    if (/\/(logout|logged-out|signout|sign-out)(\/|$|\?)/i.test(u.pathname)) return false;
    if (isGeneratedDynamicPath(u.pathname)) return false;
    if (DOWNLOAD_EXT_RE.test(u.pathname)) return false;
    if (API_PATH_RE.test(u.pathname)) return false;
    if (EXPORT_PATH_RE.test(u.pathname)) return false;
    return true;
  } catch {
    return false;
  }
}

type AuthFlowType = 'public' | 'login' | 'register' | 'password_reset' | 'verification' | 'invite' | 'session_recovery';
type AuthDiscoverySource = 'entry' | 'seed_path' | 'page_link' | 'auth_click';
type AuthDiscoveryProgress = (pct: number, step: string) => void | Promise<void>;
type AuthDiscoveryOptions = {
  appUrl: string;
  tenantId: string;
  projectId: string;
  jobId?: string;
  screenshotDir?: string;
  storageConfig?: any;
};

const AUTH_DISCOVERY_MAX_SCREENS = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_SCREENS ?? '') || 36;
const AUTH_DISCOVERY_MAX_QUEUE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_QUEUE ?? '') || 48;
const AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE = parseInt(process.env.CRAWLER_AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE ?? '') || 6;
const AUTH_DISCOVERY_PATHS = [
  '/',
  '/login', '/log-in', '/signin', '/sign-in', '/auth', '/auth/login', '/account/login', '/user/login', '/app/login',
  '/register', '/signup', '/sign-up', '/create-account', '/join',
  '/forgot-password', '/forgot', '/password/forgot', '/reset-password', '/password/reset',
  '/verify-email', '/invite',
  '/pricing', '/contact', '/about', '/help', '/docs',
];
const AUTH_PATH_RE = /\/(login|log-in|signin|sign-in|auth|oauth|sso|account\/login|user\/login|app\/login|register|signup|sign-up|create-account|forgot|reset|password|verify-email|invite)(\/|$|\?)/i;
const PUBLIC_PATH_RE = /\/($|pricing|contact|about|help|docs|features|security|privacy|terms)(\/|$|\?)/i;

function compactSlug(input: string, fallback = 'state'): string {
  const slug = input.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 48);
  return slug || fallback;
}

function titleFromUrl(url: string, fallback = 'Page'): string {
  try {
    const u = new URL(url);
    const parts = u.pathname.split('/').filter(Boolean);
    if (parts.length === 0) return 'Home';
    return parts
      .slice(-2)
      .join(' ')
      .replace(/[-_]+/g, ' ')
      .replace(/\b\w/g, (m) => m.toUpperCase());
  } catch {
    return fallback;
  }
}

function flowTypeForUrlOrText(value: string): AuthFlowType {
  const v = value.toLowerCase();
  if (/(forgot|reset|password)/.test(v)) return 'password_reset';
  if (/(register|signup|sign up|create account|join)/.test(v)) return 'register';
  if (/(verify|verification|otp|mfa|2fa|invite)/.test(v)) return v.includes('invite') ? 'invite' : 'verification';
  if (/(session-expired|expired|unauthorized|forbidden|401|403|logout|logged-out)/.test(v)) return 'session_recovery';
  if (/(login|log in|signin|sign in|auth|sso|oauth)/.test(v)) return 'login';
  return 'public';
}

function shouldSeedAuthDiscoveryUrl(url: string, origin: string): boolean {
  try {
    const u = new URL(url);
    if (!sameSite(u.toString(), origin)) return false;
    if (/\/(logout|logged-out|signout|sign-out|delete|remove|destroy)(\/|$|\?)/i.test(u.pathname)) return false;
    return AUTH_PATH_RE.test(u.pathname) || PUBLIC_PATH_RE.test(u.pathname);
  } catch {
    return false;
  }
}

function normalizeAuthUrl(url: string): string {
  try {
    const u = new URL(url);
    u.hash = '';
    return `${u.origin}${u.pathname.replace(/\/$/, '') || '/'}${u.search}`;
  } catch {
    return url;
  }
}

async function capturePageHtml(page: any): Promise<string | null> {
  let html: string | null = null;

  try {
    if (typeof page.content === 'function') {
      const raw = await page.content();
      if (typeof raw === 'string' && raw.trim()) html = raw;
    }
  } catch (err: any) {
    logger.warn({ err: String(err?.message ?? err) }, '[crawler] page.content() failed; trying DOM evaluate fallback');
  }

  if (!html) {
    try {
      html = await page.evaluate(() => {
        const type = document.doctype;
        const doctype = type
          ? '<!DOCTYPE ' + type.name +
            (type.publicId ? ' PUBLIC "' + type.publicId + '"' : '') +
            (type.systemId ? ' "' + type.systemId + '"' : '') +
            '>'
          : '<!DOCTYPE html>';
        return doctype + '\n' + (document.documentElement?.outerHTML ?? document.body?.outerHTML ?? '');
      });
      if (typeof html !== 'string' || !html.trim()) html = null;
    } catch (err: any) {
      logger.warn({ err: String(err?.message ?? err) }, '[crawler] DOM snapshot capture failed');
      return null;
    }
  }

  if (!html) return null;

  // Inline all accessible stylesheets so the snapshot renders correctly offline
  // (srcDoc iframes have a null origin — external CSS loads fail cross-origin).
  try {
    html = await page.evaluate(() => {
      // Strip script tags — execution is blocked by CSP/sandbox anyway; removing reduces size and avoids leaking inline tokens.
      document.querySelectorAll('script').forEach((s: any) => s.remove());
      for (const sheet of Array.from(document.styleSheets)) {
        try {
          // cssRules throws SecurityError for cross-origin sheets — skip those
          const cssText = Array.from(sheet.cssRules).map((r: any) => r.cssText).join('\n');
          const style = document.createElement('style');
          style.textContent = cssText;
          const owner = sheet.ownerNode as Element | null;
          owner?.replaceWith(style);
        } catch { /* cross-origin CDN sheet — leave <link> in place */ }
      }
      const doctype = (() => {
        const dt = document.doctype;
        return dt ? `<!DOCTYPE ${dt.name}>` : '<!DOCTYPE html>';
      })();
      return doctype + '\n' + (document.documentElement?.outerHTML ?? '');
    });
    if (typeof html !== 'string' || !html.trim()) html = null;
  } catch (err: any) {
    logger.warn({ err: String(err?.message ?? err) }, '[crawler] CSS inlining failed; using raw HTML');
    // html already has the raw value from page.content() — usable as-is
  }

  return html;
}

/** Launch Chrome on a random CDP port and return its WebSocket debugger URL. */
async function launchChromeCDP(
  execPath: string,
  extraArgs: string[],
  proxy?: { server: string; username?: string; password?: string },
): Promise<{ wsUrl: string; proc: any }> {
  // Use a random port in 9200-9900 range to avoid clashes between concurrent crawls.
  const port = 9200 + Math.floor(Math.random() * 700);

  const browserServer = await stealthChromium.launchServer({
    headless: true,
    executablePath: execPath,
    // Use Playwright's native proxy option — it handles 407 auth challenges automatically.
    // The --proxy-server Chrome flag does NOT support credentials with special chars (commas).
    ...(proxy ? { proxy } : {}),
    args: [
      ...extraArgs,
      `--remote-debugging-port=${port}`,
      '--remote-allow-origins=*',
      '--no-first-run',
      '--no-default-browser-check',
      '--window-size=1280,900',
    ],
  });

  // Poll until Chrome's debug port is ready (up to 10 s).
  const deadline = Date.now() + 10_000;
  while (Date.now() < deadline) {
    await new Promise((r) => setTimeout(r, 300));
    try {
      const res = await fetch(`http://127.0.0.1:${port}/json/version`);
      if (res.ok) {
        const { webSocketDebuggerUrl } = await res.json() as { webSocketDebuggerUrl: string };
        return { wsUrl: webSocketDebuggerUrl, proc: browserServer };
      }
    } catch { /* not ready */ }
  }
  await browserServer.close();
  throw new Error(`Chrome failed to open CDP port ${port} within 10 s`);
}

/**
 * Lightweight pre-crawl link discovery: loads the app URL in a headless (proxy-aware) browser
 * and returns same-site `<a href>` links. Complements sitemap discovery for SPA/app sites that
 * have no sitemap.xml. Best-effort — returns [] on any failure (bot wall, timeout, etc.).
 * chromium.launch({ proxy }) handles proxy 407 auth itself (same client), so no CDP handler needed.
 */
export async function discoverLinksViaBrowser(
  appUrl: string,
  opts?: { proxyUrl?: string; maxLinks?: number },
): Promise<string[]> {
  let proxy: { server: string; username?: string; password?: string } | undefined;
  const purl = opts?.proxyUrl;
  if (purl && purl !== 'auto') {
    const m = purl.match(/^(https?|socks5?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
    if (m) {
      const [, scheme, user, pass, hostPort] = m;
      proxy = user
        ? { server: `${scheme}://${hostPort}`, username: decodeURIComponent(user), password: decodeURIComponent(pass ?? '') }
        : { server: `${scheme}://${hostPort}` };
    } else {
      proxy = { server: purl };
    }
  }
  let executablePath: string | undefined;
  try { executablePath = chromium.executablePath(); } catch { /* fall back to CHROME_PATH */ }
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  let browser: any = null;
  try {
    browser = await chromium.launch({
      headless: true,
      ...(executablePath ? { executablePath } : {}),
      ...(proxy ? { proxy } : {}),
      args: ['--no-sandbox', '--disable-dev-shm-usage'],
    });
    const page = await browser.newPage();
    await page.goto(appUrl, { waitUntil: 'networkidle', timeout: 30_000 }).catch(() => { });
    await page.waitForTimeout(2_500); // let SPA / Web Components hydrate before scraping links
    const hrefs: string[] = await page.evaluate((origin: string) => {
      const found = new Set<string>();
      const walk = (root: Document | ShadowRoot | Element) => {
        (root as any).querySelectorAll('a[href]').forEach((a: any) => {
          const h: string = a.href || a.getAttribute('href') || '';
          if (h && (h.startsWith(origin) || h.startsWith('/'))) found.add(h);
        });
        (root as any).querySelectorAll('*').forEach((el: any) => {
          if (el.shadowRoot) walk(el.shadowRoot);
        });
      };
      walk(document);
      return [...found];
    }, new URL(appUrl).origin).catch(() => [] as string[]);
    return [...new Set(hrefs.filter((h) => sameSite(h, appUrl)))].slice(0, opts?.maxLinks ?? 500);
  } catch {
    return [];
  } finally {
    if (browser) await browser.close().catch(() => { });
  }
}

/**
 * Run a Lighthouse audit against a URL and return category scores (0-100 integers).
 * Requires ENABLE_LIGHTHOUSE=true env var — disabled by default to avoid slowing crawls.
 * Returns null on any error or when disabled.
 */
async function runLighthouse(url: string): Promise<{
  perf: number; a11y: number; seo: number; bp: number; report: object;
} | null> {
  if (!process.env.ENABLE_LIGHTHOUSE) return null;
  try {
    const { default: lighthouse } = await import('lighthouse');
    const chromeLauncher = await import('chrome-launcher');
    const chrome = await chromeLauncher.launch({
      chromeFlags: ['--headless', '--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'],
    });
    try {
      const result = await lighthouse(url, {
        port: chrome.port,
        output: 'json',
        logLevel: 'error',
        onlyCategories: ['performance', 'accessibility', 'seo', 'best-practices'],
      } as any);
      if (!result?.lhr?.categories) return null;
      const cats = result.lhr.categories as Record<string, { score: number | null }>;
      const score = (key: string) => Math.round((cats[key]?.score ?? 0) * 100);
      return {
        perf: score('performance'),
        a11y: score('accessibility'),
        seo: score('seo'),
        bp: score('best-practices'),
        report: result.lhr as object,
      };
    } finally {
      await chrome.kill();
    }
  } catch (err) {
    console.warn('[crawler] Lighthouse run failed (non-fatal):', err);
    return null;
  }
}

const ElementSchema = z.object({
  elements: z.array(z.object({
    meaning: z.string(),
    role: z.string(),
    expectedData: z.string().optional(),
  })),
});

const LinksSchema = z.object({
  links: z.array(z.string()),
});

const TwoFASchema = z.object({
  has2FA: z.boolean(),
  fieldLabel: z.string().optional(),
});

// Shared across all crawl phases (lite-worker, main BFS, P3a, P6).
// PHP/legacy apps render session-expired content inline without URL redirect.
const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;

export async function crawlProject(opts: {
  tenantId: string;
  projectId: string;
  appUrl: string;
  /** The viewport used for this crawl pass */
  viewport?: 'DESKTOP' | 'MOBILE_PORTRAIT' | 'TABLET';
  credentials?: Credentials;
  /** Playwright storageState JSON string — if present, session is restored and AI login is skipped. */
  storageState?: string;
  screenshotDir?: string;
  /** Cloud storage config for immediate screenshot upload. If set, screenshots are uploaded to cloud immediately after capture and cloudUrl is persisted. */
  storageConfig?: { provider: string; bucket: string; region?: string; accountId?: string; endpointUrl?: string; accessKeyId?: string; secretAccessKey?: string; prefix?: string; publicUrl?: string } | null;
  /** When true, run a fast URL-only discovery crawl — skips screenshots, DB saves, and AI element extraction. Returns discoveredUrls in the result object. */
  discoveryMode?: boolean;
  maxScreens?: number;
  /** Seed URLs added to the initial crawl queue — link discovery still runs from each page. */
  startUrls?: string[];
  /** When provided, crawl ONLY these exact URLs (no link-following, no full screen clear). Used for selected-screen recrawl. */
  selectiveUrls?: string[];
  /** When true, run the interactive probe after static crawl to capture real validation errors, toasts, and modals. */
  interactive?: boolean;
  /** Optional abort signal — abort() cancels the crawl between page navigations. */
  abortSignal?: AbortSignal;
  /** Platform job ID — stored on screens so we can diff which were found in each crawl. */
  jobId?: string;
  /** Device profiles to capture screenshots for after the main DESKTOP crawl.
   *  Values: 'MOBILE_PORTRAIT' | 'TABLET'
   *  The main crawl always runs as DESKTOP. Extra profiles trigger a screenshot-only pass.
   */
  deviceProfiles?: string[];
  /** HTTP Basic Auth credentials — injected as Authorization header on same-origin requests. */
  httpBasicUsername?: string;
  httpBasicPassword?: string;
  /** Bearer/API token — injected as a request header and into localStorage. Skips AI login. */
  authToken?: string;
  /** Header name for authToken (default: 'Authorization'). */
  authTokenHeader?: string;
  /** Prefix for authToken value (default: 'Bearer '). */
  authTokenPrefix?: string;
  /** CAPTCHA solving API key (2captcha or CapSolver). */
  captchaSolverApiKey?: string;
  /** CAPTCHA solving service provider (default: '2captcha'). */
  captchaSolverProvider?: '2captcha' | 'capsolver';
  /**
   * Path to a Chrome user data directory for profile inheritance.
   * When set, the crawler exports the existing session from this profile and injects it,
   * so the agent is already logged in. Works for local/self-hosted deployments only.
   * Ignored on cloud/Docker (headless env cannot open another Chrome instance safely).
   */
  chromeProfilePath?: string;
  /** MailSlurp API key for email OTP auto-retrieval (GAP 5). */
  mailslurpApiKey?: string;
  /** MailSlurp inbox ID to poll for OTP emails (GAP 5). */
  mailslurpInboxId?: string;
  /** SSO provider for ROPC token flow — 'okta' | 'azure_ad' | 'generic_oidc' (GAP 18). */
  ssoProvider?: string;
  /** SSO domain or Azure tenant ID (GAP 18). */
  ssoDomain?: string;
  /** SSO OAuth2 client ID (GAP 18). */
  ssoClientId?: string;
  /** SSO OAuth2 client secret (GAP 18). */
  ssoClientSecret?: string;
  /** SSO scopes (space-separated, default: 'openid profile') (GAP 18). */
  ssoScope?: string;
  /** HTTP/HTTPS/SOCKS5 proxy URL — e.g. 'http://user:pass@proxy.example.com:8080' or 'socks5://proxy:1080'.
   *  Useful for sites that block datacenter IPs (Akamai, Cloudflare Bot Manager). */
  proxyUrl?: string;
  /** Tiered proxy configuration for automatic per-domain escalation (free → datacenter → residential → premium). */
  proxyTiers?: ProxyTierConfig[];
  /** Use Camoufox (Firefox) as a cookie bootstrapper before the main Chromium crawl.
   *  Camoufox visits the URL first to solve Cloudflare JS challenges and extract session
   *  cookies, which are then injected into the Chromium crawl as storageState.
   *  Enable via Project Settings → Crawl Config → useFirefox: true. */
  useFirefox?: boolean;
  /** Natural language login instructions for Stagehand AI login. E.g. "Click Sign In in the nav, fill email, click Continue". */
  loginInstructions?: string;
  /** Skyvern vision fallback config from workspace settings (overrides SKYVERN_API_URL/KEY env vars). */
  skyvernConfig?: SkyvernConfig;
  /**
   * Max screens to crawl per segment (default 50).
   * After this limit the crawl pauses and remaining URLs are saved so the
   * next job can continue. Total crawl is unlimited — just runs in multiple jobs.
   */
  segmentSize?: number;
  /** URLs still queued from a previous segment — injected by the job-queue auto-continuation. */
  resumeUrls?: string[];
  /** Segment number (1-based) — used for progress display only. */
  segmentIndex?: number;
  /** Seed the crawl queue from the project's sitemap.xml, in addition to link-following. */
  useSitemap?: boolean;
  /** Explicit sitemap URL override — defaults to `${origin}/sitemap.xml` when useSitemap is set. */
  sitemapUrl?: string;
  /** Crawl queue traversal order. Default BFS — unchanged behavior from before this option existed. */
  traversalStrategy?: TraversalStrategy;
  /** BEST_FIRST only: URLs containing any of these (case-insensitive) are prioritized. */
  bestFirstKeywords?: string[];
  /** When true, skip URLs disallowed by the site's robots.txt. Default false. */
  respectRobotsTxt?: boolean;
  /** When true, dynamically slow crawl speed when server response latency is high (Scrapy-style AutoThrottle). Default false. */
  autoThrottle?: boolean;
  /** Regex pattern — only crawl URLs whose path+query match this pattern. */
  urlFilter?: string;
  /** Already-captured URLs from a previous failed job — pre-seeded into visited map so they're never re-crawled. */
  skipUrls?: string[];
  /** DOM hashes collected in a prior segment — prevents re-screenshotting structurally identical pages across segments. */
  knownDomHashes?: string[];
  /** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */
  onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise<void>;
  /** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */
  onUrlsDiscovered?: (urls: string[]) => Promise<void>;
}, onProgress?: (pct: number, step: string) => Promise<void>) {
  const tuning = await getCrawlTuning();

  if (opts.screenshotDir) await fs.mkdir(opts.screenshotDir, { recursive: true });

  // Upload screenshot to cloud storage if configured, returns cloudUrl or undefined
  const uploadToCloud = async (localPath: string, key: string): Promise<string | undefined> => {
    if (!opts.storageConfig) return undefined;
    if (!existsSyncFs(localPath)) return undefined;
    try {
      const { uploadScreenshotToTenantStorage } = await import('@detiq/app-brain');
      return await uploadScreenshotToTenantStorage(opts.storageConfig as any, localPath, key);
    } catch {
      return undefined;
    }
  };

  const isDocker = !!process.env.PLAYWRIGHT_IN_DOCKER || process.env.NODE_ENV === 'production';

  // Stagehand v3 uses Vercel AI SDK internally. Model format: "aisdk-provider/model-name".
  // Each provider maps to a native AI SDK client; OpenAI-compat baseURL used for non-native ones.
  // Provider → AI SDK prefix mapping:
  //   anthropic → anthropic, openai → openai, gemini → google, grok → xai,
  //   groq → groq (native), openrouter → openai+baseURL, ollama → ollama, nvidia → openai+baseURL

  let stagehandModel = config.llmModel.includes('/') ? config.llmModel : `openai/${config.llmModel}`;
  // clientOptions spread into Stagehand model object (apiKey, baseURL, etc.)
  let stagehandModelOpts: Record<string, string | undefined> = {};
  let stagehandConfigured = false;
  let configuredProvider = '';

  try {
    // getTaskAgentLLMConfig: tenant config → platform admin config → null (proper fallback chain)
    const crawlLLM = await getTaskAgentLLMConfig(opts.tenantId, 'crawl');
    if (crawlLLM) {
      configuredProvider = crawlLLM.provider;
      const hasKey = crawlLLM.apiKey !== undefined && crawlLLM.apiKey !== null;
      const apiKey = hasKey ? crawlLLM.apiKey : undefined;
      const baseUrl = crawlLLM.baseUrl || undefined;
      const m = crawlLLM.model;

      if (m && (hasKey || crawlLLM.provider === 'ollama' || crawlLLM.provider === 'bedrock' || crawlLLM.provider === 'vertex')) {
        stagehandConfigured = true;
        switch (crawlLLM.provider) {
          case 'anthropic':
            stagehandModel = m.includes('/') ? m : `anthropic/${m}`;
            stagehandModelOpts = apiKey ? { apiKey } : {};
            break;
          case 'openai':
            stagehandModel = m.includes('/') ? m : `openai/${m}`;
            stagehandModelOpts = apiKey ? { apiKey } : {};
            break;
          case 'gemini':
            stagehandModel = `google/${m}`;
            stagehandModelOpts = apiKey ? { apiKey } : {};
            break;
          case 'grok':
            stagehandModel = `xai/${m}`;
            stagehandModelOpts = apiKey ? { apiKey } : {};
            break;
          case 'openrouter':
            stagehandModel = `openai/${m}`;
            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || process.env.CRAWLER_OPENROUTER_BASE_URL || 'https://openrouter.ai/api/v1' };
            break;
          case 'ollama':
            stagehandModel = `ollama/${m}`;
            stagehandModelOpts = { baseURL: baseUrl || process.env.CRAWLER_OLLAMA_BASE_URL || 'http://localhost:11434' };
            break;
          case 'nvidia':
            stagehandModel = `openai/${m.startsWith('openai/') ? m.slice(7) : m}`;
            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), baseURL: baseUrl || process.env.CRAWLER_NVIDIA_BASE_URL || 'https://integrate.api.nvidia.com/v1' };
            break;
          case 'groq':
            stagehandModel = `groq/${m}`;
            stagehandModelOpts = apiKey ? { apiKey } : {};
            break;
          case 'mistral':
            stagehandModel = `mistral/${m}`;
            stagehandModelOpts = apiKey ? { apiKey } : {};
            break;
          case 'deepseek':
            stagehandModel = `deepseek/${m}`;
            stagehandModelOpts = apiKey ? { apiKey } : {};
            break;
          case 'perplexity':
            stagehandModel = `perplexity/${m}`;
            stagehandModelOpts = apiKey ? { apiKey } : {};
            break;
          case 'togetherai':
            stagehandModel = `togetherai/${m}`;
            stagehandModelOpts = apiKey ? { apiKey } : {};
            break;
          case 'bedrock':
            // Uses AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_REGION from env
            stagehandModel = `bedrock/${m}`;
            stagehandModelOpts = {};
            break;
          case 'vertex':
            // Uses GOOGLE_APPLICATION_CREDENTIALS or GOOGLE_VERTEX_PROJECT from env
            stagehandModel = `vertex/${m}`;
            stagehandModelOpts = {};
            break;
          case 'azure':
            stagehandModel = `azure/${m}`;
            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), ...(baseUrl ? { baseURL: baseUrl } : {}) };
            break;
          default:
            stagehandModel = `openai/${m}`;
            stagehandModelOpts = { ...(apiKey ? { apiKey } : {}), ...(baseUrl ? { baseURL: baseUrl } : {}) };
        }
      }
    }
  } catch { /* ignore DB errors, fall through to env var below */ }

  // Env var fallback — if DB lookup failed or returned no key
  if (!stagehandConfigured) {
    // Honor STAGEHAND_MODEL env var to determine preferred provider.
    const envModel = process.env.STAGEHAND_MODEL ?? '';
    const preferAnthropic = envModel.includes('claude') || (!envModel && !process.env.OPENAI_API_KEY);
    if (preferAnthropic && process.env.ANTHROPIC_API_KEY) {
      stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
      const defaultAnthropicModel = process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-sonnet-4-6';
      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || defaultAnthropicModel}`;
      stagehandConfigured = true;
    } else if (process.env.OPENAI_API_KEY) {
      stagehandModelOpts = { apiKey: process.env.OPENAI_API_KEY };
      const defaultOpenaiModel = process.env.CRAWLER_DEFAULT_MODEL ?? 'gpt-4o';
      stagehandModel = envModel.includes('/') ? envModel : `openai/${envModel || defaultOpenaiModel}`;
      stagehandConfigured = true;
    } else if (process.env.ANTHROPIC_API_KEY) {
      stagehandModelOpts = { apiKey: process.env.ANTHROPIC_API_KEY };
      const defaultAnthropicModel2 = process.env.CRAWLER_DEFAULT_MODEL ?? 'claude-sonnet-4-6';
      stagehandModel = envModel.includes('/') ? envModel : `anthropic/${envModel || defaultAnthropicModel2}`;
      stagehandConfigured = true;
    }
  }

  const elementLLM = await AppBrain.getTaskLLMConfig(opts.tenantId, 'elementExtraction').catch(() => null);
  const linkLLM = await AppBrain.getTaskLLMConfig(opts.tenantId, 'linkDiscovery').catch(() => null);
  const projectForProfile = await AppBrain.getProjectMap(opts.tenantId, opts.projectId).catch(() => null);
  const profileProjectName = projectForProfile?.name ?? null;
  const profileAppType = projectForProfile?.appType ?? null;
  const domainProfile = classifyDomainProfile({
    appUrl: opts.appUrl,
    projectName: profileProjectName,
    appType: profileAppType,
    seedText: [
      ...(opts.startUrls ?? []),
      opts.loginInstructions ?? '',
      ...(opts.bestFirstKeywords ?? []),
    ].join(' '),
  });
  const routeDecision = chooseCrawlEngines({
    appUrl: opts.appUrl,
    appType: profileAppType,
    hasCredentials: !!opts.credentials,
    hasStorageState: !!opts.storageState,
    hasAuthToken: !!opts.authToken,
    hasSkyvern: !!opts.skyvernConfig,
    hasProxy: !!opts.proxyUrl,
    profile: domainProfile,
  });

  AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
    jobId: opts.jobId,
    type: 'CRAWL_PROFILE',
    payload: { profile: domainProfile, routeDecision },
  }).catch((err: any) => console.warn('[crawler] profile evidence save failed:', err?.message ?? err));

  if (!stagehandConfigured && !opts.discoveryMode) {
    // Discovery mode never calls Stagehand AI — DOM link extraction only.
    // Skip this guard so URL discovery works even without an AI provider configured.
    throw new Error(
      configuredProvider
        ? `No API key configured for provider "${configuredProvider}". Go to Workspace → Settings → AI Provider.`
        : 'No AI provider configured. Go to Workspace → Settings → AI Provider to set up an AI provider for browser crawling.'
    );
  }

  // Anti-detect args applied on all platforms.
  // '--disable-blink-features=AutomationControlled' removes navigator.webdriver at the
  // Chromium level — more reliable than JS overrides alone.
  // '--disable-http2' defeats Akamai/Cloudflare JA3/JA4 TLS fingerprinting on datacenter IPs.
  const stealthArgs = [
    '--disable-blink-features=AutomationControlled',
    '--disable-http2',
    '--lang=en-US,en',
    '--window-size=1366,768',
    '--disable-features=IsolateOrigins,site-per-process',
  ];

  const chromeArgs = isDocker
    ? [
      '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu',
      ...stealthArgs,
    ]
    : [...stealthArgs];
  // Resolve proxy: tiered manager > 'auto' free list > explicit URL
  let resolvedProxyUrl = opts.proxyUrl;
  let tieredProxyManager: TieredProxyManager | undefined;
  if (opts.proxyTiers?.length) {
    tieredProxyManager = new TieredProxyManager(opts.proxyTiers);
    const domain = (() => { try { return new URL(opts.appUrl).hostname; } catch { return opts.appUrl; } })();
    resolvedProxyUrl = tieredProxyManager.selectProxy(domain) ?? resolvedProxyUrl;
    logger.info({ domain, proxyUrl: resolvedProxyUrl }, '[crawler] tiered proxy selected');
  } else if (resolvedProxyUrl === 'auto') {
    logger.info('[crawler] proxyUrl=auto: selecting free proxy from public list...');
    resolvedProxyUrl = (await pickFreeProxy()) ?? undefined;
    if (resolvedProxyUrl) {
      logger.info({ proxyUrl: resolvedProxyUrl }, '[crawler] auto-proxy selected');
    } else {
      logger.warn('[crawler] auto-proxy: no working proxy found, proceeding without proxy');
    }
  }
  // Parse proxy URL into Playwright's native proxy format.
  // We do NOT use --proxy-server Chrome arg because:
  //   1. Chrome ignores credentials in --proxy-server URLs (no auth support via flag)
  //   2. Commas in username (DataImpulse country codes) break Chrome's proxy-list parser
  // Playwright's launchServer({ proxy }) handles 407 auth challenges correctly.
  let playwrightProxy: { server: string; username?: string; password?: string } | undefined;
  if (resolvedProxyUrl) {
    try {
      // Manual parse: new URL() chokes on commas in the username part
      const proxyMatch = resolvedProxyUrl.match(/^(https?|socks5?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
      if (proxyMatch) {
        const [, scheme, rawUser, rawPass, hostPort] = proxyMatch;
        const server = `${scheme}://${hostPort}`;
        const username = rawUser ? decodeURIComponent(rawUser) : undefined;
        const password = rawPass ? decodeURIComponent(rawPass) : undefined;
        playwrightProxy = username ? { server, username, password } : { server };
        logger.info({ server, hasAuth: !!username }, '[crawler] proxy configured (Playwright native)');
      } else {
        playwrightProxy = { server: resolvedProxyUrl };
      }
    } catch {
      playwrightProxy = { server: resolvedProxyUrl };
    }
  }

  // Stagehand 3.x uses chrome-launcher under the hood, which fails in Docker
  // because it doesn't pass --no-sandbox and uses a different spawn environment.
  // Fix: spawn Chrome ourselves with the right flags, get its CDP WS URL,
  // then connect Stagehand to the already-running browser via cdpUrl.
  let executablePath: string | undefined;
  try { executablePath = chromium.executablePath(); } catch { /* use CHROME_PATH */ }

  if (!executablePath) throw new Error('No Chrome binary found. Set CHROME_PATH or install Playwright browsers.');

  // GAP 3: Chrome Profile Inheritance — local/self-hosted deployments only.
  // Launch a temporary persistent context to export the storageState from the user's Chrome profile,
  // then inject it so the normal session-restore path handles auth (no AI login needed).
  let effectiveStorageState = opts.storageState;
  if (opts.chromeProfilePath && !isDocker) {
    logger.info({ chromeProfilePath: opts.chromeProfilePath }, '[crawler] Exporting session from Chrome profile');
    try {
      const tempCtx = await chromium.launchPersistentContext(opts.chromeProfilePath, {
        headless: true,
        args: [...chromeArgs, '--no-first-run', '--no-default-browser-check'],
        executablePath,
      });
      const exportedState = await tempCtx.storageState();
      await tempCtx.close().catch(() => { });
      if ((exportedState.cookies?.length ?? 0) > 0 || (exportedState.origins?.length ?? 0) > 0) {
        effectiveStorageState = JSON.stringify(exportedState);
        logger.info({ cookies: exportedState.cookies?.length ?? 0, origins: exportedState.origins?.length ?? 0 }, '[crawler] Chrome profile session exported and injected');
      } else {
        logger.warn('[crawler] Chrome profile had no session data — continuing without profile auth');
      }
    } catch (profileErr) {
      logger.warn({ err: profileErr }, '[crawler] Could not load Chrome profile — continuing without profile auth');
    }
  } else if (opts.chromeProfilePath && isDocker) {
    logger.warn('[crawler] chromeProfilePath is set but will be ignored in Docker/cloud environments');
  }

  // ── Option 3: Camoufox bootstrap ─────────────────────────────────────────────
  // Firefox with built-in fingerprint randomisation visits the target first to solve
  // any Cloudflare JS challenge, then we extract the cf_clearance cookie and inject it
  // into the Chromium storageState so Chrome inherits the cleared session.
  if (opts.useFirefox) {
    logger.info('[crawler] useFirefox: launching Camoufox to bootstrap Cloudflare cookies...');
    try {
      // Dynamic import prevents ESM/CJS conflict at API startup when this module is imported
      const { NewBrowser: camoufoxNewBrowser } = await import('camoufox');
      const pw = require('playwright');
      const foxBrowser = await camoufoxNewBrowser(pw, true);
      const foxPage = await foxBrowser.newPage();
      await foxPage.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 });
      // Give Cloudflare JS challenge up to 8s to complete and set cf_clearance cookie
      await foxPage.waitForTimeout(8_000);
      const foxState = await foxPage.context().storageState();
      await foxBrowser.close().catch(() => { });
      if ((foxState.cookies?.length ?? 0) > 0) {
        // Merge Camoufox cookies into any existing storageState
        if (effectiveStorageState) {
          try {
            const existing = JSON.parse(effectiveStorageState);
            const merged = { ...existing, cookies: [...(existing.cookies ?? []), ...foxState.cookies] };
            effectiveStorageState = JSON.stringify(merged);
          } catch {
            effectiveStorageState = JSON.stringify(foxState);
          }
        } else {
          effectiveStorageState = JSON.stringify(foxState);
        }
        logger.info({ cookies: foxState.cookies.length }, '[crawler] Camoufox bootstrap complete — cf_clearance injected into Chromium');
      } else {
        logger.warn('[crawler] Camoufox returned no cookies — continuing without bootstrap');
      }
    } catch (foxErr) {
      logger.warn({ err: foxErr }, '[crawler] Camoufox bootstrap failed — continuing without it');
    }
  }

  const { wsUrl: cdpUrl, proc: chromeProc } = await launchChromeCDP(executablePath, chromeArgs, playwrightProxy);

  // Accumulate token usage from Stagehand's internal LLM calls (extract/act/observe).
  // Stagehand doesn't expose per-call usage externally; we parse it from its logger.
  let stagehandPromptTokens = 0;
  let stagehandCompletionTokens = 0;
  const stagehandLogger = (line: { category?: string; message: string; auxiliary?: Record<string, { value: string; type: string }> }) => {
    if (line.message === 'response' && line.auxiliary?.response?.value) {
      try {
        const resp = JSON.parse(line.auxiliary.response.value);
        if (resp?.usage) {
          stagehandPromptTokens += resp.usage.prompt_tokens ?? 0;
          stagehandCompletionTokens += resp.usage.completion_tokens ?? 0;
        }
      } catch { /* ignore parse errors */ }
    }
  };

  const stagehand = new Stagehand({
    env: 'LOCAL',
    model: { modelName: stagehandModel as any, ...stagehandModelOpts },
    serverCache: true,
    verbose: 1,
    logger: stagehandLogger,
    localBrowserLaunchOptions: { cdpUrl, headless: true },
    domSettleTimeout: parseInt(process.env.CRAWLER_DOM_SETTLE_TIMEOUT_MS ?? '') || 2000,
  });

  await stagehand.init();
  await onProgress?.(5, 'Browser launched');
  // In Stagehand v3, stagehand.page is undefined when connected via CDP.
  // Use resolvePage() to get the active V3Page (which is a full Playwright proxy).
  const page = await (stagehand as any).resolvePage();
  // Cap all Playwright operations so one stuck page can't freeze the crawl.
  try { page.setDefaultTimeout(tuning.defaultTimeoutMs || 45_000); } catch { /* proxy may not expose this */ }
  try { page.setDefaultNavigationTimeout(tuning.navTimeoutMs || 30_000); } catch { /* proxy may not expose this */ }
  // Stagehand V3Page proxy only supports the 'console' event (page.on throws for others).
  // rawPage will be wired to a real Playwright Page via the CDP connection established below.
  // Declared as `let` so the network-observer block can populate it.
  let rawPage: any = null; // populated after playwrightBrowser connects

  // ── Proxy auth ───────────────────────────────────────────────────────────────
  // Proxy 407 auth is answered via a CDP Fetch.continueWithAuth handler installed on
  // the connectOverCDP page below (see "proxy auth via CDP Fetch handler"). page.authenticate()
  // is a Puppeteer API — it does NOT exist on Playwright's connectOverCDP page and throws,
  // leaving 407 challenges unanswered (net::ERR_INVALID_AUTH_CREDENTIALS).

  // ── Stealth: defeat bot detection (Cloudflare, Akamai, DataDome) ────────────
  // addInitScript runs in every new document context BEFORE any page JS executes.
  // This overrides navigator.webdriver before anti-bot scripts can read it.
  await page.addInitScript(() => {
    // Core automation flags
    Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
    Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });

    // Realistic plugin list (Cloudflare checks length > 0)
    const fakePlugins = ['PDF Viewer', 'Chrome PDF Viewer', 'Chromium PDF Viewer', 'Microsoft Edge PDF Viewer', 'WebKit built-in PDF'];
    Object.defineProperty(navigator, 'plugins', {
      get: () => Object.assign(fakePlugins.map((name, i) => ({ name, filename: 'internal-pdf-viewer', description: '', length: 1, item: () => null, namedItem: () => null })), { item: (i: number) => null, namedItem: () => null, refresh: () => { } }),
    });

    // @ts-ignore
    window.chrome = { runtime: {}, loadTimes: () => ({}), csi: () => ({}), app: {} };

    // Cloudflare checks navigator.permissions.query — headless returns "denied", real = "prompt"
    const origQuery = window.navigator.permissions?.query?.bind(navigator.permissions);
    if (origQuery) {
      // @ts-ignore
      navigator.permissions.query = (params: any) =>
        params?.name === 'notifications'
          ? Promise.resolve({ state: 'prompt', onchange: null } as PermissionStatus)
          : origQuery(params);
    }

    // Cloudflare checks Notification.permission — "denied" = bot signal
    try { Object.defineProperty(Notification, 'permission', { get: () => 'default' }); } catch { }

    // navigator.connection — absence signals headless
    // @ts-ignore
    if (!navigator.connection) Object.defineProperty(navigator, 'connection', { get: () => ({ rtt: 50, downlink: 10, effectiveType: '4g', saveData: false }) });

    // Remove Playwright/CDP globals leaked into page context
    // @ts-ignore
    delete window.__playwright; delete (window as any).__pw_manual; delete (window as any).__PW_inspect;
  });

  // ── Web Vitals capture: LCP + CLS ────────────────────────────────────────────
  // LCP/CLS entries are only delivered to a PerformanceObserver registered with
  // buffered:true — performance.getEntriesByType(...) after the fact returns [].
  // addInitScript re-runs on every new document (survives page.goto navigations),
  // so the observers are always registered before the page paints. The crawl loop
  // reads the stashed values off window.__zetaLCP / window.__zetaCLS after load.
  await page.addInitScript(() => {
    try {
      // @ts-ignore — custom stash properties
      window.__zetaLCP = null;
      // @ts-ignore
      window.__zetaCLS = 0;
      new PerformanceObserver((list) => {
        const entries = list.getEntries();
        const last = entries[entries.length - 1] as any;
        // @ts-ignore
        if (last) window.__zetaLCP = last.startTime;
      }).observe({ type: 'largest-contentful-paint', buffered: true });
      new PerformanceObserver((list) => {
        for (const entry of list.getEntries() as any[]) {
          // Ignore shifts triggered by user input (crawler clicks/scrolls).
          // @ts-ignore
          if (!entry.hadRecentInput) window.__zetaCLS += entry.value;
        }
      }).observe({ type: 'layout-shift', buffered: true });
    } catch { /* older engines without these entry types — leave stashed values at defaults */ }
  });

  // ── API/Bearer token injection ────────────────────────────────────────────
  // If an authToken is provided, inject it as an HTTP header on every request
  // and store it in localStorage so client-side code can pick it up.
  if (opts.authToken) {
    const headerName = opts.authTokenHeader ?? 'Authorization';
    const prefix = opts.authTokenPrefix ?? 'Bearer ';
    await page.setExtraHTTPHeaders({ [headerName]: `${prefix}${opts.authToken}` });
    await page.addInitScript((token: string) => {
      localStorage.setItem('auth_token', token);
      localStorage.setItem('access_token', token);
      localStorage.setItem('token', token);
    }, opts.authToken);
  }

  // ── Web Vitals capture: CLS via buffered PerformanceObserver ─────────────────
  // CLS entries are only delivered to a PerformanceObserver; this init script
  // re-runs on every new document. LCP is captured per-page in the crawl loop.
  await page.addInitScript(() => {
    try {
      // @ts-ignore — custom stash property
      window.__zetaCLS = 0;
      new PerformanceObserver((list) => {
        for (const entry of list.getEntries() as any[]) {
          // Ignore shifts triggered by user input (crawler clicks/scrolls).
          // @ts-ignore
          if (!entry.hadRecentInput) window.__zetaCLS += entry.value;
        }
      }).observe({ type: 'layout-shift', buffered: true });
    } catch { /* Older engines without layout-shift entry type — leave at 0 */ }
  });

  // Intercept SPA route changes (history.pushState / replaceState) to discover JS-router routes
  // that have no <a href> elements — common in React Router, Next.js, Vue Router apps.
  await page.addInitScript(() => {
    (window as any).__zetaDiscoveredRoutes = (window as any).__zetaDiscoveredRoutes ?? [];
    const push = history.pushState.bind(history);
    const replace = history.replaceState.bind(history);
    history.pushState = function (data: any, unused: string, url?: string | URL | null) {
      const result = push(data, unused, url);
      if (url && typeof url === 'string') (window as any).__zetaDiscoveredRoutes.push(url);
      return result;
    };
    history.replaceState = function (data: any, unused: string, url?: string | URL | null) {
      const result = replace(data, unused, url);
      if (url && typeof url === 'string') (window as any).__zetaDiscoveredRoutes.push(url);
      return result;
    };
  });

  // Inject canvas/WebGL fingerprint overrides + hardware concurrency spoof
  await injectFingerprintOverrides(page).catch(() => { });

  // ── Network interception: capture REST/GraphQL API endpoints ────────────────
  // Stagehand v3's page proxy only supports the 'console' event — to intercept
  // network requests we connect a second Playwright client to the same CDP URL.
  const capturedApiEndpoints: ApiEndpointObservationInput[] = [];
  // P3a: URLs flagged for re-visit after async content detected (drained after main BFS)
  const revisitQueue: string[] = [];
  // P4: XHR-discovered page URL candidates (relative paths from JSON responses)
  const xhrDiscoveredPaths = new Set<string>();

  // eslint-disable-next-line @typescript-eslint/no-explicit-any
  let playwrightBrowser: any = null;
  try {
    playwrightBrowser = await chromium.connectOverCDP(cdpUrl);
    const pwContext = playwrightBrowser.contexts()[0];
    if (pwContext) {
      // P4: pass XHR URL discovery callback — adds relative paths to xhrDiscoveredPaths
      attachNetworkObserver(pwContext, capturedApiEndpoints, (relPath) => xhrDiscoveredPaths.add(relPath));
      // Popup/new-window handler — fires for every window.open() call.
      // Handles two cases:
      //  1. OAuth/SSO provider windows — wait for close, then continue login flow.
      //  2. App-generated popups (help, report, lightbox-in-window) — screenshot + save as screen.
      const OAUTH_PROVIDER_RE = /google\.|github\.|microsoft\.|facebook\.|twitter\.|linkedin\.|okta\.|auth0\.|ping\.|onelogin\.|sso\.|oauth\.|\/authorize\?|\/oauth2\//i;
      pwContext.on('page', (popup: any) => {
        // Wait for the popup to navigate to a real URL (about:blank initially)
        popup.waitForURL((u: string) => !u.startsWith('about:'), { timeout: 10_000 })
          .then(async () => {
            const popupUrl = popup.url();
            if (OAUTH_PROVIDER_RE.test(popupUrl)) {
              // OAuth flow — just wait for it to close
              logger.info({ url: popupUrl }, '[crawler] OAuth popup navigated to provider');
              popup.waitForEvent('close', { timeout: 120_000 })
                .then(() => logger.info('[crawler] OAuth popup closed — SSO flow complete'))
                .catch(() => logger.warn('[crawler] OAuth popup timed out or errored'));
              return;
            }
            // App popup — capture as a screen (skip API/export/download URLs)
            try {
              const _popupPath = (() => { try { return new URL(popupUrl).pathname; } catch { return ''; } })();
              if (API_PATH_RE.test(_popupPath) || EXPORT_PATH_RE.test(_popupPath) || DOWNLOAD_EXT_RE.test(_popupPath)) return;
            } catch {}
            logger.info({ url: popupUrl }, '[crawler] App popup detected — capturing');
            try {
              await popup.waitForLoadState('domcontentloaded', { timeout: 10_000 }).catch(() => {});
              await popup.waitForLoadState('networkidle', { timeout: 6_000 }).catch(() => {});
              const popupTitle = await popup.title().catch(() => '');
              const popupName = popupTitle || (() => { try { return new URL(popupUrl).pathname.split('/').filter(Boolean).pop() || 'Popup'; } catch { return 'Popup'; } })();
              const popupHash = popupUrl.replace(/[^a-z0-9]/gi, '').slice(-12);
              const popupShotPath = path.join(opts.screenshotDir ?? '/tmp', `popup-${popupHash}.png`);
              await waitForScreenshotReady(popup, popup);
              await popup.screenshot({ path: popupShotPath, fullPage: false }).catch(() => {});
              const popupScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, popupName, popupUrl, opts.jobId, 'LINK_FOLLOW').catch(() => null);
              if (popupScreen) {
                const cloudUrl = await uploadToCloud(popupShotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(popupShotPath)}`).catch(() => undefined);
                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, popupScreen.id, popupShotPath, popupScreen.version, 'DESKTOP', cloudUrl).catch(() => {});
                // GAP 4: DOM snapshot for popup capture
                const popupDom = await capturePageHtml(popup);
                if (popupDom) {
                  const popupDomPath = popupShotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
                  await fs.writeFile(popupDomPath, popupDom, 'utf8');
                  const popupDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(popupDomPath)}`;
                  AppBrain.updateScreenDomSnapshot(opts.tenantId, popupScreen.id, popupDomPath, await uploadToCloud(popupDomPath, popupDomKey).catch(() => undefined)).catch(() => {});
                }
                const popupEls = await domFallbackElements(popup, popup).catch(() => []);
                for (const el of popupEls) {
                  AppBrain.saveElement(opts.tenantId, opts.projectId, popupScreen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {});
                }
                logger.info({ url: popupUrl, name: popupName, elementCount: popupEls.length }, '[crawler] App popup screen saved');
              }
            } catch (popupErr: any) {
              logger.warn({ url: popupUrl, err: popupErr?.message }, '[crawler] App popup capture failed — non-fatal');
            }
          })
          .catch(() => {}); // popup may close before navigating (race condition)
      });
      // Cancel any file downloads triggered by link navigation — the crawler captures
      // pages, not binary files. Without this, clicking download links opens the browser
      // download UI and stalls the crawl waiting for a page load that never comes.
      pwContext.on('download', (dl: any) => { dl.cancel().catch(() => {}); });

      // Route-level interception: immediately abort navigation to known API/export/download
      // paths that would trigger file downloads. Uses URL-only matching (no double-fetch)
      // to avoid the extra network round-trip that was causing "Failed to fetch" crashes.
      // The download event handler above catches any remaining downloads that slip through.
      try {
        await pwContext.route('**/*', async (route: any, request: any) => {
          const resourceType: string = request.resourceType?.() ?? '';
          if (resourceType !== 'document') { await route.continue().catch(() => {}); return; }
          try {
            const u = new URL(request.url());
            if (API_PATH_RE.test(u.pathname) || EXPORT_PATH_RE.test(u.pathname) || DOWNLOAD_EXT_RE.test(u.pathname)) {
              logger.info({ url: request.url() }, '[crawler] route: aborting API/export/download navigation');
              await route.abort('failed').catch(() => {});
              return;
            }
          } catch {}
          await route.continue().catch(() => {});
        });
      } catch (routeErr: any) {
        logger.warn({ err: routeErr?.message }, '[crawler] route interception setup failed — downloads may occur');
      }
    }
    // Wire rawPage to the real Playwright page from this CDP connection.
    // This gives us a true Playwright Page that supports all events (.on('response'), etc.)
    // while `page` (Stagehand) is still used for AI actions (act/extract/observe).
    const pwContext2 = playwrightBrowser?.contexts()?.[0];
    if (pwContext2) {
      const pwPages = pwContext2.pages();
      rawPage = pwPages.find((p: any) => p.url() !== 'about:blank') ?? pwPages[0] ?? rawPage;
    }

    // ── Proxy auth via CDP Fetch handler ──────────────────────────────────────
    // launchServer({ proxy }) applies the proxy at the Chrome flag level, but its
    // credentials are only honored by Playwright's OWN launching client. Since we
    // drive the browser through a separate connectOverCDP client, the proxy 407 goes
    // unanswered (net::ERR_INVALID_AUTH_CREDENTIALS). Answer it via the CDP Fetch domain,
    // then disable interception — Chrome caches the proxy credentials for the rest of the session.
    if (playwrightProxy?.username && pwContext2 && rawPage && rawPage !== page) {
      try {
        const proxyCdp = await pwContext2.newCDPSession(rawPage);
        await proxyCdp.send('Fetch.enable', { handleAuthRequests: true, patterns: [{ urlPattern: '*' }] });
        proxyCdp.on('Fetch.requestPaused', (e: any) => {
          proxyCdp.send('Fetch.continueRequest', { requestId: e.requestId }).catch(() => { });
        });
        let proxyAuthed = false;
        proxyCdp.on('Fetch.authRequired', async (e: any) => {
          const isProxy = e.authChallenge?.source === 'Proxy';
          try {
            await proxyCdp.send('Fetch.continueWithAuth', {
              requestId: e.requestId,
              authChallengeResponse: isProxy
                ? { response: 'ProvideCredentials', username: playwrightProxy!.username as string, password: playwrightProxy!.password ?? '' }
                : { response: 'Default' },
            });
          } catch { /* ignore */ }
          if (isProxy && !proxyAuthed) {
            proxyAuthed = true;
            proxyCdp.send('Fetch.disable').catch(() => { });
          }
        });
        logger.info({ server: playwrightProxy.server }, '[crawler] proxy auth via CDP Fetch handler enabled');
      } catch (e: any) {
        logger.warn({ err: e?.message }, '[crawler] CDP proxy-auth setup failed — 407 may not resolve');
      }
    }
  } catch {
    // Non-fatal — crawl proceeds without API endpoint capture
  }
  // Final fallback: if rawPage is still null, use the Stagehand page (events may not work)
  if (!rawPage) {
    rawPage = page;
    logger.warn('[crawler] rawPage fell back to Stagehand proxy — CDP connectOverCDP failed; route() calls will be skipped');
  }

  // ── HTTP Basic Auth: inject Authorization header for same-origin requests ───
  // Must use rawPage (real Playwright page) — Stagehand V3Page proxy does not implement route().
  if (opts.httpBasicUsername && opts.httpBasicPassword && typeof rawPage?.route === 'function') {
    const encoded = Buffer.from(`${opts.httpBasicUsername}:${opts.httpBasicPassword}`).toString('base64');
    const appOrigin = new URL(opts.appUrl).origin;
    await rawPage.route('**/*', (route: any) => {
      const reqUrl = route.request().url();
      if (reqUrl.startsWith(appOrigin)) {
        route.continue({
          headers: { ...route.request().headers(), 'Authorization': `Basic ${encoded}` },
        });
      } else {
        route.continue();
      }
    });
  }

  // ── Crawler login bypass: inject internal secret header on login requests ────
  // Prevents the per-email rate limiter from blocking re-auth attempts made by
  // the crawler during long crawl sessions. The API verifies the secret before
  // skipping the limiter.
  const crawlerSecret = process.env.CRAWLER_INTERNAL_SECRET;
  if (crawlerSecret && typeof rawPage?.route === 'function') {
    const loginPattern = `${new URL(opts.appUrl).origin}**/auth/login*`;
    await rawPage.route(loginPattern, (route: any) => {
      route.continue({
        headers: { ...route.request().headers(), 'x-zeta-crawler': crawlerSecret },
      });
    });
  }

  try {
    // Step 1: Navigate and authenticate
    await onProgress?.(8, 'Loading app');
    // Use explicit timeout — Stagehand V3Page proxy silently ignores setDefaultNavigationTimeout.
    // Fall back to 'commit' (HTTP response received) if DOM takes too long; page is still usable.
    await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(async () => {
      await page.goto(opts.appUrl, { waitUntil: 'commit', timeout: 15_000 }).catch(() => { });
    });
    // Auto-accept cookie consent banners — common EU sites show consent walls before content.
    await acceptCookieConsent(page, { waitMs: 1_000 }).catch(() => { });

    // Fail fast if the app URL itself is unreachable — datacenter IP block, DNS failure, etc.
    if (page.url().startsWith('chrome-error://')) {
      throw new Error(`Cannot reach ${opts.appUrl} — the server refused the connection (likely IP block or DNS failure). Go to Project Settings → Connectivity → Proxy and select "Auto" or enter a residential proxy URL.`);
    }
    // Fail fast if Cloudflare or similar bot-protection challenge is served instead of the real app.
    // These pages never contain testable UI — crawling them produces zero useful screens.
    const initialTitle = await page.title().catch(() => '');
    const isBotChallenge = /^(just a moment|attention required|access denied|ddos protection|cloudflare|checking your browser|enable javascript)/i.test(initialTitle);
    if (isBotChallenge) {
      throw new Error(`Cannot crawl ${opts.appUrl} — bot protection challenge detected (title: "${initialTitle}"). The site is blocking automated browsers. Go to Project Settings → Connectivity → Proxy and select "Auto" (free proxies) or enter a residential proxy URL to bypass.`);
    }
    // Wait for SPA client-side redirect to complete (e.g. Next.js useEffect router.replace to /login).
    // Without this, loginWithStagehand may run on an empty/loading root page before the form renders.
    await page.waitForLoadState('networkidle', { timeout: 10_000 }).catch(() => { });
    await waitForStableDOM(page);

    // Capture homepage fingerprint for soft-404 detection. SPA catch-all routes return HTTP 200
    // but render identical homepage content for every unrecognised path. We fingerprint here
    // (pre-auth, stable DOM) and skip any BFS page that matches during the main crawl.
    const homeFingerprintTitle = await page.title().catch(() => '');
    const homeFingerprintElemCount = await page.evaluate(() =>
      document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
    ).catch(() => -1);

    // ── Pre-auth public-page discovery ───────────────────────────────────────
    // The page is unauthenticated here. On sites where login redirects users
    // away from the marketing/landing pages (e.g. "/" → app dashboard), BFS
    // (which runs after auth) will never discover those public pages. We extract
    // nav links from the current unauthenticated view and visit/capture each one
    // before the session is applied, so the marketing pages are captured correctly.
    const preAuthPublicCaptured = new Set<string>();
    if (!opts.discoveryMode) {
      const _appOrigin = new URL(opts.appUrl).origin;
      // Simple URL normalizer for pre-auth use (canonicalize() declared later in scope)
      const _norm = (u: string) => { try { const p = new URL(u); p.hash = ''; return p.href.replace(/\/$/, ''); } catch { return u; } };
      const _startNorm = _norm(opts.appUrl);
      const _authPathRe = /\/(sign.?in|sign.?up|log.?in|register|auth|oauth|sso|forgot.?password|reset.?password)(\/|$|\?)/i;
      const _skipPathRe = /\/(terms|privacy|legal|cookie-policy|changelog|blog|imprint)([-/]|$)/i;

      const publicNavLinks: string[] = await page.evaluate((originStr: string) => {
        return Array.from(document.querySelectorAll('a[href]'))
          .map((a: Element) => {
            try {
              const u = new URL((a as HTMLAnchorElement).getAttribute('href')!, window.location.href);
              return u.origin === originStr ? u.href.split('#')[0] : '';
            } catch { return ''; }
          })
          .filter((h: string, i: number, arr: string[]) => h && arr.indexOf(h) === i)
          .slice(0, 50);
      }, _appOrigin).catch(() => [] as string[]);

      for (const link of publicNavLinks) {
        if (_norm(link) === _startNorm) continue;
        try {
          const { pathname } = new URL(link);
          if (_authPathRe.test(pathname) || _skipPathRe.test(pathname) || API_PATH_RE.test(pathname) || EXPORT_PATH_RE.test(pathname) || DOWNLOAD_EXT_RE.test(pathname)) continue;
          await page.goto(link, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => {});
          await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});
          const finalUrl = page.url();
          const finalPath = new URL(finalUrl).pathname;
          if (_authPathRe.test(finalPath)) continue; // redirected to auth — skip
          if (_norm(finalUrl) === _startNorm) continue; // SPA catch-all redirect
          // Read interactive elements first — SPA pages show 0 elements without auth.
          // Only lock as pre-auth if the page actually has content; otherwise BFS handles it authenticated.
          const preAuthElements = await readScreen(page).catch(() => [] as Awaited<ReturnType<typeof readScreen>>);
          if (preAuthElements.length === 0) {
            logger.info({ url: finalUrl }, '[crawler] pre-auth page has 0 elements — deferring to authenticated BFS');
            continue;
          }
          // Detect auth walls that have a tiny number of elements (login button, back link, spinner).
          // These pages redirect or gate on auth — main BFS must visit them with a live session.
          // Label prefix or exact match — "Back to Learn" matches "back to", "Sign In" matches "sign in"
          const AUTH_WALL_LABEL_RE = /^(sign.?in|log.?in|login|platform access|back\s+to|checking\s+auth|get started|create account|register|continue with|join now)/i;
          // Also treat pages with only `a`-role elements (no inputs/buttons) as likely auth walls
          const allAnchorOnly = preAuthElements.every(el => el.role === 'a');
          const looksLikeAuthWall = preAuthElements.length <= 3 && (allAnchorOnly || preAuthElements.every(el => AUTH_WALL_LABEL_RE.test(el.label)));
          if (looksLikeAuthWall) {
            logger.info({ url: finalUrl, elements: preAuthElements.length, labels: preAuthElements.map(e => e.label) }, '[crawler] pre-auth auth wall — deferring to authenticated BFS');
            continue;
          }
          const pathParts = finalPath.split('/').filter(Boolean);
          const screenName = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';
          const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, screenName, finalUrl, opts.jobId, 'LINK_FOLLOW').catch(() => null);
          if (!screen) continue;
          const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${screen.id}-desktop-v${screen.version}.png`);
          await page.screenshot({ path: shotPath, fullPage: true }).catch(() => {});
          const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => undefined);
          await (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', cloudUrl).catch(() => {});
          // GAP 5: DOM snapshot for pre-auth public page discovery
          const pubDom = await capturePageHtml(page);
          if (pubDom) {
            const pubDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
            await fs.writeFile(pubDomPath, pubDom, 'utf8');
            const pubDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(pubDomPath)}`;
            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, pubDomPath, await uploadToCloud(pubDomPath, pubDomKey).catch(() => undefined)).catch(() => {});
          }
          // Save elements found in the pre-auth view
          await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
          for (const el of preAuthElements) {
            await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.label, el.role, el.expectedData, undefined, undefined, true, undefined, undefined, undefined).catch(() => {});
          }
          preAuthPublicCaptured.add(finalUrl);
          logger.info({ url: finalUrl, screenId: screen.id, elements: preAuthElements.length }, '[crawler] pre-auth public page captured');
        } catch (err: any) {
          logger.warn({ url: link, err: err?.message }, '[crawler] pre-auth public page visit failed — skipping');
        }
      }

      // Navigate back to start so auth phase and fingerprint comparison are correct
      if (preAuthPublicCaptured.size > 0) {
        await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 20_000 }).catch(() => {});
        await page.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
      }
    }

    // Dedicated logged-out/auth discovery phase. This runs before credentials or
    // storageState are applied, so public pages and auth recovery flows remain
    // visible even when the main crawl later becomes authenticated.
    if (!opts.discoveryMode) {
      try {
        await runAuthDiscoveryPhase(page, opts, new URL(opts.appUrl).origin, onProgress);
      } catch (preAuthErr) {
        logger.warn({ err: preAuthErr }, '[crawler] auth-discovery phase failed — continuing with authenticated crawl');
      }
    }

    // Auth discovery opens OAuth sign-in popups (Google, GitHub, etc.) which fire CDP events that
    // can permanently corrupt the Playwright isolated world. Verify the world is intact; if not,
    // re-navigate to appUrl to force a fresh execution context before BFS begins.
    {
      const worldOk = await page.evaluate(() => true).catch(() => false);
      if (!worldOk) {
        logger.warn('[crawler] isolated world corrupted after auth discovery — re-navigating to recover');
        await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => {});
        await page.waitForLoadState('networkidle', { timeout: 6_000 }).catch(() => {});
      }
    }

    let didLogin = false; // track whether AI login was actually performed

    if (opts.authToken) {
      // Auth token already injected via setExtraHTTPHeaders + addInitScript above.
      // Skip session restore and AI login entirely.
      await onProgress?.(18, 'API token configured — skipping login');
      logger.info('authToken provided — skipping AI login');
    } else if (effectiveStorageState) {
      // ── StorageState session restore: skip AI login entirely ─────────────────
      // User authenticated once via `npx playwright codegen --save-storage=session.json`
      // and saved the resulting JSON in crawl config (encrypted at rest).
      // effectiveStorageState may also come from Chrome profile inheritance (GAP 3).
      logger.info('StorageState found — restoring session, skipping AI login');
      try {
        const state = JSON.parse(effectiveStorageState!) as {
          cookies?: Array<Record<string, any>>;
          origins?: Array<{ origin: string; localStorage: Array<{ name: string; value: string }> }>;
        };

        if (state.cookies?.length) {
          // Apply cookies to both the secondary Playwright context AND Stagehand's own context.
          // Stagehand V3 uses a separate browser context from the CDP secondary client, so
          // cookies added only to playwrightBrowser.contexts()[0] don't reach Stagehand's pages.
          const browserCtx = playwrightBrowser?.contexts()[0];
          if (browserCtx) {
            await browserCtx.addCookies(state.cookies as any);
          }
          // Also apply to Stagehand's context via its page proxy
          try {
            const stagehandCtx = (page as any).context?.();
            if (stagehandCtx && typeof stagehandCtx.addCookies === 'function') {
              await stagehandCtx.addCookies(state.cookies as any);
            }
          } catch { /* Stagehand V3 proxy may not expose context() */ }
        }

        if (state.origins?.length) {
          // Apply localStorage via addInitScript to ALL contexts:
          // 1. Secondary Playwright context (rawPage navigations)
          // 2. Stagehand's page directly (Stagehand V3 uses its own context separate from playwrightBrowser)
          // Without #2, Stagehand's BFS navigations never get localStorage → pages load unauthenticated.
          const originCtx = playwrightBrowser?.contexts()[0];
          if (originCtx) {
            await originCtx.addInitScript((origins: typeof state.origins) => {
              for (const entry of (origins ?? [])) {
                if (window.location.origin === entry.origin) {
                  for (const item of entry.localStorage) {
                    try { localStorage.setItem(item.name, item.value); } catch { /* quota/sandboxed */ }
                  }
                }
              }
            }, state.origins);
          }
          // Stagehand's page — covers all BFS navigations via Stagehand
          await page.addInitScript((origins: typeof state.origins) => {
            for (const entry of (origins ?? [])) {
              if (window.location.origin === entry.origin) {
                for (const item of entry.localStorage) {
                  try { localStorage.setItem(item.name, item.value); } catch { /* quota/sandboxed */ }
                }
              }
            }
          }, state.origins);
          // Also cover rawPage for the initial navigation below
          await rawPage.addInitScript((origins: typeof state.origins) => {
            for (const entry of (origins ?? [])) {
              if (window.location.origin === entry.origin) {
                for (const item of entry.localStorage) {
                  try { localStorage.setItem(item.name, item.value); } catch { /* quota/sandboxed */ }
                }
              }
            }
          }, state.origins);
        }

        // Navigate to appUrl with cookies now set.
        // We MUST navigate to appUrl (not reload current page) because the initial
        // goto ran without cookies and was redirected to /login. Reloading /login stays
        // at /login even with a valid cookie (public path, no auth redirect). Navigating
        // to appUrl lets server middleware see the cookie and route to the authenticated app.
        await onProgress?.(12, 'Restoring session');
        await rawPage.goto(opts.appUrl, { waitUntil: 'domcontentloaded' }).catch(() => { });
        await waitForStableDOM(page);
        // SPA root pages redirect client-side (useEffect → router.replace). Wait for it.
        if (page.url() === opts.appUrl || page.url() === opts.appUrl.replace(/\/$/, '')) {
          await waitForURLChange(page, opts.appUrl).catch(() => { });
          await waitForStableDOM(page);
        }

        // Detect if session restore failed — either by URL redirect or login form presence on root URL
        const postRestoreUrl = page.url();
        const LOGIN_PATH = /\/(login|signin|sign-in|auth|log-in|sso|oauth|account\/login)(\/|$|\?)/i;
        const ERROR_PATH = /\/(session-expired|expired|unauthorized|forbidden|401|403|error|logout|logged-out)(\/|$|\?)/i;
        const redirectedToLogin = LOGIN_PATH.test(new URL(postRestoreUrl).pathname);
        const redirectedToError = ERROR_PATH.test(new URL(postRestoreUrl).pathname);
        // Also check for login form presence — catches apps where login page is at root URL (e.g. /)
        const hasLoginForm = await page.evaluate(() => {
          const inputs = Array.from(document.querySelectorAll('input'));
          const hasEmailOrUser = inputs.some((i: any) =>
            ['email', 'username', 'user', 'login'].some((k) =>
              (i.type || '').includes(k) || (i.name || '').toLowerCase().includes(k) ||
              (i.id || '').toLowerCase().includes(k) || (i.placeholder || '').toLowerCase().includes(k)
            )
          );
          const hasPasswordField = inputs.some((i: any) => i.type === 'password');
          return hasEmailOrUser && hasPasswordField;
        }).catch(() => false);
        const sessionFailed = redirectedToLogin || redirectedToError || hasLoginForm;
        if (sessionFailed) {
          logger.warn({ url: postRestoreUrl, loginForm: hasLoginForm, errorUrl: redirectedToError }, 'StorageState restore failed — session may be expired');
          // Clear the expired session from DB so user is forced to re-capture
          await AppBrain.saveCrawlConfig(opts.tenantId, opts.projectId, { storageState: null }).catch(() => { });
          // Navigate to the login page. First try appUrl — if it still redirects to an error/login
          // path, probe common login paths at the app origin so Stagehand finds a real login form.
          await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded' }).catch(() => { });
          await waitForStableDOM(page);
          const urlAfterNav = page.url();
          const stillAtError = ERROR_PATH.test(new URL(urlAfterNav).pathname) || LOGIN_PATH.test(new URL(urlAfterNav).pathname);
          if (stillAtError) {
            const appOrigin = new URL(opts.appUrl).origin;
            for (const loginPath of ['/login', '/signin', '/sign-in', '/auth/login', '/auth']) {
              try {
                await page.goto(`${appOrigin}${loginPath}`, { waitUntil: 'domcontentloaded' });
                const hasForm = await page.evaluate(() => {
                  const inputs = Array.from(document.querySelectorAll('input'));
                  return inputs.some((i: any) => i.type === 'password');
                }).catch(() => false);
                if (hasForm) { logger.info({ loginPath }, 'Found login form at path'); break; }
              } catch { /* continue to next path */ }
            }
            await waitForStableDOM(page);
          }
          if (opts.credentials) {
            logger.info('Falling back to AI login with credentials');
            await onProgress?.(15, 'Session expired — logging in with credentials…');
            const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
            if (!loggedIn) throw new Error('[SESSION_EXPIRED] Stored session expired and credential login also failed. Check your username / password in Project Settings → Credentials.');
            didLogin = true;
          } else {
            throw new Error('[SESSION_EXPIRED] Stored session has expired. Please re-capture your session in Project Settings → Advanced: Session Capture, or add username / password as a login fallback.');
          }
        } else {
          await onProgress?.(18, 'Session restored');
          logger.info({ url: postRestoreUrl }, 'Session restored');
        }
      } catch (err) {
        const msg = String((err as Error).message ?? err);
        // Re-throw errors we threw intentionally above
        if (msg.includes('[SESSION_EXPIRED]') || msg.includes('expired')) throw err;
        logger.warn({ err }, 'StorageState restore failed, falling back to AI login');
        if (opts.credentials) {
          await onProgress?.(15, 'Session restore failed — logging in with credentials…');
          const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
          if (!loggedIn) throw new Error('[SESSION_EXPIRED] Session restore failed and credential login also failed. Check your username / password in Project Settings → Credentials.');
          didLogin = true;
        } else {
          throw new Error('[SESSION_EXPIRED] Session restore failed and no credentials were provided. Please re-capture your session in Project Settings → Advanced: Session Capture.');
        }
      }
    } else if (opts.ssoProvider && opts.ssoDomain && opts.ssoClientId && opts.credentials) {
      // ── GAP 18: SSO ROPC flow — bypass browser login entirely ────────────────
      // Get a token directly from the IdP and inject it as a Bearer header.
      await onProgress?.(12, 'SSO: fetching token from identity provider');
      try {
        const ssoToken = await getSsoToken(
          {
            provider: opts.ssoProvider as any,
            domain: opts.ssoDomain,
            clientId: opts.ssoClientId,
            clientSecret: opts.ssoClientSecret,
            scope: opts.ssoScope,
          },
          opts.credentials.username,
          opts.credentials.password
        );
        await page.setExtraHTTPHeaders({ 'Authorization': `Bearer ${ssoToken}` });
        await page.addInitScript((t: string) => {
          localStorage.setItem('access_token', t);
          localStorage.setItem('auth_token', t);
          localStorage.setItem('token', t);
        }, ssoToken);
        await page.reload({ waitUntil: 'domcontentloaded' });
        didLogin = true;
        await onProgress?.(18, 'SSO token injected');
        logger.info({ provider: opts.ssoProvider }, '[crawler] SSO token obtained and injected');
      } catch (ssoErr) {
        logger.warn({ err: ssoErr }, '[crawler] SSO token retrieval failed — falling back to browser login');
        // Fall through to browser-based login
        await onProgress?.(12, 'Logging in');
        didLogin = true;
        const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
        if (!loggedIn) {
          logger.warn('Stagehand login failed — trying Skyvern vision fallback');
          await onProgress?.(14, 'Trying vision fallback');
          const skyvernResult = await skyvernFallback({
            taskDescription: `Log in to ${opts.appUrl} using username "${opts.credentials.username}" and navigate to the main dashboard.`,
            url: opts.appUrl,
            credentials: opts.credentials,
          }, opts.skyvernConfig);
          if (skyvernResult === 'failed') {
            throw new Error(
              'Login failed on both Stagehand (DOM) and Skyvern (vision). ' +
              'Check credentials. For CAPTCHA/SSO sites, set SKYVERN_API_URL + SKYVERN_API_KEY.'
            );
          }
        }
        await onProgress?.(18, 'Logged in');
      }
    } else if (opts.credentials) {
      await onProgress?.(12, 'Logging in');
      didLogin = true;
      const loggedIn = await loginWithStagehand(stagehand, opts.credentials, opts.appUrl, { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider }, { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId }, opts.loginInstructions, { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId });
      if (!loggedIn) {
        logger.warn('Stagehand login failed — trying Skyvern vision fallback');
        await onProgress?.(14, 'Trying vision fallback');
        const skyvernResult = await skyvernFallback({
          taskDescription: `Log in to ${opts.appUrl} using username "${opts.credentials.username}" and navigate to the main dashboard.`,
          url: opts.appUrl,
          credentials: opts.credentials,
        }, opts.skyvernConfig);
        if (skyvernResult === 'failed') {
          throw new Error(
            'Login failed on both Stagehand (DOM) and Skyvern (vision). ' +
            'Check credentials. For CAPTCHA/SSO sites, set SKYVERN_API_URL + SKYVERN_API_KEY.'
          );
        }
      }
      await onProgress?.(18, 'Logged in');
    } else {
      await onProgress?.(18, 'No auth needed');
    }

    // Step 2: Crawl
    // If we just logged in (via credentials or session-fallback), capture and persist the new
    // session state so the next crawl reuses it without logging in again.
    if (didLogin) {
      try {
        const freshState = await playwrightBrowser!.contexts()[0]?.storageState();
        if (freshState) {
          await AppBrain.saveCrawlConfig(opts.tenantId, opts.projectId, { storageState: JSON.stringify(freshState) });
          logger.info('Fresh session state captured and saved after credential login');
          await onProgress?.(19, 'Session saved — will reuse on next crawl');
        }
      } catch (e) {
        logger.warn({ err: e }, 'Could not save refreshed session state — crawl will continue');
      }
      // Wait for SPA router to complete the post-login redirect before seeding the queue.
      // Without this, Next.js/React apps queue the login URL instead of the dashboard URL.
      await waitForURLChange(page, opts.appUrl);
    }
    const origin = new URL(page.url()).origin;
    const maxScreens = opts.maxScreens ?? Infinity;
    const segmentSize = opts.segmentSize ?? 9_999_999; // effectively unlimited — segments only exist for checkpointing reliability
    // selective = exact-URL-only mode (recrawl selected screens, no link discovery)
    const selective = !!(opts.selectiveUrls && opts.selectiveUrls.length > 0);

    // Mark all screens for this project as active — post-crawl we'll diff which ones weren't re-discovered.
    if (!selective) {
      await AppBrain.prepareForRecrawl(opts.tenantId, opts.projectId, opts.jobId);
      await AppBrain.clearFlows(opts.tenantId, opts.projectId);
    }
    // Canonicalize URL for dedup: strip tracking params + normalize dynamic ID segments.
    // Prevents /projects/cuid1 and /projects/cuid2 from being queued as separate screens.
    const canonicalize = (raw: string): string => {
      try {
        const u = new URL(raw);
        // Treat www and non-www as the same host — most sites redirect one to the other
        // but internal links may be inconsistent, causing duplicate crawl entries otherwise.
        u.hostname = u.hostname.replace(/^www\./, '');
        // Strip params that don't change page content
        [
          'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'ref', 'source', '_gl', 'fbclid', 'gclid',
          'ts', 'timestamp', 'cache', 'cacheBust', 'cache_bust', '_', 'nonce', 'session', 'sid',
          // App content/state params — these change the content rendered within the same
          // page template (a playground example, a search query, a tab selection) but do NOT
          // represent structurally distinct pages. Not stripping them causes URL explosion:
          // every unique ?cmd=DSN-XXXXX&example=... or ?q=foo is queued as a new screen.
          'cmd', 'example', 'query', 'q', 'search', 'keyword', 'keywords',
          'tab', 'view', 'mode', 'sort', 'order', 'direction', 'orderby',
          'page', 'p', 'offset', 'limit', 'per_page', 'size', 'rows',
          // Export format params — same endpoint, different output format, not a new screen
          'format', 'output', 'export_format', 'download_format', 'fileformat', 'file_format',
          // WooCommerce / e-commerce action params
          'add-to-cart', 'add_to_wishlist', 'quantity', 'wc-ajax',
          'min_price', 'max_price', 'removed_item', 'restore', 'remove_item',
          'undo_item', 'coupon_code', 'apply_coupon', 'added-to-cart',
          'wc_error', 'wc_notice', 'wc_notice_type', 'update_cart',
          'proceed', '_wpnonce', 'post_type', 'product_cat',
          // WooCommerce facet/layered-nav filter params — each combination generates a unique URL
          // but all render the same template; not stripping these causes infinite URL explosion
          // (N filter options × M values = exponential queue growth)
          'filter_color', 'filter_colour', 'filter_size', 'filter_brand', 'filter_category',
          'filter_rating', 'filter_stock_status', 'filter_pa_color', 'filter_pa_size',
          'filter_pa_brand', 'filter_pa_material', 'filter_pa_weight', 'filter_pa_style',
          'filter_pa_gender', 'filter_pa_age', 'filter_pa_type', 'filter_pa_pattern',
          'query_type_color', 'query_type_size', 'query_type_brand', 'query_type_pa_color',
          'query_type_pa_size', 'query_type_pa_brand', 'in_stock', 'on_sale',
          // Shopify facet filters
          'sort_by', 'constraint', 'filter.p.tag', 'filter.v.price.gte', 'filter.v.price.lte',
          'filter.p.m.global.color', 'filter.p.m.global.size',
        ].forEach(p => u.searchParams.delete(p));
        // Strip any remaining filter_* / query_type_* / filter.* params (WooCommerce/Shopify facets)
        for (const key of Array.from(u.searchParams.keys())) {
          if (/^filter[._]|^query_type_/i.test(key)) u.searchParams.delete(key);
        }
        u.searchParams.sort();
        // Normalize dynamic segments globally: UUIDs, CUIDs, long numeric IDs,
        // and hyphenated numeric record IDs under any parent route.
        u.pathname = u.pathname
          .replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}(?=\/|$)/gi, '/{id}')
          .replace(/\/c[a-z0-9]{20,}(?=\/|$)/g, '/{id}')
          .replace(/\/\d{4,}(?=\/|$)/g, '/{id}')
          // Hyphenated numeric IDs: digits-digits (e.g. 0-9638, 12-4567). Not UUIDs (hex only).
          .replace(/\/\d+-\d+(?=\/|$)/g, '/{id}')
          .replace(/\/+$/, '') || '/';
        // Hash-routed SPAs put the actual route in the fragment ("#/dashboard",
        // "#!/settings") — keep it so distinct routes dedup separately instead
        // of every hash-route collapsing onto one canonical (hashless) URL.
        if (!/^#!?\//.test(u.hash)) u.hash = '';
        return u.toString();
      } catch { return raw; }
    };

    const gitInfo = getGitInfo();

    const visited = new Map<string, string>(); // key = canonical URL
    const crawlWarnings: string[] = []; // non-fatal step failures surfaced in job result
    // Pre-populate from a previous failed job so already-captured URLs are skipped
    if (opts.skipUrls?.length) {
      for (const u of opts.skipUrls) visited.set(canonicalize(u), 'skipped');
    }
    // Pre-auth public pages already captured above — mark visited so BFS (post-auth)
    // doesn't overwrite them with authenticated redirects showing app content instead.
    for (const u of preAuthPublicCaptured) visited.set(canonicalize(u), 'pre-auth');
    let checkpointSuccessCount = 0;
    let maxSeenTotal = 0; // track high-water mark so progress % never goes backwards
    let lastEmittedPct = 20; // floor above auth-discovery + login phases (max ~19%)
    const crawlLoopStart = Date.now();
    // Global crawl timeout — prevent BFS from hanging indefinitely on large apps.
    // Default 2 hours; override via CRAWL_GLOBAL_TIMEOUT_MS env var.
    const CRAWL_GLOBAL_TIMEOUT_MS = parseInt(process.env.CRAWL_GLOBAL_TIMEOUT_MS ?? '') || 2 * 60 * 60 * 1000;
    const isCrawlTimedOut = () => (Date.now() - crawlLoopStart) >= CRAWL_GLOBAL_TIMEOUT_MS;
    const startUrl = page.url();
    // Resume from previous segment's pending queue if provided
    const resumeUrls = opts.resumeUrls ?? [];
    // Sitemap-discovered URLs are additional seeds (not a replacement for
    // link-following) — merged in below unless this is a selective
    // ("crawl only these URLs") run or a resumed segment, matching how
    // GraphQL auto-introspection — fire-and-forget, stored in crawl metadata
    const gqlIntrospectionPromise = (async () => {
      const gqlEndpoints = guessGraphQLEndpoints(opts.appUrl);
      for (const ep of gqlEndpoints) {
        const result = await introspectGraphQL(ep, { timeoutMs: 5_000 }).catch(() => null);
        if (result?.schema) {
          logger.info({ endpoint: ep, typeCount: result.schema.types.length }, '[crawler] GraphQL schema discovered');
          return result;
        }
      }
      return null;
    })();

    // seedUrls itself works. Best-effort: discoverSitemapUrls() never throws.
    const sitemapUrls = (opts.useSitemap && !selective && resumeUrls.length === 0)
      ? await discoverSitemapUrls(opts.appUrl, opts.sitemapUrl)
      : [];
    // Filter bad URLs out of resumeUrls (generated dynamic routes from prior segments)
    const filteredResumeUrls = resumeUrls.filter(u => {
      try { return !isGeneratedDynamicPath(new URL(u).pathname); } catch { return true; }
    });
    const seedUrls = selective
      ? opts.selectiveUrls!
      : filteredResumeUrls.length > 0
        ? filteredResumeUrls
        : [
          startUrl,
          // User-configured seed URLs added to queue; link discovery still runs from each.
          ...(opts.startUrls ?? []).filter((u) => canonicalize(u) !== canonicalize(startUrl)),
          ...sitemapUrls.filter((u) => canonicalize(u) !== canonicalize(startUrl)),
        ];
    const queued = new Set<string>(seedUrls.map(canonicalize));
    const queue = new CrawlFrontier(opts.traversalStrategy ?? 'BFS', seedUrls, opts.bestFirstKeywords, opts.urlFilter);
    // Selective recrawl: block all link-discovery pushes so BFS only visits the requested URLs.
    // Constructor seeding already ran above — this no-op override fires only for BFS-loop discoveries.
    if (selective) {
      queue.push = () => {};
    }
    const deduplicator = new ContentDeduplicator(parseFloat(process.env.CRAWLER_DEDUP_THRESHOLD ?? '') || 0.88);
    const deadLinkTracker = new DeadLinkTracker();
    const redirectRegistry = new RedirectRegistry();
    // Re-seed from previously-discovered screens so deep routes found by AI link
    // discovery in prior crawls aren't lost when a recrawl starts from scratch.
    // Doc 41 Option D: orphan-classified screens (no inbound nav edges) get a BEST_FIRST
    // score boost via parentElements:0 so they're visited early in the next crawl.
    // Only applies to full recrawls (not selective runs or segment resumptions).
    if (!selective && resumeUrls.length === 0) {
      try {
        const priorMap = projectForProfile ?? await AppBrain.getProjectMap(opts.tenantId, opts.projectId);
        const orphanReport = await AppBrain.getOrphanScreens(opts.tenantId, opts.projectId).catch(() => null);
        const orphanIdSet = new Set<string>(
          (orphanReport?.screens ?? [])
            .filter((s: any) => s.classification !== 'REACHABLE')
            .map((s: any) => s.id as string),
        );
        // Snapshot queued set BEFORE filter so canonical dedup works correctly during filter.
        // Without this, multiple prior screens with the same canonical (e.g. playground?cmd=X1,
        // playground?cmd=X2 → same canonical) all pass the filter since queued isn't updated yet.
        const seenReseedCanon = new Set<string>(queued);
        const priorScreens = (priorMap?.screens ?? [])
          .filter((s: any) => s.status !== 'PENDING_REMOVAL')
          .filter((s: any) => {
            const u = s.url as string | null;
            if (!u) return false;
            try {
              const { pathname } = new URL(u);
              const stripWww = (h: string) => h.replace(/^www\./i, '').toLowerCase();
              if (stripWww(new URL(u).hostname) !== stripWww(new URL(startUrl).hostname)) return false;
              // Don't re-seed generated dynamic routes — they produce duplicate templates.
              if (isGeneratedDynamicPath(pathname)) return false;
              // Don't re-seed API/export/download paths — they were misidentified as screens.
              if (!isUsefulNavigationUrl(u, startUrl)) return false;
              // Canonical dedup: mark seen immediately during filter so duplicates are caught.
              const canon = canonicalize(u);
              if (seenReseedCanon.has(canon)) return false;
              seenReseedCanon.add(canon);
              return true;
            } catch { return false; }
          });
        let orphanSeedCount = 0;
        for (const s of priorScreens) {
          const u = s.url as string;
          queued.add(canonicalize(u));
          const isOrphan = orphanIdSet.has(s.id as string);
          queue.push(u, isOrphan ? { parentElements: 0 } : undefined);
          if (isOrphan) orphanSeedCount++;
        }
        if (priorScreens.length > 0) {
          logger.info(
            { count: priorScreens.length, orphanBoosted: orphanSeedCount },
            '[crawler] re-seeded prior screen URLs from DB',
          );
        }
      } catch (err) {
        logger.warn({ err: String(err) }, '[crawler] non-fatal: could not load prior screen URLs for re-seeding');
      }
    }
    // G11: Pre-auth sitemap discovery — capture public pages before session injection.
    // Many apps have marketing/pricing/docs pages only reachable without auth.
    // We fetch the sitemap, visit each URL, and screenshot pages that DON'T redirect to login.
    if (opts.useSitemap !== false) {
      const g11SitemapUrls: string[] = [];
      const sitemapCandidates = [
        `${opts.appUrl.replace(/\/$/, '')}/sitemap.xml`,
        `${opts.appUrl.replace(/\/$/, '')}/sitemap_index.xml`,
        `${opts.appUrl.replace(/\/$/, '')}/robots.txt`,
      ];
      const LOGIN_REDIRECT_PATH = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;

      try {
        for (const candidate of sitemapCandidates) {
          try {
            const resp = await fetch(candidate, { signal: AbortSignal.timeout(8_000) }).catch(() => null);
            if (!resp?.ok) continue;
            const text = await resp.text();
            if (candidate.endsWith('robots.txt')) {
              const sitemapLines = text.split('\n').filter((l: string) => l.toLowerCase().startsWith('sitemap:'));
              for (const line of sitemapLines) {
                const u = line.split(':').slice(1).join(':').trim();
                if (u) sitemapCandidates.push(u);
              }
              continue;
            }
            const locs = text.match(/<loc>([^<]+)<\/loc>/gi) ?? [];
            for (const loc of locs) {
              const u = loc.replace(/<\/?loc>/gi, '').trim();
              if (u.startsWith('http')) g11SitemapUrls.push(u);
            }
            if (g11SitemapUrls.length > 0) break;
          } catch { /* try next candidate */ }
        }

        if (g11SitemapUrls.length > 0) {
          logger.info({ count: g11SitemapUrls.length }, '[crawler] G11: sitemap discovered public URL candidates');
          const sitemapOrigin = new URL(opts.appUrl).origin;
          let sitemapCaptured = 0;
          for (const sUrl of g11SitemapUrls) {
            if (isCrawlTimedOut()) break;
            try {
              const u = new URL(sUrl);
              if (u.origin !== sitemapOrigin) continue;
              if (API_PATH_RE.test(u.pathname) || EXPORT_PATH_RE.test(u.pathname) || DOWNLOAD_EXT_RE.test(u.pathname)) continue;
              await page.goto(sUrl, { waitUntil: 'domcontentloaded', timeout: 12_000 });
              await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
              await waitForStableDOM(page);
              const landedUrl = page.url();
              if (LOGIN_REDIRECT_PATH.test(new URL(landedUrl).pathname)) {
                const canon = canonicalize(sUrl);
                if (!queued.has(canon)) { queued.add(canon); queue.push(sUrl); }
                logger.info({ url: sUrl }, '[crawler] G11: sitemap URL requires auth — added to BFS queue');
                continue;
              }
              const pagePath = u.pathname.replace(/^\//, '') || 'home';
              const pageName = pagePath.split('/').map((s: string) => s.charAt(0).toUpperCase() + s.slice(1)).join(' ') || 'Home';
              const shotPath = path.join(opts.screenshotDir ?? '/tmp', `sitemap-preauth-${sitemapCaptured}.png`);
              await waitForScreenshotReady(page);
              await page.screenshot({ path: shotPath, fullPage: true }).catch(() => { });
              const pubScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, pageName, sUrl, opts.jobId, 'SITEMAP').catch(() => null);
              if (pubScreen) {
                const sitemapCloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => undefined);
                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, pubScreen.id, shotPath, pubScreen.version, 'DESKTOP', sitemapCloudUrl).catch(() => { });
                // GAP 2: DOM snapshot for G11 pre-auth sitemap path
                const sitemapDom = await capturePageHtml(page);
                if (sitemapDom) {
                  const sitemapDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
                  await fs.writeFile(sitemapDomPath, sitemapDom, 'utf8');
                  const sitemapDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(sitemapDomPath)}`;
                  const sitemapDomCloud = await uploadToCloud(sitemapDomPath, sitemapDomKey).catch(() => undefined);
                  AppBrain.updateScreenDomSnapshot(opts.tenantId, pubScreen.id, sitemapDomPath, sitemapDomCloud).catch(() => {});
                }
                logger.info({ url: sUrl, name: pageName }, '[crawler] G11: captured public sitemap page');
              }
              sitemapCaptured++;
            } catch (sitemapPageErr) {
              logger.warn({ url: sUrl, err: String(sitemapPageErr) }, '[crawler] G11: sitemap page failed — skipping');
            }
          }
          logger.info({ captured: sitemapCaptured }, '[crawler] G11: pre-auth sitemap capture complete');
          await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => { });
          await waitForStableDOM(page);
        }
      } catch (sitemapErr) {
        logger.warn({ err: String(sitemapErr) }, '[crawler] G11: pre-auth sitemap discovery failed — continuing');
      }
    }

    // Discovery-method tracking for orphan-page detection (see AppBrain.getOrphanScreens):
    // the entry point is never an "orphan" candidate; anything else in the initial seed
    // batch is a configured seed URL (sitemap-discovered URLs included), not something
    // reached by following a link.
    const canonicalEntryUrl = canonicalize(startUrl);
    const canonicalSitemapUrls = new Set(sitemapUrls.map(canonicalize));
    const canonicalSeedUrls = new Set(seedUrls.map(canonicalize));
    const discoveryMethodFor = (url: string): 'ENTRY_POINT' | 'SEED_URL' | 'SITEMAP' | 'LINK_FOLLOW' => {
      const c = canonicalize(url);
      if (c === canonicalEntryUrl) return 'ENTRY_POINT';
      if (canonicalSitemapUrls.has(c)) return 'SITEMAP';
      if (canonicalSeedUrls.has(c)) return 'SEED_URL';
      return 'LINK_FOLLOW';
    };
    const edges: Array<{ fromUrl: string; toUrl: string }> = [];
    const pendingTransitions: Array<{ fromScreenId: string; elementMeaning: string; targetUrl: string }> = [];
    const pendingSummaryWrites: Promise<void>[] = [];
    const discovered: Array<{
      url: string; screenId?: string; elements?: number;
      elementFacts?: Array<{ meaning: string; role: string; expectedData?: string }>;
      screenshotPath?: string; version?: number; error?: string;
    }> = [];
    const performanceMetrics: PerformanceMetric[] = [];
    const seenDomHashes = new Set<string>(opts.knownDomHashes ?? []);

    // Legal/policy-only pages — no interactive elements worth testing; skip to prevent noise
    // NOTE: /about, /contact, /careers, /help are intentionally NOT skipped — they have
    // real test targets (contact forms, job apply forms, help search, team content)
    const skipPathExtra = process.env.CRAWL_SKIP_PATH_EXTRA
      ? process.env.CRAWL_SKIP_PATH_EXTRA.split(',').map((p) => p.trim()).filter(Boolean)
      : [];
    const SKIP_PATH = skipPathExtra.length > 0
      ? new RegExp(`/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap|${skipPathExtra.join('|')})([-/]|$)`, 'i')
      : /\/(terms|privacy|dpa|legal|cookie-policy|changelog|blog|imprint|gdpr|compliance|acceptable-use|security-policy|press|sitemap)([-/]|$)/i;
    const segIdx = opts.segmentIndex ?? 1;

    const robotsCache = opts.respectRobotsTxt ? new RobotsCache() : null;
    // AutoThrottle state — exponential moving average of navigation latency
    let avgNavMs = 0;
    let navCount = 0;
    // Rate-limit circuit breaker: after 5 consecutive AI errors (429), skip AI for 3 screens then retry
    // (was: 3 errors → skip 10 screens — too aggressive for Nvidia NIM which rate-limits quickly)
    let aiErrorStreak = 0;
    let aiSkipCount = 0;
    const aiMinElements = parseInt(process.env.CRAWL_AI_LINK_MIN_ELEMENTS ?? '') || 5;
    let aiRateLimitWarned = false;
    // Hard guard: total URLs attempted (including skips) capped at maxScreens * 4
    // to prevent infinite-URL-explosion from WooCommerce/Shopify facet filter combos
    // that slip through canonicalize() if any new filter param type is encountered.
    let totalAttemptsThisSegment = 0;
    const maxAttempts = Math.max(segmentSize * 4, 200);
    // Count consecutive login-wall redirects — if too many, abort early (auth-wall site, no creds)
    let loginWallRedirectCount = 0;
    const LOGIN_WALL_ABORT_THRESHOLD = Math.max(8, Math.ceil(segmentSize * 0.4));

    let pendingSessionRestore: (() => Promise<void>) | null = null;

    // F10: Parallel lite-worker browser tabs — pull from shared queue while main loop runs.
    // Each worker uses a fresh Playwright page from the same browser context so it shares
    // cookies/session with the main page. DOM-only extraction (no Stagehand AI).
    // Default 5 = 3-5× crawl throughput on a single browser context. Set CRAWL_PAGE_PARALLELISM=1 to disable.
    const CRAWL_PAGE_PARALLELISM = Math.max(1, parseInt(process.env.CRAWL_PAGE_PARALLELISM ?? '5', 10));

    async function runLiteWorker(workerPage: any): Promise<void> {
      while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
        if (opts.abortSignal?.aborted) break;
        if (isCrawlTimedOut()) break;
        if (++totalAttemptsThisSegment > maxAttempts) break;

        const url = queue.shift();
        if (!url) break;

        const canonUrl = canonicalize(url);
        if (visited.has(canonUrl)) continue;

        try {
          const { pathname } = new URL(url);
          if (SKIP_PATH.test(pathname) || isGeneratedDynamicPath(pathname)) continue;
          // Skip API/export paths — these return data files, not navigable pages
          if (API_PATH_RE.test(pathname) || EXPORT_PATH_RE.test(pathname) || DOWNLOAD_EXT_RE.test(pathname)) continue;
        } catch { continue; }

        // Claim URL before first await — prevents duplicate processing across parallel workers
        visited.set(canonUrl, '_pending_');

        try {
          const navErr = await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: tuning.navTimeoutMs }).then(() => null as null, (e: any) => e);
          if (navErr && /timeout/i.test(String((navErr as any)?.message ?? navErr))) {
            await workerPage.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch(() => {});
          }

          // Wait for SPA JS to render components before extracting elements or running soft-404 check.
          // domcontentloaded fires before JS executes — networkidle ensures React/Vue/Angular hydration.
          await workerPage.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
          // Wait for DOM to stop mutating (React/Vue render cycles) before element extraction.
          await waitForStableDOM(workerPage, 6_000, 400).catch(() => {});
          // Dismiss cookie banners / GDPR overlays so underlying elements are accessible.
          await dismissOverlays(workerPage).catch(() => {});

          // Auth wall detection: if navigation redirected to a login page, skip this URL.
          // Lite worker cannot re-authenticate (no Stagehand access) — mark and skip.
          const lwCurrentUrl = workerPage.url();
          const lwPath = (() => { try { return new URL(lwCurrentUrl).pathname.toLowerCase(); } catch { return ''; } })();
          const lwIsAuthWall = /^\/(login|signin|sign-in|auth|authenticate|session\/new|account\/login|user\/login)/.test(lwPath) && lwCurrentUrl !== url;
          const lwHasLoginForm = lwIsAuthWall ? false : await workerPage.evaluate(() => {
            const inputs = document.querySelectorAll('input[type="password"]');
            return inputs.length > 0;
          }).catch(() => false);
          if (lwIsAuthWall) {
            logger.warn({ url, landed: lwCurrentUrl }, '[lite-worker] auth wall detected — queuing for main worker re-auth');
            visited.set(canonUrl, '_auth_wall_');
            if (!revisitQueue.includes(url)) revisitQueue.push(url);
            continue;
          }
          // lwHasLoginForm (direct sign-in/register page): process normally — capture the form elements.
          // Do NOT add to revisitQueue; P3a cannot do anything useful with a login page revisit.

          // Soft-404 detection for lite-worker: skip SPA catch-all URLs that mirror homepage
          if (homeFingerprintElemCount >= 0 && url !== startUrl) {
            const wTitle = await workerPage.title().catch(() => '');
            if (wTitle === homeFingerprintTitle && wTitle !== '') {
              const wElemCount = await workerPage.evaluate(() =>
                document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
              ).catch(() => -1);
              if (wElemCount >= 0 && wElemCount === homeFingerprintElemCount) {
                console.log(`[lite-worker] skip ${url} — soft-404 (homepage fingerprint match)`);
                visited.set(canonUrl, '_soft404_');
                continue;
              }
            }
          }

          const pathParts = new URL(url).pathname.split('/').filter(Boolean);
          const name = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';

          const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
          visited.set(canonUrl, screen.id);

          // Inline auth-wall detection: PHP/legacy apps render session-expired content without URL redirect.
          // URL-redirect case (lwIsAuthWall above) is handled by main BFS via revisitQueue.

          const lwBodyText = await workerPage.evaluate(() => (document as any).body?.innerText ?? '').catch(() => '');
          if (INLINE_AUTH_WALL_RE.test(lwBodyText)) {
            logger.warn({ url, screenId: screen.id }, '[lite-worker] Inline session-expired content detected — marking requiresAuth');
            AppBrain.updateScreenRequiresAuth(opts.tenantId, screen.id, true).catch(() => {});
            visited.set(canonUrl, '_auth_wall_');
            if (!revisitQueue.includes(url)) revisitQueue.push(url);
            continue;
          }

          const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${screen.id}-desktop-v${screen.version}.png`);

          let liteElements = await domFallbackElements(workerPage).catch(() => [] as Awaited<ReturnType<typeof domFallbackElements>>);
          if (liteElements.length === 0) {
            // domFallbackElements returned nothing — try broader visible-element selector as last resort.
            liteElements = await workerPage.evaluate(() => {
              const sel = 'button,a[href],input,select,textarea,[role],[tabindex="0"],[onclick],[data-action]';
              return Array.from(document.querySelectorAll(sel))
                .filter((el: any) => el.offsetParent !== null)
                .slice(0, 150)
                .map((el: any) => ({
                  meaning: el.getAttribute('aria-label') || el.textContent?.trim().slice(0, 100) || el.tagName.toLowerCase(),
                  role: el.getAttribute('role') || el.tagName.toLowerCase(),
                  expectedData: null, notes: null, boundingRect: null, ariaState: null, parentLandmark: null,
                }));
            }).catch(() => []) as any[];
          }
          const [elements] = await Promise.all([
            Promise.resolve(liteElements),
            workerPage.screenshot({ path: shotPath, fullPage: true })
              .then(async () => {
                const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
                return (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', cloudUrl);
              })
              .catch(() => {}),
          ]);

          // GAP 1: DOM snapshot for lite-worker path
          const lwDom = await capturePageHtml(workerPage);
          if (lwDom) {
            const lwDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
            await fs.writeFile(lwDomPath, lwDom, 'utf8');
            const lwDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(lwDomPath)}`;
            const lwDomCloud = await uploadToCloud(lwDomPath, lwDomKey).catch(() => undefined);
            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, lwDomPath, lwDomCloud).catch(() => {});
          }

          // Nav-link pass — merged into main save cycle so clearScreenElements covers them too.
          // extractElements (main BFS) never runs for lite-worker screens, so nav/footer roles
          // must be captured here. Running before clear+save prevents additive accumulation.
          const _navEls = await (workerPage.evaluate as any)(() => {
            if (typeof (globalThis as any).__name === 'undefined') (globalThis as any).__name = (fn: any) => fn;
            const seen = new Set<string>();
            const items: Array<{meaning: string; role: string; notes: string}> = [];
            function isV(el: Element) {
              const s = window.getComputedStyle(el);
              return s.display !== 'none' && s.visibility !== 'hidden' && s.opacity !== '0';
            }
            // Wide nav selector: covers semantic nav, ARIA, header links, sidebar (aside/[role=complementary]),
            // and common class-based sidebars used by admin/dashboard pages.
            const NAV_SEL = [
              'nav a', '[role="navigation"] a', 'header a', 'footer nav a',
              '[aria-label*="navigation" i] a', '[aria-label*="sidebar" i] a',
              'aside a', '[role="complementary"] a',
              '[class*="sidebar" i] a', '[class*="side-nav" i] a', '[class*="sidenav" i] a',
              '[class*="nav-menu" i] a', '[class*="navmenu" i] a', '[data-nav] a',
            ].join(', ');
            document.querySelectorAll(NAV_SEL).forEach((a: any) => {
              if (!isV(a)) return;
              const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
              const href = a.href ?? '';
              if (!label || !href || /^(javascript:|#$)/i.test(href)) return;
              let landmark = 'Site Navigation';
              let cur: any = a.parentElement;
              while (cur) {
                const t = cur.tagName?.toLowerCase();
                const r = cur.getAttribute?.('role')?.toLowerCase();
                const cl = (cur.className ?? '').toLowerCase();
                if (t === 'header' || r === 'banner') { landmark = 'Header'; break; }
                if (t === 'footer' || r === 'contentinfo') { landmark = 'Footer'; break; }
                if (t === 'aside' || r === 'complementary' || cl.includes('sidebar') || cl.includes('sidenav') || cl.includes('side-nav')) { landmark = 'Sidebar'; break; }
                if (t === 'nav' || r === 'navigation') { landmark = cur.getAttribute('aria-label')?.trim() || 'Site Navigation'; break; }
                cur = cur.parentElement;
              }
              const key = `nav|${label.toLowerCase()}|${href}`;
              if (!seen.has(key)) { seen.add(key); items.push({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark }) }); }
            });
            let fc = 0;
            document.querySelectorAll('footer a, [role="contentinfo"] a, [class*="footer" i] a').forEach((a: any) => {
              if (fc >= 20 || !isV(a)) return;
              const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
              const href = a.href ?? '';
              if (!label || !href) return;
              const key = `footer|${label.toLowerCase()}|${href}`;
              if (!seen.has(key)) { seen.add(key); items.push({ meaning: label, role: 'footer-link', notes: JSON.stringify({ href: href.slice(0, 150) }) }); fc++; }
            });
            // Grandparent-grouping fallback: detects sidebar navigation where each link is
            // individually wrapped (e.g. <div class="flex"><a>link</a></div> × many siblings).
            // Runs ALWAYS (not gated on 0 results) so header links don't block sidebar detection.
            // Deduped against hrefs already captured by the semantic pass above.
            {
              const navHrefsSeen = new Set<string>();
              for (const it of items) {
                try { const h = JSON.parse(it.notes ?? '{}').href; if (h) navHrefsSeen.add(h); } catch { /* skip */ }
              }
              const gpGroups = new Map<Element, Array<{label: string; href: string}>>();
              document.querySelectorAll('a[href]').forEach((a: any) => {
                if (!isV(a)) return;
                const href = a.href ?? '';
                if (!href || /^(javascript:|#$|mailto:|tel:)/i.test(href) || navHrefsSeen.has(href)) return;
                const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
                if (!label || label.length > 40) return;
                const parent = a.parentElement;
                const gp = parent?.parentElement;
                if (!parent || !gp) return;
                // Only match when parent wraps exactly 1 anchor — the sidebar <div><a> pattern
                if (parent.querySelectorAll('a').length !== 1) return;
                const gpTag = gp.tagName.toLowerCase();
                if (['body', 'html', 'form'].includes(gpTag)) return;
                const arr = gpGroups.get(gp) ?? [];
                arr.push({ label, href });
                gpGroups.set(gp, arr);
              });
              for (const [, links] of gpGroups.entries()) {
                if (links.length >= 4) {
                  for (const { label, href } of links) {
                    const k = `cluster|${label.toLowerCase()}|${href}`;
                    if (!seen.has(k)) { seen.add(k); items.push({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark: 'Sidebar' }) }); }
                  }
                }
              }
            }
            return items;
          }).catch(() => [] as any[]);

          const allElements = [
            ...elements,
            ..._navEls.map((ne: any) => ({ ...ne, expectedData: undefined, boundingRect: undefined, ariaState: undefined, parentLandmark: undefined })),
          ];

          // Persist lite-worker elements to DB — without this, 80% of screens have 0 DB elements.
          if (allElements.length > 0) {
            await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
            for (let _ei = 0; _ei < allElements.length; _ei += 2) {
              const _batch = allElements.slice(_ei, _ei + 2);
              await Promise.all(_batch.map((el: any) =>
                AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {})
              ));
            }
            logger.info({ url, screenId: screen.id, elements: elements.length, navLinks: _navEls.length }, '[lite-worker] screen captured');
          } else if (!revisitQueue.includes(url)) {
            // 0-element: queue for main BFS re-visit with full Stagehand AI extraction
            logger.warn({ url, screenId: screen.id }, '[lite-worker] 0 elements — queued for P3a revisit');
            revisitQueue.push(url);
          }

          // Non-mutating probes for lite-worker screens: shadow DOM, a11y tree, service worker.
          // Awaited with 15s total timeout — these are read-only and safe on the shared worker page.
          if (screen?.id) {
            const [shadowRes, a11yRes, swRes] = await Promise.race([
              Promise.allSettled([
                crawlShadowDom(workerPage),
                crawlAccessibilityTree(workerPage),
                inspectServiceWorker(workerPage),
              ]),
              new Promise<PromiseSettledResult<never>[]>((resolve) =>
                setTimeout(() => resolve([
                  { status: 'rejected' as const, reason: 'timeout' },
                  { status: 'rejected' as const, reason: 'timeout' },
                  { status: 'rejected' as const, reason: 'timeout' },
                ]), 15_000)
              ),
            ]);
            const litePayload: Record<string, unknown> = { screenId: screen.id, url };
            if (shadowRes.status === 'fulfilled' && shadowRes.value && (shadowRes.value as any).shadowHostCount > 0)
              litePayload.shadowDom = shadowRes.value;
            if (a11yRes.status === 'fulfilled' && a11yRes.value)
              litePayload.accessibilityTree = a11yRes.value;
            if (swRes.status === 'fulfilled' && (swRes.value as any).supported)
              litePayload.serviceWorker = swRes.value;
            if (Object.keys(litePayload).length > 2) {
              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
                jobId: opts.jobId, type: 'PER_SCREEN_PROBES', payload: litePayload,
              }).catch(() => {});
            }
          }

          const nextLinks: string[] = await (workerPage.evaluate as any)((originStr: string) => {
            return Array.from(document.querySelectorAll('a[href],[data-href]'))
              .map((a: any) => {
                const raw = a.getAttribute('href') || a.getAttribute('data-href') || '';
                try { return new URL(raw, window.location.href).href; }
                catch { return ''; }
              })
              .filter((h: string) => {
                if (!h) return false;
                try {
                  const u = new URL(h);
                  if (u.origin !== originStr) return false;
                  // Keep hash-router paths (#/route, #!/route) but drop plain fragment anchors (#section)
                  if (u.hash && !u.hash.startsWith('#/') && !u.hash.startsWith('#!/')) return false;
                  return true;
                } catch { return false; }
              })
              .slice(0, 200);
          }, origin).catch(() => []);

          for (const link of nextLinks) {
            if (!isUsefulNavigationUrl(link, origin)) continue;
            const cLink = canonicalize(link);
            if (!visited.has(cLink) && !queued.has(cLink)) {
              queued.add(cLink);
              queue.push(link);
              edges.push({ fromUrl: url, toUrl: link });
            }
          }

          discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
          // Lite workers also advance the progress bar — without this 4/5 workers are invisible.
          const lwEffMax = isFinite(maxScreens) ? Math.min(visited.size + queue.length, maxScreens * 3) : visited.size + queue.length;
          maxSeenTotal = Math.max(maxSeenTotal, lwEffMax);
          const lwRawPct = 20 + Math.round(Math.min(57, (visited.size / Math.max(1, maxSeenTotal)) * 57));
          const lwPct = Math.max(lastEmittedPct, lwRawPct);
          lastEmittedPct = lwPct;
          await onProgress?.(lwPct, `Crawling screens (${visited.size} mapped, ${queue.length} pending)`).catch(() => {});
        } catch (err: any) {
          logger.warn({ url, err: err?.message ?? String(err) }, '[lite-worker] URL processing failed');
          discovered.push({ url, error: err?.message ?? String(err) });
        }
      }
    }

    const liteWorkerPromises: Promise<void>[] = [];
    if (CRAWL_PAGE_PARALLELISM > 1 && playwrightBrowser && !opts.discoveryMode) {
      const pwCtx = playwrightBrowser.contexts()[0];
      for (let i = 1; i < CRAWL_PAGE_PARALLELISM; i++) {
        const wp = await pwCtx.newPage();
        await wp.addInitScript(() => { Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); }).catch(() => {});
        wp.setDefaultTimeout(tuning.defaultTimeoutMs || 45_000);
        liteWorkerPromises.push(runLiteWorker(wp).finally(() => wp.close().catch(() => {})));
      }
    }

    while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
      if (pendingSessionRestore) {
        await pendingSessionRestore();
        pendingSessionRestore = null;
      }
      if (++totalAttemptsThisSegment > maxAttempts) {
        logger.warn({ maxAttempts, visited: visited.size, queued: queue.length }, '[crawler] attempt cap reached — stopping to prevent URL explosion');
        break;
      }
      if (opts.abortSignal?.aborted) {
        logger.info('Abort signal received — stopping crawl');
        throw new Error('CRAWL_ABORTED');
      }
      if (isCrawlTimedOut()) {
        const elapsedMin = Math.round((Date.now() - crawlLoopStart) / 60_000);
        logger.warn({ elapsedMin, visited: visited.size, queued: queue.length }, '[crawler] Global timeout reached — stopping BFS to prevent runaway crawl');
        await onProgress?.(89, `Crawl timeout after ${elapsedMin}min — ${visited.size} pages captured`).catch(() => {});
        break;
      }
      // AutoThrottle: delay next request proportional to avg navigation latency
      if (opts.autoThrottle && navCount > 0) {
        const delayMs = Math.min(5_000, Math.max(0, avgNavMs * 0.5));
        if (delayMs > 100) await page.waitForTimeout(delayMs).catch(() => { });
      }

      const url = queue.shift()!;
      const canonUrl = canonicalize(url);
      const apiObservationStartIndex = capturedApiEndpoints.length;

      // Skip if already visited (canonical match — catches ID-parameterized duplicates)
      if (visited.has(canonUrl)) continue;
      // Claim immediately before any await — prevents lite workers from picking up the same URL
      // concurrently while this iteration is suspended (race that causes repeated visits).
      visited.set(canonUrl, '_pending_');
      logger.info({ url, visited: visited.size, queued: queue.length, segment: `${visited.size}/${segmentSize}` }, '[crawler] visiting URL');

      // robots.txt compliance — skip disallowed URLs
      if (robotsCache && !await robotsCache.isAllowed(url)) {
        logger.info({ url }, '[crawler] robots.txt: skipping disallowed URL');
        continue;
      }

      // Skip content/legal pages — they have no interactive form elements worth testing
      try {
        const { pathname } = new URL(url);
        if (SKIP_PATH.test(pathname)) {
          console.log(`[crawler] skip content page: ${url}`);
          continue;
        }
        // Generated ID-like path segments are usually fallback IDs or entity
        // records. They flood the catalog with duplicate page templates, so
        // skip them before screenshots or screen creation.
        if (isGeneratedDynamicPath(pathname)) {
          console.log(`[crawler] skip generated dynamic route: ${url}`);
          continue;
        }
        // Skip API/export/download paths — navigating to these triggers file downloads.
        // The main BFS uses Stagehand's context which has no route interception.
        if (API_PATH_RE.test(pathname) || EXPORT_PATH_RE.test(pathname) || DOWNLOAD_EXT_RE.test(pathname)) {
          console.log(`[crawler] skip API/export/download path: ${url}`);
          visited.set(canonUrl, '_skipped_api_');
          continue;
        }
      } catch { /* invalid url, let it fall through */ }

      try {
        // Security headers — capture from first document response via CDP.
        // Stagehand V3's page proxy only supports the 'console' event (page.on throws for others).
        // We hook directly into the underlying CDP session's Network.responseReceived event instead.
        let securityHeaders: Record<string, string | null> | null = null;
        let httpStatus: number | null = null;
        const urlBase = url.replace(/[?#].*/, '');
        const cdpSession: any = (page as any).mainSession ?? (page as any)._session ?? null;
        const secHeadersCdp = (evt: any) => {
          if (securityHeaders) return;
          try {
            const responseUrl: string = evt?.response?.url ?? '';
            if (!responseUrl.startsWith(urlBase)) return;
            httpStatus = evt?.response?.status ?? null;
            const headers: Record<string, string> = {};
            // CDP headers come as array of {name, value} objects
            for (const h of (evt?.response?.headers ?? [])) headers[h.name.toLowerCase()] = h.value;
            securityHeaders = {
              csp: headers['content-security-policy'] ?? headers['content-security-policy-report-only'] ?? null,
              hsts: headers['strict-transport-security'] ?? null,
              xContentType: headers['x-content-type-options'] ?? null,
              xFrame: headers['x-frame-options'] ?? null,
              referrer: headers['referrer-policy'] ?? null,
            };
          } catch { /* non-fatal */ }
        };
        if (cdpSession?.on) cdpSession.on('Network.responseReceived', secHeadersCdp);

        const wsDetector = attachWebSocketDetector(cdpSession);

        if (url !== page.url()) {
          // LCP/CLS observers are registered via the addInitScript above, which
          // re-runs automatically on this navigation — nothing to do here.
          const navStart = Date.now();
          // Track redirect hops during this navigation
          const redirectHops: Array<{ from: string; to: string; statusCode: number }> = [];
          const onRedirectResponse = (res: any) => {
            const st = res.status();
            if (st >= 300 && st < 400) {
              redirectHops.push({ from: res.url(), to: res.headers()['location'] ?? '', statusCode: st });
            } else if (st >= 400) {
              deadLinkTracker.record(res.url(), st, { sourceUrl: url });
            }
          };
          rawPage.on('response', onRedirectResponse);

          // Session Isolation: Clear session before crawling public auth routes
          const AUTH_PATH_REGEX = /\/(login|register|signup|sign-in|sign-up|forgot-password|reset-password)([-/]|$)/i;
          const isAuthRoute = AUTH_PATH_REGEX.test(new URL(url).pathname);
          const browserCtx = playwrightBrowser?.contexts()[0];
          
          if (isAuthRoute && browserCtx) {
            const currentState = await browserCtx.storageState();
            const hasSession = (currentState.cookies?.length ?? 0) > 0 ||
              currentState.origins?.some((o: any) => (o.localStorage?.length ?? 0) > 0);
            if (hasSession) {
              pendingSessionRestore = async () => {
                const ctx = playwrightBrowser?.contexts()[0];
                if (ctx) {
                  await ctx.addCookies(currentState.cookies);
                  await rawPage.evaluate((origins: any) => {
                    for (const entry of (origins ?? [])) {
                      if (window.location.origin === entry.origin) {
                        for (const item of entry.localStorage) {
                          try { localStorage.setItem(item.name, item.value); } catch {}
                        }
                      }
                    }
                  }, currentState.origins).catch(() => {});
                  logger.info('[crawler] restored session state after public auth route');
                }
              };
              await browserCtx.clearCookies();
              await rawPage.evaluate(() => {
                try { localStorage.clear(); sessionStorage.clear(); } catch {}
              }).catch(() => {});
              logger.info({ url }, '[crawler] temporarily cleared session for public auth route');
            }
          }

          // Try fast 15 s first; only escalate to 30 s on timeout so happy-path pages don't wait
          const navErr15 = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: tuning.navTimeoutMs }).then(() => null, (e: any) => e);
          if (navErr15) {
            if (/timeout/i.test(String(navErr15?.message ?? navErr15))) {
              await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }).catch((navErr: any) => {
                logger.warn({ url, err: navErr?.message ?? String(navErr) }, '[crawler] navigation failed — skipping');
              });
            } else {
              logger.warn({ url, err: navErr15?.message ?? String(navErr15) }, '[crawler] navigation failed — skipping');
            }
          }
          rawPage.off('response', onRedirectResponse);
          if (redirectHops.length > 0) {
            const isCircularRedirect = new Set(redirectHops.map(h => h.from)).size < redirectHops.length;
            redirectRegistry.register({
              originalUrl: url, finalUrl: page.url(),
              hops: redirectHops.map((h, i) => ({ ...h, durationMs: i === 0 ? Date.now() - navStart : 0 })),
              isCircular: isCircularRedirect,
              hopCount: redirectHops.length,
              totalDurationMs: Date.now() - navStart,
            });
            if (isCircularRedirect) {
              logger.warn({ url, hops: redirectHops.map(h => h.from) }, '[crawler] circular redirect detected — skipping URL');
              continue;
            }
          }
          // Update exponential moving average for AutoThrottle
          const navMs = Date.now() - navStart;
          navCount++;
          avgNavMs = avgNavMs + (navMs - avgNavMs) / navCount;
        }
        // chrome-error:// = TCP/HTTP connection failed (IP block, DNS failure, etc.)
        // Skip immediately — don't screenshot, don't trigger Skyvern, just move on.
        if (page.url().startsWith('chrome-error://')) {
          console.warn(`[crawler] connection failed for ${url} — skipping (chrome-error page)`);
          continue;
        }
        // Full page-ready sequence:
        // 1. load — waits for initial page load (faster than networkidle; XHR handled by waitForStableDOM)
        // 2. DOM stability — waits for React/Vue hydration to finish mutating the DOM
        // 3. Modal capture — screenshot app dialogs BEFORE dismissal (they are testable UI states)
        // 4. Overlay dismissal — removes cookie banners/modals before element extraction
        // 5. Lazy-load scroll — triggers IntersectionObserver for off-screen images/components
        await (rawPage ?? page).waitForLoadState('load', { timeout: 8_000 }).catch(() => { });
        await waitForStableDOM(page, 8_000, 400, rawPage ?? undefined);
        // If DOM is sparse after stability check (< 15 interactive elements), SPA may still be
        // fetching data — skeleton is DOM-stable but empty. Extra networkidle pass catches it.
        const sparseCheck = await (rawPage ?? page).evaluate(() =>
          document.querySelectorAll('button,a,input,select,textarea,[role="button"],[aria-label]').length
        ).catch(() => 99);
        if (sparseCheck < 15) {
          await (rawPage ?? page).waitForLoadState('networkidle', { timeout: 6_000 }).catch(() => {});
          await waitForStableDOM(page, 4_000, 300, rawPage ?? undefined);
        }
        // Wait for SPA auth-check / loading spinners to resolve before screenshotting.
        // SPAs often show a "Checking authentication..." or "Loading..." state after networkidle
        // while client-side auth verifies tokens. waitForStableDOM catches DOM mutations but
        // returns as soon as the spinner DOM is stable — this catches the spinner itself.
        // Stagehand V3 proxy may not expose waitForFunction — use try/catch, not .catch(),
        // because calling an undefined method throws synchronously before a Promise is created.
        try {
          const spinnerPage = rawPage ?? page;
          if (typeof spinnerPage.waitForFunction === 'function') {
            await spinnerPage.waitForFunction(() => {
              const body = document.body;
              if (!body) return true;
              const text = body.innerText?.toLowerCase() ?? '';
              // Text-based loading indicators
              if (
                text.includes('checking authentication') ||
                text.includes('checking auth') ||
                text.includes('authenticating') ||
                text.includes('verifying session') ||
                text.includes('please wait') ||
                text.includes('loading data') ||
                (text.includes('loading') && text.length < 200)
              ) return false;
              // ARIA-based: aria-busy on any visible element
              const busyEl = body.querySelector('[aria-busy="true"]');
              if (busyEl && (busyEl as HTMLElement).offsetParent !== null) return false;
              // Data attributes: React Query / SWR / TanStack in-flight indicators
              if (body.querySelector('[data-loading="true"],[data-pending="true"],[data-fetching="true"]')) return false;
              // Indeterminate progress bars
              if (body.querySelector('progress:not([value])')) return false;
              // Common CSS class patterns for loading skeletons
              const skeleton = body.querySelector('.skeleton,.skeleton-loader,.shimmer,.pulse-loader,.content-loader');
              if (skeleton && (skeleton as HTMLElement).offsetParent !== null) return false;
              return true;
            }, { timeout: 12_000 });
          }
        } catch { /* timed out or unsupported — screenshot whatever is there */ }

        if (!opts.discoveryMode) {
          // Capture app-generated modals BEFORE dismissing — cookie/GDPR banners excluded.
          try {
            const appModalVisible = await page.evaluate(() => {
              const dialogs = Array.from(document.querySelectorAll('[role="dialog"], [role="alertdialog"], [aria-modal="true"]'));
              return dialogs.some((el: any) => {
                const text = (el.textContent ?? '').toLowerCase();
                const isCookieBanner = /(cookie|consent|gdpr|privacy policy|accept all|we use cookies)/i.test(text);
                const isVisible = el.offsetParent !== null || el.style.display !== 'none';
                return isVisible && !isCookieBanner;
              });
            }).catch(() => false);
            if (appModalVisible) {
              const modalHash = url.replace(/[^a-z0-9]/gi, '').slice(-12);
              const modalShotPath = path.join(opts.screenshotDir ?? '/tmp', `modal-${modalHash}.png`);
              await page.screenshot({ path: modalShotPath, fullPage: false }).catch(() => { });
              const modalName = await (rawPage ?? page).evaluate(() => {
                const h = document.querySelector('[role="dialog"] h1,[role="dialog"] h2,[role="dialog"] [role="heading"]');
                return h?.textContent?.trim() ?? 'Modal';
              }).catch(() => 'Modal');
              const modalScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, modalName, url, opts.jobId, discoveryMethodFor(url)).catch(() => null);
              if (modalScreen) {
                const modalCloudUrl = await uploadToCloud(modalShotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(modalShotPath)}`).catch(() => undefined);
                await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, modalScreen.id, modalShotPath, modalScreen.version, 'DESKTOP', modalCloudUrl).catch(() => { });
                // GAP 3: DOM snapshot for modal/dialog capture
                const modalDom = await capturePageHtml(page);
                if (modalDom) {
                  const modalDomPath = modalShotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
                  await fs.writeFile(modalDomPath, modalDom, 'utf8');
                  const modalDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(modalDomPath)}`;
                  const modalDomCloud = await uploadToCloud(modalDomPath, modalDomKey).catch(() => undefined);
                  AppBrain.updateScreenDomSnapshot(opts.tenantId, modalScreen.id, modalDomPath, modalDomCloud).catch(() => {});
                }
                // Extract elements from the modal BEFORE it is dismissed — use CDP path via rawPage
                const modalElements = await domFallbackElements(page, rawPage ?? undefined).catch(() => []);
                for (const el of modalElements) {
                  AppBrain.saveElement(opts.tenantId, opts.projectId, modalScreen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {});
                }
                logger.info({ url, name: modalName, elementCount: modalElements.length }, '[crawler] captured app modal before dismiss');
              }
            }
          } catch { /* non-fatal */ }
          await dismissOverlays(page);
          // Enhanced infinite scroll — handles lazy-load, infinite lists, and virtual scrollers
          await infiniteScrollToBottom(page, { maxScrolls: 30, scrollDelay: 350, stabilizeMs: 500 }, rawPage ?? undefined).catch(() => { });
        }
        if (cdpSession?.off) cdpSession.off('Network.responseReceived', secHeadersCdp);
        wsDetector.detach();

        // 429 Rate-limit — exponential backoff with jitter before continuing
        if (httpStatus === 429) {
          const backoffMs = Math.min(60_000, 1_000 * Math.pow(2, Math.min(navCount, 6)));
          logger.warn({ url, httpStatus, backoffMs }, '[crawler] 429 rate-limited — backing off');
          await new Promise(r => setTimeout(r, backoffMs + Math.random() * 1_000));
        }

        // Skip pages that returned an HTTP 4xx/5xx response — SPAs normally return 200 for all routes,
        // so a real HTTP error means the server knows this URL is invalid.
        if (httpStatus !== null && httpStatus >= 400) {
          logger.info({ url, httpStatus }, '[crawler] skip — server returned HTTP error');
          continue;
        }

        // ── Mid-crawl session expiry detection ───────────────────────────────────
        // Only trigger re-auth when the app REDIRECTED us away from the intended URL.
        // If we intentionally navigate to /session-expired or /login — save it as a screen (it's
        // a real page worth testing: messaging, recovery buttons, error UX, etc.).
        const BFS_ERROR_PATH = /\/(session-expired|expired|unauthorized|forbidden|401|403|logout|logged-out)(\/|$|\?)/i;
        const BFS_LOGIN_PATH = /\/(login|signin|sign-in|auth|log-in|sso|account\/login)(\/|$|\?)/i;
        const landedUrl = page.url();
        const weWantedThisPage = BFS_ERROR_PATH.test(new URL(url).pathname) || BFS_LOGIN_PATH.test(new URL(url).pathname);
        const redirectedToError = !weWantedThisPage && (
          BFS_ERROR_PATH.test(new URL(landedUrl).pathname) || BFS_LOGIN_PATH.test(new URL(landedUrl).pathname)
        );
        if (redirectedToError) {
          // Track login-wall redirects: if the site sends every URL to /login without credentials,
          // abort early rather than burning all maxAttempts on the same login page.
          if (BFS_LOGIN_PATH.test(new URL(landedUrl).pathname)) {
            loginWallRedirectCount++;
            if (!opts.credentials && loginWallRedirectCount >= LOGIN_WALL_ABORT_THRESHOLD) {
              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
                jobId: opts.jobId,
                type: 'AUTH_FAILURE',
                payload: {
                  url, landed: landedUrl,
                  reason: `Login wall detected: ${loginWallRedirectCount} consecutive URLs redirected to login. Add credentials in project settings to crawl authenticated pages.`,
                },
              }).catch(() => {});
              // Checkpoint remaining queue so job-queue can auto-requeue the pending URLs
              await opts.onCheckpoint?.(
                Array.from(visited.entries()).filter(([_k, v]) => v !== '_pending_').map(([k]) => k),
                queue.filter((u: string) => !visited.has(canonicalize(u)))
              ).catch(() => {});
              throw new Error(`[AUTH_FAILURE] Login wall detected: ${loginWallRedirectCount} URLs redirected to the login page. Add credentials in project settings and retry the crawl.`);
            }
          }
          // We asked for /dashboard but landed on /session-expired — session expired mid-crawl.
          logger.warn({ requested: url, landed: landedUrl }, '[crawler] Mid-crawl redirect to auth/error — attempting re-auth');
          let reauthed = false;
          if (opts.credentials) {
            reauthed = await loginWithStagehand(
              stagehand, opts.credentials, opts.appUrl,
              { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider },
              { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId },
              undefined,
              { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId },
            ).catch(() => false);
            if (reauthed) {
              logger.info({ url }, '[crawler] Re-auth succeeded — retrying URL');
              await page.goto(url, { waitUntil: 'domcontentloaded' }).catch(() => { });
              await (rawPage ?? page).waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
              await waitForStableDOM(page, 8_000, 400, rawPage ?? undefined);
              const afterReauthUrl = page.url();
              if (BFS_ERROR_PATH.test(new URL(afterReauthUrl).pathname) || BFS_LOGIN_PATH.test(new URL(afterReauthUrl).pathname)) {
                logger.warn({ url }, '[crawler] Still redirecting after re-auth — skipping this URL');
                continue;
              }
            } else {
              logger.warn({ url }, '[crawler] Re-auth failed — stopping BFS (session unrecoverable)');
              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
                jobId: opts.jobId,
                type: 'AUTH_FAILURE',
                payload: { url, landed: landedUrl, reason: 'Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.' },
              }).catch(() => { });
              // Checkpoint remaining queue before aborting so job-queue can auto-requeue
              await opts.onCheckpoint?.(
                Array.from(visited.entries()).filter(([_k, v]) => v !== '_pending_').map(([k]) => k),
                queue.filter((u: string) => !visited.has(canonicalize(u)))
              ).catch(() => {});
              throw new Error('[AUTH_FAILURE] Re-authentication failed. Session is unrecoverable. Re-capture the session in project settings and retry the crawl.');
            }
          } else {
            // Discovery mode: skip auth-required URLs and continue discovering other links.
            // Throwing here aborts the entire BFS — in discovery mode we want to continue
            // visiting other queued URLs (e.g. sitemap seeds) that may not require auth.
            if (opts.discoveryMode) {
              logger.info({ url, landed: landedUrl }, '[discover] skipping auth-required URL — no credentials, continuing BFS');
              continue;
            }
            logger.warn('[crawler] Session expired mid-crawl, no credentials to recover — skipping URL');
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
              jobId: opts.jobId,
              type: 'AUTH_FAILURE',
              payload: { url, landed: landedUrl, reason: 'Session expired and no credentials are configured. Add credentials in project settings and re-capture the session.' },
            }).catch(() => { });
            // Checkpoint remaining queue before aborting so job-queue can auto-requeue
            await opts.onCheckpoint?.(
              Array.from(visited.entries()).filter(([_k, v]) => v !== '_pending_').map(([k]) => k),
              queue.filter((u: string) => !visited.has(canonicalize(u)))
            ).catch(() => {});
            throw new Error('[AUTH_FAILURE] Session expired and no credentials are configured. Add credentials in project settings and re-capture the session.');
          }
        }
        // If weWantedThisPage (intentional /login, /session-expired, /404 navigation) —
        // fall through and save it as a real screen. Error pages are legitimate test targets.
        if (!redirectedToError && landedUrl !== url) {
          logger.info({ requested: url, landed: landedUrl }, '[crawler] navigated with redirect');
        }
        // Successful non-login page visit: reset the consecutive login-wall counter
        if (!redirectedToError) loginWallRedirectCount = 0;

        // Discovery mode fast-path: record URL + extract DOM links only; skip screenshots/DB/AI.
        // IMPORTANT: DOM hash dedup is intentionally DISABLED in discovery mode — SPAs use identical
        // page shells for every route, so hashing would skip all pages after the first one.
        if (opts.discoveryMode) {
          visited.set(canonicalize(url), url); // key=canonical for dedup; value=original URL returned to caller

          // ── SPA hydration wait ───────────────────────────────────────────────────
          // 'domcontentloaded' fires before JS executes, so shadow DOM doesn't exist yet.
          // Wait for network idle to ensure JS bundles are downloaded + executed,
          // then add a brief buffer for component rendering (setTimeout/rAF-deferred work).
          await (rawPage ?? page).waitForLoadState('networkidle', { timeout: 10_000 }).catch(() => { });
          await page.waitForTimeout(500).catch(() => { });

          // ── Link extraction — three approaches, merged ───────────────────────────
          // 1. Playwright native locator (auto-pierces shadow DOM in Playwright 1.20+)
          //    Works universally for React, Vue, Angular, Polymer, Lit, Stencil, FAST.
          // 2. BFS shadow DOM traversal via evaluate() — catches any gaps from approach 1.
          // 3. SPA route manifests (Next.js __NEXT_DATA__, React Router registries).
          const rawLinksSet = new Set<string>();
          const addLink = (h: string) => {
            if (!h || h === '#' || h.startsWith('javascript:') || h.startsWith('mailto:') || h.startsWith('tel:')) return;
            try {
              const href = new URL(h, origin).toString();
              if (isUsefulNavigationUrl(href, origin)) rawLinksSet.add(href);
            } catch {
              // ignore malformed links
            }
          };

          // 1️⃣ Playwright locator — shadow-DOM-piercing (primary approach)
          const pwPage = rawPage ?? page;
          await pwPage.locator('a[href]').evaluateAll((anchors: Element[]) =>
            anchors.map((a: any) => a.href || a.getAttribute('href') || '')
          ).then((hrefs: string[]) => hrefs.forEach(addLink)).catch(() => { });

          await pwPage.locator('[data-href],[data-url],[data-link]').evaluateAll((els: Element[]) =>
            els.map((el: any) => el.dataset?.href || el.dataset?.url || el.dataset?.link || '')
          ).then((hrefs: string[]) => hrefs.forEach(addLink)).catch(() => { });

          await pwPage.locator('[href]:not(a):not(link):not(base):not(script):not(style)').evaluateAll((els: Element[]) =>
            els.map((el: any) => el.getAttribute('href') || '')
          ).then((hrefs: string[]) => hrefs.forEach(addLink)).catch(() => { });

          const playwrightCount = rawLinksSet.size;
          console.log(`[discover] playwright-locator found ${playwrightCount} links for ${url}`);

          // 2️⃣ BFS shadow DOM evaluate — fallback / complement for stubborn SPAs
          //    Runs regardless so it can catch elements the locator misses.
          const evalPage = rawPage ?? page;
          await evalPage.evaluate((o: string) => {
            try {
              const hrefs: string[] = [];
              const add = (h: string) => {
                if (h && h !== '#' && !h.startsWith('javascript:') && !h.startsWith('mailto:') && !h.startsWith('tel:')) {
                  if (h.startsWith(o) || h.startsWith('/')) hrefs.push(h);
                }
              };
              const queue: any[] = [document];
              const seenRoots = new WeakSet<any>();
              seenRoots.add(document);
              let shadowRootsFound = 0;
              while (queue.length > 0) {
                const root: any = queue.shift();
                if (!root) continue;
                try {
                  root.querySelectorAll('a[href]').forEach((a: any) => add(a.href || a.getAttribute('href') || ''));
                  root.querySelectorAll('[data-href],[data-url],[data-link]').forEach((el: any) => {
                    add((el.dataset && (el.dataset.href || el.dataset.url || el.dataset.link)) || '');
                  });
                  root.querySelectorAll('[href]:not(a):not(link):not(base):not(script):not(style)').forEach((el: any) => {
                    const h = el.getAttribute('href') || '';
                    if (h && !h.startsWith('#')) add(h);
                  });
                  root.querySelectorAll('*').forEach((el: any) => {
                    try {
                      if (el.shadowRoot && !seenRoots.has(el.shadowRoot)) {
                        seenRoots.add(el.shadowRoot);
                        queue.push(el.shadowRoot);
                        shadowRootsFound++;
                      }
                    } catch (_) { /* closed shadow root — skip */ }
                  });
                } catch (_) { /* element threw — skip */ }
              }
              return { hrefs, shadowRootsFound };
            } catch { return { hrefs: [] as string[], shadowRootsFound: 0 }; }
          }, origin).then((result: any) => {
            (result?.hrefs ?? []).forEach(addLink);
            if (result?.shadowRootsFound > 0) {
              console.log(`[discover] bfs found ${result.shadowRootsFound} shadow roots, added ${rawLinksSet.size - playwrightCount} extra links for ${url}`);
            }
          }).catch(() => { });

          // 3️⃣ SPA route manifests
          await evalPage.evaluate(() => {
            const routes: string[] = [];
            try {
              const nd = (window as any).__NEXT_DATA__;
              if (nd?.page) routes.push(nd.page);
            } catch (_) { /* non-fatal */ }
            try {
              const w = window as any;
              const rr = w.__reactRouterRoutes || (w.__reactRouterManifest?.routes);
              if (rr) {
                const collect = (r: any) => {
                  if (r?.path && r.path !== '*') routes.push(r.path.startsWith('/') ? r.path : '/' + r.path);
                  if (r?.children) r.children.forEach(collect);
                };
                (Array.isArray(rr) ? rr : Object.values(rr)).forEach(collect);
              }
            } catch (_) { /* non-fatal */ }
            return routes;
          }).then((routes: any[]) => {
            (routes ?? []).forEach(p => {
              try { addLink(new URL(p, origin).href); } catch { /* ignore */ }
            });
          }).catch(() => { });

          const rawLinks = Array.from(rawLinksSet);
          console.log(`[discover] ${url} → rawLinks=${rawLinks.length} origin=${origin} pageUrl=${page.url()}`);
          if (rawLinks.length > 0) console.log(`[discover] sample links: ${rawLinks.slice(0, 5).join(' | ')}`);
          let newlyQueued = 0;
          for (const link of rawLinks) {
            try {
              const resolved = link.startsWith('/') ? `${origin}${link}` : link;
              const u = new URL(resolved);
              // Keep hash-router fragments (#/route, #!/route) — strip plain anchors (#section)
              if (!/^#!?\//.test(u.hash)) u.hash = '';
              const clean = u.toString();
              if (!isUsefulNavigationUrl(clean, origin)) continue;
              // Dedup: use canonical form to avoid crawling 1000 product detail pages separately
              const norm = canonicalize(clean);
              if (!visited.has(norm) && !queued.has(norm)) { queued.add(norm); queue.push(clean); newlyQueued++; }
            } catch { /* ignore malformed */ }
          }
          if (newlyQueued > 0) console.log(`[discover] queued ${newlyQueued} new URLs, queue size=${queue.length}`);
          const pct = 10 + Math.round((visited.size / Math.max(1, opts.maxScreens ?? 1000)) * 80);
          await onProgress?.(Math.min(pct, 89), `Discovered ${visited.size} URLs…`);
          // Stream partial results to caller so they can be persisted/shown incrementally
          if (opts.onUrlsDiscovered) {
            await opts.onUrlsDiscovered(Array.from(visited.values())).catch(() => { });
          }
          continue;
        }

        // Normal crawl mode: DOM hash dedup — detect duplicate-structure pages (SPA 404s, template clones).
        // Skip dedup when hash is empty (computeDomHash failed) — two failed-hash pages would
        // incorrectly match each other and the second would be silently dropped.
        const preDomHash = await computeDomHash(page);
        if (preDomHash !== '' && seenDomHashes.has(preDomHash)) {
          console.log(`[crawler] skip ${url} — duplicate DOM structure (same content as prior page)`);
          continue;
        }
        const pageTitle = await page.title().catch(() => '');
        if (/\b(not found|404|page not found|this page could not be found)\b/i.test(pageTitle)) {
          console.log(`[crawler] skip ${url} — error page detected (title: "${pageTitle}")`);
          seenDomHashes.add(preDomHash); // register so future 404 pages also hit hash-dedup
          continue;
        }
        // Soft-404: SPA catch-all routes return 200 but render identical homepage content.
        // Skip any non-homepage URL whose title + interactive element count matches the homepage.
        if (homeFingerprintElemCount >= 0 && url !== startUrl && pageTitle === homeFingerprintTitle && pageTitle !== '') {
          const pElemCount = await page.evaluate(() =>
            document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
          ).catch(() => -1);
          if (pElemCount >= 0 && pElemCount === homeFingerprintElemCount) {
            console.log(`[crawler] skip ${url} — soft-404 (homepage fingerprint match, title="${pageTitle}" elems=${pElemCount})`);
            seenDomHashes.add(preDomHash);
            continue;
          }
        }
        // Do NOT add real pages to seenDomHashes — server-rendered sites (WooCommerce, WordPress)
        // use the same template for many legitimate pages; registering the hash would skip them all.

        // Collect Navigation Timing + Paint Timing immediately after DOM stable.
        // LCP/CLS come from the buffered PerformanceObservers registered in the
        // init script above (the synchronous timeline never holds LCP entries).
        const rawPerf = await page.evaluate(() => {
          const nav = performance.getEntriesByType('navigation')[0] as PerformanceNavigationTiming | undefined;
          const paints = performance.getEntriesByType('paint');
          // Prefer pre-registered observer value (window.__zetaLCP) over buffered entries.
          // Buffered entries are often empty if the observer wasn't registered before navigation.
          const lcpFromObserver: number | null = (window as any).__zetaLCP ?? null;
          const lcpEntries = performance.getEntriesByType('largest-contentful-paint');
          const lcpFromEntries = lcpEntries.length > 0
            ? Math.round((lcpEntries[lcpEntries.length - 1] as any).startTime)
            : undefined;
          // @ts-ignore — stashed by the CLS init script
          // @ts-ignore — stashed by the web-vitals init script
          const stashedLcp: number | null = (window as any).__zetaLCP ?? null;
          // @ts-ignore
          const stashedCls: number | undefined = (window as any).__zetaCLS;
          return {
            ttfb: nav ? Math.round(nav.responseStart - nav.fetchStart) : undefined,
            fcp: paints.find(p => p.name === 'first-contentful-paint')?.startTime
              ? Math.round(paints.find(p => p.name === 'first-contentful-paint')!.startTime)
              : undefined,
            lcp: lcpFromObserver != null ? Math.round(lcpFromObserver) : lcpFromEntries,
            cls: typeof stashedCls === 'number' ? Math.round(stashedCls * 1000) / 1000 : undefined,
          };
        }).catch(() => null);

        // Accessibility audit via axe-core (WCAG 2.0 A/AA + best-practice)
        const a11yResult = await (async () => {
          try {
            const _require = createRequire(import.meta.url);
            const axePath = _require.resolve('axe-core');
            // page.evaluate() with script content bypasses CSP (DevTools Runtime.evaluate).
            // page.addScriptTag({ path }) is blocked by strict-CSP on many production sites.
            const axeContent = readFileSync(axePath, 'utf8');
            await page.evaluate(axeContent);
            const result = await page.evaluate(async () => {
              return (window as any).axe.run(document, {
                runOnly: { type: 'tag', values: ['wcag2a', 'wcag2aa', 'best-practice'] },
                resultTypes: ['violations'],
              });
            });
            // Full violation objects for DB storage (Phase 4.4)
            const violations = result.violations.map((v: any) => ({
              impact: v.impact as string,
              rule: v.id as string,
              description: v.description as string,
              // First offending node's outer HTML (truncated later in saveA11yViolations)
              nodeHtml: (v.nodes?.[0]?.html as string | undefined) ?? undefined,
              wcagTags: (v.tags as string[]).filter((t: string) => t.startsWith('wcag') || t.startsWith('best-')),
            }));
            const criticalCount = violations.filter((v: any) => v.impact === 'critical').length;
            return { violations, violationCount: violations.length, criticalCount };
          } catch (err) {
            logger.warn({ err: String((err as any)?.message ?? err) }, '[crawler] axe-core a11y audit failed (non-fatal)');
            return null;
          }
        })();
        const a11yScore = a11yResult
          ? Math.max(0, 100 - (a11yResult.criticalCount * 20) - ((a11yResult.violationCount - a11yResult.criticalCount) * 5))
          : null;


        // Screen name strategy (SPA-aware):
        //   1. Title with page-specific segment (e.g. "Dashboard | ZeTA")
        //   2. aria-current="page" active nav item (reliable for SPAs)
        //   3. Page-specific headings that are NOT the brand name
        //   4. URL path segments (skipping cuid/UUIDs/numerics)
        const name = await (async () => {
          let brandName = '';

          // Compute reliable URL path-based name — skip cuid/UUID/numeric segments
          let pathName = 'Home';
          try {
            const u = new URL(url);
            const hostname = u.hostname.replace(/^www\./, '').split('.')[0]!.toLowerCase();
            const allSegs = u.pathname.split('/').filter(Boolean);
            const IS_ID = (s: string) =>
              /^\d+$/.test(s) ||                   // numeric
              /^[0-9a-f]{8,}$/i.test(s) ||         // hex id
              /^[a-z]{1,2}[0-9a-z]{20,}$/i.test(s); // cuid/cuid2 (e.g. cm...)

            const readable = allSegs
              .filter((s) => !IS_ID(s))
              .map((s) => s.charAt(0).toUpperCase() + s.slice(1).replace(/[-_]/g, ' '));

            if (readable.length > 0) {
              const last = readable[readable.length - 1]!;
              const GENERIC = ['edit', 'new', 'create', 'view', 'show', 'index', 'detail', 'details'];
              if (GENERIC.includes(last.toLowerCase()) && readable.length > 1) {
                pathName = `${readable[readable.length - 2]} · ${last}`;
              } else {
                pathName = last;
              }
            } else if (allSegs.length > 0) {
              // All segments were IDs — use the last non-trivial one
              const lastSeg = allSegs[allSegs.length - 1]!;
              pathName = lastSeg.slice(0, 8).toUpperCase();
            }

            // Append significant query params
            const SIG_PARAMS = ['product_cat', 'category', 'cat', 'type', 'tag', 'brand', 'filter', 'tab', 'section', 'view', 'play', 'mode', 'id', 'page'];
            const paramSuffix = SIG_PARAMS
              .map((p) => { const v = u.searchParams.get(p); return v ? ` · ${v.replace(/[-_]/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase())}` : ''; })
              .find(Boolean) ?? '';
            pathName = pathName + paramSuffix;

            // Detect brand name from hostname for later comparisons
            brandName = hostname;
          } catch { /* fall through */ }

          // 1. Document title — use only if it has a page-specific segment
          try {
            const title = await page.title();
            if (title && title.length > 0 && title.length < 120) {
              const parts = title.split(/\s*[|—\-–·]\s*/);
              // Store brand candidate (usually last segment)
              if (parts.length > 1) brandName = (parts[parts.length - 1] ?? brandName).trim().toLowerCase();

              if (parts.length > 1) {
                // "Dashboard | ZeTA" → "Dashboard"
                const candidate = parts[0].trim();
                if (candidate.length > 2 && !/^\d[\d\s]*$/.test(candidate)) return candidate;
              }
              // Single-segment title — only use if multi-word, not brand-name-like, and not the generic SPA app title
              const candidate = parts[0].trim();
              const isBrand = candidate.split(/\s+/).length === 1 &&
                (candidate.toLowerCase().includes(brandName) || candidate.toLowerCase() === brandName);
              const isSiteGenericTitle = homeFingerprintTitle !== '' && candidate === homeFingerprintTitle;
              if (!isBrand && !isSiteGenericTitle && candidate.split(/\s+/).length > 1 && !/^\d[\d\s]*$/.test(candidate)) {
                return candidate;
              }
            }
          } catch { /* fall through */ }

          // 2. Active nav item — SPAs set aria-current="page" on the active route link
          try {
            const activeNav = await page.locator('[aria-current="page"]').first().textContent({ timeout: tuning.defaultTimeoutMs }).catch(() => null);
            if (activeNav) {
              const t = activeNav.trim();
              if (t.length > 1 && t.length < 60 && t.toLowerCase() !== brandName) return t;
            }
          } catch { /* fall through */ }

          // 3. Page-specific heading — skip if it's just the brand name
          try {
            const selectors = [
              'main h1', '[role="main"] h1', '.page-title', '.page-header h1', '.content h1',
              'main h2', '[role="main"] h2', '.page-header h2',
            ];
            for (const sel of selectors) {
              const h = await page.locator(sel).first().textContent({ timeout: tuning.defaultTimeoutMs }).catch(() => null);
              if (!h) continue;
              const t = h.trim();
              if (t.length < 2 || t.length > 80) continue;
              // Skip if it's the brand name (exact or contains hostname)
              if (t.toLowerCase() === brandName) continue;
              if (brandName && t.toLowerCase().replace(/\s/g, '').includes(brandName.replace(/\s/g, ''))) continue;
              return t;
            }
          } catch { /* fall through */ }

          // Safety: if the only candidate is the brand/hostname, fall back to URL path.
          try {
            const hostname = new URL(url).hostname.replace(/^www\./, '').split('.')[0]!.toLowerCase();
            if (pathName !== 'Home' || name.toLowerCase() === hostname || name.toLowerCase().replace(/\s/g, '') === hostname) {
              // name is still brand-name-equivalent — use URL-derived pathName
              if (name.toLowerCase() === hostname || name.toLowerCase().replace(/\s/g, '') === hostname) {
                return pathName;
              }
            }
          } catch { /* ignore */ }

          return pathName;
        })();
        const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, url, opts.jobId, discoveryMethodFor(url));
        visited.set(canonicalize(url), screen.id);

        const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${screen.id}-desktop-v${screen.version}.png`);

        // Wait for skeleton loaders / aria-busy spinners to resolve before screenshotting and analysis.
        // Stagehand V3 proxy does NOT support page.waitForFunction — use page.evaluate polling instead.
        await (async () => {
          const deadline = Date.now() + 4_000;
          while (Date.now() < deadline) {
            const ready = await page.evaluate(() => {
              const busy = document.querySelectorAll('[aria-busy="true"], [data-loading="true"]');
              const skeletons = document.querySelectorAll('.skeleton, [class*="skeleton"], [class*="Skeleton"]');
              return busy.length === 0 && skeletons.length === 0;
            }).catch(() => true); // if evaluate fails, assume ready
            if (ready) break;
            await new Promise(r => setTimeout(r, 200));
          }
        })();

        // Fire-and-forget per-page analyses — non-blocking, failures silently swallowed.
        // screen is now declared; page still has current page state and is fully hydrated.
        void (async () => {
          const screenId = screen.id;
          const [seoR, contrastR, flagsR, langR, formsR, a11yManualR] = await Promise.allSettled([
            checkSeo(page, url),
            auditColorContrast(page, url),
            detectFeatureFlags(page, url),
            detectLanguage(page, url),
            detectForms(page),
            auditAccessibility(page, url),
          ]);
          const base = { screenId, jobId: opts.jobId };
          if (seoR.status === 'fulfilled')
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'SEO_CHECK', payload: seoR.value }).catch(() => { });
          if (contrastR.status === 'fulfilled' && (contrastR.value as any).issues?.length)
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'COLOR_CONTRAST', payload: contrastR.value }).catch(() => { });
          if (flagsR.status === 'fulfilled' && (flagsR.value as any).flags?.length)
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'FEATURE_FLAGS', payload: flagsR.value }).catch(() => { });
          if (langR.status === 'fulfilled')
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'LANGUAGE_DETECTION', payload: langR.value }).catch(() => { });
          if (formsR.status === 'fulfilled' && Array.isArray(formsR.value) && formsR.value.length)
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'FORMS_DETECTED', payload: { forms: formsR.value } }).catch(() => { });
          if (a11yManualR.status === 'fulfilled' && (a11yManualR.value as any).violations?.length)
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, { ...base, type: 'A11Y_MANUAL_AUDIT', payload: a11yManualR.value }).catch(() => { });
        })();

        // Freeze all CSS animations/transitions so screenshots are stable across runs
        try { await page.addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; animation-delay: 0s !important; transition-duration: 0s !important; transition-delay: 0s !important; animation-play-state: paused !important; animation-fill-mode: both !important; }' }); } catch { /* Stagehand proxy may not expose addStyleTag */ }
        // Freeze JS-driven animations (requestAnimationFrame loops, setInterval animations)
        await page.evaluate(() => {
          try {
            const noop = () => 0;
            Object.defineProperty(window, 'requestAnimationFrame', { value: noop, configurable: true });
            Object.defineProperty(window, 'cancelAnimationFrame', { value: noop, configurable: true });
          } catch { /* property may not be configurable */ }
        }).catch(() => { });

        // Replay-first: check for a cached link-discovery recipe before calling AI Stagehand
        const canonUrl = canonicalize(url);
        const linkRecipe = await AppBrain.getActionRecipe(
          opts.tenantId, opts.projectId, new URL(url).hostname, `link-discovery:${canonUrl}`
        ).catch(() => null);
        const cachedLinks: string[] | null = (linkRecipe && (linkRecipe as any).successRate >= 0.7)
          ? (((linkRecipe as any).steps as any[])
            .map((s: any) => s.href)
            .filter((href: string | undefined): href is string => !!href && isUsefulNavigationUrl(href, origin)))
          : null;
        if (cachedLinks) {
          logger.info({ url, cachedCount: cachedLinks.length }, '[crawler] link-discovery: using cached recipe — AI will still run and merge');
        }

        // Emit progress now — navigation done, page stable.
        // Use the same visited-count formula as the post-capture block so progress
        // advances on EVERY URL visit (not just on successful screen saves).
        {
          const effMax = isFinite(maxScreens) ? Math.min(visited.size + queue.length, maxScreens * 3) : visited.size + queue.length;
          maxSeenTotal = Math.max(maxSeenTotal, effMax);
          const rawPrePct = 20 + Math.round(Math.min(57, (visited.size / Math.max(1, maxSeenTotal)) * 57));
          const preCapturePct = Math.max(lastEmittedPct, rawPrePct);
          lastEmittedPct = preCapturePct;
          await onProgress?.(preCapturePct, `Crawling ${visited.size} of ~${visited.size + queue.length} · ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);
        }

        await waitForScreenshotReady(page, rawPage ?? undefined);

        // P3a: Async content detection — count AFTER networkidle+lazy-load so lazy-fetched content
        // is included. Previously captured before waitForScreenshotReady which caused AI skip logic
        // to see skeleton element counts instead of real populated counts.
        const preDomCount = await page.evaluate(() =>
          document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
        ).catch(() => 0);
        await onProgress?.(lastEmittedPct, `Capturing ${(() => { try { return new URL(url).pathname || '/'; } catch { return '/'; } })()}`);

        // FM-4: capture device pixel ratio for coordinate scaling in element highlighting
        const pageDpr: number = await (rawPage ?? page).evaluate(() => window.devicePixelRatio ?? 1).catch(() => 1);

        // Extract elements first so we can highlight them in the screenshot
        const elements = await extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch((err: any) => {
          console.error('[crawler] extractElements failed:', err);
          return [] as Awaited<ReturnType<typeof extractElements>>;
        });

        // Inject bounding boxes for the screenshot
        await page.evaluate((elems: any[]) => {
          const container = document.createElement('div');
          container.id = '__zeta_element_highlights';
          container.style.position = 'absolute';
          container.style.top = '0';
          container.style.left = '0';
          container.style.width = '100%';
          container.style.height = '100%';
          container.style.pointerEvents = 'none';
          container.style.zIndex = '2147483647';
          
          elems.forEach((el, index) => {
            if (!el.boundingRect) return;
            const rect = el.boundingRect;
            const box = document.createElement('div');
            box.style.position = 'absolute';
            box.style.left = (rect.x + window.scrollX) + 'px';
            box.style.top = (rect.y + window.scrollY) + 'px';
            box.style.width = rect.width + 'px';
            box.style.height = rect.height + 'px';
            box.style.border = '2px solid rgba(255, 0, 0, 0.8)';
            box.style.backgroundColor = 'rgba(255, 0, 0, 0.1)';
            box.style.boxSizing = 'border-box';
            
            const label = document.createElement('div');
            label.textContent = el.role || 'element';
            label.style.position = 'absolute';
            label.style.top = '-16px';
            label.style.left = '-2px';
            label.style.backgroundColor = 'rgba(255, 0, 0, 0.9)';
            label.style.color = '#FFF';
            label.style.fontSize = '10px';
            label.style.padding = '2px 4px';
            label.style.borderRadius = '2px';
            label.style.whiteSpace = 'nowrap';
            label.style.fontFamily = 'monospace';
            box.appendChild(label);
            
            container.appendChild(box);
          });
          document.body.appendChild(container);
        }, elements).catch(() => {});

        // Parallelize: screenshot + link discovery + AT capture
        const [, nextLinks, atElements] = await Promise.all([
          page.screenshot({ path: shotPath, fullPage: true })
            .then(async () => {
              const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
              return (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', cloudUrl, pageDpr);
            })
            .catch((err: any) => console.error('[crawler] page.screenshot failed:', err)),
          // Always run AI link discovery (even when cache exists) — cache is merged in, not used as a replacement.
          // Skipping AI when cache exists was a bug: stale cache from broken crawls permanently blocked discovery.
          // Note: preDomCount < aiMinElements guard removed — it was backwards: sparse pages (SPAs) need AI most.
          discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined)
            .then((freshLinks) => {
              if (aiSkipCount > 0) aiSkipCount--;
              aiErrorStreak = 0;
              const merged = (cachedLinks ? Array.from(new Set([...cachedLinks, ...freshLinks])) : freshLinks)
                .filter((href) => isUsefulNavigationUrl(href, origin));
              logger.info({ url, domLinks: freshLinks.length, cached: cachedLinks?.length ?? 0, merged: merged.length }, '[crawler] link-discovery: result');
              // Always persist fresh recipe so stale cache gets overwritten with new AI discoveries
              if (merged.length > 0) {
                AppBrain.saveActionRecipe(opts.tenantId, opts.projectId, {
                  domain: new URL(url).hostname,
                  purpose: `link-discovery:${canonUrl}`,
                  fingerprint: sha256(canonUrl),
                  steps: merged.map((href) => ({ action: 'link', href })),
                  successRate: 0.8,
                }).catch(() => { });
              }
              return merged;
            })
            .catch((err: any) => {
              const fallback: string[] = err?.domLinks ?? [];
              if (err?.isRateLimit) {
                aiErrorStreak++;
                if (aiErrorStreak >= 5) { aiSkipCount = 3; aiErrorStreak = 0; logger.warn('[crawler] 5 consecutive AI rate limits — pausing AI link discovery for next 3 screens then retrying'); }
                if (!aiRateLimitWarned) {
                  aiRateLimitWarned = true;
                  onProgress?.(20, 'AI quota exhausted — falling back to DOM-only link discovery').catch(() => { });
                }
              }
              return fallback;
            }),
          readScreen(rawPage ?? page).catch(() => [] as Awaited<ReturnType<typeof readScreen>>),
        ]);

        // Cleanup highlights
        await page.evaluate(() => {
          const container = document.getElementById('__zeta_element_highlights');
          if (container) container.remove();
        }).catch(() => {});
        logger.info({ url, screenId: screen.id, version: screen.version, elements: elements.length, links: nextLinks.length, atSample: atElements.slice(0, 3).map((a: any) => a.label) }, '[crawler] screen captured');
        console.log('[crawler] AT:', JSON.stringify(atElements.slice(0, 5)));
        // Fire-and-forget keyboard-nav probe — records Tab sequence and focus-indicator issues per screen
        probeKeyboardNav(page).then((kbResult) => {
          if (kbResult.focusableCount > 0) {
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
              screenId: screen.id, jobId: opts.jobId, type: 'KEYBOARD_NAV', payload: kbResult,
            }).catch(() => { });
          }
        }).catch(() => { });

        // Per-screen probes: split read-only (awaited, safe on main page) vs mutating (isolated page).
        // READ-ONLY probes are awaited before BFS navigates — DOM must reflect current screen.
        // MUTATING probes (dropdowns/CRUD/filters/advanced) open a fresh page to isolate state changes.
        {
          // Resolve browser context from rawPage (actual Playwright page, not Stagehand proxy).
          const safeContext: import('playwright').BrowserContext | null =
            typeof (rawPage as any)?.context === 'function'
              ? (rawPage as any).context()
              : typeof (page as any)?.context === 'function'
              ? (page as any).context()
              : playwrightBrowser?.contexts()[0] ?? null;

          // --- Read-only probes: awaited with 25s timeout, never mutate page state ---
          const roTimeout = new Promise<PromiseSettledResult<never>[]>((resolve) =>
            setTimeout(() => resolve([
              { status: 'rejected' as const, reason: 'probe-timeout' },
              { status: 'rejected' as const, reason: 'probe-timeout' },
              { status: 'rejected' as const, reason: 'probe-timeout' },
              { status: 'rejected' as const, reason: 'probe-timeout' },
              { status: 'rejected' as const, reason: 'probe-timeout' },
            ]), 25_000)
          );
          const [nestedCtxRes, shadowRes, a11yRes, perfRes, swRes] = await Promise.race([
            Promise.allSettled([
              crawlNestedContexts(rawPage ?? page, safeContext, url),
              crawlShadowDom(rawPage ?? page),
              crawlAccessibilityTree(rawPage ?? page),
              collectPerfMetrics(rawPage ?? page, url),
              inspectServiceWorker(rawPage ?? page),
            ]),
            roTimeout,
          ]);

          const roPayload: Record<string, unknown> = { screenId: screen.id, url };
          if (nestedCtxRes.status === 'fulfilled' && nestedCtxRes.value && (nestedCtxRes.value as any).totalContextsFound > 0)
            roPayload.nestedContexts = nestedCtxRes.value;
          if (shadowRes.status === 'fulfilled' && shadowRes.value && (shadowRes.value as any).shadowHostCount > 0)
            roPayload.shadowDom = shadowRes.value;
          if (a11yRes.status === 'fulfilled' && a11yRes.value)
            roPayload.accessibilityTree = a11yRes.value;
          if (perfRes.status === 'fulfilled' && (perfRes.value as any).requestCount > 0)
            roPayload.performance = perfRes.value;
          if (swRes.status === 'fulfilled' && (swRes.value as any).supported)
            roPayload.serviceWorker = swRes.value;
          if (Object.keys(roPayload).length > 2) {
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
              jobId: opts.jobId, type: 'PER_SCREEN_PROBES', payload: roPayload,
            }).catch(() => {});
          }

          // --- Mutating probes (interactive): isolated page, fire-and-forget, only when interactive ---
          if (screen?.id && opts.interactive !== false && safeContext) {
            void (async () => {
              let probePage: import('playwright').Page | null = null;
              try {
                // Cap concurrent probe pages: if too many already open, skip to prevent resource exhaustion.
                const openProbePages = safeContext.pages().length;
                if (openProbePages > 8) {
                  logger.warn({ url, openProbePages }, '[crawler] probe page cap reached — skipping mutating probes for this screen');
                  return;
                }
                probePage = await safeContext.newPage();
                await probePage.goto(url, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => {});
                await probePage.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => {});
                await waitForStableDOM(probePage as any, 4_000, 400).catch(() => {});

                const muTimeout = new Promise<PromiseSettledResult<never>[]>((resolve) =>
                  setTimeout(() => resolve([
                    { status: 'rejected' as const, reason: 'probe-timeout' },
                    { status: 'rejected' as const, reason: 'probe-timeout' },
                    { status: 'rejected' as const, reason: 'probe-timeout' },
                    { status: 'rejected' as const, reason: 'probe-timeout' },
                  ]), 60_000)
                );
                const [dropEnumRes, crudRes, filterRes, advancedRes] = await Promise.race([
                  Promise.allSettled([
                    enumerateDropdowns(probePage),
                    probeCrudFlows(probePage, name),
                    crawlFiltersAndPagination(probePage),
                    probeAdvancedUI(probePage),
                  ]),
                  muTimeout,
                ]);

                const muPayload: Record<string, unknown> = { screenId: screen.id, url };
                if (dropEnumRes.status === 'fulfilled' && (dropEnumRes.value as any).totalOptionsFound > 0)
                  muPayload.dropdowns = dropEnumRes.value;
                if (crudRes.status === 'fulfilled' && ((crudRes.value as any).flows?.length > 0 || (crudRes.value as any).toastsObserved?.length > 0))
                  muPayload.crudFlows = crudRes.value;
                if (filterRes.status === 'fulfilled' && (filterRes.value as any).filters?.length > 0)
                  muPayload.filtersAndPagination = filterRes.value;
                if (advancedRes.status === 'fulfilled' && advancedRes.value &&
                    ((advancedRes.value as any).datePickers?.length > 0 || (advancedRes.value as any).sliders?.length > 0 ||
                     (advancedRes.value as any).typeaheads?.length > 0 || (advancedRes.value as any).dragDrop?.library !== 'none'))
                  muPayload.advancedUI = advancedRes.value;
                if (Object.keys(muPayload).length > 2) {
                  AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
                    jobId: opts.jobId, type: 'PER_SCREEN_PROBES_MUTATING', payload: muPayload,
                  }).catch(() => {});
                }
              } catch {
                // non-fatal — probe failures must not block BFS
              } finally {
                await probePage?.close().catch(() => {});
              }
            })();
          }
        }

        // Inline auth-wall detection: PHP/legacy apps render session-expired content
        // without changing the URL, bypassing all URL-path-based guards above.
        const INLINE_AUTH_WALL_RE = /your login session (was|has been) expired|session (has )?expired[^.]*click|please (log ?in|sign ?in) to continue|you (are|were) logged out|your session has timed out/i;
        const pageBodyText = await page.evaluate(() => (document as any).body?.innerText ?? '').catch(() => '');
        if (INLINE_AUTH_WALL_RE.test(pageBodyText)) {
          logger.warn({ url, screenId: screen.id }, '[crawler] Inline session-expired content detected — skipping DOM snapshot, marking requiresAuth');
          AppBrain.updateScreenRequiresAuth(opts.tenantId, screen.id, true).catch(() => {});
          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
            screenId: screen.id, jobId: opts.jobId,
            type: 'AUTH_FAILURE',
            payload: { url, reason: 'Inline session-expired page rendered without URL redirect. Re-authenticate and re-crawl.' },
          }).catch(() => {});
        } else {

        // Capture DOM snapshot for structural analysis, and extract Markdown/text
        // content from the same DOM (crawl4AI-parity text output) — no extra fetch.
        try {
          const domContent = await capturePageHtml(page);
          if (!domContent) {
            logger.warn({ url, screenId: screen.id }, '[crawler] markdown extraction skipped: DOM snapshot unavailable');
          } else {
            const domPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
            await fs.writeFile(domPath, domContent, 'utf8');
            // Upload DOM snapshot to cloud alongside PNG, then persist both paths
            (async () => {
              const domKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(domPath)}`;
              const domCloudUrl = await uploadToCloud(domPath, domKey).catch(() => undefined);
              AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, domPath, domCloudUrl).catch(() => {});
            })();

            // Supplement markdown with iFrame and Shadow DOM content
            const [iframeContent, shadowContent] = await Promise.all([
              extractIframeContent(page).catch(() => []),
              pierceShadowDom(page).catch(() => ''),
            ]);
            const supplemental = [
              ...iframeContent.map((f: any) => f.markdown).filter(Boolean),
              shadowContent,
            ].join('\n\n').trim();

            const extracted = extractMarkdown(
              supplemental ? domContent + '\n<!-- iframe/shadow -->\n' + supplemental.slice(0, 3_000) : domContent,
              url
            );
            let fitMarkdown: string | undefined;
            if (extracted) {
              const mdPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.md');
              fitMarkdown = buildFitMarkdown(extracted.markdown, [
                ...domainProfile.importantScreenPatterns,
                ...domainProfile.workflowGoals,
                ...(opts.bestFirstKeywords ?? []),
              ]);
              const { text: scrubbed, scrubbed: hadPii } = scrubPii(extracted.markdown);
              if (hadPii) logger.info({ url }, '[crawler] PII detected and redacted from markdown');
              const dedupResult = deduplicator.check(url, scrubbed);
              if (dedupResult.isDuplicate) {
                logger.info({ url, similarUrl: dedupResult.similarUrl, score: dedupResult.similarityScore }, '[crawler] near-duplicate page skipped');
              }
              await fs.writeFile(mdPath, scrubbed, 'utf8');

              // Generate a one-sentence AI summary from the markdown — fire-and-forget
              const summaryPromise = (async () => {
                try {
                  const llmCfg = await getCachedCrawlLLM(opts.tenantId);
                  if (!llmCfg) return undefined;
                  const snippet = extracted.markdown.slice(0, 1200);
                  // Skip LLM when content is too thin — output would be useless
                  if (snippet.trim().length < 80) return undefined;
                  const summarySeed = deterministicSeed(`${opts.tenantId}:${opts.projectId ?? ''}:page-summary:${url}`);
                  const { text, usage } = await askTracked(
                    'You are a QA analyst. Output ONLY a single complete sentence (no preamble, no quotes) describing what this screen does or shows. Be specific. Stop after the first sentence.',
                    `Screen: "${name}"\nURL: ${url}\n\nContent:\n${snippet}`,
                    llmCfg,
                    200,
                    undefined,
                    { temperature: 0, seed: summarySeed },
                  );
                  if (usage) {
                    recordLLMCall({
                      tenantId: opts.tenantId,
                      projectId: opts.projectId,
                      jobId: opts.jobId,
                      agentType: 'crawl_summary',
                      label: `page_summary:${name}`,
                      usage,
                    }).catch(() => { });
                  }
                  // Keep only the first sentence if the LLM returned multiple
                  const clean = text.trim().replace(/^["']|["']$/g, '');
                  const firstSentence = clean.match(/^[^.!?]+[.!?]/)?.[0] ?? clean;
                  return firstSentence.trim();
                } catch { return undefined; }
              })();

              AppBrain.updateScreenMarkdown(opts.tenantId, opts.projectId, screen.id, {
                markdownPath: mdPath,
                markdownMethod: extracted.method,
                markdownWordCount: extracted.wordCount,
                markdown: fitMarkdown ?? extracted.markdown,
                heading: extracted.title || name,
              }).catch((err: any) => console.warn('[crawler] markdown extraction save failed:', err));

              pendingSummaryWrites.push(
                summaryPromise.then((aiSummary) => {
                  if (aiSummary) AppBrain.updateScreenAiSummary(opts.tenantId, screen.id, aiSummary).catch((e: any) => {
                    logger.warn({ screenId: screen.id, err: String(e?.message ?? e) }, '[crawler] aiSummary DB save failed');
                  });
                }).catch((e: any) => {
                  logger.warn({ screenId: screen.id, err: String(e?.message ?? e) }, '[crawler] aiSummary generation failed');
                })
              );
            }

            const structuredJson = summarizeStructuredElements(atElements);
            const artifact = buildCrawlArtifactEnvelope({
              tenantId: opts.tenantId,
              projectId: opts.projectId,
              jobId: opts.jobId,
              url,
              canonicalUrl: canonicalize(url),
              screenId: screen.id,
              screenshotPath: shotPath,
              domPath,
              domContent,
              markdown: extracted?.markdown,
              fitMarkdown,
              accessibilityElements: atElements,
              structuredJson,
              evidence: [
                { type: 'domain_profile', payload: { industry: domainProfile.industry, confidence: domainProfile.confidence } },
                { type: 'engine_route', payload: routeDecision },
                { type: 'screen_fingerprints', payload: { urlCanonicalHash: sha256(canonicalize(url)) } },
              ],
            });
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
              screenId: screen.id,
              jobId: opts.jobId,
              type: 'SCREEN_ARTIFACT_ENVELOPE',
              payload: artifact,
            }).catch((err: any) => console.warn('[crawler] artifact evidence save failed:', err?.message ?? err));
          }
        } catch (err: any) {
          logger.warn({ url, screenId: screen.id, err: String(err?.message ?? err) }, '[crawler] markdown extraction failed');
        }
        } // end: inline auth-wall else

        // Persist a11y score + individual violations — fire-and-forget, non-fatal
        if (a11yScore !== null) {
          AppBrain.updateScreenA11y(opts.tenantId, screen.id, { a11yScore })
            .catch((err: any) => console.warn('[crawler] a11y score update failed:', err));
        }
        if (a11yResult !== null && a11yResult.violations.length > 0) {
          AppBrain.saveA11yViolations(opts.tenantId, opts.projectId, screen.id, a11yResult.violations, opts.jobId)
            .catch((err: any) => console.warn('[crawler] a11y violations save failed:', err));
        }

        // DOM hash was registered before saveScreen — no action needed here.

        // Build a role→selectorHint lookup from AT data so fingerprints can include tag hints
        const atByRole = new Map<string, string>();
        for (const at of atElements) {
          if (!atByRole.has(at.role)) atByRole.set(at.role, at.selectorHint);
        }

        // Auto-create DESKTOP visual baseline on first crawl (no-op on recrawls)
        AppBrain.autoCreateVisualBaseline(opts.tenantId, opts.projectId, screen.id, 'DESKTOP', shotPath, gitInfo.branch, gitInfo.commit || undefined)
          .catch(() => { });

        // Phase 2: compute element hash for stale-test detection
        const newElementHash = computeElementHash(elements);
        AppBrain.updateElementHash(opts.tenantId, screen.id, newElementHash, (screen as any).elementHash ?? null)
          .catch((err: any) => console.warn('[crawler] element hash update failed:', err));

        // Phase 3: perceptual screenshot diff — non-blocking
        if (screen.version > 1) {
          (async () => {
            try {
              const prevShot = await AppBrain.getPreviousScreenshotPath(opts.tenantId, screen.id, screen.version - 1);
              if (prevShot) {
                const { screenshotDiffPercent } = await import('./phash.js');
                const { pHashDistance, changeLevel } = await screenshotDiffPercent(prevShot, shotPath);
                await AppBrain.saveScreenDiff(opts.tenantId, opts.projectId, screen.id, screen.version - 1, screen.version, pHashDistance, changeLevel);
              }
            } catch (err) {
              console.warn('[crawler] pHash diff skipped:', err);
            }
          })();
        }

        // VisualDiff pipeline — compare against approved baseline (not previous screenshot)
        if (screen.version > 1) {
          (async () => {
            try {
              const baseline = await AppBrain.getVisualBaselineForDiff(opts.tenantId, screen.id, 'DESKTOP', gitInfo.branch);
              if (baseline && baseline.storagePath !== shotPath) {
                // Resolve baseline file path — local copy may have been rotated; fall back to cloud download
                let baselinePath: string = baseline.storagePath;
                let tempBaselinePath: string | null = null;
                const localExists = await fs.access(baseline.storagePath).then(() => true, () => false);
                if (!localExists) {
                  if (baseline.cloudUrl) {
                    try {
                      const resp = await fetch(baseline.cloudUrl);
                      if (resp.ok) {
                        const buf = Buffer.from(await resp.arrayBuffer());
                        tempBaselinePath = `${shotPath}.baseline-tmp.png`;
                        await fs.writeFile(tempBaselinePath, buf);
                        baselinePath = tempBaselinePath;
                        logger.info({ screenId: screen.id }, '[crawler] VisualDiff baseline downloaded from cloud storage');
                      } else {
                        logger.warn({ screenId: screen.id, status: resp.status }, '[crawler] VisualDiff skipped — baseline cloud fetch returned non-200');
                        return;
                      }
                    } catch (dlErr) {
                      logger.warn({ screenId: screen.id, err: String(dlErr) }, '[crawler] VisualDiff skipped — baseline cloud download failed');
                      return;
                    }
                  } else {
                    logger.warn({ screenId: screen.id }, '[crawler] VisualDiff skipped — baseline not on disk and no cloudUrl available');
                    return;
                  }
                }
                try {
                const { screenshotDiffPercent, gridLayoutDiff } = await import('./phash.js');
                const [{ pHashDistance, changeLevel }, layoutChanges, elementShifts] = await Promise.all([
                  screenshotDiffPercent(baselinePath, shotPath, baseline.ignoredRegions),
                  gridLayoutDiff(baselinePath, shotPath, baseline.ignoredRegions),
                  AppBrain.computeElementLayoutDiff(opts.tenantId, screen.id, 'DESKTOP'),
                ]);
                let verdict = pHashVerdict(pHashDistance, changeLevel, [
                  ...layoutChanges,
                  ...elementShifts.map((s) => ({ location: 'element', severity: s.severity, description: s.description })),
                ]);

                // pHash is a cheap, always-on filter; only spend an LLM call on the
                // screens it flags as worth a closer look. It also can't tell
                // meaningful UI changes apart from anti-aliasing/font-rendering noise,
                // which is exactly what Visual AI comparison exists to filter out.
                if (verdict.changeType === 'SIGNIFICANT' || verdict.changeType === 'CRITICAL') {
                  try {
                    const regions = baseline.ignoredRegions ?? [];
                    // Same mask the pHash passes already applied — otherwise a
                    // manually-masked region (timestamp, ad banner, live counter)
                    // still reaches the AI verdict unmasked and can get flagged.
                    const [baselineBuffer, newBuffer] = regions.length > 0
                      ? await (async () => {
                        const { applyMask } = await import('./phash.js');
                        return Promise.all([applyMask(baselinePath, regions), applyMask(shotPath, regions)]);
                      })()
                      : await Promise.all([
                        fs.readFile(baselinePath),
                        fs.readFile(shotPath),
                      ]);
                    const aiResult = await aiVisualCompare({
                      baselineImageBase64: baselineBuffer.toString('base64'),
                      newImageBase64: newBuffer.toString('base64'),
                      screenName: screen.name,
                      screenPurpose: 'Application screen captured during automated crawl',
                      tenantId: opts.tenantId,
                      projectId: opts.projectId,
                    });
                    verdict = mergeAiVisualVerdict(verdict, aiResult, screen.name);
                  } catch (aiErr) {
                    logger.warn({ screenId: screen.id, err: String(aiErr) }, '[crawler] AI visual compare failed, keeping pHash-only verdict');
                  }
                }

                let { changeType, recommendation, changes: allChanges, bugReportDraft, aiSummary } = verdict;

                // R7: Apply per-project visual diff threshold — suppress diffs below the threshold
                if (changeType !== 'NONE') {
                  try {
                    const project = await AppBrain.getProject(opts.tenantId, opts.projectId);
                    const threshold = (project as any)?.visualDiffThreshold ?? null;
                    if (threshold !== null && pHashDistance < threshold) {
                      changeType = 'NONE';
                    }
                  } catch { /* non-fatal — use verdict as-is */ }
                }

                if (changeType !== 'NONE') {
                  let newScreenshotCloudUrl: string | undefined;
                  let diffImageCloudUrl: string | undefined;
                  try {
                    const { getTenantStorageConfig, uploadScreenshotToTenantStorage, buildArtifactKey } = await import('@detiq/app-brain');
                    const storageCfg = await getTenantStorageConfig(opts.tenantId).catch(() => null);
                    if (storageCfg) {
                      const key = buildArtifactKey(opts.tenantId, opts.projectId, 'visual-diffs', `${screen.id}-v${screen.version}.png`, storageCfg.prefix);
                      newScreenshotCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, shotPath, key);

                      // R5: Generate and upload pixelmatch diff image (changed pixels in red)
                      try {
                        const { generatePixelmatchDiff } = await import('./phash.js');
                        const diffBuffer = await generatePixelmatchDiff(baselinePath, shotPath, baseline.ignoredRegions ?? []);
                        const diffTmpPath = `${shotPath}.diff.png`;
                        await fs.writeFile(diffTmpPath, diffBuffer);
                        const diffKey = buildArtifactKey(opts.tenantId, opts.projectId, 'visual-diffs', `${screen.id}-v${screen.version}-diff.png`, storageCfg.prefix);
                        diffImageCloudUrl = await uploadScreenshotToTenantStorage(storageCfg, diffTmpPath, diffKey);
                        fs.unlink(diffTmpPath).catch(() => {});
                      } catch (diffErr) {
                        logger.warn({ screenId: screen.id, err: String(diffErr) }, '[crawler] pixelmatch diff generation failed, skipping');
                      }
                    }
                  } catch { /* non-fatal — falls back to local path */ }
                  await AppBrain.saveVisualDiff(opts.tenantId, {
                    baselineId: baseline.id,
                    newScreenshotPath: shotPath,
                    newScreenshotCloudUrl,
                    diffImageCloudUrl,
                    diffScore: pHashDistance,
                    changeType,
                    changes: allChanges,
                    recommendation,
                    bugReportDraft,
                    aiSummary,
                  });
                }
                } finally {
                  if (tempBaselinePath) fs.unlink(tempBaselinePath).catch(() => {});
                }
              }
            } catch (err: any) {
              console.warn('[crawler] VisualDiff pipeline failed:', err);
            }
          })();
        }

        const linkMappings = await extractLinkMappings(page, origin);

        // Auth-wall guard: check if the page drifted to a login/error URL during the async
        // element extraction window (30-90s). If so, the `elements` array contains login-page
        // elements — saving them would corrupt the screen's element catalog.
        const postExtractUrl = (() => { try { return page.url(); } catch { return url; } })();
        const postExtractParsed = (() => { try { return new URL(postExtractUrl); } catch { return new URL(url); } })();
        const driftedToAuthWall = !weWantedThisPage && (
          BFS_LOGIN_PATH.test(postExtractParsed.pathname) || BFS_ERROR_PATH.test(postExtractParsed.pathname)
        );
        if (driftedToAuthWall) {
          logger.warn({ url, postExtractUrl }, '[crawler] Page drifted to auth wall during element extraction — skipping element save to prevent login-page element contamination');
          AppBrain.updateScreenRequiresAuth(opts.tenantId, screen.id, true).catch(() => {});
          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
            screenId: screen.id,
            jobId: opts.jobId,
            type: 'AUTH_FAILURE',
            payload: { url, landed: postExtractUrl, reason: 'Session expired during element extraction. Re-capture session and re-crawl this page.' },
          }).catch(() => {});
        } else {
        if (elements.length === 0) {
          // Extraction returned nothing — keep existing elements rather than wiping them.
          // Causes: page.evaluate failure, contentFallbackElements threw, SPA not yet hydrated.
          logger.warn({ url, screenId: screen.id }, '[crawler] extractElements returned 0 — preserving existing elements to avoid data loss');
          // Queue for re-visit with longer wait — SPA content may not have hydrated yet
          if (!revisitQueue.includes(url)) {
            revisitQueue.push(url);
            logger.info({ url }, '[crawler] 0-element page queued for re-visit with extended hydration wait');
          }
        } else {
        // Clear existing elements before re-saving — prevents duplicates on re-crawl
        await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
        // Batch element saves: 2 concurrent DB writes (4 caused pool saturation at 5 workers)
        for (let _ei = 0; _ei < elements.length; _ei += 2) {
          const _batch = elements.slice(_ei, _ei + 2);
          await Promise.all(_batch.map(async (el: any) => {
            const saved = await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark);
            // Populate ElementFingerprint — fire-and-forget, failures are non-fatal
            const label = el.meaning;
            const role = el.role ?? 'unknown';
            const selectorHint = atByRole.get(role) ?? role;
            const selectors = [
              { type: 'role-label', value: `${role}:${label}`, confidence: 0.9 },
              { type: 'aria-label', value: label, confidence: 0.85 },
              { type: 'tag-hint', value: selectorHint, confidence: 0.6 },
            ];
            await AppBrain.saveElementFingerprint(opts.tenantId, opts.projectId, (saved as any).id, selectors).catch(console.warn);
          }));
        }
        } // end inner else: elements.length > 0
        for (const lm of linkMappings) {
          const safety = assessActionSafety(lm.label, domainProfile);
          if (!safety.allowed) {
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
              screenId: screen.id,
              jobId: opts.jobId,
              type: 'ACTION_BLOCKED_BY_SAFETY_POLICY',
              payload: { label: lm.label, href: lm.href, decision: safety, profile: domainProfile.industry },
            }).catch(() => { });
            continue;
          }
          pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: lm.label, targetUrl: lm.href });
          AppBrain.saveActionRecipe(opts.tenantId, opts.projectId, {
            domain: new URL(url).hostname,
            purpose: `navigate:${lm.label}`,
            fingerprint: sha256(`${canonicalize(url)}:${lm.label}:${canonicalize(lm.href)}`),
            steps: [{ action: 'click', label: lm.label, targetUrl: lm.href, fromUrl: url }],
            successRate: 0.5,
          }).catch(() => { });

          if (!selective) {
            try {
              const targetPath = new URL(lm.href).pathname;
              const canonLmHref = canonicalize(lm.href);
              if (
                !SKIP_PATH.test(targetPath) &&
                isUsefulNavigationUrl(lm.href, origin) &&
                !queued.has(canonLmHref) &&
                !visited.has(canonLmHref) &&
                queued.size < maxScreens * 3
              ) {
                if (robotsCache && !await robotsCache.isAllowed(lm.href)) continue;
                queued.add(canonLmHref);
                queue.push(lm.href, { parentElements: elements.length });
                edges.push({ fromUrl: url, toUrl: lm.href });
                logger.info({ fromUrl: url, newUrl: lm.href, label: lm.label, queueSize: queue.length }, '[crawler] anchor mapping added to queue');
              }
            } catch { /* ignore invalid link mapping */ }
          }
        }
        if (!selective) {
          for (const href of nextLinks) {
            edges.push({ fromUrl: url, toUrl: href });
            // Skip WooCommerce/e-commerce action URLs — these are AJAX/form actions that render
            // the same page template with a compare widget; they are not distinct page routes.
            // They would all get queued (different `id` param = different canonical URL), navigated,
            // found "duplicate DOM", and burn through totalAttemptsThisSegment without any gain.
            try {
              const hrefParams = new URL(href).searchParams;
              const actionVal = hrefParams.get('action') ?? '';
              if (/^yith-|^wc-|^vc_|^elementor/.test(actionVal)) continue;
            } catch { /* invalid URL, let it fall through */ }
            const canonHref = canonicalize(href);
            if (!queued.has(canonHref) && !visited.has(canonHref) && queued.size < maxScreens * 3) {
              // Don't flood the queue with generated ID-like route segments.
              try {
                const hrefPath = new URL(href).pathname;
                if (isGeneratedDynamicPath(hrefPath)) continue;
              } catch { /* ignore invalid URLs */ }
              if (robotsCache && !await robotsCache.isAllowed(href)) continue;
              queued.add(canonHref);
              queue.push(href, { parentElements: elements.length }); // navigate to actual URL, dedup by canonical
              logger.info({ newUrl: href, queueSize: queue.length }, '[crawler] new URL added to queue');
            }
          }
          logger.info({ fromUrl: url, discovered: nextLinks.length, queueAfter: queue.length, visitedSoFar: visited.size }, '[crawler] link processing done');

          // Harvest SPA routes captured via history.pushState/replaceState interception
          try {
            const jsRoutes = await page.evaluate(() => {
              const routes = (window as any).__zetaDiscoveredRoutes ?? [];
              (window as any).__zetaDiscoveredRoutes = [];
              return routes as string[];
            }).catch(() => [] as string[]);
            for (const route of jsRoutes) {
              try {
                const abs = route.startsWith('http') ? route : `${origin}${route.startsWith('/') ? '' : '/'}${route}`;
                const canonRoute = canonicalize(abs);
                if (!queued.has(canonRoute) && !visited.has(canonRoute)) {
                  queued.add(canonRoute);
                  queue.push(abs);
                  logger.info({ route: abs }, '[crawler] history.pushState: discovered SPA route');
                }
              } catch { /* ignore invalid routes */ }
            }
          } catch { /* non-fatal */ }

          // P3a: Post-extraction DOM count — detect async-loaded content.
          // Only check sparse pages (preDomCount < 25) where async widgets are likely.
          // If DOM grew significantly during AI extraction, re-visit later with networkidle.
          const asyncContentCheckMinDom = parseInt(process.env.CRAWLER_ASYNC_CONTENT_MIN_DOM ?? '') || 25;
          if (preDomCount < asyncContentCheckMinDom && !revisitQueue.includes(url)) {
            try {
              const postDomCount = await page.evaluate(() =>
                document.querySelectorAll('button,a,input,select,textarea,[role="button"],[onclick],[aria-label]').length
              ).catch(() => 0);
              const asyncGrowthRatio = parseFloat(process.env.CRAWLER_ASYNC_GROWTH_RATIO ?? '') || 1.3;
              const asyncGrowthFloor = parseInt(process.env.CRAWLER_ASYNC_GROWTH_FLOOR ?? '') || 4;
              if (postDomCount > preDomCount * asyncGrowthRatio + asyncGrowthFloor) {
                revisitQueue.push(url);
                logger.info({ url, preDomCount, postDomCount }, '[P3a] Async content detected — scheduled for re-visit');
              }
            } catch { /* non-fatal */ }
          }

          // P2: Static JS router detection — scan bundle for declared client-side routes.
          // Only run on first 5 pages (bundle is the same across pages; scanning every page is wasteful).
          if (visited.size <= (parseInt(process.env.CRAWLER_SPA_ROUTER_DETECT_LIMIT ?? '') || 5)) {
            try {
              const jsDetectedRoutes = await detectJsRoutes(page, origin);
              for (const route of jsDetectedRoutes) {
                const canonRoute = canonicalize(route);
                if (isUsefulNavigationUrl(route, origin) && !queued.has(canonRoute) && !visited.has(canonRoute) && queued.size < maxScreens * 3) {
                  queued.add(canonRoute);
                  queue.push(route, { parentElements: elements.length });
                  logger.info({ route }, '[P2] JS router: discovered route added to queue');
                }
              }
            } catch { /* non-fatal */ }
          }

          // P4: Drain XHR-discovered page URL candidates (accumulated by network-observer callback).
          // These are relative paths extracted from JSON API response bodies.
          for (const relPath of [...xhrDiscoveredPaths]) {
            xhrDiscoveredPaths.delete(relPath);
            try {
              const abs = `${origin}${relPath}`;
              const canon = canonicalize(abs);
              if (isUsefulNavigationUrl(abs, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 3) {
                queued.add(canon);
                queue.push(abs);
                logger.info({ url: abs }, '[P4] XHR: page URL from JSON response added to queue');
              }
            } catch { /* ignore invalid paths */ }
          }

          // Gap 5: Skyvern URL-discovery bootstrap for pages that defeat DOM link extraction.
          // Triggered per-page when: zero DOM links found + fewer than 5 elements (dead-end / canvas page).
          // Skyvern returns URLs it discovered → added to queue for ZeTa's own BFS crawl (not saved directly).
          // ZeTa does all actual crawling — Skyvern is only a URL-discovery override layer.
          if (!selective && nextLinks.length === 0 && elements.length < (parseInt(process.env.CRAWL_SKYVERN_PAGE_MIN_ELEMENTS ?? '') || 5) && opts.skyvernConfig) {
            try {
              logger.info({ url }, 'Gap 5: thin page detected — asking Skyvern to discover navigation URLs');
              const skyvernNav = await skyvernNavigate(
                url,
                ['Find and return all navigation links, menu items, and page URLs accessible from this screen'],
                opts.skyvernConfig,
              );
              for (const skyvernUrl of skyvernNav.visitedUrls) {
                const canonSkyvernUrl = canonicalize(skyvernUrl);
                if (!queued.has(canonSkyvernUrl) && !visited.has(canonSkyvernUrl) && queued.size < maxScreens * 3) {
                  logger.info({ url: skyvernUrl }, 'Gap 5: Skyvern-discovered URL added to ZeTa crawl queue');
                  queued.add(canonSkyvernUrl);
                  queue.push(skyvernUrl); // ZeTa will crawl this normally
                  // Wire the KAG edge so the Skyvern-discovered page isn't flagged as an orphan
                  pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: 'Skyvern-discovered link', targetUrl: skyvernUrl });
                }
              }
            } catch { /* non-fatal — Skyvern unavailable */ }
          }
        }
        } // end auth-wall guard else

        if (rawPerf) {
          performanceMetrics.push({ url, screenName: name, ...rawPerf });
        }

        const screenApiEndpoints = capturedApiEndpoints.slice(apiObservationStartIndex);
        if (screenApiEndpoints.length > 0) {
          AppBrain.saveApiEndpointObservations(opts.tenantId, opts.projectId, screen.id, screenApiEndpoints)
            .then(() => AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
              screenId: screen.id,
              jobId: opts.jobId,
              type: 'API_ENDPOINT_OBSERVATIONS',
              payload: { endpoints: screenApiEndpoints },
            }))
            .catch((err: any) => console.warn('[crawler] API endpoint observation save failed:', err?.message ?? err));
        }

        // Persist performance snapshot (web vitals + optional Lighthouse) — fire-and-forget
        if (opts.jobId && (rawPerf || process.env.ENABLE_LIGHTHOUSE)) {
          (async () => {
            try {
              const lhScores = await runLighthouse(url);
              await AppBrain.savePerformanceSnapshot(opts.tenantId, screen.id, opts.jobId!, {
                lcp: rawPerf?.lcp,
                fcp: rawPerf?.fcp,
                cls: rawPerf?.cls,
                ttfb: rawPerf?.ttfb,
                viewport: opts.viewport ?? 'DESKTOP',
                ...(lhScores ? {
                  lighthousePerf: lhScores.perf,
                  lighthouseA11y: lhScores.a11y,
                  lighthouseSEO: lhScores.seo,
                  lighthouseBP: lhScores.bp,
                  lighthouseReport: lhScores.report,
                } : {}),
              });
            } catch (perfErr) {
              console.warn('[crawler] savePerformanceSnapshot failed (non-fatal):', perfErr);
            }
          })();
        }

        discovered.push({ url, screenId: screen.id, elements: elements.length, elementFacts: elements, screenshotPath: shotPath, version: screen.version });
        // Progress: 20% baseline + up to 60% across discovered screens.
        // Cap maxSeenTotal at maxScreens * 3 so unbounded link discovery does not
        // collapse the denominator and freeze progress in the 20-22% band.
        const effectiveMax = isFinite(maxScreens) ? Math.min(visited.size + queue.length, maxScreens * 3) : visited.size + queue.length;
        maxSeenTotal = Math.max(maxSeenTotal, effectiveMax);
        const rawPct = 20 + Math.round(Math.min(60, (visited.size / Math.max(1, maxSeenTotal)) * 60));
        const clampedPct = Math.max(lastEmittedPct, rawPct);
        lastEmittedPct = clampedPct;
        await onProgress?.(clampedPct, `Crawling screens (${visited.size} mapped, ${queue.length} pending)`);
        logger.info({ name, elements: elements.length }, 'Screen crawled');
        // Checkpoint every 10 successes so a failed job can be resumed from close to where it stopped
        if (++checkpointSuccessCount % 5 === 0) {
          await opts.onCheckpoint?.(
            Array.from(visited.entries()).filter(([_k, v]) => v !== '_pending_').map(([k]) => k),
            queue.filter(u => !visited.has(canonicalize(u)))
          );
        }
      } catch (err: any) {
        logger.warn({ url, err: err?.message ?? String(err), stack: err?.stack?.split('\n').slice(0, 5).join(' | ') }, 'Screen crawl failed');
        discovered.push({ url, error: err?.message ?? String(err) });
      }
    }

    // Wait for any lite workers still processing URLs from the shared queue
    if (liteWorkerPromises.length > 0) await Promise.all(liteWorkerPromises);

    // Discovery mode: return collected URLs and skip all post-crawl analysis phases.
    // Use visited.values() (original URLs) not .keys() (canonical forms with /{id} placeholders).
    // Filter out sentinel strings (_pending_, _auth_wall_, _soft404_, etc.) that mark
    // in-progress or skipped entries — only keep actual http(s) URLs.
    if (opts.discoveryMode) {
      return {
        discoveredUrls: Array.from(visited.values()).filter((v: string) => v.startsWith('http')),
        screensFound: 0,
        screensErrored: 0,
        screensSkippedToNextSegment: 0,
        isPartial: false,
        discovered: [],
        apiEndpoints: [],
        performanceMetrics: [],
        pendingUrls: [],
        segmentIndex: segIdx,
        hasMoreSegments: false,
        domHashes: Array.from(seenDomHashes),
      };
    }

    // Gap 5: Skyvern post-crawl top-up for canvas/non-DOM apps where total screens < 10.
    // Skyvern discovers URLs → ZeTa's own BFS crawl processes each one normally.
    // Skyvern is ONLY a URL-discovery layer here — it does NOT save screens directly.
    // Use only successfully-saved screens (screenId present), not total discovered including errors.
    const successfulScreenCount = discovered.filter(d => d.screenId).length;
    const skyvernMinScreens = parseInt(process.env.CRAWL_SKYVERN_MIN_SCREENS ?? '') || 10;
    if (!selective && successfulScreenCount < skyvernMinScreens && opts.skyvernConfig) {
      await onProgress?.(78, 'Few screens found — using Skyvern to discover additional navigation URLs');
      const currentUrl = page.url();
      const skyvernNav = await skyvernNavigate(
        currentUrl,
        [
          'Explore the main navigation and visit each top-level section',
          'Click any sidebar or top menu items to discover sub-pages',
        ],
        opts.skyvernConfig,
      );
      const skyvernUrls = skyvernNav.visitedUrls.filter((u) => !visited.has(canonicalize(u)));
      if (skyvernUrls.length > 0) {
        logger.info({ count: skyvernUrls.length }, 'Gap 5: Skyvern found new URLs — processing with ZeTa native crawler');
        // Process each URL with ZeTa's own Playwright-based extraction
        for (const skyvernUrl of skyvernUrls) {
          if (isCrawlTimedOut() || visited.size >= maxScreens) break;
          try {
            await page.goto(skyvernUrl, { waitUntil: 'domcontentloaded', timeout: 20000 });
            await new Promise<void>(r => setTimeout(r, 1500));
            const skyvernName = new URL(skyvernUrl).pathname.split('/').filter(Boolean).pop() ?? 'Screen';
            const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, skyvernName, skyvernUrl, opts.jobId, 'SKYVERN_AI');
            const shotPath = `${opts.screenshotDir ?? '/tmp'}/${screen.id}-v${screen.version}.png`;
            await waitForScreenshotReady(page, rawPage ?? undefined);
            await page.screenshot({ path: shotPath, fullPage: true });
            const skyvernCloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`).catch(() => undefined);
            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', skyvernCloudUrl);
            // GAP 7: DOM snapshot for Skyvern primary URLs
            const skyvernDom = await capturePageHtml(page);
            if (skyvernDom) {
              const skyvernDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
              await fs.writeFile(skyvernDomPath, skyvernDom, 'utf8');
              const skyvernDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(skyvernDomPath)}`;
              const skyvernDomCloud = await uploadToCloud(skyvernDomPath, skyvernDomKey).catch(() => undefined);
              AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, skyvernDomPath, skyvernDomCloud).catch(() => {});
            }
            const [elems, links, linkMappings] = await Promise.all([
              extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined),
              discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined),
              extractLinkMappings(page, origin),
            ]);
            for (const el of elems) {
              await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
            }
            visited.set(canonicalize(skyvernUrl), screen.id);
            discovered.push({ url: skyvernUrl, screenId: screen.id, elements: elems.length, version: screen.version });
            // Wire KAG element->screen transitions for this Skyvern-discovered page too —
            // without this, any page only ever reached via Skyvern always looks orphaned
            // (zero inbound edges) even when a real DOM link exists elsewhere pointing to it.
            for (const lm of linkMappings) {
              pendingTransitions.push({ fromScreenId: screen.id, elementMeaning: lm.label, targetUrl: lm.href });
            }
            // Also enqueue links Skyvern page found for further ZeTa crawling
            for (const href of links) {
              const c = canonicalize(href);
              if (isUsefulNavigationUrl(href, origin) && !queued.has(c) && !visited.has(c)) { queued.add(c); queue.push(href); }
            }
          } catch (err) {
            logger.warn({ url: skyvernUrl, err: String(err) }, 'Gap 5: ZeTa crawl of Skyvern URL failed');
          }
        }
        // Drain any newly queued URLs from Skyvern pages
        while (queue.length > 0 && visited.size < maxScreens && !isCrawlTimedOut()) {
          const href = queue.shift()!;
          const canonHref = canonicalize(href);
          if (visited.has(canonHref)) continue;
          if (!isUsefulNavigationUrl(href, origin)) continue;
          try {
            await page.goto(href, { waitUntil: 'domcontentloaded', timeout: 20000 });
            await new Promise<void>(r => setTimeout(r, 1000));
            const nm = new URL(href).pathname.split('/').filter(Boolean).pop() ?? 'Screen';
            const sc = await AppBrain.saveScreen(opts.tenantId, opts.projectId, nm, href, opts.jobId, 'SKYVERN_AI');
            const sp = `${opts.screenshotDir ?? '/tmp'}/${sc.id}-v${sc.version}.png`;
            await waitForScreenshotReady(page, rawPage ?? undefined);
            await page.screenshot({ path: sp, fullPage: true });
            const skyvernDrainCloudUrl = await uploadToCloud(sp, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(sp)}`).catch(() => undefined);
            await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, sc.id, sp, sc.version, 'DESKTOP', skyvernDrainCloudUrl);
            // GAP 8: DOM snapshot for Skyvern drain-queue URLs
            const skyvernDrainDom = await capturePageHtml(page);
            if (skyvernDrainDom) {
              const skyvernDrainDomPath = sp.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
              await fs.writeFile(skyvernDrainDomPath, skyvernDrainDom, 'utf8');
              const skyvernDrainDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(skyvernDrainDomPath)}`;
              const skyvernDrainDomCloud = await uploadToCloud(skyvernDrainDomPath, skyvernDrainDomKey).catch(() => undefined);
              AppBrain.updateScreenDomSnapshot(opts.tenantId, sc.id, skyvernDrainDomPath, skyvernDrainDomCloud).catch(() => {});
            }
            const [elems2, linkMappings2] = await Promise.all([
              extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined),
              extractLinkMappings(page, origin),
            ]);
            for (const el of elems2) {
              await AppBrain.saveElement(opts.tenantId, opts.projectId, sc.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
            }
            visited.set(canonHref, sc.id);
            discovered.push({ url: href, screenId: sc.id, elements: elems2.length, version: sc.version });
            for (const lm of linkMappings2) {
              pendingTransitions.push({ fromScreenId: sc.id, elementMeaning: lm.label, targetUrl: lm.href });
            }
          } catch { /* non-fatal */ }
        }
      }
    }

    if (pendingSessionRestore) {
      await pendingSessionRestore();
      pendingSessionRestore = null;
    }

    // P1: URL Pattern LLM Completion — post-crawl LLM prediction of undiscovered URLs.
    // Single LLM call predicts missing routes from discovered URL patterns, HEAD-validates each.
    // Saves URL_CANDIDATES evidence so the ActivityDrawer shows what was found.
    try {
      const { runUrlPatternCompletion } = await import('./url-pattern-completer.js');
      const llmCfgForPatterns = await getCachedCrawlLLM(opts.tenantId);
      const urlPatternCandidates = await runUrlPatternCompletion(
        Array.from(visited.keys()),
        opts.appUrl,
        llmCfgForPatterns,
      );
      if (urlPatternCandidates.length > 0) {
        AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
          jobId: opts.jobId,
          type: 'URL_CANDIDATES',
          payload: {
            candidates: urlPatternCandidates,
            hits: urlPatternCandidates.filter(c => c.status === 'hit').length,
            total: urlPatternCandidates.length,
          },
        }).catch(() => { });
        logger.info(
          { hits: urlPatternCandidates.filter(c => c.status === 'hit').length, total: urlPatternCandidates.length },
          '[P1] URL pattern completion saved to evidence',
        );
      }
    } catch (err) {
      const msg = String(err);
      logger.warn({ err: msg }, '[P1] URL pattern completion failed — non-fatal, continuing');
      crawlWarnings.push(`url-completer skipped: ${msg.includes('401') || msg.toLowerCase().includes('api key') || msg.toLowerCase().includes('authentication') ? 'LLM API key invalid or missing' : msg}`);
    }

    // P3a: Async content re-visit — revisit pages where DOM grew significantly during extraction.
    // Uses networkidle wait + longer stability period to capture fully-loaded dashboard content.
    if (revisitQueue.length > 0) {
      await onProgress?.(79, `Re-visiting ${revisitQueue.length} page(s) with async content…`);
      logger.info({ count: revisitQueue.length }, '[P3a] Starting async content re-visit pass');
      for (const revisitUrl of revisitQueue.slice(0, 20)) { // cap at 20 re-visits; also guard global timeout
        if (isCrawlTimedOut()) break;
        const rawScreenId = visited.get(canonicalize(revisitUrl));
        // Auth-wall pages are stored as '_auth_wall_' — not a real screen ID.
        // Navigate with the authenticated main page; it will redirect to the real content.
        const isAuthWallRevisit = rawScreenId === '_auth_wall_';
        if (!rawScreenId) continue;
        try {
          await page.goto(revisitUrl, { waitUntil: 'networkidle', timeout: 25_000 });
          await new Promise<void>(r => setTimeout(r, 2_000));
          await waitForScreenshotReady(page, rawPage ?? undefined);

          // After navigation, check where we actually landed (auth may redirect)
          const landedUrl = (rawPage ?? page).url?.() ?? revisitUrl;
          const landedCanon = canonicalize(landedUrl);
          let existingScreenId = isAuthWallRevisit
            ? (visited.get(landedCanon) ?? null)  // use screen at redirected URL if known
            : rawScreenId;

          // If auth-wall redirect landed on an unvisited page, save it as a new screen
          if (isAuthWallRevisit && (!existingScreenId || existingScreenId.startsWith('_')) && !visited.has(landedCanon)) {
            const pathParts = new URL(landedUrl).pathname.split('/').filter(Boolean);
            const screenName = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';
            const newScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, screenName, landedUrl, opts.jobId, 'LINK_FOLLOW').catch(() => null);
            if (newScreen) {
              visited.set(landedCanon, newScreen.id);
              existingScreenId = newScreen.id;
            }
          }

          if (!existingScreenId || existingScreenId.startsWith('_')) {
            logger.warn({ url: revisitUrl, landed: landedUrl }, '[P3a] Re-visit skipped — could not resolve screen ID');
            continue;
          }

          const shotPath2 = `${opts.screenshotDir ?? '/tmp'}/${existingScreenId}-revisit.png`;
          await page.screenshot({ path: shotPath2, fullPage: true });
          const p3aCloudUrl = await uploadToCloud(shotPath2, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath2)}`).catch(() => undefined);
          await (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, existingScreenId, shotPath2, 1, 'DESKTOP', p3aCloudUrl).catch(() => {});
          // GAP 10: DOM snapshot for P3a auth-wall revisit
          const p3aDom = await capturePageHtml(page);
          if (p3aDom) {
            const p3aDomPath = shotPath2.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
            await fs.writeFile(p3aDomPath, p3aDom, 'utf8');
            const p3aDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(p3aDomPath)}`;
            const p3aDomCloud = await uploadToCloud(p3aDomPath, p3aDomKey).catch(() => undefined);
            AppBrain.updateScreenDomSnapshot(opts.tenantId, existingScreenId, p3aDomPath, p3aDomCloud).catch(() => {});
          }
          // Re-extract elements with the async-loaded content now visible
          const revisitElements = await extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch(() => []);
          if (revisitElements.length > 0) {
            await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, existingScreenId).catch(() => {});
            for (const el of revisitElements) {
              await AppBrain.saveElement(opts.tenantId, opts.projectId, existingScreenId, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
            }
          } else {
            // P3a got 0 elements — check if it's an inline session-expired page.
            const p3aBodyText = await page.evaluate(() => (document as any).body?.innerText ?? '').catch(() => '');
            if (INLINE_AUTH_WALL_RE.test(p3aBodyText)) {
              logger.warn({ url: revisitUrl, screenId: existingScreenId }, '[P3a] Re-visit got 0 elements — inline session-expired content, marking requiresAuth');
              AppBrain.updateScreenRequiresAuth(opts.tenantId, existingScreenId, true).catch(() => {});
            }
          }
          // Harvest any new links revealed by async content
          const revisitLinks = await discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false, rawPage ?? undefined).catch(() => []);
          for (const href of revisitLinks) {
            const canon = canonicalize(href);
            if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 3) {
              queued.add(canon);
              queue.push(href);
            }
          }
          logger.info({ url: revisitUrl, landed: landedUrl, elements: revisitElements.length, newLinks: revisitLinks.length }, '[P3a] Re-visit complete');
        } catch (err) {
          logger.warn({ url: revisitUrl, err: String(err) }, '[P3a] Re-visit failed — non-fatal');
        }
      }
    }

    // P3b: Extended BFS — process any links discovered during P3a revisit pass.
    // Without this, auth-wall pages redirect to real app pages, AI discovers new links,
    // but those links are pushed to queue and NEVER processed (main BFS already exited).
    if (queue.length > 0 && visited.size < maxScreens) {
      const p3bCount = Math.min(queue.length, maxScreens - visited.size);
      logger.info({ newLinks: p3bCount }, '[P3b] Extended BFS — processing links discovered during revisit pass');
      await onProgress?.(82, `Extended crawl: ${p3bCount} newly discovered page(s)…`);
      while (queue.length > 0 && visited.size < maxScreens && visited.size < segmentSize) {
        if (isCrawlTimedOut()) break;
        const href = queue.shift();
        if (!href) break;
        const canonHref = canonicalize(href);
        if (visited.has(canonHref)) continue;
        if (!isUsefulNavigationUrl(href, origin)) continue;
        visited.set(canonHref, '_pending_');
        try {
          await page.goto(href, { waitUntil: 'networkidle', timeout: 25_000 }).catch(() => {});
          await waitForScreenshotReady(page, rawPage ?? undefined);
          const pathParts = new URL(href).pathname.split('/').filter(Boolean);
          const nm = pathParts.map((p: string) => p.charAt(0).toUpperCase() + p.slice(1).replace(/[-_]/g, ' ')).join(' — ') || 'Home';
          const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, nm, href, opts.jobId, 'LINK_FOLLOW');
          visited.set(canonHref, screen.id);
          const sp = `${opts.screenshotDir ?? '/tmp'}/${screen.id}-v${screen.version}.png`;
          await page.screenshot({ path: sp, fullPage: true }).catch(() => {});
          const cloudUrl3 = await uploadToCloud(sp, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(sp)}`).catch(() => '');
          await (AppBrain.saveScreenshot as any)(opts.tenantId, opts.projectId, screen.id, sp, screen.version, 'DESKTOP', cloudUrl3).catch(() => {});
          // GAP 6: DOM snapshot for P3b extended BFS
          const p3bDom = await capturePageHtml(page);
          if (p3bDom) {
            const p3bDomPath = sp.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
            await fs.writeFile(p3bDomPath, p3bDom, 'utf8');
            const p3bDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(p3bDomPath)}`;
            const p3bDomCloud = await uploadToCloud(p3bDomPath, p3bDomKey).catch(() => undefined);
            AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, p3bDomPath, p3bDomCloud).catch(() => {});
          }
          const [p3bElems, p3bLinks] = await Promise.all([
            extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch(() => []),
            discoverLinks(stagehand, origin, page, linkLLM ?? undefined, aiSkipCount > 0, rawPage ?? undefined).catch(() => []),
          ]);
          if (p3bElems.length > 0) {
            await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
            for (let _ei3 = 0; _ei3 < p3bElems.length; _ei3 += 2) {
              const _b3 = p3bElems.slice(_ei3, _ei3 + 2);
              await Promise.all(_b3.map((el: any) =>
                AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => {})
              ));
            }
          }
          for (const link of p3bLinks) {
            const c = canonicalize(link);
            if (isUsefulNavigationUrl(link, origin) && !queued.has(c) && !visited.has(c) && queued.size < maxScreens * 3) {
              queued.add(c); queue.push(link);
            }
          }
          discovered.push({ url: href, screenId: screen.id, elements: p3bElems.length, version: screen.version });
          logger.info({ url: href, elements: p3bElems.length, newLinks: p3bLinks.length }, '[P3b] Extended BFS screen captured');
        } catch (err) {
          logger.warn({ url: href, err: String(err) }, '[P3b] Extended BFS URL failed');
          visited.set(canonHref, '_error_');
        }
      }
    }

    // P6: Gap Pass — LLM gap analysis on discovered screens to find structurally missing pages.
    // Runs after P3a re-visits so the gap analysis has the most complete screen list possible.
    // Confirmed gap URLs (HEAD=200) are visited directly with the existing browser.
    if (visited.size >= (parseInt(process.env.CRAWLER_GAP_PASS_MIN_SCREENS ?? '') || 3)) {
      try {
        await onProgress?.(80, 'Running gap analysis…');
        const llmCfgForGap = await getCachedCrawlLLM(opts.tenantId);
        const discoveredForGap = Array.from(visited.entries()).map(([url, screenId]) => {
          const d = discovered.find(dd => dd.screenId === screenId);
          return { url, name: d ? (new URL(url).pathname.split('/').filter(Boolean).pop() ?? 'screen') : undefined, elements: d?.elements };
        });
        const gapCandidates = await runGapAnalysis(discoveredForGap, opts.appUrl, llmCfgForGap);
        const confirmedGaps = gapCandidates.filter(g => g.confirmedUrl && !visited.has(canonicalize(g.confirmedUrl!)));

        if (confirmedGaps.length > 0) {
          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
            jobId: opts.jobId,
            type: 'URL_CANDIDATES',
            payload: {
              source: 'gap-pass',
              candidates: gapCandidates.map(g => ({ url: g.confirmedUrl ?? g.urlPatterns[0], confidence: g.confidence, reason: g.rationale, status: g.confirmedUrl ? 'hit' : 'miss' })),
              hits: confirmedGaps.length,
              total: gapCandidates.length,
            },
          }).catch(() => { });
          logger.info({ confirmed: confirmedGaps.length, total: gapCandidates.length }, '[P6] Gap pass: visiting confirmed gap URLs');

          // Visit each confirmed gap URL with the existing browser (depth=1, no link traversal)
          const gapLoginPattern = /\/(login|signin|sign-in|auth|log-in)(\/|$|\?)/i;
          for (const gap of confirmedGaps.slice(0, parseInt(process.env.CRAWLER_GAP_PASS_MAX_VISITS ?? '') || 25)) {
            if (isCrawlTimedOut() || visited.size >= maxScreens) break;
            const gapUrl = gap.confirmedUrl!;
            try {
              await page.goto(gapUrl, { waitUntil: 'domcontentloaded', timeout: 20_000 });
              // Skip if session expired — don't save a login-page screenshot as the gap result
              if (gapLoginPattern.test(page.url())) {
                logger.warn({ gapUrl, landed: page.url() }, '[P6] Gap URL redirected to login — skipping');
                continue;
              }
              await new Promise<void>(r => setTimeout(r, 1_500));
              const gapName = gap.category;
              const gapScreen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, gapName, gapUrl, opts.jobId, 'ENTRY_POINT');
              const gapShot = `${opts.screenshotDir ?? '/tmp'}/${gapScreen.id}-v${gapScreen.version}.png`;
              await waitForScreenshotReady(page, rawPage ?? undefined);
              await page.screenshot({ path: gapShot, fullPage: true });
              const gapCloudUrl = await uploadToCloud(gapShot, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(gapShot)}`).catch(() => undefined);
              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, gapScreen.id, gapShot, gapScreen.version, 'DESKTOP', gapCloudUrl);
              // GAP 9: DOM snapshot for P6 gap-pass
              const gapDom = await capturePageHtml(page);
              if (gapDom) {
                const gapDomPath = gapShot.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
                await fs.writeFile(gapDomPath, gapDom, 'utf8');
                const gapDomKey = `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(gapDomPath)}`;
                const gapDomCloud = await uploadToCloud(gapDomPath, gapDomKey).catch(() => undefined);
                AppBrain.updateScreenDomSnapshot(opts.tenantId, gapScreen.id, gapDomPath, gapDomCloud).catch(() => {});
              }
              const [gapElems, gapLinks, gapLinkMaps] = await Promise.all([
                extractElements(stagehand, page, elementLLM ?? undefined, rawPage ?? undefined).catch(() => []),
                discoverLinks(stagehand, origin, page, linkLLM ?? undefined, false, rawPage ?? undefined).catch(() => []),
                extractLinkMappings(page, origin).catch(() => []),
              ]);
              for (const el of gapElems) {
                await AppBrain.saveElement(opts.tenantId, opts.projectId, gapScreen.id, el.meaning, el.role, el.expectedData, el.notes, undefined, true, el.boundingRect, el.ariaState, el.parentLandmark).catch(() => { });
              }
              if (gapElems.length === 0) {
                const gapBodyText = await page.evaluate(() => (document as any).body?.innerText ?? '').catch(() => '');
                if (INLINE_AUTH_WALL_RE.test(gapBodyText)) {
                  logger.warn({ url: gapUrl, screenId: gapScreen.id }, '[P6] Gap URL got 0 elements — inline session-expired content, marking requiresAuth');
                  AppBrain.updateScreenRequiresAuth(opts.tenantId, gapScreen.id, true).catch(() => {});
                }
              }
              visited.set(canonicalize(gapUrl), gapScreen.id);
              discovered.push({ url: gapUrl, screenId: gapScreen.id, elements: gapElems.length, version: gapScreen.version });
              for (const lm of gapLinkMaps) {
                pendingTransitions.push({ fromScreenId: gapScreen.id, elementMeaning: lm.label, targetUrl: lm.href });
              }
              // Enqueue gap page's links for further crawl (within maxScreens budget)
              for (const href of gapLinks) {
                const canon = canonicalize(href);
                if (isUsefulNavigationUrl(href, origin) && !queued.has(canon) && !visited.has(canon) && queued.size < maxScreens * 3) {
                  queued.add(canon); queue.push(href);
                }
              }
              logger.info({ url: gapUrl, elements: gapElems.length, category: gap.category }, '[P6] Gap URL captured');
            } catch (err) {
              logger.warn({ url: gapUrl, err: String(err) }, '[P6] Gap URL visit failed — non-fatal');
            }
          }
        } else {
          logger.info({ total: gapCandidates.length }, '[P6] Gap pass: no confirmed gaps found');
        }
      } catch (err) {
        const msg = String(err);
        logger.warn({ err: msg }, '[P6] Gap pass failed — non-fatal, continuing');
        crawlWarnings.push(`gap-analysis skipped: ${msg.includes('401') || msg.toLowerCase().includes('api key') || msg.toLowerCase().includes('authentication') ? 'LLM API key invalid or missing' : msg}`);
      }
    }

    // After crawl: mark screens not re-discovered as PENDING_REMOVAL (non-destructive diff)
    // Only run if crawl actually found screens — 0 discovered means failed/aborted run, not evidence of removal
    if (!selective && visited.size > 0) {
      // Only pass canonical URLs of screens that were actually visited (saveScreen called).
      // Exclude marker strings (_skipped_api_, _pending_, _soft404_, pre-auth, skipped)
      // so that skipped API/export paths don't pollute the discoveredSet and prevent removal
      // of old API-path screens from previous crawls.
      const VISIT_MARKER_RE = /^_|^pre-auth$|^skipped$/;
      const discoveredUrls = Array.from(visited.entries())
        .filter(([, v]) => typeof v === 'string' && !VISIT_MARKER_RE.test(v))
        .map(([k]) => k);
      await AppBrain.markRemovedScreens(opts.tenantId, opts.projectId, discoveredUrls, opts.jobId);
    }

    // Step 3: Save flows
    await onProgress?.(82, 'Saving app map');
    for (const edge of edges) {
      const fromId = visited.get(canonicalize(edge.fromUrl));
      const toId = visited.get(canonicalize(edge.toUrl));
      if (fromId && toId && fromId !== toId) {
        await AppBrain.saveFlow(opts.tenantId, opts.projectId, `${edge.fromUrl} -> ${edge.toUrl}`, fromId, toId);
      }
    }

    // Step 4: Wire KAG element-level transitions
    // For each <a href> link captured from each screen, resolve the href to a visited screenId
    // and store transitionsToScreenId on the matching element.
    await onProgress?.(84, 'Wiring navigation graph (KAG)');
    // Secondary index: pathname → [canonicalUrl, ...] for subset-param fuzzy matching.
    // A link to /product?id=5 should resolve to a page visited at /product?id=5&session=abc
    // even though their canonical forms differ (session param not in the strip list).
    const visitedByPath = new Map<string, string[]>();
    for (const [canonUrl] of visited) {
      try {
        const p = new URL(canonUrl).pathname;
        const arr = visitedByPath.get(p);
        if (arr) arr.push(canonUrl); else visitedByPath.set(p, [canonUrl]);
      } catch { /* invalid canonical — skip */ }
    }
    for (const pt of pendingTransitions) {
      let toScreenId = visited.get(canonicalize(pt.targetUrl));
      if (!toScreenId) {
        // Fuzzy fallback: find a visited URL at the same pathname whose params are a superset
        // of the link's params (handles extra session/tracking params on the visited side).
        try {
          const targetCanon = new URL(canonicalize(pt.targetUrl));
          const candidates = visitedByPath.get(targetCanon.pathname) ?? [];
          for (const cand of candidates) {
            const candUrl = new URL(cand);
            let allMatch = true;
            for (const [k, v] of targetCanon.searchParams) {
              if (candUrl.searchParams.get(k) !== v) { allMatch = false; break; }
            }
            if (allMatch) { toScreenId = visited.get(cand); break; }
          }
        } catch { /* non-fatal */ }
      }
      if (toScreenId && toScreenId !== pt.fromScreenId) {
        await AppBrain.setElementTransition(opts.tenantId, pt.fromScreenId, pt.elementMeaning, toScreenId).catch(() => { });
        await AppBrain.saveScreenTransition(opts.tenantId, opts.projectId, {
          fromScreenId: pt.fromScreenId,
          toScreenId,
          action: {
            type: 'click',
            label: pt.elementMeaning,
            targetUrl: pt.targetUrl,
            source: 'crawl-link-mapping',
          },
          confidence: 0.8,
          jobId: opts.jobId,
        }).catch(() => { });
      }
    }
    const transitionCount = pendingTransitions.filter(pt => {
      if (visited.get(canonicalize(pt.targetUrl))) return true;
      try {
        const tc = new URL(canonicalize(pt.targetUrl));
        return (visitedByPath.get(tc.pathname) ?? []).some(cand => {
          const cu = new URL(cand);
          for (const [k, v] of tc.searchParams) { if (cu.searchParams.get(k) !== v) return false; }
          return true;
        });
      } catch { return false; }
    }).length;
    logger.info({ transitionCount }, 'KAG: wired element→screen transitions');
    AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
      jobId: opts.jobId,
      type: 'WORKFLOW_GRAPH_SUMMARY',
      payload: {
        transitionCount,
        screenCount: visited.size,
        profile: domainProfile.industry,
      },
    }).catch(() => { });

    if (capturedApiEndpoints.length > 0) {
      logger.info({ count: capturedApiEndpoints.length }, 'Captured unique API endpoint patterns');
    }

    if (visited.size >= 5) {
      await onProgress?.(90, `App map saved · ${visited.size} screens, ${edges.length} flows wired`);
    }

    // P5: Export session cookies BEFORE session health check so we can reuse them even if
    // the main browser session is stale. The cookie jar stays valid for fresh browser contexts
    // even when the CDP-connected Stagehand page shows a redirect to /login.
    let mainStorageState: any;
    try {
      mainStorageState = await playwrightBrowser!.contexts()[0]?.storageState();
    } catch { /* non-fatal */ }

    // P5: Orphan page auth re-verify — confirm session is still alive before tail-end viewport pass.
    // The main BFS can span many minutes; the session may have expired by the time we reach here.
    // If expired, attempt re-auth via Stagehand before proceeding.
    // Creative resilience: even if the main browser is dead, we still run the viewport pass using
    // the exported cookie jar — fresh browsers are often accepted by servers even after CDP session loss.
    let sessionAliveForViewports = true;
    try {
      await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 });
      const sessionCheckUrl = page.url();
      const loginKeywords = /login|signin|sign-in|\/auth[/?#]/i;
      if (loginKeywords.test(sessionCheckUrl)) {
        logger.warn({ landed: sessionCheckUrl }, '[P5] Session expired before viewport pass — attempting re-auth');
        let reauthed = false;
        if (opts.credentials) {
          reauthed = await loginWithStagehand(
            stagehand, opts.credentials, opts.appUrl,
            { apiKey: opts.captchaSolverApiKey, provider: opts.captchaSolverProvider },
            { apiKey: opts.mailslurpApiKey, inboxId: opts.mailslurpInboxId },
            undefined,
            { tenantId: opts.tenantId, projectId: opts.projectId, jobId: opts.jobId },
          ).catch(() => false);
        }
        if (reauthed) {
          // Refresh cookie jar with newly authenticated session
          try { mainStorageState = await playwrightBrowser!.contexts()[0]?.storageState(); } catch { /* keep prior */ }
          logger.info('[P5] Re-auth succeeded — proceeding with viewport pass');
        } else {
          sessionAliveForViewports = false;
          const hasCookies = (mainStorageState?.cookies?.length ?? 0) > 0;
          logger.warn({ landed: sessionCheckUrl, hasCookies }, '[P5] Re-auth failed — will attempt viewport pass with exported cookies');
          if (!hasCookies) {
            AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
              jobId: opts.jobId,
              type: 'AUTH_FAILURE',
              payload: {
                url: opts.appUrl,
                landed: sessionCheckUrl,
                reason: 'Session expired before mobile/tablet viewport pass and re-auth failed. Re-capture the session in project settings.',
              },
            }).catch(() => { });
          }
        }
      } else {
        logger.info({ url: sessionCheckUrl }, '[P5] Session alive — proceeding with viewport pass');
      }
    } catch (err) {
      logger.warn({ err: String(err) }, '[P5] Session health check navigation failed — proceeding with viewport pass anyway');
    }

    // Step 5: Multi-viewport pass — screenshot + DOM element extraction at each device profile.
    // Only DESKTOP (the main crawl above) runs Stagehand AI navigation and link discovery.
    // Mobile/tablet viewports navigate to already-discovered screens and re-extract elements
    // with their viewport-specific bounding rects.
    //
    // IMPORTANT: We launch a fresh Playwright browser per viewport profile instead of calling
    // playwrightBrowser.newContext() on the CDP-connected instance. CDP-connected browsers
    // share the DevTools session with Stagehand and cannot reliably set device emulation
    // (isMobile, hasTouch, viewport, deviceScaleFactor) on new contexts — the flags are
    // silently ignored, so screenshots come out at desktop dimensions regardless of profile.
    const extraProfiles = (opts.deviceProfiles ?? []).filter((p) => p !== 'DESKTOP');
    // Run the viewport pass even when the main session is dead, as long as we exported cookies —
    // fresh browsers often authenticate successfully with the saved cookie jar.
    const hasExportedAuth = (mainStorageState?.cookies?.length ?? 0) > 0;
    if ((sessionAliveForViewports || hasExportedAuth) && extraProfiles.length > 0 && discovered.length > 0) {
      const { devices, chromium: pwChromium } = await import('playwright');
      const DEVICE_MAP: Record<string, any> = {
        // Override viewport heights: Playwright's device descriptors use viewport-minus-browser-chrome
        // (659 for iPhone 15, ~980 for iPad Pro 11). Override to full logical screen dimensions so
        // screenshots match what users see on device and mobile-detection scripts pass.
        MOBILE_PORTRAIT: { ...devices['iPhone 15'], viewport: { width: 393, height: 852 } },
        TABLET: { ...devices['iPad Pro 11'], viewport: { width: 834, height: 1194 } },
      };

      for (const profile of extraProfiles) {
        const deviceConfig = DEVICE_MAP[profile];
        if (!deviceConfig) continue;
        await onProgress?.(95, `Capturing ${profile} screenshots + elements…`);

        // Launch a dedicated browser for this viewport — no CDP sharing
        let mobileBrowser: import('playwright').Browser | null = null;
        try {
          mobileBrowser = await pwChromium.launch({
            headless: true,
            executablePath,
            args: chromeArgs,
          });
          const mobileCtx = await mobileBrowser.newContext({
            ...deviceConfig,
            ignoreHTTPSErrors: true,
            ...(mainStorageState ? { storageState: mainStorageState } : {}),
          });
          mobileCtx.on('download', (dl: any) => { dl.cancel().catch(() => {}); });
          const mobilePage = await mobileCtx.newPage();

          for (const disc of discovered) {
            if (isCrawlTimedOut()) break;
            if (!disc.screenId) continue;
            try {
              await mobilePage.goto(disc.url, { waitUntil: 'domcontentloaded', timeout: 20_000 });
              await mobilePage.waitForLoadState('networkidle', { timeout: 8_000 }).catch(() => { });
              await dismissOverlays(mobilePage);
              try { await mobilePage.addStyleTag({ content: '*, *::before, *::after { animation-duration: 0s !important; transition-duration: 0s !important; }' }); } catch { /* ignore */ }
              // mobilePage is a real Playwright page — pass it as rawPage so CDP wheel events
              // trigger virtual scrollers (react-window / TanStack Virtual) at mobile viewports.
              await infiniteScrollToBottom(mobilePage, { maxScrolls: 20, scrollDelay: 300, stabilizeMs: 400 }, mobilePage).catch(() => { });
              const shotPath = path.join(opts.screenshotDir ?? '/tmp', `${disc.screenId}-${profile.toLowerCase()}-v${disc.version}.png`);
              await waitForScreenshotReady(mobilePage, mobilePage);
              // Screenshot and DOM element extraction run in parallel
              const [, vpElements] = await Promise.all([
                mobilePage.screenshot({ path: shotPath, fullPage: true }),
                domFallbackElements(mobilePage, mobilePage),
              ]);
              const cloudUrl = await uploadToCloud(shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
              await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, disc.screenId, shotPath, disc.version!, profile, cloudUrl);
              AppBrain.autoCreateVisualBaseline(opts.tenantId, opts.projectId, disc.screenId, profile, shotPath, gitInfo.branch, gitInfo.commit || undefined).catch(() => { });
              // Save viewport-specific element records (bounding rects differ per viewport)
              for (const el of vpElements) {
                AppBrain.saveElement(
                  opts.tenantId, opts.projectId, disc.screenId,
                  el.meaning, el.role, el.expectedData, el.notes,
                  undefined, true, el.boundingRect, el.ariaState, el.parentLandmark,
                  profile,
                ).catch(() => { });
              }
            } catch (err) {
              logger.warn({ profile, url: disc.url, err: String(err) }, `[P5] ${profile} screenshot failed — skipping screen`);
              AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
                jobId: opts.jobId,
                screenId: disc.screenId,
                type: 'VIEWPORT_SCREENSHOT_FAILURE',
                payload: { profile, url: disc.url, reason: String(err) },
              }).catch(() => { });
            }
          }
          await mobilePage.close().catch(() => { });
          await mobileCtx.close().catch(() => { });
        } catch (profileErr) {
          logger.warn({ profile, err: String(profileErr) }, `[P5] ${profile} browser launch failed`);
        } finally {
          await mobileBrowser?.close().catch(() => { });
        }
      }
    }

    // Step 6: Interactive probe — always-on (C-06) to capture real validation errors, toasts, and modals.
    // Previously gated on opts.interactive or sparse crawls; now unconditional for accurate expected-result generation.
    const shouldRunInteractiveProbe = true;
    if (shouldRunInteractiveProbe) {
      await onProgress?.(88, 'Running interactive probe (capturing real errors & modals)');
      const probeInputs = discovered
        .filter((d) => d.screenId)
        .map((d) => ({ url: d.url, screenId: d.screenId!, elements: d.elementFacts ?? [] }));
      await probeAllScreens(page, opts.tenantId, opts.projectId, probeInputs, onProgress);
    }

    // Step 7: End-of-crawl state graph coverage report.
    // Per-screen probes (dropdown, CRUD, filter, DnD, nested-contexts, shadow-DOM,
    // accessibility-tree, performance, service-worker) run per-screen inside the BFS
    // loop above and are saved as PER_SCREEN_PROBES evidence.
    try {
      await onProgress?.(90, 'Generating state graph coverage report');
      const stateGraph = new ScreenTransitionGraph();
      const coveragePayload: Record<string, unknown> = {
        stateGraph: stateGraph.getCoverageReport(),
        totalScreensCrawled: discovered.filter(d => !!d.screenId).length,
      };
      AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
        jobId: opts.jobId,
        type: 'CRAWL_COVERAGE_REPORT',
        payload: coveragePayload,
      }).catch(() => { });
    } catch (extErr) {
      logger.warn({ err: String(extErr) }, '[coverage-report] non-fatal error');
    }

    await onProgress?.(97, 'Running post-crawl analysis');

    // ── Per-crawl summaries (fire-and-forget, non-blocking) ────────────────
    const successScreens = discovered.filter(d => !!d.screenId);
    ; (async () => {
      try {
        // 1. Sitemap XML
        if (successScreens.length > 0) {
          const sitemapUrls = screensToSitemapUrls(successScreens.map(d => ({ url: d.url })));
          const sitemapPath = path.join(opts.screenshotDir ?? '/tmp', 'sitemap.xml');
          await writeSitemap(sitemapUrls, sitemapPath);
          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
            jobId: opts.jobId, type: 'SITEMAP_WRITTEN', payload: { path: sitemapPath, urlCount: sitemapUrls.length },
          }).catch(() => { });
        }

        // 2. OpenAPI spec from captured API endpoints
        if (capturedApiEndpoints.length > 0) {
          const spec = generateOpenApiSpec(capturedApiEndpoints, { title: `${opts.projectId} API`, version: '1.0.0' });
          const yamlStr = specToYaml(spec);
          const specPath = path.join(opts.screenshotDir ?? '/tmp', 'openapi.yaml');
          await fs.writeFile(specPath, yamlStr, 'utf8');
          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
            jobId: opts.jobId, type: 'OPENAPI_SPEC', payload: { path: specPath, endpointCount: capturedApiEndpoints.length },
          }).catch(() => { });
        }

        // 3. Dead link summary
        const deadStats = deadLinkTracker.stats();
        if (deadStats.total > 0) {
          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
            jobId: opts.jobId, type: 'DEAD_LINKS', payload: { ...deadStats, links: deadLinkTracker.getDeadLinks() },
          }).catch(() => { });
        }

        // 4. Redirect chain summary
        const redirectStats = redirectRegistry.stats();
        if (redirectStats.withRedirects > 0) {
          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
            jobId: opts.jobId, type: 'REDIRECT_CHAINS', payload: { ...redirectStats, problematic: redirectRegistry.getProblematic() },
          }).catch(() => { });
        }

        // 5. Playwright test generation (first 5 screens as sample)
        if (successScreens.length > 0) {
          const sampleScreens = successScreens.slice(0, 5).map(d => ({
            id: d.screenId!, url: d.url, name: d.url,
          }));
          const playwrightScript = generatePlaywrightTest(sampleScreens);
          const testPath = path.join(opts.screenshotDir ?? '/tmp', 'generated.spec.ts');
          await fs.writeFile(testPath, playwrightScript, 'utf8');
          AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
            jobId: opts.jobId, type: 'PLAYWRIGHT_TEST_GENERATED', payload: { path: testPath, screenCount: sampleScreens.length },
          }).catch(() => { });
        }

        // 6. Webhook notification (if configured on project)
        const webhookUrl = (opts as any).webhookUrl;
        if (webhookUrl) {
          const notifier = new WebhookNotifier({ url: webhookUrl, secret: (opts as any).webhookSecret });
          await notifier.notify('crawl.completed', opts.jobId ?? '', opts.projectId, opts.tenantId, {
            screensFound: successScreens.length,
            apiEndpoints: capturedApiEndpoints.length,
            deadLinks: deadLinkTracker.stats().total,
          }).catch(() => { });
        }
      } catch { /* post-crawl analysis is non-fatal */ }
    })();

    await onProgress?.(98, 'Finalizing');
    // Remaining queue = URLs discovered but not yet crawled due to segment limit
    const wasTimeout = isCrawlTimedOut();
    const pendingUrls = queue.filter((u) => !visited.has(canonicalize(u)));
    // All fully-processed URLs (exclude '_pending_' entries interrupted by timeout/abort)
    const visitedUrls = Array.from(visited.entries())
      .filter(([_k, v]) => v !== '_pending_')
      .map(([k]) => k);
    const hasMoreSegments = pendingUrls.length > 0;
    if (wasTimeout && pendingUrls.length > 0) {
      const elapsedMin = Math.round((Date.now() - crawlLoopStart) / 60_000);
      crawlWarnings.push(`global-timeout: crawl stopped after ${elapsedMin} minutes with ${pendingUrls.length} URLs remaining`);
    }
    const screensSucceeded = discovered.filter(d => !!d.screenId).length;
    const screensErrored = discovered.filter(d => !!d.error && !d.screenId).length;

    // Drain pending AI summary writes before marking the job done — ensures descriptions
    // are visible immediately after crawl completes. Cap wait at 30 s per screen, 60 s total.
    if (pendingSummaryWrites.length > 0) {
      await Promise.race([
        Promise.allSettled(pendingSummaryWrites),
        new Promise<void>((resolve) => setTimeout(resolve, Math.min(60_000, pendingSummaryWrites.length * 3_000))),
      ]);
    }

    // Detect systematic element/link extraction failure across all screens
    if (screensSucceeded >= 3) {
      const screensWithElements = discovered.filter(d => (d.elements ?? 0) > 0 && d.screenId).length;
      if (screensWithElements === 0) {
        crawlWarnings.push(
          `element-extraction: 0 elements found across all ${screensSucceeded} screens — ` +
          `DOM evaluate may be failing (check server logs for "page.evaluate threw" warnings)`,
        );
      }
    }

    return {
      screensFound: screensSucceeded,           // successful captures only (not errors)
      screensErrored,                           // pages visited but failed to capture
      screensSkippedToNextSegment: pendingUrls.length,
      isPartial: hasMoreSegments,               // true = more segments auto-queued
      discovered,
      apiEndpoints: capturedApiEndpoints,
      performanceMetrics,
      pendingUrls,                              // feed into next segment's resumeUrls
      visitedUrls,                              // all fully-processed URLs for skipUrls in continuation
      wasTimeout,                               // true when global timeout triggered the BFS break
      segmentIndex: segIdx,                     // which segment this was
      hasMoreSegments,
      domHashes: Array.from(seenDomHashes),     // carry forward to next segment to avoid re-screenshotting
      crawlWarnings: crawlWarnings.length > 0 ? crawlWarnings : undefined,
    };
  } finally {
    // Log accumulated Stagehand LLM usage (extract/act/observe calls) to cost ledger.
    if (stagehandPromptTokens > 0 || stagehandCompletionTokens > 0) {
      const totalTokens = stagehandPromptTokens + stagehandCompletionTokens;
      const shProvider = stagehandModel.includes('/') ? stagehandModel.split('/')[0] : configuredProvider || 'openai';
      const shModel = stagehandModel.includes('/') ? stagehandModel.split('/').slice(1).join('/') : stagehandModel;
      const shUsage = {
        provider: shProvider,
        model: shModel,
        promptTokens: stagehandPromptTokens,
        completionTokens: stagehandCompletionTokens,
        cacheReadTokens: 0,
        cacheWriteTokens: 0,
        totalTokens,
      };
      recordLLMCall({
        tenantId: opts.tenantId,
        projectId: opts.projectId,
        jobId: opts.jobId,
        agentType: 'crawl_stagehand',
        label: 'stagehand_browser_actions',
        usage: { ...shUsage, estimatedCostUsd: estimateCost(shUsage, `${shProvider}/${shModel}`) },
      }).catch(() => { });
    }
    await playwrightBrowser?.close().catch(() => { });
    await stagehand.close().catch(() => { });
    await chromeProc.close().catch(() => { });
  }
}

// Elements whose meanings indicate UI state toggles or noise — filtered from both DOM and AI results.
const ELEMENT_NOISE = /^(cookie|consent|privacy|accept|decline|dismiss|close|copy|©|v\d|switch to (dark|light)|dark mode|light mode|toggle (dark|light|theme)|theme (toggle|switch)|share|tweet|follow|subscribe to newsletter|skip to|back to top|\s*$)/i;

async function extractElements(stagehand: Stagehand, page: any, llmCfg?: { provider: string; model: string; apiKey?: string; baseUrl?: string } | null, rawPage?: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>> {
  // llmCfg reserved for per-task model selection (future: create task-scoped Stagehand)
  // DOM-first: fast, no LLM cost, covers server-rendered and most SPAs.
  // Threshold >= 2 so a lone dark-mode button doesn't suppress AI fallback for rich pages.
  // rawPage: real Playwright CDP page used for page.evaluate() — avoids Stagehand's proxy
  // re-bundling the function with esbuild (which injects __name, breaking browser eval).
  const domElements = await domFallbackElements(page, rawPage).catch((err: any) => {
    console.warn('[crawler] domFallbackElements failed (page context issue or DOM error):', err?.message ?? err);
    return [] as Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>;
  });

  // FM-8: extract interactive elements from visible same-origin iframes
  const iframeElements: typeof domElements = [];
  try {
    const iframes = page.frames ? page.frames() : [];
    for (const frame of iframes) {
      if (frame === page.mainFrame?.()) continue; // skip main frame
      const iframeUrl: string = await frame.url().catch(() => '');
      if (!iframeUrl || iframeUrl === 'about:blank' || iframeUrl.startsWith('javascript:')) continue;
      const frameEls = await domFallbackElements(frame).catch(() => []);
      // Tag iframe elements with source context
      for (const el of frameEls.slice(0, 30)) {
        iframeElements.push({ ...el, notes: el.notes ? `${el.notes} [in iframe]` : '[in iframe]' });
      }
    }
  } catch { /* iframes may be cross-origin or inaccessible — skip silently */ }

  // FM-7: reveal dynamic elements — click/hover trigger elements that show dropdowns or modals
  // Only run when base DOM found elements (not on blank pages) to avoid spurious interactions.
  let dynamicElements: typeof domElements = [];
  if (domElements.length > 0) {
    try {
      dynamicElements = await page.evaluate(() => {
        const out: { meaning: string; role: string; notes: string }[] = [];
        const seen = new Set<string>();
        // Look for select/combobox/listbox that aren't already captured as interactive elements
        const triggers = document.querySelectorAll<HTMLElement>('[role="combobox"],[role="listbox"] [role="option"],[aria-haspopup]');
        triggers.forEach((el) => {
          const label = (el.getAttribute('aria-label') || el.textContent?.trim() || '').slice(0, 80);
          if (!label || seen.has(label)) return;
          // Don't duplicate what's already captured
          const role = el.getAttribute('role') ?? el.tagName.toLowerCase();
          seen.add(label);
          out.push({ meaning: label, role, notes: '[dynamic/revealed]' });
        });
        return out.slice(0, 30);
      }).catch(() => []);
    } catch { dynamicElements = []; }
  }

  // Always capture content elements (headings, meta, cards, nav, FAQ) regardless of interactive count.
  // Previously this was fallback-only; now both paths run together so static sections are
  // always visible to the test generator even on pages with forms.
  const contentElements = await contentFallbackElements(page).catch((err: any) => {
    console.warn('[crawler] contentFallbackElements failed (page context issue):', err?.message ?? err);
    return [] as Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number> }>;
  });

  // Cross-source text deduplication: strips wrapper phrases like "Button to expand '...' details"
  // and "... FAQ button" so duplicate FAQ buttons from DOM + content sources are collapsed.
  const crossSourceDedup = <T extends { meaning: string; role: string }>(arr: T[]): T[] => {
    const stripPhrases = (s: string) => s
      .replace(/^button to expand ['"]?|['"]?\s*details$/gi, '')
      .replace(/\s+(faq\s*)?button$/i, '')
      .replace(/^button:\s*/i, '')
      .toLowerCase().trim().slice(0, 80);
    const ROLE_PRIORITY: Record<string, number> = { 'faq-question': 10, 'faq-answer': 9, 'h1': 8, 'h2': 7, 'h3': 6, 'h4': 5, 'heading': 8, button: 3, a: 2 };
    const normMap = new Map<string, number>();
    const result: T[] = [];
    for (const el of arr) {
      const norm = stripPhrases(el.meaning);
      if (!norm) { result.push(el); continue; }
      const existingIdx = normMap.get(norm);
      if (existingIdx === undefined) { normMap.set(norm, result.length); result.push(el); }
      else {
        const exPri = ROLE_PRIORITY[result[existingIdx].role] ?? 1;
        const thisPri = ROLE_PRIORITY[el.role] ?? 1;
        if (thisPri > exPri) result[existingIdx] = el;
      }
    }
    return result;
  };

  if (domElements.length >= 2) return crossSourceDedup([...domElements, ...iframeElements, ...dynamicElements, ...contentElements]);

  // Static/marketing page — no interactive widgets; use content elements.
  if (contentElements.length > 0) return crossSourceDedup([...domElements, ...contentElements]);

  // AI fallback: DOM found nothing or only noise — try AI (pure SPA, shadow DOM, canvas-based).
  try {
    const extracted = await Promise.race([
      stagehand.extract(
        'Extract ONLY the meaningful form and action elements on this page — inputs, selects, textareas, and action buttons. ' +
        'For each: a clear human-readable purpose (e.g. "Email address input", "Submit password reset button"), ' +
        'its role (button/input/select/checkbox/textarea), and for inputs the expected data type ' +
        '(valid_email/password/phone/date/search_term/text). ' +
        'SKIP: navigation links, header/footer links, cookie banners, theme toggles (dark mode/light mode), ' +
        'social share buttons, version strings, decorative elements, any button inside a nav or header. ' +
        'Focus ONLY on main content forms and primary action buttons. Return at most 20 elements.',
        ElementSchema as any
      ),
      new Promise<null>((resolve) => setTimeout(() => resolve(null), 30_000)),
    ]);
    if (!extracted) return domElements;
    // Apply noise filter to AI results — AI doesn't always follow skip instructions
    const filtered = (extracted as any).elements.filter((e: any) => !ELEMENT_NOISE.test(e.meaning ?? ''));
    return filtered.slice(0, 40);
  } catch {
    return domElements;
  }
}

async function discoverLinks(stagehand: Stagehand, origin: string, page: any, llmCfg?: { provider: string; model: string; apiKey?: string; baseUrl?: string } | null, skipAi = false, rawPage?: any): Promise<string[]> {
  // Expand collapsed nav menus (hamburger, aria-haspopup, Radix closed triggers) so hidden routes
  // become part of the DOM before querying. This catches SPA nav dropdowns that only render on click.
  const navExpanded = await expandNavMenus(page);

  // DOM-first: <a href> links — standard apps. AI supplements when DOM coverage is < 20 unique links.
  // Threshold of 20 (not 8) because nav sidebars quickly provide 8+ links while missing deep content URLs.
  const domLinks = await domFallbackLinks(page, origin, rawPage);

  // Interact with tabs, carousels, and accordions to reveal hidden content and discover more links.
  const interactiveLinks = await interactAndCollectLinks(page, origin);

  // Dismiss any opened menus so page is clean for screenshot/element extraction after this call.
  if (navExpanded) {
    page.keyboard?.press('Escape').catch(() => { });
    await page.evaluate(() => {
      // Also click any currently-open triggers to close them (toggle pattern)
      document.querySelectorAll<HTMLElement>('[aria-expanded="true"],[data-state="open"],[data-headlessui-state="open"]').forEach(el => {
        try { el.click(); } catch { /* ignore */ }
      });
    }).catch(() => { });
  }

  const allDomLinks = Array.from(new Set([...domLinks, ...interactiveLinks]));

  if (allDomLinks.length >= 50 || skipAi) return allDomLinks;

  // AI complement: DOM found fewer than 50 unique links — SPA likely has JS-router pages not in <a href>.
  // Always merge DOM + AI so we never lose already-discovered links.
  // 12s timeout: allows 1 OpenAI retry (4.8s delay) but bails before the 2nd retry wastes more time.
  try {
    const extracted = await Promise.race([
      stagehand.extract(
        `Find all in-app navigation links for this application (origin: ${origin}). ` +
        'Include: main nav, sidebar, content links, feature pages, settings pages. ' +
        'Exclude: logout, sign out, delete, external sites, anchor-only links (#), mailto:, javascript:.',
        LinksSchema as any
      ),
      new Promise<null>((resolve) => setTimeout(() => resolve(null), 12_000)),
    ]);
    if (!extracted) return allDomLinks;
    const { links } = extracted as any;
    const currentUrl: string = page.url?.() ?? '';
    const aiLinks = links
      .map((href: string) => {
        try { const u = new URL(href, currentUrl); if (!/^#!?\//.test(u.hash)) u.hash = ''; return u.toString(); }
        catch { return null; }
      })
      .filter((u: string | null): u is string => {
        if (!u) return false;
        return isUsefulNavigationUrl(u, origin);
      });
    // Always merge DOM + AI — AI complements, never replaces
    return Array.from(new Set([...allDomLinks, ...aiLinks])).filter((href) => isUsefulNavigationUrl(href, origin));
  } catch (err: any) {
    // Detect OpenAI rate-limit errors — signal caller via thrown special error so circuit breaker can trip
    const is429 = err?.status === 429 || String(err?.message ?? '').includes('429') || String(err?.message ?? '').includes('rate limit');
    if (is429) throw Object.assign(new Error('RATE_LIMITED'), { isRateLimit: true, domLinks: allDomLinks });
    return allDomLinks;
  }
}

export async function domFallbackElements(page: any, rawPage?: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }>> {
  // Named function so we can .toString() it for CDP Runtime.evaluate fallback
  function _browserEval() {
    const out: { meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number>; ariaState?: Record<string, string>; parentLandmark?: string }[] = [];
    const seen = new Set<string>();
    // UI chrome noise: cookie banners, theme toggles, version strings, social/share buttons
    const noise = /^(cookie|consent|privacy|accept|decline|dismiss|close|copy|©|v\d|switch to (dark|light)|dark mode|light mode|toggle (dark|light|theme)|theme (toggle|switch)|share|tweet|follow|subscribe to newsletter|skip to|back to top|\s*$)/i;

    const querySelectorAllDeep = (root: any, selector: string): Element[] => {
      const results: Element[] = [];
      const queue: any[] = [root];
      while (queue.length > 0) {
        const current = queue.shift();
        Array.from(current.querySelectorAll(selector)).forEach((el: any) => results.push(el));
        current.querySelectorAll('*').forEach((el: any) => {
          if (el.shadowRoot) queue.push(el.shadowRoot);
        });
      }
      return results;
    };

    // Expanded selector — all interactive ARIA roles + native form elements + rich editors + bare nav links
    const selector = 'input:not([type=hidden]),select,textarea,button,a[href],[role="button"],[role="checkbox"],[role="radio"],[role="combobox"],[role="slider"],[role="spinbutton"],[role="switch"],[role="tab"],[role="dialog"],[role="alertdialog"],[role="menu"],[role="menuitem"],[role="menuitemcheckbox"],[role="menuitemradio"],[role="listbox"],[role="option"],[role="treeitem"],[role="searchbox"],[role="gridcell"],[role="link"],[contenteditable="true"]';

    querySelectorAllDeep(document, selector).forEach((el: any) => {
      const style = window.getComputedStyle(el);
      // Enhanced actionability checks
      if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0' || style.pointerEvents === 'none') return;
      if (el.hasAttribute('inert') || el.getAttribute('aria-disabled') === 'true') return;

      const rect = el.getBoundingClientRect();
      const hasAccessibleLabel = !!(el.getAttribute('aria-label') || el.getAttribute('aria-labelledby'));
      if (rect.width === 0 && rect.height === 0 && !hasAccessibleLabel) return;

      // Skip elements inside 3rd-party chat widgets, cookie banners, floating overlays — these appear on EVERY page
      if (el.closest('[id*="intercom"],[class*="intercom-"],[id*="crisp"],[class*="crisp-"],[id*="freshchat"],[id*="freshdesk"],[id*="drift"],[class*="drift-"],[id*="hubspot-"],[class*="hs-chat"],[class*="hs-widget"],[id*="tawk"],[class*="tawk-"],[id*="chat-widget"],[class*="chat-widget"],[class*="chatbot"],[id*="zopim"],[id*="ze-snippet"],[class*="cookie-banner"],[class*="cookie-consent"],[class*="gdpr-banner"],[id*="cookie-law"],[class*="cookie-law"],[id*="popup-form"],[class*="popup-form"],[class*="floating-form"],[class*="sticky-cta"],[id*="wm-ipp"],[class*="wm-ipp"]')) return;

      const tag = el.tagName.toLowerCase();
      const ariaRole = el.getAttribute('role');
      const isActionEl = tag === 'button' || tag === 'a' || ariaRole === 'button';

      // Skip plain anchor links inside <footer> only — footer chrome (sitemap, legal links) has no test value.
      // Header nav links ARE test targets (SPA primary navigation lives in <header>) — do NOT skip.
      // Do NOT skip <nav> elements: SPA dashboard sidebars live in <nav> and are primary test targets.
      if (tag === 'a') {
        let p = el.parentElement;
        while (p) {
          if (p.tagName?.toLowerCase() === 'footer') return;
          p = p.parentElement;
        }
      }

      // Label resolution: explicit <label for>, implicit wrapping <label>, aria-label, placeholder
      let labelText = '';
      if (el.id) {
        const linkedLabel = document.querySelector(`label[for="${el.id}"]`);
        if (linkedLabel) labelText = linkedLabel.textContent?.trim() || '';
      }
      // Implicit label: input is a child of <label> (no for= attribute needed)
      if (!labelText) {
        const wrappingLabel = el.closest('label');
        if (wrappingLabel) {
          const clone = wrappingLabel.cloneNode(true) as Element;
          clone.querySelectorAll('input,select,textarea,button').forEach((c: any) => c.remove());
          labelText = clone.textContent?.replace(/\s+/g, ' ').trim() || '';
        }
      }

      const label = (
        el.getAttribute('aria-label') ||
        el.getAttribute('placeholder') ||
        labelText ||
        el.innerText?.replace(/\s+/g, ' ').trim() ||
        el.getAttribute('name') ||
        ''
      ).slice(0, 80);

      if (!label || noise.test(label) || label.length > 70) return;

      const role = tag === 'input' ? (el.type || 'text') : el.hasAttribute('contenteditable') ? 'rich-text' : (ariaRole || tag);
      const key = `${role}:${label.toLowerCase()}`;
      if (seen.has(key)) return;
      seen.add(key);

      let expectedData: string | undefined;
      if (tag === 'input') {
        if (el.type === 'email' || /email/i.test(label)) expectedData = 'valid_email';
        else if (el.type === 'password' || /password/i.test(label)) expectedData = 'password';
        else if (el.type === 'tel' || /phone|mobile/i.test(label)) expectedData = 'phone';
        else if (el.type === 'date' || /\bdate\b/i.test(label)) expectedData = 'date';
        else if (/\b(code|otp|token)\b/i.test(label)) expectedData = 'one_time_code';
        else if (/\b(search|query)\b/i.test(label)) expectedData = 'search_term';
        else if (el.type === 'number' || /\b(amount|quantity|count|age|year)\b/i.test(label)) expectedData = 'number';
        else if (el.type === 'url' || /\b(url|website|link)\b/i.test(label)) expectedData = 'url';
        else if (el.type === 'file') expectedData = 'file';
      }

      // Multi-Attribute Identity
      const attrs: Record<string, string> = {};
      if (el.id) attrs.id = el.id;
      if (el.name) attrs.name = el.name;
      if (el.getAttribute('data-testid')) attrs['data-testid'] = el.getAttribute('data-testid');

      // HTML5 validation attributes — ground test boundary values in real constraints
      if (tag === 'input' || tag === 'textarea') {
        if (el.required) attrs.required = 'true';
        if (el.pattern) attrs.pattern = el.pattern;
        if (el.minLength > 0) attrs.minLength = String(el.minLength);
        if (el.maxLength > 0 && el.maxLength < 524288) attrs.maxLength = String(el.maxLength);
        if (el.min !== '') attrs.min = String(el.min);
        if (el.max !== '') attrs.max = String(el.max);
        if (el.step && el.step !== 'any') attrs.step = String(el.step);
        if (el.accept) attrs.accept = el.accept;
        if (el.multiple) attrs.multiple = 'true';
        if (el.readOnly) attrs.readOnly = 'true';
        if (el.disabled) attrs.disabled = 'true';
        if (el.autocomplete && el.autocomplete !== 'on') attrs.autocomplete = el.autocomplete;
      }
      // CSS class hint — first component-library class for COMPONENT file detection
      const cssClass = (el.className || '')
        .split(' ')
        .map((c: string) => c.trim())
        .find((c: string) => c.length > 2 && !/^(true|false|active|disabled|hidden|visible)$/i.test(c));
      if (cssClass) attrs.cssClass = cssClass;

      // GAP 4: Select option enumeration — test generator knows valid values
      if (tag === 'select' && el.options?.length > 0 && el.options.length <= 60) {
        const opts = Array.from(el.options).map((o: any) => {
          const entry: Record<string, any> = { v: o.value, l: o.text.trim() };
          if (o.selected) entry.s = true;
          return entry;
        }).filter((o: any) => o.l);
        if (opts.length > 0) attrs.selectOptions = JSON.stringify(opts);
      }

      // GAP 6: aria-describedby help text — captures real constraint text (e.g. "min 8 chars")
      const describedById = el.getAttribute('aria-describedby');
      if (describedById) {
        const helpEl = document.getElementById(describedById.split(' ')[0]);
        const helpText = helpEl?.textContent?.trim().slice(0, 200);
        if (helpText) attrs.helpText = helpText;
      }

      // GAP 1: Form grouping — inputs/buttons tagged with their containing form
      const form = el.closest('form');
      if (form) {
        if ((form as any).id) attrs.formId = (form as any).id;
        const formAction = form.getAttribute('action');
        if (formAction && !formAction.startsWith('javascript:')) attrs.formAction = formAction.slice(0, 120);
        const formMethod = (form as any).method;
        if (formMethod) attrs.formMethod = formMethod.toUpperCase();
      }
      const fieldset = el.closest('fieldset');
      if (fieldset) {
        const legendText = fieldset.querySelector('legend')?.textContent?.trim().slice(0, 60);
        if (legendText) attrs.fieldsetGroup = legendText;
      }

      // GAP 7: Button classification — type + destructive intent
      if (tag === 'button' || ariaRole === 'button') {
        if (el.type && el.type !== 'button') attrs.buttonType = el.type; // submit | reset
        if (/\b(delete|remove|destroy|revoke|clear|reset|discard|cancel)\b/i.test(label)) attrs.isDestructive = 'true';
      }

      // ARIA state snapshot — promoted to first-class field (Phase 1)
      const stateMap: Record<string, string> = {};
      const ariaExpanded = el.getAttribute('aria-expanded');
      const ariaChecked = el.getAttribute('aria-checked');
      const ariaPressed = el.getAttribute('aria-pressed');
      const ariaSelected = el.getAttribute('aria-selected');
      const ariaInvalid = el.getAttribute('aria-invalid');
      const ariaCurrent = el.getAttribute('aria-current');
      if (ariaExpanded !== null) stateMap.expanded = ariaExpanded;
      if (ariaChecked !== null) stateMap.checked = ariaChecked;
      if (ariaPressed !== null) stateMap.pressed = ariaPressed;
      if (ariaSelected !== null) stateMap.selected = ariaSelected;
      if (ariaInvalid && ariaInvalid !== 'false') stateMap.invalid = ariaInvalid;
      if (ariaCurrent) stateMap.current = ariaCurrent;
      if (el.getAttribute('aria-required') === 'true' && !attrs.required) stateMap.required = 'true';

      // Parent landmark — promoted to first-class field (Phase 1)
      const getRegion = (node: any): string | undefined => {
        let p = node.parentElement;
        while (p && p !== document.body) {
          const ptag = p.tagName?.toLowerCase();
          const prole = p.getAttribute?.('role');
          if (prole === 'dialog' || prole === 'alertdialog') return 'modal';
          if (prole === 'main' || ptag === 'main') return 'main';
          if (prole === 'complementary' || ptag === 'aside') return 'aside';
          if (prole === 'navigation' || ptag === 'nav') return 'nav';
          if (prole === 'banner' || ptag === 'header') return 'header';
          if (prole === 'contentinfo' || ptag === 'footer') return 'footer';
          p = p.parentElement;
        }
        return undefined;
      };
      const parentLandmark = getRegion(el);

      // GAP 8: Rich text editor detection
      if (el.hasAttribute('contenteditable')) {
        const cls = ((el.className || '') + ' ' + (el.getAttribute('data-type') || '')).toLowerCase();
        const editorType = cls.includes('tiptap') || cls.includes('prosemirror') ? 'tiptap' :
          cls.includes('cm-') || cls.includes('codemirror') ? 'codemirror' :
            (window as any).Quill ? 'quill' :
              typeof (window as any).monaco !== 'undefined' ? 'monaco' : 'contenteditable';
        attrs.editorType = editorType;
      }

      const notes = Object.keys(attrs).length > 0 ? JSON.stringify(attrs) : undefined;
      const boundingRect = {
        x: Math.round(rect.x), y: Math.round(rect.y),
        width: Math.round(rect.width), height: Math.round(rect.height),
        top: Math.round(rect.top), left: Math.round(rect.left),
        right: Math.round(rect.right), bottom: Math.round(rect.bottom),
      };
      out.push({
        meaning: label, role, expectedData, notes,
        boundingRect,
        ariaState: Object.keys(stateMap).length > 0 ? stateMap : undefined,
        parentLandmark: parentLandmark ?? undefined,
      });
    });

    // Form fields first, then buttons
    const priority = (e: typeof out[0]) =>
      ['email', 'password', 'text', 'tel', 'date', 'number', 'select', 'textarea', 'url', 'file'].includes(e.role) ? 0 : 1;
    out.sort((a, b) => priority(a) - priority(b));
    return out; // Cap removed
  }
  // CDP Runtime.evaluate bypasses Playwright's isolated world entirely.
  // The "__name is not defined" error comes from Playwright's own UtilityScript being absent
  // from the isolated world (destroyed by navigation events in Stagehand's CDP session).
  // Solution: when rawPage (a real Playwright Page) is available, use CDP Runtime.evaluate
  // as the PRIMARY path — no isolated world needed, no __name dependency.
  //
  // esbuild injects __name() calls for named arrow-function constants inside _browserEval
  // (e.g. querySelectorAllDeep). When _browserEval.toString() is sent to the browser,
  // the browser throws "ReferenceError: __name is not defined". Fix: prepend a tiny
  // __name polyfill that just returns the function (name-setting is non-critical).
  const _browserEvalStr = `(function(){const __name=(fn,n)=>{try{Object.defineProperty(fn,'name',{value:n,configurable:true})}catch(e){}return fn};return(${_browserEval.toString()})()})();`;

  if (rawPage && typeof rawPage.context === 'function') {
    try {
      const ctx = rawPage.context();
      if (typeof ctx?.newCDPSession === 'function') {
        const cdp = await ctx.newCDPSession(rawPage);
        try {
          const evalResult = await cdp.send('Runtime.evaluate', {
            expression: _browserEvalStr,
            returnByValue: true,
            awaitPromise: false,
          });
          if (evalResult?.exceptionDetails) {
            console.warn('[crawler] domFallbackElements CDP primary error:', evalResult.exceptionDetails?.exception?.description ?? evalResult.exceptionDetails?.text);
          } else if (Array.isArray(evalResult?.result?.value)) {
            return (evalResult.result.value as any[]).map((e: any) => ({ ...e, isInteractive: true }));
          }
        } finally {
          cdp.detach().catch(() => {});
        }
      }
    } catch (cdpPrimaryErr: any) {
      console.warn('[crawler] domFallbackElements CDP primary failed, trying page.evaluate:', cdpPrimaryErr?.message ?? String(cdpPrimaryErr));
    }
  }

  // Fallback: pass the pre-serialized string expression so esbuild's __name() wrapper
  // for nested named functions never reaches the browser (Function.prototype.toString
  // returns the source including __name() calls for any nested named arrow constants).
  // Playwright accepts a JS expression string directly — no isolated-world __name leakage.
  const evalPage = rawPage ?? page;
  const _doEvaluate = (): Promise<any> => evalPage.evaluate(_browserEvalStr as any);
  let result: any[];
  try {
    result = await _doEvaluate();
  } catch (err1: any) {
    // Second attempt after a brief wait — isolated world may need a tick to settle
    await new Promise<void>(r => setTimeout(r, 500));
    try {
      result = await _doEvaluate();
    } catch (err2: any) {
      console.warn('[crawler] domFallbackElements page.evaluate threw (attempt 2):', err2?.message ?? String(err2));
      return [];
    }
  }
  return (result ?? []).map((e: any) => ({ ...e, isInteractive: true }));
}

/** Capture content anatomy for EVERY page — not just static ones.
 *  Headings, meta tags, cards, chips, FAQ items, nav/footer structure, section text.
 *  Previously fallback-only; now always runs alongside domFallbackElements so
 *  the test generator sees ALL page content, not just interactive widgets. */
async function contentFallbackElements(page: any): Promise<Array<{ meaning: string; role: string; expectedData?: string; notes?: string; boundingRect?: Record<string, number> }>> {
  return page.evaluate(() => {
    const out: { meaning: string; role: string; notes?: string; boundingRect?: Record<string, number> }[] = [];
    const seen = new Set<string>();
    const captureRect = (el: Element): Record<string, number> | undefined => {
      try {
        const r = el.getBoundingClientRect();
        if (r.width === 0 && r.height === 0) return undefined;
        return { x: r.x, y: r.y, width: r.width, height: r.height, top: r.top, right: r.right, bottom: r.bottom, left: r.left };
      } catch { return undefined; }
    };
    const addUniq = (item: { meaning: string; role: string; notes?: string }, el?: Element) => {
      const key = `${item.role}:${item.meaning.toLowerCase().slice(0, 60)}`;
      if (!seen.has(key) && item.meaning.trim().length > 1) {
        seen.add(key);
        out.push(el ? { ...item, boundingRect: captureRect(el) } : item);
      }
    };
    const isVisible = (el: Element) => {
      const s = window.getComputedStyle(el);
      return s.display !== 'none' && s.visibility !== 'hidden' && s.opacity !== '0' && (el as HTMLElement).offsetParent !== null;
    };

    // 1. Page-level meta (title, description, og:, canonical)
    const title = document.title?.trim();
    if (title) addUniq({ meaning: title, role: 'page-title', notes: JSON.stringify({ tag: 'title' }) });
    const desc = document.querySelector('meta[name="description"]')?.getAttribute('content')?.trim();
    if (desc) addUniq({ meaning: desc, role: 'meta-description', notes: JSON.stringify({ tag: 'meta[name=description]' }) });
    const ogTitle = document.querySelector('meta[property="og:title"]')?.getAttribute('content')?.trim();
    if (ogTitle) addUniq({ meaning: ogTitle, role: 'og-title', notes: JSON.stringify({ tag: 'meta[property=og:title]' }) });
    const canonical = document.querySelector('link[rel="canonical"]')?.getAttribute('href')?.trim();
    if (canonical) addUniq({ meaning: canonical, role: 'canonical-url', notes: JSON.stringify({ tag: 'link[rel=canonical]' }) });

    // 2. All headings h1–h6 — full page structure
    document.querySelectorAll('h1, h2, h3, h4, h5, h6').forEach((h: any) => {
      if (!isVisible(h)) return;
      const text = h.textContent?.replace(/\s+/g, ' ').trim().slice(0, 100);
      if (text) addUniq({ meaning: text, role: h.tagName.toLowerCase(), notes: JSON.stringify({ level: h.tagName }) }, h);
    });

    // 3. FAQ / accordion — <details>/<summary>, ARIA pattern, and class-name scan
    document.querySelectorAll('details, [data-state="closed"], [data-state="open"]').forEach((el: any) => {
      if (!isVisible(el)) return;
      const summary = el.tagName === 'DETAILS'
        ? el.querySelector('summary')
        : el.querySelector('[class*="trigger" i], [class*="header" i], [class*="question" i], [class*="title" i]') ?? el;
      const qText = summary?.textContent?.replace(/\s+/g, ' ').trim().slice(0, 120);
      if (qText && qText.length > 3) {
        addUniq({ meaning: qText, role: 'faq-question', notes: JSON.stringify({ expanded: el.hasAttribute('open') || el.getAttribute('data-state') === 'open' }) });
      }
      // Answer: for <details> it's the non-summary child; for React accordions try both child content and next sibling
      const contentEl = el.tagName === 'DETAILS'
        ? Array.from(el.children).find((c: any) => c.tagName !== 'SUMMARY')
        : (el.querySelector('[class*="content" i], [class*="answer" i], [class*="panel" i], [class*="body" i]:not(body)') ?? el.nextElementSibling);
      if (contentEl) {
        const aText = (contentEl as any).textContent?.replace(/\s+/g, ' ').trim().slice(0, 200);
        if (aText && aText.length > 5 && aText !== qText) addUniq({ meaning: aText, role: 'faq-answer', notes: JSON.stringify({ question: qText }) });
      }
    });
    // 3b. FAQ by class-name container scan — catches sites without ARIA or data-state attributes
    document.querySelectorAll('[class*="faq" i]:not(script):not(style):not(nav), [id*="faq" i]:not(script):not(style)').forEach((container: any) => {
      if (!isVisible(container)) return;
      const qEls = container.querySelectorAll('[class*="question" i], [class*="trigger" i], summary, dt, [class*="heading" i]');
      const aEls = container.querySelectorAll('[class*="answer" i], [class*="content" i]:not(main):not(article), [class*="panel" i], dd');
      qEls.forEach((q: any) => {
        const qText = q.textContent?.replace(/\s+/g, ' ').trim().slice(0, 120);
        if (qText && qText.length > 3) addUniq({ meaning: qText, role: 'faq-question', notes: JSON.stringify({ source: 'class-faq' }) });
      });
      aEls.forEach((a: any) => {
        const aText = a.textContent?.replace(/\s+/g, ' ').trim().slice(0, 200);
        if (aText && aText.length > 5) addUniq({ meaning: aText, role: 'faq-answer', notes: JSON.stringify({ source: 'class-faq' }) });
      });
    });

    // 4. Card and chip content (component-library agnostic)
    document.querySelectorAll('[class*="card" i]:not(nav):not(header):not(footer), [class*="chip" i], [class*="badge" i], [class*="tag" i], [class*="pill" i]').forEach((el: any) => {
      if (!isVisible(el)) return;
      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
      if (text && text.length > 2 && text.length < 200) addUniq({ meaning: text, role: 'content-card' }, el);
    });

    // 5. Feature / service list items inside main content (capped per category)
    const mainContent = document.querySelector('main, [role="main"], article') ?? document.body;
    let listItemCount = 0;
    mainContent.querySelectorAll('li').forEach((li: any) => {
      if (listItemCount >= 50) return;
      if (!isVisible(li)) return;
      const text = li.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
      if (text && text.length > 3 && text.length < 200) { addUniq({ meaning: text, role: 'list-item' }, li); listItemCount++; }
    });

    // 6. First paragraph in each section (key descriptive copy)
    mainContent.querySelectorAll('section > p:first-of-type, article > p:first-of-type, [class*="section" i] > p:first-of-type').forEach((p: any) => {
      if (!isVisible(p)) return;
      const text = p.textContent?.replace(/\s+/g, ' ').trim().slice(0, 250);
      if (text && text.length > 10) addUniq({ meaning: text, role: 'section-paragraph' }, p);
    });

    // 7. CTA section text (standalone paragraphs near buttons that aren't form fields)
    document.querySelectorAll('[class*="cta" i] p, [class*="call-to-action" i] p, section:not(form) > p').forEach((p: any) => {
      if (!isVisible(p)) return;
      const text = p.textContent?.replace(/\s+/g, ' ').trim().slice(0, 300);
      if (text && text.length > 10 && text.length < 400) addUniq({ meaning: text, role: 'cta-section' });
    });

    // 8. Navigation links — explicitly captured (excluded from interactive element selector)
    // Use a lighter visibility check: skip offsetParent because nav is often inside a responsive
    // container (e.g. hidden lg:grid) that may be display:none at the crawler's viewport,
    // causing offsetParent to be null even though the link exists in the page's nav structure.
    const isNavVisible = (el: Element) => {
      const s = window.getComputedStyle(el);
      return s.display !== 'none' && s.visibility !== 'hidden' && s.opacity !== '0';
    };
    const navSel = [
      'nav a', '[role="navigation"] a', 'header a', 'header nav a', 'footer nav a',
      '[aria-label*="navigation" i] a', '[aria-label*="menu" i] a', '[aria-label*="sidebar" i] a',
      'aside a', '[role="complementary"] a',
      '[class*="sidebar" i] a', '[class*="side-nav" i] a', '[class*="sidenav" i] a',
      '[class*="nav-menu" i] a', '[class*="navmenu" i] a', '[data-nav] a',
    ].join(', ');
    const navLinksFoundCount = { n: 0 };
    document.querySelectorAll(navSel).forEach((a: any) => {
      if (!isNavVisible(a)) return;
      const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
      const href = a.href;
      if (!label || !href || /^(javascript:|#$)/i.test(href)) return;
      let landmark = 'Site Navigation';
      let el: any = a.parentElement;
      while (el) {
        const tag = el.tagName?.toLowerCase();
        const role = el.getAttribute?.('role')?.toLowerCase();
        const cl = (el.className ?? '').toLowerCase();
        if (tag === 'header' || role === 'banner') { landmark = 'Header'; break; }
        if (tag === 'footer' || role === 'contentinfo') { landmark = 'Footer'; break; }
        if (tag === 'aside' || role === 'complementary' || cl.includes('sidebar') || cl.includes('sidenav') || cl.includes('side-nav')) { landmark = 'Sidebar'; break; }
        if (tag === 'nav' || role === 'navigation') {
          const directLabel = el.getAttribute('aria-label')?.trim() ?? '';
          const labelId = el.getAttribute('aria-labelledby');
          const referencedLabel = labelId ? (document.getElementById(labelId)?.textContent?.trim() ?? '') : '';
          landmark = directLabel || referencedLabel || 'Site Navigation';
          break;
        }
        el = el.parentElement;
      }
      addUniq({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark }) });
      navLinksFoundCount.n++;
    });
    // Grandparent-grouping fallback: detects sidebar navigation where each link is individually
    // wrapped (e.g. <div class="flex"><a>link</a></div> × many siblings). Runs ALWAYS so header
    // links don't block sidebar detection. Deduped against hrefs already in the semantic pass.
    {
      const navHrefsAlready = new Set<string>();
      for (const item of out) {
        if (item.role === 'nav-link') {
          try { const h = JSON.parse(item.notes ?? '{}').href; if (h) navHrefsAlready.add(h); } catch { /* skip */ }
        }
      }
      const gpGroups2 = new Map<Element, Array<{label: string; href: string}>>();
      document.querySelectorAll('a[href]').forEach((a: any) => {
        if (!isNavVisible(a)) return;
        const href = a.href ?? '';
        if (!href || /^(javascript:|#$|mailto:|tel:)/i.test(href) || navHrefsAlready.has(href)) return;
        const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
        if (!label || label.length > 40) return;
        const parent = a.parentElement;
        const gp = parent?.parentElement;
        if (!parent || !gp) return;
        if (parent.querySelectorAll('a').length !== 1) return;
        const gpTag = gp.tagName.toLowerCase();
        if (['body', 'html', 'form'].includes(gpTag)) return;
        const arr = gpGroups2.get(gp) ?? [];
        arr.push({ label, href });
        gpGroups2.set(gp, arr);
      });
      for (const [, links] of gpGroups2.entries()) {
        if (links.length >= 4) {
          for (const { label, href } of links) {
            addUniq({ meaning: label, role: 'nav-link', notes: JSON.stringify({ href: href.slice(0, 150), landmark: 'Sidebar' }) });
          }
        }
      }
    }

    // 9. Breadcrumb items
    document.querySelectorAll('[aria-label*="breadcrumb" i] a, [class*="breadcrumb" i] a, nav[aria-label*="breadcrumb" i] a').forEach((a: any) => {
      const label = a.textContent?.replace(/\s+/g, ' ').trim().slice(0, 60);
      const href = a.href;
      if (label) addUniq({ meaning: label, role: 'breadcrumb-item', notes: JSON.stringify({ href: href?.slice(0, 150) }) });
    });

    // 10. Footer links and contact info (capped at 25)
    // Matches semantic <footer>, ARIA contentinfo, and common div/section footer patterns
    let footerLinkCount = 0;
    const footerEls = new Set<Element>();
    document.querySelectorAll('footer, [role="contentinfo"], [class*="footer" i]:not(script):not(style):not(nav), [id*="footer" i]:not(script):not(style)').forEach((f: any) => footerEls.add(f));
    const footerLinks: Element[] = [];
    footerEls.forEach((f) => f.querySelectorAll('a').forEach((a: Element) => footerLinks.push(a)));
    // Deduplicate links that appear in multiple matched footer containers
    const seenFooterHrefs = new Set<string>();
    footerLinks.filter((a: any) => {
      const href = a.href ?? '';
      if (seenFooterHrefs.has(href)) return false;
      seenFooterHrefs.add(href);
      return true;
    }).forEach((a: any) => {
      if (footerLinkCount >= 25) return;
      if (!isVisible(a)) return;
      const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
      const href = a.href ?? '';
      if (!label) return;
      if (/^mailto:/i.test(href)) {
        addUniq({ meaning: label || href, role: 'footer-contact', notes: JSON.stringify({ href }) }); footerLinkCount++;
      } else if (/^tel:/i.test(href)) {
        addUniq({ meaning: label || href, role: 'footer-contact', notes: JSON.stringify({ href }) }); footerLinkCount++;
      } else if (label.length > 0) {
        addUniq({ meaning: label, role: 'footer-link', notes: JSON.stringify({ href: href.slice(0, 150) }) }); footerLinkCount++;
      }
    });

    // 11. All image types — img tags (with or without alt), picture/source, CSS background-image, lazy-load patterns
    let imgCount = 0;
    // Lighter visibility check for images: skip only display:none or visibility:hidden.
    // Do NOT check offsetParent (fails for position:fixed heroes) or opacity (kills fade-in animations).
    const isImgVisible = (el: any) => {
      const s = window.getComputedStyle(el);
      return s.display !== 'none' && s.visibility !== 'hidden';
    };
    // 11a. <img> — both with and without alt (capped at 20)
    document.querySelectorAll('img').forEach((img: any) => {
      if (imgCount >= 20) return;
      if (!isImgVisible(img)) return;
      const alt = img.getAttribute('alt')?.trim().slice(0, 80) ?? '';
      const src = (img.getAttribute('src') || img.getAttribute('data-src') || img.getAttribute('data-lazy-src') || img.getAttribute('data-original') || '').slice(0, 120);
      // Also grab first URL from srcset (Next.js Image component uses srcset directly on <img>)
      const srcsetFirst = img.getAttribute('srcset')?.split(',')[0]?.trim().split(' ')[0]?.slice(0, 120) ?? '';
      const resolvedSrc = src || srcsetFirst;
      if (!resolvedSrc && !alt) return;
      const label = alt || resolvedSrc.split('/').pop()?.split('?')[0]?.slice(0, 60) || 'Image';
      addUniq({ meaning: label, role: alt ? 'image' : 'image-no-alt', notes: JSON.stringify({ src: resolvedSrc, alt, hasAlt: !!alt }) });
      imgCount++;
    });
    // 11b. <picture> / <source srcset> — responsive images
    document.querySelectorAll('picture source[srcset]').forEach((src: any) => {
      if (imgCount >= 25) return;
      const srcset = src.getAttribute('srcset')?.split(',')[0]?.trim().split(' ')[0]?.slice(0, 120) ?? '';
      const media = src.getAttribute('media') ?? '';
      if (srcset) { addUniq({ meaning: `Responsive image${media ? ' (' + media + ')' : ''}: ${srcset.split('/').pop()?.split('?')[0] ?? ''}`, role: 'picture-source', notes: JSON.stringify({ srcset: srcset.slice(0, 150), media }) }); imgCount++; }
    });
    // 11c. CSS background-image via inline style and computed style (capped at 15)
    let bgCount = 0;
    document.querySelectorAll('[style*="background-image"], [style*="background:"]').forEach((el: any) => {
      if (bgCount >= 15) return;
      if (!isImgVisible(el)) return;
      const style = el.getAttribute('style') ?? '';
      const match = style.match(/url\(['"]?([^'")\s]+)['"]?\)/i);
      if (match) {
        const url = match[1].slice(0, 120);
        const role = el.tagName.toLowerCase();
        const context = (el.getAttribute('aria-label') || el.getAttribute('title') || el.className.split(' ').find((c: string) => c.length > 3 && !['true','false'].includes(c)) || '').slice(0, 60);
        addUniq({ meaning: context || `BG image on <${role}>: ${url.split('/').pop()?.split('?')[0] ?? url}`, role: 'bg-image', notes: JSON.stringify({ url, element: role, context }) });
        bgCount++;
      }
    });
    // 11d. Computed background-image on key visual elements (expanded class list)
    document.querySelectorAll('[class*="hero" i], [class*="banner" i], [class*="cover" i], [class*="header-bg" i], [class*="card" i]:not(nav), [class*="feature" i], [class*="service" i], [class*="about" i], [class*="team" i], section').forEach((el: any) => {
      if (bgCount >= 15) return;
      if (!isImgVisible(el)) return;
      try {
        const computed = window.getComputedStyle(el).backgroundImage;
        if (computed && computed !== 'none' && computed.includes('url(')) {
          const match = computed.match(/url\(['"]?([^'")\s]+)['"]?\)/i);
          if (match) {
            const url = match[1].slice(0, 120);
            const tag = el.tagName.toLowerCase();
            const cls = Array.from(el.classList as DOMTokenList).find((c: any) => typeof c === 'string' && c.length > 3 && !/^(true|false|active|visible)$/.test(c)) ?? '';
            addUniq({ meaning: `CSS bg image on .${cls || tag}: ${url.split('/').pop()?.split('?')[0] ?? url}`, role: 'bg-image', notes: JSON.stringify({ url, cssClass: cls, element: tag }) });
            bgCount++;
          }
        }
      } catch { /* getComputedStyle not always available */ }
    });
    // 11e. Lazy-load data attribute patterns (data-bg, data-background, data-lazy)
    document.querySelectorAll('[data-bg], [data-background], [data-background-image]').forEach((el: any) => {
      if (bgCount >= 15) return;
      if (!isVisible(el)) return;
      const url = (el.getAttribute('data-bg') || el.getAttribute('data-background') || el.getAttribute('data-background-image') || '').slice(0, 120);
      if (url) { addUniq({ meaning: `Lazy BG image: ${url.split('/').pop()?.split('?')[0] ?? url}`, role: 'bg-image', notes: JSON.stringify({ url, pattern: 'data-bg' }) }); bgCount++; }
    });

    // 12. Content links outside nav/header/footer (capped at 40)
    const navTags = new Set(['nav', 'header', 'footer']);
    let contentLinkCount = 0;
    document.querySelectorAll('a[href]').forEach((a: any) => {
      if (contentLinkCount >= 40) return;
      if (!isVisible(a)) return;
      let p = a.parentElement;
      while (p) { if (navTags.has(p.tagName?.toLowerCase())) return; p = p.parentElement; }
      const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 80);
      const href = a.href;
      if (label && href && !/^(#$|javascript:|mailto:|tel:)/i.test(href)) {
        const isExternal = !href.includes(location.hostname);
        addUniq({ meaning: label, role: isExternal ? 'external-link' : 'internal-link', notes: JSON.stringify({ href: href.slice(0, 120) }) });
        contentLinkCount++;
      }
    });

    // 13. Animated / interactive content markers
    const hasAnimation = !!document.querySelector('[class*="animate"], [class*="motion"], [class*="transition"], [class*="scroll-"], [data-aos], [data-scroll]');
    if (hasAnimation) addUniq({ meaning: 'CSS animations / scroll effects detected on page', role: 'animation-layer', notes: JSON.stringify({ detected: true }) });
    if (document.querySelector('video')) addUniq({ meaning: 'Video element present', role: 'video', notes: JSON.stringify({ count: document.querySelectorAll('video').length }) });
    if (document.querySelector('canvas')) addUniq({ meaning: 'Canvas element present', role: 'canvas' });

    // C-07: Section structure as named semantic units
    document.querySelectorAll('main section, article section, [role="main"] section').forEach((el: any) => {
      if (!isVisible(el)) return;
      const heading = el.querySelector('h1,h2,h3,h4')?.textContent?.trim() ?? '';
      const itemCount = el.querySelectorAll('li, [class*="card" i], [class*="chip" i]').length;
      const paragraphs = el.querySelectorAll('p').length;
      const meaning = heading || `Section (${itemCount} items)`;
      if (meaning) addUniq({ meaning, role: 'page-section', notes: JSON.stringify({ heading, itemCount, paragraphs, id: el.id || el.getAttribute('data-section') }) });
    });

    // 14. Extended meta tags (og:description, og:image, twitter card)
    const ogDesc = document.querySelector('meta[property="og:description"]')?.getAttribute('content')?.trim();
    if (ogDesc) addUniq({ meaning: ogDesc, role: 'og-description', notes: JSON.stringify({ tag: 'meta[property=og:description]' }) });
    const ogImage = document.querySelector('meta[property="og:image"]')?.getAttribute('content')?.trim();
    if (ogImage) addUniq({ meaning: ogImage, role: 'og-image', notes: JSON.stringify({ tag: 'meta[property=og:image]', src: ogImage }) });
    const twitterTitle = document.querySelector('meta[name="twitter:title"]')?.getAttribute('content')?.trim();
    if (twitterTitle) addUniq({ meaning: twitterTitle, role: 'twitter-title', notes: JSON.stringify({ tag: 'meta[name=twitter:title]' }) });
    const twitterDesc = document.querySelector('meta[name="twitter:description"]')?.getAttribute('content')?.trim();
    if (twitterDesc) addUniq({ meaning: twitterDesc, role: 'twitter-description', notes: JSON.stringify({ tag: 'meta[name=twitter:description]' }) });
    const jsonFonts: string[] = [];
    document.querySelectorAll('link[rel="preload"][as="font"], link[href*="fonts.googleapis"]').forEach((l: any) => {
      const url = l.href?.slice(0, 120);
      if (url) jsonFonts.push(url);
    });
    if (jsonFonts.length) addUniq({ meaning: `Font imports: ${jsonFonts.length} font file(s)`, role: 'font-stack', notes: JSON.stringify({ fonts: jsonFonts.slice(0, 5) }) });

    // 15. JSON-LD structured data — extract FAQ answers, deliverables, tech stack, metrics
    document.querySelectorAll('script[type="application/ld+json"], script[class*="tsr"]').forEach((s: any) => {
      try {
        const data = JSON.parse(s.textContent ?? '{}');
        const items = Array.isArray(data) ? data : [data];
        for (const item of items) {
          // FAQ answers from JSON-LD (often NOT rendered in collapsed accordions)
          const faqArr = item.mainEntity ?? item.faqs ?? item.questions ?? item.faq ?? [];
          (Array.isArray(faqArr) ? faqArr : []).forEach((f: any) => {
            const q = f.name ?? f.question ?? '';
            const a = f.acceptedAnswer?.text ?? f.answer ?? '';
            if (q) addUniq({ meaning: String(q).slice(0, 150), role: 'faq-question', notes: JSON.stringify({ source: 'json-ld' }) });
            if (a) addUniq({ meaning: String(a).slice(0, 200), role: 'faq-answer', notes: JSON.stringify({ question: String(q).slice(0, 80), source: 'json-ld' }) });
          });
          // Deliverables with descriptions
          const deliverables = item.deliverables ?? item.services ?? item.offerings ?? [];
          (Array.isArray(deliverables) ? deliverables : []).forEach((d: any) => {
            const title = d.title ?? d.name ?? '';
            const desc = d.description ?? d.desc ?? '';
            if (title) addUniq({ meaning: `${title}${desc ? ': ' + desc.slice(0, 120) : ''}`, role: 'deliverable', notes: JSON.stringify({ title, description: String(desc).slice(0, 200) }) });
          });
          // Tech stack items
          const stack = item.stack ?? item.technologies ?? item.tools ?? [];
          (Array.isArray(stack) ? stack : []).forEach((tool: any) => {
            const name = typeof tool === 'string' ? tool : (tool.name ?? '');
            if (name) addUniq({ meaning: name, role: 'tech-stack-item', notes: JSON.stringify({ source: 'json-ld' }) });
          });
          // Outcome/metric cards
          const outcomes = item.outcomes ?? item.metrics ?? item.stats ?? [];
          (Array.isArray(outcomes) ? outcomes : []).forEach((m: any) => {
            const metric = m.metric ?? m.value ?? '';
            const label = m.label ?? m.name ?? '';
            if (metric && label) addUniq({ meaning: `${metric} — ${label}`, role: 'metric-card', notes: JSON.stringify({ metric, label }) });
          });
          // Features list
          const features = item.features ?? item.capabilities ?? [];
          (Array.isArray(features) ? features : []).forEach((f: any) => {
            const name = typeof f === 'string' ? f : (f.name ?? '');
            if (name) addUniq({ meaning: name, role: 'feature-badge', notes: JSON.stringify({ source: 'json-ld' }) });
          });
          // HowTo steps (tutorials, guides, recipes)
          if (item['@type'] === 'HowTo' || item['@type'] === 'Recipe') {
            const steps = item.step ?? item.steps ?? [];
            (Array.isArray(steps) ? steps : []).forEach((step: any, idx: number) => {
              const text = step.text ?? step.description ?? (typeof step === 'string' ? step : '');
              const name = step.name ?? `Step ${idx + 1}`;
              if (text) addUniq({ meaning: `${name}: ${String(text).slice(0, 200)}`, role: 'howto-step', notes: JSON.stringify({ stepIndex: idx + 1, name }) });
            });
          }
          // BreadcrumbList — page hierarchy for navigation test generation
          if (item['@type'] === 'BreadcrumbList') {
            const crumbs = item.itemListElement ?? [];
            (Array.isArray(crumbs) ? crumbs : []).forEach((crumb: any) => {
              const name = crumb.name ?? crumb.item?.name ?? '';
              const url = crumb.item?.['@id'] ?? crumb.item ?? '';
              if (name) addUniq({ meaning: name, role: 'breadcrumb-item', notes: JSON.stringify({ position: crumb.position, url }) });
            });
          }
          // Event details (conference, webinar, workshop)
          if (item['@type'] === 'Event') {
            const name = item.name ?? '';
            const startDate = item.startDate ?? '';
            const location = item.location?.name ?? item.location?.address?.streetAddress ?? '';
            if (name) addUniq({ meaning: name, role: 'event-name', notes: JSON.stringify({ startDate, location, source: 'json-ld' }) });
          }
          // JobPosting — for job boards and career pages
          if (item['@type'] === 'JobPosting') {
            const title = item.title ?? '';
            const desc = item.description ?? '';
            const employer = item.hiringOrganization?.name ?? '';
            if (title) addUniq({ meaning: `${title}${employer ? ' at ' + employer : ''}`, role: 'job-posting', notes: JSON.stringify({ description: String(desc).slice(0, 200) }) });
          }
          // Review / AggregateRating — for product and service pages
          if (item['@type'] === 'Review') {
            const body = item.reviewBody ?? '';
            const rating = item.reviewRating?.ratingValue ?? '';
            const author = item.author?.name ?? '';
            if (body) addUniq({ meaning: String(body).slice(0, 200), role: 'review-text', notes: JSON.stringify({ rating, author }) });
          }
        }
      } catch { /* non-fatal — malformed JSON-LD */ }
    });

    // 16. Tab panels — capture visible and aria-labeled panels (process steps, tabbed content)
    document.querySelectorAll('[role="tabpanel"], [data-state="active"][id*="panel"], [id*="panel"][aria-labelledby]').forEach((el: any) => {
      if (!isVisible(el)) return;
      const heading = el.querySelector('h2, h3, h4')?.textContent?.trim() ?? '';
      const desc = el.querySelector('p')?.textContent?.replace(/\s+/g, ' ').trim().slice(0, 200) ?? '';
      const badges = Array.from(el.querySelectorAll('[class*="badge" i], [class*="tag" i], [class*="pill" i], [class*="chip" i], [class*="rounded-full" i]')).map((b: any) => b.textContent?.trim()).filter(Boolean).slice(0, 8);
      const meaning = heading || desc;
      if (meaning) addUniq({ meaning: meaning.slice(0, 150), role: 'tab-panel', notes: JSON.stringify({ heading, description: desc, badges, panelId: el.id }) });
    });

    // 17. FAQ accordion: aria-controls pattern — find answer panel via aria-controls attribute
    // Include both aria-expanded buttons and plain aria-controls (some FAQ components omit aria-expanded)
    document.querySelectorAll('[aria-controls]').forEach((btn: any) => {
      if (!isVisible(btn)) return;
      const panelId = btn.getAttribute('aria-controls');
      if (!panelId) return;
      const panel = document.getElementById(panelId);
      if (!panel) return;
      const qText = btn.textContent?.replace(/\s+/g, ' ').trim().slice(0, 120);
      // Read panel text even if hidden (accordion may be collapsed)
      const aText = panel.textContent?.replace(/\s+/g, ' ').trim().slice(0, 200);
      if (qText && qText.length > 3) addUniq({ meaning: qText, role: 'faq-question', notes: JSON.stringify({ panelId, expanded: btn.getAttribute('aria-expanded') }) });
      if (aText && aText.length > 5 && aText !== qText) addUniq({ meaning: aText, role: 'faq-answer', notes: JSON.stringify({ question: qText?.slice(0, 80), source: 'aria-controls' }) });
    });

    // 18. Tech stack / tool pills — monospace pill elements outside nav/footer
    const mainEl = document.querySelector('main, [role="main"], article') ?? document.body;
    mainEl.querySelectorAll('[class*="mono" i]:not(code):not(pre), [class*="pill" i]:not(nav), [class*="chip" i]:not(nav), [class*="tag" i]:not(nav), [class*="badge" i]:not(nav)').forEach((el: any) => {
      if (!isVisible(el)) return;
      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 60);
      if (text && text.length > 1 && text.length < 60 && !text.includes(' ')) {
        // Single-word or hyphenated tool name
        addUniq({ meaning: text, role: 'tech-stack-item', notes: JSON.stringify({ source: 'dom-pill' }) });
      } else if (text && text.length > 1 && text.length < 40) {
        addUniq({ meaning: text, role: 'feature-badge', notes: JSON.stringify({ source: 'dom-pill' }) });
      }
    });

    // 19. Related/linked service cards in main content (cards with links to other pages)
    mainEl.querySelectorAll('a[href][class*="card" i], [class*="card" i] a[href], [class*="service" i][href], a[class*="service" i]').forEach((el: any) => {
      if (!isVisible(el)) return;
      const anchor = el.tagName === 'A' ? el : el.querySelector('a');
      if (!anchor) return;
      const href = anchor.href;
      if (!href || /^(#|javascript:|mailto:|tel:)/i.test(href)) return;
      const heading = el.querySelector('h2, h3, h4')?.textContent?.trim() ?? '';
      const desc = el.querySelector('p')?.textContent?.replace(/\s+/g, ' ').trim().slice(0, 120) ?? '';
      const label = anchor.textContent?.replace(/\s+/g, ' ').trim().slice(0, 80) ?? '';
      const meaning = heading || label || 'Linked card';
      if (meaning && meaning !== 'Linked card' || (heading && desc)) {
        addUniq({ meaning: `${heading || meaning}${desc ? ': ' + desc : ''}`.slice(0, 180), role: 'service-card', notes: JSON.stringify({ href: href.slice(0, 150), heading, description: desc }) });
      }
    });

    // 20. Metric/outcome cards — structured numeric data cards
    mainEl.querySelectorAll('[class*="metric" i], [class*="outcome" i], [class*="stat" i], [class*="kpi" i]').forEach((el: any) => {
      if (!isVisible(el)) return;
      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 100);
      if (text && text.length > 2 && text.length < 120) addUniq({ meaning: text, role: 'metric-card', notes: JSON.stringify({ source: 'dom-class' }) });
    });

    // 21. Screen-reader-only headings (sr-only) — accessibility landmarks hidden from visual view
    document.querySelectorAll('.sr-only, [class*="sr-only" i], [class*="visually-hidden" i]').forEach((el: any) => {
      const tag = el.tagName?.toLowerCase() ?? '';
      if (!/^h[1-6]$/.test(tag)) return;
      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 100);
      if (text) addUniq({ meaning: text, role: `${tag}-sr-only`, notes: JSON.stringify({ srOnly: true, tag }) });
    });

    // 22. CSS custom property (design token) detection
    // Step 1: enumerate :root / html rules from all same-origin stylesheets
    try {
      const cssTokens: Record<string, string> = {};
      const rootStyle = window.getComputedStyle(document.documentElement);
      // Enumerate from stylesheets (catches ALL custom properties, not just a hardcoded list)
      try {
        for (let si = 0; si < document.styleSheets.length; si++) {
          try {
            const sheet = document.styleSheets[si];
            const rules = sheet.cssRules;
            for (let ri = 0; ri < rules.length; ri++) {
              const rule = rules[ri] as any;
              if (rule.selectorText === ':root' || rule.selectorText === 'html' || rule.selectorText === ':root, html') {
                const style = rule.style;
                for (let pi = 0; pi < style.length; pi++) {
                  const prop = style[pi];
                  if (prop.startsWith('--') && Object.keys(cssTokens).length < 30) {
                    const val = style.getPropertyValue(prop).trim();
                    if (val) cssTokens[prop] = val.slice(0, 60);
                  }
                }
              }
            }
          } catch { /* cross-origin sheet, skip */ }
        }
      } catch { /* no styleSheets access */ }
      // Step 2: fall back to computed value probe with common framework naming schemes
      // Covers: plain CSS, Tailwind v4, Bootstrap, Material UI, Shadcn/ui, custom brands
      const probePrefixes = ['', '--color-', '--bs-', '--brand-', '--clr-', '--theme-', '--md-sys-color-'];
      const probeNames = ['primary', 'secondary', 'accent', 'background', 'foreground', 'surface', 'text', 'muted', 'border', 'ring', 'card', 'destructive', 'warning', 'success'];
      const probeFonts = ['--font-sans', '--font-mono', '--font-display', '--font-heading', '--font-body', '--font-family-base'];
      for (const pfx of probePrefixes) {
        for (const name of probeNames) {
          const v = `${pfx}${name}`;
          if (!cssTokens[v]) {
            const val = rootStyle.getPropertyValue(v).trim();
            if (val) cssTokens[v] = val.slice(0, 60);
          }
        }
      }
      for (const v of probeFonts) {
        if (!cssTokens[v]) {
          const val = rootStyle.getPropertyValue(v).trim();
          if (val) cssTokens[v] = val.slice(0, 60);
        }
      }
      if (Object.keys(cssTokens).length > 0) {
        const limited = Object.fromEntries(Object.entries(cssTokens).slice(0, 25));
        addUniq({ meaning: `Design tokens: ${Object.keys(limited).join(', ')}`, role: 'design-tokens', notes: JSON.stringify(limited) });
      }
    } catch { /* computed styles not available */ }

    // 23. Data tables — headers + row count (critical for business/admin pages)
    document.querySelectorAll('table').forEach((table: any) => {
      if (!isVisible(table)) return;
      const headers = Array.from(table.querySelectorAll('thead th, thead td, tr:first-child th')).map((th: any) => th.textContent?.trim()).filter(Boolean).slice(0, 8);
      const rowCount = table.querySelectorAll('tbody tr').length;
      if (headers.length > 0) addUniq({ meaning: `Table: ${headers.join(' · ')}`, role: 'data-table', notes: JSON.stringify({ headers, rowCount }) });
    });

    // 24. Definition lists — term/description pairs (used for specs, glossaries, FAQs, metadata)
    document.querySelectorAll('dl').forEach((dl: any) => {
      if (!isVisible(dl)) return;
      const terms = Array.from(dl.querySelectorAll('dt')).map((dt: any) => dt.textContent?.trim()).filter(Boolean).slice(0, 10);
      if (terms.length > 0) addUniq({ meaning: `Definition list: ${terms.slice(0, 4).join(', ')}`, role: 'definition-list', notes: JSON.stringify({ terms: terms.slice(0, 10) }) });
    });

    // 25. Blockquotes / testimonials / reviews / pull quotes
    let quoteCount = 0;
    document.querySelectorAll('blockquote, [class*="testimonial" i], [class*="review" i]:not(script), [class*="quote" i]:not([class*="icon"]), [class*="pullquote" i]').forEach((el: any) => {
      if (quoteCount >= 10) return;
      if (!isVisible(el)) return;
      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 200);
      const author = el.querySelector('[class*="author" i], [class*="name" i], cite')?.textContent?.trim().slice(0, 60) ?? '';
      if (text && text.length > 20) { addUniq({ meaning: `${text.slice(0, 160)}${author ? ' — ' + author : ''}`, role: 'testimonial', notes: JSON.stringify({ author }) }); quoteCount++; }
    });

    // 26. Alert / notice / notification banners (non-interactive, but key content for test assertions)
    document.querySelectorAll('[role="alert"], [role="status"], [role="banner"]:not(header), .alert, .notice, .notification, .warning-banner, .error-banner, .success-banner, [class*="alert-" i]:not(button), [class*="toast" i]:not(button), [class*="snackbar" i]').forEach((el: any) => {
      if (!isVisible(el)) return;
      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
      if (text && text.length > 5) addUniq({ meaning: text, role: 'alert-banner', notes: JSON.stringify({ ariaRole: el.getAttribute('role') }) }, el);
    });

    // 27. Pricing / plan cards — structured pricing content
    let pricingCount = 0;
    document.querySelectorAll('[class*="pricing" i], [class*="plan" i]:not(main):not(article), [class*="tier" i]:not(nav), [class*="package" i]:not(script)').forEach((el: any) => {
      if (pricingCount >= 8) return;
      if (!isVisible(el)) return;
      const name = el.querySelector('h2,h3,h4,[class*="plan-name" i],[class*="tier-name" i]')?.textContent?.trim().slice(0, 60) ?? '';
      const price = el.querySelector('[class*="price" i],[class*="amount" i]')?.textContent?.trim().slice(0, 40) ?? '';
      if (name || price) { addUniq({ meaning: `${name || 'Plan'}${price ? ': ' + price : ''}`, role: 'pricing-plan', notes: JSON.stringify({ planName: name, price }) }); pricingCount++; }
    });

    // 28. Pagination — page navigation for multi-page content
    const paginationEl = document.querySelector('[role="navigation"][aria-label*="paginat" i], [class*="pagination" i], [aria-label*="paginat" i]');
    if (paginationEl && isVisible(paginationEl)) {
      const pages = Array.from(paginationEl.querySelectorAll('a, button')).map((p: any) => p.textContent?.trim()).filter(Boolean).slice(0, 10);
      if (pages.length > 0) addUniq({ meaning: `Pagination: ${pages.join(', ')}`, role: 'pagination', notes: JSON.stringify({ pages }) });
    }

    // 29. Code blocks / pre blocks — critical for developer documentation pages
    let codeCount = 0;
    document.querySelectorAll('pre code, [class*="code-block" i], [class*="highlight" i] pre').forEach((el: any) => {
      if (codeCount >= 5) return;
      if (!isVisible(el)) return;
      const lang = el.getAttribute('class')?.match(/language-(\w+)/)?.[1] ?? '';
      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 100);
      if (text && text.length > 10) { addUniq({ meaning: `Code example${lang ? ' (' + lang + ')' : ''}: ${text.slice(0, 80)}`, role: 'code-block', notes: JSON.stringify({ language: lang }) }); codeCount++; }
    });

    // 30. Dropdown / select menu items exposed via ARIA listbox/option
    let dropdownCount = 0;
    document.querySelectorAll('[role="listbox"] [role="option"], [role="option"]:not([role="listbox"] *)').forEach((el: any) => {
      if (dropdownCount >= 20) return;
      if (!isVisible(el)) return;
      const text = el.textContent?.trim().slice(0, 80);
      if (text) { addUniq({ meaning: text, role: 'dropdown-option', notes: JSON.stringify({ selected: el.getAttribute('aria-selected') }) }); dropdownCount++; }
    });

    // 31. Timeline / process steps (common on landing pages)
    let timelineCount = 0;
    document.querySelectorAll('[class*="timeline" i] li, [class*="timeline" i] [class*="step" i], [class*="process" i] li, [class*="steps" i] li').forEach((el: any) => {
      if (timelineCount >= 10) return;
      if (!isVisible(el)) return;
      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
      if (text && text.length > 5) { addUniq({ meaning: text, role: 'timeline-step', notes: JSON.stringify({ source: 'dom-class' }) }); timelineCount++; }
    });

    // 32. Team members / person cards — social proof and about pages
    let teamCount = 0;
    document.querySelectorAll('[class*="team-member" i], [class*="team-card" i], [class*="person-card" i], [class*="bio" i][class*="card" i]').forEach((el: any) => {
      if (teamCount >= 10) return;
      if (!isVisible(el)) return;
      const name = el.querySelector('[class*="name" i], h3, h4')?.textContent?.trim().slice(0, 60) ?? '';
      const role = el.querySelector('[class*="role" i], [class*="title" i], [class*="position" i]')?.textContent?.trim().slice(0, 60) ?? '';
      if (name) { addUniq({ meaning: `${name}${role ? ' — ' + role : ''}`, role: 'team-member', notes: JSON.stringify({ name, role }) }); teamCount++; }
    });

    // 33. Video embeds — YouTube, Vimeo, Wistia, Loom iframes (src is readable even cross-origin)
    document.querySelectorAll('iframe').forEach((iframe: any) => {
      const src = iframe.getAttribute('src') || iframe.getAttribute('data-src') || '';
      const title = iframe.getAttribute('title') || iframe.getAttribute('aria-label') || '';
      if (!src) return;
      if (/youtube\.com\/embed|youtu\.be\/|youtube-nocookie\.com/i.test(src)) {
        const videoId = src.match(/\/embed\/([^/?&]+)/)?.[1] ?? '';
        addUniq({ meaning: title || `YouTube video${videoId ? ' (ID: ' + videoId + ')' : ''}`, role: 'video-embed', notes: JSON.stringify({ platform: 'youtube', src: src.slice(0, 200), title, videoId }) });
      } else if (/vimeo\.com\/video\//i.test(src)) {
        const videoId = src.match(/video\/(\d+)/)?.[1] ?? '';
        addUniq({ meaning: title || `Vimeo video${videoId ? ' (ID: ' + videoId + ')' : ''}`, role: 'video-embed', notes: JSON.stringify({ platform: 'vimeo', src: src.slice(0, 200), title, videoId }) });
      } else if (/wistia\.com|loom\.com|dailymotion\.com|twitch\.tv|brightcove\.com/i.test(src)) {
        const platform = /wistia/.test(src) ? 'wistia' : /loom/.test(src) ? 'loom' : /dailymotion/.test(src) ? 'dailymotion' : /twitch/.test(src) ? 'twitch' : 'brightcove';
        addUniq({ meaning: title || `${platform} video`, role: 'video-embed', notes: JSON.stringify({ platform, src: src.slice(0, 200), title }) });
      } else if (/maps\.google\.com|google\.com\/maps\/embed/i.test(src)) {
        const q = new URLSearchParams(src.split('?')[1] ?? '').get('q') ?? new URLSearchParams(src.split('?')[1] ?? '').get('query') ?? '';
        addUniq({ meaning: title || `Google Maps${q ? ': ' + decodeURIComponent(q).slice(0, 80) : ''}`, role: 'map-embed', notes: JSON.stringify({ provider: 'google', src: src.slice(0, 200), query: q }) });
      } else if (/openstreetmap\.org|mapbox\.com|here\.com|maps\.apple\.com/i.test(src)) {
        const provider = /openstreetmap/.test(src) ? 'OpenStreetMap' : /mapbox/.test(src) ? 'Mapbox' : /here\.com/.test(src) ? 'HERE Maps' : 'Apple Maps';
        addUniq({ meaning: title || `${provider} embed`, role: 'map-embed', notes: JSON.stringify({ provider, src: src.slice(0, 200), title }) });
      }
    });

    // 34. SVG content — only meaningful SVGs (have title/aria-label AND are large enough to be illustrations)
    document.querySelectorAll('svg').forEach((svg: any) => {
      if (!isVisible(svg)) return;
      const titleEl = svg.querySelector('title');
      const label = svg.getAttribute('aria-label')?.trim() || titleEl?.textContent?.trim() || '';
      if (!label) return;
      const rect = svg.getBoundingClientRect();
      if (rect.width < 48 && rect.height < 48) return; // Skip icon-size SVGs
      addUniq({ meaning: label.slice(0, 100), role: 'svg-image', notes: JSON.stringify({ width: Math.round(rect.width), height: Math.round(rect.height) }) });
    });

    // 35. Animated counters — read data-target/data-count BEFORE animation completes
    let counterCount = 0;
    document.querySelectorAll('[data-count],[data-target],[data-to],[data-countup],[class*="counter" i]:not(button),[class*="count-up" i],[class*="odometer" i]').forEach((el: any) => {
      if (counterCount >= 15) return;
      if (!isVisible(el)) return;
      const targetVal = el.getAttribute('data-count') || el.getAttribute('data-target') || el.getAttribute('data-to') || el.getAttribute('data-countup') || '';
      const currentText = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 40) ?? '';
      const contextLabel = el.closest('[class*="stat" i],[class*="metric" i],[class*="kpi" i],[class*="counter-item" i]')?.textContent?.replace(/\s+/g, ' ').trim().slice(0, 80) ?? '';
      const value = targetVal || currentText;
      if (value && value !== '0') {
        addUniq({ meaning: contextLabel.slice(0, 100) || `Counter: ${value}`, role: 'animated-counter', notes: JSON.stringify({ targetValue: targetVal, displayedValue: currentText }) });
        counterCount++;
      }
    });

    // 36. Client / partner logos — distinguish from generic images
    let logoCount = 0;
    document.querySelectorAll('[class*="client" i] img,[class*="partner" i] img,[class*="logo-grid" i] img,[class*="clients-" i] img,[class*="brand-logo" i] img,[class*="trust-logo" i] img,[class*="sponsor" i] img').forEach((img: any) => {
      if (logoCount >= 20) return;
      if (!isVisible(img)) return;
      const alt = img.getAttribute('alt')?.trim().slice(0, 60) ?? '';
      const src = (img.getAttribute('src') || img.getAttribute('data-src') || '').slice(0, 120);
      if (!alt && !src) return;
      addUniq({ meaning: alt || src.split('/').pop()?.split('?')[0]?.slice(0, 50) || 'Client logo', role: 'client-logo', notes: JSON.stringify({ alt, src, hasAlt: !!alt }) });
      logoCount++;
    });

    // 37. Awards / certifications / trust badges
    let awardCount = 0;
    document.querySelectorAll('[class*="award" i],[class*="certif" i]:not(script):not(link),[class*="accredit" i],[class*="trust-badge" i],[class*="recognition" i],[class*="badge" i][class*="award" i]').forEach((el: any) => {
      if (awardCount >= 10) return;
      if (!isVisible(el)) return;
      const text = el.textContent?.replace(/\s+/g, ' ').trim().slice(0, 80);
      const imgEl = el.querySelector('img');
      const label = text || imgEl?.getAttribute('alt')?.trim() || '';
      if (label && label.length > 2) { addUniq({ meaning: label, role: 'award-badge', notes: JSON.stringify({ hasImage: !!imgEl }) }); awardCount++; }
    });

    // 38. A/B testing detection — captures active experiments (not variant content)
    try {
      const abTools: string[] = [];
      if ((window as any).optimizely || (window as any).Optimizely) abTools.push('Optimizely');
      if ((window as any).VWO || (window as any)._vwo_code) abTools.push('VWO');
      if ((window as any).__ABTasty || (window as any).ABTasty) abTools.push('AB Tasty');
      if (document.querySelector('script[src*="optimize.js"]')) abTools.push('Google Optimize');
      if ((window as any).splitbee) abTools.push('Splitbee');
      if ((window as any).__GROWTHBOOK__ || document.querySelector('script[src*="growthbook"]')) abTools.push('GrowthBook');
      if ((window as any).Kameleoon) abTools.push('Kameleoon');
      if (abTools.length > 0) {
        const optExp = (window as any).optimizely?.get?.('state')?.getActiveExperimentIds?.() ?? [];
        addUniq({ meaning: `A/B testing: ${abTools.join(', ')}${optExp.length ? ' (' + optExp.length + ' active experiments)' : ''}`, role: 'ab-test-layer', notes: JSON.stringify({ tools: abTools, activeExperiments: optExp.slice(0, 10) }) });
      }
    } catch { /* best-effort */ }

    // 39. Geo / personalization detection
    try {
      const signals: string[] = [];
      if ((window as any).__geo || (window as any).__country || (window as any).__region) signals.push('window geo globals');
      if (document.querySelector('[data-geo],[data-country],[data-region],[data-locale]')) signals.push('data-geo DOM attributes');
      const ipScripts = Array.from(document.querySelectorAll('script[src]')).filter((s: any) => /ipinfo|maxmind|ipstack|ipgeolocation/i.test(s.getAttribute('src') ?? '')).length;
      if (ipScripts > 0) signals.push('IP geolocation script');
      if ((window as any).Intl?.DateTimeFormat?.()?.resolvedOptions?.()?.timeZone) signals.push(`timezone: ${(window as any).Intl.DateTimeFormat().resolvedOptions().timeZone}`);
      if (signals.length > 1) {
        addUniq({ meaning: `Personalization/geo signals: ${signals.join('; ')}`, role: 'personalization-layer', notes: JSON.stringify({ signals }) });
      }
    } catch { /* best-effort */ }

    // 40. Hover-revealed content — read DOM text of CSS-hidden children on hover triggers
    let hoverCount = 0;
    document.querySelectorAll('[class*="group"]:not(nav):not(ul):not(ol),[class*="hover-trigger" i],[data-hover],[class*="has-dropdown" i]').forEach((trigger: any) => {
      if (hoverCount >= 8) return;
      if (!isVisible(trigger)) return;
      const hiddenChildren = Array.from(trigger.querySelectorAll('[class*="group-hover"],[class*="hover-content" i],[class*="dropdown-menu" i],[class*="submenu" i]')).filter((c: any) => {
        try { const s = window.getComputedStyle(c); return s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1; } catch { return false; }
      });
      for (const child of hiddenChildren.slice(0, 3)) {
        const text = (child as any).textContent?.replace(/\s+/g, ' ').trim().slice(0, 150);
        if (text && text.length > 3) {
          addUniq({ meaning: text, role: 'hover-content', notes: JSON.stringify({ triggerClass: (trigger.className?.toString() ?? '').slice(0, 60) }) });
          hoverCount++;
        }
      }
    });

    return out.slice(0, 400); // Per-category caps above; raised global cap to 400 for new categories
  }).then((elems: any[]) => elems.map((e: any) => ({ ...e, isInteractive: false }))).catch(() => []);
}

/**
 * Click through tabs, advance carousels, open accordions — collect links that only appear
 * inside those interactive panels. Does not navigate away from the current page.
 */
async function interactAndCollectLinks(page: any, origin: string): Promise<string[]> {
  const collected = new Set<string>();
  const sleep = (ms: number) => new Promise<void>(r => setTimeout(r, ms));

  const collectCurrentLinks = async () => {
    const links: string[] = await page.evaluate((origin: string) => {
      const out: string[] = [];
      const logout = /logout|log out|sign out|delete/i;
      document.querySelectorAll<HTMLAnchorElement>('a[href]').forEach(a => {
        if (logout.test(a.textContent ?? '')) return;
        try {
          const u = new URL(a.href);
          if (u.origin === origin) { if (u.hash && !/^#!?\//.test(u.hash)) u.hash = ''; out.push(u.toString()); }
        } catch { /* ignore */ }
      });
      return out;
    }, origin).catch(() => []);
    links.forEach(l => collected.add(l));
  };

  try {
    // ── Tabs: click each tab individually, collect links from revealed panel ──
    const tabCount: number = await page.evaluate(() =>
      document.querySelectorAll('[role="tab"]').length
    ).catch(() => 0);

    for (let i = 0; i < Math.min(tabCount, 30); i++) {
      try {
        const clicked = await page.evaluate((idx: number) => {
          const tabs = document.querySelectorAll<HTMLElement>('[role="tab"]');
          const tab = tabs[idx];
          if (!tab) return false;
          const s = window.getComputedStyle(tab);
          const r = tab.getBoundingClientRect();
          if (s.display === 'none' || r.width < 4 || r.height < 4) return false;
          tab.click();
          return true;
        }, i).catch(() => false);
        if (clicked) {
          await sleep(350);
          await collectCurrentLinks();
        }
      } catch { /* skip */ }
    }

    // ── Carousels: advance next button up to 8 times, collect links each slide ──
    const carouselNextSel = [
      'button[aria-label*="next" i]', 'button[aria-label*="forward" i]',
      'button[aria-label*="right" i]', '[data-direction="next"]',
      '[class*="carousel" i] button[class*="next" i]',
      '[class*="slider" i] button[class*="next" i]',
    ].join(', ');

    for (let i = 0; i < 8; i++) {
      const advanced: boolean = await page.evaluate((sel: string) => {
        const btn = document.querySelector<HTMLElement>(sel);
        if (!btn) return false;
        const s = window.getComputedStyle(btn);
        const r = btn.getBoundingClientRect();
        if (s.display === 'none' || r.width < 4 || r.height < 4) return false;
        btn.click();
        return true;
      }, carouselNextSel).catch(() => false);
      if (!advanced) break;
      await sleep(300);
      await collectCurrentLinks();
    }

    // ── Accordions / disclosure panels: open collapsed sections ──────────────
    const accordionCount: number = await page.evaluate(() =>
      document.querySelectorAll('details:not([open]) > summary, [data-state="closed"] > button, [aria-expanded="false"][class*="accordion" i]').length
    ).catch(() => 0);

    for (let i = 0; i < Math.min(accordionCount, 20); i++) {
      try {
        await page.evaluate((idx: number) => {
          const items = document.querySelectorAll<HTMLElement>(
            'details:not([open]) > summary, [data-state="closed"] > button, [aria-expanded="false"][class*="accordion" i]'
          );
          const el = items[idx];
          if (el) { try { el.click(); } catch { /* ignore */ } }
        }, i).catch(() => { });
        await sleep(250);
        await collectCurrentLinks();
      } catch { /* skip */ }
    }
  } catch { /* best-effort */ }

  return Array.from(collected);
}

/** Click collapsed nav triggers (hamburger menus, aria-haspopup buttons, Radix/Headless UI toggles)
 *  to reveal routes that only exist in the DOM after user interaction.
 *  Returns true if anything was clicked so caller can dismiss afterward. */
async function expandNavMenus(page: any): Promise<boolean> {
  try {
    const clicked = await page.evaluate(() => {
      const isVisible = (el: HTMLElement) => {
        const s = window.getComputedStyle(el);
        if (s.display === 'none' || s.visibility === 'hidden' || parseFloat(s.opacity) < 0.1) return false;
        const r = el.getBoundingClientRect();
        return r.width > 4 && r.height > 4;
      };
      const inBadContext = (el: HTMLElement) => {
        let p = el.parentElement;
        while (p) {
          const tag = p.tagName?.toLowerCase();
          const role = p.getAttribute('role') ?? '';
          if (['table', 'select'].includes(tag ?? '') ||
            ['dialog', 'alertdialog', 'combobox', 'listbox', 'grid', 'treegrid'].includes(role)) return true;
          p = p.parentElement;
        }
        return false;
      };
      const targets = new Set<HTMLElement>();
      // aria-haspopup: canonical menu-trigger pattern
      document.querySelectorAll<HTMLElement>('[aria-haspopup]:not([aria-expanded="true"])').forEach(el => {
        if (isVisible(el) && !inBadContext(el)) targets.add(el);
      });
      // aria-expanded=false: collapsed nav sections
      document.querySelectorAll<HTMLElement>('[aria-expanded="false"]').forEach(el => {
        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadContext(el)) targets.add(el);
      });
      // Radix UI / Headless UI closed triggers
      document.querySelectorAll<HTMLElement>('[data-state="closed"],[data-headlessui-state="closed"]').forEach(el => {
        if ((el.tagName === 'BUTTON' || el.getAttribute('role') === 'button') && isVisible(el) && !inBadContext(el)) targets.add(el);
      });
      let count = 0;
      targets.forEach(el => { try { (el as HTMLElement).click(); count++; } catch { /* ignore */ } });
      return count;
    });
    if (clicked > 0) await new Promise(r => setTimeout(r, 500));

    // ── Hover sweep: reveal CSS :hover dropdown menus ──────────────────────
    // Pure CSS hover menus need real mouse moves — JS click alone won't trigger :hover
    try {
      const navHandles = await page.$$(
        'nav > ul > li, nav > li, header > ul > li, header > nav > ul > li, ' +
        '[role="navigation"] > ul > li, [role="menubar"] > [role="menuitem"], ' +
        '[class*="nav"] > ul > li, [class*="menu"] > li'
      ).catch(() => [] as any[]);
      for (const h of (navHandles as any[]).slice(0, 40)) {
        try { await h.hover({ force: true, timeout: 600 }); await page.waitForTimeout(280); } catch { /* skip invisible */ }
      }
      if ((navHandles as any[]).length > 0) {
        await page.mouse.move(2, 2); // dismiss hover state
        await page.waitForTimeout(150);
      }
    } catch { /* non-fatal */ }

    return (clicked as number) > 0;
  } catch { return false; }
}

async function domFallbackLinks(page: any, origin: string, rawPage?: any): Promise<string[]> {
  // Named function so .toString() gives the raw function body for CDP evaluation.
  // esbuild wraps nested named arrow-function constants with __name() — prepend a
  // tiny polyfill so the browser has it when we serialize this function as a string.
  function _linkEval(originStr: string) {
    const logout = /logout|log out|sign out|delete/i;
    const isRouteHash = (hash: string) => /^#!?\//.test(hash);
    const seen = new Set<string>();
    const add = (rawHref: string, textContent: string) => {
      if (logout.test(textContent)) return;
      if (!rawHref) return;
      if (rawHref.startsWith('#') && !isRouteHash(rawHref)) return;
      if (rawHref.startsWith('mailto:') || rawHref.startsWith('javascript:')) return;
      try {
        const u = new URL(rawHref, location.href);
        if (!isRouteHash(u.hash)) u.hash = '';
        if (new URL(u).origin === originStr) seen.add(u.toString());
      } catch { /* ignore */ }
    };
    const allRoots: Array<Document | ShadowRoot> = [document];
    const rootSeen = new WeakSet<Document | ShadowRoot>();
    rootSeen.add(document);
    while (allRoots.length > 0) {
      const root = allRoots.shift();
      if (!root) continue;
      try {
        root.querySelectorAll('a[href]').forEach((a) => add((a as HTMLAnchorElement).href || a.getAttribute('href') || '', a.textContent || ''));
        root.querySelectorAll('[href]:not(a):not(link):not(base):not(script)').forEach((el) => {
          add(el.getAttribute('href') || '', el.textContent || '');
        });
        root.querySelectorAll('[data-href]').forEach((el) => {
          add(el.getAttribute('data-href') || '', el.textContent || '');
        });
        // Form[action] GET forms: JSP/MVC apps navigate via form submissions (not anchor links)
        root.querySelectorAll('form[action]').forEach((form) => {
          const method = (form.getAttribute('method') || 'get').toLowerCase();
          if (method !== 'post') add(form.getAttribute('action') || '', form.getAttribute('aria-label') || '');
        });
        root.querySelectorAll('*').forEach((el) => {
          try {
            if (el.shadowRoot && !rootSeen.has(el.shadowRoot)) {
              rootSeen.add(el.shadowRoot);
              allRoots.push(el.shadowRoot);
            }
          } catch (_) { /* closed/cross-origin shadow root */ }
        });
      } catch (_) { /* element threw — skip root */ }
    }
    return Array.from(seen);
  }

  const _linkEvalStr = `(function(){const __name=(fn,n)=>{try{Object.defineProperty(fn,'name',{value:n,configurable:true})}catch(e){}return fn};return(${_linkEval.toString()})(${JSON.stringify(origin)})})();`;

  // CDP primary — bypasses Stagehand isolated world entirely
  if (rawPage && typeof rawPage.context === 'function') {
    try {
      const ctx = rawPage.context();
      if (typeof ctx?.newCDPSession === 'function') {
        const cdp = await ctx.newCDPSession(rawPage);
        try {
          const evalResult = await cdp.send('Runtime.evaluate', {
            expression: _linkEvalStr,
            returnByValue: true,
            awaitPromise: false,
          });
          if (!evalResult?.exceptionDetails && Array.isArray(evalResult?.result?.value)) {
            return evalResult.result.value as string[];
          }
          if (evalResult?.exceptionDetails) {
            console.warn('[crawler] domFallbackLinks CDP error:', evalResult.exceptionDetails?.exception?.description ?? evalResult.exceptionDetails?.text);
          }
        } finally {
          cdp.detach().catch(() => {});
        }
      }
    } catch (cdpErr: any) {
      console.warn('[crawler] domFallbackLinks CDP failed:', cdpErr?.message ?? String(cdpErr));
    }
  }

  // Fallback: pass string expression so esbuild __name wrapper never reaches browser
  const evalPage = rawPage ?? page;
  return (evalPage.evaluate(_linkEvalStr as any) as Promise<string[]>).catch((err: any) => {
    console.warn('[crawler] domFallbackLinks page.evaluate threw:', err?.message ?? String(err));
    return [];
  });
}

/** Extract label→href pairs for all in-app anchor links — used to seed KAG element transitions. */
async function extractLinkMappings(page: any, origin: string): Promise<Array<{ label: string; href: string }>> {
  return page.evaluate((origin: string) => {
    const logout = /logout|log out|sign out|delete/i;
    return Array.from(document.querySelectorAll('a[href]'))
      .filter((a: any) => !logout.test(a.textContent ?? ''))
      .map((a: any) => {
        const label = (
          a.getAttribute('aria-label') ||
          a.textContent?.replace(/\s+/g, ' ').trim() ||
          a.getAttribute('title') ||
          ''
        ).slice(0, 80);
        const href = (() => { try { const u = new URL(a.href); if (!/^#!?\//.test(u.hash)) u.hash = ''; return u.toString(); } catch { return ''; } })();
        return { label, href };
      })
      .filter(({ label, href }: { label: string; href: string }) =>
        label.length > 2 && href.length > 0 &&
        (() => { try { return new URL(href).origin === origin; } catch { return false; } })()
      );
  }, origin).catch(() => []);
}

/** 
 * Computes a structural hash of the DOM to detect repeating screens/components.
 * Strips out text and focuses only on the sequence of semantic/form tags.
 */
async function computeDomHash(page: any): Promise<string> {
  return page.evaluate(() => {
    let hashStr = '';
    const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT);
    let node;
    while ((node = walker.nextNode())) {
      const el = node as HTMLElement;
      // Only include structural/semantic tags that matter for UI layouts
      if (/^(MAIN|SECTION|NAV|FORM|INPUT|SELECT|TEXTAREA|BUTTON|TABLE|TR|TD|TH|UL|LI|A|H1|H2|H3)$/.test(el.tagName)) {
        // Count structural nesting depth — distinguishes modal dialogs (deeply nested)
        // from top-level nav/forms (shallow), preventing false duplicate matches.
        let depth = 0;
        let parent = el.parentElement;
        while (parent) {
          if (/^(MAIN|SECTION|NAV|FORM|INPUT|SELECT|TEXTAREA|BUTTON|TABLE|TR|TD|TH|UL|LI|A|H1|H2|H3)$/.test(parent.tagName)) depth++;
          parent = parent.parentElement;
        }
        hashStr += el.tagName + depth + '|';
      }
    }
    // Very simple hashing function (djb2 style)
    let hash = 5381;
    for (let i = 0; i < hashStr.length; i++) {
      hash = (hash * 33) ^ hashStr.charCodeAt(i);
    }
    return (hash >>> 0).toString(16);
  }).catch(() => '');
}

async function waitForURLChange(page: any, fromUrl: string, timeout = 10_000): Promise<void> {
  const deadline = Date.now() + timeout;
  while (Date.now() < deadline) {
    const current = page.url();
    try {
      const from = new URL(fromUrl);
      const now = new URL(current);
      if (now.pathname !== from.pathname || now.origin !== from.origin) return;
    } catch {
      if (current !== fromUrl) return;
    }
    await new Promise(r => setTimeout(r, 200));
  }
  // Timed out — proceed with current URL (may still be login page)
}

/**
 * Dismiss common overlay patterns (cookie banners, GDPR consent, chat widgets, newsletter modals)
 * before element extraction — prevents banner elements from polluting the element model.
 */
async function dismissOverlays(page: any): Promise<void> {
  const selectors = [
    // OneTrust / CookieBot / TrustArc (most common GDPR banners)
    '#onetrust-accept-btn-handler',
    '#accept-cookie-consent',
    '.cc-btn.cc-allow',
    '[data-cookiebanner] button',
    '[id*="cookie-accept"]',
    '[class*="cookie-accept"]',
    '[id*="cookieAccept"]',
    '[class*="cookieAccept"]',
    'button[aria-label*="Accept cookies" i]',
    'button[aria-label*="Accept all" i]',
    'button[title*="Accept" i][class*="cookie" i]',
    // Intercom / Drift / HubSpot chat
    '[class*="intercom-close"]',
    '#hubspot-messages-iframe-container [aria-label*="close" i]',
    // Generic overlay close buttons
    '[role="dialog"] button[aria-label="Close" i]',
    '[role="dialog"] button[aria-label="Dismiss" i]',
    // Newsletter popups
    'button[class*="modal-close"]',
    'button[class*="popup-close"]',
    '[class*="newsletter"] button[class*="close"]',
  ];
  for (const sel of selectors) {
    try {
      const el = page.locator(sel).first();
      if (await el.isVisible({ timeout: 300 }).catch(() => false)) {
        await el.click({ timeout: 500, force: true }).catch(() => { });
      }
    } catch { /* overlay not present — skip */ }
  }
  // Brief settle after dismissals — Node.js timer, no dependency on page.waitForTimeout proxy
  await new Promise<void>(r => setTimeout(r, 300));
}

/**
 * Scroll the full page height to trigger IntersectionObserver-based lazy loads
 * (images, components, infinite-scroll content). Scrolls back to top before capture.
 */
async function scrollToTriggerLazyLoad(page: any, rawPage?: any): Promise<void> {
  const evalPage = rawPage ?? page;
  try {
    // Scroll the full page in viewport-sized steps.
    // window.scrollTo handles standard scroll containers and triggers IntersectionObserver.
    const { scrollHeight, innerHeight, innerWidth } = await evalPage.evaluate(() => ({
      scrollHeight: Math.min(document.body.scrollHeight, 12_000),
      innerHeight: window.innerHeight || 800,
      innerWidth: window.innerWidth || 1280,
    })).catch(() => ({ scrollHeight: 0, innerHeight: 800, innerWidth: 1280 }));

    if (scrollHeight > innerHeight) {
      // CDP wheel events trigger virtual scrollers (react-window / TanStack Virtual / AG Grid)
      // which ignore window.scrollTo but listen on their own container via wheel events.
      let cdp: any = null;
      if (rawPage && typeof rawPage.context === 'function') {
        try {
          const ctx = rawPage.context();
          if (typeof ctx?.newCDPSession === 'function') cdp = await ctx.newCDPSession(rawPage);
        } catch { cdp = null; }
      }

      const step = Math.max(innerHeight, 400);
      for (let y = step; y < scrollHeight; y += step) {
        await evalPage.evaluate((pos: number) => window.scrollTo(0, pos), y).catch(() => {});
        if (cdp) {
          await cdp.send('Input.dispatchMouseEvent', {
            type: 'mouseWheel',
            x: Math.floor(innerWidth / 2),
            y: Math.floor(innerHeight / 2),
            deltaX: 0,
            deltaY: step,
            modifiers: 0,
          }).catch(() => {});
        }
        // Node.js-side wait — does NOT depend on page.waitForTimeout (Stagehand proxy may lack it)
        await new Promise<void>(r => setTimeout(r, 150));
      }
      // Scroll back to top
      await evalPage.evaluate(() => window.scrollTo(0, 0)).catch(() => {});
      if (cdp) cdp.detach().catch(() => {});
    }

    // Let lazy-triggered content render before extraction
    await new Promise<void>(r => setTimeout(r, 400));
    // Wait for networkidle: IntersectionObserver fires on scroll but async data fetches
    // (React Query, SWR, Apollo) triggered by it may still be in-flight. networkidle ensures
    // all XHR/fetch requests spawned by the scroll complete before DOM stability check.
    const scrollWaitPage = rawPage ?? page;
    if (typeof scrollWaitPage.waitForLoadState === 'function') {
      await scrollWaitPage.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => {});
    }
    // Re-check DOM stability (shadow-piercing) after scroll-triggered mutations + data fetches
    await waitForStableDOM(page, 4_000, 500, rawPage);
  } catch { /* scroll failed — non-fatal, proceed */ }
}

async function waitForStableDOM(page: any, maxWait = 8_000, stabilityMs = 400, rawPage?: any): Promise<void> {
  const evalPage = rawPage ?? page; // prefer rawPage — avoids Stagehand proxy + __name isolated-world issues
  const deadline = Date.now() + maxWait;
  let lastCount = -1;
  let stableAt = 0;
  let evalErrors = 0;
  while (Date.now() < deadline) {
    const count: number = await evalPage.evaluate(() => {
      // Shadow-piercing counter: querySelectorAll('*') only counts light-DOM elements;
      // shadow roots are separate trees. Counting both ensures changes inside web components
      // (e.g. Material UI, Lit, Shoelace) reset the stability timer correctly.
      let c = 0;
      const visit = (root: Document | ShadowRoot) => {
        const els = root.querySelectorAll('*');
        c += els.length;
        els.forEach((el: any) => { if (el.shadowRoot) visit(el.shadowRoot); });
      };
      visit(document);
      return c;
    }).catch(() => -1);
    if (count === -1) {
      // evaluate failed (isolated world broken) — after 3 consecutive failures, wait stabilityMs and bail
      if (++evalErrors >= 3) {
        await new Promise(r => setTimeout(r, stabilityMs));
        return;
      }
    } else {
      evalErrors = 0;
      if (count !== lastCount) {
        lastCount = count;
        stableAt = Date.now();
      } else if (Date.now() - stableAt >= stabilityMs) {
        return;
      }
    }
    await new Promise(r => setTimeout(r, 100));
  }
  // Timed out — DOM may still be loading, proceed anyway
}

async function waitForScreenshotReady(page: any, rawPage?: any): Promise<void> {
  const hardDeadline = Date.now() + 30_000;
  const withHardCap = <T>(p: Promise<T>): Promise<T | void> =>
    Promise.race([p, new Promise<void>((resolve) => setTimeout(resolve, Math.max(0, hardDeadline - Date.now())))]);
  // Use rawPage for load-state waits — Stagehand proxy may not expose waitForLoadState
  const waitPage = rawPage ?? page;
  const evalPage = rawPage ?? page;
  try {
    await withHardCap(waitPage.waitForLoadState('domcontentloaded', { timeout: 5_000 }).catch(() => { }));
    await withHardCap(waitPage.waitForLoadState('networkidle', { timeout: 12_000 }).catch(() => { }));
    await withHardCap(evalPage.evaluate(() => {
      if (document.readyState === 'complete') return true;
      return new Promise<boolean>((resolve) => {
        const done = () => resolve(true);
        window.addEventListener('load', done, { once: true });
        setTimeout(done, 4_000);
      });
    }).catch(() => { }));
    await withHardCap(evalPage.evaluate(async () => {
      const fonts = (document as any).fonts;
      if (fonts?.ready) await fonts.ready;
    }).catch(() => { }));
    await withHardCap(scrollToTriggerLazyLoad(rawPage ?? page, rawPage));
    await withHardCap(waitForStableDOM(page, 4_000, 700, rawPage));
    await withHardCap(evalPage.evaluate(async () => {
      const pending = Array.from(document.images).filter((img) => !img.complete);
      if (pending.length === 0) return;
      await Promise.race([
        Promise.all(pending.map((img) => new Promise<void>((resolve) => {
          img.addEventListener('load', () => resolve(), { once: true });
          img.addEventListener('error', () => resolve(), { once: true });
        }))),
        new Promise<void>((resolve) => setTimeout(resolve, 4_000)),
      ]);
    }).catch(() => { }));
    await withHardCap(waitForStableDOM(page, 2_000, 500, rawPage));
  } catch { /* readiness is best-effort; never fail the crawl for a continuously mutating page */ }
}

async function hasMeaningfulAuthOrPublicState(page: any, requestedUrl: string): Promise<boolean> {
  const pathLooksRelevant = (() => {
    try {
      const pathname = new URL(page.url?.() ?? requestedUrl).pathname;
      return AUTH_PATH_RE.test(pathname) || PUBLIC_PATH_RE.test(pathname);
    } catch {
      return false;
    }
  })();

  const domState = await page.evaluate(() => {
    const text = (document.body?.innerText ?? '').toLowerCase();
    const inputs = Array.from(document.querySelectorAll('input')) as HTMLInputElement[];
    const hasPassword = inputs.some((i) => i.type === 'password');
    const hasIdentityInput = inputs.some((i) => {
      const meta = `${i.type} ${i.name} ${i.id} ${i.placeholder} ${i.autocomplete}`.toLowerCase();
      return /(email|username|user|login|phone|otp|code|password)/.test(meta);
    });
    const hasAuthCopy = /(sign in|log in|login|sign up|create account|register|forgot password|reset password|verify email|one-time|otp|continue with email)/i.test(text);
    const hasPublicNav = /(pricing|contact|features|documentation|docs|about|privacy|terms)/i.test(text);
    // Soft-404: page says "not found" / "404" with no real auth form
    const isSoft404 = !hasPassword && !hasIdentityInput &&
      /(page not found|404|this page (could not|doesn'?t) (be found|exist)|we couldn'?t find|no longer exists|been moved|doesn'?t exist)/i.test(text);
    return { hasPassword, hasIdentityInput, hasAuthCopy, hasPublicNav, isSoft404 };
  }).catch(() => ({ hasPassword: false, hasIdentityInput: false, hasAuthCopy: false, hasPublicNav: false, isSoft404: false }));

  // Soft-404 with no auth form — path match alone is not enough
  if (domState.isSoft404) return false;

  const domLooksRelevant = domState.hasPassword || domState.hasIdentityInput || domState.hasAuthCopy || domState.hasPublicNav;
  return pathLooksRelevant || domLooksRelevant;
}

async function discoverAuthCandidates(page: any, origin: string): Promise<{
  links: Array<{ href: string; label: string; flowType: AuthFlowType }>;
  clicks: Array<{ selector: string; label: string; flowType: AuthFlowType }>;
}> {
  return page.evaluate((siteOrigin: string) => {
    type BrowserFlowType = 'public' | 'login' | 'register' | 'password_reset' | 'verification' | 'invite' | 'session_recovery';
    const authTextRe = /(sign\s*in|log\s*in|login|sign\s*up|signup|register|create account|forgot password|reset password|use email|continue with email|back to login|verify|invite|get started|start free trial)/i;
    const skipTextRe = /(logout|log out|sign out|delete|remove|discard|cancel subscription|payment|billing)/i;
    const authPathRe = /\/(login|log-in|signin|sign-in|auth|oauth|sso|account\/login|user\/login|app\/login|register|signup|sign-up|create-account|forgot|reset|password|verify-email|invite)(\/|$|\?)/i;
    const publicPathRe = /\/($|pricing|contact|about|help|docs|features|security|privacy|terms)(\/|$|\?)/i;
    const sameSiteBrowser = (href: string) => {
      try {
        const u = new URL(href, location.href);
        const s = new URL(siteOrigin);
        const strip = (h: string) => h.replace(/^www\./i, '');
        return strip(u.hostname) === strip(s.hostname);
      } catch {
        return false;
      }
    };
    const flowTypeFor = (value: string): BrowserFlowType => {
      const v = value.toLowerCase();
      if (/(forgot|reset|password)/.test(v)) return 'password_reset';
      if (/(register|signup|sign up|create account|join)/.test(v)) return 'register';
      if (/(verify|verification|otp|mfa|2fa|invite)/.test(v)) return v.includes('invite') ? 'invite' : 'verification';
      if (/(session-expired|expired|unauthorized|forbidden|401|403|logout|logged-out)/.test(v)) return 'session_recovery';
      if (/(login|log in|signin|sign in|auth|sso|oauth)/.test(v)) return 'login';
      return 'public';
    };
    const textOf = (el: Element) => (
      el.textContent ||
      el.getAttribute('aria-label') ||
      el.getAttribute('title') ||
      el.getAttribute('placeholder') ||
      ''
    ).replace(/\s+/g, ' ').trim();
    const isVisible = (el: Element) => {
      const rect = (el as HTMLElement).getBoundingClientRect?.();
      const style = window.getComputedStyle(el as HTMLElement);
      return !!rect && rect.width > 0 && rect.height > 0 && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0';
    };
    const links = new Map<string, { href: string; label: string; flowType: BrowserFlowType }>();
    const addHref = (hrefRaw: string | null, labelRaw: string) => {
      if (!hrefRaw || /^(javascript|mailto|tel):/i.test(hrefRaw)) return;
      let href = '';
      try { href = new URL(hrefRaw, location.href).toString(); } catch { return; }
      if (!sameSiteBrowser(href)) return;
      const u = new URL(href);
      const label = labelRaw || u.pathname;
      if (skipTextRe.test(label) || skipTextRe.test(u.pathname)) return;
      if (!authTextRe.test(label) && !authPathRe.test(u.pathname) && !publicPathRe.test(u.pathname)) return;
      const key = `${u.origin}${u.pathname.replace(/\/$/, '') || '/'}${u.search}`;
      if (!links.has(key)) links.set(key, { href, label, flowType: flowTypeFor(`${label} ${u.pathname}`) });
    };

    const walk = (root: Document | ShadowRoot) => {
      root.querySelectorAll('a[href]').forEach((el) => addHref((el as HTMLAnchorElement).href || el.getAttribute('href'), textOf(el)));
      root.querySelectorAll('[data-href],[data-url],[data-link]').forEach((el: any) => addHref(el.dataset?.href || el.dataset?.url || el.dataset?.link || '', textOf(el)));
      root.querySelectorAll('*').forEach((el: any) => { if (el.shadowRoot) walk(el.shadowRoot); });
    };
    walk(document);

    const clicks: Array<{ selector: string; label: string; flowType: BrowserFlowType }> = [];
    const clickEls = Array.from(document.querySelectorAll('button,[role="button"],a:not([href]),[data-state],[aria-haspopup]')) as HTMLElement[];
    let idx = 0;
    for (const el of clickEls) {
      if (clicks.length >= 12) break;
      if (!isVisible(el)) continue;
      const label = textOf(el);
      if (!label || !authTextRe.test(label) || skipTextRe.test(label)) continue;
      const type = (el.getAttribute('type') || '').toLowerCase();
      const inCredentialForm = !!el.closest('form')?.querySelector('input[type="password"],input[type="email"],input[name*="user" i],input[name*="email" i]');
      const submitLike = type === 'submit' || /^(login|log in|sign in|continue)$/i.test(label);
      // Avoid submitting empty login forms. Links/buttons that change auth state, such as
      // "Forgot password" or "Create account", are safe to click even inside the form.
      if (inCredentialForm && submitLike && !/(forgot|reset|register|sign up|create account|back to login|use email)/i.test(label)) continue;
      const attr = `zeta-auth-${idx++}`;
      el.setAttribute('data-zeta-auth-discovery-id', attr);
      clicks.push({ selector: `[data-zeta-auth-discovery-id="${attr}"]`, label, flowType: flowTypeFor(label) });
    }

    return { links: Array.from(links.values()).slice(0, 32), clicks };
  }, origin).catch(() => ({ links: [], clicks: [] }));
}

async function captureAuthDiscoveryScreen(
  page: any,
  opts: AuthDiscoveryOptions,
  requestedUrl: string,
  source: AuthDiscoverySource,
  ordinal: number,
  trigger?: { label?: string; flowType?: AuthFlowType; sameUrlState?: boolean },
): Promise<{ captured: boolean; finalUrl: string; flowType: AuthFlowType }> {
  const finalUrl = page.url?.() ?? requestedUrl;
  if (finalUrl.startsWith('chrome-error://')) return { captured: false, finalUrl, flowType: 'public' };
  if (!(await hasMeaningfulAuthOrPublicState(page, requestedUrl))) {
    return { captured: false, finalUrl, flowType: flowTypeForUrlOrText(`${requestedUrl} ${finalUrl} ${trigger?.label ?? ''}`) };
  }

  const flowType = trigger?.flowType ?? flowTypeForUrlOrText(`${requestedUrl} ${finalUrl} ${trigger?.label ?? ''}`);
  const title = await page.title().catch(() => '');
  const baseName = title?.trim() && !/^(untitled|about:blank)$/i.test(title.trim())
    ? title.trim().slice(0, 72)
    : titleFromUrl(finalUrl);
  const prefix = flowType === 'public' ? 'Public' : `Auth ${flowType.replace('_', ' ')}`;
  const name = `${prefix} · ${trigger?.label ? trigger.label.slice(0, 48) : baseName}`.replace(/\s+/g, ' ');
  const screenUrl = trigger?.sameUrlState
    ? `${finalUrl.split('#')[0]}#zeta-auth-flow=${compactSlug(trigger.label ?? `${flowType}-${ordinal}`)}`
    : finalUrl;
  const shotPath = path.join(opts.screenshotDir ?? '/tmp', `auth-discovery-${opts.jobId ?? 'run'}-${ordinal}.png`);

  await waitForScreenshotReady(page);
  await page.screenshot({ path: shotPath, fullPage: true }).catch(() => { });
  const screen = await AppBrain.saveScreen(opts.tenantId, opts.projectId, name, screenUrl, opts.jobId, source === 'entry' ? 'ENTRY_POINT' : 'SEED_URL').catch(() => null);
  if (!screen) return { captured: false, finalUrl, flowType };
  let authCloudUrl: string | undefined;
  if (opts.storageConfig && existsSyncFs(shotPath)) {
    try {
      const { uploadScreenshotToTenantStorage } = await import('@detiq/app-brain');
      authCloudUrl = await uploadScreenshotToTenantStorage(opts.storageConfig as any, shotPath, `${opts.tenantId}/${opts.projectId}/screenshots/${path.basename(shotPath)}`);
    } catch { /* non-fatal */ }
  }
  if (existsSyncFs(shotPath)) {
    await AppBrain.saveScreenshot(opts.tenantId, opts.projectId, screen.id, shotPath, screen.version, 'DESKTOP', authCloudUrl).catch(() => { });
  }
  // GAP 11: DOM snapshot for auth-discovery phase
  const authDomContent = await capturePageHtml(page);
  if (authDomContent && opts.screenshotDir) {
    const authDomPath = shotPath.replace(/\.(png|jpg|jpeg)$/i, '.dom.html');
    await fs.writeFile(authDomPath, authDomContent, 'utf8');
    let authDomCloudUrl: string | undefined;
    if (opts.storageConfig) {
      try {
        const { uploadScreenshotToTenantStorage } = await import('@detiq/app-brain');
        authDomCloudUrl = await uploadScreenshotToTenantStorage(opts.storageConfig as any, authDomPath, `${opts.tenantId}/${opts.projectId}/dom-snapshots/${path.basename(authDomPath)}`);
      } catch { /* non-fatal */ }
    }
    await AppBrain.updateScreenDomSnapshot(opts.tenantId, screen.id, authDomPath, authDomCloudUrl).catch(() => {});
  }
  // Extract and save interactive elements (inputs, buttons, links) for this auth-discovery screen.
  // Without this, login pages and auth flows always show 0 elements in Crawl Captures.
  const authDiscoveryElements = await readScreen(page).catch(() => [] as Awaited<ReturnType<typeof readScreen>>);
  if (authDiscoveryElements.length > 0) {
    await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
    for (const el of authDiscoveryElements) {
      await AppBrain.saveElement(opts.tenantId, opts.projectId, screen.id, el.label, el.role, el.expectedData, undefined, undefined, true, undefined, undefined, undefined).catch(() => {});
    }
    logger.info({ screenId: screen.id, elements: authDiscoveryElements.length }, '[crawler] auth-discovery: saved elements');
  }
  await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
    screenId: screen.id,
    jobId: opts.jobId,
    type: 'AUTH_DISCOVERY',
    payload: {
      requestedUrl,
      finalUrl,
      screenUrl,
      authState: flowType === 'public' ? 'logged_out' : 'auth_flow',
      flowType,
      source,
      triggerLabel: trigger?.label ?? null,
    },
  }).catch(() => { });
  logger.info({ requestedUrl, finalUrl, screenUrl, flowType, source }, '[crawler] auth-discovery: captured screen');
  return { captured: true, finalUrl, flowType };
}

async function runAuthDiscoveryPhase(
  page: any,
  opts: AuthDiscoveryOptions,
  origin: string,
  onProgress?: AuthDiscoveryProgress,
): Promise<{ captured: number; discoveredUrls: string[] }> {
  const tuning = await getCrawlTuning();
  const queue: Array<{ url: string; source: AuthDiscoverySource }> = [];
  const queued = new Set<string>();
  const capturedKeys = new Set<string>();
  const discoveredUrls = new Set<string>();
  const enqueue = (url: string, source: AuthDiscoverySource) => {
    if (!shouldSeedAuthDiscoveryUrl(url, origin)) return;
    const key = normalizeAuthUrl(url);
    if (queued.has(key) || queue.length >= AUTH_DISCOVERY_MAX_QUEUE) return;
    queued.add(key);
    queue.push({ url, source });
  };

  enqueue(opts.appUrl, 'entry');
  for (const p of AUTH_DISCOVERY_PATHS) {
    try { enqueue(new URL(p, origin).toString(), 'seed_path'); } catch { /* ignore bad seed */ }
  }

  // Capture homepage DOM fingerprint for soft-404 detection.
  // SPA catch-all sites return 200 for every path but render identical content.
  // If a seeded path produces the same element count + title as the homepage, it's
  // a phantom route — skip it so auth paths don't pollute the discovered URL list.
  let homeTitle = '';
  let homeElemCount = -1;
  try {
    await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => null);
    await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
    homeTitle = await page.title().catch(() => '');
    homeElemCount = await page.evaluate(() =>
      document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
    ).catch(() => -1);
  } catch { /* best-effort */ }

  let captured = 0;
  let ordinal = 0;
  await onProgress?.(10, 'Discovering public and auth flows');

  while (queue.length > 0 && captured < AUTH_DISCOVERY_MAX_SCREENS) {
    const current = queue.shift()!;
    const _authPct = 10 + Math.round((captured / AUTH_DISCOVERY_MAX_SCREENS) * 7);
    const _authPath = current.url.replace(/^https?:\/\/[^/]+/, '') || '/';
    await onProgress?.(_authPct, `Auth discovery · ${_authPath} · ${captured} found`);
    try {
      const response = await page.goto(current.url, { waitUntil: 'domcontentloaded', timeout: tuning.navTimeoutMs }).catch(() => null);
      if (response && response.status && response.status() >= 400) continue;
      await page.waitForLoadState('networkidle', { timeout: 6_000 }).catch(() => { });
      await waitForStableDOM(page);
      await acceptCookieConsent(page, { waitMs: 400 }).catch(() => { });

      // Soft-404 detection: SPA returns 200 for unknown paths but renders homepage.
      // Skip if title + element count match the homepage fingerprint exactly.
      if (current.source === 'seed_path' && homeElemCount >= 0) {
        const pageTitle = await page.title().catch(() => '');
        const elemCount = await page.evaluate(() =>
          document.querySelectorAll('button,a,input,select,textarea,[role="button"]').length
        ).catch(() => -1);
        if (pageTitle === homeTitle && elemCount === homeElemCount) continue;
      }

      const finalUrl = page.url?.() ?? current.url;
      const captureKey = normalizeAuthUrl(finalUrl);
      if (!capturedKeys.has(captureKey)) {
        const result = await captureAuthDiscoveryScreen(page, opts, current.url, current.source, ordinal++);
        if (result.captured) {
          captured++;
          capturedKeys.add(captureKey);
          discoveredUrls.add(result.finalUrl);
        }
      }

      const candidates = await discoverAuthCandidates(page, origin);
      for (const link of candidates.links) enqueue(link.href, 'page_link');

      const baseUrlForClicks = finalUrl;
      const clickLimit = Math.min(candidates.clicks.length, AUTH_DISCOVERY_MAX_CLICKS_PER_PAGE);
      for (let i = 0; i < clickLimit && captured < AUTH_DISCOVERY_MAX_SCREENS; i++) {
        await page.goto(baseUrlForClicks, { waitUntil: 'domcontentloaded', timeout: 12_000 }).catch(() => { });
        await page.waitForLoadState('networkidle', { timeout: 4_000 }).catch(() => { });
        await waitForStableDOM(page);
        const freshCandidates = await discoverAuthCandidates(page, origin);
        const click = freshCandidates.clicks[i];
        if (!click) continue;
        const beforeUrl = page.url?.() ?? baseUrlForClicks;
        await page.locator(click.selector).first().click({ timeout: 4_000 }).catch(() => { });
        await page.waitForLoadState('networkidle', { timeout: 5_000 }).catch(() => { });
        await waitForStableDOM(page);
        const afterUrl = page.url?.() ?? beforeUrl;
        const sameUrlState = normalizeAuthUrl(beforeUrl) === normalizeAuthUrl(afterUrl);
        const stateKey = sameUrlState
          ? `${normalizeAuthUrl(afterUrl)}::${compactSlug(click.label)}`
          : normalizeAuthUrl(afterUrl);
        if (capturedKeys.has(stateKey)) continue;
        const result = await captureAuthDiscoveryScreen(page, opts, beforeUrl, 'auth_click', ordinal++, {
          label: click.label,
          flowType: click.flowType,
          sameUrlState,
        });
        if (result.captured) {
          captured++;
          capturedKeys.add(stateKey);
          discoveredUrls.add(result.finalUrl);
          const nested = await discoverAuthCandidates(page, origin);
          for (const link of nested.links) enqueue(link.href, 'page_link');
        }
      }
    } catch (err) {
      logger.warn({ url: current.url, err: String((err as Error)?.message ?? err) }, '[crawler] auth-discovery: URL failed — skipping');
    }
  }

  await page.goto(opts.appUrl, { waitUntil: 'domcontentloaded', timeout: 15_000 }).catch(() => { });
  await waitForStableDOM(page).catch(() => { });
  await AppBrain.saveCrawlEvidence(opts.tenantId, opts.projectId, {
    jobId: opts.jobId,
    type: 'AUTH_DISCOVERY_SUMMARY',
    payload: { captured, discoveredUrls: Array.from(discoveredUrls), maxScreens: AUTH_DISCOVERY_MAX_SCREENS },
  }).catch(() => { });
  logger.info({ captured, discovered: discoveredUrls.size }, '[crawler] auth-discovery: phase complete');
  return { captured, discoveredUrls: Array.from(discoveredUrls) };
}