/**
 * ZeTa Native App Crawler — Appium-based crawler for Android APK and iOS IPA.
 *
 * Architecture:
 *   Appium Server (running at APPIUM_SERVER_URL) → WebDriverIO session
 *   → UiAutomator2 (Android) / XCUITest (iOS)
 *   → accessibility tree XML → ZeTa Element records
 *
 * TODO: requires Appium server running at APPIUM_SERVER_URL env var
 * TODO: requires `webdriverio` package — add "webdriverio": "^9.x" to package.json
 *
 * Device farm support:
 *   - local_emulator: Appium server + local emulator/simulator
 *   - aws_device_farm: presigned S3 URL uploaded to Device Farm, session via WDIO
 *   - browserstack: BrowserStack Automate Appium endpoint
 */

import fs from 'node:fs/promises';
import path from 'node:path';
import os from 'node:os';
import { withTenant } from '@detiq/database';
import { AppBrain, getTenantStorageConfig, uploadScreenshotToTenantStorage, type TenantStorageConfig } from '@detiq/app-brain';
import { config } from '@detiq/core';
import { logger } from './logger.js';

// ---------------------------------------------------------------------------
// Public types
// ---------------------------------------------------------------------------

export interface NativeCrawlerOptions {
  tenantId: string;
  projectId: string;
  jobId: string;
  platform: 'android' | 'ios' | 'windows' | 'macos';
  appS3Path: string;         // s3://bucket/path/app.apk or s3://bucket/path/app.ipa
  packageName?: string;      // com.example.app (Android) or bundle ID (iOS)
  deviceModel?: string;      // optional device preference
  osVersion?: string;     // Appium platformVersion (e.g. "14.0" for Android, "17.0" for iOS)
  appActivity?: string;   // Android: activity to launch (e.g. ".MainActivity")
  appWaitActivity?: string; // Android: activity to wait for after launch
  deviceFarm: 'local_emulator' | 'aws_device_farm' | 'browserstack';
  // Device farm connection credentials (from workspace settings, overrides env vars)
  appiumServerUrl?: string;
  browserstackUsername?: string;
  browserstackAccessKey?: string;
  awsAccessKeyId?: string;
  awsSecretAccessKey?: string;
  awsRegion?: string;
  awsDeviceFarmProjectArn?: string;
  // Auth credentials for Gap 1: native app login
  credentials?: {
    username?: string;
    password?: string;
    usernameFieldLabel?: string; // hint if default detection fails, e.g. "Email"
    passwordFieldLabel?: string;
    loginButtonLabel?: string;
  };
  // Vision fallback AI config for Gap 2: when accessibility tree is empty
  visionLlmConfig?: { provider: string; model: string; apiKey?: string; baseUrl?: string } | null;
  // Gesture config for Gap 3
  gestureConfig?: {
    enableSwipe?: boolean;    // default true
    enableLongPress?: boolean; // default true
  };
  // MITM proxy for Gap 4 (when available)
  mitmProxyPort?: number;
}

/** A native UI element extracted from the accessibility tree. */
interface NativeElement {
  role: string;
  label: string;
  bounds: NativeBounds;
  platform: 'android' | 'ios' | 'windows' | 'macos';
  resourceId?: string;   // Android: resource-id attribute
  className?: string;    // Android: widget class / iOS: XCUIElement type
}

interface NativeBounds {
  x: number;
  y: number;
  width: number;
  height: number;
}

// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------

/**
 * Build Appium desired capabilities for the given platform.
 * localAppPath is the local path after downloading from S3 (or a presigned URL
 * for device farms that fetch the binary themselves).
 */
function buildCapabilities(opts: NativeCrawlerOptions, localAppPath: string): Record<string, unknown> {
  const appiumServerUrl = opts.appiumServerUrl ?? process.env.APPIUM_SERVER_URL ?? 'http://127.0.0.1:4723';
  logger.info({ appiumServerUrl, platform: opts.platform }, 'Building Appium capabilities');

  // BrowserStack early-return: https://hub-cloud.browserstack.com/wd/hub
  if (opts.deviceFarm === 'browserstack' && (opts.platform === 'android' || opts.platform === 'ios')) {
    const bstackOptions = {
      userName: opts.browserstackUsername ?? process.env.BROWSERSTACK_USERNAME ?? '',
      accessKey: opts.browserstackAccessKey ?? process.env.BROWSERSTACK_ACCESS_KEY ?? '',
      deviceName: opts.deviceModel ?? (opts.platform === 'android' ? 'Google Pixel 7' : 'iPhone 15'),
      osVersion: opts.osVersion ?? (opts.platform === 'android' ? '13.0' : '17'),
      realMobile: true,
      projectName: 'ZeTa-QIP',
      buildName: `${opts.platform}-crawl`,
      sessionName: opts.jobId,
    };
    if (opts.platform === 'android') {
      return {
        'platformName': 'android',
        'appium:automationName': 'UiAutomator2',
        'appium:app': localAppPath,
        'appium:newCommandTimeout': 120,
        'appium:autoGrantPermissions': true,
        ...(opts.appActivity ? { 'appium:appActivity': opts.appActivity } : {}),
        ...(opts.packageName ? { 'appium:appPackage': opts.packageName } : {}),
        'bstack:options': bstackOptions,
      };
    }
    return {
      'platformName': 'ios',
      'appium:automationName': 'XCUITest',
      'appium:app': localAppPath,
      'appium:newCommandTimeout': 120,
      ...(opts.packageName ? { 'appium:bundleId': opts.packageName } : {}),
      'bstack:options': bstackOptions,
    };
  }

  if (opts.platform === 'android') {
    return {
      'appium:platformName': 'Android',
      'appium:deviceName': opts.deviceModel ?? 'Pixel 7',
      'appium:app': localAppPath,
      'appium:automationName': 'UiAutomator2',
      'appium:autoGrantPermissions': true,
      'appium:newCommandTimeout': 120,
      'appium:platformVersion': opts.osVersion ?? '',
      ...(opts.appActivity ? { 'appium:appActivity': opts.appActivity } : {}),
      ...(opts.appWaitActivity ? { 'appium:appWaitActivity': opts.appWaitActivity } : {}),
      'appium:eventTimings': true,   // capture launch time telemetry
      ...(opts.packageName ? { 'appium:appPackage': opts.packageName } : {}),
    };
  }

  if (opts.platform === 'windows') {
    return {
      'platformName': 'Windows',
      'appium:automationName': 'Windows',
      'appium:app': localAppPath,
      'appium:deviceName': 'WindowsPC',
      'appium:newCommandTimeout': 120,
      ...(opts.packageName ? { 'appium:appTopLevelWindow': opts.packageName } : {}),
    };
  }

  if (opts.platform === 'macos') {
    return {
      'platformName': 'mac',
      'appium:automationName': 'Mac2',
      'appium:app': localAppPath,
      'appium:deviceName': 'Mac',
      'appium:newCommandTimeout': 120,
      ...(opts.packageName ? { 'appium:bundleId': opts.packageName } : {}),
    };
  }

  // iOS
  return {
    'appium:platformName': 'iOS',
    'appium:deviceName': opts.deviceModel ?? 'iPhone 15',
    'appium:app': localAppPath,
    'appium:automationName': 'XCUITest',
    'appium:newCommandTimeout': 120,
    'appium:platformVersion': opts.osVersion ?? '',
    ...(opts.packageName ? { 'appium:bundleId': opts.packageName } : {}),
  };
}

/**
 * Upload a locally downloaded app binary to BrowserStack App Automate.
 * Returns the bs:// app URL to use as the Appium capability value.
 */
async function uploadToBrowserStack(localPath: string, opts: NativeCrawlerOptions): Promise<string> {
  const username = opts.browserstackUsername ?? process.env.BROWSERSTACK_USERNAME ?? '';
  const accessKey = opts.browserstackAccessKey ?? process.env.BROWSERSTACK_ACCESS_KEY ?? '';
  const credentials = Buffer.from(`${username}:${accessKey}`).toString('base64');

  const fileBuffer = await fs.readFile(localPath);
  const filename = localPath.split('/').pop() ?? 'app.apk';

  const form = new (global as any).FormData();
  form.append('file', new Blob([new Uint8Array(fileBuffer)]), filename);

  logger.info({ localPath, filename }, 'Uploading app binary to BrowserStack');
  const response = await fetch('https://api-cloud.browserstack.com/app-automate/upload', {
    method: 'POST',
    headers: { Authorization: `Basic ${credentials}` },
    body: form,
  });

  if (!response.ok) {
    throw new Error(`BrowserStack upload failed: ${response.status} ${await response.text()}`);
  }
  const json = (await response.json()) as { app_url?: string };
  if (!json.app_url) throw new Error('BrowserStack upload did not return app_url');
  logger.info({ appUrl: json.app_url }, 'App uploaded to BrowserStack');
  return json.app_url; // e.g. "bs://1234abc..."
}

/**
 * Download app binary from S3 to a local temp file.
 * For BrowserStack, also uploads the binary to App Automate and returns the bs:// URL.
 * Returns the local file path (or bs:// URL for BrowserStack).
 */
async function downloadAppFromS3(appS3Path: string, platform: 'android' | 'ios' | 'windows' | 'macos', opts: NativeCrawlerOptions, storageCfg?: TenantStorageConfig | null): Promise<string> {
  // Local disk fallback — APK stored on-server when no cloud storage is configured
  if (appS3Path.startsWith('local://')) {
    const localPath = appS3Path.slice('local://'.length);
    logger.info({ localPath }, 'Using locally stored app binary');
    if (opts.deviceFarm === 'browserstack') {
      return uploadToBrowserStack(localPath, opts);
    }
    return localPath;
  }

  // Parse  s3://bucket/key
  const withoutScheme = appS3Path.replace(/^s3:\/\//, '');
  const slashIdx = withoutScheme.indexOf('/');
  if (slashIdx === -1) {
    throw new Error(`Invalid S3 path: ${appS3Path}. Expected format: s3://bucket/path/to/file`);
  }
  const bucket = withoutScheme.slice(0, slashIdx);
  const key = withoutScheme.slice(slashIdx + 1);
  const extMap: Record<string, string> = { android: '.apk', ios: '.ipa', windows: '.exe', macos: '.app' };
  const ext = extMap[platform] ?? '';
  const tmpPath = path.join(os.tmpdir(), `zeta-native-${Date.now()}${ext}`);

  logger.info({ bucket, key, tmpPath, provider: storageCfg?.provider ?? 'env' }, 'Downloading app binary from storage');

  const { S3Client, GetObjectCommand } = await import('@aws-sdk/client-s3' as any);
  const isR2 = storageCfg?.provider === 'R2';
  const s3ClientCfg: Record<string, any> = {
    region: isR2 ? 'auto' : (storageCfg?.region ?? process.env.AWS_REGION ?? 'us-east-1'),
  };
  if (storageCfg?.accessKeyId && storageCfg?.secretAccessKey) {
    s3ClientCfg.credentials = { accessKeyId: storageCfg.accessKeyId, secretAccessKey: storageCfg.secretAccessKey };
  }
  if (isR2 && storageCfg?.accountId) {
    s3ClientCfg.endpoint = `https://${storageCfg.accountId}.r2.cloudflarestorage.com`;
    s3ClientCfg.forcePathStyle = true;
  }
  const s3 = new S3Client(s3ClientCfg);
  const response = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));

  // Stream the body to disk
  const { Readable } = await import('node:stream');
  const { pipeline } = await import('node:stream/promises');
  const fh = await fs.open(tmpPath, 'w');
  try {
    await pipeline(Readable.from(response.Body as any), fh.createWriteStream());
  } finally {
    await fh.close();
  }

  logger.info({ tmpPath }, 'App binary downloaded');

  // For BrowserStack, upload the binary and return the bs:// URL
  if (opts.deviceFarm === 'browserstack') {
    return uploadToBrowserStack(tmpPath, opts);
  }
  return tmpPath;
}

/**
 * Parse the Appium XML page source into a flat list of NativeElement records.
 *
 * Android UiAutomator2 XML looks like:
 *   <node text="Login" resource-id="com.example:id/btn_login"
 *         class="android.widget.Button" bounds="[100,400][980,520]"
 *         clickable="true" enabled="true" />
 *
 * iOS XCUITest XML looks like:
 *   <XCUIElementTypeButton name="Login" label="Login"
 *         x="50" y="300" width="280" height="44"
 *         enabled="true" visible="true" />
 */
function parseAccessibilityTree(xmlSource: string, platform: 'android' | 'ios' | 'windows' | 'macos'): NativeElement[] {
  const elements: NativeElement[] = [];

  if (platform === 'windows') {
    // WinAppDriver returns UIAutomation XML like:
    // <Button Name="OK" AutomationId="OkButton" IsEnabled="true" IsOffscreen="false"/>
    const WIN_INTERACTIVE = new Set(['Button', 'Edit', 'ComboBox', 'CheckBox', 'RadioButton', 'Slider', 'ListItem', 'MenuItem', 'Hyperlink']);
    const nodeRegex = /<(\w+)\s([^>]*)\/>/g;
    let match: RegExpExecArray | null;
    while ((match = nodeRegex.exec(xmlSource)) !== null) {
      const typeName = match[1] ?? '';
      if (!WIN_INTERACTIVE.has(typeName)) continue;
      const attrs = match[2] ?? '';
      const get = (name: string) => new RegExp(`\\b${name}="([^"]*)"`, 'i').exec(attrs)?.[1] ?? '';
      if (get('IsEnabled') === 'false' || get('IsOffscreen') === 'true') continue;
      const label = get('Name') || get('AutomationId') || '';
      if (!label) continue;
      const role = winTypeToRole(typeName);
      // BoundingRectangle="x:0 y:0 width:100 height:30" or similar
      const brStr = get('BoundingRectangle');
      const brMatch = /x:(\d+)\s+y:(\d+)\s+width:(\d+)\s+height:(\d+)/i.exec(brStr);
      const bounds = brMatch
        ? { x: parseInt(brMatch[1]), y: parseInt(brMatch[2]), width: parseInt(brMatch[3]), height: parseInt(brMatch[4]) }
        : { x: 0, y: 0, width: 100, height: 30 };
      elements.push({ role, label, bounds, platform: 'windows' as any, className: typeName });
    }
  }

  if (platform === 'macos') {
    const MAC_INTERACTIVE = new Set([
      'XCUIElementTypeButton', 'XCUIElementTypeTextField', 'XCUIElementTypeSecureTextField',
      'XCUIElementTypeCheckBox', 'XCUIElementTypeRadioButton', 'XCUIElementTypePopUpButton',
      'XCUIElementTypeComboBox', 'XCUIElementTypeSlider', 'XCUIElementTypeMenuItem',
      'XCUIElementTypeSearchField', 'XCUIElementTypeLink',
    ]);
    const nodeRegex = /<(XCUIElementType\w+)\s([^>]*)\/>/g;
    let match: RegExpExecArray | null;
    while ((match = nodeRegex.exec(xmlSource)) !== null) {
      const typeName = match[1] ?? '';
      if (!MAC_INTERACTIVE.has(typeName)) continue;
      const attrs = match[2] ?? '';
      const get = (name: string) => new RegExp(`\\b${name}="([^"]*)"`, 'i').exec(attrs)?.[1] ?? '';
      if (get('enabled') === 'false') continue;
      const label = get('title') || get('label') || get('value') || '';
      if (!label) continue;
      const x = parseFloat(get('x') || '0');
      const y = parseFloat(get('y') || '0');
      const width = parseFloat(get('width') || '0');
      const height = parseFloat(get('height') || '0');
      const role = macTypeToRole(typeName);
      elements.push({ role, label, bounds: { x, y, width, height }, platform: 'macos' as any, className: typeName });
    }
  }

  if (platform === 'android') {
    // Match interactive node elements
    const INTERACTIVE_CLASSES = new Set([
      'android.widget.Button',
      'android.widget.EditText',
      'android.widget.CheckBox',
      'android.widget.RadioButton',
      'android.widget.Switch',
      'android.widget.Spinner',
      'android.widget.TextView',    // tappable text
      'android.widget.ImageButton',
      'android.widget.ToggleButton',
      'androidx.recyclerview.widget.RecyclerView',
      'com.google.android.material.bottomnavigation.BottomNavigationItemView',
      'android.widget.ImageView',  // tappable images (nav icons)
    ]);

    // Simple regex-based node extraction — avoids adding an XML parser dependency.
    // This is intentionally minimal; a full implementation would use a proper XML parser.
    const nodeRegex = /<node\s([^>]*)\/?>/g;
    let match: RegExpExecArray | null;
    while ((match = nodeRegex.exec(xmlSource)) !== null) {
      const attrs = match[1] ?? '';
      const get = (name: string): string => {
        const m = new RegExp(`\\b${name}="([^"]*)"`, 'i').exec(attrs);
        return m ? m[1] ?? '' : '';
      };

      const className = get('class');
      const clickable = get('clickable') === 'true';
      const enabled = get('enabled') === 'true';

      if (!enabled) continue;
      if (!clickable && !INTERACTIVE_CLASSES.has(className)) continue;

      const label = get('content-desc') || get('text') || get('resource-id').split('/').pop() || '';
      if (!label) continue;

      const resourceId = get('resource-id');
      const boundsStr = get('bounds'); // "[x1,y1][x2,y2]"
      const bounds = parseAndroidBounds(boundsStr);
      if (!bounds) continue;

      const role = classToRole(className);
      elements.push({ role, label, bounds, platform: 'android', resourceId, className });
    }
  } else {
    // iOS — XCUITest element types
    const INTERACTIVE_TYPES = new Set([
      'XCUIElementTypeButton',
      'XCUIElementTypeTextField',
      'XCUIElementTypeSecureTextField',
      'XCUIElementTypeSwitch',
      'XCUIElementTypeSlider',
      'XCUIElementTypeCheckBox',
      'XCUIElementTypeRadioButton',
      'XCUIElementTypePickerWheel',
      'XCUIElementTypeSearchField',
      'XCUIElementTypeCell',
      'XCUIElementTypeLink',
      'XCUIElementTypeTabBarItem',
      'XCUIElementTypeNavigationBar',
    ]);

    const nodeRegex = /<(XCUIElementType\w+)\s([^>]*)\/>/g;
    let match: RegExpExecArray | null;
    while ((match = nodeRegex.exec(xmlSource)) !== null) {
      const typeName = match[1] ?? '';
      if (!INTERACTIVE_TYPES.has(typeName)) continue;

      const attrs = match[2] ?? '';
      const get = (name: string): string => {
        const m = new RegExp(`\\b${name}="([^"]*)"`, 'i').exec(attrs);
        return m ? m[1] ?? '' : '';
      };

      if (get('enabled') === 'false') continue;

      const label = get('label') || get('name') || get('value') || '';
      if (!label) continue;

      const x = parseFloat(get('x') || '0');
      const y = parseFloat(get('y') || '0');
      const width = parseFloat(get('width') || '0');
      const height = parseFloat(get('height') || '0');
      if (width === 0 && height === 0) continue;

      const role = xcuiTypeToRole(typeName);
      elements.push({ role, label, bounds: { x, y, width, height }, platform: 'ios', className: typeName });
    }
  }

  return elements;
}

/** Parse Android bounds string "[x1,y1][x2,y2]" → NativeBounds */
function parseAndroidBounds(boundsStr: string): NativeBounds | null {
  const m = /\[(\d+),(\d+)\]\[(\d+),(\d+)\]/.exec(boundsStr);
  if (!m) return null;
  const x1 = parseInt(m[1] ?? '0', 10);
  const y1 = parseInt(m[2] ?? '0', 10);
  const x2 = parseInt(m[3] ?? '0', 10);
  const y2 = parseInt(m[4] ?? '0', 10);
  return { x: x1, y: y1, width: x2 - x1, height: y2 - y1 };
}

/** Map Android widget class names to ZeTa element roles */
function classToRole(className: string): string {
  if (className.includes('Button'))   return 'button';
  if (className.includes('EditText')) return 'input';
  if (className.includes('CheckBox')) return 'checkbox';
  if (className.includes('RadioButton')) return 'radio';
  if (className.includes('Switch') || className.includes('Toggle')) return 'switch';
  if (className.includes('Spinner')) return 'select';
  if (className.includes('TextView')) return 'text';
  return className.split('.').pop()?.toLowerCase() ?? 'unknown';
}

/** Map XCUITest element type strings to ZeTa element roles */
function xcuiTypeToRole(xcuiType: string): string {
  if (xcuiType.includes('Button'))        return 'button';
  if (xcuiType.includes('TextField'))     return 'input';
  if (xcuiType.includes('SecureTextField')) return 'input';
  if (xcuiType.includes('Switch'))        return 'switch';
  if (xcuiType.includes('Slider'))        return 'slider';
  if (xcuiType.includes('CheckBox'))      return 'checkbox';
  if (xcuiType.includes('RadioButton'))   return 'radio';
  if (xcuiType.includes('PickerWheel'))   return 'select';
  if (xcuiType.includes('SearchField'))   return 'input';
  return xcuiType.replace('XCUIElementType', '').toLowerCase();
}

/** Map WinAppDriver UIAutomation element type names to ZeTa element roles */
function winTypeToRole(winType: string): string {
  if (winType === 'Button')      return 'button';
  if (winType === 'Edit')        return 'input';
  if (winType === 'ComboBox')    return 'select';
  if (winType === 'CheckBox')    return 'checkbox';
  if (winType === 'RadioButton') return 'radio';
  if (winType === 'Slider')      return 'slider';
  if (winType === 'ListItem')    return 'option';
  if (winType === 'MenuItem')    return 'menuitem';
  if (winType === 'Hyperlink')   return 'link';
  return winType.toLowerCase();
}

/** Map Mac2Driver XCUIElementType strings to ZeTa element roles */
function macTypeToRole(xcuiType: string): string {
  if (xcuiType.includes('Button'))          return 'button';
  if (xcuiType.includes('SecureTextField')) return 'input';
  if (xcuiType.includes('TextField'))       return 'input';
  if (xcuiType.includes('CheckBox'))        return 'checkbox';
  if (xcuiType.includes('RadioButton'))     return 'radio';
  if (xcuiType.includes('PopUpButton'))     return 'select';
  if (xcuiType.includes('ComboBox'))        return 'select';
  if (xcuiType.includes('Slider'))          return 'slider';
  if (xcuiType.includes('MenuItem'))        return 'menuitem';
  if (xcuiType.includes('SearchField'))     return 'input';
  if (xcuiType.includes('Link'))            return 'link';
  return xcuiType.replace('XCUIElementType', '').toLowerCase();
}

// ─── Gap 1: Native Auth ──────────────────────────────────────────────────────

/** Detect if the current screen is a login/auth screen and fill credentials. */
async function attemptNativeLogin(
  driver: any,
  platform: 'android' | 'ios' | 'windows' | 'macos',
  credentials: NonNullable<NativeCrawlerOptions['credentials']>,
): Promise<boolean> {
  try {
    const pageXml: string = await driver.getPageSource();
    const elements = parseAccessibilityTree(pageXml, platform);

    const inputEls = elements.filter((e) => e.role === 'input');
    if (inputEls.length === 0) return false;

    // Detect login screen heuristic: has at least one text input and one
    // "password" input (SecureTextField on iOS, password type on Android)
    const isPasswordField = (e: NativeElement) =>
      e.className?.toLowerCase().includes('secure') ||
      e.className?.toLowerCase().includes('password') ||
      e.label?.toLowerCase().match(/password|passcode|pin|secret/i) != null;

    const hasPasswordInput = elements.some(isPasswordField);
    if (!hasPasswordInput && inputEls.length < 2) return false;

    logger.info({ platform }, 'Native auth: login screen detected — filling credentials');

    // Fill username field: first non-password input
    const unLabel = credentials.usernameFieldLabel?.toLowerCase() ?? '';
    const usernameEl = inputEls.find((e) => {
      if (isPasswordField(e)) return false;
      if (unLabel) return e.label?.toLowerCase().includes(unLabel);
      return e.label?.toLowerCase().match(/email|username|user|login|phone|mobile/i) != null || true;
    }) ?? inputEls[0];

    if (usernameEl && credentials.username) {
      const uSelector = usernameEl.resourceId
        ? `id=${usernameEl.resourceId}`
        : `~${usernameEl.label}`;
      try {
        const uEl = await driver.$(uSelector);
        await uEl.click();
        await driver.pause(300);
        await uEl.clearValue();
        await uEl.setValue(credentials.username);
      } catch { /* non-fatal */ }
    }

    // Fill password field
    const pwEl = elements.find(isPasswordField);
    if (pwEl && credentials.password) {
      const pSelector = pwEl.resourceId ? `id=${pwEl.resourceId}` : `~${pwEl.label}`;
      try {
        const pEl = await driver.$(pSelector);
        await pEl.click();
        await driver.pause(300);
        await pEl.clearValue();
        await pEl.setValue(credentials.password);
      } catch { /* non-fatal */ }
    }

    // Dismiss keyboard then tap login button
    try { await driver.hideKeyboard(); } catch { /* not always available */ }
    await driver.pause(500);

    const loginLabel = credentials.loginButtonLabel ?? '';
    const loginBtn = elements.find((e) =>
      e.role === 'button' &&
      (loginLabel
        ? e.label?.toLowerCase().includes(loginLabel.toLowerCase())
        : e.label?.toLowerCase().match(/log\s?in|sign\s?in|submit|continue|next|connect/i) != null)
    );
    if (loginBtn) {
      const btnSelector = loginBtn.resourceId ? `id=${loginBtn.resourceId}` : `~${loginBtn.label}`;
      try {
        await driver.$(btnSelector).click();
        await driver.pause(3000); // wait for auth response
        logger.info({ platform }, 'Native auth: login button tapped — waiting for app');
        return true;
      } catch { /* non-fatal */ }
    }
    return false;
  } catch (err) {
    logger.warn({ err }, 'Native auth attempt failed — continuing without login');
    return false;
  }
}

// ─── Gap 3: Gesture Navigation ───────────────────────────────────────────────

/** Perform a swipe gesture on the screen. */
async function performSwipe(
  driver: any,
  platform: 'android' | 'ios' | 'windows' | 'macos',
  direction: 'left' | 'right' | 'up' | 'down',
): Promise<void> {
  try {
    const size: { width: number; height: number } = await driver.getWindowSize();
    const { width: W, height: H } = size;
    const midX = Math.round(W / 2);
    const midY = Math.round(H / 2);

    if (platform === 'android') {
      // Use W3C actions for Android
      const startX = direction === 'left' ? Math.round(W * 0.8) : direction === 'right' ? Math.round(W * 0.2) : midX;
      const startY = direction === 'up' ? Math.round(H * 0.8) : direction === 'down' ? Math.round(H * 0.2) : midY;
      const endX = direction === 'left' ? Math.round(W * 0.2) : direction === 'right' ? Math.round(W * 0.8) : midX;
      const endY = direction === 'up' ? Math.round(H * 0.2) : direction === 'down' ? Math.round(H * 0.8) : midY;
      await driver.touchAction([
        { action: 'press', x: startX, y: startY },
        { action: 'wait', ms: 300 },
        { action: 'moveTo', x: endX, y: endY },
        { action: 'release' },
      ]);
    } else if (platform === 'ios') {
      await driver.execute('mobile: swipe', { direction });
    } else {
      // Windows/macOS: use pointer actions
      await driver.execute('mobile: scroll', { direction });
    }
    await driver.pause(500);
  } catch { /* gesture not supported on this platform/driver — non-fatal */ }
}

/** Long-press an element by its selector. */
async function performLongPress(driver: any, selector: string, durationMs = 1200): Promise<void> {
  try {
    const el = await driver.$(selector);
    await driver.touchAction([
      { action: 'longPress', element: el, ms: durationMs },
      { action: 'release' },
    ]);
    await driver.pause(600);
  } catch { /* non-fatal */ }
}

// ─── Gap 2: Vision Fallback ───────────────────────────────────────────────────

/** When the accessibility tree yields zero interactive elements, use LLM vision to describe the screen. */
async function visionFallbackDescribeScreen(
  screenshotBase64: string,
  llmConfig: { provider: string; model: string; apiKey?: string; baseUrl?: string },
): Promise<{ screenDescription: string; suggestedElements: Array<{ role: string; label: string }> }> {
  try {
    const Anthropic = (await import('@anthropic-ai/sdk')).default;
    if (!llmConfig.apiKey) throw new Error('No LLM API key configured. Configure an Anthropic key in Workspace → Settings → AI Provider.');
    const client = new Anthropic({ apiKey: llmConfig.apiKey });
    const response = await client.messages.create({
      model: llmConfig.model ?? 'claude-haiku-4-5-20251001',
      max_tokens: 512,
      messages: [{
        role: 'user',
        content: [
          { type: 'image', source: { type: 'base64', media_type: 'image/png', data: screenshotBase64 } },
          { type: 'text', text: 'This is a screenshot from a mobile app. The accessibility tree returned no elements. Describe what you see on screen in 1-2 sentences, then list ALL interactive UI elements (buttons, inputs, links, toggles) as JSON: [{"role":"button","label":"Sign In"},...]' },
        ],
      }],
    });
    const text = (response.content[0] as any)?.text ?? '';
    const jsonMatch = text.match(/\[[\s\S]*\]/);
    const elements = jsonMatch ? JSON.parse(jsonMatch[0]) : [];
    return { screenDescription: text.split('[')[0]?.trim() ?? '', suggestedElements: elements };
  } catch (err) {
    logger.warn({ err }, 'Vision fallback LLM call failed');
  }
  return { screenDescription: 'Screen captured (accessibility tree empty — vision fallback unavailable)', suggestedElements: [] };
}

// ─── Gap 4: MITM Proxy API Capture ───────────────────────────────────────────

/** Start a lightweight proxy server to intercept native app API traffic. */
async function startMitmProxy(port: number): Promise<{ captured: Array<{ method: string; url: string; status?: number }>; stop: () => void }> {
  const captured: Array<{ method: string; url: string; status?: number }> = [];
  try {
    const net = await import('net');
    // Simple TCP intercept marker — full MITM requires SSL cert installation on device.
    // This records connection attempts; full response interception requires device trust cert.
    const server = net.createServer((socket) => {
      socket.once('data', (data) => {
        const raw = data.toString('utf8', 0, 512);
        const match = /^(GET|POST|PUT|DELETE|PATCH|HEAD)\s+(https?:\/\/[^\s]+)/i.exec(raw);
        if (match) captured.push({ method: match[1]!, url: match[2]! });
        socket.destroy();
      });
    });
    server.listen(port, '127.0.0.1');
    const stop = () => server.close();
    logger.info({ port }, 'MITM proxy listener started (connection-level only — install device trust cert for full interception)');
    return { captured, stop };
  } catch (err) {
    logger.warn({ err }, 'MITM proxy start failed — API capture disabled');
    return { captured, stop: () => {} };
  }
}

/** Update the PlatformJob status/progress in DB — non-fatal. */
async function updateProgress(tenantId: string, jobId: string, progress: number, step: string): Promise<void> {
  try {
    await withTenant(tenantId, (tx) =>
      tx.platformJob.update({
        where: { id: jobId },
        data: { progress, result: { step } as any },
      })
    );
  } catch (err) {
    logger.warn({ err }, 'Failed to update job progress (non-fatal)');
  }
}

// ---------------------------------------------------------------------------
// Main export
// ---------------------------------------------------------------------------

/**
 * Run a native app crawl session via Appium.
 *
 * Flow:
 *   1. Download APK/IPA from S3 to local temp dir
 *   2. Build Appium WebDriverIO capabilities for the target platform
 *   3. Create WebDriverIO remote session (Appium server at APPIUM_SERVER_URL)
 *   4. For each discovered screen:
 *      a. Get page source XML → parse elements
 *      b. Take screenshot → save to screenshotDir → persist via AppBrain
 *      c. Save screen + elements to DB
 *      d. Tap nav elements to discover next screens
 *   5. Update PlatformJob as DONE (or FAILED on error)
 */
export async function runNativeCrawl(opts: NativeCrawlerOptions): Promise<void> {
  const { tenantId, projectId, jobId, platform } = opts;
  const screenshotDir = path.join(config.screenshotDir, projectId);
  await fs.mkdir(screenshotDir, { recursive: true });

  let tmpAppPath: string | null = null;

  // Mark job RUNNING
  await withTenant(tenantId, (tx) =>
    tx.platformJob.update({
      where: { id: jobId },
      data: { status: 'RUNNING', startedAt: new Date() },
    })
  );

  const tenantStorageCfg = await getTenantStorageConfig(tenantId).catch(() => null);

  try {
    // ── Step 1: Download app binary from S3 / R2 ────────────────────────────
    await updateProgress(tenantId, jobId, 5, 'Downloading app binary');
    tmpAppPath = await downloadAppFromS3(opts.appS3Path, platform, opts, tenantStorageCfg);

    // ── Step 2: Build Appium capabilities ───────────────────────────────────
    await updateProgress(tenantId, jobId, 10, 'Building Appium session capabilities');
    const capabilities = buildCapabilities(opts, tmpAppPath);

    // ── Step 3: Start WebDriverIO session ───────────────────────────────────
    // TODO: add "webdriverio": "^9.x" to packages/crawler/package.json
    // TODO: requires Appium server running at APPIUM_SERVER_URL env var
    await updateProgress(tenantId, jobId, 15, 'Starting Appium session');

    const appiumServerUrl = opts.appiumServerUrl ?? process.env.APPIUM_SERVER_URL ?? 'http://127.0.0.1:4723';

    // Pre-check connectivity before wdio throws an opaque EAI_AGAIN
    try {
      const probe = await fetch(`${appiumServerUrl}/status`, { signal: AbortSignal.timeout(5000) });
      if (!probe.ok) throw new Error(`HTTP ${probe.status}`);
    } catch (e: any) {
      throw new Error(
        `Appium server at ${appiumServerUrl} is not reachable (${e.message}). ` +
        `Start it with: cd /opt/zeta-ai && docker compose -f docker-compose.prod.yml -f docker-compose.appium.yml up -d appium`
      );
    }

    // TODO: add "webdriverio": "^9.x" to packages/crawler/package.json before deploying.
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const wdio: any = await import('webdriverio' as any);
    const driver = await wdio.remote({
      hostname: new URL(appiumServerUrl).hostname,
      port: parseInt(new URL(appiumServerUrl).port || '4723', 10),
      path: '/',
      capabilities: capabilities as any,
      logLevel: 'warn',
    });

    logger.info({ platform, deviceFarm: opts.deviceFarm }, 'Appium session started');

    try {
      // Update DB with Appium session ID
      const sessionId: string = driver.sessionId;
      await withTenant(tenantId, (tx) =>
        (tx as any).platformJob.update({
          where: { id: jobId },
          data: { appiumSessionId: sessionId },
        })
      );

      // ── Step 4: Crawl screens ─────────────────────────────────────────────
      await updateProgress(tenantId, jobId, 20, 'App launched — extracting initial screen');

      // Gap 4: Start MITM proxy for API traffic capture
      let mitmStop: (() => void) | null = null;
      const mitmCaptured: Array<{ method: string; url: string; status?: number }> = [];
      if (opts.mitmProxyPort) {
        const mitm = await startMitmProxy(opts.mitmProxyPort);
        mitmStop = mitm.stop;
        mitmCaptured.push(...mitm.captured);
      }

      // Allow the app to reach its initial state
      await driver.pause(2000);

      // Gap 1: Attempt native login before BFS begins
      if (opts.credentials?.username && opts.credentials?.password) {
        const loggedIn = await attemptNativeLogin(driver, platform, opts.credentials);
        if (loggedIn) await driver.pause(2500); // wait for post-login navigation
      }

      // Capture launch screen then navigate forward via discoverable nav elements.
      const MAX_SCREENS = 30;
      let screensFound = 0;
      const tappedSelectors = new Set<string>();

      for (let i = 0; i < MAX_SCREENS; i++) {
        const pct = 20 + Math.round((i / MAX_SCREENS) * 60);
        await updateProgress(tenantId, jobId, pct, `Capturing screen ${i + 1}`);

        // a. Get screen title / name heuristic
        let screenName = `Screen ${i + 1}`;
        try {
          // Android: use current activity name; iOS: use app title from accessibility tree
          // Windows/macOS: use driver.getTitle() for the window title
          if (platform === 'android') {
            const activity: string = await (driver as any).getCurrentActivity();
            screenName = activity.split('.').pop() ?? screenName;
          } else if (platform === 'ios') {
            // iOS: attempt to read the navigation bar title
            const titleEl = await driver.$('~NavigationBarTitle').catch(() => null);
            if (titleEl) {
              const titleText = await (titleEl as any).getText().catch(() => '');
              if (titleText) screenName = titleText;
            }
          } else {
            // windows / macos: use window title
            const winTitle: string = await driver.getTitle().catch(() => '');
            if (winTitle) screenName = winTitle;
          }
        } catch { /* non-fatal — use default name */ }

        // b. Take screenshot
        const screenshotBase64: string = await driver.takeScreenshot();
        const screenshotBuffer = Buffer.from(screenshotBase64, 'base64');

        // DPR normalization: resize physical pixels → logical pixels
        let screenshotBuf = screenshotBuffer;
        try {
          const sharp = (await import('sharp')).default;
          const meta = await sharp(screenshotBuffer).metadata();
          const logicalSize: { width: number; height: number } = await (driver as any).getWindowSize().catch(() => ({ width: meta.width ?? 0, height: meta.height ?? 0 }));
          const physW = meta.width ?? 0;
          const logW = logicalSize.width;
          if (physW > 0 && logW > 0 && physW > logW) {
            const logH = Math.round((meta.height ?? physW) * (logW / physW));
            screenshotBuf = Buffer.from(await sharp(screenshotBuffer).resize(logW, logH, { fit: 'fill' }).png().toBuffer());
          }
        } catch { /* sharp unavailable or resize failed — use raw buffer */ }

        // Save screen record first (to get screen.id)
        const screen = await AppBrain.saveScreen(tenantId, projectId, screenName);
        const shotPath = path.join(screenshotDir, `${screen.id}-native-v${screen.version}.png`);
        await fs.writeFile(shotPath, screenshotBuf);
        let cloudUrl: string | undefined;
        if (tenantStorageCfg) {
          const s3Key = `${tenantId}/${projectId}/screenshots/${screen.id}-native-v${screen.version}.png`;
          cloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, shotPath, s3Key).catch(() => undefined);
        }
        await AppBrain.saveScreenshot(tenantId, projectId, screen.id, shotPath, screen.version, 'NATIVE', cloudUrl);

        // c. Parse accessibility tree → save elements
        // Scroll down to reveal below-fold elements, then scroll back up before parsing
        try {
          if (platform === 'android') {
            await driver.execute('mobile: scroll', { direction: 'down' });
          } else if (platform === 'ios') {
            await driver.execute('mobile: scroll', { direction: 'down', element: null });
          }
          await driver.pause(400);
        } catch { /* scroll not always available — non-fatal */ }

        const pageXml: string = await driver.getPageSource();
        const nativeElements = parseAccessibilityTree(pageXml, platform);
        try {
          if (platform === 'android') {
            await driver.execute('mobile: scroll', { direction: 'up' });
          } else if (platform === 'ios') {
            await driver.execute('mobile: scroll', { direction: 'up', element: null });
          }
          await driver.pause(300);
        } catch { /* non-fatal */ }
        logger.info({ screenName, elements: nativeElements.length }, 'Native screen parsed');

        // Gap 2: Vision fallback when accessibility tree is empty (Flutter / game engines)
        if (nativeElements.length === 0 && opts.visionLlmConfig) {
          logger.info({ screenName }, 'Empty accessibility tree — triggering vision LLM fallback');
          const { suggestedElements } = await visionFallbackDescribeScreen(screenshotBase64, opts.visionLlmConfig);
          for (const el of suggestedElements) {
            nativeElements.push({
              role: el.role, label: el.label, resourceId: '', className: '',
              bounds: { x: 0, y: 0, width: 0, height: 0 }, platform,
            });
          }
        }

        for (const el of nativeElements) {
          const boundingRect = {
            x: Math.round(el.bounds.x),
            y: Math.round(el.bounds.y),
            width: Math.round(el.bounds.width),
            height: Math.round(el.bounds.height),
          };
          const notes = JSON.stringify({
            ...(el.resourceId ? { resourceId: el.resourceId } : {}),
            ...(el.className ? { className: el.className } : {}),
            platform: el.platform,
          });
          await AppBrain.saveElement(
            tenantId,
            projectId,
            screen.id,
            el.label,       // meaning
            el.role,
            undefined,      // expectedData
            notes,
            undefined,      // text
            true,           // isInteractive
            boundingRect,
            undefined,      // ariaState
            undefined,      // parentLandmark
            'NATIVE',       // viewport
          );
        }

        screensFound++;

        // d. Navigate to next screen — basic BFS via nav-like elements
        const navElements = nativeElements.filter(el =>
          ['button', 'text'].includes(el.role) ||
          el.label?.match(/menu|home|back|next|continue|tab|nav|dashboard|settings|profile|more|list|search|help|about/i)
        );

        for (const navEl of navElements.slice(0, 15)) {
          if (screensFound >= MAX_SCREENS) break;
          try {
            // Build a selector for this element
            let selector: string;
            if (platform === 'android' && navEl.resourceId) {
              selector = `id=${navEl.resourceId}`;
            } else if (navEl.label) {
              selector = `~${navEl.label}`;
            } else {
              continue;
            }

            if (tappedSelectors.has(selector)) continue;
            tappedSelectors.add(selector);

            // Capture pre-tap screen identity
            let preActivity = '';
            try {
              if (platform === 'android') {
                preActivity = await (driver as any).getCurrentActivity();
              } else {
                preActivity = await driver.getTitle();
              }
            } catch { /* non-fatal */ }

            await driver.$(selector).click();
            await driver.pause(1500);

            // Android crash detection via logcat
            if (platform === 'android') {
              try {
                const logs: any[] = await (driver as any).getLogs('logcat');
                const crashLog = logs.find((l: any) =>
                  typeof l.message === 'string' && /FATAL EXCEPTION|Process.*has died|ANR in/i.test(l.message)
                );
                if (crashLog) {
                  logger.error({ jobId, crash: String(crashLog.message).slice(0, 500) }, 'App crash detected during crawl');
                  break; // stop navigating from this screen
                }
              } catch { /* logcat not available on all environments — non-fatal */ }
            }
            // iOS crash log detection
            if (platform === 'ios') {
              const crashLogs: any[] = await (driver as any).getLogs('crashlog').catch(() => []);
              if (crashLogs.length > 0 && crashLogs.some((l: any) => l.message?.includes('Exception') || l.message?.includes('crash'))) {
                logger.warn({ screenName }, 'iOS crash log detected — stopping crawl');
                break;
              }
            }

            // Check if screen changed
            let postActivity = '';
            try {
              if (platform === 'android') {
                postActivity = await (driver as any).getCurrentActivity();
              } else {
                postActivity = await driver.getTitle();
              }
            } catch { /* non-fatal */ }

            if (postActivity === preActivity) continue; // same screen — skip

            // Check MAX_SCREENS before saving the new screen
            if (screensFound >= MAX_SCREENS) break;

            // New screen — capture it
            const newScreenName = postActivity
              ? postActivity.split('.').pop() ?? `Screen ${screensFound + 1}`
              : `Screen ${screensFound + 1}`;
            const newScreenshot: string = await driver.takeScreenshot();
            const newScreen = await AppBrain.saveScreen(tenantId, projectId, newScreenName);
            const newShotPath = path.join(screenshotDir, `${newScreen.id}-native-v${newScreen.version}.png`);
            await fs.writeFile(newShotPath, Buffer.from(newScreenshot, 'base64'));
            let newCloudUrl: string | undefined;
            if (tenantStorageCfg) {
              const s3Key = `${tenantId}/${projectId}/screenshots/${newScreen.id}-native-v${newScreen.version}.png`;
              newCloudUrl = await uploadScreenshotToTenantStorage(tenantStorageCfg, newShotPath, s3Key).catch(() => undefined);
            }
            await AppBrain.saveScreenshot(tenantId, projectId, newScreen.id, newShotPath, newScreen.version, 'NATIVE', newCloudUrl);

            const newXml: string = await driver.getPageSource();
            const newElements = parseAccessibilityTree(newXml, platform);
            for (const el of newElements) {
              await AppBrain.saveElement(
                tenantId, projectId, newScreen.id,
                el.label, el.role, undefined,
                JSON.stringify({ ...(el.resourceId ? { resourceId: el.resourceId } : {}), ...(el.className ? { className: el.className } : {}), platform: el.platform }),
                undefined, true,
                { x: Math.round(el.bounds.x), y: Math.round(el.bounds.y), width: Math.round(el.bounds.width), height: Math.round(el.bounds.height) },
                undefined, undefined, 'NATIVE',
              );
            }
            screensFound++;

            // Navigate back so subsequent nav attempts start from a known state
            try { await driver.back(); await driver.pause(800); } catch { /* non-fatal */ }
          } catch { /* element may have disappeared or tap failed — skip */ }
        }

        // Gap 3: Gesture-based discovery — swipe to reveal off-screen content
        if (opts.gestureConfig?.enableSwipe && screensFound < MAX_SCREENS) {
          for (const dir of ['left', 'right', 'up', 'down'] as const) {
            await performSwipe(driver, platform, dir);
            const postSwipeXml = await driver.getPageSource().catch(() => '');
            if (postSwipeXml && postSwipeXml !== pageXml) break; // new content found
          }
        }

        // Nav pass complete for this screen — continue BFS to next iteration.
      }

      // ── Step 5: Mark job DONE ─────────────────────────────────────────────
      if (mitmStop) mitmStop();
      await driver.deleteSession();
      await updateProgress(tenantId, jobId, 99, `Crawl complete — ${screensFound} screen(s) captured`);
      await withTenant(tenantId, (tx) =>
        tx.platformJob.update({
          where: { id: jobId },
          data: {
            status: 'DONE',
            progress: 100,
            finishedAt: new Date(),
            result: { screensFound, platform } as any,
          },
        })
      );
      logger.info({ jobId, screensFound, platform }, 'Native crawl complete');

    } catch (sessionErr) {
      // Best-effort: clean up Appium session before re-throwing
      try { await driver.deleteSession(); } catch { /* ignore */ }
      throw sessionErr;
    }

  } catch (err: any) {
    logger.error({ jobId, err: String(err) }, 'Native crawl failed');
    await withTenant(tenantId, (tx) =>
      tx.platformJob.update({
        where: { id: jobId },
        data: {
          status: 'FAILED',
          error: String(err?.message ?? err),
          finishedAt: new Date(),
        },
      })
    ).catch(() => {});
    throw err;
  } finally {
    // Clean up local temp file
    if (tmpAppPath) {
      fs.unlink(tmpAppPath).catch(() => {});
    }
  }
}