/** * P6: Gap Pass — LLM-guided second crawl pass * * After the primary BFS crawl completes, this module: * 1. Sends the discovered screen list to an LLM for gap analysis * 2. HEAD-validates each suggested URL * 3. Returns confirmed gap URLs for targeted re-crawl * * The caller (crawler.ts) is responsible for actually visiting the confirmed * URLs with the existing browser — this module only identifies them. */ import { logger as rootLogger } from './logger.js'; const logger = rootLogger.child({ module: 'gap-pass' }); export interface GapCandidate { category: string; urlPatterns: string[]; confidence: number; rationale: string; confirmedUrl: string | null; headStatus: number | null; } interface DiscoveredScreen { url: string; name?: string; elements?: number; } interface LlmCfg { provider: string; model: string; apiKey?: string; baseUrl?: string; } // Only openai and anthropic are supported here. NVIDIA, groq, ollama, etc. → skip, fall to env. /** * Resolve effective LLM. Priority: openai → anthropic → skip unsupported providers. * Falls back to env vars when platform provider is not supported. */ function resolveTextLLM(llmCfg: LlmCfg | null | undefined): { provider: 'openai' | 'anthropic'; apiKey: string; model: string; baseUrl?: string } | null { if (llmCfg?.provider === 'openai' && llmCfg.apiKey) { return { provider: 'openai', apiKey: llmCfg.apiKey, model: llmCfg.model, baseUrl: llmCfg.baseUrl }; } if (llmCfg?.provider === 'anthropic' && llmCfg.apiKey) { return { provider: 'anthropic', apiKey: llmCfg.apiKey, model: llmCfg.model }; } // Unsupported provider (nvidia, groq, etc.) or no key — env fallback, openai first. if (process.env.OPENAI_API_KEY) { return { provider: 'openai', apiKey: process.env.OPENAI_API_KEY, model: 'gpt-4o-mini' }; } if (process.env.ANTHROPIC_API_KEY) { return { provider: 'anthropic', apiKey: process.env.ANTHROPIC_API_KEY, model: 'claude-haiku-4-5-20251001' }; } return null; } /** * Run LLM gap analysis + HEAD validation. * Returns candidates with `confirmedUrl` set for those where HEAD returned 200. */ export async function runGapAnalysis( screens: DiscoveredScreen[], appUrl: string, llmCfg: LlmCfg | null | undefined, ): Promise { const effective = resolveTextLLM(llmCfg); if (!effective || screens.length < 2) return []; const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })(); if (!origin) return []; // Cap context — only send url + name, not full element data const screenList = screens .slice(0, 80) .map(s => { try { return new URL(s.url).pathname + (s.name ? ` (${s.name})` : ''); } catch { return s.url; } }) .join('\n'); const systemPrompt = 'You are a web application coverage analyst. Respond with valid JSON only.'; const userPrompt = `You are analyzing the screen map of a web application to find structural coverage gaps. Discovered screens (${screens.length} total): ${screenList} App URL: ${appUrl} Based on these screens and common patterns for this type of web application, identify screen categories that are likely present but missing. Focus on pages that would exist in a typical SaaS or web app but are not represented above. Respond with ONLY valid JSON, no markdown: {"gaps":[{"category":"User profile edit","urlPatterns":["/profile/edit","/account/profile"],"confidence":0.8,"rationale":"Profile view found but no edit page"},{"category":"Billing history","urlPatterns":["/settings/billing","/account/billing"],"confidence":0.7,"rationale":"Settings section found but no billing page"}]}`; try { let text = ''; if (effective.provider === 'openai') { const { OpenAI } = await import('openai'); const client = new OpenAI({ apiKey: effective.apiKey, ...(effective.baseUrl ? { baseURL: effective.baseUrl } : {}) }); const response = await client.chat.completions.create({ model: effective.model ?? 'gpt-4o-mini', max_tokens: 1024, temperature: 0, messages: [ { role: 'system', content: systemPrompt }, { role: 'user', content: userPrompt }, ], }); text = response.choices[0]?.message?.content ?? ''; } else { const Anthropic = (await import('@anthropic-ai/sdk')).default; const client = new Anthropic({ apiKey: effective.apiKey }); const response = await client.messages.create({ model: effective.model ?? 'claude-haiku-4-5-20251001', max_tokens: 1024, temperature: 0, // @ts-ignore system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }], messages: [{ role: 'user', content: userPrompt }], }); text = (response.content[0] as any)?.text ?? ''; } const jsonMatch = text.match(/\{[\s\S]*\}/); if (!jsonMatch) return []; const parsed = JSON.parse(jsonMatch[0]); const rawGaps: Array<{ category: string; urlPatterns: string[]; confidence: number; rationale: string }> = parsed.gaps ?? []; if (!Array.isArray(rawGaps) || rawGaps.length === 0) return []; logger.info({ gaps: rawGaps.length, provider: effective.provider }, '[P6] Gap analysis: LLM returned candidates'); // HEAD-validate each URL pattern const results: GapCandidate[] = []; for (const gap of rawGaps.slice(0, 15)) { let confirmedUrl: string | null = null; let headStatus: number | null = null; for (const pattern of (gap.urlPatterns ?? []).slice(0, 4)) { const fullUrl = pattern.startsWith('http') ? pattern : `${origin}${pattern.startsWith('/') ? '' : '/'}${pattern}`; try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 6_000); const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null); clearTimeout(timeout); headStatus = res?.status ?? null; if (res?.ok) { confirmedUrl = fullUrl; break; } } catch { /* non-fatal */ } } results.push({ category: gap.category, urlPatterns: gap.urlPatterns, confidence: gap.confidence, rationale: gap.rationale, confirmedUrl, headStatus }); logger.info({ category: gap.category, confirmedUrl, headStatus }, '[P6] Gap candidate checked'); } const confirmed = results.filter(r => r.confirmedUrl).length; logger.info({ total: results.length, confirmed }, '[P6] Gap pass analysis complete'); return results; } catch (err) { logger.warn({ err: String(err) }, '[P6] Gap analysis failed — non-fatal'); return []; } }