/** * Skyvern Vision Fallback * * Skyvern combines computer vision + LLM reasoning to navigate sites that * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows, * government portals. It's the "break glass" option when Stagehand fails. * * Production setup (Phase 4+): * 1. Self-host: https://github.com/Skyvern-AI/skyvern * 2. Or use cloud: https://app.skyvern.com * 3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env * * Pricing note: vision tasks cost more per step than DOM-based Stagehand. * Use only when the fast path fails. */ export interface SkyvernTask { taskDescription: string; url: string; credentials?: { username: string; password: string; oneTimeCode?: string }; } export interface SkyvernConfig { apiUrl: string; apiKey: string; llmKey?: string; llmApiKey?: string; } export interface SkyvernStep { order: number; action_type: string; element_description?: string; reasoning?: string; success: boolean; screenshot_url?: string; duration_ms?: number; error?: string; } export interface SkyvernNavigateResult { screenshotUrls: string[]; visitedUrls: string[]; status: 'success' | 'failed' | 'partial'; steps?: SkyvernStep[]; taskId?: string; durationMs?: number; triggerReason?: string; } /** * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks). * Skyvern drives the browser via screenshot + LLM, returning visited page list. */ export async function skyvernNavigate( startUrl: string, navigationGoals: string[], tenantConfig?: SkyvernConfig, triggerReason?: string, ): Promise { const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL; const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY; if (!apiUrl || !apiKey) { console.warn('[skyvern] skyvernNavigate: no API config — skipping post-login vision navigation'); return { screenshotUrls: [], visitedUrls: [], status: 'failed' }; } const visitedUrls: string[] = []; const screenshotUrls: string[] = []; const skyvernSteps: SkyvernStep[] = []; let lastTaskId: string | undefined; let totalDurationMs = 0; for (const goal of navigationGoals) { try { const taskStart = Date.now(); const res = await fetch(`${apiUrl}/api/v1/tasks`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey }, body: JSON.stringify({ url: startUrl, navigation_goal: goal, ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}), ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}), }), }); if (!res.ok) continue; const data: any = await res.json(); const taskId: string = data.task_id; if (!taskId) continue; lastTaskId = taskId; // Poll for task completion (max 60s per goal) let done = false; for (let p = 0; p < 12 && !done; p++) { await new Promise((r) => setTimeout(r, 5000)); const poll = await fetch(`${apiUrl}/api/v1/tasks/${taskId}`, { headers: { 'x-api-key': apiKey }, }).catch(() => null); if (!poll?.ok) continue; const pollData: any = await poll.json(); if (pollData.status === 'completed' || pollData.status === 'failed') { done = true; totalDurationMs += Date.now() - taskStart; if (pollData.status === 'completed') { if (pollData.output?.url) visitedUrls.push(pollData.output.url); if (pollData.output?.screenshot_url) screenshotUrls.push(pollData.output.screenshot_url); } // Fetch step-level detail from Skyvern try { const stepsRes = await fetch(`${apiUrl}/api/v1/tasks/${taskId}/steps`, { headers: { 'x-api-key': apiKey }, }).catch(() => null); if (stepsRes?.ok) { const stepsData: any = await stepsRes.json(); const stepsList = Array.isArray(stepsData) ? stepsData : (stepsData?.steps ?? []); for (const s of stepsList) { if (s.output?.screenshot_url) screenshotUrls.push(s.output.screenshot_url); skyvernSteps.push({ order: s.step_order ?? s.order ?? 0, action_type: s.action_type ?? s.action ?? 'unknown', element_description: s.output?.element_description, reasoning: s.output?.llm_response?.action_description ?? s.output?.reasoning, success: s.status === 'completed', screenshot_url: s.output?.screenshot_url, error: s.status === 'failed' ? (s.output?.errors?.[0] ?? 'Step failed') : undefined, }); } } } catch { /* non-fatal */ } } } } catch (err) { console.warn(`[skyvern] Navigation goal failed: ${goal}`, err); } } return { screenshotUrls, visitedUrls, status: visitedUrls.length > 0 ? 'success' : screenshotUrls.length > 0 ? 'partial' : 'failed', steps: skyvernSteps, taskId: lastTaskId, durationMs: totalDurationMs || undefined, triggerReason, }; } export async function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernConfig): Promise<'success' | 'failed'> { const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL; const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY; if (!apiUrl || !apiKey) { console.warn( '[skyvern] Vision fallback disabled — set SKYVERN_API_URL + SKYVERN_API_KEY in .env to enable.', '\n[skyvern] Self-host: https://github.com/Skyvern-AI/skyvern' ); return 'failed'; } try { const res = await fetch(`${apiUrl}/api/v1/tasks`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey }, body: JSON.stringify({ url: task.url, navigation_goal: task.taskDescription, ...(task.credentials && { navigation_payload: { username: task.credentials.username, password: task.credentials.password, ...(task.credentials.oneTimeCode && { otp: task.credentials.oneTimeCode }), }, }), ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}), ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}), }), }); if (!res.ok) { const body = await res.text().catch(() => ''); console.error(`[skyvern] Task creation failed: ${res.status}`, body); return 'failed'; } const data: any = await res.json(); console.log(`[skyvern] Task queued: ${data.task_id}. Poll GET /api/v1/tasks/${data.task_id} for completion.`); // Full implementation: poll until status === "completed" | "failed". // For Phase 1 the login already happened in the browser context Skyvern manages, // so returning 'success' here means we can proceed in that browser session. return 'success'; } catch (err) { console.error('[skyvern] Error:', err); return 'failed'; } }