/** * Vision Matcher Types * * Types for vision-based element finding using screenshot analysis * with vision models (GPT-4V, Claude Vision). */ /** * Vision model provider type */ export type VisionProvider = 'openai' | 'anthropic'; /** * Configuration for the vision matcher */ export interface VisionMatcherConfig { /** Preferred vision model provider (default: first available) */ provider?: VisionProvider; /** Maximum retries for failed vision requests */ maxRetries?: number; /** Timeout for vision API calls in ms (default: 30000) */ timeoutMs?: number; /** Enable caching of vision results (default: true) */ enableCache?: boolean; /** Maximum cached positions per domain (default: 100) */ maxCachePerDomain?: number; /** Cache TTL in ms (default: 1 hour) */ cacheTtlMs?: number; /** Enable learning from successful matches (default: true) */ enableLearning?: boolean; /** Minimum confidence threshold to accept a match (default: 0.7) */ minConfidence?: number; } /** * Request to find an element using vision */ export interface VisionFindRequest { /** Screenshot as base64 PNG */ screenshotBase64: string; /** Natural language description of the element to find */ elementDescription: string; /** Optional context about the page */ pageContext?: { url?: string; title?: string; domain?: string; }; /** Optional hint about element type */ elementType?: 'button' | 'link' | 'input' | 'text' | 'image' | 'checkbox' | 'radio' | 'dropdown' | 'other'; /** Optional previous selectors that failed (for context) */ failedSelectors?: string[]; } /** * Bounding box for an element */ export interface BoundingBox { /** X coordinate of top-left corner */ x: number; /** Y coordinate of top-left corner */ y: number; /** Width of the element */ width: number; /** Height of the element */ height: number; } /** * Result of a vision-based element find */ export interface VisionFindResult { /** Whether an element was found */ found: boolean; /** Confidence score (0-1) */ confidence: number; /** Bounding box of the found element */ boundingBox?: BoundingBox; /** Center point for clicking */ centerPoint?: { x: number; y: number; }; /** Natural language explanation of the match */ explanation?: string; /** Suggested CSS selector based on visual analysis */ suggestedSelector?: string; /** Alternative matches found */ alternatives?: Array<{ boundingBox: BoundingBox; confidence: number; explanation?: string; }>; /** Time taken for the vision analysis */ durationMs: number; /** Vision provider used */ provider: VisionProvider; /** Error message if failed */ error?: string; } /** * Cached position for a learned element */ export interface LearnedPosition { /** Element description that was searched for */ elementDescription: string; /** Domain where this was found */ domain: string; /** Bounding box from successful find */ boundingBox: BoundingBox; /** Confidence from original match */ confidence: number; /** Suggested selector from vision analysis */ suggestedSelector?: string; /** How many times this position was successfully used */ successCount: number; /** How many times this position failed */ failCount: number; /** Last time this position was used successfully */ lastSuccessTimestamp: number; /** When this position was first learned */ createdTimestamp: number; } /** * Event types emitted by VisionMatcher */ export type VisionMatcherEventType = 'element_found' | 'element_not_found' | 'position_learned' | 'position_used' | 'position_failed' | 'cache_hit' | 'cache_miss' | 'api_error'; /** * Event data for VisionMatcher events */ export interface VisionMatcherEvent { type: VisionMatcherEventType; domain?: string; elementDescription?: string; confidence?: number; boundingBox?: BoundingBox; provider?: VisionProvider; error?: string; durationMs?: number; timestamp: number; } /** * Vision matcher statistics */ export interface VisionMatcherStats { /** Total find requests */ totalRequests: number; /** Successful finds */ successfulFinds: number; /** Failed finds */ failedFinds: number; /** Cache hits */ cacheHits: number; /** Learned positions count */ learnedPositions: number; /** Average confidence of successful matches */ averageConfidence: number; /** Average response time in ms */ averageResponseTime: number; /** Requests by provider */ requestsByProvider: Record; /** Success rate by domain */ successRateByDomain: Record; } /** * Export format for vision matcher learned positions */ export interface VisionMatcherExport { version: '1.0'; exportedAt: string; positions: LearnedPosition[]; stats: VisionMatcherStats; } //# sourceMappingURL=vision-matcher.d.ts.map