/** * Learning Engine - Advanced learning system for the LLM Browser * * Features: * - Temporal confidence decay * - Content structure learning (selector patterns) * - Selector fallback chains * - Failure context learning * - Content change frequency tracking * - Cross-domain pattern transfer * - Response validation * - Pagination pattern detection * * Uses PersistentStore for: * - Debounced writes (batches rapid learning calls) * - Atomic writes (temp file + rename for corruption safety) */ import type { EnhancedApiPattern, EnhancedKnowledgeBaseEntry, SelectorPattern, PaginationPattern, ContentLoadingPatternEntry, DomainGroup, FailureContext, LearningEvent, ConfidenceDecayConfig, ApiPattern, SuccessProfile, AnomalyFalsePositive, PatternSource, KnowledgePack, KnowledgeExportOptions, KnowledgeImportOptions, KnowledgeImportResult, CaptchaMetrics } from '../types/index.js'; import type { AntiPattern, FailureCategory } from '../types/api-patterns.js'; import type { ApiPatternRegistry } from './api-pattern-learner.js'; import type { ContentIntelligence } from './content-intelligence.js'; import type { SemanticPatternMatcher, SimilarPattern } from './semantic-pattern-matcher.js'; import type { PatternHealthConfig, PatternHealthNotification, PatternHealth, HealthCheckOptions, HealthCheckResult } from '../types/pattern-health.js'; import type { WebSocketPattern, WebSocketConnection } from '../types/websocket-patterns.js'; import type { ContentChangePattern, ContentChangeAnalysis, PollRecommendation } from '../types/content-change.js'; /** * Options for learning an API pattern (CX-006) */ export interface LearnApiPatternOptions { /** How this pattern was learned */ source?: PatternSource; /** URL where the pattern was discovered (e.g., OpenAPI spec URL) */ sourceUrl?: string; /** ID of the source pattern (for transferred patterns) */ sourcePatternId?: string; /** Additional metadata about the source */ sourceMetadata?: Record; } export declare class LearningEngine { private entries; private domainGroups; private learningEvents; private store; private decayConfig; private semanticMatcher; /** * Persisted anti-patterns (LI-002) * These are high-confidence anti-patterns that survive restarts */ private antiPatterns; /** * Secondary index for O(1) pattern lookup by domain+pathname (P-002) * Maps `${domain}:${pathname}` to array of patterns (multiple patterns can share pathname) * This avoids O(N) iteration through all patterns for a domain */ private pathnameIndex; /** * Pattern health tracker (FEAT-002) * Monitors learned pattern health over time and detects degradation */ private healthTracker; /** * WebSocket pattern learner (FEAT-003) * Learns WebSocket patterns from captured connections */ private wsPatternLearner; /** * Content change predictor (GAP-011) * Learns content update patterns and predicts when content will change */ private changePredictor; constructor(filePath?: string, decayConfig?: ConfidenceDecayConfig, healthConfig?: Partial); initialize(): Promise; /** * Generate index key for pathname lookup */ private getPathnameIndexKey; /** * Extract pathname from a pattern endpoint, returns null if invalid */ private extractPathnameFromEndpoint; /** * Add a pattern to the pathname index */ private indexPattern; /** * Remove a pattern from the pathname index */ private unindexPattern; /** * Rebuild the entire pathname index from entries * Called after load() or when index integrity is questionable */ private rebuildPathnameIndex; /** * Set the semantic pattern matcher for similarity-based search * This enables findPatternAsync to use semantic search as a fallback */ setSemanticMatcher(matcher: SemanticPatternMatcher | null): void; /** * Check if semantic matching is available */ hasSemanticMatcher(): boolean; /** * Find a pattern matching a URL with semantic fallback * * Search order: * 1. Exact match (fastest) - same pathname * 2. Prefix match - pathname starts with pattern * 3. Semantic match (if matcher available) - similar patterns by embedding * * @param url The URL to find a pattern for * @param options Options for semantic matching * @returns The best matching pattern or null */ findPatternAsync(url: string, options?: { minSimilarity?: number; }): Promise<{ pattern: EnhancedApiPattern | null; matchType: 'exact' | 'prefix' | 'semantic' | 'none'; similarity?: number; }>; /** * Find similar patterns using semantic search * * @param url The URL to find similar patterns for * @param limit Maximum number of results * @param minSimilarity Minimum similarity threshold (0-1) * @returns Array of similar patterns with scores */ findSimilarPatterns(url: string, limit?: number, minSimilarity?: number): Promise; /** * Convert a LearnedPattern to EnhancedApiPattern format */ private convertToEnhancedPattern; /** * Apply confidence decay to all patterns based on time since last verification */ applyConfidenceDecay(): void; private confidenceToNumber; private numberToConfidence; /** * Learn a successful selector for a content type */ learnSelector(domain: string, selector: string, contentType: SelectorPattern['contentType'], urlPattern?: string, semanticSelector?: import('../types/index.js').SemanticSelector): void; /** * Record a selector failure */ recordSelectorFailure(domain: string, selector: string, contentType: SelectorPattern['contentType']): void; /** * Get the best selector chain for a content type * Returns CSS selectors only (legacy method for backward compatibility) * @deprecated Use getSelectorPatterns instead for semantic selector support */ getSelectorChain(domain: string, contentType: SelectorPattern['contentType']): string[]; /** * Get the best selector patterns for a content type (including semantic selectors) */ getSelectorPatterns(domain: string, contentType: SelectorPattern['contentType']): import('../types/index.js').SelectorPattern[]; /** * Record a failure with context for learning */ recordFailure(domain: string, failure: Omit): void; /** * Record a successful fetch with details about what worked * This builds the success profile for a domain */ recordSuccess(domain: string, details: { tier: 'intelligence' | 'lightweight' | 'playwright'; strategy?: string; responseTime: number; contentLength: number; hasStructuredData?: boolean; hasFrameworkData?: boolean; hasBypassableApis?: boolean; }): void; /** * Get the success profile for a domain */ getSuccessProfile(domain: string): SuccessProfile | null; /** * Check if we should use the success profile for a domain * Returns the profile if it's reliable enough to use */ getReliableSuccessProfile(domain: string): SuccessProfile | null; /** * Record lazy-load pattern detection for a domain * This helps decide whether to skip lightweight tier or use scroll simulation on future visits */ recordLazyLoadDetection(domain: string, details: { patterns: string[]; confidence: 'low' | 'medium' | 'high'; lightweightSucceeded?: boolean; }): void; /** * Get lazy-load profile for a domain * Returns learned lazy-loading patterns */ getLazyLoadProfile(domain: string): import('../types/index.js').LazyLoadProfile | null; /** * Check if a domain requires scroll simulation (high-confidence lazy loading) */ requiresScrollSimulation(domain: string): boolean; /** Threshold for CAPTCHA rate to trigger avoidance */ private static readonly CAPTCHA_AVOIDANCE_THRESHOLD; /** How long to avoid a CAPTCHA-heavy domain (24 hours) */ private static readonly CAPTCHA_AVOIDANCE_DURATION_MS; /** Minimum requests before calculating CAPTCHA rate */ private static readonly MIN_REQUESTS_FOR_CAPTCHA_RATE; /** * Record a request to a domain (for CAPTCHA rate calculation) * Call this on every request to track baseline rate */ recordDomainRequest(domain: string): void; /** * Record a CAPTCHA encounter for a domain * This updates the CAPTCHA rate and may trigger avoidance */ recordCaptchaEncounter(domain: string, captchaType?: string): void; /** * Update CAPTCHA rate and avoidance status for a domain entry */ private updateCaptchaRate; /** * Check if a domain should be avoided due to high CAPTCHA rate * Use this before selecting a source to prefer alternatives */ shouldAvoidDueToCaptha(domain: string): boolean; /** * Get CAPTCHA metrics for a domain */ getCaptchaMetrics(domain: string): CaptchaMetrics | null; /** * Get list of domains that should be avoided due to CAPTCHA * Useful for source selection and routing decisions */ getCaptchaAvoidedDomains(): string[]; private static readonly ROLLING_WINDOW_SIZE; private static readonly QUALITY_EXCELLENT_THRESHOLD; private static readonly QUALITY_GOOD_THRESHOLD; private static readonly QUALITY_FAIR_THRESHOLD; private static readonly QUALITY_POOR_THRESHOLD; /** * Record a successful extraction from a domain */ recordExtractionSuccess(domain: string, responseTimeMs: number, completenessScore: number): void; /** * Record a failed request to a domain */ recordExtractionFailure(domain: string, errorType: string, responseTimeMs?: number): void; /** * Initialize source quality metrics with defaults */ private initializeSourceQualityMetrics; /** * Update derived metrics (rates, scores, tiers) */ private updateSourceQualityMetrics; /** * Get source quality metrics for a domain */ getSourceQualityMetrics(domain: string): import('../types/index.js').SourceQualityMetrics | null; /** * Get all domains sorted by quality score * Useful for source selection and routing decisions */ getDomainsByQuality(minScore?: number): Array<{ domain: string; metrics: import('../types/index.js').SourceQualityMetrics; }>; /** * Get domains that should be avoided due to poor quality */ getLowQualityDomains(): string[]; /** * Get a summary of source quality for dashboard display */ getSourceQualitySummary(): { totalDomains: number; byTier: Record; topSources: Array<{ domain: string; score: number; }>; problemSources: Array<{ domain: string; score: number; topError: string; }>; }; /** * Record a false positive in anomaly detection * * Called when anomaly detection triggered but we successfully extracted * substantial content, proving the detection was incorrect. * * @param domain Domain where false positive occurred * @param anomalyType Type of anomaly that was incorrectly detected * @param triggerReasons Reasons that triggered the detection (e.g., "cloudflare") * @param actualContentLength How much content was actually extracted */ recordAnomalyFalsePositive(domain: string, anomalyType: 'challenge_page' | 'error_page' | 'empty_content' | 'redirect_notice' | 'captcha' | 'rate_limited', triggerReasons: string[], actualContentLength: number): void; /** * Check if an anomaly type has been flagged as unreliable for this domain * * Returns true if we should skip or de-weight this anomaly detection * because it has produced false positives before. * * @param domain Domain to check * @param anomalyType Type of anomaly being detected * @returns Whether this detection is unreliable for this domain */ isAnomalyDetectionUnreliable(domain: string, anomalyType: 'challenge_page' | 'error_page' | 'empty_content' | 'redirect_notice' | 'captcha' | 'rate_limited'): boolean; /** * Get all anomaly false positives for a domain */ getAnomalyFalsePositives(domain: string): AnomalyFalsePositive[]; /** * Classify an error into a failure type */ classifyError(error: Error, responseStatus?: number): FailureContext['type']; /** * Get failure patterns for a domain */ getFailurePatterns(domain: string): { mostCommonType: FailureContext['type'] | null; recentFailureRate: number; shouldBackoff: boolean; }; /** * Record a content check and whether it changed */ recordContentCheck(domain: string, urlPattern: string, content: string, changed: boolean): void; /** * Get recommended refresh interval for a URL pattern */ getRecommendedRefreshInterval(domain: string, urlPattern: string): number; private hashContent; /** * Get the domain group for a domain */ getDomainGroup(domain: string): DomainGroup | null; /** * Transfer learned patterns from one domain to another in the same group */ transferPatterns(fromDomain: string, toDomain: string): boolean; /** * Get shared patterns for a domain's group */ getSharedPatterns(domain: string): DomainGroup['sharedPatterns'] | null; /** * Learn validation rules from successful responses */ learnValidator(domain: string, content: string, urlPattern?: string): void; /** * Validate content against learned rules */ validateContent(domain: string, content: string, urlPattern?: string): { valid: boolean; reasons: string[]; }; /** * Universal content anomaly detection - works without prior learning * Detects challenge pages, errors, and suspicious content patterns */ detectContentAnomalies(content: string, url: string, expectedTopic?: string): { isAnomaly: boolean; anomalyType?: 'challenge_page' | 'error_page' | 'empty_content' | 'redirect_notice' | 'captcha' | 'rate_limited'; confidence: number; reasons: string[]; suggestedAction?: 'wait' | 'retry' | 'use_session' | 'change_agent' | 'skip'; waitTimeMs?: number; }; /** * Learn pagination pattern from URL and page content */ learnPaginationPattern(domain: string, urls: string[], pattern: Partial): void; private detectPaginationFromUrls; private extractUrlBase; /** * Get pagination pattern for a URL */ getPaginationPattern(domain: string, url: string): PaginationPattern | null; /** * Learn content loading patterns for a domain */ learnContentLoadingPatterns(domain: string, patterns: ContentLoadingPatternEntry[]): void; /** * Get content loading patterns for a domain */ getContentLoadingPatterns(domain: string): ContentLoadingPatternEntry[]; /** * Get the best content loading pattern for waiting */ getBestContentLoadingPattern(domain: string): ContentLoadingPatternEntry | null; /** * Record success for a content loading pattern */ recordContentLoadingSuccess(domain: string, patternId: string, responseTime: number): void; /** * Record failure for a content loading pattern */ recordContentLoadingFailure(domain: string, patternId: string, reason: string): void; /** * Learn or update an API pattern with enhanced tracking and provenance (CX-006) */ learnApiPattern(domain: string, pattern: ApiPattern, options?: LearnApiPatternOptions): void; /** * Record API pattern verification success */ verifyApiPattern(domain: string, endpoint: string, method: string): void; /** * Record API pattern failure */ recordApiPatternFailure(domain: string, endpoint: string, method: string, failure: Omit): void; /** * Get health status for a specific pattern */ getPatternHealth(domain: string, endpoint: string): PatternHealth | null; /** * Get all patterns with non-healthy status */ getUnhealthyPatterns(): Array<{ domain: string; endpoint: string; health: PatternHealth; }>; /** * Get all recent health notifications */ getHealthNotifications(): PatternHealthNotification[]; /** * Clear all health notifications */ clearHealthNotifications(): void; /** * Perform manual health check for a pattern */ checkPatternHealth(domain: string, endpoint: string, options?: HealthCheckOptions): HealthCheckResult | null; /** * Get health statistics summary */ getHealthStats(): { total: number; healthy: number; degraded: number; failing: number; broken: number; }; /** * Export health data for persistence */ exportHealthData(): Record; /** * Import health data from persistence */ importHealthData(data: Record): void; /** * Learn WebSocket pattern from captured connection */ learnWebSocketPattern(connection: WebSocketConnection, domain: string): void; /** * Get WebSocket patterns for a domain */ getWebSocketPatterns(domain?: string): WebSocketPattern[]; /** * Verify WebSocket pattern (after successful replay) */ verifyWebSocketPattern(domain: string, endpoint: string, protocol: string): void; /** * Record WebSocket pattern failure */ recordWebSocketPatternFailure(domain: string, endpoint: string, protocol: string, failure: Omit): void; private getOrCreateEntry; private recordLearningEvent; /** * Get knowledge base statistics */ getStats(): { totalDomains: number; totalApiPatterns: number; totalSelectors: number; totalValidators: number; bypassablePatterns: number; domainGroups: string[]; recentLearningEvents: LearningEvent[]; }; /** * Get entry for a domain */ getEntry(domain: string): EnhancedKnowledgeBaseEntry | null; /** * Get all domain names that have been learned * (LI-003: Used by learning effectiveness metrics) */ getAllDomains(): string[]; /** * Get all API patterns for a domain * (KnowledgeBase compatibility method) */ getPatterns(domain: string): EnhancedApiPattern[]; /** * Get high-confidence patterns that can bypass browser * (KnowledgeBase compatibility method) */ getBypassablePatterns(domain: string): EnhancedApiPattern[]; /** * Find a pattern matching a URL * (KnowledgeBase compatibility method) * * Optimized with O(1) pathname index lookup (P-002): * - Exact pathname match: O(1) via index * - Prefix match fallback: O(N) only when exact match fails */ findPattern(url: string): EnhancedApiPattern | null; /** * Update success rate for a pattern * (KnowledgeBase compatibility method) */ updateSuccessRate(domain: string, endpoint: string, success: boolean): void; /** * Clear all learned data * (KnowledgeBase compatibility method) */ clear(): void; /** * Alias for learnApiPattern to maintain KnowledgeBase compatibility * (KnowledgeBase used learn() method) */ learn(domain: string, patterns: ApiPattern[]): void; /** * Check if an anti-pattern should be persisted * Only high-confidence anti-patterns are persisted */ private shouldPersistAntiPattern; /** * Persist a high-confidence anti-pattern * Called by ApiPatternRegistry when an anti-pattern is created * @returns true if the anti-pattern was persisted, false if not eligible */ persistAntiPattern(antiPattern: AntiPattern): boolean; /** * Get all persisted anti-patterns (active only) */ getPersistedAntiPatterns(): AntiPattern[]; /** * Clear a persisted anti-pattern (e.g., after user provides authentication) */ clearPersistedAntiPattern(antiPatternId: string): boolean; /** * Get anti-patterns for a specific domain */ getAntiPatternsForDomain(domain: string): AntiPattern[]; /** * Record a pattern failure for feedback loop * This updates pattern confidence and tracks failure context */ recordPatternFailure(domain: string, patternId: string, failureCategory: FailureCategory, errorMessage?: string): void; /** * Map FailureCategory to FailureContext type */ private mapFailureCategoryToContextType; /** * Get anti-pattern statistics */ getAntiPatternStats(): { total: number; active: number; byCategory: Record; byDomain: Record; }; private load; /** * Migrate data from legacy knowledge-base.json to enhanced format * This is a one-time migration that preserves existing learned patterns */ private migrateFromLegacyKnowledgeBase; private save; /** * Flush any pending writes to disk immediately */ flush(): Promise; private exitHandlersInstalled; /** * Set up process exit handlers to ensure patterns are persisted on shutdown. * Call this once after creating the LearningEngine to ensure graceful shutdown. */ setupExitHandlers(): void; /** * Get statistics about persisted patterns. * Useful for verifying that patterns survived across sessions. */ getPersistedPatternStats(): { totalDomains: number; domainsWithSelectors: number; totalSelectors: number; domainsWithApiPatterns: number; totalApiPatterns: number; totalAntiPatterns: number; lastSaved: number | null; oldestPattern: { domain: string; createdAt: number; } | null; newestPattern: { domain: string; createdAt: number; } | null; }; /** * Verify that a specific domain's patterns are persisted. * Returns true if the domain has persisted patterns that will survive restart. */ verifyDomainPersistence(domain: string): { exists: boolean; hasSelectorChains: boolean; hasApiPatterns: boolean; hasAntiPatterns: boolean; lastUpdated: number | null; isPersisted: boolean; }; /** * Force immediate save of all data. * Use this when you need to guarantee patterns are saved (e.g., after important learning). */ forceSave(): Promise; /** * Export full knowledge base for debugging */ exportKnowledgeBase(): Promise; /** Unsubscribe function for pattern registry */ private patternRegistryUnsubscribe?; /** * Subscribe to an ApiPatternRegistry to receive and persist anti-pattern events. * When the registry creates an anti-pattern, LearningEngine will persist it. * @returns Unsubscribe function */ subscribeToPatternRegistry(registry: ApiPatternRegistry): () => void; /** * Load persisted anti-patterns into an ApiPatternRegistry. * This should be called after the registry is initialized. */ loadPersistedAntiPatternsInto(registry: ApiPatternRegistry): Promise; /** * Wire to a ContentIntelligence instance for anti-pattern feedback. * This subscribes to the ContentIntelligence's pattern registry and loads * persisted anti-patterns into it. * @returns Unsubscribe function and count of loaded anti-patterns */ wireToContentIntelligence(contentIntelligence: ContentIntelligence): Promise<{ unsubscribe: () => void; loadedAntiPatterns: number; }>; /** * Check if a domain matches a glob-like pattern * Uses centralized url-pattern-matcher utility (D-007) */ private matchesDomainPattern; /** * Export knowledge base as a portable pack */ exportKnowledgePack(options?: KnowledgeExportOptions): KnowledgePack; /** * Serialize knowledge pack to JSON string */ serializeKnowledgePack(pack: KnowledgePack, pretty?: boolean): string; /** * Import knowledge base from a pack */ importKnowledgePack(packJson: string, options?: KnowledgeImportOptions): Promise; /** * Downgrade confidence level (high -> medium -> low) */ private downgradeConfidence; /** * Calculate confidence downgrade steps from adjustment factor * 1.0 = no change, 0.5 = 1 step down, 0.25 = 2 steps down */ private getConfidenceDowngradeSteps; /** * Prepare an entry for import (reset metrics, adjust confidence) */ private prepareEntryForImport; /** * Create a unique key for a selector chain */ private getSelectorChainKey; /** * Merge two domain entries */ private mergeEntries; /** * Get anti-patterns for export */ getAntiPatterns(): AntiPattern[]; /** * Get content change pattern for a URL */ getContentChangePattern(domain: string, urlPattern: string): ContentChangePattern | null; /** * Analyze content change pattern and get recommendations */ analyzeContentChangePattern(domain: string, urlPattern: string): ContentChangeAnalysis; /** * Check if we should poll this content now */ shouldCheckContentNow(domain: string, urlPattern: string): PollRecommendation; /** * Get next predicted change time for a URL */ getNextPredictedChange(domain: string, urlPattern: string): number | null; /** * Get all content change patterns */ getAllContentChangePatterns(): ContentChangePattern[]; /** * Export content change patterns for persistence */ exportContentChangePatterns(): Record; /** * Import content change patterns from persistence */ importContentChangePatterns(data: Record): void; } export declare const learningEngine: LearningEngine; //# sourceMappingURL=learning-engine.d.ts.map