/** * Dynamic Handler Integration * * Connects the DynamicHandlerRegistry with ContentIntelligence and TieredFetcher. * This module: * 1. Records observations from successful extractions * 2. Records failures to learn quirks * 3. Provides extraction strategies from learned handlers */ import { DynamicHandlerRegistry } from './registry.js'; import type { HandlerTemplate, SiteQuirks, ExtractionRule, ApiPattern } from './types.js'; import type { ExtractionStrategy, ContentResult } from '../content-intelligence.js'; /** * Context for an extraction attempt */ export interface ExtractionContext { url: string; domain: string; html?: string; headers?: Record; } /** * Recommendation from the dynamic handler system */ export interface ExtractionRecommendation { /** Recommended template type */ template: HandlerTemplate; /** Site-specific quirks to apply */ quirks?: SiteQuirks; /** Custom extraction rules */ rules: ExtractionRule[]; /** Discovered API patterns */ apis: ApiPattern[]; /** Confidence in this recommendation */ confidence: number; /** Recommended headers */ headers?: Record; /** Rate limit to apply (requests per second) */ rateLimit?: number; } /** * Dynamic Handler Integration * * Provides methods to integrate the dynamic handler system with * ContentIntelligence and TieredFetcher. */ export declare class DynamicHandlerIntegration { private registry; private autoSave; constructor(registry?: DynamicHandlerRegistry); /** * Enable auto-save persistence */ enablePersistence(options?: { path?: string; saveDelayMs?: number; }): void; /** * Get extraction recommendation for a URL * * Call this before attempting extraction to get guidance * on the best approach for this site. */ getRecommendation(context: ExtractionContext): ExtractionRecommendation; /** * Record a successful extraction * * Call this after ContentIntelligence successfully extracts content * to teach the system about this site. */ recordSuccess(url: string, strategy: ExtractionStrategy, result: ContentResult, details?: { html?: string; apiCalls?: Array<{ url: string; method: string; status: number; responseType: string; }>; selectorsUsed?: string[]; jsonPaths?: string[]; duration: number; }): void; /** * Record an extraction failure * * Call this when extraction fails to learn quirks about the site. */ recordFailure(url: string, error: string, context: { statusCode?: number; headers?: Record; strategy?: string; duration?: number; }): void; /** * Check if the system has learned about a domain */ hasLearnedDomain(domain: string): boolean; /** * Get quirks for a domain */ getQuirks(domain: string): SiteQuirks | undefined; /** * Manually update quirks for a domain */ updateQuirks(domain: string, quirks: Partial): void; /** * Get statistics about what the system has learned */ getStats(): { totalHandlers: number; totalQuirks: number; totalObservations: number; topDomains: Array<{ domain: string; observations: number; }>; }; /** * Force save (useful before shutdown) */ save(): void; /** * Clean up (call on shutdown) */ dispose(): void; /** * Get the underlying registry */ getRegistry(): DynamicHandlerRegistry; /** * Sync all learned rate limits to the rate limiter * Call this after loading persisted data to apply learned limits */ syncAllRateLimits(): void; /** * Sync a single domain's rate limit to the rate limiter */ private syncRateLimitToDomain; private extractDomain; } /** * Singleton integration instance with persistence */ export declare const dynamicHandlerIntegration: DynamicHandlerIntegration; /** * Initialize the integration with persistence * Call this at application startup */ export declare function initializeDynamicHandlers(options?: { persistencePath?: string; saveDelayMs?: number; }): DynamicHandlerIntegration; /** * Shutdown the integration (save and cleanup) * Call this before application exit */ export declare function shutdownDynamicHandlers(): void; /** * Apply quirks to fetch options * * Helper function to apply learned quirks to fetch options */ export declare function applyQuirksToFetchOptions(quirks: SiteQuirks | undefined, options: { headers?: Record; timeout?: number; }): typeof options; /** * Convert template to recommended extraction strategy */ export declare function templateToStrategy(template: HandlerTemplate): ExtractionStrategy | null; //# sourceMappingURL=integration.d.ts.map