import { HttpClient } from '../utils/http-client.js';
export interface CrawlOptions {
    crawlStrategy?: 'bfs' | 'dfs';
    maxDepth?: number;
    maxPages?: number;
    includePaths?: string[];
    excludePaths?: string[];
    rateLimit?: number;
    maxRetries?: number;
    retryDelay?: number;
    useBrowserAutomation?: boolean;
    browserType?: 'puppeteer' | 'playwright';
    spaStrategy?: 'smart' | 'auto' | 'manual';
    skipLlmsTxt?: boolean;
    workers?: number;
    checkpoint?: {
        enabled: boolean;
        interval: number;
        file?: string;
    };
    resume?: boolean;
}
export interface CrawledPage {
    url: string;
    title: string;
    content: string;
    depth: number;
    sections: Array<{
        title: string;
        content: string;
        anchor?: string;
    }>;
    navigationLinks: Array<{
        text: string;
        url: string;
        isInternal: boolean;
    }>;
    headings: Array<{
        level: string;
        text: string;
        id?: string;
    }>;
    codeSamples: Array<{
        code: string;
        language: string;
    }>;
}
export interface LinkDiscoveryStats {
    totalLinksFound: number;
    linksFiltered: {
        notContent: number;
        externalDomain: number;
        alreadyVisited: number;
        excludedPattern: number;
        depthLimit: number;
    };
    linksQueued: number;
    pagesDiscovered: number;
    pagesCrawled: number;
}
export type AbandonReason = 'insufficient_content' | 'media_only' | 'empty_pages' | 'no_structured_content';
export interface CrawlResult {
    pages: CrawledPage[];
    totalPages: number;
    maxDepthReached: number;
    errors: Array<{
        url: string;
        error: string;
    }>;
    linkDiscoveryStats: LinkDiscoveryStats;
    abandoned?: boolean;
    abandonReason?: AbandonReason;
}
export declare class DocumentationCrawler {
    private browser;
    private browserAdapter;
    private urlQueue;
    private crawledPages;
    private errors;
    private crawlStartTime;
    private lastProgressTime;
    private consecutiveErrors;
    private options;
    private baseUrl;
    private linkDiscoveryStats;
    private checkpointManager?;
    private pagesSinceLastCheckpoint;
    private readonly SAFETY_LIMITS;
    private readonly DOCUMENTATION_PATTERNS;
    private readonly EXCLUDED_PATTERNS;
    constructor(httpClient?: HttpClient);
    /**
     * Crawl documentation starting from a root URL
     * Uses HTTP client (axios) exclusively - no browser automation
     * For SPA sites that require JavaScript rendering, use Cursor/Claude's built-in browser tools
     * Supports both BFS (breadth-first) and DFS (depth-first) crawl strategies
     */
    crawl(rootUrl: string, options?: CrawlOptions): Promise<CrawlResult>;
    /**
     * Sequential crawling (single-threaded)
     */
    private crawlSequential;
    /**
     * Parallel crawling with multiple workers
     */
    private crawlWithWorkers;
    /**
     * Process a single page (shared by both sequential and parallel crawling)
     */
    private processPage;
    /**
     * Discover documentation links from a crawled page
     */
    private discoverDocumentationLinks;
    /**
     * Check if a path should be crawled (permissive - only exclude clearly non-content paths)
     */
    private isDocumentationPath;
    /**
     * Check if a path should be excluded
     */
    private shouldExclude;
    /**
     * Check if crawled content is sufficient for skill generation
     * Enhanced with multi-dimensional quality metrics
     */
    private canGenerateSkill;
    /**
     * Evaluate content quality with multi-dimensional metrics
     */
    private evaluateContentQuality;
    /**
     * Check if should continue crawling based on content quality
     */
    private shouldContinueCrawling;
    /**
     * Fetch a page with retry logic
     * Supports HTML pages and Markdown files
     */
    private fetchPageWithRetry;
    /**
     * Extract content from Markdown file
     * Converts Markdown structure to WebDocumentationPage format
     */
    private extractMarkdownContent;
    /**
     * Parse Markdown content into structured data
     */
    private parseMarkdown;
    /**
     * Classify error type for better error messages
     */
    private classifyError;
    /**
     * Check if an error is retryable
     */
    private isRetryableError;
    /**
     * Get error breakdown by type
     */
    private getErrorBreakdown;
    /**
     * Try to detect and use llms.txt for optimized crawling
     */
    private tryLlmsTxt;
    /**
     * Check if a URL is valid for crawling
     */
    private isValidUrl;
    /**
     * Save checkpoint
     */
    private saveCheckpoint;
    /**
     * Load checkpoint and restore state
     */
    private loadCheckpoint;
    /**
     * Clear checkpoint after successful crawl
     */
    private clearCheckpoint;
    /**
     * Sanitize filename for checkpoint
     */
    private sanitizeFilename;
    /**
     * Initialize browser adapter if needed
     */
    private initializeBrowserAdapter;
    /**
     * Fetch page using browser automation
     */
    private fetchPageWithBrowser;
    /**
     * Parse HTML content to extract structured data
     */
    private parseHtmlContent;
    /**
     * Cleanup resources (checkpoint, browser, etc.)
     */
    cleanup(): Promise<void>;
    /**
     * ✅ 检查是否应该中止爬取（安全限制）
     */
    private shouldAbortCrawl;
    /**
     * Delay helper for rate limiting
     */
    private delay;
}
//# sourceMappingURL=doc-crawler.d.ts.map