/**
 * Web content source implementation.
 *
 * Processes web pages via sitemap crawling, extracting content
 * and creating chunks for embedding. Supports Divi and Docusaurus sites.
 *
 * Features:
 * - Sitemap parsing (index and flat sitemaps)
 * - Rate-limited page fetching
 * - HTML to Markdown conversion
 * - Content preprocessing and deduplication
 * - Optional LLM-based content enhancement
 * - Optional image processing with vision LLM
 */
import { BaseContentSource } from '../base.js';
import type { SourceContext, SourceResult } from '../types.js';
import { WebSourceConfig } from './config.js';
/**
 * Web content source.
 *
 * Crawls websites via sitemap, extracts content, and creates
 * chunks ready for embedding.
 */
export declare class WebSource extends BaseContentSource<WebSourceConfig> {
    readonly type = "web";
    readonly name = "Web (Sitemap)";
    /**
     * Process the web source and return chunks.
     */
    process(config: WebSourceConfig, context: SourceContext): Promise<SourceResult>;
    /**
     * Validate and transform raw configuration.
     */
    validateConfig(config: unknown): WebSourceConfig;
    /** Minimum chunk size for good embedding quality */
    private static readonly MIN_CHUNK_SIZE;
    /**
     * Create chunks from LLM-processed page.
     */
    private createChunksFromProcessedPage;
    /**
     * Create simple chunks without LLM processing.
     */
    private createSimpleChunks;
    /**
     * Merge small adjacent sections to improve embedding quality.
     */
    private mergeSmallSections;
    /**
     * Clean raw content for better embedding.
     */
    private cleanRawContent;
    /**
     * Split content into sections respecting max size.
     */
    private splitIntoSections;
    /**
     * Split oversized content into smaller chunks.
     */
    private splitOversizedContent;
    /**
     * Infer section and category from URL path.
     */
    private inferSectionCategory;
    /**
     * Extract version and guide from URL path.
     */
    private extractGuideAndVersion;
}
export type { SitemapEntry, FetchResult, ExtractedPage, FetcherOptions, ExtractorOptions, WebSourceOptions, PreprocessorOptions, } from './types.js';
export { DIVI_CONTENT_SELECTORS, DIVI_EXCLUDE_SELECTORS, DOCUSAURUS_CONTENT_SELECTORS, DOCUSAURUS_EXCLUDE_SELECTORS, } from './types.js';
export type { WebSourceConfig, ImageProcessingConfig, PageTypeConfig, PreprocessingConfig, } from './config.js';
export { WebSourceConfigSchema, WEB_SOURCE_DEFAULTS } from './config.js';
export { parseSitemapIndex, parseSitemap, filterUrls } from './sitemap.js';
export { fetchPages, fetchAllPages } from './fetcher.js';
export { extractContent, isContentMeaningful } from './extractor.js';
export { preprocessContent, deduplicateContent, isChunkMeaningful, identifyBoilerplate, } from './preprocessor.js';
export { processPageWithLLM, detectPageType } from './llm-processor.js';
export { extractImages, filterImages, type ExtractedImage } from './images.js';
export { processImagesWithVision, formatImageDescriptions, type ProcessedImage, type VisionProcessingOptions, } from './vision.js';
export { ImageCache, type CacheEntry, type ImageCacheConfig, type CacheStats } from './image-cache.js';
//# sourceMappingURL=index.d.ts.map