/**
 * Web source configuration types and validation.
 *
 * This module defines the configuration schema for web sources,
 * including sitemap parsing, content extraction, preprocessing,
 * and image processing options.
 */
import { z } from 'zod';
import type { BaseSourceConfig } from '../types.js';
import type { ChunkFilterOptions } from '../../chunking/types.js';
/**
 * Page-type specific configuration for image processing.
 *
 * Allows fine-tuning image processing behavior for different types
 * of pages (e.g., floor plans may need more images processed).
 */
export interface PageTypeConfig {
    /** URL pattern to match (regex string) */
    urlPattern: string;
    /** Override max images for this page type */
    maxPerPage?: number;
    /** Process all significant images (ignore maxPerPage) */
    processAll?: boolean;
    /** Custom prompt template key for vision processing */
    promptTemplate?: string;
    /** Minimum dimensions override */
    minWidth?: number;
    minHeight?: number;
}
/**
 * Image processing configuration for vision LLM analysis.
 */
export interface ImageProcessingConfig {
    /** Enable image processing */
    enabled: boolean;
    /** Minimum image width to process (default: 300) */
    minWidth?: number;
    /** Minimum image height to process (default: 200) */
    minHeight?: number;
    /** Maximum images per page (default: 3) */
    maxPerPage?: number;
    /** URL patterns to exclude from image processing */
    excludePatterns?: string[];
    /** Vision model to use */
    model?: string;
    /** Page-type specific overrides */
    pageTypes?: PageTypeConfig[];
    /** Cache settings */
    cache?: {
        /** Enable caching of image descriptions (default: true) */
        enabled?: boolean;
        /** Cache directory (default: .cache/images) */
        directory?: string;
        /** Cache TTL in days (default: 30) */
        ttlDays?: number;
    };
}
/**
 * Content preprocessing configuration.
 *
 * Options for cleaning and normalizing extracted content
 * before chunking and embedding.
 */
export interface PreprocessingConfig {
    /** Remove common CTA phrases like "Learn More", "Get Started" */
    removeCTAs?: boolean;
    /** Remove phone numbers */
    removePhoneNumbers?: boolean;
    /** Normalize Unicode characters (curly quotes, dashes, etc.) */
    normalizeUnicode?: boolean;
    /** Clean orphaned markdown formatting */
    cleanMarkdown?: boolean;
    /** Custom regex patterns to remove */
    customPatterns?: string[];
    /** Minimum word count for a line to be kept (0 = keep all) */
    minLineWords?: number;
}
/**
 * Web source configuration.
 *
 * Configuration for processing web pages via sitemap crawling.
 * Supports Divi and Docusaurus sites with custom selectors.
 */
export interface WebSourceConfig extends BaseSourceConfig {
    type: 'web';
    /** Sitemap index URL (or flat sitemap URL) */
    sitemapUrl: string;
    /** Only process these child sitemaps (if using sitemap index) */
    includeSitemaps?: string[];
    /** URL patterns to include (if specified, only matching URLs are processed) */
    includePatterns?: string[];
    /** URL patterns to exclude */
    excludePatterns?: string[];
    /** CSS selectors for main content areas */
    contentSelectors?: string[];
    /** CSS selectors for elements to remove */
    excludeSelectors?: string[];
    /** Requests per second (default: 2) */
    rateLimit?: number;
    /** Content preprocessing options */
    preprocessing?: PreprocessingConfig;
    /** Remove elements hidden via inline styles (default: true) */
    removeHiddenElements?: boolean;
    /** Deduplicate repeated content blocks (default: true) */
    deduplicate?: boolean;
    /** Image processing configuration */
    imageProcessing?: ImageProcessingConfig;
    /** Enable LLM-based content processing */
    useLLM?: boolean;
    /** LLM model for content extraction */
    llmModel?: string;
    /** LLM request timeout in ms */
    llmTimeout?: number;
    /** LLM processing concurrency */
    llmConcurrency?: number;
    /** Chunk filtering options */
    chunkFilter?: ChunkFilterOptions;
    /** Prepend page title to each chunk content (default: false) */
    prependTitle?: boolean;
}
/**
 * Zod schema for web source configuration.
 */
export declare const WebSourceConfigSchema: z.ZodObject<{
    type: z.ZodLiteral<"web">;
    sitemapUrl: z.ZodString;
    includeSitemaps: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
    includePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
    excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
    contentSelectors: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
    excludeSelectors: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
    rateLimit: z.ZodOptional<z.ZodNumber>;
    categories: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
    guide: z.ZodOptional<z.ZodString>;
    preprocessing: z.ZodOptional<z.ZodObject<{
        removeCTAs: z.ZodOptional<z.ZodBoolean>;
        removePhoneNumbers: z.ZodOptional<z.ZodBoolean>;
        normalizeUnicode: z.ZodOptional<z.ZodBoolean>;
        cleanMarkdown: z.ZodOptional<z.ZodBoolean>;
        customPatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
        minLineWords: z.ZodOptional<z.ZodNumber>;
    }, "strip", z.ZodTypeAny, {
        normalizeUnicode?: boolean | undefined;
        removeCTAs?: boolean | undefined;
        removePhoneNumbers?: boolean | undefined;
        cleanMarkdown?: boolean | undefined;
        customPatterns?: string[] | undefined;
        minLineWords?: number | undefined;
    }, {
        normalizeUnicode?: boolean | undefined;
        removeCTAs?: boolean | undefined;
        removePhoneNumbers?: boolean | undefined;
        cleanMarkdown?: boolean | undefined;
        customPatterns?: string[] | undefined;
        minLineWords?: number | undefined;
    }>>;
    removeHiddenElements: z.ZodOptional<z.ZodBoolean>;
    deduplicate: z.ZodOptional<z.ZodBoolean>;
    imageProcessing: z.ZodOptional<z.ZodObject<{
        enabled: z.ZodBoolean;
        minWidth: z.ZodOptional<z.ZodNumber>;
        minHeight: z.ZodOptional<z.ZodNumber>;
        maxPerPage: z.ZodOptional<z.ZodNumber>;
        excludePatterns: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
        model: z.ZodOptional<z.ZodString>;
        pageTypes: z.ZodOptional<z.ZodArray<z.ZodObject<{
            urlPattern: z.ZodString;
            maxPerPage: z.ZodOptional<z.ZodNumber>;
            processAll: z.ZodOptional<z.ZodBoolean>;
            promptTemplate: z.ZodOptional<z.ZodString>;
            minWidth: z.ZodOptional<z.ZodNumber>;
            minHeight: z.ZodOptional<z.ZodNumber>;
        }, "strip", z.ZodTypeAny, {
            urlPattern: string;
            maxPerPage?: number | undefined;
            processAll?: boolean | undefined;
            promptTemplate?: string | undefined;
            minWidth?: number | undefined;
            minHeight?: number | undefined;
        }, {
            urlPattern: string;
            maxPerPage?: number | undefined;
            processAll?: boolean | undefined;
            promptTemplate?: string | undefined;
            minWidth?: number | undefined;
            minHeight?: number | undefined;
        }>, "many">>;
        cache: z.ZodOptional<z.ZodObject<{
            enabled: z.ZodOptional<z.ZodBoolean>;
            directory: z.ZodOptional<z.ZodString>;
            ttlDays: z.ZodOptional<z.ZodNumber>;
        }, "strip", z.ZodTypeAny, {
            enabled?: boolean | undefined;
            directory?: string | undefined;
            ttlDays?: number | undefined;
        }, {
            enabled?: boolean | undefined;
            directory?: string | undefined;
            ttlDays?: number | undefined;
        }>>;
    }, "strip", z.ZodTypeAny, {
        enabled: boolean;
        excludePatterns?: string[] | undefined;
        model?: string | undefined;
        cache?: {
            enabled?: boolean | undefined;
            directory?: string | undefined;
            ttlDays?: number | undefined;
        } | undefined;
        maxPerPage?: number | undefined;
        minWidth?: number | undefined;
        minHeight?: number | undefined;
        pageTypes?: {
            urlPattern: string;
            maxPerPage?: number | undefined;
            processAll?: boolean | undefined;
            promptTemplate?: string | undefined;
            minWidth?: number | undefined;
            minHeight?: number | undefined;
        }[] | undefined;
    }, {
        enabled: boolean;
        excludePatterns?: string[] | undefined;
        model?: string | undefined;
        cache?: {
            enabled?: boolean | undefined;
            directory?: string | undefined;
            ttlDays?: number | undefined;
        } | undefined;
        maxPerPage?: number | undefined;
        minWidth?: number | undefined;
        minHeight?: number | undefined;
        pageTypes?: {
            urlPattern: string;
            maxPerPage?: number | undefined;
            processAll?: boolean | undefined;
            promptTemplate?: string | undefined;
            minWidth?: number | undefined;
            minHeight?: number | undefined;
        }[] | undefined;
    }>>;
    useLLM: z.ZodOptional<z.ZodBoolean>;
    llmModel: z.ZodOptional<z.ZodString>;
    llmTimeout: z.ZodOptional<z.ZodNumber>;
    llmConcurrency: z.ZodOptional<z.ZodNumber>;
    chunkFilter: z.ZodOptional<z.ZodObject<{
        skipSections: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
        minContentLength: z.ZodOptional<z.ZodNumber>;
        maxLinkRatio: z.ZodOptional<z.ZodNumber>;
        stripLeadingH1: z.ZodOptional<z.ZodBoolean>;
    }, "strip", z.ZodTypeAny, {
        skipSections?: string[] | undefined;
        minContentLength?: number | undefined;
        maxLinkRatio?: number | undefined;
        stripLeadingH1?: boolean | undefined;
    }, {
        skipSections?: string[] | undefined;
        minContentLength?: number | undefined;
        maxLinkRatio?: number | undefined;
        stripLeadingH1?: boolean | undefined;
    }>>;
    prependTitle: z.ZodOptional<z.ZodBoolean>;
}, "strip", z.ZodTypeAny, {
    type: "web";
    sitemapUrl: string;
    guide?: string | undefined;
    preprocessing?: {
        normalizeUnicode?: boolean | undefined;
        removeCTAs?: boolean | undefined;
        removePhoneNumbers?: boolean | undefined;
        cleanMarkdown?: boolean | undefined;
        customPatterns?: string[] | undefined;
        minLineWords?: number | undefined;
    } | undefined;
    includeSitemaps?: string[] | undefined;
    includePatterns?: string[] | undefined;
    excludePatterns?: string[] | undefined;
    contentSelectors?: string[] | undefined;
    excludeSelectors?: string[] | undefined;
    rateLimit?: number | undefined;
    removeHiddenElements?: boolean | undefined;
    deduplicate?: boolean | undefined;
    categories?: Record<string, string> | undefined;
    useLLM?: boolean | undefined;
    llmModel?: string | undefined;
    llmTimeout?: number | undefined;
    llmConcurrency?: number | undefined;
    imageProcessing?: {
        enabled: boolean;
        excludePatterns?: string[] | undefined;
        model?: string | undefined;
        cache?: {
            enabled?: boolean | undefined;
            directory?: string | undefined;
            ttlDays?: number | undefined;
        } | undefined;
        maxPerPage?: number | undefined;
        minWidth?: number | undefined;
        minHeight?: number | undefined;
        pageTypes?: {
            urlPattern: string;
            maxPerPage?: number | undefined;
            processAll?: boolean | undefined;
            promptTemplate?: string | undefined;
            minWidth?: number | undefined;
            minHeight?: number | undefined;
        }[] | undefined;
    } | undefined;
    chunkFilter?: {
        skipSections?: string[] | undefined;
        minContentLength?: number | undefined;
        maxLinkRatio?: number | undefined;
        stripLeadingH1?: boolean | undefined;
    } | undefined;
    prependTitle?: boolean | undefined;
}, {
    type: "web";
    sitemapUrl: string;
    guide?: string | undefined;
    preprocessing?: {
        normalizeUnicode?: boolean | undefined;
        removeCTAs?: boolean | undefined;
        removePhoneNumbers?: boolean | undefined;
        cleanMarkdown?: boolean | undefined;
        customPatterns?: string[] | undefined;
        minLineWords?: number | undefined;
    } | undefined;
    includeSitemaps?: string[] | undefined;
    includePatterns?: string[] | undefined;
    excludePatterns?: string[] | undefined;
    contentSelectors?: string[] | undefined;
    excludeSelectors?: string[] | undefined;
    rateLimit?: number | undefined;
    removeHiddenElements?: boolean | undefined;
    deduplicate?: boolean | undefined;
    categories?: Record<string, string> | undefined;
    useLLM?: boolean | undefined;
    llmModel?: string | undefined;
    llmTimeout?: number | undefined;
    llmConcurrency?: number | undefined;
    imageProcessing?: {
        enabled: boolean;
        excludePatterns?: string[] | undefined;
        model?: string | undefined;
        cache?: {
            enabled?: boolean | undefined;
            directory?: string | undefined;
            ttlDays?: number | undefined;
        } | undefined;
        maxPerPage?: number | undefined;
        minWidth?: number | undefined;
        minHeight?: number | undefined;
        pageTypes?: {
            urlPattern: string;
            maxPerPage?: number | undefined;
            processAll?: boolean | undefined;
            promptTemplate?: string | undefined;
            minWidth?: number | undefined;
            minHeight?: number | undefined;
        }[] | undefined;
    } | undefined;
    chunkFilter?: {
        skipSections?: string[] | undefined;
        minContentLength?: number | undefined;
        maxLinkRatio?: number | undefined;
        stripLeadingH1?: boolean | undefined;
    } | undefined;
    prependTitle?: boolean | undefined;
}>;
/**
 * Default values for web source configuration.
 */
export declare const WEB_SOURCE_DEFAULTS: {
    rateLimit: number;
    removeHiddenElements: boolean;
    deduplicate: boolean;
    preprocessing: {
        removeCTAs: boolean;
        normalizeUnicode: boolean;
        cleanMarkdown: boolean;
    };
    imageProcessing: {
        minWidth: number;
        minHeight: number;
        maxPerPage: number;
        model: string;
        cache: {
            enabled: boolean;
            directory: string;
            ttlDays: number;
        };
    };
    llmModel: string;
    llmTimeout: number;
    llmConcurrency: number;
    /** Don't prepend title by default - it's already in metadata */
    prependTitle: boolean;
    /** Chunk filter defaults for web content */
    chunkFilter: {
        skipSections: string[];
        minContentLength: number;
        maxLinkRatio: number;
        stripLeadingH1: boolean;
    };
};
//# sourceMappingURL=config.d.ts.map