import { IOptimizationModule, OptimizationResult } from './IOptimizationModule.js';
import { ITokenCounter } from '../interfaces/ITokenCounter.js';
/**
 * Production-ready deduplication optimization module.
 *
 * This module removes duplicate sentences and paragraphs from text while
 * preserving the original order and context. It uses robust algorithms
 * suitable for production use:
 * - Proper sentence boundary detection (Intl.Segmenter with fallback)
 * - Fuzzy matching for near-duplicates (Levenshtein distance)
 * - Semantic-aware code block preservation
 * - Formatting preservation for non-duplicate content
 *
 * Ideal for:
 * - Removing repeated boilerplate text
 * - Cleaning up copy-paste artifacts
 * - Removing redundant explanations
 * - Consolidating repeated information
 *
 * @example
 * ```typescript
 * const tokenCounter = new TokenCounter();
 * const deduplicationModule = new DeduplicationModule(tokenCounter, {
 *   caseSensitive: false,        // Ignore case when comparing
 *   minSentenceLength: 10,       // Only dedupe sentences with 10+ chars
 *   preserveFirst: true,         // Keep first occurrence
 *   similarityThreshold: 0.9,    // 90% similarity for fuzzy matching
 *   deduplicateParagraphs: true  // Also dedupe at paragraph level
 * });
 *
 * const result = await deduplicationModule.apply(textWithDuplicates);
 * console.log(`Removed ${result.metadata?.duplicatesRemoved} duplicates`);
 * console.log(`Saved ${result.savings} tokens`);
 * ```
 */
export declare class DeduplicationModule implements IOptimizationModule {
    private readonly tokenCounter;
    private readonly options?;
    readonly name = "deduplication";
    /**
     * Create a deduplication module.
     *
     * @param tokenCounter - Token counter for measuring savings
     * @param options - Configuration options
     */
    constructor(tokenCounter: ITokenCounter, options?: {
        /**
         * Case-sensitive comparison
         * @default true
         */
        caseSensitive?: boolean;
        /**
         * Minimum sentence length to consider for deduplication
         * @default 5
         */
        minSentenceLength?: number;
        /**
         * Preserve first occurrence (true) or last occurrence (false)
         * @default true
         */
        preserveFirst?: boolean;
        /**
         * Similarity threshold for fuzzy matching (0-1)
         * - 1.0 = exact matching only
         * - 0.9 = 90% similarity required
         * - 0.8 = 80% similarity required
         * Lower values catch more near-duplicates but may have false positives
         * @default 1.0
         */
        similarityThreshold?: number;
        /**
         * Also deduplicate at paragraph level
         * @default false
         */
        deduplicateParagraphs?: boolean;
        /**
         * Preserve code blocks (don't deduplicate content between ``` markers)
         * @default true
         */
        preserveCodeBlocks?: boolean;
    } | undefined);
    /**
     * Apply deduplication to the input text.
     *
     * This method:
     * 1. Counts tokens in the original text
     * 2. Extracts and preserves code blocks
     * 3. Splits text into sentences/paragraphs using proper tokenization
     * 4. Identifies duplicates using exact or fuzzy matching
     * 5. Removes duplicates while preserving formatting
     * 6. Restores code blocks
     * 7. Counts tokens in the deduplicated text
     * 8. Returns detailed results with statistics
     *
     * @param text - The text to deduplicate
     * @returns Optimization result with deduplication statistics
     */
    apply(text: string): Promise<OptimizationResult>;
    /**
     * Deduplicate sentences in text using proper sentence boundary detection.
     *
     * Uses Intl.Segmenter (Node 16+) for accurate sentence splitting that handles
     * abbreviations correctly. Falls back to regex-based splitting on older Node versions.
     *
     * @param text - Text to process
     * @returns Deduplicated text and statistics
     */
    private deduplicateSentences;
    /**
     * Deduplicate paragraphs in text while preserving original formatting.
     *
     * Uses capturing groups to preserve the exact separators between paragraphs,
     * so non-duplicate content retains its original spacing.
     *
     * @param text - Text to process
     * @returns Deduplicated text and statistics
     */
    private deduplicateParagraphs;
    /**
     * Split text into sentences using proper sentence boundary detection.
     *
     * Uses Intl.Segmenter on Node 16+ for accurate splitting that handles:
     * - Abbreviations (Dr., U.S., etc.)
     * - Decimal numbers (1.5, $10.99)
     * - Ellipses (...)
     * - Quotations and punctuation
     *
     * Falls back to regex-based splitting on older Node versions.
     *
     * @param text - Text to split
     * @returns Array of sentence strings
     */
    private splitSentences;
    /**
     * Calculate normalized Levenshtein distance between two strings.
     *
     * Returns similarity score from 0 (completely different) to 1 (identical).
     * Uses character-level edit distance normalized by max string length.
     *
     * @param s1 - First string
     * @param s2 - Second string
     * @returns Similarity score between 0 and 1
     */
    private calculateSimilarity;
}
//# sourceMappingURL=DeduplicationModule.d.ts.map