/**
 * Tokenizer service for Context7-style token management
 *
 * Provides token estimation, smart truncation, and section extraction
 * for controlling response size in LLM contexts.
 */
import type { TokenInfo, ArticleSection } from '../types.js';
/**
 * Result of truncation operation
 */
export interface TruncateResult {
    content: string;
    tokenInfo: TokenInfo;
    remainingSections?: ArticleSection[];
}
/**
 * Result of summary extraction
 */
export interface SummaryResult {
    title: string;
    summary: string;
    outline: ArticleSection[];
    totalTokens: number;
    estimatedReadTime: number;
    tokenInfo: TokenInfo;
}
/**
 * Result of section extraction
 */
export interface ExtractSectionResult {
    content: string;
    section: ArticleSection | null;
    tokenInfo: TokenInfo;
}
/**
 * Estimate token count for a text string
 * Uses different ratios for code blocks vs normal text
 */
export declare function estimateTokens(text: string | null | undefined): number;
/**
 * Create a TokenInfo object
 */
export declare function createTokenInfo(content: string, maxTokens: number, truncated?: boolean): TokenInfo;
/**
 * Generate a unique section ID with duplicate suffix handling
 * Shared between extractSections() and extractSection() for consistency
 */
export declare function generateSectionId(title: string, index: number, idCounts: Map<string, number>): string;
/**
 * Extract sections (headings) from Markdown content
 * Generates unique slugified IDs with duplicate suffix handling
 */
export declare function extractSections(content: string): ArticleSection[];
/**
 * Generate a section ID (slug) from heading title
 */
export declare function slugify(title: string): string;
/**
 * Extract a specific section from Markdown content
 */
export declare function extractSection(content: string, sectionIdentifier: string, maxTokens?: number): ExtractSectionResult;
/**
 * Smart truncation that preserves document structure
 * - Preserves paragraph boundaries
 * - Correctly closes code blocks
 * - Lists remaining sections when truncated
 */
export declare function truncateToTokenLimit(content: string, maxTokens?: number, preExtractedSections?: ArticleSection[]): TruncateResult;
/**
 * Extract summary from Markdown content
 * Returns the first paragraph and an outline of sections with token estimates
 */
export declare function extractSummary(content: string, title: string, maxTokens?: number): SummaryResult;
/**
 * Calculate pagination info
 */
export declare function calculatePagination(totalItems: number, page: number, pageSize: number): {
    page: number;
    pageSize: number;
    totalPages: number;
    totalItems: number;
    hasNext: boolean;
    hasPrev: boolean;
    startIndex: number;
    endIndex: number;
    requestedPage: number;
    pageWasClamped: boolean;
};
/**
 * Truncate an already-paginated list to fit within a token budget.
 * Items are dropped from the end once cumulative tokens exceed maxTokens.
 */
export declare function truncateListByTokens<T>(items: T[], maxTokens: number, toString: (item: T) => string): {
    items: T[];
    tokenCount: number;
    truncated: boolean;
};
/**
 * Build a human-readable note when the requested page was clamped.
 */
export declare function buildPaginationNote(pagination: {
    pageWasClamped: boolean;
    requestedPage: number;
    totalPages: number;
}): string | undefined;
/**
 * Truncate an array of items to fit within token limit
 * Returns items that fit and pagination info
 */
export declare function truncateItemsToTokenLimit<T>(items: T[], maxTokens: number, itemToString: (item: T) => string, page?: number, pageSize?: number): {
    items: T[];
    tokenInfo: TokenInfo;
    pagination: {
        page: number;
        pageSize: number;
        totalPages: number;
        totalItems: number;
        hasNext: boolean;
        hasPrev: boolean;
    };
};
//# sourceMappingURL=tokenizer.d.ts.map