interface Candidate { score: number; elem: any; } /** * ### HTML-to-Main-Content Extractor #1 * The function extracts main content with regex patterns, cleaning HTML, scoring nodes * based on content indicators like paragraphs and id/class names, selecting * the top candidate, extracting it, and cleaning up content around it. * * * 1. Define regular expressions: * - Various regex patterns are defined to identify content and non-content areas. * * 2. Define helper functions: * - normalizeSpaces: Normalizes whitespace in a string. * - stripTags: Removes all HTML tags from a string. * - getTextLength: Calculates the length of text after stripping tags. * - calculateLinkDensity: Calculates the ratio of link text to total text. * * 3. Clean HTML: * - Remove unlikely candidates (e.g., ads, sidebars) from the HTML. * * 4. Define scoring function: * - scoreNode: Assigns a score to an HTML node based on content and attributes. * - Increases score for positive indicators (e.g., article, body, content tags). * - Decreases score for negative indicators (e.g., hidden, footer, sidebar tags). * - Adds to score based on paragraph tags and text length. * * 5. Find and score candidate nodes: * - Identify potential content nodes in the cleaned HTML. * - Score each node using the scoreNode function. * * 6. Select top candidate: * - Sort candidates by score and select the highest-scoring node. * * 7. Extract content: * - Use regex to extract content around the top candidate node. * * 8. Clean up extracted content: * - Remove script and style tags and their contents. * - Process anchor tags based on content density. * - Keep only specific HTML tags (a, p, img, h1-h6, ul, ol, li). * - Remove excess whitespace from the final content. * * [Article Extraction Benchmark](https://trafilatura.readthedocs.io/en/latest/evaluation.html) * * @example * var url = "https://www.nytimes.com/2024/08/28/business/telegram-ceo-pavel-durov-charged.html" * const html = await (await fetch(url)).text(); * var articleContent = extractMainContentFromHTML(html); * @param {Object} [options] * @param {number} options.minContentLength default=140 - Minimum length of content to be considered valid * @param {number} options.minScore default=20 - Minimum score for content to be considered valid * @param {number} options.minTextLength default=25 - Minimum length of text to be considered valid * @param {number} options.retryLength default=250 - Length to retry content extraction if initial attempt fails * @returns {string} Extracted HTML string of main content * @author [vtempest (2025)](https://github.com/vtempest) * Based on [Mozilla Readability (2015)](https://github.com/mozilla/readability) * @category Extract */ export declare function extractMainContentFromHTML(html: string, options?: { minContentLength?: number; minScore?: number; minTextLength?: number; retryLength?: number; }): string; /** * Calculates the link density of an element. * @param {Element} elem - The element to calculate link density for * @returns {number} The link density (ratio of link text length to total text length) */ export declare function getLinkDensity(elem: any): number; /** * Calculates the weight of an element based on its class and id attributes. * @param {Element} elem - The element to calculate weight for * @param {RegExp} positiveRe - Regular expression for positive indicators * @param {RegExp} negativeRe - Regular expression for negative indicators * @returns {number} The calculated weight */ export declare function classWeight(elem: any, positiveRe: RegExp, negativeRe: RegExp): number; /** * Scores a node based on its tag name and attributes. * @param {Element} elem - The element to score * @param {RegExp} positiveRe - Regular expression for positive indicators * @param {RegExp} negativeRe - Regular expression for negative indicators * @returns {Object} An object containing the score and the element */ export declare function scoreNode(elem: any, positiveRe: RegExp, negativeRe: RegExp): Candidate; /** * Sanitizes the content by removing unwanted elements and cleaning remaining elements. * @param {Element} node - The node to sanitize * @param {Object} candidates - Object containing scored candidates * @param {RegExp} videoRe - Regular expression for video URLs * @param {RegExp} positiveRe - Regular expression for positive indicators * @param {RegExp} negativeRe - Regular expression for negative indicators * @param {number} minTextLength - Minimum text length to consider * @returns {Element} The sanitized node */ export declare function sanitize(node: any, candidates: Record, videoRe: RegExp, positiveRe: RegExp, negativeRe: RegExp, minTextLength: number): any; export {};