import * as cheerio from "cheerio"; import type { CheerioAPI } from "cheerio"; import { marked } from "marked"; import { patterns } from "./patterns"; import { htmlToText } from "./htmlToText"; import { htmlToMarkdown } from "./htmlToMarkdown"; import { cleanEmailHtml } from "./htmlCleaner"; export interface GetHtmlOptions { /** If true, returns raw HTML without cleaning. Default: false (cleaned). */ raw?: boolean; } /* eslint-disable @typescript-eslint/no-explicit-any */ type CheerioNode = any; // Regex for clearing text - compiled once for performance const CLEAR_TEXT_REGEX = /[><\s\r\t\xa0]+/g; /** * Extract original tag names with their casing from HTML */ function extractOriginalTagCasing(html: string): Map { const tagMap = new Map(); const tagRegex = /<\/?([a-zA-Z][a-zA-Z0-9]*)/g; let match; while ((match = tagRegex.exec(html)) !== null) { const tagName = match[1]; const lowerTag = tagName.toLowerCase(); // Keep the first occurrence's casing (usually the opening tag) if (!tagMap.has(lowerTag)) { tagMap.set(lowerTag, tagName); } } return tagMap; } /** * Restore original tag casing in processed HTML */ function restoreTagCasing( html: string, originalCasing: Map ): string { let result = html; for (const [lowerTag, originalTag] of originalCasing) { if (lowerTag !== originalTag) { // Replace opening tags result = result.replace( new RegExp(`<${lowerTag}(\\s|>|\\/)`, "gi"), (match) => `<${originalTag}${match.slice(lowerTag.length + 1)}` ); // Replace closing tags result = result.replace( new RegExp(``, "gi"), `` ); } } return result; } /** * Extract inner HTML without the wrapper * and convert to self-closing tags like Python's BeautifulSoup */ function getInnerHtml($: CheerioAPI, originalHtml: string): string { // Extract original tag casing before cheerio normalizes it const originalCasing = extractOriginalTagCasing(originalHtml); // Check if original HTML had html/body/head tags const hadHtmlTag = /]/i.test(originalHtml); const hadBodyTag = /]/i.test(originalHtml); const hadHeadTag = /]/i.test(originalHtml); // Extract whitespace patterns from original const htmlBodyWhitespace = originalHtml.match(/]*>([\s]*)([\s]*)<\/html>/i)?.[1] || ""; let html: string; if (hadHtmlTag || hadBodyTag) { // Original had structure, keep it but normalize html = $.html(); // Remove if original didn't have it if (!hadHeadTag) { html = html.replace(/<\/head>/g, ""); } // Restore whitespace between html/body tags if (htmlBodyWhitespace) { html = html.replace( /]*)>${htmlBodyWhitespace}<\/html>/i, `${bodyCloseWhitespace}` ); } } else { // Original was a fragment, extract just body contents html = $("body").html() || ""; } // Convert to self-closing tags like BeautifulSoup html = html.replace(/
/gi, "
"); html = html.replace(/

/gi, "

"); // Normalize excessive whitespace (more than 4 consecutive newlines) like BeautifulSoup html = html.replace(/\n{5,}/g, "\n\n\n\n"); // Restore original tag casing html = restoreTagCasing(html, originalCasing); return html.trim(); } export class Unquote { private originalHtml: string | null; private originalText: string | null; private _html: string | null = null; private _cleanHtml: string | null = null; private _text: string | null = null; private _markdown: string | null = null; private _quoteHtml: string | null = null; private _parsed: boolean = false; private _cachedCheerio: CheerioAPI | null = null; constructor(html: string | null, text: string | null) { if (!html && !text) { throw new Error("You must provide at least one of html or text"); } this.originalHtml = html ? html.replace(/\xa0/g, " ") : null; this.originalText = text ? text.replace(/\xa0/g, " ") : null; } /** * Get cached Cheerio instance, creating it on first access */ private getCheerio(): CheerioAPI | null { if (!this.originalHtml) return null; if (!this._cachedCheerio) { this._cachedCheerio = cheerio.load(this.originalHtml); } return this._cachedCheerio; } /** * Capture HTML of elements before removing them and store as quote */ private captureAndRemove($: CheerioAPI, ...elements: CheerioNode[]): void { const captured: string[] = []; for (const el of elements) { if (el && el.length) { captured.push($.html(el)); el.remove(); } } if (captured.length) { this._quoteHtml = captured.join(""); } } /** * Get the HTML content without quoted replies. * @param options.raw - If true, returns raw HTML. Default: false (cleaned). */ getHtml(options: GetHtmlOptions = {}): string | null { const { raw = false } = options; this.ensureParsed(); if (this._html === null) { if (this.originalHtml) { this._html = this.originalHtml; } else if (this.originalText) { this._html = this.textToHtml(this.originalText); } } // Return raw if requested if (raw) { return this._html; } // Return cleaned version (cached) if (this._cleanHtml === null && this._html) { this._cleanHtml = cleanEmailHtml(this._html); } return this._cleanHtml; } getText(): string | null { this.ensureParsed(); if (this._text === null) { if (this.originalText) { this._text = this.originalText; } else if (this.originalHtml) { this._text = htmlToText(this.originalHtml).trim(); } } return this._text; } getMarkdown(): string | null { this.ensureParsed(); if (this._markdown === null) { // Use cleaned HTML for markdown conversion const html = this.getHtml(); if (html) { this._markdown = htmlToMarkdown(html).trim(); } } return this._markdown; } /** * Get the quoted/replied content that was removed. * Returns the HTML of the quoted reply section, or null if no quote was found. * @param options.raw - If true, returns raw HTML. Default: false (cleaned). */ getQuote(options: GetHtmlOptions = {}): string | null { const { raw = false } = options; this.ensureParsed(); if (!this._quoteHtml) return null; return raw ? this._quoteHtml : cleanEmailHtml(this._quoteHtml); } /** * Ensure parse() has been called, calling it if not yet done */ private ensureParsed(): void { if (!this._parsed) { this.parse(); } } private parseHtml($: CheerioAPI): boolean { // Moz (must be before Apple) const moz = $("div.moz-cite-prefix"); if (moz.length) { const nextSibling = moz.next('blockquote[type="cite"]'); if (nextSibling.length) { this.captureAndRemove($, moz, nextSibling); return true; } } // Freshdesk const freshdesk = $("div.freshdesk_quote"); if (freshdesk.length) { this.captureAndRemove($, freshdesk); return true; } // Front const front = $(".front-blockquote"); if (front.length) { this.captureAndRemove($, front); return true; } // Missive const missive = $("div.missive_quote"); if (missive.length) { this.captureAndRemove($, missive); return true; } // Outreach const outreach = $("div.outreach-quote"); if (outreach.length) { this.captureAndRemove($, outreach); return true; } // Hubspot const hubspot = $("div.hs_reply"); if (hubspot.length) { this.captureAndRemove($, hubspot); return true; } // Spark const spark = $('[name="messageReplySection"]'); if (spark.length) { this.captureAndRemove($, spark); return true; } // Gmail const gmail = $(".gmail_attr"); if (gmail.length) { const parent = gmail.parent(); const parentClasses = parent.attr("class") || ""; if ( parentClasses.includes("gmail_quote_container") || parentClasses.includes("gmail_quote") ) { this.captureAndRemove($, parent); return true; } } // Another Gmail const gmail2 = $("div.gmail_quote"); if (gmail2.length) { const parent = gmail2.parent(); const parentClasses = parent.attr("class") || ""; if (parentClasses.includes("gmail_extra")) { this.captureAndRemove($, parent); return true; } } // Gmail, fallback - handle blockquote.gmail_quote inside div.gmail_quote const gmailFallback = $("blockquote.gmail_quote"); if (gmailFallback.length) { // Check if blockquote is inside a div.gmail_quote - if so, remove the whole div const parentDiv = gmailFallback.closest("div.gmail_quote"); if (parentDiv.length) { this.captureAndRemove($, parentDiv); } else { this.captureAndRemove($, gmailFallback); } return true; } // Gmail div without blockquote but with quote content (cleanup empty gmail_quote divs) const gmailDivOnly = $("div.gmail_quote"); if (gmailDivOnly.length) { // Remove if it only contains whitespace or is empty after other processing const textContent = gmailDivOnly.text().trim(); if (!textContent || gmailDivOnly.children().length === 0) { this.captureAndRemove($, gmailDivOnly); return true; } } // Yahoo const yahoo = $("div.yahoo_quoted"); if (yahoo.length) { this.captureAndRemove($, yahoo); return true; } // Ymail const ymail = $("div#ymail_android_signature"); if (ymail.length) { const nextEls = ymail.nextAll().not("script, style"); this.captureAndRemove($, ymail, nextEls); return true; } // Yahoo quoted mail const ymail2 = $("p.yahoo-quoted-begin"); if (ymail2.length) { const nextEls = ymail2.nextAll().not("script, style"); this.captureAndRemove($, ymail2, nextEls); return true; } // GetFernand.com const fernand = $("div.fernand_quote"); if (fernand.length) { this.captureAndRemove($, fernand); return true; } // Intercom const intercom = $("div.history"); if (intercom.length) { this.captureAndRemove($, intercom); return true; } // Reply const reply = $("p#reply-intro"); if (reply.length) { const blockquote = reply.next('blockquote[type="cite"]'); if (blockquote.length) { this.captureAndRemove($, reply, blockquote); return true; } } // MsOffice const msoffice = $("div#mail-editor-reference-message-container"); if (msoffice.length) { this.captureAndRemove($, msoffice); return true; } // MsOutlook // Find all potential Outlook quote divs by structure, then validate style with regex // This handles spacing variations like "border-top:solid" vs "border-top: solid" const msoutlookCandidates = $("div[style]>p.MsoNormal>b"); let msoutlook: ReturnType | null = null; msoutlookCandidates.each((_, el) => { const parentDiv = $(el).parent().parent(); const styleAttr = parentDiv.attr("style") || ""; // Match border:none (or border : none) followed by border-top:solid (or border-top : solid) // Handles spaces before and after colons and semicolons if (/border\s*:\s*none\s*[;,]\s*border-top\s*:\s*solid/i.test(styleAttr)) { msoutlook = $(el); return false; // break the loop } }); if (msoutlook && $(msoutlook).length) { const msoRoot = $(msoutlook).parent().parent(); const style = (msoRoot.attr("style") || "") .replace(/cm/g, "in") .replace(/pt/g, "in") .replace(/mm/g, "in"); // Normalize spaces for the endsWith check: // - Remove spaces around colons and semicolons // - Collapse multiple spaces to single space // - Trim leading/trailing whitespace const normalizedStyle = style .replace(/\s*:\s*/g, ":") .replace(/\s*;\s*/g, ";") .replace(/\s+/g, " ") .trim(); if (normalizedStyle.endsWith(" 1.0in;padding:3.0in 0in 0in 0in")) { let root = msoRoot; const parentContents = msoRoot.parent().children(); const htmlElements = parentContents.filter((_, el) => $.html(el).startsWith("<") ); if (htmlElements.length === 1) { root = msoRoot.parent(); } // Capture quote before removing const nextEls = root.nextAll(); this._quoteHtml = $.html(root) + nextEls .map((_, el) => $.html(el)) .get() .join(""); nextEls.remove(); root.remove(); return true; } } // Outlook const outlook = $("div#divRplyFwdMsg"); if (outlook.length) { const prevHr = outlook.prevAll("hr").first(); if (prevHr.length) { const nextEls = outlook.nextAll(); this._quoteHtml = $.html(prevHr) + $.html(outlook) + nextEls .map((_, el) => $.html(el)) .get() .join(""); nextEls.remove(); outlook.remove(); prevHr.remove(); return true; } } // ProtonMail const proton = $(".protonmail_quote"); if (proton.length) { this.captureAndRemove($, proton); return true; } // Trix const trix = $("div.trix-content>blockquote"); if (trix.length) { this.captureAndRemove($, trix); return true; } // ZMail const zmail = $("div.zmail_extra"); if (zmail.length) { const previous = zmail.prev(); if (previous.hasClass("zmail_extra_hr")) { this.captureAndRemove($, previous, zmail); } else { this.captureAndRemove($, zmail); } return true; } // Zendesk const zendesk = $("div.quotedReply>blockquote"); if (zendesk.length) { this.captureAndRemove($, zendesk.parent()); return true; } // Zoho const zoho = $('div[title="beforequote:::"]'); if (zoho.length) { const nextEls = zoho.nextAll(); const prevSibling = zoho.prev(); const hasPrevSep = prevSibling.text().trim().startsWith("---"); this._quoteHtml = (hasPrevSep ? $.html(prevSibling) : "") + $.html(zoho) + nextEls .map((_, el) => $.html(el)) .get() .join(""); nextEls.remove(); if (hasPrevSep) { prevSibling.remove(); } zoho.remove(); return true; } // Notion const notion = $("blockquote.notion-mail-quote"); if (notion.length) { this.captureAndRemove($, notion); return true; } // Tutanota const tutanota = $("blockquote.tutanota_quote"); if (tutanota.length) { const prevDiv = tutanota.prev("div"); if (prevDiv.length) { this.captureAndRemove($, prevDiv, tutanota); } else { this.captureAndRemove($, tutanota); } return true; } // Some odd Yahoo ydp const ydp = $('div[class$="yahoo_quoted"]'); if (ydp.length) { const id = ydp.attr("id") || ""; if (id.includes("yahoo_quoted")) { this.captureAndRemove($, ydp); return true; } } // QT const qt = $('blockquote[type="cite"]#qt'); if (qt.length) { this.captureAndRemove($, qt); return true; } // Alimail const alimail = $("div.alimail-quote"); if (alimail.length) { const parent = alimail.parent(); if (parent.is("blockquote")) { this.captureAndRemove($, parent); return true; } } // Some Apple version const apple = $( 'html[class*="apple-mail"] blockquote[type="cite"]>div[dir]' ); if (apple.length) { const blockquoteParent = apple.parent(); let previousSibling = blockquoteParent.prev(); // Skip text nodes while (previousSibling.length && !previousSibling.is("*")) { previousSibling = previousSibling.prev(); } if (previousSibling.length && previousSibling.attr("dir")) { const childBlockquote = previousSibling.find("blockquote"); if (childBlockquote.length) { this._quoteHtml = $.html(previousSibling) + $.html(blockquoteParent); previousSibling.remove(); } else { this._quoteHtml = $.html(blockquoteParent); } } else { this._quoteHtml = $.html(blockquoteParent); } blockquoteParent.remove(); return true; } // Apple interchange const appleIc = $("br.Apple-interchange-newline"); if (appleIc.length) { const parent = appleIc.parent(); if (parent.is("blockquote")) { this.captureAndRemove($, parent); return true; } } // Another apple const apple2 = $('meta[name="x-apple-disable-message-reformatting"]'); if (apple2.length) { const blockquote = apple2.closest("blockquote"); if (blockquote.length) { const prevDiv = blockquote.prev("div"); if (prevDiv.length) { const nestedBlockquote = prevDiv.find('blockquote[type="cite"]'); if (nestedBlockquote.length) { this._quoteHtml = $.html(prevDiv) + $.html(blockquote); prevDiv.remove(); } else { this._quoteHtml = $.html(blockquote); } } else { this._quoteHtml = $.html(blockquote); } blockquote.remove(); return true; } } // OneComWebmail const onecom = $("div.oneComWebmail-html"); if (onecom.length) { const parent = onecom.parent(); if (parent.is("blockquote")) { this.captureAndRemove($, parent); } } // NH* const nh = $("div.nh_extra"); if (nh.length) { this.captureAndRemove($, nh); } // GWP starting classname const gwp = $('div[id^="gwp"]'); if (gwp.length) { const parent = gwp.parent(); const tagName = parent.prop("tagName") || ""; if (tagName.includes("@")) { this.captureAndRemove($, parent); return true; } } return false; } private clearText(text: string): string { return text.replace(CLEAR_TEXT_REGEX, ""); } parse(): boolean { if (this._parsed) return false; this._parsed = true; this._text = this.originalText; this._html = this.originalHtml; const $ = this.getCheerio(); if (this._html && $) { if (this.parseHtml($)) { this._html = getInnerHtml($, this.originalHtml!); this._text = htmlToText(this._html).trim(); // Reset cached cheerio since HTML changed this._cachedCheerio = null; return true; } // Try to locate any class="*quote*" and debug it const quoteElements = $('[class*="quote"]'); if (quoteElements.length) { this.quoteFound($); } // Try to locate any class="*sign*" and debug it const signElements = $('[class*="sign"]'); if (signElements.length) { this.signFound($); } } if (!this._text && this._html) { this._text = htmlToText(this._html).trim(); } // Content based data using regex let match: RegExpMatchArray | null = null; for (const pattern of patterns) { if (this._text) { match = this._text.match(pattern); if (match) { break; } } } if (!match) { if (this._text) { this.noPatternsFound(this._text); } return false; } // Get the index of the match const matchIndex = this._text!.indexOf(match[0]); this._text = this._text!.substring(0, matchIndex).trim(); if (this._html) { // Use fresh cheerio load for manipulation after pattern match const $fresh = cheerio.load(this._html); let content = ""; let matchingTag: CheerioNode | null = null; const lookupText = this.clearText(match[0]); // Iterate through all text nodes const textNodes: CheerioNode[] = []; const collectTextNodes = (node: CheerioNode) => { node.contents().each((_: number, el: any) => { const $el = $fresh(el); if (el.type === "text") { textNodes.push($el); } else if (el.type === "tag") { collectTextNodes($el); } }); }; collectTextNodes($fresh.root()); for (const textNode of textNodes) { const currentText = textNode.text(); if (!currentText) continue; content += this.clearText(currentText); if (content.includes(lookupText)) { matchingTag = textNode; break; } } if (matchingTag) { // Capture elements after the matching tag's parent before removing const afterElements = matchingTag.parent().nextAll(); const capturedAfter: string[] = []; afterElements.each((_: number, el: any) => { capturedAfter.push($fresh.html(el)); }); afterElements.remove(); // Also capture and remove the blockquote that follows if present const blockquoteAfter = matchingTag.parent().next("blockquote"); if (blockquoteAfter.length) { capturedAfter.push($fresh.html(blockquoteAfter)); blockquoteAfter.remove(); } // Go up from the text node until we find an element that contains ONLY the quote // If findIndex === 0, the element contains only the quote, remove it // If findIndex > 0, the element has other content, keep the previous element let currentTag: CheerioNode | null = matchingTag; let previousTag: CheerioNode | null = null; let found = false; let tagToRemove: CheerioNode | null = null; while (currentTag && currentTag.length) { const tagContent = currentTag.text(); const clearedContent = this.clearText(tagContent); const findIndex = clearedContent.indexOf(lookupText); if (findIndex === 0) { // This element contains only the quote text // Keep traversing up to find the highest element that only contains quote tagToRemove = currentTag; previousTag = currentTag; currentTag = currentTag.parent(); found = true; } else if (findIndex > 0) { // This element has other content before the quote // Remove the previous element (which only had quote) if (tagToRemove) { found = true; } break; } else { // Quote not found at all - shouldn't happen break; } } if (found && tagToRemove && tagToRemove.length) { // Check if it's not the root const tagName = tagToRemove.prop("tagName"); if ( tagName && tagName.toLowerCase() !== "html" && tagName.toLowerCase() !== "body" ) { // Capture the quote HTML before removing this._quoteHtml = $fresh.html(tagToRemove) + capturedAfter.join(""); const parentOfRemoved: CheerioNode | null = tagToRemove.parent(); tagToRemove.remove(); // Clean up empty parent elements let current = parentOfRemoved; while (current && current.length) { const currentTagName = current.prop("tagName"); if ( !currentTagName || currentTagName.toLowerCase() === "html" || currentTagName.toLowerCase() === "body" ) { break; } if (!current.text().trim() && !current.find("img").length) { const nextParent = current.parent(); current.remove(); current = nextParent; } else { break; } } } } else if (capturedAfter.length) { // If no tag to remove but we captured after elements, store them as quote this._quoteHtml = capturedAfter.join(""); } this._html = getInnerHtml($fresh, this.originalHtml!); } else { // Rebuild the html from the text this._html = this.textToHtml(this._text); } } // Reset cached cheerio since HTML may have changed this._cachedCheerio = null; return true; } textToHtml(data: string | null): string | null { if (!data) { return null; } return (marked.parse(data, { async: false }) as string).trim(); } /** Override this method to handle cases when a quote class is found but not processed */ protected quoteFound(_$: CheerioAPI): void { // Can be overridden } /** Override this method to handle cases when a signature class is found but not processed */ protected signFound(_$: CheerioAPI): void { // Can be overridden } /** Override this method to handle cases when no patterns matched */ protected noPatternsFound(_text: string): void { // Can be overridden } } export class VerboseUnquote extends Unquote { protected quoteFound($: CheerioAPI): void { console.log("Quote found in HTML structure"); console.log($.html().substring(0, 100)); } protected signFound($: CheerioAPI): void { console.log("Signature found in HTML structure"); console.log($.html().substring(0, 100)); } protected noPatternsFound(text: string): void { console.log("No patterns found in text"); console.log(text.substring(0, 100)); } }