// @ts-nocheck /** * @module research/extractor/html-to-content/html-utils * @description Research library module. */ /** * Converts URL-safe escaped HTML codes like &"'`’ & to standard HTML or in reverse. * @param {string} str - The string to process. * @param {boolean} toStandardHTML default=true - If true, converts url-safe codes * to standard HTML. If false, converts standard HTML to url-safe codes. * @return {string} The processed string. * @category HTML Utilities * @example * var normalHTML = convertURLSafeHTMLToHTML('<p>This & that © 2023 '+ * '"Quotes"'Apostrophes' €100 ☺</p>', true) * console.log(normalHTML) // "

This & that \u00a9 2023 "Quotes" 'Apostrophes' \u20ac100 \u263a

" */ export function convertURLSafeHTMLToHTML(str, toStandardHTML = true) { const entityMap = { "&": "&", "<": "<", ">": ">", '"': """, " ": " ", "'": "'", "`": "`", "\u00a2": "¢", "\u00a3": "£", "\u00a5": "¥", "\u20ac": "€", "\u00a9": "©", "\u00ae": "®", "\u2122": "™", }; // Add numeric character references for Latin-1 Supplement characters for (let i = 160; i <= 255; i++) { entityMap[String.fromCharCode(i)] = `&#${i};`; } if (toStandardHTML) { // Create a reverse mapping for unescaping const reverseEntityMap = Object.fromEntries( Object.entries(entityMap).map(([k, v]) => [v, k]) ); // Add alternative representations reverseEntityMap["'"] = "'"; reverseEntityMap["«"] = "\u00ab"; reverseEntityMap["»"] = "\u00bb"; // Regex to match all types of HTML entities const entityRegex = new RegExp( Object.keys(reverseEntityMap).join("|") + "|&#[0-9]+;|&#x[0-9a-fA-F]+;", "g" ); str = str.replace(entityRegex, (entity) => { if (entity.startsWith("&#x")) { // Convert hexadecimal numeric character reference return String.fromCharCode(parseInt(entity.slice(3, -1), 16)); } else if (entity.startsWith("&#")) { // Convert decimal numeric character reference return String.fromCharCode(parseInt(entity.slice(2, -1), 10)); } // Convert named entity return reverseEntityMap[entity] || entity; }); str = str.replace(/[\u0300-\u036f]/g, ""); //special chars return str; } else { // Regex to match all characters that need to be escaped const charRegex = new RegExp(`[${Object.keys(entityMap).join("")}]`, "g"); return str.replace(charRegex, (char) => entityMap[char]); } } /** * Convert relative URL to absolute URL using base URL. * @param {string} base base url of the domain * @param {string} relative partial urls like ../images/image.jpg #hash * @returns {string} absolute URL * @example * var absoluteURL = convertURLToAbsoluteURL('https://example.com', 'images/image.jpg') * console.log(absoluteURL) // Returns: "https://example.com/images/image.jpg" * var absoluteURL = convertURLToAbsoluteURL('https://example.com', '//images/image.jpg') * console.log(absoluteURL) // Returns: "https:images/image.jpg" * @category HTML Utilities * @author [vtempest (2025)](https://github.com/vtempest) */ export function convertURLToAbsoluteURL(base, relative) { // remove the %20 codes like data:image/svg+xml,%3Csvg%20x relative = decodeURI(relative); base = decodeURI(base); if ( relative.includes("data:") || relative.startsWith("#") || relative.startsWith("http") ) return relative; // Remove hash from base URL base = base.replace(/#.*$/, ""); // If relative URL starts with '//', add scheme from base if (relative.startsWith("//")) return base.split("://")[0] + ":" + relative; // If relative URL starts with '/', replace everything after the host in base if (relative[0] === "/") { const matchdomain = base.match(/^(https?:\/\/[^\/]+)/i); const domain = matchdomain ? matchdomain[1] : null; return domain + relative; } // Handle relative URLs if (relative.startsWith("../")) { base = base.replace(/\/[^\/]+$/, ""); while (relative.substring(0, 3) === "../") { relative = relative.substring(3); base = base.replace(/\/[^\/]+$/, ""); } relative = relative.replace(/^\.\//, ""); } // Combine base and relative // if (relative.startsWith("/")) { base = base.replace(/\/[^\/]+$/, ""); return base.replace(/\/+$/, "") + relative; } else { return base.split("/").slice(0, -1).join("/") + "/" + relative; } } import { marked } from "marked"; import Prism from "prismjs"; import "prismjs/components/prism-markup"; import "prismjs/components/prism-css"; import "prismjs/components/prism-javascript"; import "prismjs/components/prism-typescript"; import "prismjs/components/prism-jsx"; import "prismjs/components/prism-tsx"; import "prismjs/components/prism-python"; import "prismjs/components/prism-bash"; import "prismjs/components/prism-json"; import "prismjs/components/prism-yaml"; import "prismjs/components/prism-markdown"; import "prismjs/components/prism-sql"; import "prismjs/components/prism-rust"; import "prismjs/components/prism-go"; import "prismjs/components/prism-java"; import "prismjs/components/prism-c"; import "prismjs/components/prism-cpp"; // Configure marked once at module load with Prism.js syntax highlighting. // marked v17 removed the `highlight` option from setOptions, so highlighting // is wired via a custom `code` renderer instead. marked.use({ renderer: { code({ text, lang }) { const language = lang && Prism.languages[lang] ? lang : null; const highlighted = language ? Prism.highlight(text, Prism.languages[language], language) : text .replace(/&/g, "&") .replace(//g, ">"); const cls = language ? ` class="language-${language}"` : ""; return `
${highlighted}
\n`; }, }, }); /** * Converts Markdown text to HTML. It handles the following Markdown elements: * - Headers (h1 to h6) * - Bold text * - Italic text * - Unordered lists * - Ordered lists * - Paragraphs * - Images * - Links * - Code blocks * @param {string} content - The Markdown or HTML content to be converted. * @param {boolean} toHtml - default=true - If true, converts Markdown to HTML. * If false, converts HTML to Markdown. * @returns {string} The resulting HTML string. * @category HTML Utilities * @example * const markdown = "# Header\n\nThis is **bold** and *italic* text.\n\n* List item 1\n* List item 2"; * const html = convertMarkdownToHTML(markdown); * console.log(html); * // Output: * //

Header

* //

This is bold and italic text.

* // */ export function convertMarkdownToHTML(content, toHtml = true) { if (!toHtml) return convertHTMLToMarkdown(content); return content?.length ? marked.parse(content) : ""; } /** * Escape the HTML-significant characters in a raw string so it can be * safely embedded inside generated HTML (used for code spans/blocks). * @param {string} str * @returns {string} */ function escapeHTMLChars(str) { return String(str) .replace(/&/g, "&") .replace(//g, ">"); } /** * Apply inline-level Markdown regexp replacements (images, links, bold, * italic, strikethrough) to a single already-block-parsed line of text. * Inline code spans are expected to already be swapped out for placeholders * so their contents are never touched here. * @param {string} text * @returns {string} */ function applyInlineMarkdown(text) { return text // Images: ![alt](src "title") -- must run before links .replace( /!\[([^\]]*)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)/g, (_m, alt, src, title) => `${alt}` ) // Links: [text](href "title") .replace( /\[([^\]]+)\]\(([^)\s]+)(?:\s+"([^"]*)")?\)/g, (_m, label, href, title) => `${label}` ) // Bold: **text** or __text__ .replace(/\*\*([^*]+)\*\*/g, "$1") .replace(/__([^_]+)__/g, "$1") // Italic: *text* or _text_ (avoid matching inside words for `_`) .replace(/\*([^*\n]+)\*/g, "$1") .replace(/(^|[^A-Za-z0-9_])_([^_\n]+)_(?=[^A-Za-z0-9_]|$)/g, "$1$2") // Strikethrough: ~~text~~ .replace(/~~([^~]+)~~/g, "$1"); } /** * Convert a Markdown document to formatted HTML using regular expressions to * detect Markdown syntax. Unlike {@link convertMarkdownToHTML} (which relies on * the `marked` library), this is a dependency-free, self-contained converter * intended for post-processing content returned as Markdown (e.g. from the * JINA reader fallback in the scraper). * * Supported block elements: ATX headers (`#`..`######`), fenced code blocks * (```lang), blockquotes (`>`), unordered lists (`-`, `*`, `+`), ordered lists * (`1.`, `1)`), horizontal rules (`---`, `***`, `___`) and paragraphs. * Supported inline elements: bold, italic, strikethrough, inline code, images * and links. * * @param {string} markdown - The Markdown content to convert. * @returns {string} The resulting formatted HTML string. * @category HTML Utilities * @example * convertMarkdownToFormattedHTML("# Title\n\nSome **bold** text."); * // => "

Title

\n

Some bold text.

" */ export function convertMarkdownToFormattedHTML(markdown) { if (!markdown || typeof markdown !== "string") return ""; let text = markdown.replace(/\r\n?/g, "\n"); // 1. Pull fenced code blocks out first so their contents are never parsed // as Markdown. Each is replaced by a placeholder restored at the end. const codeBlocks = []; text = text.replace( /```([^\n`]*)\n([\s\S]*?)```/g, (_m, lang, code) => { const language = (lang || "").trim(); const cls = language ? ` class="language-${language}"` : ""; const body = escapeHTMLChars(code.replace(/\n$/, "")); codeBlocks.push(`
${body}
`); return `\u0000CB${codeBlocks.length - 1}\u0000`; } ); // 2. Pull inline code spans out next for the same reason. const inlineCodes = []; text = text.replace(/`([^`\n]+)`/g, (_m, code) => { inlineCodes.push(`${escapeHTMLChars(code)}`); return `\u0000IC${inlineCodes.length - 1}\u0000`; }); const lines = text.split("\n"); const out = []; let inUl = false; let inOl = false; let inBlockquote = false; let paragraph = []; const flushParagraph = () => { if (paragraph.length) { out.push(`

${applyInlineMarkdown(paragraph.join(" "))}

`); paragraph = []; } }; const closeLists = () => { if (inUl) { out.push(""); inUl = false; } if (inOl) { out.push(""); inOl = false; } }; const closeBlockquote = () => { if (inBlockquote) { out.push(""); inBlockquote = false; } }; for (const line of lines) { // Standalone fenced-code-block placeholder line const cb = line.match(/^\u0000CB(\d+)\u0000$/); if (cb) { flushParagraph(); closeLists(); closeBlockquote(); out.push(codeBlocks[Number(cb[1])]); continue; } // Blank line closes open blocks if (/^\s*$/.test(line)) { flushParagraph(); closeLists(); closeBlockquote(); continue; } // Horizontal rule: ---, ***, ___ (3+) if (/^\s*([-*_])(?:\s*\1){2,}\s*$/.test(line)) { flushParagraph(); closeLists(); closeBlockquote(); out.push("
"); continue; } // ATX header: # .. ###### const header = line.match(/^\s*(#{1,6})\s+(.+?)\s*#*\s*$/); if (header) { flushParagraph(); closeLists(); closeBlockquote(); const level = header[1].length; out.push(`${applyInlineMarkdown(header[2])}`); continue; } // Blockquote: > text const bq = line.match(/^\s*>\s?(.*)$/); if (bq) { flushParagraph(); closeLists(); if (!inBlockquote) { out.push("
"); inBlockquote = true; } out.push(`

${applyInlineMarkdown(bq[1])}

`); continue; } closeBlockquote(); // Unordered list item: -, *, + const ul = line.match(/^\s*[-*+]\s+(.+)$/); if (ul) { flushParagraph(); if (inOl) { out.push(""); inOl = false; } if (!inUl) { out.push(""); inUl = false; } if (!inOl) { out.push("
    "); inOl = true; } out.push(`
  1. ${applyInlineMarkdown(ol[1])}
    2. `); continue; } // Otherwise accumulate into the current paragraph closeLists(); paragraph.push(line.trim()); } flushParagraph(); closeLists(); closeBlockquote(); let html = out.join("\n"); // Restore inline code placeholders html = html.replace(/\u0000IC(\d+)\u0000/g, (_m, i) => inlineCodes[Number(i)]); return html.trim(); } export function convertHTMLToMarkdown(html) { var markdown = html // Convert headers .replace(/(.*?)<\/h[1-6]>/g, (match, level, content) => { return "#".repeat(parseInt(level)) + " " + content.trim() + "\n\n"; }) // Convert bold text .replace(/(.*?)<\/strong>/g, "**$1**") .replace(/(.*?)<\/b>/g, "**$1**") // Convert italic text .replace(/(.*?)<\/em>/g, "*$1*") // Convert unordered lists .replace(/