import * as cheerio from 'cheerio'; import TurndownService from 'turndown'; import { truncateUtf16 } from './bounded-text'; import { MAX_CONTENT_SIZE } from './constants'; // Create a singleton turndown instance const turndownService = new TurndownService({ codeBlockStyle: 'fenced', fence: '```', emDelimiter: '*', strongDelimiter: '**', linkStyle: 'inlined', }); // Customize heading style (use # instead of underlining) turndownService.addRule('heading', { filter: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'], replacement(content, node) { const level = parseInt(node.nodeName.substring(1), 10); return `\n\n${'#'.repeat(level)} ${content}\n\n`; }, }); /** Schemes we never want to emit as live links (XSS / data-exfil vectors). */ const UNSAFE_SCHEME = /^\s*(?:javascript|data|vbscript):/i; /** * Make a URL safe to embed in a markdown `(...)` link destination. * The TUI markdown parser (src/tui/markdown/parse.ts) accepts only ONE level * of balanced parens and treats a stray `)`, whitespace, or any deeper nesting * as the end of the destination. Encode the characters the parser can't handle * so links never truncate: spaces, unbalanced `)`, and parens nested past one * level. A single level of balanced parens (e.g. Wikipedia URLs) survives. */ function sanitizeLinkUrl(href: string): string { let result = ''; let depth = 0; for (const ch of href) { if (ch === ' ') { result += '%20'; } else if (ch === '(') { if (depth >= 1) { // Nesting past one level — the parser can't follow it, so encode. result += '%28'; } else { depth++; result += ch; } } else if (ch === ')') { if (depth > 0) { depth--; result += ch; } else { // Unbalanced closing paren would terminate the link early — encode it. result += '%29'; } } else { result += ch; } } return result; } // Handle links: drop unsafe schemes and keep destinations parseable by the TUI // markdown parser. The title attribute is intentionally dropped: the standard // `(url "title")` form puts a space inside the destination, which the TUI parser // treats as the end of the URL, so emitting it would truncate the link. turndownService.addRule('links', { filter: 'a', replacement(content, node) { const href = node.getAttribute('href'); if (!href) return content; // Render javascript:/data:/vbscript: links as plain text, never as live links. if (UNSAFE_SCHEME.test(href)) return content; return `[${content}](${sanitizeLinkUrl(href)})`; }, }); /** * Convert HTML to markdown */ export function htmlToMarkdown(html: string): string { if (!html) return ''; return turndownService.turndown(html); } /** * Whether stored feed/article content would render as visible text or media. * Raw whitespace, entity-only values, and empty HTML containers are not useful * content even though their source strings are technically non-empty. */ export function hasRenderableContent(content: string | null | undefined): content is string { if (typeof content !== 'string') return false; const bounded = truncateUtf16(content, MAX_CONTENT_SIZE) ?? ''; return bounded.trim().length > 0 && htmlToMarkdown(bounded).trim().length > 0; } /** * Strip HTML tags and return plain text. * Uses cheerio so HTML entities (&, ’, ...) are decoded rather than * left as raw entity text in summaries. */ export function stripHtml(html: string): string { if (!html) return ''; return cheerio.load(html).text().trim(); } /** * Check if string contains HTML tags */ export function containsHtml(str: string): boolean { return /<[a-z][\s\S]*>/i.test(str); } /** * Escape special characters for XML output */ export function escapeXml(str: string): string { // XML 1.0 accepts TAB, LF, and CR among C0 controls, plus scalar values in // its documented ranges. Iterate by Unicode code point so valid astral // characters survive while lone surrogate halves and terminal controls do // not make an otherwise escaped OPML document invalid (or unsafe to print). const xmlSafe = Array.from(str) .filter((character) => { const codePoint = character.codePointAt(0); return ( codePoint !== undefined && (codePoint === 0x9 || codePoint === 0xa || codePoint === 0xd || (codePoint >= 0x20 && codePoint <= 0xd7ff) || (codePoint >= 0xe000 && codePoint <= 0xfffd) || (codePoint >= 0x10000 && codePoint <= 0x10ffff)) ); }) .join(''); return xmlSafe .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"') .replace(/'/g, '''); }