import { Readability } from '@mozilla/readability'; import { JSDOM, VirtualConsole } from 'jsdom'; import type { ExtractedContent } from '../types.js'; export type ReadableExtractionMode = 'readability' | 'fallback'; export type SafeReadableExtraction = { mode: ReadableExtractionMode; content: ExtractedContent; }; export function extractReadableContent(html: string, maxLength = 4000): ExtractedContent { let stylesheetError: Error | undefined; const virtualConsole = new VirtualConsole(); virtualConsole.on('jsdomError', (error) => { if (!stylesheetError && error.message.includes('Could not parse CSS stylesheet')) { stylesheetError = error; } }); const dom = new JSDOM(html, { url: 'https://example.com', virtualConsole }); if (stylesheetError) { throw stylesheetError; } const article = new Readability(dom.window.document).parse(); const rawText = (article?.textContent ?? dom.window.document.body.textContent ?? '').trim(); const text = rawText.slice(0, maxLength); const fallbackTitle = dom.window.document.title || undefined; return { title: article?.title ?? fallbackTitle, byline: article?.byline || undefined, text }; } function decodeHtmlEntities(text: string): string { return text .replace(/ /gi, ' ') .replace(/&/gi, '&') .replace(/</gi, '<') .replace(/>/gi, '>') .replace(/"/gi, '"') .replace(/'/gi, "'") .replace(/'/gi, "'") .replace(///gi, '/') .replace(/(\d+);/g, (_, code) => String.fromCharCode(Number(code))) .replace(/([\da-f]+);/gi, (_, code) => String.fromCharCode(parseInt(code, 16))); } function extractTitle(html: string): string | undefined { const match = html.match(/