import * as nodeHtmlParser from 'node-html-parser' import Turndown from 'turndown' // @ts-ignore import * as turndownPluginGfm from 'turndown-plugin-gfm' import { PageMetadata } from '../../shared/lib/PagesMetadata' import { TransformerConfig } from '../types/config.types' const turndown = new Turndown({ headingStyle: 'atx', fence: '```', emDelimiter: '_', strongDelimiter: '**', linkStyle: 'inlined', linkReferenceStyle: 'full', codeBlockStyle: 'fenced', bulletListMarker: '-', }) turndown.use([turndownPluginGfm.tables, turndownPluginGfm.strikethrough]) const removeElement = ( node: nodeHtmlParser.HTMLElement, ...selectors: string[] ) => { for (const selector of selectors) { const elements = node.querySelectorAll(selector) elements.forEach((el) => el.remove()) } } export const parsePageHtmlContent = async ( html: string, pageMetadata: PageMetadata, config: TransformerConfig, ) => { const parsed = nodeHtmlParser.parse(html, { blockTextElements: { script: false, style: false, }, }) // const head = parsed.querySelector('head') const article = parsed.querySelector('body article') if (!article) { return null } removeElement(article, '.nextra-breadcrumb', 'svg', 'img') article.querySelectorAll('h1, h2, h3, h4, h5, h6').forEach((el) => { removeElement(el, 'a') }) article.querySelectorAll('a').forEach((el) => { el.innerHTML = el.textContent.replaceAll('\n', ' ') }) const metadata: Record = pageMetadata.frontMatter || {} metadata.title = pageMetadata.title metadata.__path__ = JSON.stringify(pageMetadata.path) if (metadata.keywords) { metadata.keywords = metadata.keywords .split(',') .map((keyword: string) => keyword.trim()) } return { metadata, html: article.outerHTML, } }