import { parser as htmlParser } from "@lezer/html" import type { ExtractedComments, ProseBreak } from "./comments.ts" import type { MarkdownHtmlComment } from "./markdown.ts" // These elements hold code, markup, or preformatted content, never prose. // `code` is also inline, so a semicolon in inline code stays quiet while the // sentence around it still reads as one sentence. const IGNORED_CONTENT_TAGS = new Set(["code", "pre", "script", "style", "textarea"]) // Phrasing elements that wrap words inside one sentence. // They never start a prose block, so a sentence split across them counts as one sentence. // Every other element starts a prose block. // That rule separates a heading, a list item, a table cell, and bare text in a `div`. const INLINE_TAGS = new Set([ "a", "abbr", "b", "bdi", "bdo", "cite", "code", "data", "del", "dfn", "em", "i", "img", "ins", "kbd", "label", "mark", "q", "rp", "rt", "ruby", "s", "samp", "small", "span", "strong", "sub", "sup", "time", "u", "var", "wbr", ]) /** * Find every single-line HTML comment in the document with the lezer parser. * The markdown finder cannot see comments that markdown would read as indented code. */ export function htmlComments(source: string): readonly MarkdownHtmlComment[] { const comments: MarkdownHtmlComment[] = [] const lineStarts = [0] for (let offset = 0; offset < source.length; offset++) { if (source.charCodeAt(offset) === 0x0a) lineStarts.push(offset + 1) } htmlParser.parse(source).iterate({ enter(ref) { if (ref.name !== "Comment") return const text = source.slice(ref.from, ref.to) if (text.includes("\n")) return let lineIndex = lineStarts.length - 1 while (lineIndex > 0 && (lineStarts[lineIndex] as number) > ref.from) lineIndex -= 1 const lineStart = lineStarts[lineIndex] as number comments.push({ line: lineIndex + 1, startColumn: ref.from - lineStart, endColumn: ref.to - lineStart, text, }) }, }) return comments } interface OpenElement { readonly tag: string readonly inline: boolean } /** * Keep the text nodes of an HTML document and blank every other byte with a space. * Attribute values, comments, entity references, tags, and ignored element content all go. * Each prose block becomes one prose break, so sentences never join across a block edge. * The masked lines keep the width of the source lines, so positions stay exact. */ export function extractHtmlProse(source: string): ExtractedComments { const keep = new Uint8Array(source.length) const blockEdges: number[] = [] const open: OpenElement[] = [] let ignoreDepth = 0 htmlParser.parse(source).iterate({ enter(ref) { if (ref.name === "Element") { const tagName = ref.node.firstChild?.getChild("TagName") const tag = tagName === null || tagName === undefined ? "" : source.slice(tagName.from, tagName.to).toLowerCase() const inline = INLINE_TAGS.has(tag) if (!inline) blockEdges.push(ref.from) if (IGNORED_CONTENT_TAGS.has(tag)) ignoreDepth += 1 open.push({ tag, inline }) return } if (ref.name === "Text" && ignoreDepth === 0) keep.fill(1, ref.from, ref.to) }, leave(ref) { if (ref.name !== "Element") return const element = open.pop() if (element === undefined) return if (IGNORED_CONTENT_TAGS.has(element.tag)) ignoreDepth -= 1 if (!element.inline) blockEdges.push(ref.to) }, }) blockEdges.sort((left, right) => left - right) const lines = source.split("\n") const contentStarts: number[] = [] const proseBreaks: ProseBreak[] = [] let lineStart = 0 let edgeIndex = 0 let blockIndex = -1 const maskedLines = lines.map((line, lineIndex) => { const characters = new Array(line.length).fill(" ") let contentStart = -1 for (let column = 0; column < line.length; column += 1) { const offset = lineStart + column while (edgeIndex < blockEdges.length && (blockEdges[edgeIndex] as number) <= offset) { edgeIndex += 1 } if (keep[offset] === 0) continue const character = source[offset] as string characters[column] = character if (/\s/u.test(character)) continue if (contentStart === -1) contentStart = column if (edgeIndex === blockIndex) continue if (blockIndex !== -1) proseBreaks.push({ line: lineIndex, column }) blockIndex = edgeIndex } lineStart += line.length + 1 contentStarts.push(contentStart === -1 ? line.length : contentStart) return characters.join("") }) return { lines: maskedLines, contentStarts, proseBreaks, lineComments: [] } }