import { truncateUtf16 } from '../../utils/bounded-text'; import { MAX_HTTP_URL_SIZE } from '../../utils/constants'; import { unescapeMarkdownPunctuation } from '../../utils/markdown-unescape'; import { sanitizeTerminalText } from '../../utils/terminal'; const MAX_TABLE_ROWS = 100; const MAX_TABLE_COLUMNS = 32; const MAX_LIST_INDENT_LEVEL = 32; // ArticleReader already caps stored article text at MAX_CONTENT_SIZE. Keep the // standalone parser bounded too, while allowing the 100KB adversarial-parser // regression and modest callers that use it outside the reader. const MAX_MARKDOWN_INPUT_SIZE = 1_000_000; export interface InlinePart { type: 'text' | 'bold' | 'italic' | 'code' | 'strikethrough' | 'link'; content: string; url?: string; linkIndex?: number; } export interface MarkdownBlock { type: | 'h1' | 'h2' | 'h3' | 'h4' | 'h5' | 'h6' | 'paragraph' | 'blockquote' | 'ulist' | 'olist' | 'codeblock' | 'hr' | 'image' | 'table' | 'blank'; content: string; rawContent?: string; language?: string; level?: number; number?: number; rows?: string[][]; rawRows?: string[][]; tableInlineParts?: InlinePart[][][]; inlineParts?: InlinePart[]; blockquoteLines?: InlinePart[][]; omittedRows?: number; omittedColumns?: number; } export interface ExtractedLink { url: string; text: string; linkIndex: number; } export interface ParsedDocument { blocks: MarkdownBlock[]; links: ExtractedLink[]; } /** True only for absolute web URLs that are safe to hand to the OS browser. */ export function isSafeExternalUrl(value: string): boolean { if (value.length > MAX_HTTP_URL_SIZE) return false; try { const url = new URL(value); return ( (url.protocol === 'http:' || url.protocol === 'https:') && url.hostname.length > 0 && url.username.length === 0 && url.password.length === 0 ); } catch { return false; } } export function shortDomain(url: string): string { try { const u = new URL(url); return u.hostname.replace(/^www\./, ''); } catch { return url.length > 30 ? `${url.slice(0, 30)}...` : url; } } export function parseMarkdown(text: string): MarkdownBlock[] { const boundedText = truncateUtf16(text, MAX_MARKDOWN_INPUT_SIZE) ?? ''; const lines = sanitizeTerminalText(boundedText, { preserveNewlines: true }).split('\n'); const result: MarkdownBlock[] = []; let inCodeBlock = false; let codeLanguage = ''; let codeLines: string[] = []; let consecutiveBlanks = 0; for (let i = 0; i < lines.length; i++) { const rawLine = lines[i] as string; if (inCodeBlock) { if (rawLine.startsWith('```')) { result.push({ type: 'codeblock', content: codeLines.join('\n'), language: codeLanguage, }); inCodeBlock = false; codeLines = []; consecutiveBlanks = 0; } else { codeLines.push(rawLine); } continue; } if (rawLine.startsWith('```')) { codeLanguage = rawLine.slice(3).trim(); inCodeBlock = true; codeLines = []; consecutiveBlanks = 0; continue; } // NOTE: we deliberately do NOT unescape here. Block detection and inline // tokenization must run against the raw (Turndown-escaped) line so that // escaped punctuation like "5 \* 3" or "\[text\](url)" is never treated as // markdown syntax. Unescaping happens later, per plain-text segment, in // parseInline / via unescapeMarkdownPunctuation on already-extracted text // (headings, image alt, table cells). const line = rawLine; if (line.trim() === '') { consecutiveBlanks++; if (consecutiveBlanks <= 1) { result.push({ type: 'blank', content: '' }); } continue; } consecutiveBlanks = 0; const headingMatch = line.match(/^(#{1,6})\s+(.+)/); if (headingMatch) { const level = (headingMatch[1] as string).length; const tag = `h${level}` as MarkdownBlock['type']; const rawContent = headingMatch[2] as string; result.push({ type: tag, content: unescapeMarkdownPunctuation(rawContent), rawContent }); continue; } if (/^(-{3,}|_{3,}|\*{3,})$/.test(line.trim())) { result.push({ type: 'hr', content: '' }); continue; } if (line.startsWith('> ') || line === '>') { const bqLines: string[] = [line.startsWith('> ') ? line.slice(2) : '']; while (i + 1 < lines.length) { const next = lines[i + 1] as string; if (next.startsWith('> ') || next === '>') { bqLines.push(next.startsWith('> ') ? next.slice(2) : ''); i++; } else { break; } } result.push({ type: 'blockquote', content: bqLines.join('\n'), }); continue; } const imgMatch = line.match(/^!\[([^\]]*)\]\(([^)]+)\)/); if (imgMatch) { result.push({ type: 'image', content: unescapeMarkdownPunctuation(imgMatch[1] || '') || 'image' }); continue; } if (line.includes('|') && line.trim().startsWith('|')) { const tableRows: string[][] = []; const rawTableRows: string[][] = []; let omittedRows = 0; let omittedColumns = 0; let ti = i; while (ti < lines.length) { const tline = (lines[ti] as string).trim(); if (!tline.startsWith('|')) break; if (/^\|[-:\s|]+\|$/.test(tline)) { ti++; continue; } const rawCells = tline .split('|') .slice(1, -1) .map((cell) => cell.trim()); const cells = rawCells.map(unescapeMarkdownPunctuation); omittedColumns = Math.max(omittedColumns, Math.max(0, cells.length - MAX_TABLE_COLUMNS)); if (cells.length > 0 && tableRows.length < MAX_TABLE_ROWS) { tableRows.push(cells.slice(0, MAX_TABLE_COLUMNS)); rawTableRows.push(rawCells.slice(0, MAX_TABLE_COLUMNS)); } else if (cells.length > 0) { omittedRows++; } ti++; } if (tableRows.length > 0) { result.push({ type: 'table', content: '', rows: tableRows, rawRows: rawTableRows, omittedRows, omittedColumns, }); i = ti - 1; continue; } } // A leading "\?" tolerates Turndown's defensive escaping of list markers // (it emits "\- item" / "1\. item"), which were genuine list items in the // source HTML and should re-detect as lists. Marker chars elsewhere stay // literal — only this anchored, marker-shaped position re-activates. const ulistMatch = line.match(/^(\s*)\\?[*\-+]\s+(.+)/); if (ulistMatch) { const indent = Math.min(ulistMatch[1]?.length ?? 0, MAX_LIST_INDENT_LEVEL * 2); result.push({ type: 'ulist', content: ulistMatch[2] ?? '', level: Math.floor(indent / 2), }); continue; } const olistMatch = line.match(/^(\s*)(\d+)\\?\.\s+(.+)/); if (olistMatch) { const indent = Math.min(olistMatch[1]?.length ?? 0, MAX_LIST_INDENT_LEVEL * 2); result.push({ type: 'olist', content: olistMatch[3] ?? '', level: Math.floor(indent / 2), number: Number.parseInt(olistMatch[2] as string, 10), }); continue; } result.push({ type: 'paragraph', content: line }); } if (inCodeBlock && codeLines.length > 0) { result.push({ type: 'codeblock', content: codeLines.join('\n'), language: codeLanguage, }); } return result; } interface BracketLink { label: string; url: string; end: number; } export interface InlineParseMetrics { /** Deterministic count of indexed characters, cursor advances, and tokenizer iterations. */ steps: number; } interface MarkerCursor { positions: number[]; index: number; } interface InlineScanIndex { bracketEnds: MarkerCursor; boldDelimiters: MarkerCursor; italicDelimiters: MarkerCursor; codeDelimiters: MarkerCursor; strikeDelimiters: MarkerCursor; matchingParenEnds: Int32Array; } function buildInlineScanIndex(text: string, metrics?: InlineParseMetrics): InlineScanIndex { const bracketEnds: number[] = []; const boldDelimiters: number[] = []; const italicDelimiters: number[] = []; const codeDelimiters: number[] = []; const strikeDelimiters: number[] = []; const matchingParenEnds = new Int32Array(text.length); const parenthesisStack: number[] = []; for (let i = 0; i < text.length; i++) { if (metrics) metrics.steps++; const character = text[i]; if (character === ']') bracketEnds.push(i); if (character === '`') codeDelimiters.push(i); if (character === '*' && text[i + 1] === '*') boldDelimiters.push(i); if (character === '*' && text[i - 1] !== '\\' && text[i - 1] !== '*' && text[i + 1] !== '*') { italicDelimiters.push(i); } if (character === '~' && text[i + 1] === '~') strikeDelimiters.push(i); if (character && /\s/.test(character)) { parenthesisStack.length = 0; } else if (character === '(') { parenthesisStack.push(i); } else if (character === ')') { const opening = parenthesisStack.pop(); if (opening !== undefined) matchingParenEnds[opening] = i + 1; } } return { bracketEnds: { positions: bracketEnds, index: 0 }, boldDelimiters: { positions: boldDelimiters, index: 0 }, italicDelimiters: { positions: italicDelimiters, index: 0 }, codeDelimiters: { positions: codeDelimiters, index: 0 }, strikeDelimiters: { positions: strikeDelimiters, index: 0 }, matchingParenEnds, }; } function nextMarker(cursor: MarkerCursor, minimum: number, metrics?: InlineParseMetrics): number { while (cursor.index < cursor.positions.length && (cursor.positions[cursor.index] as number) < minimum) { cursor.index++; if (metrics) metrics.steps++; } return cursor.positions[cursor.index] ?? -1; } function parseBracketLink( text: string, start: number, image: boolean, scanIndex: InlineScanIndex, metrics?: InlineParseMetrics, ): BracketLink | null { const labelStart = start + (image ? 2 : 1); const labelEnd = nextMarker(scanIndex.bracketEnds, labelStart, metrics); if (labelEnd < 0 || text[labelEnd + 1] !== '(') return null; const urlOpen = labelEnd + 1; const end = scanIndex.matchingParenEnds[urlOpen] ?? 0; return end > 0 ? { label: text.slice(labelStart, labelEnd), url: text.slice(urlOpen + 1, end - 1), end } : null; } function parseBareUrl(text: string, start: number): { url: string; trailer: string; end: number } | null { let end = start; let depth = 0; while (end < text.length) { const character = text[end]; if (!character || /[\s<>]/.test(character)) break; if (character === '(') depth++; if (character === ')') depth--; end++; } let urlEnd = end; while (urlEnd > start) { const last = text[urlEnd - 1]; if (last && /[.,;:!?]/.test(last)) { urlEnd--; continue; } if (last === ')' && depth < 0) { urlEnd--; depth++; continue; } break; } return urlEnd > start ? { url: text.slice(start, urlEnd), trailer: text.slice(urlEnd, end), end } : null; } /** * Linear inline tokenizer. Delimiter positions and balanced parentheses are * indexed once, then consumed through monotonic cursors so malformed openers * never rescan the remaining suffix. */ export function parseInline( rawText: string, startLinkIndex = 0, metrics?: InlineParseMetrics, ): { parts: InlinePart[]; nextLinkIndex: number } { const text = sanitizeTerminalText(truncateUtf16(rawText, MAX_MARKDOWN_INPUT_SIZE) ?? ''); const parts: InlinePart[] = []; let linkIndex = startLinkIndex; let plainStart = 0; let plainPrefix = ''; const scanIndex = buildInlineScanIndex(text, metrics); const pushPlain = (end: number) => { const source = text.slice(plainStart, end); if (plainPrefix || source) { parts.push({ type: 'text', content: unescapeMarkdownPunctuation(plainPrefix + source) }); } plainPrefix = ''; }; for (let i = 0; i < text.length; ) { if (metrics) metrics.steps++; const previous = text[i - 1]; const escaped = previous === '\\'; let end = -1; if (!escaped && text.startsWith('![', i)) { const image = parseBracketLink(text, i, true, scanIndex, metrics); if (image) { pushPlain(i); if (image.label) parts.push({ type: 'text', content: unescapeMarkdownPunctuation(image.label) }); i = image.end; plainStart = i; continue; } } if (!escaped && text[i] === '[' && previous !== '!') { const link = parseBracketLink(text, i, false, scanIndex, metrics); if (link?.label && link.url) { pushPlain(i); parts.push({ type: 'link', content: unescapeMarkdownPunctuation(link.label), url: link.url, linkIndex }); linkIndex++; i = link.end; plainStart = i; continue; } } if (!escaped && text.startsWith('**', i)) { end = nextMarker(scanIndex.boldDelimiters, i + 2, metrics); if (end > i + 2) { pushPlain(i); parts.push({ type: 'bold', content: unescapeMarkdownPunctuation(text.slice(i + 2, end)) }); i = end + 2; plainStart = i; continue; } } if (!escaped && text[i] === '*' && previous !== '*' && text[i + 1] !== '*') { end = nextMarker(scanIndex.italicDelimiters, i + 1, metrics); if (end > i + 1) { pushPlain(i); parts.push({ type: 'italic', content: unescapeMarkdownPunctuation(text.slice(i + 1, end)) }); i = end + 1; plainStart = i; continue; } } if (!escaped && text[i] === '`') { end = nextMarker(scanIndex.codeDelimiters, i + 1, metrics); if (end > i + 1) { pushPlain(i); parts.push({ type: 'code', content: text.slice(i + 1, end) }); i = end + 1; plainStart = i; continue; } } if (!escaped && text.startsWith('~~', i)) { end = nextMarker(scanIndex.strikeDelimiters, i + 2, metrics); if (end > i + 2) { pushPlain(i); parts.push({ type: 'strikethrough', content: unescapeMarkdownPunctuation(text.slice(i + 2, end)) }); i = end + 2; plainStart = i; continue; } } if (!escaped && previous !== '(' && (text.startsWith('https://', i) || text.startsWith('http://', i))) { const bare = parseBareUrl(text, i); if (bare) { pushPlain(i); parts.push({ type: 'link', content: shortDomain(bare.url), url: bare.url, linkIndex }); linkIndex++; plainPrefix = bare.trailer; i = bare.end; plainStart = i; continue; } } i++; } pushPlain(text.length); return { parts, nextLinkIndex: linkIndex }; } /** * Parse a markdown document into blocks + a flat link list. * * The blocks returned have inline content pre-parsed (`inlineParts` for * paragraph/list, `blockquoteLines` for blockquote). Each link gets a stable * `linkIndex` matching its position in `links`, used by hint mode to map * keypresses back to URLs. */ export function parseDocument(content: string): ParsedDocument { const blocks = parseMarkdown(content); const links: ExtractedLink[] = []; let linkCounter = 0; const recordLinks = (parts: InlinePart[]) => { for (const part of parts) { if ( part.type === 'link' && part.url !== undefined && part.linkIndex !== undefined && isSafeExternalUrl(part.url) ) { links.push({ url: part.url, text: part.content, linkIndex: part.linkIndex }); } } }; for (const block of blocks) { if (block.type === 'paragraph' || block.type === 'ulist' || block.type === 'olist' || /^h[1-6]$/.test(block.type)) { const { parts, nextLinkIndex } = parseInline(block.rawContent ?? block.content, linkCounter); block.inlineParts = parts; linkCounter = nextLinkIndex; recordLinks(parts); } else if (block.type === 'blockquote') { const lines = block.content.split('\n'); const lineParts: InlinePart[][] = []; for (const line of lines) { const { parts, nextLinkIndex } = parseInline(line, linkCounter); lineParts.push(parts); linkCounter = nextLinkIndex; recordLinks(parts); } block.blockquoteLines = lineParts; } else if (block.type === 'table') { const tableParts: InlinePart[][][] = []; for (const row of block.rawRows ?? block.rows ?? []) { const rowParts: InlinePart[][] = []; for (const cell of row) { const { parts, nextLinkIndex } = parseInline(cell, linkCounter); rowParts.push(parts); linkCounter = nextLinkIndex; recordLinks(parts); } tableParts.push(rowParts); } block.tableInlineParts = tableParts; } } return { blocks, links }; }