import { execFileSync } from "node:child_process"; import TurndownService from "turndown"; export interface Section { level: number; title: string; content: string; priority: number; sizeBytes: number; } export const PRIORITY_KEYWORDS: Record = { 100: ["overview", "introduction", "about", "description"], 90: ["install", "installation", "setup", "getting started", "get started"], 85: ["quickstart", "quick start", "tutorial", "guide", "walkthrough"], 80: ["usage", "example", "examples", "how to", "howto"], 70: ["api", "reference", "methods", "functions", "classes"], 60: ["configuration", "config", "options", "settings", "parameters"], 50: ["advanced", "tips", "best practices", "patterns"], 40: ["changelog", "history", "releases", "versions"], }; function looksLikeRst(rst: string): boolean { const top = rst.split(/\r?\n/).slice(0, 20).join("\n"); return /(^|\n)\s*\.\.\s+\w+::/.test(rst) || /::/.test(top); } function stripIndent(lines: string[], indent: number): string[] { return lines.map((line) => { if (!line.trim()) return ""; let count = 0; let index = 0; while (index < line.length && count < indent && (line[index] === " " || line[index] === "\t")) { count += line[index] === "\t" ? 4 : 1; index += 1; } return line.slice(index); }); } function consumeIndentedBlock(lines: string[], startIndex: number): { block: string[]; nextIndex: number } { const block: string[] = []; let i = startIndex; while (i < lines.length && !lines[i].trim()) { i += 1; } if (i >= lines.length) { return { block, nextIndex: i }; } const firstLine = lines[i]; const indentMatch = firstLine.match(/^(\s+)/); const indent = indentMatch ? indentMatch[1].length : 0; while (i < lines.length) { const line = lines[i]; if (!line.trim()) { block.push(""); i += 1; continue; } const leading = line.match(/^(\s*)/)?.[1].length ?? 0; if (leading < indent) { break; } block.push(line.slice(indent)); i += 1; } return { block, nextIndex: i }; } export function htmlToMarkdown(html: string, baseUrl = ""): string { const turndown = new TurndownService({ headingStyle: "atx", bulletListMarker: "-", codeBlockStyle: "fenced", }); turndown.remove(["script", "style"]); let markdown = turndown.turndown(html); if (baseUrl) { markdown = convertRelativeUrls(markdown, baseUrl); } return markdown.trim(); } export function convertRstToMarkdown(rst: string): string { if (!looksLikeRst(rst)) { return rst; } try { const lines = rst.replace(/\r\n/g, "\n").split("\n"); const out: string[] = []; for (let i = 0; i < lines.length; i += 1) { const line = lines[i]; const linkLine = line.replace(/`([^`<]+)\s+<([^>]+)>`_/g, "[$1]($2)"); let match = line.match(/^\s*\.\.\s+code-block::\s*([^\s]+)\s*$/); if (match) { const lang = match[1]; const { block, nextIndex } = consumeIndentedBlock(lines, i + 1); out.push(`\`\`\`${lang}`); out.push(...stripIndent(block, 0)); out.push("```"); i = nextIndex - 1; continue; } match = line.match(/^\s*\.\.\s+image::\s*(\S+)\s*$/); if (match) { out.push(`![image](${match[1]})`); continue; } match = line.match(/^\s*\.\.\s+(note|warning)::\s*(.*)$/); if (match) { const message = match[2].trim(); if (message) { out.push(`> ${message}`); } else { out.push("> "); } const { block, nextIndex } = consumeIndentedBlock(lines, i + 1); for (const blockLine of block) { out.push(blockLine ? `> ${blockLine}` : "> "); } i = nextIndex - 1; continue; } match = line.match(/^\s*\.\.\s+literalinclude::\s*(\S+)\s*$/); if (match) { out.push("```"); out.push(``); out.push("```"); const { nextIndex } = consumeIndentedBlock(lines, i + 1); i = nextIndex - 1; continue; } if (/^\s*\.\.\s+\w+::/.test(line)) { const { nextIndex } = consumeIndentedBlock(lines, i + 1); i = nextIndex - 1; continue; } out.push(linkLine); } let markdown = out.join("\n"); if (markdown.includes("::")) { try { markdown = execFileSync("pandoc", ["-f", "rst", "-t", "markdown"], { input: rst, encoding: "utf8", }); } catch { return rst; } } return markdown.trim(); } catch { return rst; } } export function extractSections(markdown: string): Section[] { if (!markdown || !markdown.trim()) { return []; } const sections: Section[] = []; const lines = markdown.split("\n"); let currentSectionLines: string[] = []; let currentLevel = 0; let currentTitle = ""; for (const line of lines) { const headingMatch = line.match(/^(#{1,6})\s+(.+)$/); if (headingMatch) { if (currentSectionLines.length > 0) { const sectionContent = currentSectionLines.join("\n"); sections.push({ level: currentLevel, title: currentTitle, content: sectionContent, priority: scoreSection(currentTitle), sizeBytes: Buffer.byteLength(sectionContent, "utf8"), }); } currentLevel = headingMatch[1].length; currentTitle = headingMatch[2].trim(); currentSectionLines = [line]; } else { currentSectionLines.push(line); } } if (currentSectionLines.length > 0) { const sectionContent = currentSectionLines.join("\n"); sections.push({ level: currentLevel, title: currentTitle, content: sectionContent, priority: scoreSection(currentTitle), sizeBytes: Buffer.byteLength(sectionContent, "utf8"), }); } return sections; } export function scoreSection(title: string): number { if (!title) { return 30; } const titleLower = title.toLowerCase(); const priorities = Object.entries(PRIORITY_KEYWORDS) .map(([score, keywords]) => [Number(score), keywords] as const) .sort((a, b) => b[0] - a[0]); for (const [score, keywords] of priorities) { if (keywords.some((keyword) => titleLower.includes(keyword))) { return score; } } return 30; } export function smartTruncate(text: string, maxBytes: number): string { if (!text) { return ""; } if (Buffer.byteLength(text, "utf8") <= maxBytes) { return text; } const encoded = Buffer.from(text, "utf8"); let truncatePoint = maxBytes; const decoder = new TextDecoder("utf-8", { fatal: true }); let truncated = ""; while (truncatePoint > 0) { try { truncated = decoder.decode(encoded.subarray(0, truncatePoint)); break; } catch { truncatePoint -= 1; } } if (!truncated && truncatePoint <= 0) { return ""; } const lastPara = truncated.lastIndexOf("\n\n"); if (lastPara > maxBytes * 0.7) { const result = `${truncated.slice(0, lastPara).trim()}\n\n...`; if (Buffer.byteLength(result, "utf8") <= maxBytes) { return result; } } for (const punct of [".\n", "!\n", "?\n"]) { const lastSent = truncated.lastIndexOf(punct); if (lastSent > maxBytes * 0.7) { const result = `${truncated.slice(0, lastSent + 1).trim()}\n\n...`; if (Buffer.byteLength(result, "utf8") <= maxBytes) { return result; } } } const lastSpace = truncated.lastIndexOf(" "); if (lastSpace > 0) { const result = `${truncated.slice(0, lastSpace).trim()}...`; if (Buffer.byteLength(result, "utf8") <= maxBytes) { return result; } } if (maxBytes <= 3) { return ".".repeat(maxBytes); } const ellipsisTarget = maxBytes - 3; truncatePoint = ellipsisTarget; while (truncatePoint > 0) { try { truncated = decoder.decode(encoded.subarray(0, truncatePoint)); break; } catch { truncatePoint -= 1; } } if (!truncated && truncatePoint <= 0) { return maxBytes <= 3 ? ".".repeat(maxBytes) : "..."; } return `${truncated.trim()}...`; } export function prioritizeSections(sections: Section[], maxBytes = 20480): string { if (!sections.length) { return ""; } const first = sections[0]; if (first.sizeBytes > maxBytes) { return smartTruncate(first.content, maxBytes); } const result = [first]; let remainingBytes = maxBytes - first.sizeBytes; const sortedSections = sections.slice(1).sort((a, b) => b.priority - a.priority); for (const section of sortedSections) { if (section.sizeBytes <= remainingBytes) { result.push(section); remainingBytes -= section.sizeBytes; } } const selected = new Set(result); const ordered = sections.filter((section) => selected.has(section)); return ordered.map((section) => section.content).join("\n\n"); } export function convertRelativeUrls(markdown: string, baseUrl: string): string { if (!baseUrl) { return markdown; } baseUrl = baseUrl.replace(/\/+$/, ""); const convertUrl = (url: string, allowMailto: boolean): string => { if (url.startsWith("http://") || url.startsWith("https://") || url.startsWith("#") || (allowMailto && url.startsWith("mailto:"))) { return url; } if (url.startsWith("/")) { const domainMatch = baseUrl.match(/(https?:\/\/[^/]+)/); if (domainMatch) { return `${domainMatch[1]}${url}`; } } return `${baseUrl}/${url}`; }; markdown = markdown.replace(/\[([^\]]+)\]\(([^)]+)\)/g, (_match, text: string, url: string) => { return `[${text}](${convertUrl(url, true)})`; }); markdown = markdown.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (_match, alt: string, url: string) => { return `![${alt}](${convertUrl(url, false)})`; }); return markdown; }