import type { SourceDialect } from "./types.ts" export interface ProseBreak { readonly line: number readonly column: number } export interface LineCommentSpan { readonly line: number readonly markerStart: number readonly contentStart: number readonly endColumn: number } export interface ExtractedComments { readonly lines: readonly string[] readonly contentStarts: readonly number[] readonly proseBreaks: readonly ProseBreak[] readonly lineComments: readonly LineCommentSpan[] } const blankLine = (line: string): string => " ".repeat(line.length) const consumeSeparator = (line: string, index: number): number => line[index] === " " || line[index] === "\t" ? index + 1 : index const hasEscapedLineBreak = (line: string): boolean => { let index = line.endsWith("\r") ? line.length - 1 : line.length let backslashes = 0 while (line[index - 1] === "\\") { backslashes++ index-- } return backslashes % 2 !== 0 } const isRustLifetime = (line: string, index: number): boolean => { if (line[index] !== "'" || !/[A-Za-z_]/.test(line[index + 1] ?? "")) return false let end = index + 2 while (/[A-Za-z0-9_]/.test(line[end] ?? "")) end++ return line[end] !== "'" } type MultilineLiteral = | { readonly kind: "escaped"; readonly terminator: string } | { readonly kind: "literal"; readonly terminator: string } | { readonly kind: "verbatim" } const hasTokenBoundary = (line: string, index: number): boolean => index === 0 || !/[A-Za-z0-9_]/.test(line[index - 1] ?? "") const multilineLiteralAt = ( line: string, index: number, ): { literal: MultilineLiteral; end: number } | null => { if (!hasTokenBoundary(line, index)) return null const csharp = line.slice(index).match(/^(?:\$@|@\$|@)"/) if (csharp !== null) { return { literal: { kind: "verbatim" }, end: index + csharp[0].length } } const rust = line.slice(index).match(/^(?:br|r)(#{0,255})"/) if (rust !== null) { return { literal: { kind: "literal", terminator: `"${rust[1] as string}` }, end: index + rust[0].length, } } const cpp = line.slice(index).match(/^(?:u8|u|U|L)?R"([^ ()\\\t\r\n]{0,16})\(/) if (cpp !== null) { return { literal: { kind: "literal", terminator: `)${cpp[1] as string}"` }, end: index + cpp[0].length, } } return null } interface TemplateContext { mode: "text" | "expression" braceDepth: number } export function extractSlashComments( text: string, dialect: SourceDialect = "general", ): ExtractedComments { let blockDepth = 0 let multilineLiteral: MultilineLiteral | null = null let continuedLineQuote: "'" | '"' | null = null const templateStack: TemplateContext[] = [] const javascript = dialect === "javascript" const nestedBlocks = dialect === "nested-slash" let previousComment: "line" | "block" | null = null const contentStarts: number[] = [] const proseBreaks: ProseBreak[] = [] const lineComments: LineCommentSpan[] = [] const lines = text.split("\n").map((line, lineIndex) => { const out = new Array(line.length).fill(" ") let contentStart = line.length let lineQuote = continuedLineQuote continuedLineQuote = null let i = 0 const markContentStart = (index: number) => { contentStart = Math.min(contentStart, index) } const beginComment = (kind: "line" | "block", index: number) => { if (previousComment !== null && (kind === "block" || previousComment === "block")) { proseBreaks.push({ line: lineIndex, column: index }) } previousComment = kind } if (blockDepth > 0) { while (line[i] === " " || line[i] === "\t") i++ if (line[i] === "*" && line[i + 1] !== "/") i++ i = consumeSeparator(line, i) markContentStart(i) } while (i < line.length) { const ch = line[i] as string const next = line[i + 1] if (blockDepth > 0) { if (nestedBlocks && ch === "/" && next === "*") { blockDepth++ i += 2 continue } if (ch === "*" && next === "/") { blockDepth-- i += 2 continue } out[i] = ch i++ continue } if (multilineLiteral !== null) { if (multilineLiteral.kind === "escaped" && ch === "\\") { i += 2 continue } if (multilineLiteral.kind === "verbatim") { if (line.startsWith('""', i)) { i += 2 continue } if (ch === '"') { multilineLiteral = null i++ continue } i++ continue } if (line.startsWith(multilineLiteral.terminator, i)) { i += multilineLiteral.terminator.length multilineLiteral = null continue } i++ continue } const template = templateStack.at(-1) if (template?.mode === "text") { if (ch === "\\") { i += 2 continue } if (line.startsWith("${", i)) { template.mode = "expression" template.braceDepth = 0 i += 2 continue } if (ch === "`") templateStack.pop() i++ continue } if (lineQuote !== null) { if (ch === "\\") { i += 2 continue } if (ch === lineQuote) lineQuote = null i++ continue } if (template?.mode === "expression") { if (ch === "{") { template.braceDepth++ i++ continue } if (ch === "}") { if (template.braceDepth === 0) template.mode = "text" else template.braceDepth-- i++ continue } } const boundedLiteral = multilineLiteralAt(line, i) if (boundedLiteral !== null) { multilineLiteral = boundedLiteral.literal i = boundedLiteral.end continue } if (line.startsWith('"""', i)) { multilineLiteral = { kind: "literal", terminator: '"""' } i += 3 continue } if (ch === "`") { if (javascript) templateStack.push({ mode: "text", braceDepth: 0 }) else multilineLiteral = { kind: "escaped", terminator: "`" } i++ continue } if (ch === '"' || (ch === "'" && !isRustLifetime(line, i))) { lineQuote = ch i++ continue } if (ch === "/" && next === "/") { const markerStart = i i += 2 while (line[i] === "/" || line[i] === "!") i++ i = consumeSeparator(line, i) lineComments.push({ line: lineIndex + 1, markerStart, contentStart: i, endColumn: line.length, }) beginComment("line", i) markContentStart(i) for (; i < line.length; i++) out[i] = line[i] as string break } if (ch === "/" && next === "*") { blockDepth = 1 i += 2 while (line[i] === "*" && line[i + 1] !== "/") i++ i = consumeSeparator(line, i) beginComment("block", i) markContentStart(i) continue } i++ } if (lineQuote !== null && hasEscapedLineBreak(line)) continuedLineQuote = lineQuote contentStarts.push(contentStart) return out.join("") }) return { lines, contentStarts, proseBreaks, lineComments } } interface Heredoc { readonly delimiter: string readonly terminatorIndent: "none" | "tabs" | "whitespace" } interface ParsedHeredoc extends Heredoc { readonly end: number } const parseHeredoc = (line: string, start: number): ParsedHeredoc | null => { if (!line.startsWith("<<", start) || line[start + 2] === "<") return null let index = start + 2 let terminatorIndent: Heredoc["terminatorIndent"] = "none" if (line[index] === "-") { terminatorIndent = "tabs" index++ } while (line[index] === " " || line[index] === "\t") index++ let delimiter = "" while (index < line.length && !/[\s;&|()<>]/.test(line[index] as string)) { const ch = line[index] as string if (ch === "'" || ch === '"') { const quote = ch const close = line.indexOf(quote, index + 1) if (close === -1) return null delimiter += line.slice(index + 1, close) index = close + 1 continue } if (ch === "\\") { if (index + 1 >= line.length) return null delimiter += line[index + 1] as string index += 2 continue } delimiter += ch index++ } return delimiter === "" ? null : { delimiter, terminatorIndent, end: index } } const parseRubyHeredoc = (line: string, start: number): ParsedHeredoc | null => { if (!line.startsWith("<<", start) || line[start + 2] === "<") return null let index = start + 2 let terminatorIndent: Heredoc["terminatorIndent"] = "none" if (line[index] === "-" || line[index] === "~") { terminatorIndent = "whitespace" index++ } const quote = line[index] === "'" || line[index] === '"' || line[index] === "`" ? line[index] : null if (quote !== null) index++ const delimiterStart = index while (/[A-Za-z0-9_]/.test(line[index] ?? "")) index++ const delimiter = line.slice(delimiterStart, index) if (delimiter === "" || !/[A-Za-z_]/.test(delimiter[0] as string)) return null if (quote !== null) { if (line[index] !== quote) return null index++ } return { delimiter, terminatorIndent, end: index } } const isShellCommentStart = (line: string, index: number): boolean => index === 0 || /[\s;|&()]/.test(line[index - 1] ?? "") const isYamlCommentStart = (line: string, index: number): boolean => index === 0 || /[ \t]/.test(line[index - 1] ?? "") interface YamlBlockScalar { readonly parentIndent: number readonly explicitIndent: number | undefined readonly contentIndent: number | undefined } const leadingSpaces = (line: string): number => line.length - line.replace(/^ */u, "").length const YAML_BLOCK_SCALAR_CONTEXT = /(?:^[ \t]*(?:[-?:][ \t]+)*|:[ \t]+)(?:[&!][^\s]+[ \t]+)*$/u const YAML_QUOTED_SCALAR_CONTEXT = /(?:^[ \t]*(?:(?:---|\.\.\.)[ \t]+)?(?:[-?:][ \t]+)*|:[ \t]+|[[{,][ \t]*)(?:[&!][^\s,[\]{}]+[ \t]+)*$/u const isYamlQuotedScalarStart = (line: string, index: number): boolean => YAML_QUOTED_SCALAR_CONTEXT.test(line.slice(0, index)) interface DelimitedLiteral { readonly open: string | null readonly close: string depth: number } const RUBY_PERCENT_LITERAL_TYPES = new Set(["q", "Q", "w", "W", "i", "I", "x", "r", "s"]) const PAIRED_DELIMITERS: Readonly> = { "(": ")", "[": "]", "{": "}", "<": ">", } const rubyPercentLiteralAt = ( line: string, index: number, ): { readonly literal: DelimitedLiteral; readonly end: number } | null => { if (line[index] !== "%" || !RUBY_PERCENT_LITERAL_TYPES.has(line[index + 1] ?? "")) return null const delimiter = line[index + 2] if (delimiter === undefined || /[A-Za-z0-9\s]/u.test(delimiter)) return null const close = PAIRED_DELIMITERS[delimiter] ?? delimiter return { literal: { open: close === delimiter ? null : delimiter, close, depth: 1 }, end: index + 3, } } const yamlBlockScalarAt = ( line: string, index: number, ): { readonly explicitIndent: number | undefined } | null => { if (line[index] !== "|" && line[index] !== ">") return null const prefix = line.slice(0, index) if (!YAML_BLOCK_SCALAR_CONTEXT.test(prefix)) return null const suffix = line.slice(index + 1) const match = suffix.match(/^((?:[+-][1-9]?|[1-9][+-]?)?)[ \t]*(?:#.*)?\r?$/u) if (match === null) return null const digit = match[1]?.match(/[1-9]/u)?.[0] return { explicitIndent: digit === undefined ? undefined : Number(digit) } } export function extractHashComments( text: string, dialect: SourceDialect = "general", ): ExtractedComments { let multilineQuote: "'''" | '"""' | null = null let shellQuote: "'" | '"' | null = null let continuedLineQuote: "'" | '"' | null = null let yamlQuote: "'" | '"' | null = null let rubyPercentLiteral: DelimitedLiteral | null = null let parameterDepth = 0 let arithmeticDepth = 0 const heredocs: Heredoc[] = [] const contentStarts: number[] = [] const lineComments: LineCommentSpan[] = [] const perl = dialect === "perl" const ruby = dialect === "ruby" const shell = dialect === "shell" const yaml = dialect === "yaml" let yamlBlockScalar: YamlBlockScalar | null = null const lines = text.split("\n").map((line, lineIndex) => { if (yamlBlockScalar !== null) { if (line.trim() === "") { contentStarts.push(line.length) return blankLine(line) } const indent = leadingSpaces(line) const requiredIndent = yamlBlockScalar.explicitIndent === undefined ? yamlBlockScalar.contentIndent : yamlBlockScalar.parentIndent + yamlBlockScalar.explicitIndent if (requiredIndent === undefined && indent > yamlBlockScalar.parentIndent) { yamlBlockScalar = { ...yamlBlockScalar, contentIndent: indent } contentStarts.push(line.length) return blankLine(line) } if (requiredIndent !== undefined && indent >= requiredIndent) { contentStarts.push(line.length) return blankLine(line) } yamlBlockScalar = null } const activeHeredoc = heredocs[0] if (activeHeredoc !== undefined) { const normalized = line.endsWith("\r") ? line.slice(0, -1) : line const candidate = activeHeredoc.terminatorIndent === "tabs" ? normalized.replace(/^\t+/, "") : activeHeredoc.terminatorIndent === "whitespace" ? normalized.trimStart() : normalized if (candidate === activeHeredoc.delimiter) heredocs.shift() contentStarts.push(line.length) return blankLine(line) } if (lineIndex === 0 && line.startsWith("#!")) { contentStarts.push(line.length) return blankLine(line) } const out = new Array(line.length).fill(" ") const pendingHeredocs: Heredoc[] = [] let contentStart = line.length let lineQuote = yaml ? yamlQuote : continuedLineQuote continuedLineQuote = null let i = 0 while (i < line.length) { const ch = line[i] as string if (multilineQuote !== null) { if (line.startsWith(multilineQuote, i)) { i += multilineQuote.length multilineQuote = null continue } i++ continue } if (rubyPercentLiteral !== null) { if (ch === "\\") { i += 2 continue } if (rubyPercentLiteral.open !== null && ch === rubyPercentLiteral.open) { rubyPercentLiteral.depth++ i++ continue } if (ch === rubyPercentLiteral.close) { rubyPercentLiteral.depth-- if (rubyPercentLiteral.depth === 0) rubyPercentLiteral = null } i++ continue } if (shellQuote !== null) { if (ch === "\\" && shellQuote === '"') { i += 2 continue } if (ch === shellQuote) shellQuote = null i++ continue } if (lineQuote !== null) { if (ch === "\\" && (!yaml || lineQuote === '"')) { i += 2 continue } if (yaml && lineQuote === "'" && ch === "'" && line[i + 1] === "'") { i += 2 continue } if (ch === lineQuote) lineQuote = null i++ continue } if ( !shell && !yaml && !ruby && !perl && (line.startsWith("'''", i) || line.startsWith('"""', i)) ) { multilineQuote = line.slice(i, i + 3) as "'''" | '"""' i += 3 continue } const percentLiteral = ruby ? rubyPercentLiteralAt(line, i) : null if (percentLiteral !== null) { rubyPercentLiteral = percentLiteral.literal i = percentLiteral.end continue } if (ch === '"' || ch === "'") { if (shell) shellQuote = ch else if (!yaml || isYamlQuotedScalarStart(line, i)) lineQuote = ch i++ continue } if (shell && line.startsWith("${", i)) { parameterDepth++ i += 2 continue } if (shell && parameterDepth > 0 && ch === "}") { parameterDepth-- i++ continue } if (shell && line.startsWith("$((", i)) { arithmeticDepth++ i += 3 continue } if (shell && arithmeticDepth > 0 && line.startsWith("))", i)) { arithmeticDepth-- i += 2 continue } if ( (shell || ruby || perl) && parameterDepth === 0 && arithmeticDepth === 0 && line.startsWith("<<", i) ) { const heredoc = ruby ? parseRubyHeredoc(line, i) : parseHeredoc(line, i) if (heredoc !== null) { pendingHeredocs.push({ delimiter: heredoc.delimiter, terminatorIndent: heredoc.terminatorIndent, }) i = heredoc.end continue } } if (yaml) { const scalar = yamlBlockScalarAt(line, i) if (scalar !== null) { yamlBlockScalar = { parentIndent: leadingSpaces(line), explicitIndent: scalar.explicitIndent, contentIndent: undefined, } } } if ( ch === "#" && parameterDepth === 0 && arithmeticDepth === 0 && line[i - 1] !== "$" && (!shell || isShellCommentStart(line, i)) && (!yaml || isYamlCommentStart(line, i)) ) { const markerStart = i let j = i + 1 while (line[j] === "#") j++ j = consumeSeparator(line, j) lineComments.push({ line: lineIndex + 1, markerStart, contentStart: j, endColumn: line.length, }) contentStart = j for (; j < line.length; j++) out[j] = line[j] as string break } i++ } if (yaml) yamlQuote = lineQuote else if (ruby || perl || (lineQuote !== null && hasEscapedLineBreak(line))) { continuedLineQuote = lineQuote } heredocs.push(...pendingHeredocs) contentStarts.push(contentStart) return out.join("") }) return { lines, contentStarts, proseBreaks: [], lineComments } }