/* * This file is part of TREB. * * TREB is free software: you can redistribute it and/or modify it under the * terms of the GNU General Public License as published by the Free Software * Foundation, either version 3 of the License, or (at your option) any * later version. * * TREB is distributed in the hope that it will be useful, but WITHOUT ANY * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more * details. * * You should have received a copy of the GNU General Public License along * with TREB. If not, see . * * Copyright 2022-2026 trebco, llc. * info@treb.app * */ // support for entities removed, as it adds a lot of bloat. we // could easily support numbers (+hex), but html entities require // a table. see // // https://html.spec.whatwg.org/entities.json // // I don't think we need this, since we're not html -- you can just // enter utf8 characters. we might think about emoji entities, though, // along the lines of // // https://github.com/markdown-it/markdown-it-emoji // // import * as he from 'he'; export interface StringFormat { strong?: boolean; emphasis?: boolean; strike?: boolean; // TODO pre?: boolean; // TODO } interface NewlineToken extends StringFormat { type: 'newline'; text: string; } interface WhitespaceToken extends StringFormat { type: 'whitespace'; text: string; } interface TextToken extends StringFormat { type: 'text'; text: string; } interface DelimeterToken extends StringFormat { type: 'delimeter'; text: string; left_flanking?: boolean; right_flanking?: boolean; char: string; length: number; } type Token = NewlineToken | WhitespaceToken | TextToken | DelimeterToken; /** * this external type only has text information */ export interface FormattedString extends StringFormat { text: string; } /** * utility for formatting markdown strings. we split text into tokens * by format. implemented as a factory/singleton, stateless. * * note: in case it's not clear, where I reference MD rules, I mean * CommonMark. we may add some GFM as well (strike?). * * UPDATE: moving into the parser lib, since it's a parser. even though * it's totally independent. (has no deps, though, nice). */ export class MDParser { private static _instance: MDParser = new MDParser(); protected constructor() { // ... } public static get instance(): MDParser { return this._instance; } /** * given some formatted text (output of the `Parse` method), return HTML. * FIXME: is this used outside of testing? seems like we're wasting bytes. * * also the way this works adds extra tags if you have nested styles. not * an issue if it's just for testing though. * * update to optionally not add breaking spaces (
). we need this for * containers that are set to white-space: pre-line, where we will already * get a linebreak. * */ public HTML(formatted: FormattedString[][], options: { br?: boolean } = {}): string { // default options options = { br: true, ...options }; const lines: string[] = []; for (const line of formatted) { const text: string[] = []; for (const element of line) { if (element.pre) { text.push('
'); }
        if (element.emphasis) { text.push(''); }
        if (element.strong) { text.push(''); }
        if (element.strike) { text.push(''); }

        text.push(element.text);

        if (element.strike) { text.push(''); }
        if (element.strong) { text.push(''); }
        if (element.emphasis) { text.push(''); }
        if (element.pre) { text.push('
'); } } lines.push(text.join('')); } return lines.join(options.br ? '
\n' : '\n'); } /** * this is a replacement for the Parse() method, if you don't actually * want to parse markdown. the aim is to have a unified result format, * even if we're not handling md. */ public Dummy(text = ''): FormattedString[][] { return text.split(/\n/).map(text => [{ text }]); } /** * given some input text, creates a set of text tokens with * emphasis/strong emphasis applied. splits into lines (the * outer array). whitespace (other than newlines) is preserved. */ public Parse(text = ''): FormattedString[][] { // first pass: tokenize const tokens = this.Tokenize(text); // for the most part, MD emphapsis/strong can be parsed as if it were // using open/close tags. that's not strictly the case, however, and // there is at least one situation that has some required ambiguity. // MD does specify "left-flanking" and "right-flanking" delimiter runs, // which can open or close formatting, respectively (and a delimeter run // may be both left- and right-flanking). // second pass: assign those flanks on delimeters. from CM spec: /* A left-flanking delimiter run is a delimiter run that is (1) not followed by Unicode whitespace, and either (2a) not followed by a punctuation character, or (2b) followed by a punctuation character and preceded by Unicode whitespace or a punctuation character. For purposes of this definition, the beginning and the end of the line count as Unicode whitespace. A right-flanking delimiter run is a delimiter run that is (1) not preceded by Unicode whitespace, and either (2a) not preceded by a punctuation character, or (2b) preceded by a punctuation character and followed by Unicode whitespace or a punctuation character. For purposes of this definition, the beginning and the end of the line count as Unicode whitespace. */ // FIXME: could this not be consolidated with "apply formatting", below? or // is the concern that if we do that, we might calculate more than once for // any given token? it might still be more efficient... for (let i = 0; i < tokens.length; i++) { const token = tokens[i]; if (token.type === 'delimeter') { const preceding = tokens[i-1]; const following = tokens[i+1]; const preceded_by_whitespace = !preceding || preceding.type === 'whitespace' || preceding.type === 'newline'; const preceded_by_punctuation = preceding && preceding.type === 'text' && /[^\w\d]$/.test(preceding.text); const followed_by_whitespace = !following || following.type === 'whitespace' || following.type === 'newline'; const followed_by_punctuation = following && following.type === 'text' && /^[^\w\d]/.test(following.text); token.left_flanking = ((!followed_by_whitespace) && ((!followed_by_punctuation) || preceded_by_whitespace)); token.right_flanking = ((!preceded_by_whitespace) && ((!preceded_by_punctuation) || (followed_by_whitespace || followed_by_punctuation))); } } // next pass does left/right token matching and applies formatting this.ApplyFormatting(tokens); // last pass consolidates text with like formats, scrubs used tokens // (actually changes _unused_ tokens -> text), and splits into lines. /* const formatted: FormattedString[][] = this.Consolidate(tokens); for (const line of formatted) { for (const token of line) { token.text = he.decode(token.text); } } return formatted ; // this.Consolidate(tokens) as FormattedString[][]; */ return this.Consolidate(tokens) as FormattedString[][]; } /** is this worth a function call? will it get inlined? */ protected IsWhitespace(char: string): boolean { return char === ' ' || char === '\t'; } /** is this worth a function call? will it get inlined? */ protected IsNewline(char: string): boolean { return char === '\r' || char === '\n'; } /** is this worth a function call? will it get inlined? */ protected IsDelimeter(char: string): boolean { return char === '*' || char === '_' || char === '~'; } /** * consolidate text with common formatting. splits into lines (newlines are not rendered). */ protected Consolidate(tokens: Token[]): TextToken[][] { const result: TextToken[][] = []; const format: StringFormat = {}; let line: TextToken[] = []; let current_token: TextToken = {type: 'text', text: ''}; for (const token of tokens) { if (token.type === 'newline') { if (current_token.text.length) { line.push(current_token); } current_token = {...format, text: '', type: 'text'}; result.push(line); line = []; } else { // yuck // can we have a method? or maybe this should be a bitmask, // so we can use === and only have to worry about 0 if ((!!format.strong !== !!token.strong) || (!!format.emphasis !== !!token.emphasis) || (!!format.strike !== !!token.strike)) { format.strong = !!token.strong; format.emphasis = !!token.emphasis; format.strike = !!token.strike; if (current_token.text.length) { line.push(current_token); } current_token = {...format, text: '', type: 'text'}; } switch (token.type) { case 'text': case 'whitespace': current_token.text += token.text; break; case 'delimeter': for (let i = 0; i < token.length; i++) { current_token.text += token.char; } break; } } } if (current_token.text.length) { line.push(current_token); } if (line.length) { result.push(line); } return result; } /** * */ protected ApplyFormatting(tokens: Token[], open?: DelimeterToken): {index: number, token?: DelimeterToken} { // if we're called with no opening token, that's the start // of the text block and formatting is clear (no emphasis). // console.info("AF", "open", open); let index = 0; const length = tokens.length; for (index = 0; index < length; index++) { const token = tokens[index]; if (token.type === 'delimeter') { // check if this token can close (all or in part) our opening tag. // if so, return closing token and index. note that we are checking // length > 0 here; that is because operations may reduce the // "available length" when processing. if (open && token.right_flanking && open.char === token.char && token.length > 0) { // console.info(" ", "close", token); return {index, token}; } // if not, see if we can start a new block if (token.left_flanking) { const result = this.ApplyFormatting(tokens.slice(index + 1), token); if (result.token) { // what format do we apply? it depends on the MIN of open, close, // because it may be a partial close or it may have extra characters. const format = Math.min(result.token.length, token.length); // what format to we apply to the contained block? depends on the // CLOSING delimeter, which may be < the opening delimeter. const strike = token.char === '~'; const emphasis = !strike && !!(format % 2); const strong = !strike && (format >= 2); /* const formats: string[] = []; if (emphasis) formats.push('emphasis'); if (strong) formats.push('strong'); console.info('applying', formats, 'to tokens from', index + 1, 'to', index + result.index, `(len ${length})`); */ // apply this format to all the handled tokens (inclusive) for (let i = index + 1; i <= index + result.index; i++) { tokens[i].strong = (!!tokens[i].strong) || strong; tokens[i].emphasis = (!!tokens[i].emphasis) || emphasis; tokens[i].strike = (!!tokens[i].strike) || strike; } // now we have to handle two separate cases. // one, the closing delimeter is shorter than the opening delimeter. // this happens if you have composite formatting (generally three, but // could be more) and only partially close, like // // ___something_ strange__ // // in that case, we want to handle the opening delimeter again, but // only with the remaining length. // // for case two, closing length >= opening length. in this case, we // reduce the lengths of both tokens but don't handle the opening again. // in the case of === length, both tokens should basically disappear. // if close > open, then the remaining balance will be treated as text. // then we can jump ahead by the handled amount. result.token.length -= format; token.length -= format; if (token.length > 0) { index--; // repeat } else { index += result.index; } } } } } // console.info("finished"); return {index}; } /** * */ protected Tokenize(text = ''): Token[] { const tokens: Token[] = []; const length = text.length; // first pass parse converts text into tokens // FIXME: our escape rule is not quite right -- escape turns out to // be pretty complicated, see CM spec @ 6.1. punting for the time being, // we just always escape the next character. let index = 0; let escape = false; let current_token = ''; // implicit text token for (index = 0; index < length; index++) { const char = text[index]; // we do this three times, but it's kind of hard to fold properly if (this.IsWhitespace(char)) { if (current_token) { tokens.push({ type: 'text', text: current_token }); } let tmp = char; for (;;) { // while (true) { const next_char = text[index+1]; if (this.IsWhitespace(next_char)) { tmp += next_char; index++; } else { break; } } tokens.push({ type: 'whitespace', text: tmp, }) escape = false; current_token = ''; } else if (this.IsNewline(char)) { if (current_token) { tokens.push({ type: 'text', text: current_token }); } tokens.push({ type: 'newline', text: char, }) let tmp = ''; for (;;) { // while (true) { const next_char = text[index+1]; if (this.IsNewline(next_char)) { tmp += next_char; index++; } else { break; } } if (tmp.length) { tokens.push({ type: 'newline', text: tmp, }); } escape = false; current_token = ''; } else if (escape) { current_token += char; escape = false; } else if (this.IsDelimeter(char)) { if (current_token) { tokens.push({ type: 'text', text: current_token }); } let tmp = char; for (;;) { // while (true) { const next_char = text[index+1]; if (next_char === char) { // delimeters do not mix tmp += next_char; index++; } else { break; } } tokens.push({ type: 'delimeter', text: tmp, char, length: tmp.length, }) escape = false; current_token = ''; } else if (char === '\\') { escape = true; } else { current_token += char; } } if (current_token) { tokens.push({type: 'text', text: current_token}); } return tokens; } }