import { describe, expect, test } from "bun:test" import { probeConversion } from "./conversion-probe.mjs" import { snapshotFromHtml } from "./source-snapshot.mjs" // One table per class of conversion artifact. Each row is one form: the raw source, the converted // text (BUILTIN = the built-in snapshot of the raw HTML), and the exact `altered` and `lost` lists. // `attributes: false` is the built-in snapshot path, which writes no attribute text and no syntax. type Row = { form: string; raw: string; converted: string; altered: string[]; lost: string[]; attributes?: boolean } const BUILTIN = "\u0000builtin" function check(row: Row) { const builtin = row.converted === BUILTIN const converted = builtin ? snapshotFromHtml(row.raw) : row.converted const result = probeConversion(row.raw, converted, { attributes: builtin ? false : (row.attributes ?? true) }) expect({ altered: result.altered, lost: result.lost }).toEqual({ altered: row.altered, lost: row.lost }) } const rows = (table: Row[]) => table.map((row) => [row.form, row] as const) const OL = "
  1. Alpha
  2. Beta
" const PANDOC_NOTE = '

Margin 12%1

  1. Filing.back

' const CONVERTER_DIGITS: Row[] = [ { form: "ol -> '1. '", raw: OL, converted: "1. Alpha\n2. Beta", altered: [], lost: [] }, { form: "ol -> '1) '", raw: OL, converted: "1) Alpha\n2) Beta", altered: [], lost: [] }, { form: "ol -> '(1) '", raw: OL, converted: "(1) Alpha\n(2) Beta", altered: [], lost: [] }, { form: "KO ol with numbers in the items", raw: "
  1. \uB9E4\uCD9C 3\uC870 \uC6D0
  2. \uC601\uC5C5\uC774\uC775 1,250\uC5B5 \uC6D0
", converted: "1. \uB9E4\uCD9C 3\uC870 \uC6D0\n2. \uC601\uC5C5\uC774\uC775 1,250\uC5B5 \uC6D0", altered: [], lost: [] }, { form: "indented markers (spaces, tab)", raw: OL, converted: " 1. Alpha\n\t2. Beta", altered: [], lost: [] }, { form: "ol start=3", raw: '
  1. x
  2. y
', converted: "3. x\n4. y", altered: [], lost: [] }, { form: "ol start=3 renumbered from 1 by the converter", raw: '
  1. x
', converted: "1. x", altered: ["1"], lost: [] }, { form: "nested ol", raw: "
  1. A
    1. B
    2. C
  2. D
", converted: "1. A\n 1. B\n 2. C\n2. D", altered: [], lost: [] }, { form: "ol inside a blockquote", raw: "
  1. A
", converted: "> 1. A", altered: [], lost: [] }, { form: "footnote list (references ol)", raw: '

Revenue rose 4%.[1]

  1. Annual report.
', converted: "Revenue rose 4%.[[1]](#cite_note-1)\n\n1. Annual report.", altered: [], lost: [] }, { form: "pandoc footnote -> [^1] and [^1]: definition", raw: PANDOC_NOTE, converted: "Margin 12%[^1]\n\n[^1]: Filing.", altered: [], lost: [] }, { form: "pandoc footnote -> markdownify list", raw: PANDOC_NOTE, converted: "Margin 12%[1](#fn1)\n\n1. Filing.[back](#fnref1)", altered: [], lost: [] }, { form: "KO footnote list", raw: '

\uB9E4\uCD9C 3,234\uC5B5 \uC6D01

  1. \uC0AC\uC5C5\uBCF4\uACE0\uC11C
', converted: "\uB9E4\uCD9C 3,234\uC5B5 \uC6D0[1](#n1)\n\n1. \uC0AC\uC5C5\uBCF4\uACE0\uC11C", altered: [], lost: [] }, { form: "table of contents as nested ol", raw: '
  1. Intro
    1. Scope
  2. Method
', converted: "1. [Intro](#intro)\n 1. [Scope](#scope)\n2. [Method](#method)", altered: [], lost: [] }, { form: "table of contents numbers written in the source", raw: '', converted: "- [1 Intro](#s1)\n - [1.1 Scope](#s1-1)", altered: [], lost: [] }, { form: "table of contents number the converter invented", raw: '', converted: "- [1.1 Scope](#scope)", altered: ["1.1"], lost: [] }, { form: "heading number written in the source", raw: "

3. Results

", converted: "## 3. Results", altered: [], lost: [] }, { form: "KO heading number written in the source", raw: "

2\uC7A5 \uACB0\uACFC

", converted: "## 2\uC7A5 \uACB0\uACFC", altered: [], lost: [] }, { form: "heading number the converter invented", raw: "

Results

", converted: "## 3 Results", altered: ["3"], lost: [] }, { form: "setext heading '2. Method' beside an ol", raw: "

2. Method

  1. x
  2. y
", converted: "2. Method\n---------\n\n1. x\n2. y", altered: [], lost: [] }, { form: "numbered paragraph, escaped by the converter", raw: "

1. Alpha

", converted: "1\\. Alpha", altered: [], lost: [] }, { form: "numbered paragraph, unescaped", raw: "

1. Alpha

", converted: "1. Alpha", altered: [], lost: [] }, { form: "bare '1. ' with no ol in the source", raw: "

Alpha

", converted: "1. Alpha", altered: ["1"], lost: [] }, { form: "ul source written as '1. '", raw: "", converted: "1. Alpha", altered: ["1"], lost: [] }, { form: "KO bare '1. ' with no ol", raw: "

\uB9E4\uCD9C \uC99D\uAC00

", converted: "1. \uB9E4\uCD9C \uC99D\uAC00", altered: ["1"], lost: [] }, { form: "more markers than ol items", raw: "
  1. Alpha
", converted: "1. Alpha\n2. Beta", altered: ["2"], lost: [] }, { form: "lazy numbering '1. 1. 1.' over a three-item ol", raw: "
  1. a
  2. b
  3. c
", converted: "1. a\n1. b\n1. c", altered: [], lost: [] }, { form: "lazy numbering at start=3", raw: '
  1. x
  2. y
', converted: "3. x\n3. y", altered: [], lost: [] }, { form: "KO lazy numbering", raw: "
  1. \uB9E4\uCD9C
  2. \uBE44\uC6A9
", converted: "1. \uB9E4\uCD9C\n1. \uBE44\uC6A9", altered: [], lost: [] }, { form: "lazy numbering with more markers than ol items", raw: "
  1. a
  2. b
", converted: "1. a\n1. b\n1. c", altered: ["1"], lost: [] }, { form: "built-in path: no marker exemption", raw: "
  1. Alpha
", converted: "1. Alpha", altered: ["1"], lost: [], attributes: false }, { form: "inline link target digits", raw: '

Report

', converted: "[Report](https://example.com/2024/03/report-4)", altered: [], lost: [] }, { form: "image src, alt and title digits", raw: '

Chart 2023

', converted: '![Chart 2023](chart-640x480.png "Fig 4")', altered: [], lost: [] }, { form: "autolink", raw: '

https://example.com/2024

', converted: "", altered: [], lost: [] }, { form: "reference-style links with numeric labels", raw: '

report and data

', converted: "[report][1] and [data][2]\n\n[1]: https://example.com/r/7\n[2]: https://example.com/r/9", altered: [], lost: [] }, { form: "KO reference-style link", raw: '

\uBCF4\uACE0\uC11C

', converted: "[\uBCF4\uACE0\uC11C][1]\n\n[1]: https://example.kr/b/2024", altered: [], lost: [] }, { form: "reference label with no definition", raw: "

report

", converted: "[report][1]", altered: ["1"], lost: [] }, { form: "reference definition with a title", raw: '

report

', converted: '[report][1]\n\n[1]: https://example.com/r/7 "Fig 4"', altered: [], lost: [] }, { form: "footnote definition text is prose, not link syntax", raw: '

Margin1 per source.

  1. Rate 4.22.
', converted: "Margin[^1] per source.\n\n[^1]: Rate 4.22.", altered: ["4.22"], lost: ["4.2", "2"] }, { form: "KO footnote definition text is prose", raw: '

\uB9E4\uCD9C1 \uCD9C\uCC98

  1. \uB9E4\uCD9C 3,234\uC5B5 \uC6D0
', converted: "\uB9E4\uCD9C[^1] \uCD9C\uCC98\n\n[^1]: \uB9E4\uCD9C 3,234\uC5B5 \uC6D0", altered: ["3,234"], lost: ["3,23", "4"] }, { form: "footnote definition with no use keeps its label a candidate", raw: "

Filing.

", converted: "[^1]: Filing.", altered: ["1"], lost: [] }, { form: "attribute digit written into prose", raw: '

page

', converted: "page 42", altered: ["42"], lost: [] }, { form: "built-in path: attribute digit in link syntax", raw: '

page

', converted: "[page](/p/42)", altered: ["42"], lost: [], attributes: false }, { form: "href page number beside a glued footnote", raw: '

Output reached 109 kWh

book

', converted: "Output reached 109 kWh\n\n[book](https://b.example/b?pg=PA109)", altered: ["109"], lost: ["10", "9"] }, ] const SIGNS: Row[] = [ { form: "U+2212 written as a hyphen", raw: "fell \u22123.5 points", converted: "fell -3.5 points", altered: [], lost: [] }, { form: "U+2212 dropped", raw: "fell \u22123.5 points", converted: "fell 3.5 points", altered: ["3.5"], lost: ["\u22123.5"] }, { form: "hyphen-minus dropped", raw: "fell -3.5", converted: "fell 3.5", altered: ["3.5"], lost: ["-3.5"] }, { form: "en dash minus after a space dropped", raw: "fell \u20135%", converted: "fell 5%", altered: ["5"], lost: ["\u20135"] }, { form: "en dash minus written as a hyphen", raw: "fell \u20135%", converted: "fell -5%", altered: [], lost: [] }, { form: "hyphen after a space written as U+2212", raw: "change -4.2", converted: "change \u22124.2", altered: [], lost: [] }, { form: "U+2212 inside parentheses dropped", raw: "(\u22125)", converted: "(5)", altered: ["5"], lost: ["\u22125"] }, { form: "U+2212 after one letter dropped", raw: "\u0394\u22125 points", converted: "\u03945 points", altered: ["5"], lost: ["\u22125"] }, { form: "hyphen after one letter dropped", raw: "x-5", converted: "x5", altered: ["5"], lost: ["-5"] }, { form: "U+2212 after one letter written as a hyphen", raw: "\u0394\u22125", converted: "\u0394-5", altered: [], lost: [] }, { form: "en dash after one letter is a connector", raw: "A\u20135", converted: "A5", altered: [], lost: [] }, { form: "range after a digit with U+2212", raw: "2019\u22122023", converted: "2019-2023", altered: [], lost: [] }, { form: "range after a digit with an en dash", raw: "2019\u20132023", converted: "2019-2023", altered: [], lost: [] }, { form: "ISO date", raw: "2026-10-26", converted: "2026-10-26", altered: [], lost: [] }, { form: "hyphen after Hangul dropped", raw: "\uAC10\uC18C\uC728-3.5%", converted: "\uAC10\uC18C\uC728 3.5%", altered: ["3.5"], lost: ["-3.5"] }, { form: "U+2212 after Hangul dropped", raw: "\uC804\uB144 \uB300\uBE44\u22122.1%", converted: "\uC804\uB144 \uB300\uBE44 2.1%", altered: ["2.1"], lost: ["\u22122.1"] }, { form: "U+2212 after Hangul written as a hyphen", raw: "\uC99D\uAC10\uB960\u22122.1%", converted: "\uC99D\uAC10\uB960-2.1%", altered: [], lost: [] }, { form: "identifier cite_note-7", raw: "cite_note-7", converted: "cite_note7", altered: [], lost: [] }, { form: "identifier ISO-8601", raw: "ISO-8601", converted: "ISO 8601", altered: [], lost: [] }, { form: "identifier UTF-8", raw: "UTF-8 text", converted: "UTF8 text", altered: [], lost: [] }, { form: "identifier COVID-19", raw: "COVID-19 cases", converted: "COVID 19 cases", altered: [], lost: [] }, { form: "two-letter code AB-12", raw: "AB-12", converted: "AB12", altered: [], lost: [] }, { form: "digit-letter code 3x-5", raw: "3x-5", converted: "3x5", altered: [], lost: [] }, { form: "exponent minus dropped", raw: "1e-5", converted: "1e5", altered: ["5"], lost: ["-5"] }, { form: "exponent U+2212 written as a hyphen", raw: "2.5E\u22123", converted: "2.5E-3", altered: [], lost: [] }, { form: "superscript exponent flattened onto its base", raw: "

10\u22125

", converted: "10-5", altered: ["5"], lost: ["\u22125"] }, { form: "superscript exponent written with a caret", raw: "

10\u22125

", converted: "10^-5", altered: [], lost: [] }, { form: "negative cells kept", raw: "
Q4-5\u22123.2
", converted: "| Q4 | -5 | -3.2 |", altered: [], lost: [] }, { form: "negative cells lose their sign", raw: "
Q4-5\u22123.2
", converted: "| Q4 | 5 | 3.2 |", altered: ["5", "3.2"], lost: ["-5", "\u22123.2"] }, { form: "KO negative cell", raw: "
\uC99D\uAC10\u22122.1%
", converted: "| \uC99D\uAC10 | -2.1% |", altered: [], lost: [] }, { form: "built-in path: negative cell", raw: "
Q4\u22123.2
", converted: BUILTIN, altered: [], lost: [] }, { form: "plus sign carries no sign", raw: "+7", converted: "7", altered: [], lost: [] }, { form: "minus with a space after it is not a sign", raw: "change - 5", converted: "change 5", altered: [], lost: [] }, { form: "Markdown bullet before a number", raw: "", converted: "- 5 sites", altered: [], lost: [] }, { form: "Markdown bullet before a negative number", raw: "", converted: "- -5 C", altered: [], lost: [] }, ] const GROUPING: Row[] = [ { form: "td | td | td", raw: "
3250100
", converted: "| 3 | 250 | 100 |", altered: [], lost: [] }, { form: "th | td", raw: "
12345
", converted: "| 12 | 345 |", altered: [], lost: [] }, { form: "tr / tr", raw: "
12
345
", converted: "| 12 |\n| 345 |", altered: [], lost: [] }, { form: "ul li / li", raw: "", converted: "- 3\n- 250", altered: [], lost: [] }, { form: "ol li / li", raw: "
  1. 3
  2. 250
", converted: "1. 3\n2. 250", altered: [], lost: [] }, { form: "p / p", raw: "

3

250

", converted: "3\n\n250", altered: [], lost: [] }, { form: "div / div", raw: "
12
345
", converted: "12\n345", altered: [], lost: [] }, { form: "h2 / p", raw: "

12

345

", converted: "## 12\n\n345", altered: [], lost: [] }, { form: "dt / dd", raw: "
12
345
", converted: "12\n: 345", altered: [], lost: [] }, { form: "blockquote / p", raw: "
12

345

", converted: "> 12\n\n345", altered: [], lost: [] }, { form: "br -> line break", raw: "

12
345

", converted: "12\n345", altered: [], lost: [] }, { form: "br kept as
in a Markdown cell", raw: "
2,500
300
", converted: "| 2,500
300 |", altered: [], lost: [] }, { form: "br flattened to a space (real artifact)", raw: "

12
345

", converted: "12 345", altered: ["12 345"], lost: ["12", "345"] }, { form: "li holding two paragraphs, flattened", raw: "", converted: "in 2023 500 sites", altered: [], lost: [] }, { form: "comma group then a space, one paragraph", raw: "

1,250\n300

", converted: "1,250 300", altered: [], lost: [] }, { form: "comma group | next cell", raw: "
1,250300
", converted: "| 1,250 | 300 |", altered: [], lost: [] }, { form: "period decimal | next cell", raw: "
1.234567
", converted: "| 1.234 | 567 |", altered: [], lost: [] }, { form: "KO td | td | td", raw: "
\uB9E4\uCD9C3,234\uC5B5\uC6D0
", converted: "| \uB9E4\uCD9C | 3,234 | \uC5B5\uC6D0 |", altered: [], lost: [] }, { form: "KO li / li", raw: "", converted: "- 12\n- 345\uBA85", altered: [], lost: [] }, { form: "KO p / p", raw: "

12

345\uC5B5 \uC6D0

", converted: "12\n\n345\uC5B5 \uC6D0", altered: [], lost: [] }, { form: "spaces inside one cell group", raw: "
1 234 5678
", converted: "| 1 234 567 | 8 |", altered: [], lost: [] }, { form: "NBSP groups", raw: "1\u00a0234\u00a0567", converted: "1 234 567", altered: [], lost: [] }, { form: "narrow NBSP groups", raw: "1\u202f234\u202f567", converted: "1 234 567", altered: [], lost: [] }, { form: "thin space entity groups", raw: "

1 234 567

", converted: "1 234 567", altered: [], lost: [] }, { form: "figure space entity groups", raw: "

1 234

", converted: "1 234", altered: [], lost: [] }, { form: "span-split groups", raw: "

1 234 567

", converted: "1 234 567", altered: [], lost: [] }, { form: "source newline between spans is a space", raw: "

1\n234

", converted: "1 234", altered: [], lost: [] }, { form: "KO space group with a unit", raw: "

1 234 567\uC6D0

", converted: "1 234 567\uC6D0", altered: [], lost: [] }, { form: "KO comma group in a cell", raw: "
3,234\uC5B5 \uC6D0
", converted: "| 3,234\uC5B5 \uC6D0 |", altered: [], lost: [] }, { form: "converter hard wrap inside a space group", raw: "

Population 1 234 567 people

", converted: "Population 1 234\n567 people", altered: [], lost: [] }, { form: "KO hard wrap inside a space group", raw: "

\uC778\uAD6C 1 234 567\uBA85

", converted: "\uC778\uAD6C 1 234\n567\uBA85", altered: [], lost: [] }, { form: "hard wrap between two separate numbers", raw: "

in 2023 500 sites

", converted: "in 2023\n500 sites", altered: [], lost: [] }, { form: "a wrap is not rejoined when the raw side holds both halves too", raw: "

12 345 and 12 and 345

", converted: "12\n345 and 12 and 345", altered: ["12", "345"], lost: ["12 345"] }, { form: "a blank line never rejoins a group", raw: "

Population 1 234 567 people

", converted: "Population 1 234\n\n567 people", altered: ["1 234", "567"], lost: ["1 234 567"] }, { form: "footnote glued onto a space group", raw: "

1 2345

", converted: "1 2345", altered: ["1", "2345"], lost: ["1 234", "5"] }, { form: "built-in path: td | td | td", raw: "
3250100
", converted: BUILTIN, altered: [], lost: [] }, { form: "built-in path: li / li", raw: "", converted: BUILTIN, altered: [], lost: [] }, { form: "built-in path: br inside a paragraph is joined by the snapshot, so it is flagged", raw: "

12
345

", converted: BUILTIN, altered: ["12 345"], lost: ["12", "345"] }, { form: "built-in path: br between cells' lines", raw: "
12
345
", converted: BUILTIN, altered: [], lost: [] }, { form: "built-in path: KO space group in a cell", raw: "
1 234 567\uC6D08
", converted: BUILTIN, altered: [], lost: [] }, ] // Markdown markup a converter writes beside a sign must not change how the sign reads: the text the // markup closes (or opens) decides, as it does in the raw HTML. One row per markup x dash x context. type Shape = { form: string; html: (c: string) => string; md: (c: string) => string; opaque?: boolean } const MARKUP: Shape[] = [ { form: "**", html: (c) => `${c}`, md: (c) => `**${c}**` }, { form: "*", html: (c) => `${c}`, md: (c) => `*${c}*` }, { form: "_", html: (c) => `${c}`, md: (c) => `_${c}_` }, { form: "__", html: (c) => `${c}`, md: (c) => `__${c}__` }, { form: "~~", html: (c) => `${c}`, md: (c) => `~~${c}~~` }, { form: "`", html: (c) => `${c}`, md: (c) => `\`${c}\`` }, { form: "``", html: (c) => `${c}`, md: (c) => `\`\`${c}\`\`` }, { form: "[](url)", html: (c) => `${c}`, md: (c) => `[${c}](/c)` }, { form: "[][ref]", html: (c) => `${c}`, md: (c) => `[${c}][c]` }, { form: "[] shortcut", html: (c) => `${c}`, md: (c) => `[${c}]` }, ] const DASHES = [["hyphen", "-"], ["en dash", "\u2013"], ["U+2212", "\u2212"]] as const const CLOSED = [ { form: "a digit", text: "2019", gap: "", after: "2023", signed: (_dash: string) => false }, { form: "a Latin word", text: "COVID", gap: "", after: "19", signed: (_dash: string) => false }, { form: "one Latin letter", text: "x", gap: "", after: "5", signed: (dash: string) => dash !== "\u2013" }, { form: "KO Hangul", text: "\uAC10\uC18C", gap: "", after: "3", signed: (_dash: string) => true }, { form: "a space", text: "fell", gap: " ", after: "5", signed: (_dash: string) => true }, ] // Every closer shape a CommonMark converter writes after text: inline links and images with a // "title", 'title' or (title), a target with balanced parentheses or in <...>, collapsed and full // reference forms, a linked image, and backslash escapes. An escaped closer is skipped like the raw // character it stands for; an escaped opener (`opaque`) is a real character, so every dash after it is a sign. const IMG = 'x `${c}`, md: (c) => `[${c}](/c "t")` }, { form: "[](url 'title')", html: (c) => `${c}`, md: (c) => `[${c}](/c 't')` }, { form: "[](url (title))", html: (c) => `${c}`, md: (c) => `[${c}](/c (t))` }, { form: '[](url "title with \\" and )")', html: (c) => `${c}`, md: (c) => `[${c}](/c "a \\" )")` }, { form: "[](url_(x))", html: (c) => `${c}`, md: (c) => `[${c}](/F_(x))` }, { form: '[](url_(x) "title")', html: (c) => `${c}`, md: (c) => `[${c}](/F_(x) "t")` }, { form: "[]()", html: (c) => `${c}`, md: (c) => `[${c}]()` }, { form: "[][] collapsed", html: (c) => `${c}`, md: (c) => `[${c}][]` }, { form: '[**text**](url "title")', html: (c) => `${c}`, md: (c) => `[**${c}**](/c "t")` }, { form: '**[text](url "title")**', html: (c) => `${c}`, md: (c) => `**[${c}](/c "t")**` }, { form: "![](src)", html: (c) => `${c}${IMG}>`, md: (c) => `${c}![x](i.png)` }, { form: '![](src "title")', html: (c) => `${c}${IMG} title="t">`, md: (c) => `${c}![x](i.png "t")` }, { form: "![](src 'title')", html: (c) => `${c}${IMG} title="t">`, md: (c) => `${c}![x](i.png 't')` }, { form: "![](a_(b).png)", html: (c) => `${c}x`, md: (c) => `${c}![x](a_(b).png)` }, { form: '![](a_(b).png "title")', html: (c) => `${c}x`, md: (c) => `${c}![x](a_(b).png "t")` }, { form: "![]()", html: (c) => `${c}x`, md: (c) => `${c}![x]()` }, { form: "![][ref]", html: (c) => `${c}${IMG}>`, md: (c) => `${c}![x][i]` }, { form: "[![](src)](url)", html: (c) => `${c}${IMG}>`, md: (c) => `${c}[![x](i.png)](/p)` }, { form: "**![](src)**", html: (c) => `${c}${IMG}>`, md: (c) => `${c}**![x](i.png)**` }, ...["*", "_", "]", ")", "~", "`"].map((x): Shape => ({ form: `escaped \\${x}`, html: (c) => `${c}${x}`, md: (c) => `${c}\\${x}` })), ...["[", "("].map((x): Shape => ({ form: `escaped \\${x} (an opener)`, html: (c) => `${c}${x}`, md: (c) => `${c}\\${x}`, opaque: true })), ] const closerRows = (shapes: Shape[]): Row[] => shapes.flatMap((m) => DASHES.flatMap(([dash, d]) => CLOSED.flatMap((c) => { const raw = `

${m.html(c.text)}${c.gap}${d}${c.after}

` const kept: Row = { form: `${m.form} closing ${c.form}, then a ${dash}`, raw, converted: `${m.md(c.text)}${c.gap}${d}${c.after}`, altered: [], lost: [] } if (!(m.opaque || c.signed(d))) return [kept] return [kept, { form: `${m.form} closing ${c.form}, ${dash} dropped`, raw, converted: `${m.md(c.text)}${c.gap}${c.after}`, altered: [c.after], lost: [`${d}${c.after}`] }] }))) const SIGN_MARKUP: Row[] = [ ...closerRows(MARKUP), ...closerRows(CLOSER_SHAPES), ...MARKUP.flatMap((m) => [ ...DASHES.flatMap(([dash, d]) => [ { form: `${m.form} around a ${dash} number after a space`, raw: `

fell ${m.html(`${d}5`)}

`, converted: `fell ${m.md(`${d}5`)}`, altered: [], lost: [] }, { form: `${m.form} around a ${dash} number at line start`, raw: `

${m.html(`${d}5`)} pts

`, converted: `${m.md(`${d}5`)} pts`, altered: [], lost: [] }, { form: `${m.form} opened between a ${dash} and its digits`, raw: `

fell ${d}${m.html("5")}

`, converted: `fell ${d}${m.md("5")}`, altered: [], lost: [] }, ]), { form: `${m.form} around a number whose U+2212 was dropped`, raw: `

fell ${m.html("\u22125")}

`, converted: `fell ${m.md("5")}`, altered: ["5"], lost: ["\u22125"] }, ]), { form: "reviewer: **2019**-2023", raw: "

2019-2023

", converted: "**2019**-2023", altered: [], lost: [] }, { form: "reviewer: *UTF*-8", raw: "

UTF-8

", converted: "*UTF*-8", altered: [], lost: [] }, { form: "reviewer: `v1`-2", raw: "

v1-2

", converted: "`v1`-2", altered: [], lost: [] }, { form: "image before a minus: the alt text is not the context", raw: '

Q chart\u22125

', converted: "Q ![chart](i.png)-5", altered: [], lost: [] }, { form: "KO image before a minus", raw: '

\uC99D\uAC10 \uADF8\uB9BC\u22122.1%

', converted: "\uC99D\uAC10 ![\uADF8\uB9BC](i.png)-2.1%", altered: [], lost: [] }, { form: "autolink closer before one letter's minus", raw: '

https://x.example/a\u22125

', converted: "-5", altered: [], lost: [] }, { form: "identifier kept through markup: **cite_note**-7", raw: "

cite_note-7

", converted: "**cite_note**-7", altered: [], lost: [] }, { form: "two letters split by markup stay an identifier", raw: "

ab\u22125

", converted: "a**b**-5", altered: [], lost: [] }, { form: "reviewer r5: titled link before an en dash range", raw: 'From 1990\u20132000', converted: 'From [1990](/wiki/1990 "1990")\u20132000', altered: [], lost: [] }, { form: "reviewer r5: titled link before a hyphen range", raw: 'From 1990-2000', converted: 'From [1990](/wiki/1990 "1990")-2000', altered: [], lost: [] }, { form: "reviewer r5: titled link on a word", raw: '

ab-5

', converted: '[ab](/x "t")-5', altered: [], lost: [] }, { form: "reviewer r5: titled image after a word", raw: '

abx-5

', converted: 'ab![x](i.png "t")-5', altered: [], lost: [] }, { form: "reviewer r5: image target with parentheses after a space", raw: '

fell x-5

', converted: "fell ![x](a_(b).png)-5", altered: [], lost: [] }, { form: "reviewer r5: escaped brackets", raw: "x[3]-5", converted: "x\\[3\\]-5", altered: [], lost: [] }, { form: "reviewer r5: escaped asterisk", raw: "foo*-5", converted: "foo\\*-5", altered: [], lost: [] }, { form: "reviewer r5: escaped underscore", raw: "ab_-5", converted: "ab\\_-5", altered: [], lost: [] }, { form: "link target digits after balanced parentheses are link syntax", raw: '

pic

', converted: "[pic](/wiki/File_(2019)_v9.png)", altered: [], lost: [] }, { form: "titled link target digits are link syntax", raw: '

pic

', converted: '[pic](/r/7 "Fig (4)")', altered: [], lost: [] }, ] // Raw inline HTML a converter leaves inside its Markdown (pandoc's gfm writer keeps , , // sized , and
): read with the html-lite tag reader, a sup, sub or break tag is a // boundary as on the raw side (`opaque`: every dash after it is a sign), any other tag or comment // is dropped with its attributes, like a Markdown closer or opener. const HTML_SHAPES: Shape[] = [ { form: "", html: (c) => `${c}`, md: (c) => `${c}`, opaque: true }, { form: "", html: (c) => `${c}`, md: (c) => `${c}`, opaque: true }, { form: "
after the text", html: (c) => `${c}
`, md: (c) => `${c}
`, opaque: true }, { form: "
after the text", html: (c) => `${c}
`, md: (c) => `${c}
`, opaque: true }, { form: "", html: (c) => `${c}`, md: (c) => `${c}` }, { form: '', html: (c) => `${c}`, md: (c) => `${c}` }, { form: " written for ", html: (c) => `${c}`, md: (c) => `${c}` }, { form: '', html: (c) => `${c}`, md: (c) => `${c}` }, { form: '', html: (c) => `${c}`, md: (c) => `${c}` }, { form: ' after the text', html: (c) => `${c}`, md: (c) => `${c}` }, { form: " after the text", html: (c) => `${c}x`, md: (c) => `${c}x` }, { form: " after the text", html: (c) => `${c}`, md: (c) => `${c}` }, { form: "****", html: (c) => `${c}`, md: (c) => `**${c}**` }, { form: "[link](url)", html: (c) => `${c}`, md: (c) => `[${c}](/c)` }, ] const RAW_HTML: Row[] = [ ...closerRows(HTML_SHAPES), { form: "reviewer r6: pandoc 10-5", raw: "

10-5 m

", converted: "10-5 m", altered: [], lost: [] }, { form: "reviewer r6: pandoc 10U+2212 5", raw: "

10\u22125 m

", converted: "10\u22125 m", altered: [], lost: [] }, { form: "reviewer r6: pandoc 109", raw: "

Output reached 109 kWh

", converted: "Output reached 109 kWh", altered: [], lost: [] }, { form: "reviewer r6: a-5", raw: "

a-5

", converted: "a-5", altered: [], lost: [] }, { form: 'reviewer r6: ', raw: '

', converted: '', altered: [], lost: [] }, { form: "reviewer r6: before a number", raw: "

5 apples

", converted: " 5 apples", altered: [], lost: [] }, { form: "KO before a number", raw: "

\uC0AC\uACFC 5\uAC1C

", converted: " \uC0AC\uACFC 5\uAC1C", altered: [], lost: [] }, { form: "reviewer r6: 2019a-2023", raw: "

2019a-2023

", converted: "2019a-2023", altered: [], lost: [] }, { form: "KO 3,2341\uC5B5 kept as raw HTML", raw: "

\uB9E4\uCD9C 3,2341\uC5B5 \uC6D0

", converted: "\uB9E4\uCD9C 3,2341\uC5B5 \uC6D0", altered: [], lost: [] }, { form: "a superscript the converter flattened is still altered", raw: "

10-5 m

", converted: "10-5 m", altered: ["5"], lost: ["-5"] }, { form: "a glued footnote is still altered next to raw HTML", raw: "

4.22 bn

", converted: "4.22 bn", altered: ["4.22"], lost: ["4.2", "2"] }, { form: "an img width written into prose is still altered", raw: "

5

", converted: "320 5", altered: ["320"], lost: [] }, { form: "an autolink is not a tag", raw: '

https://x.example/2024

', converted: "", altered: [], lost: [] }, { form: "an escaped \\< is text, not a tag", raw: "

a<b>7

", converted: "a\\7", altered: [], lost: [] }, { form: "a < with no tag end stays text", raw: "

a <sup 5

", converted: "a a <span

5 x>

", converted: "a ", altered: [], lost: [] }, ] // A reference definition's title may sit alone on the next line (CommonMark): its digits are link // syntax there too. Only the title is read across the line break; text after it makes it prose. const REF_TITLES: Row[] = [ ...[['"Fig 2019"', "double quotes"], ["'Fig 2019'", "single quotes"], ["(Fig 2019)", "parentheses"]].flatMap(([title, form]) => [ { form: `title in ${form} on the next line, indented`, raw: '

fig

', converted: `[fig][1]\n\n[1]: /x\n ${title}`, altered: [], lost: [] }, { form: `title in ${form} on the next line, not indented`, raw: '

fig

', converted: `[fig][1]\n\n[1]: /x\n${title}\n`, altered: [], lost: [] }, ]), { form: "KO title on the next line", raw: '

\uADF8\uB9BC

', converted: '[\uADF8\uB9BC][1]\n\n[1]: /x\n "\uADF8\uB9BC 2019"', altered: [], lost: [] }, { form: "two definitions, the first with its title on the next line", raw: '

a b

', converted: '[a][1] [b][2]\n\n[1]: /x\n "Fig 2019"\n[2]: /y/7', altered: [], lost: [] }, { form: "a quoted line with text after it is prose, not a title", raw: '

fig

', converted: '[fig][1]\n\n[1]: /x\n"Fig 2019" was cited', altered: ["2019"], lost: [] }, { form: "a next-line title digit the source does not hold is altered", raw: '

fig

', converted: '[fig][1]\n\n[1]: /x\n "Fig 2019"', altered: ["2019"], lost: [] }, ] // Declared out of scope (references/fact-ledger.md): constructs outside the CommonMark inline grammar, // the html-lite tag set and the named-entity table. Each row ASSERTS the safe-direction report, so // the boundary is tested: the numbers are altered and `source snapshot --replace` recovers. const OUT_OF_SCOPE: Row[] = [ { form: "custom converter syntax: a Hugo figure shortcode", raw: '
', converted: '{{< figure src="chart-2019.png" width="640" >}}', altered: ["2019", "640"], lost: [] }, { form: "custom converter syntax: pandoc image attributes {width=640}", raw: '

x

', converted: "![x](i.png){width=640}", altered: ["640"], lost: [] }, { form: "LaTeX: \\frac{3}{4} for a vulgar fraction", raw: "

\u00BE of sites

", converted: "$\\frac{3}{4}$ of sites", altered: ["3", "4"], lost: [] }, { form: "KO LaTeX: \\frac{3}{4}", raw: "

\uC0AC\uC774\uD2B8\uC758 \u00BE

", converted: "\uC0AC\uC774\uD2B8\uC758 $\\frac{3}{4}$", altered: ["3", "4"], lost: [] }, { form: "MathJax: \\sqrt[3]{8} for a cube root", raw: "

\u221B8

", converted: "\\(\\sqrt[3]{8}\\)", altered: ["3"], lost: [] }, { form: "non-CommonMark table: AsciiDoc cols", raw: "
AB
", converted: '[cols="1,3"]\n|===\n| A | B\n|===', altered: ["1,3"], lost: [] }, { form: "non-CommonMark table: a colspan cell repeated per column", raw: '
5
', converted: "| 5 | 5 |", altered: ["5"], lost: [] }, { form: "footnote extension beyond pandoc's: a numbered back-reference title", raw: '

Margin 12%a

Filing.

', converted: 'Margin 12%[^a]\n\n[^a]: Filing. [\u21A9](#fnref-a "Jump back to footnote 2 in the text")', altered: ["2"], lost: [] }, { form: "reference definition with its destination on the next line", raw: '

fig

', converted: "[fig][1]\n\n[1]:\n /x/2019", altered: ["2019"], lost: [] }, { form: "an unescaped & whose text spells a reference (turndown)", raw: "

Type &minus;5 to negate

", converted: "Type −5 to negate", altered: ["\u22125"], lost: ["5"] }, { form: "a legacy reference without its semicolon ( )", raw: "

1 234 567

", converted: "1\u00A0234 567", altered: ["1 234 567"], lost: ["1", "234 567"] }, { form: "a numeric reference padded past 6 hex digits", raw: "

−5

", converted: "\u22125", altered: ["\u22125"], lost: ["00002212", "5"] }, ] // Named character references whose decoded character the probe reads as a sign or a separator (or // as markup), decoded on the raw side as a converter decodes them: each in a sign position and a // separator position, EN and KO, kept as the decoded character, plus the sign position with the // character dropped (a sign is lost; any other character changes nothing). const ENTITIES: [string, string][] = [ ["minus", "\u2212"], ["ndash", "\u2013"], ["mdash", "\u2014"], ["hyphen", "\u2010"], ["dash", "\u2010"], ["thinsp", "\u2009"], ["ThinSpace", "\u2009"], ["numsp", "\u2007"], ["puncsp", "\u2008"], ["hairsp", "\u200A"], ["VeryThinSpace", "\u200A"], ["ensp", "\u2002"], ["emsp", "\u2003"], ["emsp13", "\u2004"], ["emsp14", "\u2005"], ["MediumSpace", "\u205F"], ["nbsp", "\u00A0"], ["NonBreakingSpace", "\u00A0"], ["comma", ","], ["period", "."], ["plus", "+"], ["lowbar", "_"], ["ast", "*"], ["midast", "*"], ["grave", "`"], ["DiacriticalGrave", "`"], ["lpar", "("], ["rpar", ")"], ["lsqb", "["], ["lbrack", "["], ["rsqb", "]"], ["rbrack", "]"], ] const SIGN_CHARS = new Set(["\u2212", "\u2013"]) const ENTITY_ROWS: Row[] = ENTITIES.flatMap(([name, ch]) => { const e = `&${name};` return [ { form: `&${name}; in a sign position`, raw: `

fell ${e}5 pts

`, converted: `fell ${ch}5 pts`, altered: [], lost: [] }, { form: `KO &${name}; in a sign position`, raw: `

\uAC10\uC18C${e}3%

`, converted: `\uAC10\uC18C${ch}3%`, altered: [], lost: [] }, { form: `&${name}; as a separator`, raw: `

1${e}234${e}567 people

`, converted: `1${ch}234${ch}567 people`, altered: [], lost: [] }, { form: `KO &${name}; as a separator`, raw: `

\uC778\uAD6C 1${e}234${e}567\uBA85

`, converted: `\uC778\uAD6C 1${ch}234${ch}567\uBA85`, altered: [], lost: [] }, { form: `&${name}; in a sign position, dropped`, raw: `

fell ${e}5 pts

`, converted: "fell 5 pts", altered: SIGN_CHARS.has(ch) ? ["5"] : [], lost: SIGN_CHARS.has(ch) ? [`${ch}5`] : [] }, { form: `KO &${name}; in a sign position, dropped`, raw: `

\uAC10\uC18C${e}3%

`, converted: "\uAC10\uC18C3%", altered: SIGN_CHARS.has(ch) ? ["3"] : [], lost: SIGN_CHARS.has(ch) ? [`${ch}3`] : [] }, { form: `&${name}; kept as the reference in a sign position`, raw: `

fell ${e}5 pts

`, converted: `fell ${e}5 pts`, altered: [], lost: [] }, { form: `KO &${name}; kept as the reference in a sign position`, raw: `

\uAC10\uC18C${e}3%

`, converted: `\uAC10\uC18C${e}3%`, altered: [], lost: [] }, { form: `&${name}; kept as the reference as a separator`, raw: `

1${e}234${e}567 people

`, converted: `1${e}234${e}567 people`, altered: [], lost: [] }, { form: `KO &${name}; kept as the reference as a separator`, raw: `

\uC778\uAD6C 1${e}234${e}567\uBA85

`, converted: `\uC778\uAD6C 1${e}234${e}567\uBA85`, altered: [], lost: [] }, { form: `&${name}; in a verbatim plain-text copy (sign position)`, raw: `fell ${e}5 pts`, converted: `fell ${e}5 pts`, altered: [], lost: [] }, { form: `&${name}; in a verbatim plain-text copy (separator)`, raw: `1${e}234${e}567 people`, converted: `1${e}234${e}567 people`, altered: [], lost: [] }, ] }) ENTITY_ROWS.push( { form: "reviewer r5:  − in a text with no tags", raw: "x −3.5%", converted: "x \u22123.5%", altered: [], lost: [] }, { form: "reviewer r5:   groups in a text with no tags", raw: "1 234 567", converted: "1\u2009234\u2009567", altered: [], lost: [] }, { form: "reviewer r5:  − in a paragraph", raw: "

x −3.5%

", converted: "x\u00A0\u22123.5%", altered: [], lost: [] }, { form: "an unterminated reference in a text with no tags stays literal", raw: "a < b & c 5", converted: "a < b & c 5", altered: [], lost: [] }, { form: "reviewer r6: 'Rate −5 and 1 234' copied verbatim", raw: "Rate −5 and 1 234", converted: "Rate −5 and 1 234", altered: [], lost: [] }, { form: "KO verbatim copy with − and  ", raw: "\uC99D\uAC10 −5%, \uC778\uAD6C 1 234\uBA85", converted: "\uC99D\uAC10 −5%, \uC778\uAD6C 1 234\uBA85", altered: [], lost: [] }, { form: "a sign the converted side spells as another reference is still dropped", raw: "

fell −5 pts

", converted: "fell  5 pts", altered: ["5"], lost: ["\u22125"] }, { form: "a separator spelled as a reference on the converted side still joins two numbers", raw: "

12

345

", converted: "12 345", altered: ["12 345"], lost: ["12", "345"] }, ) describe("conversion probe classes (one row per form, EN and KO)", () => { test.each(rows(CONVERTER_DIGITS))("converter-emitted digits: %s", (_form, row) => check(row)) test.each(rows(SIGNS))("signs and separators around numbers: %s", (_form, row) => check(row)) test.each(rows(GROUPING))("grouping boundaries: %s", (_form, row) => check(row)) test.each(rows(SIGN_MARKUP))("sign context through Markdown markup: %s", (_form, row) => check(row)) test.each(rows(ENTITY_ROWS))("named character references on both sides: %s", (_form, row) => check(row)) test.each(rows(RAW_HTML))("raw inline HTML in the converted Markdown: %s", (_form, row) => check(row)) test.each(rows(REF_TITLES))("reference definition titles on the next line: %s", (_form, row) => check(row)) test.each(rows(OUT_OF_SCOPE))("declared out of scope (reported as altered): %s", (_form, row) => check(row)) })