/** * Streaming-path tests for the unified markdown engine (plan §D1): * atomic-block splitting, fence tail-completion, card-block cache * exclusion, and — critically — that the COMPLETION render (streaming * flipped off) equals the whole-document parse for every cross-block * construct (late reference defs, loose-list numbering, footnote-ish * splits, blockquote-internal blank lines, streamed tables). */ import React from 'react' import { describe, it, expect, vi } from 'vitest' import type { Components } from 'react-markdown' import { render, act } from '@testing-library/react' import { splitStreamingBlocks, completeStreamingTail } from '../streaming' import { MarkdownEngine } from '../engine' vi.mock('mermaid', () => ({ default: { initialize: vi.fn(), render: vi.fn(async () => ({ svg: '' })), }, })) async function renderHtml(ui: React.ReactElement): Promise { let container: HTMLElement await act(async () => { const result = render(ui) container = result.container await new Promise((r) => setTimeout(r, 0)) }) return container!.innerHTML } describe('splitStreamingBlocks', () => { it('splits plain paragraphs at blank lines', () => { const blocks = splitStreamingBlocks('para one\n\npara two\n\npara three') expect(blocks.map((b) => b.memoizable)).toEqual([true, true, false]) expect(blocks[0].text).toContain('para one') expect(blocks[2].text).toContain('para three') }) it('never cuts inside a fenced code block', () => { const blocks = splitStreamingBlocks('```js\ncode\n\nmore code\n```\n\nafter') expect(blocks).toHaveLength(2) expect(blocks[0].text).toContain('more code') }) it('never cuts a loose ordered list apart', () => { const md = '1. first\n\n2. second\n\n3. third' const blocks = splitStreamingBlocks(md) expect(blocks).toHaveLength(1) }) it('never cuts a multi-paragraph blockquote apart', () => { const md = '> first para\n\n> second para' expect(splitStreamingBlocks(md)).toHaveLength(1) }) it('does not cut before an indented continuation line', () => { const md = '- item one\n\n continuation of item one\n\nnext para' const blocks = splitStreamingBlocks(md) expect(blocks[0].text).toContain('continuation') }) it('marks card:// and mention:// blocks non-memoizable (stale-card fix)', () => { const blocks = splitStreamingBlocks( 'plain text block\n\nSee [card://blog:x] here\n\nCheck @device:r1 via mention\n\ntail', ) expect(blocks.map((b) => b.memoizable)).toEqual([true, false, true, false]) // mention text without mention:// scheme stays memoizable; the scheme // appears post-remark, so the splitter keys on the raw marker too: const withScheme = splitStreamingBlocks('a [x](mention://device:r1) link\n\ntail') expect(withScheme[0].memoizable).toBe(false) }) it('marks reference-definition/use and footnote blocks non-memoizable', () => { const blocks = splitStreamingBlocks( 'See [text][ref] please\n\n[ref]: https://example.com\n\nA footnote[^1] here\n\ntail', ) expect(blocks.map((b) => b.memoizable)).toEqual([false, false, false, false]) }) it('keys blocks by position so identical blocks never alias', () => { const blocks = splitStreamingBlocks('---\n\n---\n\nend') expect(blocks.map((b) => b.index)).toEqual([0, 1, 2]) }) // --- fence-marker fidelity (reviewer-reported failures) --- it('does not let ~~~ close a ``` fence', () => { const md = '```text\nline\n~~~\n\nstill in code\n```\n\nafter' const blocks = splitStreamingBlocks(md) // The whole code block stays ONE unit; only the trailing prose splits. expect(blocks).toHaveLength(2) expect(blocks[0].text).toContain('still in code') expect(blocks[1].text.trim()).toBe('after') }) it('does not let ``` close a ~~~ fence', () => { const blocks = splitStreamingBlocks('~~~text\nline\n```\n\nstill in code\n~~~\n\nafter') expect(blocks).toHaveLength(2) expect(blocks[0].text).toContain('still in code') }) it('ignores fences inside 4-space-indented code (CommonMark caps indent at 3)', () => { const blocks = splitStreamingBlocks(' ```\n not a fence\n\nreal para\n\ntail') expect(blocks.length).toBeGreaterThan(1) expect(blocks.some((b) => b.text.includes('real para'))).toBe(true) }) it('requires the closer to be at least as long as the opener', () => { // The inner ``` is CONTENT of the ````-opened block. const blocks = splitStreamingBlocks('````\n```\n\ninner\n````\n\nafter') expect(blocks).toHaveLength(2) expect(blocks[0].text).toContain('inner') }) // --- raw HTML block integrity --- it('never cuts inside an unbalanced raw HTML block', () => { const blocks = splitStreamingBlocks( '
\nMore\n\nbody text\n\n
\n\ntail', ) expect(blocks).toHaveLength(2) expect(blocks[0].text).toContain('body text') expect(blocks[0].text).toContain('') // Raw HTML units are never cached. expect(blocks.every((b) => !b.memoizable)).toBe(true) }) it('never cuts inside a MULTI-LINE start tag', () => { // `HTML_TAG_RE` needs `>` on the same line, so `` never // opened the container: the splitter cut at every blank line and marked // the fragments memoizable — the container's body escaped the div and // the broken render got cached. const blocks = splitStreamingBlocks('\n\nbody\n\n\n\nafter') expect(blocks).toHaveLength(2) expect(blocks[0].text).toContain('body') expect(blocks[0].text).toContain('') expect(blocks.every((b) => !b.memoizable)).toBe(true) }) it('releases the latch when a container closes MID-LINE', () => { // A close tag not at column 0..3 (`some text `) never // decremented, so `htmlDepth` latched >0 and the whole rest of the // message collapsed into ONE non-memoizable unit for the rest of the // stream. const blocks = splitStreamingBlocks( '
\nstuff\n\nsome text
\n\nafter\n\nmore\n\ntail', ) expect(blocks).toHaveLength(4) expect(blocks[0].text).toContain('') expect(blocks[0].memoizable).toBe(false) // Cutting resumed — the plain paragraphs after the container cache again. expect(blocks.map((b) => b.memoizable)).toEqual([false, true, true, false]) }) it('sees raw HTML indented to a list item content column', () => { // The old column-0..3 gate missed this entirely, so the unit was marked // memoizable despite containing raw HTML. const blocks = splitStreamingBlocks( '1. step\n\n
\n x\n
\n\ntail', ) expect(blocks[0].text).toContain('
') expect(blocks[0].memoizable).toBe(false) }) it('caps the latch so an unclosed container cannot disable cutting forever', () => { const paras = Array.from({ length: 300 }, (_, i) => `para ${i}`).join('\n\n') const blocks = splitStreamingBlocks(`
\n\n${paras}`) // Past the cap the splitter resumes cutting… expect(blocks.length).toBeGreaterThan(1) // …but nothing inside the still-open container is ever cached. expect(blocks.every((b) => !b.memoizable)).toBe(true) }) it('reports each unit’s document-wide startLine', () => { const blocks = splitStreamingBlocks('one\n\ntwo\n\nthree') expect(blocks.map((b) => b.startLine)).toEqual([1, 3, 5]) }) }) describe('completeStreamingTail', () => { it('closes an unterminated fence', () => { expect(completeStreamingTail('text\n\n```js\nconst x =')).toMatch(/```$/) }) it('leaves balanced fences alone', () => { const md = '```js\ncode\n```' expect(completeStreamingTail(md)).toBe(md) }) it('does NOT touch stray emphasis/link openers (no false-positive close)', () => { for (const md of ['2 * 3 is six', 'a lone _ under', 'an open [ bracket', 'tick ` alone']) { expect(completeStreamingTail(md)).toBe(md) } }) it('does not append a bogus fence when ``` and ~~~ both appear (closed block)', () => { const md = '```\n~~~\n```' expect(completeStreamingTail(md)).toBe(md) }) it('does not append a bogus fence to a ```-block containing a ~~~ line', () => { const md = '```text\nline\n~~~\n\nstill in code\n```\n\nafter' expect(completeStreamingTail(md)).toBe(md) }) it('closes an unterminated ~~~ fence with ~~~ (the RECORDED opener)', () => { expect(completeStreamingTail('~~~py\nx =')).toBe('~~~py\nx =\n~~~') }) it('closes with a run at least as long as the opener', () => { expect(completeStreamingTail('````\nstuff')).toBe('````\nstuff\n````') }) }) describe('MarkdownEngine streaming vs completion equivalence', () => { const textOf = (html: string) => html .replace(//, '') .replace(/<[^>]+>/g, '') .replace(/\s+/g, '') const CASES: Record = { 'loose list keeps numbering': { md: '1. first\n\n2. second\n\n3. third' }, // Reference links are the DOCUMENTED acceptable mid-stream transient: // the use renders literally until the def arrives; completion self-heals. 'late reference definition resolves': { md: 'See [the docs][ref] now.\n\n[ref]: https://example.com/late', transientTextDivergence: true, }, 'blockquote with internal blank line': { md: '> first\n>\n> second' }, 'gfm table': { md: '| A | B |\n|---|---|\n| 1 | 2 |' }, 'multi-block document': { md: '# Title\n\npara one\n\n- a\n- b\n\npara two' }, } for (const [name, { md, transientTextDivergence }] of Object.entries(CASES)) { it(`completion render equals whole-document parse: ${name}`, async () => { const streamingHtml = await renderHtml() const completedHtml = await renderHtml() const wholeHtml = await renderHtml() // Completion IS the authoritative whole-document parse — always. expect(completedHtml).toBe(wholeHtml) if (!transientTextDivergence) { expect(textOf(streamingHtml)).toBe(textOf(wholeHtml)) } }) } /** * The `textOf` comparison above cannot see this class of divergence: when a * list is cut apart, every remaining item becomes its own `
    `/`
      ` with * no `start`, so the TEXT is identical while the visible ordinals restart at * "1." for every item. The assertion therefore has to be on list STRUCTURE. * * The shape is the single most common assistant answer — numbered steps whose * items carry a code fence, a second paragraph, a table, or a nested list — and * the broken units were `memoizable`, so the wrong render was CACHED for the * rest of the turn rather than self-healing on the next token. */ const LIST_CONTINUATION_CASES: Record = { 'ordered list with per-item code fences': '1. Install:\n\n ```bash\n npm i\n ```\n\n2. Run it.\n\n3. Done.\n', 'ordered list with 4-space-indented fences': '1. Install:\n\n ```bash\n npm i\n ```\n\n2. Run it.\n', 'list item with a second paragraph': '- item\n\n second para of item\n\n- item2\n', 'list item with a table continuation': '1. Options:\n\n | a | b |\n |---|---|\n | 1 | 2 |\n\n2. Next step.\n', 'list item with a nested sublist': '- item\n - nested\n\n- item2\n', 'a list that follows an unrelated paragraph': '- one\n\n- two\n\nplain para\n\n- three\n\n cont\n\n- four\n', } for (const [name, md] of Object.entries(LIST_CONTINUATION_CASES)) { it(`never cuts a list apart mid-stream: ${name}`, async () => { const streamingHtml = await renderHtml() const wholeHtml = await renderHtml() const listCount = (html: string) => (html.match(/<[ou]l[\s>]/g) ?? []).length expect(listCount(streamingHtml)).toBe(listCount(wholeHtml)) expect(textOf(streamingHtml)).toBe(textOf(wholeHtml)) }) } it('splits a list with a continuation block into ONE unit', () => { const md = '1. Install:\n\n ```bash\n npm i\n ```\n\n2. Run it.\n\n3. Done.\n' expect(splitStreamingBlocks(md)).toHaveLength(1) }) it('still cuts between a list and a genuinely separate following block', () => { // The guard must not become "never cut once a list appeared" — a paragraph // after a list is a real boundary and its unit must stay memoizable. const blocks = splitStreamingBlocks('- one\n\n- two\n\nplain para\n\ntail para\n') expect(blocks.length).toBeGreaterThan(1) expect(blocks[0].text).toContain('- one') expect(blocks[0].memoizable).toBe(true) }) it('streaming wrapper is aria-live polite', async () => { const html = await renderHtml() expect(html).toContain('aria-live="polite"') const done = await renderHtml() expect(done).not.toContain('aria-live') }) it('raw HTML block renders identically mid-stream and completed', async () => { const md = '
      \nMore\n\nbody text\n\n
      \n\ntail' const streamingHtml = await renderHtml() const wholeHtml = await renderHtml() // Strip only the streaming aria-live wrapper; the disclosure widget and // its body must be structurally identical (body INSIDE
      , the // closing tag not dropped). const stripWrapper = (html: string) => html .replace('
      ', '') .replace('
      ', '') expect(streamingHtml).toContain(' { const html = await renderHtml( ]+>/g, '')).toContain('const partial =') }) }) // --------------------------------------------------------------------------- // Atomic-block memo (regression: the memo never hit) // --------------------------------------------------------------------------- describe('streaming atomic-block memoization', () => { /** * Module-scope so the override map's IDENTITY is stable across rerenders — * otherwise the test would be measuring its own instability instead of the * engine's. The counter proves memo HITS: without it, DOM reconciliation * reuses the same nodes either way, so node identity alone cannot fail. */ let paragraphRenders = 0 const COUNTING_OVERRIDES: Partial = { // eslint-disable-next-line @typescript-eslint/no-explicit-any p: ({ children }: any) => { paragraphRenders++ return

      {children}

      }, } it('does NOT re-render completed blocks when the tail grows', async () => { const base = 'alpha para\n\nbeta para\n\ngamma tail' paragraphRenders = 0 let container!: HTMLElement let rerender!: (ui: React.ReactElement) => void await act(async () => { const result = render( , ) container = result.container rerender = result.rerender await new Promise((r) => setTimeout(r, 0)) }) const firstParagraph = container.querySelector('p') expect(firstParagraph?.textContent).toBe('alpha para') expect(paragraphRenders).toBe(3) const before = paragraphRenders await act(async () => { rerender( , ) await new Promise((r) => setTimeout(r, 0)) }) // Only the live tail may re-parse. Before the NO_BROKEN_LINKS fix the // `components` memo re-created every render, so all THREE re-rendered. expect(paragraphRenders - before).toBe(1) // …and the completed block's DOM node is the very same element. expect(Object.is(container.querySelector('p'), firstParagraph)).toBe(true) }) it('survives heading ids: appending prose does not bust the block memo', async () => { // The heading-id map is rebuilt on EVERY token (its input is the growing // `processedContent`). It therefore reaches the heading renderers via // CONTEXT, never through the `components` memo — otherwise its changing // identity would re-create `components` per token and every completed // block's `memo` would bail, exactly the regression above. const base = '## Setup\n\nalpha para\n\nbeta para\n\ngamma tail' paragraphRenders = 0 let container!: HTMLElement let rerender!: (ui: React.ReactElement) => void await act(async () => { const result = render( , ) container = result.container rerender = result.rerender await new Promise((r) => setTimeout(r, 0)) }) const firstParagraph = container.querySelector('p') const before = paragraphRenders await act(async () => { rerender( , ) await new Promise((r) => setTimeout(r, 0)) }) expect(paragraphRenders - before).toBe(1) expect(Object.is(container.querySelector('p'), firstParagraph)).toBe(true) // A tail that DOES add a heading still gets correct, unique ids. await act(async () => { rerender( , ) await new Promise((r) => setTimeout(r, 0)) }) const ids = Array.from(container.querySelectorAll('h2')).map((el) => el.id) expect(ids).toEqual(['setup', 'setup-2']) expect(new Set(ids).size).toBe(ids.length) }) })