// // Copyright 2025 DXOS.org // import { ensureSyntaxTree, syntaxTree } from '@codemirror/language'; import { EditorState } from '@codemirror/state'; import { describe, test } from 'vitest'; import { trim } from '@dxos/util'; import { extendedMarkdown } from './extended-markdown.ts'; import TEXT from './testing/text.md?raw'; import { type XmlWidgetRegistry, getXmlInnerText, getXmlTextChild, xmlTags } from './xml-tags.ts'; import { type Tag, nodeToJson } from './xml-util.ts'; type ParsedElement = Tag & { /** Element spans the entire document (opening through closing tag). */ complete: boolean; }; /** * Helper to extract all parsed XML elements from a document. * Checks completeness by verifying the Element node has a CloseTag child. */ const parseElements = (doc: string, registry: XmlWidgetRegistry = {}): ParsedElement[] => { const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry }), xmlTags({ registry })], }); const tree = ensureSyntaxTree(state, doc.length, 5000) ?? syntaxTree(state); const elements: ParsedElement[] = []; tree.iterate({ enter: (node) => { if (node.type.name === 'Element') { const tag = nodeToJson(state, node.node); if (tag) { const hasCloseTag = !!node.node.getChild('CloseTag'); const hasSelfClose = !!node.node.getChild('SelfClosingTag'); elements.push({ ...tag, complete: hasCloseTag || hasSelfClose, }); } return false; } }, }); return elements; }; describe('nodeToJson', () => { test('should parse a simple element', ({ expect }) => { const xml = trim` # Test `; const elements = parseElements(xml); expect(elements).toHaveLength(1); expect(elements[0]).toMatchObject({ _tag: 'test', id: '123', foo: '100', }); }); test('should parse tag with single-line content', ({ expect }) => { const xml = trim` The user is asking about markdown. `; const registry = { reasoning: { block: true } }; const elements = parseElements(xml, registry); expect(elements).toHaveLength(1); expect(elements[0]._tag).toBe('reasoning'); expect(elements[0].complete).toBe(true); expect(elements[0].children).toEqual(['The user is asking about markdown.']); }); test('should parse tag with multi-line content without blank lines', ({ expect }) => { const xml = trim` The user is asking about markdown. This is a follow-up thought. `; const registry = { reasoning: { block: true } }; const elements = parseElements(xml, registry); expect(elements).toHaveLength(1); expect(elements[0]._tag).toBe('reasoning'); expect(elements[0].complete).toBe(true); }); // BUG: Blank lines inside XML tags cause the markdown parser to split the content // into multiple blocks (HTMLBlock + Paragraph), preventing the XML mixed parser from // seeing the full element. The first HTMLBlock gets an incomplete Element (no CloseTag), // and the closing tag ends up in a Paragraph that's never parsed as XML. // // Syntax tree with blank lines: // Document [0-28] → Element (incomplete: OpenTag + Text + ⚠ — no CloseTag) // Paragraph [30-60] → "Second paragraph.\n" test('should parse tag with blank lines in content', ({ expect }) => { const xml = trim` The user is asking me to think deeply about what markdown is. But given the context of our conversation - we've been trying to scrape slab data from a website - I think they might be hinting at something specific. `; const registry = { reasoning: { block: true } }; const elements = parseElements(xml, registry); expect(elements).toHaveLength(1); expect(elements[0]._tag).toBe('reasoning'); expect(elements[0].complete).toBe(true); }); // Regression: when reasoning text contains escaped XML (e.g. `<foo>`), Lezer XML // splits the inline content into Text + EntityReference + Text + EntityReference + Text // siblings. The walker used to skip entity references and push each text segment as a // separate child, so `getXmlTextChild(children)` only saw the prefix before the first // `<` (e.g. ``"The user sent `"``). test('should preserve text containing entity references as a single child', ({ expect }) => { const xml = trim` The user sent \`<foo>\`, which appears to be an XML-like tag. `; const registry = { reasoning: { block: true } }; const elements = parseElements(xml, registry); expect(elements).toHaveLength(1); expect(elements[0]._tag).toBe('reasoning'); expect(elements[0].children).toEqual(['The user sent ``, which appears to be an XML-like tag.']); }); // Same fault as the entity-reference case above, for a nested ELEMENT rather than an entity: the // parser splits contents into alternating text and element children, so `getXmlTextChild` returns // the prefix and drops the rest. The reader saw a heading and an opening fence, then nothing. test('should read the whole text of a tag holding a nested element', ({ expect }) => { const xml = trim` Completed the checklist: 1. [ ] Review new messages. done. `; const registry = { synthetic: { block: true } }; const [element] = parseElements(xml, registry); const children = element.children ?? []; // The split is what the fault is made of: text, then the element, then text. expect(children).toHaveLength(3); expect(getXmlTextChild(children)).toBe('Completed the checklist:\n'); // The nested tag contributes its text but not its markup. expect(getXmlInnerText(children)).toBe('Completed the checklist:\n1. [ ] Review new messages.\ndone.'); }); test('should decode numeric character references', ({ expect }) => { const xml = trim` arrow → and hex → `; const registry = { reasoning: { block: true } }; const elements = parseElements(xml, registry); expect(elements).toHaveLength(1); expect(elements[0].children).toEqual(['arrow → and hex →']); }); test('should parse tag with multiple blank lines in content', ({ expect }) => { const xml = trim` First paragraph. Second paragraph after two blank lines. `; const registry = { reasoning: { block: true } }; const elements = parseElements(xml, registry); expect(elements).toHaveLength(1); expect(elements[0]._tag).toBe('reasoning'); expect(elements[0].complete).toBe(true); }); // Regression: when an unregistered XML tag like `` appears earlier in the doc, // the markdown parser treats it as a paragraph that lazy-continues into subsequent lines, // and a later multi-line tag (e.g. `` whose body contains an ordered list) gets // its HTMLBlock truncated to its first line — losing the closing tag and breaking widget // rendering. Including the surrounding tag in the registry as a block-only entry (no // factory/Component) lets `xmlBlockParsers` keep each tag as its own block. test('multi-line reasoning is complete when preceded by a registered prompt block', ({ expect }) => { const xml = [ 'summarize the posts', '', 'multi line content', '1. "First" - desc', '5. "Last" - desc', ].join('\n'); const registry = { prompt: { block: true }, reasoning: { block: true }, toolCall: { block: true }, }; const elements = parseElements(xml, registry); expect(elements.map((e) => `${e._tag}${e.complete ? '' : '!'}`)).toEqual(['prompt', 'toolCall', 'reasoning']); }); test('should parse text.md', ({ expect }) => { // Mirror the live ChatThread registry, including `prompt` as a block-only entry so // unregistered-paragraph lazy-continuation does not break later multi-line tags. const registry = { prompt: { block: true }, reasoning: { block: true }, status: { block: true }, toolCall: { block: true }, name: { block: true }, }; const elements = parseElements(TEXT, registry); const tags = elements.map((element) => element._tag); expect(tags).toEqual([ 'prompt', 'reasoning', 'status', 'toolCall', 'toolCall', 'reasoning', 'status', 'toolCall', 'reasoning', 'prompt', 'reasoning', 'prompt', 'name', ]); // Every element must be `complete` so the widget renderer wraps it. The bug we fixed: // third multi-line `` (lines 9-14) used to be truncated by markdown's // OrderedList parser breaking the HTMLBlock, leaving an Element node with no CloseTag. expect(elements.filter((e) => !e.complete)).toEqual([]); }); });