// // Copyright 2025 DXOS.org // import { syntaxTree } from '@codemirror/language'; import { EditorState } from '@codemirror/state'; import { type SyntaxNode } from '@lezer/common'; import { describe, test } from 'vitest'; import { trim } from '@dxos/util'; import { extendedMarkdown } from './extended-markdown.ts'; import { nodeToJson } from './xml-util.ts'; const testRegistry = { prompt: { block: true }, suggestion: { block: true }, choice: { block: true }, toolkit: { block: true }, toolCall: { block: true }, summary: { block: true }, reasoning: { block: true }, stats: { block: true }, reference: { block: false }, select: { block: true }, }; describe('extended-markdown', () => { // Flaky. test.skip('tree', async ({ expect }) => { const doc = trim` Hello Hi there! What can I do for you? Summarize tools `; const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] }); const nodes: SyntaxNode[] = []; const tree = syntaxTree(state); tree.iterate({ enter: (node) => { if (node.type.name === 'Element') { nodes.push(node.node); return false; // Stop traversal. } }, }); expect(nodes).toHaveLength(4); expect( nodes.map((node) => ({ content: doc.slice(node.from, node.to), data: nodeToJson(state, node.node), })), ).toEqual([ { content: '\n Hello\n', data: { _tag: 'prompt', children: ['Hello'], }, }, { content: 'Summarize tools', data: { _tag: 'suggestion', children: ['Summarize tools'], }, }, { content: '\n \n \n', data: { _tag: 'choice', children: [ { _tag: 'option', children: ['Summarize tools'], }, { _tag: 'option', children: ['Retry'], }, ], }, }, { content: '', data: { _tag: 'toolkit', }, }, ]); }); test('setext heading disabled', () => { const doc = trim` This should NOT be a heading = Another line that should NOT be a heading - `; const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] }); const tree = syntaxTree(state); tree.iterate({ enter: (node) => { if (node.type.name === 'SetextHeading') { throw new Error('SetextHeading should be disabled!'); } }, }); }); test('self-closing tags with attributes should not consume subsequent content', ({ expect }) => { const doc = trim` ## Architecture Regular paragraph. `; const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] }); const tree = syntaxTree(state); const nodeNames: string[] = []; tree.iterate({ enter: (node) => { nodeNames.push(node.name); }, }); // The heading must be parsed as ATXHeading2, not swallowed into an HTMLBlock. expect(nodeNames).toContain('ATXHeading2'); }); test('self-closing tags without attributes should parse correctly', ({ expect }) => { const doc = trim` ## Heading After Self-Close Paragraph text. `; const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] }); const tree = syntaxTree(state); const nodeNames: string[] = []; tree.iterate({ enter: (node) => { nodeNames.push(node.name); }, }); expect(nodeNames).toContain('ATXHeading2'); }); test('self-closing tags with trailing text should not consume subsequent content', ({ expect }) => { const doc = trim` trailing text ## Heading After Trailing `; const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] }); const tree = syntaxTree(state); const nodeNames: string[] = []; tree.iterate({ enter: (node) => { nodeNames.push(node.name); }, }); // The trailing text line should not be swallowed as an HTMLBlock. expect(nodeNames).toContain('ATXHeading2'); }); test('mixed self-closing and block tags should not break markdown parsing', ({ expect }) => { const doc = trim` What is markdown? The user is asking about markdown. Some explanation text. Analyzed 42 files. ## Architecture The project follows a modular pattern. 3 tool uses ยท 12.4k tokens ### Sub-heading More text. `; const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] }); const tree = syntaxTree(state); const nodeNames: string[] = []; tree.iterate({ enter: (node) => { nodeNames.push(node.name); }, }); expect(nodeNames).toContain('ATXHeading2'); expect(nodeNames).toContain('ATXHeading3'); }); // // TODO(burdon): All tests below should test the tree. // test('should parse standard markdown elements', ({ expect }) => { const doc = trim` # Heading 1 ## Heading 2 ### Heading 3 This is a paragraph with **bold** and *italic* text. \`\`\`javascript const code = 'block'; \`\`\` `; const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] }); const tree = state.sliceDoc(0, state.doc.length); // Verify the document is parsed without errors. expect(tree).toBe(doc); expect(state.doc.lines).toBeGreaterThan(1); }); test('should parse custom XML tags', ({ expect }) => { const doc = trim` # Document with Custom Tags This is a custom prompt block Regular paragraph text. Regular paragraph text. Multi-line prompt content `; const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] }); const tree = state.sliceDoc(0, state.doc.length); // Verify the document contains custom tags. expect(tree).toContain(''); expect(tree).toContain(''); expect(tree).toContain(''); }); test('should handle mixed markdown and XML content', ({ expect }) => { const doc = trim` # Mixed Content This is a paragraph with a inline prompt tag. ## Section 2 This prompt contains **markdown** formatting \`\`\` This should not be parsed as XML inside code block \`\`\` `; const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] }); const tree = state.sliceDoc(0, state.doc.length); // Verify mixed content is preserved. expect(tree).toBe(doc); }); test('should not parse XML tags in code blocks', ({ expect }) => { const doc = trim` \`\`\`xml This is inside a code block \`\`\` \`inline code\` `; const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] }); const tree = state.sliceDoc(0, state.doc.length); // Verify code blocks are preserved as-is. expect(tree).toContain('```xml'); expect(tree).toContain('This is inside a code block'); }); test('should handle nested and complex structures', ({ expect }) => { const doc = trim` # Complex Document First prompt with some content ## Lists and Prompts - Item 1 - Item 2 with embedded prompt - Item 3 Another prompt after the list ### Code Example \`\`\`typescript const example = 'not parsed'; \`\`\` `; const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] }); const tree = state.sliceDoc(0, state.doc.length); // Verify complex structures are handled. expect(tree).toBe(doc); expect(tree).toMatch(/Item 2 with embedded prompt<\/prompt>/); }); test('should handle empty and edge cases', ({ expect }) => { const emptyDoc = ''; const emptyState = EditorState.create({ doc: emptyDoc, extensions: [extendedMarkdown({ registry: testRegistry })], }); expect(emptyState.doc.length).toBe(0); const onlyPromptDoc = 'Only prompt content'; const onlyPromptState = EditorState.create({ doc: onlyPromptDoc, extensions: [extendedMarkdown({ registry: testRegistry })], }); expect(onlyPromptState.sliceDoc(0)).toBe(onlyPromptDoc); const unclosedPromptDoc = 'Unclosed prompt'; const unclosedState = EditorState.create({ doc: unclosedPromptDoc, extensions: [extendedMarkdown({ registry: testRegistry })], }); expect(unclosedState.sliceDoc(0)).toBe(unclosedPromptDoc); }); });