//
// Copyright 2025 DXOS.org
//
import { syntaxTree } from '@codemirror/language';
import { EditorState } from '@codemirror/state';
import { type SyntaxNode } from '@lezer/common';
import { describe, test } from 'vitest';
import { trim } from '@dxos/util';
import { extendedMarkdown } from './extended-markdown.ts';
import { nodeToJson } from './xml-util.ts';
const testRegistry = {
prompt: { block: true },
suggestion: { block: true },
choice: { block: true },
toolkit: { block: true },
toolCall: { block: true },
summary: { block: true },
reasoning: { block: true },
stats: { block: true },
reference: { block: false },
select: { block: true },
};
describe('extended-markdown', () => {
// Flaky.
test.skip('tree', async ({ expect }) => {
const doc = trim`
Hello
Hi there!
What can I do for you?
Summarize tools
`;
const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] });
const nodes: SyntaxNode[] = [];
const tree = syntaxTree(state);
tree.iterate({
enter: (node) => {
if (node.type.name === 'Element') {
nodes.push(node.node);
return false; // Stop traversal.
}
},
});
expect(nodes).toHaveLength(4);
expect(
nodes.map((node) => ({
content: doc.slice(node.from, node.to),
data: nodeToJson(state, node.node),
})),
).toEqual([
{
content: '\n Hello\n',
data: {
_tag: 'prompt',
children: ['Hello'],
},
},
{
content: 'Summarize tools',
data: {
_tag: 'suggestion',
children: ['Summarize tools'],
},
},
{
content: '\n \n \n',
data: {
_tag: 'choice',
children: [
{
_tag: 'option',
children: ['Summarize tools'],
},
{
_tag: 'option',
children: ['Retry'],
},
],
},
},
{
content: '',
data: {
_tag: 'toolkit',
},
},
]);
});
test('setext heading disabled', () => {
const doc = trim`
This should NOT be a heading
=
Another line that should NOT be a heading
-
`;
const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] });
const tree = syntaxTree(state);
tree.iterate({
enter: (node) => {
if (node.type.name === 'SetextHeading') {
throw new Error('SetextHeading should be disabled!');
}
},
});
});
test('self-closing tags with attributes should not consume subsequent content', ({ expect }) => {
const doc = trim`
## Architecture
Regular paragraph.
`;
const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] });
const tree = syntaxTree(state);
const nodeNames: string[] = [];
tree.iterate({
enter: (node) => {
nodeNames.push(node.name);
},
});
// The heading must be parsed as ATXHeading2, not swallowed into an HTMLBlock.
expect(nodeNames).toContain('ATXHeading2');
});
test('self-closing tags without attributes should parse correctly', ({ expect }) => {
const doc = trim`
## Heading After Self-Close
Paragraph text.
`;
const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] });
const tree = syntaxTree(state);
const nodeNames: string[] = [];
tree.iterate({
enter: (node) => {
nodeNames.push(node.name);
},
});
expect(nodeNames).toContain('ATXHeading2');
});
test('self-closing tags with trailing text should not consume subsequent content', ({ expect }) => {
const doc = trim`
trailing text
## Heading After Trailing
`;
const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] });
const tree = syntaxTree(state);
const nodeNames: string[] = [];
tree.iterate({
enter: (node) => {
nodeNames.push(node.name);
},
});
// The trailing text line should not be swallowed as an HTMLBlock.
expect(nodeNames).toContain('ATXHeading2');
});
test('mixed self-closing and block tags should not break markdown parsing', ({ expect }) => {
const doc = trim`
What is markdown?
The user is asking about markdown.
Some explanation text.
Analyzed 42 files.
## Architecture
The project follows a modular pattern.
3 tool uses ยท 12.4k tokens
### Sub-heading
More text.
`;
const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] });
const tree = syntaxTree(state);
const nodeNames: string[] = [];
tree.iterate({
enter: (node) => {
nodeNames.push(node.name);
},
});
expect(nodeNames).toContain('ATXHeading2');
expect(nodeNames).toContain('ATXHeading3');
});
//
// TODO(burdon): All tests below should test the tree.
//
test('should parse standard markdown elements', ({ expect }) => {
const doc = trim`
# Heading 1
## Heading 2
### Heading 3
This is a paragraph with **bold** and *italic* text.
\`\`\`javascript
const code = 'block';
\`\`\`
`;
const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] });
const tree = state.sliceDoc(0, state.doc.length);
// Verify the document is parsed without errors.
expect(tree).toBe(doc);
expect(state.doc.lines).toBeGreaterThan(1);
});
test('should parse custom XML tags', ({ expect }) => {
const doc = trim`
# Document with Custom Tags
This is a custom prompt block
Regular paragraph text.
Regular paragraph text.
Multi-line
prompt content
`;
const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] });
const tree = state.sliceDoc(0, state.doc.length);
// Verify the document contains custom tags.
expect(tree).toContain('');
expect(tree).toContain('');
expect(tree).toContain('');
});
test('should handle mixed markdown and XML content', ({ expect }) => {
const doc = trim`
# Mixed Content
This is a paragraph with a inline prompt tag.
## Section 2
This prompt contains **markdown** formatting
\`\`\`
This should not be parsed as XML inside code block
\`\`\`
`;
const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] });
const tree = state.sliceDoc(0, state.doc.length);
// Verify mixed content is preserved.
expect(tree).toBe(doc);
});
test('should not parse XML tags in code blocks', ({ expect }) => {
const doc = trim`
\`\`\`xml
This is inside a code block
\`\`\`
\`inline code\`
`;
const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] });
const tree = state.sliceDoc(0, state.doc.length);
// Verify code blocks are preserved as-is.
expect(tree).toContain('```xml');
expect(tree).toContain('This is inside a code block');
});
test('should handle nested and complex structures', ({ expect }) => {
const doc = trim`
# Complex Document
First prompt with some content
## Lists and Prompts
- Item 1
- Item 2 with embedded prompt
- Item 3
Another prompt after the list
### Code Example
\`\`\`typescript
const example = 'not parsed';
\`\`\`
`;
const state = EditorState.create({ doc, extensions: [extendedMarkdown({ registry: testRegistry })] });
const tree = state.sliceDoc(0, state.doc.length);
// Verify complex structures are handled.
expect(tree).toBe(doc);
expect(tree).toMatch(/Item 2 with embedded prompt<\/prompt>/);
});
test('should handle empty and edge cases', ({ expect }) => {
const emptyDoc = '';
const emptyState = EditorState.create({
doc: emptyDoc,
extensions: [extendedMarkdown({ registry: testRegistry })],
});
expect(emptyState.doc.length).toBe(0);
const onlyPromptDoc = 'Only prompt content';
const onlyPromptState = EditorState.create({
doc: onlyPromptDoc,
extensions: [extendedMarkdown({ registry: testRegistry })],
});
expect(onlyPromptState.sliceDoc(0)).toBe(onlyPromptDoc);
const unclosedPromptDoc = 'Unclosed prompt';
const unclosedState = EditorState.create({
doc: unclosedPromptDoc,
extensions: [extendedMarkdown({ registry: testRegistry })],
});
expect(unclosedState.sliceDoc(0)).toBe(unclosedPromptDoc);
});
});