import { A_Scope } from '@adaas/a-concept'; import { AreNode } from '@adaas/are/node/AreNode.entity'; import { AreSyntax } from '@adaas/are/syntax/AreSyntax.context'; import { AreSyntaxInitOptions, AreSyntaxTokenMatch } from '@adaas/are/syntax/AreSyntax.types'; import { AreTokenizer } from '@adaas/are/tokenizer/AreTokenizer.component'; import { AreContext } from '@adaas/are/component/Are.context'; jest.retryTimes(0); function htmlElementMatcher( source: string, from: number, to: number, build: (raw: string, content: string, position: number, closing: string) => AreSyntaxTokenMatch ): AreSyntaxTokenMatch | null { let index = from while (index < to) { const tagStart = source.indexOf('<', index) if (tagStart === -1 || tagStart >= to) return null // skip comments, closing tags, doctype, processing instructions if (source.startsWith(' — priority 10, not nested { opening: '', component: AreNode, priority: 10, nested: false, extract: (raw) => ({ content: raw.slice(4, -3).trim() }), }, // interpolations {{ expr }} — priority 9, not nested { opening: '{{', closing: '}}', component: InterpolationNode, priority: 9, nested: false, extract: (_, match) => ({ key: match.content }), }, // HTML elements — custom matcher handles tag-aware open/close detection { matcher: htmlElementMatcher, component: AreNode, priority: 4, }, // plain text fallback { component: TextNode, priority: 0, }, ], } describe('AreTokenizer Tests', () => { it('Should tokenize AreNode Correctly', async () => { const node = new AreNode({ /** Full matched string including delimiters */ raw: ``, /** Content between delimiters */ content: '
Hello Test
', /** The opening delimiter that matched */ opening: '<', /** The closing delimiter that matched */ closing: '>', /** Start position in source string */ position: 0, /** Data extracted via rule.extract */ payload: { entity: 'custom' } }); const testCope = new A_Scope({ name: 'test', entities: [node], components: [AreTokenizer], fragments: [new AreSyntax(HTML_CONFIG), new AreContext(' ')], }); node.tokenize(); expect(node.children.length).toBe(1); const divNode = node.children[0]; expect(divNode.content).toBe(' Hello Test'); divNode.tokenize(); expect(divNode.children.length).toBe(2); const spanNode = divNode.children[0]; expect(spanNode.content).toBe('Hello'); const textNode = divNode.children[1]; expect(textNode.content).toBe('Test'); }); it('Should return no children on empty content', async () => { const node = new AreNode({ /** Full matched string including delimiters */ raw: ``, /** Content between delimiters */ content: '', /** The opening delimiter that matched */ opening: '<', /** The closing delimiter that matched */ closing: '>', /** Start position in source string */ position: 0, /** Data extracted via rule.extract */ payload: { entity: 'custom' } }); const testCope = new A_Scope({ name: 'test', entities: [node], components: [AreTokenizer], fragments: [new AreSyntax(HTML_CONFIG), new AreContext(' ')], }); node.tokenize(); expect(node.children.length).toBe(0); }); it('Should return no children on empty content', async () => { const node = new AreNode({ /** Full matched string including delimiters */ raw: ``, /** Content between delimiters */ content: '', /** The opening delimiter that matched */ opening: '<', /** The closing delimiter that matched */ closing: '>', /** Start position in source string */ position: 0, /** Data extracted via rule.extract */ payload: { entity: 'custom' } }); const testCope = new A_Scope({ name: 'test', entities: [node], components: [AreTokenizer], fragments: [new AreSyntax(HTML_CONFIG), new AreContext(' ')], }); node.tokenize(); expect(node.children.length).toBe(0); }); it('Should return nested nodes based on content', async () => { const node = new AreNode({ /** Full matched string including delimiters */ raw: ``, /** Content between delimiters */ content: 'Do: {{btn2}} Make
Hello Test
', /** The opening delimiter that matched */ opening: '<', /** The closing delimiter that matched */ closing: '>', /** Start position in source string */ position: 0, /** Data extracted via rule.extract */ payload: { entity: 'custom' } }); const testCope = new A_Scope({ name: 'test', entities: [node], components: [AreTokenizer], fragments: [new AreSyntax(HTML_CONFIG), new AreContext(' ')], }); node.tokenize(); expect(node.children.length).toBe(4); expect(node.children[0]).toBeInstanceOf(TextNode); expect(node.children[1]).toBeInstanceOf(InterpolationNode); }); });