import { tokenize, nonWhitespace } from '../parser/lexer'; import { TokenKind } from '../parser/types'; function kinds(input: string): TokenKind[] { return nonWhitespace(tokenize(input)) .filter((t) => t.kind !== TokenKind.EOF) .map((t) => t.kind); } function values(input: string): string[] { return nonWhitespace(tokenize(input)) .filter((t) => t.kind !== TokenKind.EOF) .map((t) => t.value); } describe('Loki lexer', () => { // ── Basic tokens ── it('tokenizes stream selector braces', () => { expect(kinds('{}')).toEqual([TokenKind.LCurly, TokenKind.RCurly]); }); it('tokenizes pipe', () => { expect(kinds('|')).toEqual([TokenKind.Pipe]); }); it('tokenizes parentheses', () => { expect(kinds('()')).toEqual([TokenKind.LParen, TokenKind.RParen]); }); it('tokenizes brackets', () => { expect(kinds('[]')).toEqual([TokenKind.LBracket, TokenKind.RBracket]); }); it('tokenizes comma', () => { expect(kinds(',')).toEqual([TokenKind.Comma]); }); // ── Operators ── it('tokenizes = and != and =~ and !~', () => { expect(kinds('= != =~ !~')).toEqual([TokenKind.Equals, TokenKind.NotEquals, TokenKind.RegexEquals, TokenKind.RegexNotEquals]); }); it('tokenizes comparison operators', () => { expect(kinds('> >= < <=')).toEqual([TokenKind.GreaterThan, TokenKind.GreaterEquals, TokenKind.LessThan, TokenKind.LessEquals]); }); // ── Pipe operators ── it('tokenizes line filter operators', () => { expect(kinds('|= |~')).toEqual([TokenKind.PipeEquals, TokenKind.PipeRegex]); }); it('tokenizes bare pipe after != / !~', () => { // | != and | !~ are two tokens each const t = nonWhitespace(tokenize('| != "err"')).filter((x) => x.kind !== TokenKind.EOF); expect(t.map((x) => x.kind)).toEqual([TokenKind.Pipe, TokenKind.NotEquals, TokenKind.String]); }); // ── Strings ── it('tokenizes double-quoted string', () => { const t = nonWhitespace(tokenize('"hello"')); expect(t[0].kind).toBe(TokenKind.String); expect(t[0].value).toBe('"hello"'); }); it('tokenizes single-quoted string', () => { const t = nonWhitespace(tokenize("'hello'")); expect(t[0].kind).toBe(TokenKind.String); expect(t[0].value).toBe("'hello'"); }); it('tokenizes backtick string', () => { const t = nonWhitespace(tokenize('`hello`')); expect(t[0].kind).toBe(TokenKind.String); expect(t[0].value).toBe('`hello`'); }); it('handles unclosed string', () => { const t = nonWhitespace(tokenize('"unclosed')); expect(t[0].kind).toBe(TokenKind.String); expect(t[0].value).toBe('"unclosed'); }); // ── Identifiers ── it('tokenizes identifiers', () => { expect(kinds('job')).toEqual([TokenKind.Ident]); expect(values('status')).toEqual(['status']); }); // ── Duration ── it('tokenizes duration values', () => { expect(kinds('5m')).toEqual([TokenKind.Duration]); expect(values('5m')).toEqual(['5m']); expect(values('1h30m')).toEqual(['1h', '30m']); expect(values('300ms')).toEqual(['300ms']); }); // ── Bytes ── it('tokenizes bytes values', () => { expect(kinds('42MB')).toEqual([TokenKind.Bytes]); expect(values('42MB')).toEqual(['42MB']); expect(values('1.5KiB')).toEqual(['1.5KiB']); }); // ── Numbers ── it('tokenizes numbers', () => { expect(kinds('123')).toEqual([TokenKind.Number]); expect(values('45.6')).toEqual(['45.6']); }); // ── Comments ── it('tokenizes # comments', () => { const t = nonWhitespace(tokenize('# this is a comment\n{job="nginx"}')).filter((x) => x.kind !== TokenKind.EOF); // Comment is filtered out by nonWhitespace expect(t.map((x) => x.kind)).toEqual([TokenKind.LCurly, TokenKind.Ident, TokenKind.Equals, TokenKind.String, TokenKind.RCurly]); }); // ── Full LogQL query ── it('tokenizes a full log query', () => { const t = nonWhitespace(tokenize('{job="nginx"} |= "error" | json | status >= 400')).filter((x) => x.kind !== TokenKind.EOF); expect(t.map((x) => x.kind)).toEqual([ TokenKind.LCurly, TokenKind.Ident, TokenKind.Equals, TokenKind.String, TokenKind.RCurly, TokenKind.PipeEquals, TokenKind.String, TokenKind.Pipe, TokenKind.Ident, TokenKind.Pipe, TokenKind.Ident, TokenKind.GreaterEquals, TokenKind.Number, ]); }); it('tokenizes a metric query', () => { const t = nonWhitespace(tokenize('rate({job="nginx"}[5m])')); const kindsArr = t.map((x) => x.kind); expect(kindsArr).toContain(TokenKind.Ident); // rate expect(kindsArr).toContain(TokenKind.LParen); expect(kindsArr).toContain(TokenKind.LCurly); expect(kindsArr).toContain(TokenKind.LBracket); expect(kindsArr).toContain(TokenKind.Duration); expect(kindsArr).toContain(TokenKind.RBracket); expect(kindsArr).toContain(TokenKind.RParen); }); it('tokenizes multiple matchers', () => { const t = nonWhitespace(tokenize('{job="nginx", app="api"}')).filter((x) => x.kind !== TokenKind.EOF); expect(t.map((x) => x.kind)).toEqual([ TokenKind.LCurly, TokenKind.Ident, TokenKind.Equals, TokenKind.String, TokenKind.Comma, TokenKind.Ident, TokenKind.Equals, TokenKind.String, TokenKind.RCurly, ]); }); });