import { TokenKind } from './types'; import type { Token, LokiQuery, StreamSelector, Matcher, PipelineStage, LineFilter, MetricExpr, Grouping, ParseError } from './types'; import { tokenize, nonWhitespace } from './lexer'; // ─── Parser class ───────────────────────────────────────────────── class Parser { private tokens: Token[]; private pos = 0; errors: ParseError[] = []; constructor(tokens: Token[]) { this.tokens = tokens; } private peek(offset = 0): Token { const idx = this.pos + offset; return idx < this.tokens.length ? this.tokens[idx] : this.tokens[this.tokens.length - 1]; } private advance(): Token { if (this.pos >= this.tokens.length) { // Past the end — return the last token (EOF sentinel) to avoid undefined access return this.tokens[this.tokens.length - 1]; } return this.tokens[this.pos++]; } private expect(kind: TokenKind): Token { const token = this.advance(); if (token.kind !== kind) { this.errors.push({ message: `Expected ${TokenKind[kind]} but got ${token.kind !== TokenKind.EOF ? token.value : 'EOF'}`, start: token.start, end: token.end, }); } return token; } private expectAny(kinds: TokenKind[]): Token { const token = this.advance(); if (!kinds.includes(token.kind)) { this.errors.push({ message: `Expected one of [${kinds.map((k) => TokenKind[k]).join(', ')}] but got ${token.value || 'EOF'}`, start: token.start, end: token.end, }); } return token; } private match(kind: TokenKind): boolean { if (this.peek().kind === kind) { this.advance(); return true; } return false; } // ── Top level ── parse(): LokiQuery { this.pos = 0; this.errors = []; let streamSelector: StreamSelector | null = null; if (this.peek().kind === TokenKind.LCurly) { streamSelector = this.parseStreamSelector(); } const pipeline: PipelineStage[] = []; while (this.isPipeStart()) { const stage = this.parsePipelineStage(); if (stage) pipeline.push(stage); } let metricExpr: MetricExpr | null = null; if (this.peek().kind === TokenKind.Ident && this.peek(1).kind === TokenKind.LParen) { metricExpr = this.parseMetricExpr(); } return { streamSelector, pipeline, metricExpr, errors: this.errors, }; } // ── Stream selector: { label op "value", ... } ── private parseStreamSelector(): StreamSelector { const matchers: Matcher[] = []; this.expect(TokenKind.LCurly); while (this.peek().kind !== TokenKind.RCurly && this.peek().kind !== TokenKind.EOF) { // Skip comma if (this.match(TokenKind.Comma)) continue; // Skip whitespace/comments (should already be filtered out) const labelTok = this.expectAny([TokenKind.Ident, TokenKind.String]); if (labelTok.kind === TokenKind.EOF) break; const opTok = this.expectAny([TokenKind.Equals, TokenKind.NotEquals, TokenKind.RegexEquals, TokenKind.RegexNotEquals]); if (opTok.kind === TokenKind.EOF) break; const valTok = this.expectAny([TokenKind.String, TokenKind.Ident, TokenKind.Number]); if (valTok.kind === TokenKind.EOF) break; matchers.push({ label: labelTok.value, operator: opTok.value, value: valTok.value, }); } this.expect(TokenKind.RCurly); return { matchers }; } // ── Pipeline ── private isPipeStart(): boolean { const k = this.peek().kind; return k === TokenKind.Pipe || k === TokenKind.PipeEquals || k === TokenKind.PipeRegex; } private parsePipelineStage(): PipelineStage | null { const tok = this.peek(); const kind = tok.kind; if (kind === TokenKind.PipeEquals) { this.advance(); const valTok = this.peek().kind === TokenKind.String ? this.advance() : null; return { type: 'lineFilter', operator: '|=', value: valTok?.value ?? '', } as LineFilter; } if (kind === TokenKind.PipeRegex) { this.advance(); const valTok = this.peek().kind === TokenKind.String ? this.advance() : null; return { type: 'lineFilter', operator: '|~', value: valTok?.value ?? '', } as LineFilter; } if (kind === TokenKind.Pipe) { this.advance(); // consume | const next = this.peek(); // Line filter: != or !~ if (next.kind === TokenKind.NotEquals) { this.advance(); const valTok = this.peek().kind === TokenKind.String ? this.advance() : null; return { type: 'lineFilter', operator: '!=', value: valTok?.value ?? '' } as LineFilter; } if (next.kind === TokenKind.RegexNotEquals) { this.advance(); const valTok = this.peek().kind === TokenKind.String ? this.advance() : null; return { type: 'lineFilter', operator: '!~', value: valTok?.value ?? '' } as LineFilter; } // Keyword-based stages if (next.kind === TokenKind.Ident) { const keyword = next.value; if (keyword === 'json') { this.advance(); return this.parseJsonParser(); } if (keyword === 'logfmt') { this.advance(); return this.parseLogfmtParser(); } if (keyword === 'regexp') { this.advance(); return this.parseRegexpParser(); } if (keyword === 'pattern') { this.advance(); return this.parseRegexpParser(); } if (keyword === 'unpack') { this.advance(); return { type: 'jsonParser' } as PipelineStage; } if (keyword === 'decolorize') { this.advance(); return { type: 'unrecognized', keyword: 'decolorize', rest: '' } as PipelineStage; } if (keyword === 'line_format' || keyword === 'label_format' || keyword === 'drop' || keyword === 'keep' || keyword === 'unwrap') { this.advance(); return { type: 'unrecognized', keyword, rest: '' } as PipelineStage; } // Not a pipeline keyword → treat as start of a label filter: | label op value return this.parseLabelFilter(); } return null; } return null; } // ── Parsers ── private parseJsonParser(): PipelineStage { const expressions: string[] = []; // Optional: | json field1, field2="expr" while (this.peek().kind === TokenKind.Ident || this.peek().kind === TokenKind.String) { const tok = this.advance(); expressions.push(tok.value); this.match(TokenKind.Comma); // optional comma } return { type: 'jsonParser', expressions: expressions.length > 0 ? expressions : undefined }; } private parseLogfmtParser(): PipelineStage { const expressions: string[] = []; while (this.peek().kind === TokenKind.Ident || this.peek().kind === TokenKind.String) { const tok = this.advance(); expressions.push(tok.value); this.match(TokenKind.Comma); } return { type: 'logfmtParser', expressions: expressions.length > 0 ? expressions : undefined }; } private parseRegexpParser(): PipelineStage { const patternTok = this.peek().kind === TokenKind.String ? this.advance() : null; return { type: 'regexpParser', pattern: patternTok?.value ?? '' }; } // ── Label filter: label op value [and/or label op value] ── private parseLabelFilter(): PipelineStage { const conditions: import('./types').LabelFilterCondition[] = []; while (this.peek().kind !== TokenKind.EOF && !this.isPipeStart() && this.peek().kind !== TokenKind.RParen) { const labelTok = this.expectAny([TokenKind.Ident, TokenKind.String]); if (labelTok.kind === TokenKind.EOF) break; const opTok = this.expectAny([ TokenKind.Equals, TokenKind.NotEquals, TokenKind.RegexEquals, TokenKind.RegexNotEquals, TokenKind.GreaterThan, TokenKind.GreaterEquals, TokenKind.LessThan, TokenKind.LessEquals, ]); if (opTok.kind === TokenKind.EOF) break; const valTok = this.expectAny([TokenKind.String, TokenKind.Number, TokenKind.Duration, TokenKind.Bytes, TokenKind.Ident]); if (valTok.kind === TokenKind.EOF) break; conditions.push({ label: labelTok.value, operator: opTok.value, value: valTok.value, }); // Optional and/or if (this.peek().kind === TokenKind.Ident && (this.peek().value === 'and' || this.peek().value === 'or')) { this.advance(); } else { break; } } return { type: 'labelFilter', conditions }; } // ── Metric expression: func([5m]) ── private parseMetricExpr(): MetricExpr { const nameTok = this.advance(); // function name this.expect(TokenKind.LParen); // Could be nested query or range let range = ''; if (this.peek().kind === TokenKind.LCurly) { // Nested query: sum(rate({...}[5m])) // Simple parse: skip until '[' or ')' while (this.peek().kind !== TokenKind.EOF && this.peek().kind !== TokenKind.RParen) { if (this.peek().kind === TokenKind.LBracket) break; this.advance(); } } // Parse range: [5m] if (this.peek().kind === TokenKind.LBracket) { this.advance(); // consume [ const rangeStart = this.pos; while (this.peek().kind !== TokenKind.RBracket && this.peek().kind !== TokenKind.EOF) { this.advance(); } range = this.peek().kind === TokenKind.RBracket ? this.tokens .slice(rangeStart, this.pos) .map((t) => t.value) .join('') : ''; this.expect(TokenKind.RBracket); } this.expect(TokenKind.RParen); // Parse optional grouping: by(...) / without(...) let grouping: Grouping | undefined; if (this.peek().kind === TokenKind.Ident && (this.peek().value === 'by' || this.peek().value === 'without')) { grouping = this.parseGrouping(); } return { func: nameTok.value, range, grouping }; } private parseGrouping(): Grouping { const by = this.peek().value === 'by'; this.advance(); // consume by/without this.expect(TokenKind.LParen); const labels: string[] = []; while (this.peek().kind !== TokenKind.RParen && this.peek().kind !== TokenKind.EOF) { if (this.match(TokenKind.Comma)) continue; const tok = this.expectAny([TokenKind.Ident, TokenKind.String]); if (tok.kind !== TokenKind.EOF) labels.push(tok.value); } this.expect(TokenKind.RParen); return { by, labels }; } } // ─── Public API ─────────────────────────────────────────────────── export function parse(input: string): LokiQuery { const rawTokens = tokenize(input); const tokens = nonWhitespace(rawTokens); const parser = new Parser(tokens); return parser.parse(); }