import { TokenKind, Token } from './Lexer'; /** * Parsing context types that the TokenStream can track */ export declare const ParsingContext: { readonly NONE: "none"; readonly ARRAY_LITERAL: "array_literal"; readonly OBJECT_LITERAL: "object_literal"; readonly FUNCTION_CALL: "function_call"; readonly SUBEXPRESSION: "subexpression"; readonly BLOCK: "block"; readonly FUNCTION_DEFINITION: "function_definition"; readonly STRING_LITERAL: "string_literal"; }; export type ParsingContext = typeof ParsingContext[keyof typeof ParsingContext]; export declare class TokenStream { private tokens; private position; private contextStack; private lastNextPosition; private consecutiveNextCalls; private positionHistory; private operationsSinceLastAdvance; private lastPositionValue; /** * Maximum number of consecutive next() calls allowed at the same position before throwing */ static readonly MAX_CONSECUTIVE_NEXT_AT_SAME_POSITION = 3; /** * Maximum number of operations without position advancement before detecting stuck * Only counts operations that should advance (next, match, expect, etc.) */ static readonly MAX_OPERATIONS_WITHOUT_ADVANCE = 100; /** * Debug mode flag - set to true to enable logging * Can be controlled via VITE_DEBUG environment variable or set programmatically * Usage: TokenStream.debug = true; */ static debug: boolean; /** * Create a new TokenStream * @param tokens - Array of tokens to stream * @param startIndex - Optional starting index (default 0) */ constructor(tokens: Token[], startIndex?: number); /** * Create a new TokenStream starting from the given index * Useful for sub-parsing operations that need to start from a specific position * @param index - Starting index in the token array * @returns New TokenStream instance starting at the given index */ cloneFrom(index: number): TokenStream; /** * Reset stuck detection counters (useful when manually advancing position) */ resetStuckDetection(): void; /** * Set the current position directly * @param position - New position */ setPosition(position: number): void; /** * Get the current token without consuming it * @returns The current token, or null if at end */ current(): Token | null; /** * Look ahead at a token without consuming it * @param offset - Number of tokens to look ahead (default 0 = current token) * @returns The token at the given offset, or null if beyond end */ peek(offset?: number): Token | null; /** * Consume and return the current token * @returns The current token, or null if at end * @throws Error if called multiple times at the same position (indicates infinite loop) */ next(): Token | null; /** * Check if the stream appears to be stuck (no position advancement) * @param operation - Name of the operation being performed * @throws Error if stuck condition detected */ private checkStuckCondition; /** * Check if we're at the end of the token stream * @returns True if at EOF or beyond */ isAtEnd(): boolean; /** * Check if the current token matches the given kind or text, without consuming it * @param kindOrText - TokenKind enum or string text to match * @returns True if the current token matches */ check(kindOrText: TokenKind | string): boolean; /** * If the current token matches, consume it and return true * Otherwise, return false without consuming * * @param kindOrText - TokenKind enum or string text to match * @returns True if matched and consumed, false otherwise */ match(kindOrText: TokenKind | string): boolean; /** * Expect the current token to match, consume it, and return it * If it doesn't match, throw an error * * @param kindOrText - TokenKind enum or string text to expect * @param message - Optional custom error message * @returns The consumed token * @throws Error if token doesn't match */ expect(kindOrText: TokenKind | string, message?: string): Token; /** * Save the current position for potential backtracking * @returns The current position index */ save(): number; /** * Restore a previously saved position (backtrack) * @param pos - The position to restore to */ restore(pos: number): void; /** * Skip tokens of a specific kind (useful for skipping newlines, comments, etc.) * @param kind - The TokenKind to skip * @returns Number of tokens skipped */ skip(kind: TokenKind): number; /** * Skip all newline tokens * @returns Number of newlines skipped */ skipNewlines(): number; /** * Consume tokens until a specific kind or text is found (exclusive) * @param kindOrText - The kind or text to stop at * @returns Array of consumed tokens (not including the stop token) */ consumeUntil(kindOrText: TokenKind | string): Token[]; /** * Collect all tokens on the current line (until NEWLINE or EOF) * @returns Array of tokens on the current line */ collectLine(): Token[]; /** * Get the current position in the stream * @returns Current position index */ getPosition(): number; /** * Get the total number of tokens in the stream * @returns Total token count */ getLength(): number; /** * Get all remaining tokens without consuming them * @returns Array of remaining tokens */ remaining(): Token[]; /** * Create a sub-stream from a range of tokens * Useful for parsing sub-expressions or blocks * * @param start - Start position (inclusive) * @param end - End position (exclusive), defaults to current position * @returns New TokenStream instance */ slice(start: number, end?: number): TokenStream; /** * Format current position for error messages * @returns String like "line 10, column 5" */ formatPosition(): string; /** * Push a parsing context onto the context stack * @param context - The parsing context to enter */ pushContext(context: ParsingContext): void; /** * Pop the most recent parsing context from the stack * @returns The context that was removed, or NONE if stack was empty */ popContext(): ParsingContext; /** * Get the current parsing context (top of the stack) * @returns The current parsing context, or NONE if no context is active */ getCurrentContext(): ParsingContext; /** * Check if we're currently in a specific parsing context * @param context - The context to check for * @returns True if we're in the given context (at any level) */ isInContext(context: ParsingContext): boolean; /** * Get all active contexts (the entire stack) * @returns Array of contexts from bottom to top */ getContextStack(): ParsingContext[]; /** * Clear all parsing contexts */ clearContexts(): void; /** * Create a new TokenStream with the same tokens and contexts * Useful for sub-parsing that should inherit context * @param startIndex - Optional starting index (defaults to current position) * @returns New TokenStream instance with copied context */ cloneWithContext(startIndex?: number): TokenStream; /** * Skip whitespace (newlines) and comments * Advances the stream past any consecutive newline or comment tokens */ skipWhitespaceAndComments(): void; }