/** * Trivia attachment algorithm for CST construction. * * Takes the full token stream (with whitespace and comments preserved) and * wraps each "real" token in a CstToken that carries its leading and trailing * trivia. Uses the split convention: * * - Same-line trivia (up to and including the first newline) is trailing * trivia of the previous real token. * - Subsequent-line trivia is leading trivia of the next real token. * * This matches the convention used by Roslyn, swift-syntax, and rust-analyzer. */ import type { Token } from '../tokenizer/token'; import type { CstToken, TriviaNode } from './types'; export declare function isTrivia(token: Token): boolean; /** * Get the raw source text for a token. Most tokens store raw text in * token[1], but a few strip their prefix during tokenization: * - Atom: value "ok" → raw ":ok" * - EffectName: value "my.eff" → raw "@my.eff" * - MacroPrefix: value "name" → raw "#name" */ export declare function rawTokenText(token: Token): string; export declare function toTriviaNode(token: Token): TriviaNode; /** * Split a trivia sequence at the first newline boundary. * * Everything up to and including the first `\n` character goes into * `trailing`. Everything after goes into `leading`. If a single trivia * node spans the boundary (e.g. whitespace " \n "), it is split into * two nodes at the newline. * * If no newline exists in the trivia, everything is trailing (same-line). */ export declare function splitTriviaAtNewline(trivia: TriviaNode[]): { trailing: TriviaNode[]; leading: TriviaNode[]; }; export interface AttachTriviaResult { /** Real tokens wrapped with their leading/trailing trivia. */ tokens: CstToken[]; /** Trivia after the last real token that belongs to subsequent lines * (file-level trailing comments, final newlines). Stored on CstProgram. */ trailingTrivia: TriviaNode[]; } /** * Attach trivia (whitespace, comments) to real tokens using the split * convention. * * @param fullTokens Token stream from `tokenize(source, true, ...)`. * Must include whitespace and comment tokens (debug mode). */ export declare function attachTrivia(fullTokens: Token[]): AttachTriviaResult; /** Concatenate a single CstToken back to its source representation. */ export declare function cstTokenText(token: CstToken): string; /** * Reconstruct the full source from CstTokens + trailing trivia. * Used for losslessness verification: `printTokens(attachTrivia(tokenize(src))) === src`. */ export declare function printTokens(result: AttachTriviaResult): string;