/** * Derived tokenization groundwork: primitive kernels and lexical-token families. * * `collectAlphabet()` is the preserved primitive-terminal scanner groundwork: * it knows every `literal`, `keywords`, and `regex` in the composed grammar and * assigns each one a small integer KERNEL id. Those ids are implementation * details, not parser tokens. In particular, unwrapping * `token(sequence(identifier, optional(literal('('))))` must never publish an * identifier id followed by a `(` id: `token()` defines ONE contiguous source * token and ONE CST leaf for the complete range. * * `collectLexicalAlphabet()` is the production-facing boundary. It assigns ids * to authored `token()` families, normalizes effect-free bodies into canonical * lexical IR, interns equivalent recognizers, and records the downstream * `dispatch()` outcome classifiers over the same range. Primitive terminals * stay private recognizer machinery. * * Two things are deliberately separated, because conflating them is what makes * a naive derived scanner useless: * * - The token ID SPACE is GLOBAL. One id per distinct terminal across the whole * grammar, so an id can be compared, switched on, and indexed with. * - The CANDIDATE SET is LOCAL, per decision point. Global maximal munch over * the whole alphabet is wrong: a css alphabet contains construct-local * long-run terminals (`[^()]+`, `[^{}/]+`, the `scanTo` raw-prelude runs) * that swallow the document. Measured: global munch yields SEVEN tokens for a * 123 KB stylesheet. A decision point consults only the terminals that can * actually start one of its arms. * * Nothing here decides HOW a token is recognized (see `token-scanner.ts`); this * module only decides WHAT the tokens are and WHICH ones each site may see. */ import type { AutoNotCheck, Combinator, FirstSet, ParserDef } from '../types.ts'; /** A terminal in the derived alphabet, with its globally assigned id. */ export type TokenTerminal = { kind: 'literal'; id: number; value: string; caseInsensitive: boolean; } | { kind: 'keywords'; id: number; words: readonly string[]; caseInsensitive: boolean; boundary: string | undefined; } | { kind: 'regex'; id: number; source: string; flags: string; }; /** Reserved ids. Real terminals start at `FIRST_TERMINAL_ID`. */ export declare const TOK_EOF = 0; export declare const TOK_UNKNOWN = 1; export declare const TOK_WS = 2; export declare const FIRST_TERMINAL_ID = 3; export type Alphabet = { terminals: TokenTerminal[]; /** Dedup key → id, so the same terminal spelled twice gets ONE id. */ byKey: Map; /** The combinator that contributed each id (first one wins), for candidate sets. */ originOf: Map>; }; /** Effect-free recognition IR underneath one authored `token()` boundary. */ export type LexicalIr = { readonly kind: 'literal'; readonly value: string; readonly caseInsensitive: boolean; } | { readonly kind: 'keywords'; readonly words: readonly string[]; readonly caseInsensitive: boolean; readonly boundary: string | undefined; } | { readonly kind: 'regex'; readonly source: string; readonly flags: string; } | { readonly kind: 'sequence'; readonly parts: readonly LexicalIr[]; } | { readonly kind: 'choice'; readonly arms: readonly LexicalIr[]; } | { readonly kind: 'repeat'; readonly body: LexicalIr; readonly min: number; readonly max: number | null; readonly greedy: boolean; readonly mode: 'possessive' | 'backtracking'; } | { readonly kind: 'assert'; readonly positive: boolean; readonly body: LexicalIr; } | { readonly kind: 'balanced'; readonly open: string; readonly close: string; readonly strict: boolean; readonly raw: boolean; /** Ordered explicit skippers; ambient trivia/scanSkip precede them unless raw. */ readonly skip: readonly LexicalIr[]; }; /** First complete executable lexical body. It is intentionally a semantic * shape rather than an opcode: the encoder relocates the regex into the table * pool only after TOKEN wins the compiler-only cost comparison. */ export type OptionalSuffixLexBody = { readonly kind: 'regex-optional-code-unit'; readonly source: string; readonly flags: string; readonly suffix: string; }; export type ExecutableOptionalSuffixLexBody = OptionalSuffixLexBody & { /** Match the two authored terminal line-publication points exactly. */ readonly baseCanMatchNewline: boolean; readonly suffixCanMatchNewline: boolean; /** Failure authority belongs to the resolved final regex, not the lazy * construction thunk that originally named it. */ readonly expected: readonly string[]; }; /** A direct authored terminal under token(). The boundary shell still owns * source slicing/CST; this body owns exactly the terminal recognizer and lines. */ export type ExecutableTerminalLexBody = { readonly kind: 'regex-terminal'; readonly source: string; readonly flags: string; readonly canMatchNewline: boolean; readonly expected: readonly string[]; }; export type ExecutableOrdered4LexBody = { readonly kind: 'ordered4-terminals'; readonly commonFirstSet: FirstSet; readonly commonMissExpected: readonly string[]; readonly expected: readonly string[]; readonly arms: readonly [ ExecutableTerminalLexBody, ExecutableTerminalLexBody, ExecutableTerminalLexBody, ExecutableTerminalLexBody ]; }; export type ExecutableNot2LexBody = { readonly kind: 'not2-ordered4-terminal'; readonly guard0Expected: readonly string[]; readonly guard1Expected: readonly string[]; readonly terminalExpected: readonly string[]; readonly guard0: ExecutableOrdered4LexBody['arms']; readonly guard1: ExecutableTerminalLexBody; readonly terminal: ExecutableTerminalLexBody; }; export type ExecutableLexNode = { readonly kind: 'terminal'; readonly terminal: ExecutableTerminalLexBody; } | { readonly kind: 'sequence'; readonly parts: readonly ExecutableLexNode[]; } | { readonly kind: 'choice'; readonly arms: readonly ExecutableLexNode[]; readonly firstSets: readonly FirstSet[]; readonly armExpected: readonly (readonly string[])[]; readonly expected: readonly string[]; } | { readonly kind: 'not'; readonly body: ExecutableLexNode; readonly expected: readonly string[]; } | { readonly kind: 'optional'; readonly body: ExecutableLexNode; }; export type ExecutableFixedLexBody = { readonly kind: 'fixed-tree'; readonly root: ExecutableLexNode; }; /** A balanced token whose recognition remains owned by the canonical scanner * pool. The selected shell removes only the losing token/parent topology; it * never restates ambient scanSkip, recovery, or skipper semantics. */ export type ExecutableBalancedLexBody = { readonly kind: 'balanced-scan'; readonly open: string; readonly close: string; readonly raw: boolean; readonly strict: boolean; readonly ownSkip: readonly Combinator[]; }; export type ExecutableLexBody = ExecutableOptionalSuffixLexBody | ExecutableTerminalLexBody | ExecutableOrdered4LexBody | ExecutableNot2LexBody | ExecutableFixedLexBody | ExecutableBalancedLexBody; export type LexBodyCandidate = { readonly strategy: 'character'; readonly estimatedOps: number; } | { readonly strategy: 'token'; readonly estimatedOps: number; readonly body: OptionalSuffixLexBody; }; /** Exact recognizer shape admitted by the first end-to-end Stage-C tranche. */ export declare function optionalSuffixLexBody(ir: LexicalIr): OptionalSuffixLexBody | undefined; /** Compare two COMPLETE candidates. The CHARACTER cost is the existing * token-boundary + sequence + regex + optional + literal body. The TOKEN cost * is one selected body and one materialization boundary; neither candidate is * serialized until this decision returns. */ export declare function selectOptionalSuffixLexBody(ir: LexicalIr): LexBodyCandidate; /** Refuse wrapper/effect lookalikes even when normalization erases them to the * same language. This first executable body owns only the authored token * boundary around a direct sequence(regex, optional(literal-one-code-unit)). */ export declare function directOptionalSuffixTokenBody(parser: Combinator, resolve?: (name: string) => Combinator | undefined): ExecutableOptionalSuffixLexBody | undefined; /** Exact direct token terminal bodies. `keywords()` is represented by the same * canonical sticky RegExp its authored parser and OP_RX lowering use. Literal * stays CHARACTER for now: case-insensitive literal() intentionally uses ASCII * folding, which is not JavaScript RegExp `/i` semantics for non-ASCII text. */ export declare function directTerminalTokenBody(parser: Combinator, resolve?: (name: string) => Combinator | undefined): ExecutableTerminalLexBody | undefined; /** First composite selected body: the hot Less at-keyword choice. Keep this a * fixed template, not a recursive lexical interpreter. Equal finite first sets * are load-bearing: at a matching lead all four arms really run in source * order, while an excluded lead contributes only the choice's static opener * set and no probe event. */ export declare function directOrdered4TokenBody(parser: Combinator, resolve?: (name: string) => Combinator | undefined): ExecutableOrdered4LexBody | undefined; export declare function directNot2TokenBody(parser: Combinator, resolve?: (name: string) => Combinator | undefined): ExecutableNot2LexBody | undefined; export declare function directFixedTokenBody(parser: Combinator, resolve?: (name: string) => Combinator | undefined): ExecutableFixedLexBody | undefined; export declare function directBalancedTokenBody(parser: Combinator, resolve?: (name: string) => Combinator | undefined): ExecutableBalancedLexBody | undefined; export declare function directExecutableTokenBody(parser: Combinator, resolve?: (name: string) => Combinator | undefined): ExecutableLexBody | undefined; /** One canonical recognizer spec, shared by every family with equal lexical IR. */ export type LexicalRecognizer = { readonly id: number; readonly key: string; readonly ir: LexicalIr; /** Crosswalk into the compiler-wide canonical recognition-family interner. */ readonly capabilityFamilyId: number; }; /** * Spelling-specific failure/probe contract. Families may share recognition IR * without sharing where a structured spelling fails or what it expects. */ export type LexicalDiagnosticPlan = { readonly id: number; readonly body: Combinator; }; /** Pointer-free, compiler-only checkpoints inside one authored token body. */ export type LexicalDiagnosticEvent = { readonly op: 'FAIL'; readonly state: number; readonly expectedId: number; } | { readonly op: 'ASSERT'; readonly state: number; readonly innerStart: number; readonly innerEnd: number; readonly positive: boolean; readonly expectedId: number; readonly snapshotPolicy: 'saveLookaheadMark'; readonly execution: 'deferred'; } | { readonly op: 'REQUIRE'; readonly state: number; readonly childStart: number; readonly childEnd: number; readonly expectedId: number; readonly when: 'mandatory-iteration-2-to-min'; readonly committedChild: 'propagate'; readonly probe: 'child-only'; readonly anchor: 'repeat-position'; } | { readonly op: 'BAL_CLOSE_STRICT'; readonly state: number; readonly expectedId: number; readonly probe: true; readonly committed: false; readonly when: 'close-miss-after-open-body-success'; } | { readonly op: 'BAL_CLOSE_RECOVER'; readonly state: number; readonly expectedId: number; readonly probe: true; readonly committed: false; readonly when: 'close-miss-after-open-body-success'; readonly result: 'parseError'; readonly errorSpan: 'close-position'; readonly lineAnnotation: 'when-active'; readonly errorSink: 'when-active'; readonly cstErrorCapture: 'when-active'; }; /** * Stage-B2 diagnostic metadata. It is intentionally absent from TableProgram: * assertion execution and every token-boundary effect remain capability GAPs. */ export type LexicalTransitionDiagnosticPlan = { readonly id: number; readonly stateCount: number; readonly expected: readonly (readonly string[])[]; readonly events: readonly LexicalDiagnosticEvent[]; }; export type LexicalControlNode = { readonly id: number; readonly parentControlId?: number; /** Authored parser kind, or the conditional/repeated balanced skipper role. */ readonly kind: ParserDef['tag'] | 'balanced' | 'balanced-skip'; readonly stateStart: number; readonly stateEnd: number; }; export type LexicalControlPlan = { readonly id: number; /** Pointer-free source-control index over the one normalization session. */ readonly controls: readonly LexicalControlNode[]; }; /** One globally interned lexical family/base-token id. */ export type LexicalTokenFamily = { readonly id: number; readonly recognizerId: number; }; /** One authored `token()` site, with spelling-specific diagnostics/effects. */ export type LexicalTokenSite = { readonly parser: Combinator; readonly body: Combinator; /** Absent on a refused site; refused metadata allocates no diagnostic plan. */ readonly diagnosticId?: number; /** All three are absent when normalization declined. */ readonly familyId?: number; readonly recognizerId?: number; readonly refusal?: string; }; export type LexicalCapabilityStatus = { readonly kind: 'complete'; } | { readonly kind: 'gap'; readonly reason: string; } | { readonly kind: 'impossible'; readonly proof: string; }; export type LexicalCapabilityPhase = { readonly representation: LexicalCapabilityStatus; /** A complete body is constructible for every required reader/variant before selection. */ readonly executableLowering: LexicalCapabilityStatus; }; export type LexicalCapabilityObligations = { readonly recognition: LexicalCapabilityPhase; readonly diagnostics: LexicalCapabilityPhase; readonly boundaryPlan: LexicalCapabilityPhase; readonly materializationPlan: LexicalCapabilityPhase; readonly supportedVariants: LexicalCapabilityPhase; readonly bindingAndReachability: LexicalCapabilityPhase; }; /** Built-in authored token() context transaction. Compiler-only in this tranche. */ export type LexicalBoundaryPlan = { readonly id: 0; readonly kind: 'token-context-transaction'; }; /** Built-in authored token() source-range value/CST materialization. */ export type LexicalMaterializationPlan = { readonly id: 0; readonly kind: 'token-source-range'; }; /** Exact construction-time grammar wrapper policy, not merely its effective context. */ export type LexicalGrammarWrapperSpec = { readonly id: number; readonly sourceOperationId: number; readonly clearTrivia: boolean; readonly triviaBindingId?: number; readonly trackLines: 'on' | 'off' | 'inherit'; readonly captureTrivia: boolean; readonly captureTriviaKindsId?: number; readonly rootCapture: 'opaque' | 'inherit'; readonly clonePolicy: 'spread-existing-or-create-canonical'; readonly postChildPolicy: 'line-propagate-then-annotate-or-return'; }; /** Well-nested overlay over the sole LexicalIr/B2 control/state authority. */ export type LexicalWrapperFrame = { readonly id: number; readonly parentFrameId?: number; readonly controlId: number; readonly stateStart: number; readonly stateEnd: number; readonly kind: 'token'; readonly boundaryPlanId: 0; readonly materializationPlanId: 0; } | { readonly id: number; readonly parentFrameId?: number; readonly controlId: number; readonly stateStart: number; readonly stateEnd: number; readonly kind: 'grammar'; readonly wrapperSpecId: number; }; export type LexicalBoundaryTopology = { readonly id: number; /** Source-order outer-before-inner frames; recognition/control is not duplicated. */ readonly frames: readonly LexicalWrapperFrame[]; }; export type LexicalCapabilityContext = { readonly trivia: Combinator | undefined; readonly scanSkip: readonly Combinator[]; readonly trackLines: boolean; readonly captureTrivia: boolean; readonly opaqueRootCapture: boolean; readonly dynamicState: boolean; }; /** Immutable compiler occurrence context. Live scanner callbacks are policy, * never identity-bearing lexical facts or pending-range keys. */ export type LexicalContextSnapshot = { readonly id: number; readonly key: string; readonly hasTrivia: boolean; readonly hasScanSkip: boolean; readonly trackLines: boolean; readonly captureTrivia: boolean; readonly opaqueRootCapture: boolean; readonly dynamicState: boolean; }; export type LexicalOwnSkipEntry = { readonly semanticKey?: string; readonly ir?: LexicalIr; readonly status: LexicalCapabilityStatus; }; /** Construction-owned skipper children only. Ambient trivia/scanSkip never * enter this pool because their object and `.parse` lookup are observable. */ export type LexicalOwnSkipPlan = { readonly id: number; readonly entries: readonly LexicalOwnSkipEntry[]; }; export type LexicalScanConsumerPolicy = { readonly id: number; readonly siteId: number; readonly path: string; readonly contextSnapshotId: number; readonly kind: 'scanTo'; readonly raw: boolean; readonly ambient: 'enumerate-trivia-then-scanSkip-per-parse-attempt' | 'none-raw'; readonly lookup: 'dynamic-parse-property-every-skip-test'; readonly context: 'detached-per-attempt-state-errors-only'; readonly ownSkipPlanId?: number; readonly pendingReuse: 'forbidden'; readonly status: LexicalCapabilityStatus; } | { readonly id: number; readonly siteId: number; readonly path: string; readonly contextSnapshotId: number; readonly kind: 'balanced'; readonly raw: boolean; readonly ambient: 'array-identity-interior-cache' | 'none-raw'; readonly cache: 'lookup-every-attempt-enumerate-on-miss'; readonly lookup: 'dynamic-parse-property-every-skip-test'; readonly context: 'token-cleared-original'; readonly ownSkipPlanId?: number; readonly pendingReuse: 'forbidden'; readonly status: LexicalCapabilityStatus; }; /** * Compiler-only phase-A census record. These ids are deliberately not family * ids: two authored sites can share one future language while remaining two * independently auditable capability obligations. */ export type LexicalCapabilitySite = { readonly id: number; /** Final-graph compiler body identity. Never serialized as a parser pointer. */ readonly bodyId: number; /** Interned recognition-language summary; never used as a completeness id. */ readonly languageId: number; readonly path: string; /** Effective lexical scope at this occurrence; compiler-only, never a family id. */ readonly contextKey: string; readonly contextSnapshotId: number; /** Context seen by recognition after token() clears trivia/capture sinks. */ readonly recognitionContextKey: string; readonly context: LexicalCapabilityContext; readonly recognitionContext: LexicalCapabilityContext; readonly semanticKey: string; readonly atom: 'terminal' | 'token' | 'choice' | 'dispatch'; readonly parser: Combinator; readonly obligations: LexicalCapabilityObligations; /** Internal body metadata only; never claims the boundary effects obligation. */ readonly diagnosticPlanId?: number; /** Compiler-only boundary overlay; absent when exact representation declined. */ readonly boundaryTopologyId?: number; /** Control ancestry authority used by the boundary overlay. */ readonly controlPlanId?: number; /** Scan consumers owned by this atomic occurrence, in structural order. */ readonly scanConsumerPolicyIds: readonly number[]; /** Derived from `obligations`; callers cannot independently set it. */ readonly status: LexicalCapabilityStatus; }; export type LexicalCapabilityLanguage = { readonly id: number; readonly atom: LexicalCapabilitySite['atom']; readonly semanticKey: string; }; /** * One compiler-wide canonical source-range language. IDs use the lexical * family namespace even while phase B is disabled; decision plans and the * legacy token collector cross-reference this one IR interner. */ export type LexicalDecisionFamily = { readonly id: number; readonly semanticKey: string; readonly ir: LexicalIr; }; /** * A compatible view of one family range. `prefix` owns its own shorter end; * consumers must never substitute the full family end (the `a | ab` rule). */ export type LexicalDecisionOutcomeView = { readonly kind: 'whole'; readonly relation: 'equal'; } | { readonly kind: 'predicate'; readonly relation: 'equal'; readonly match: Exclude; } | { readonly kind: 'language'; readonly relation: 'equal' | 'prefix'; readonly ir: LexicalIr; }; /** Global, family-qualified atomic outcome/view identity. */ export type LexicalDecisionOutcome = { readonly id: number; readonly familyId: number; readonly semanticKey: string; readonly view: LexicalDecisionOutcomeView; }; export type LexicalDecisionAcceptance = { readonly kind: 'outcomes'; readonly outcomeIds: readonly number[]; } | { readonly kind: 'otherwise'; readonly excludingOutcomeIds: readonly number[]; } | { readonly kind: 'unrestricted'; } | { readonly kind: 'impossible'; } | { readonly kind: 'gap'; readonly reason: string; }; /** One source-ordered arm/route for one candidate family at one occurrence. */ export type LexicalDecisionArm = { readonly armId: number; readonly acceptance: LexicalDecisionAcceptance; readonly usesRouted: boolean; readonly dynamicGate: boolean; }; export type LexicalDecisionFamilyPlan = { readonly familyId: number; readonly arms: readonly LexicalDecisionArm[]; }; /** Source-ordered decision regions within which a recognized range may remain * live. A dynamic scanner is its own closed epoch: callbacks and ambient skip * policy may mutate parse state, so no lexical fact may cross that arm in * either direction. This is compiler evidence for one complete TOKEN body, * never a request to replay the CHARACTER body. */ export type LexicalDecisionReuseEpoch = { readonly armIds: readonly number[]; readonly pendingReuse: 'forbidden' | 'unproved'; readonly boundary: 'none' | 'dynamic-scan'; }; /** Occurrence-local ordered decision proof; never serialized in this tranche. */ export type LexicalDecisionSite = { readonly siteId: number; readonly atom: 'choice' | 'dispatch'; readonly path: string; readonly contextKey: string; readonly contextSnapshotId: number; readonly families: readonly LexicalDecisionFamilyPlan[]; /** Dispatch-only source-order routed policy. This remains Stage-A authority; * the effect projection references the site rather than copying route bits. */ readonly routeUsesRouted?: readonly boolean[]; /** Families not proven at this occurrence remain admitted through this * conservative relation. It is an explicit TOKEN-body route, never a request * to replay the character parser. */ readonly fallback: 'unrestricted'; /** Missing inclusion/partition proofs reduce pruning precision only. Every * authored arm still runs its own exact replacement recognizer in PEG order. */ readonly precisionNotes: readonly string[]; /** Dynamic scanner callbacks are effectful and can never publish a reusable * cross-arm lexical fact. All other sites remain unproved in this tranche. */ readonly pendingReuse: 'forbidden' | 'unproved'; readonly reuseEpochs: readonly LexicalDecisionReuseEpoch[]; }; /** Final winner-resolved fixed choice classes, shared with table encoding. */ export type LexicalChoiceClassAuthority = { readonly id: number; readonly armIds: readonly number[]; readonly firstSets: readonly FirstSet[]; }; /** A non-authored execution order such as longest literal first. */ export type LexicalChoiceExecutionOrder = { readonly id: number; readonly armIds: readonly number[]; }; /** Exact classified spelling to authored arm identity. */ export type LexicalClassifySpellingMap = { readonly id: number; readonly entries: readonly { readonly spelling: string; readonly armId: number; }[]; }; /** Transform callback identities applied to an already recognized spelling. */ export type LexicalClassifyProjectionPlan = { readonly id: number; readonly entries: readonly { readonly armId: number; readonly transformIds: readonly number[]; readonly transformSourceIds: readonly (number | -1)[]; readonly transformCount: number; }[]; }; export type LexicalExpectedAuthority = { readonly id: number; readonly values: readonly string[]; }; /** Compiler-local callback identity; never a TableProgram or public wire value. */ export type LexicalDecisionCallbackAuthority = { readonly id: number; readonly callback: ((state: unknown) => boolean) | ((value: unknown, span: { start: number; end: number; }) => unknown); }; /** Named emitted projection for a callback; compiler-only until Stage C selects. */ export type LexicalDecisionSourceAuthority = { readonly id: number; readonly source: string; }; /** Exact post-success rejection checks, interned without re-deriving them. */ export type LexicalAutoNotAuthority = { readonly id: number; readonly checks: readonly AutoNotCheck[]; }; export type LexicalFinalDecisionAuthority = { readonly id: number; readonly atom: 'choice'; readonly decisionSiteId: number; readonly authoredArmIds: readonly number[]; readonly mode: { readonly kind: 'ordered'; } | { readonly kind: 'exclusive'; readonly classAuthorityId: number; } | { readonly kind: 'longest'; readonly orderAuthorityId: number; } | { readonly kind: 'classify'; readonly superArmId: number; readonly spellingMapAuthorityId: number; readonly projectionAuthorityId: number; }; /** Static total-miss set only for strategies that never consult dynamic gates. */ readonly staticMissExpectedAuthorityId?: number; } | { readonly id: number; readonly atom: 'dispatch'; readonly decisionSiteId: number; readonly selectorBindingEdgeId: number; readonly classifierAuthorityId: number; readonly orderedRouteIds: readonly number[]; readonly noRouteExpectedAuthorityId: number; }; export type LexicalDecisionChildEffect = { readonly id: number; readonly decisionSiteId: number; readonly armId: number; readonly childBindingEdgeId: number; readonly expectedAuthorityId: number; readonly gateCallbackId?: number; readonly gateSourceId?: number; readonly autoNotAuthorityId?: number; }; export type LexicalDecisionEffectProgram = { readonly id: number; readonly decisionSiteId: number; readonly finalDecisionAuthorityId: number; readonly phase: LexicalCapabilityPhase; readonly childEffectIds: readonly number[]; /** Exact fixed-edge projections consumed by the three existing readers. */ readonly childBindingProjectionIds: readonly number[]; readonly referenceTemplateId: number; readonly capturedTemplateId: number; readonly namedTemplateId: number; readonly readerMask: number; readonly variantMask: number; readonly semanticDigest: number; readonly pendingRollbackPolicyId: 0; readonly commitPolicyId: 0; }; /** Compiler-only C2 pools. No member is a TableProgram or runtime wire field. */ export type LexicalDecisionEffectInventory = { readonly finalDecisionAuthorities: readonly LexicalFinalDecisionAuthority[]; readonly decisionEffects: readonly LexicalDecisionEffectProgram[]; readonly decisionChildEffects: readonly LexicalDecisionChildEffect[]; readonly choiceClassAuthorities: readonly LexicalChoiceClassAuthority[]; readonly choiceExecutionOrders: readonly LexicalChoiceExecutionOrder[]; readonly classifySpellingMaps: readonly LexicalClassifySpellingMap[]; readonly classifyProjectionPlans: readonly LexicalClassifyProjectionPlan[]; readonly decisionExpectedAuthorities: readonly LexicalExpectedAuthority[]; readonly decisionCallbackAuthorities: readonly LexicalDecisionCallbackAuthority[]; readonly decisionSourceAuthorities: readonly LexicalDecisionSourceAuthority[]; readonly decisionAutoNotAuthorities: readonly LexicalAutoNotAuthority[]; }; /** One fixed incoming/root edge whose direct linked-body candidates remain owed. */ export type LexicalBindingEdge = { readonly id: number; readonly path: string; readonly contextKey: string; /** Effective context at the child occurrence, after the parent boundary. */ readonly childContextKey: string; readonly contextSnapshotId: number; readonly parentTag: ParserDef['tag'] | 'root'; readonly childTag: ParserDef['tag']; readonly projectionId?: number; readonly status: LexicalCapabilityStatus; }; /** Compiler-owned fixed-edge lowering. These are numeric construction facts, * not a parse-time micro-IR: each reader consumes only its own preselected * template and the common digest proves all three describe the same edge. */ export type LexicalBindingProjection = { readonly id: number; readonly edgeId: number; readonly parentBodyId: number; readonly childBodyId: number; readonly childOrdinal: number; readonly referenceOperand: number; readonly capturedSlotId: number; readonly capturedTemplateId: number; readonly namedSymbolId: number; readonly namedTemplateId: number; /** bit 0 reference, bit 1 captured/CSP, bit 2 named/emitted. */ readonly readerMask: number; /** Strict, recovery, tracking, CST, probe, and coverage construction modes. */ readonly variantMask: number; readonly semanticDigest: number; }; /** One executable primitive-terminal candidate plus every fixed incoming edge. * This remains compiler-only until Stage C selects a complete semantic body. */ export type LexicalTerminalProjection = { readonly id: number; readonly siteId: number; readonly bodyId: number; readonly languageId: number; readonly referenceBodyId: number; readonly capturedTemplateId: number; readonly namedSymbolId: number; readonly incomingBindingProjectionIds: readonly number[]; readonly readerMask: number; readonly variantMask: number; readonly semanticDigest: number; readonly status: LexicalCapabilityStatus; }; export type LexicalOutcomeMatch = { readonly kind: 'exact'; readonly values: readonly string[]; readonly caseInsensitive: boolean; } | { readonly kind: 'startsWith' | 'endsWith'; readonly value: string; readonly caseInsensitive: boolean; } | { readonly kind: 'matches'; readonly value: string; readonly flags: string; readonly caseInsensitive: boolean; } | { readonly kind: 'otherwise'; readonly excluding: readonly Exclude[]; }; /** A globally reusable classification id for one family/range predicate. */ export type LexicalOutcomeSpec = { readonly id: number; readonly familyId: number; readonly match: LexicalOutcomeMatch; }; /** One compatible view over a family range at a particular dispatch site. */ export type LexicalTokenOutcome = { readonly id: number; readonly match: LexicalOutcomeMatch; }; /** One ordered dispatch arm. IDs classify; this route owns the branch/cut. */ export type LexicalTokenRoute = { readonly index: number; readonly kind: 'exact' | 'matcher' | 'otherwise'; readonly acceptedIds: readonly number[]; readonly matches: readonly LexicalOutcomeMatch[]; readonly parser: Combinator; readonly usesRouted: boolean; }; /** Dispatch outcomes stay site-local; ordered dispatch semantics are unchanged. */ export type LexicalTokenClassifier = { readonly dispatch: Combinator; readonly familyId: number; /** True when reaching token() crossed a wrapper whose effects must still run. */ readonly selectorEffects: boolean; /** Compatible range views, flattened and allowed to repeat an id by route. */ readonly outcomes: readonly LexicalTokenOutcome[]; /** Authoritative ordered route/cut identity. */ readonly routes: readonly LexicalTokenRoute[]; }; export type LexicalAlphabet = { /** * Compiler-only graph metadata. `diagnostics`, `sites`, `classifiers`, and * `familyIdOf` contain live combinators/Maps and are NOT TableProgram data. * A serializer must project the numeric/IR specs and relocate site references. */ readonly recognizers: readonly LexicalRecognizer[]; readonly diagnostics: readonly LexicalDiagnosticPlan[]; readonly families: readonly LexicalTokenFamily[]; readonly sites: readonly LexicalTokenSite[]; readonly outcomes: readonly LexicalOutcomeSpec[]; readonly classifiers: readonly LexicalTokenClassifier[]; readonly familyIdOf: ReadonlyMap, number>; /** Whole-final-grammar, ownership-aware phase-A capability census. */ readonly capabilities: readonly LexicalCapabilitySite[]; /** Interned language view only; completeness is always occurrence-based. */ readonly capabilityLanguages: readonly LexicalCapabilityLanguage[]; /** Every distinct fixed parent/root edge, independently GAP until linked. */ readonly bindingEdges: readonly LexicalBindingEdge[]; readonly bindingProjections: readonly LexicalBindingProjection[]; readonly terminalProjections: readonly LexicalTerminalProjection[]; /** Phase-A compatible range/arm algebra; compiler-only and occurrence-local. */ readonly decisionFamilies: readonly LexicalDecisionFamily[]; readonly decisionOutcomes: readonly LexicalDecisionOutcome[]; readonly decisions: readonly LexicalDecisionSite[]; readonly decisionEffectPlan: LexicalDecisionEffectInventory; readonly transitionDiagnostics: readonly LexicalTransitionDiagnosticPlan[]; readonly boundaryPlans: readonly LexicalBoundaryPlan[]; readonly materializationPlans: readonly LexicalMaterializationPlan[]; readonly grammarWrapperSpecs: readonly LexicalGrammarWrapperSpec[]; readonly grammarCaptureTriviaKinds: readonly (readonly string[])[]; readonly boundaryTopologies: readonly LexicalBoundaryTopology[]; readonly controlPlans: readonly LexicalControlPlan[]; readonly contextSnapshots: readonly LexicalContextSnapshot[]; readonly ownSkipPlans: readonly LexicalOwnSkipPlan[]; readonly scanConsumerPolicies: readonly LexicalScanConsumerPolicy[]; /** False means phase B is forbidden for the entire program. */ readonly capabilityComplete: boolean; }; export type LexicalCapabilityInventory = Pick; /** Numeric-only artifact projection; `TableProgram` aliases this contract. */ export type NumericLexicalPlan = { readonly recognizerOffsets: readonly number[]; readonly recognizerData: readonly number[]; readonly outcomeOffsets: readonly number[]; readonly outcomeData: readonly number[]; readonly tokenSites: readonly number[]; readonly sites: readonly number[]; readonly routes: readonly number[]; readonly accepted: readonly number[]; }; /** Lexical-family ids occupy their own published namespace. */ export declare const FIRST_LEXICAL_FAMILY_ID = 3; /** Canonical, parser-free identity for one range predicate. */ export declare function canonicalLexicalOutcomeKey(match: LexicalOutcomeMatch): string; /** Does a final rule-map winner bottom out at the reference being resolved? */ export declare function winnerWrapsReference(winner: Combinator, reference: Combinator): boolean; /** Direct sub-parsers of a def, resolving a lazy through `resolve` when its own thunk is a hole. */ export declare function tokenChildren(p: Combinator, resolve?: (name: string) => Combinator | undefined): Combinator[]; /** * Collect the alphabet over `roots`. Ids are assigned in first-encounter order, * which is stable for a given grammar and therefore safe to switch on. */ export declare function collectAlphabet(roots: ReadonlyArray>, resolve?: (name: string) => Combinator | undefined): Alphabet; /** Test-only production-seam model: snapshot the supplied derived tables while * retaining a raw graph receipt captured before derivation. A dropped raw * occurrence must fail even though the post-derivation snapshot agrees. */ export declare function assertLexicalCapabilityProductionBijection(roots: ReadonlyArray>, alphabet: Omit, resolve?: (name: string) => Combinator | undefined): void; /** * Test/diagnostic entry for the one-pass production closure guard. This helper * deliberately rebuilds the expected inventory for a supplied test artifact; * production consumes the session-local raw receipt and never walks the graph * or recomputes lexical analyses a second time. */ export declare function assertLexicalCapabilityOccurrenceBijection(roots: ReadonlyArray>, alphabet: Omit, resolve?: (name: string) => Combinator | undefined): void; /** Re-enumerate the final graph so a dropped/filtered candidate fails closed. */ export declare function assertLexicalCapabilityClosure(roots: ReadonlyArray>, alphabet: Pick & Partial>, resolve?: (name: string) => Combinator | undefined): void; /** Phase-A only: inventory obligations without constructing unused runtime families/sites. */ export declare function collectLexicalCapabilities(roots: ReadonlyArray>, resolve?: (name: string) => Combinator | undefined): LexicalCapabilityInventory; /** * Catalog authored lexical tokens and their dispatch views without confusing * private child terminals for source tokens. This is metadata only: consumers * still have to lower the canonical IR in every shipping engine before enabling * an admission path. */ export declare function collectLexicalAlphabet(roots: ReadonlyArray>, resolve?: (name: string) => Combinator | undefined): LexicalAlphabet; /** * Project the compiler graph into compact numeric pools. The caller supplies * only already-relocated table-site numbers and its existing const-pool intern; * no combinator or identity map crosses this boundary. */ export declare function serializeLexicalPlan(alphabet: LexicalAlphabet, constant: (value: unknown) => number, tokenSites: readonly number[], dispatchSites: ReadonlyArray<{ readonly dsp: number; readonly classifier: LexicalTokenClassifier; }>): NumericLexicalPlan | undefined; /** * Metadata/reference oracle for compatible same-range views. Exact/prefix/suffix * checks read char codes directly; regex matcher views slice only in this cold * oracle. Production token cursors must lower matcher plans without treating * this helper as their allocation contract. */ export declare function compatibleLexicalOutcomes(classifier: LexicalTokenClassifier, input: string, start: number, end: number): number[]; /** * Cold oracle for dispatch selection precedence. Compatible views are a set; * selected routing is still exact cases first, then matcher source order, then * otherwise, and a selected branch failure remains committed. */ export declare function selectedLexicalOutcome(classifier: LexicalTokenClassifier, input: string, start: number, end: number): { route: LexicalTokenRoute; outcomeId: number; } | undefined; /** The id already assigned to this terminal, if it is one. */ export declare function terminalId(alphabet: Alphabet, p: Combinator): number | undefined; /** * The LEADING terminal of an arm — the one a decision point would consult. Walks * through the wrappers that do not consume input, and through a sequence's * nullable/zero-width prefix. Returns undefined when the lead is not a single * derived terminal, which is the signal to stay scannerless at that site. */ export declare function leadTerminal(p: Combinator, alphabet: Alphabet, resolve?: (name: string) => Combinator | undefined, depth?: number): number | undefined; /** * The CANDIDATE SET for one decision point: the leading terminals of its arms. * `complete` is false when any arm's lead is not a derived terminal — that site * must stay scannerless, per-construct, exactly as `scanSkip` and first-set * gating already apply per region rather than globally. */ export type CandidateSet = { ids: number[]; complete: boolean; }; export declare function candidateSet(arms: ReadonlyArray>, alphabet: Alphabet, resolve?: (name: string) => Combinator | undefined): CandidateSet; //# sourceMappingURL=token-alphabet.d.ts.map