/** * The linker: compose independently-carried grammar pieces into ONE runnable rule map. * * Composition is a RULE-MAP MERGE plus a single encode. Each piece carries its * combinator graph as IR; `compose()` evaluates every piece's IR back to a rule map, * lets a later piece's name override an earlier one, and encodes the merged map ONCE. * `enc.winners` binds every by-name reference against that merged map, so overriding a * rule reroutes EVERY call to it — including calls inside a base piece's own rules * (open recursion) — with no shared scope and no relocation. Later piece wins per * rule name. * * This replaced a textual splice. The source lowering compiled each piece to namespaced * `_r_` function sources and concatenated them into one `new Function` scope, * patching `@FS:` first-set placeholders per winner. That is why the linker needed * `'unsafe-eval'`; the merge is now data, so it does not. */ import { ruleDependencies, childrenOf } from '../analysis/gating.ts' import { FUSED_HOST_MODE, FUSED_HOST_ELIDED, type HostMode } from '../cst/host-mode.ts' import { evalRuleMapIR, serializeRuleMap } from './ir-serialize.ts' import { compileLinkableTable, type LinkableTable } from './compile-linkable-table.ts' import { compileRuleMapRunnable } from '../table/compile-rule-map.ts' import { tableRules } from '../table/assemble.ts' import { GRAMMAR_REFLECTION } from '../cst/reflection.ts' import { PARSEMAN_VERSION } from '../version.ts' import type { BuildHost, Combinator, CstCollapsePredicate, ParseContext, ParseResult } from '../types.ts' import type { Runnable } from '../functional/run.ts' /** * Compile a `rules()` map to a **linkable artifact** — the composable, shippable * form (RULE_ABI_PLAN §4). A package exports `linkable(rules(g => …))`; consumers * import that artifact and `fuse([...])` it — **no source of the base grammar is * ever read**. Under the macro this is precompiled to static pieces; in the * interpreter it compiles here at load (like `compile()`). * * `ns` is a per-artifact namespace; omit it to auto-assign a process-unique one * (fine at runtime — the macro supplies a stable module-derived ns instead). */ let _nsCounter = 0 export function linkable( rulesMap: Record>, ns?: string, trivia?: Combinator, // Compile-time host mode, same meaning as `compile(g, { hostMode })`: 'ast' (default) // emits the grammar's own builders and NO positioned-CST branch; 'cst' builds every // node through the host. A linked/fused artifact is version- and mode-locked, so a // language service links its own 'cst' artifact rather than switching at parse time. hostMode?: HostMode, ): LinkableTable { const piece = compileLinkableTable( Object.entries(rulesMap), ns ?? `_lk${_nsCounter++}_`, { ...(trivia ? { trivia } : {}), ...(hostMode ? { hostMode } : {}) }, ) if (!piece) throw new Error('linkable(): this grammar cannot be compiled to a linkable artifact (contains a runtime-only parser fallback)') return piece } export type CstBuildHostOptions = { /** * Collapse transparent one-child CST wrapper nodes at build time. * - `true`: collapse any one-child node whose rawChildren also has exactly one * entry, so trivia/error boundaries are not silently dropped. * - `string[]`: collapse only these grammar node types. * - predicate: final policy hook for language-specific public CSTs. */ collapse?: boolean | readonly string[] | CstCollapsePredicate /** * Materialize `node(..., { tags })` grammar metadata onto produced CST nodes. * When omitted, tags stay in grammar reflection for zero per-node tree cost. */ tags?: boolean } function normalizeCstCollapse(collapse: CstBuildHostOptions['collapse']): CstCollapsePredicate | undefined { if (collapse === true) return () => true if (Array.isArray(collapse)) { const types = new Set(collapse) return type => types.has(type) } return typeof collapse === 'function' ? collapse : undefined } function buildCstNode( type: string, children: ReadonlyArray, _fields: unknown, span: { start: number; end: number }, _rawChildren?: ReadonlyArray, _triviaLog?: readonly number[], state?: unknown, tags?: readonly string[] | undefined, ): unknown { // Carry the grammar's `ctx.state` snapshot onto the node (null when unset) — the // CST contract includes `state` and incremental re-parse replays it on edit. return tags !== undefined && tags.length > 0 ? { _tag: 'node', type, tags, span: { ...span }, state: state ?? null, children: [...children] } : { _tag: 'node', type, span: { ...span }, state: state ?? null, children: [...children] } } /** * A generic positioned-CST build host (RULE_ABI_PLAN §7). Pass as `ctx.build` * (or `parseDoc(..., { build: cstBuildHost })`) to make ANY linkable/fused * grammar produce a uniform CST — `{ _tag:'node', type, span, state, children }` * — instead of its own eval-AST builders. This is the host the linter and IDE * drivers use; the eval driver leaves `ctx.build` unset (grammar's own builders). * * For public syntax trees, call `cstBuildHost({ collapse })`: Parseman will skip * allocating wrapper CST nodes whose single child should stand in for the rule. */ export function cstBuildHost(options?: CstBuildHostOptions): BuildHost export function cstBuildHost( type: string, children: ReadonlyArray, fields: unknown, span: { start: number; end: number }, rawChildren?: ReadonlyArray, triviaLog?: readonly number[], state?: unknown, tags?: readonly string[] | undefined, ): unknown export function cstBuildHost( typeOrOptions?: string | CstBuildHostOptions, children?: ReadonlyArray, _fields?: unknown, span?: { start: number; end: number }, rawChildren?: ReadonlyArray, triviaLog?: readonly number[], state?: unknown, _tags?: readonly string[] | undefined, ): unknown { if (typeof typeOrOptions === 'string') { return buildCstNode(typeOrOptions, children ?? [], _fields, span ?? { start: 0, end: 0 }, rawChildren, triviaLog, state) } const collapse = normalizeCstCollapse(typeOrOptions?.collapse) const materializeTags = typeOrOptions?.tags === true const host: BuildHost = ( type: string, children: ReadonlyArray | undefined, fields: unknown, span: { start: number; end: number }, rawChildren: ReadonlyArray, triviaLog: readonly number[], state: unknown, tags?: readonly string[] | undefined, // A CST/collapse host always keeps `children` (chV) — the opt-out never // applies — so `?? []` is unreachable defensive modeling for the widened type. ) => buildCstNode(type, children ?? [], fields, span, rawChildren, triviaLog, state, materializeTags ? tags : undefined) ;(host as typeof host & { _parsemanCstOutput?: true })._parsemanCstOutput = true if (collapse) host._parsemanCstCollapse = collapse return host } // `cstBuildHost` itself is also accepted as a BuildHost (without options). ;(cstBuildHost as unknown as { _parsemanCstOutput?: true })._parsemanCstOutput = true /** * A fused function receives the full ParseContext through `run()`. Direct * callers historically supplied a plain context object, so keep that usage * valid while making the function assignable to the public `Runnable` type. * Generated code treats optional framework fields as absent when they are not * provided, matching the interpreter's normal defaults. */ export type FusedRule = ( input: string, pos: number, ctx: ParseContext | Record, ) => ParseResult & { readonly value?: unknown } /** * Fuse carried pieces into a runnable rule map — the TABLE equivalent of the * textual splice this file used to perform. * * `fusedBody()` concatenated namespaced `_r_` function sources, picked a winning * function per name, and patched `@FS:` dispatch placeholders with the winner's * first-set condition — roughly 200 lines whose entire job was to make separately * lowered SOURCE agree about names. A table has no text to splice, so the merge moves * up one level onto the combinators: merge the rule maps (later piece wins), then * `encodeTable` ONCE over the merged map. `enc.winners` binds every by-name reference, * including a base piece's internal `g.Atom` that an override replaced, so open * recursion across pieces resolves without relocating a single encoded offset. * * One encode, no pools to merge, and the result is the table the merged grammar would * have produced had it been written as a single `rules()` call. */ function fuseCarried( carried: ReadonlyArray, trivia?: Combinator, hostMode?: HostMode, ): Record { const maps: Array]>> = [] for (const p of carried) { // ARTIFACT VERSION LOCK. `fusedBody` enforced this and would have taken it with it: // artifacts are version-locked and there is no cross-version read path, so an // UNSTAMPED piece and a MISMATCHED one are both refused — a stale artifact that // merely happens to still encode is exactly what this stops. if (!isIRPiece(p)) { if (typeof p.v !== 'string') { throw new Error( `parseman: artifact "${p.ns}" is UNSTAMPED (compiled before the version-lock invariant). ` + `Recompile the grammar with parseman ${PARSEMAN_VERSION}; parseman does not fuse unversioned or cross-version artifacts.`, ) } if (p.v !== PARSEMAN_VERSION) { throw new Error( `parseman: artifact "${p.ns}" was compiled with parseman ${p.v}, but is being fused with parseman ${PARSEMAN_VERSION}. ` + `Compiled grammar artifacts are version-locked — recompile the grammar with parseman ${PARSEMAN_VERSION}; parseman does not fuse across versions.`, ) } } const rules = ruleMapOfCarried(p) if (rules === undefined) { throw new Error(`compose: carried piece "${p.ns}" has no re-lowerable IR and cannot be fused`) } maps.push(rules) } const merged = mergeCarriedRuleMaps(maps) materializeDirectBuilders(merged) // COMPOSING-WINS is an OVERRIDE, not a gap-fill: the composing grammar's trivia // governs every fused rule INCLUDING inherited ones. `applyAmbient` inside // `compileRuleMap` only fills a rule that declares none, which would leave an // inherited rule still skipping its own base's whitespace after a delta re-declared // it. Safe to mutate — `evalRuleMapIR` builds fresh combinators per fuse. if (trivia) { for (const [, rule] of merged) { if (rule._meta.isTrivia) continue ;(rule._meta as { grammarTrivia?: Combinator }).grammarTrivia = trivia } } // RUNNABLE, not printable. `compose()` returns a parser; it never emits source, so // requiring a captured source per author callback would refuse every grammar built at // runtime — which have live callbacks by construction. const refusals: string[] = [] const compiled = compileRuleMapRunnable(merged, { ...(trivia ? { trivia } : {}), ...(hostMode ? { hostMode } : {}), refusals, }) if (compiled === null) { throw new Error(`compose: the merged grammar could not be encoded to a table${refusals.length ? ` — ${refusals.join('; ')}` : ''}`) } // `tableRules`, NOT `exec.ts`'s same-named `tableRules`. Both return // `Record`, so binding the interpreter here type-checked and ran // correctly — it was just the SLOW engine, on the one path (compose/fuse) that never // goes through `table/index.ts` and so never saw the `tableRules as tableRules` // re-export. Import the assembler by its own name so the binding cannot go stale again. const map = tableRules(compiled.prog) as unknown as Record // The host-mode stamp went on the fused closure before; a table carries nothing until // it is stamped, and an UNSTAMPED map reads as `{ ast, false }` so every driver // compatibility check passes vacuously. Stamped on the rule functions too, because // `run(map.Rule, …)` is handed the rule and never sees the map. for (const k of Object.keys(map)) { Object.defineProperty(map[k]!, FUSED_HOST_MODE, { value: compiled.hostMode, enumerable: false }) Object.defineProperty(map[k]!, FUSED_HOST_ELIDED, { value: compiled.hostBranchElided, enumerable: false }) } Object.defineProperty(map, FUSED_HOST_MODE, { value: compiled.hostMode, enumerable: false }) Object.defineProperty(map, FUSED_HOST_ELIDED, { value: compiled.hostBranchElided, enumerable: false }) // GRAMMAR REFLECTION, which `fusedBody` merged across pieces and stamped. A visitor // built over a composed grammar reads it to know the node types; unstamped, it reads // as an empty grammar and every visitor silently matches nothing. Object.defineProperty(map, GRAMMAR_REFLECTION, { value: compiled.reflection, enumerable: false }) return map } /** * Re-attach a DIRECT node builder that came back from IR as an inert sentinel. * * `evalRuleMapIR` deliberately refuses to evaluate a captured `buildSrc`: raw IR * interpretation must not run arbitrary source, so `_nd` installs a thrower and keeps * the text. Only a COMPILER consumer may materialize it — and the runtime fuse is one. * The source lowering did exactly this, less visibly: it inlined `buildSrc` into the * fused body and `new Function` evaluated it. The table parks callbacks in a pool * instead of inlining them, so the same materialization has to be explicit or the * sentinel reaches the pool and throws on the first parse through that node. * * Scoped to the runtime fuse. The MACRO never comes here: its encoder captures * `buildSrc` and PRINTS it, so it needs no live function at all. */ function materializeDirectBuilders(ruleMap: ReadonlyArray]>): void { const seen = new Set>() const visit = (p: Combinator): void => { if (seen.has(p)) return seen.add(p) const d = p._def as { tag: string; type?: string; build?: unknown; buildSrc?: string; buildImports?: ReadonlyArray<{ local: string; source: string; imported: string }> } if (d.tag === 'node' && typeof d.buildSrc === 'string') { // FAIL CLOSED for the runtime fuse: a builder whose free names are module // imports can be re-bound by the MACRO plugin (it re-emits the imports into the // generated module), but an in-process `(0, eval)` here has no import to give it, // so the builder would throw ReferenceError at parse time — wrong output deferred. // Refuse now, with a message that points at the build-time path. The macro never // reaches this function (it prints `buildSrc`, see the doc above). if (d.buildImports !== undefined && d.buildImports.length > 0) { throw new Error( `IR direct node builder for ${d.type ?? ''} references module import(s) ` + `${d.buildImports.map(bi => bi.local).join(', ')} that a runtime compose() cannot supply; ` + `compose at build time via the parseman macro plugin, which re-emits the imports.`, ) } try { // eslint-disable-next-line no-eval const fn = (0, eval)(`(${d.buildSrc})`) as unknown if (typeof fn === 'function') d.build = fn } catch { /* leave the sentinel: it throws with its own message on reach */ } } if (d.tag === 'lazy') { let resolved: Combinator | undefined try { resolved = (d as unknown as { thunk: () => Combinator }).thunk() } catch { return } if (resolved) visit(resolved) return } for (const child of childrenOf(p._def)) visit(child) } for (const [, rule] of ruleMap) visit(rule) } /** The combinator map behind a carried piece: IR is evaluated back, a table piece * uses the IR it always carries. `undefined` only for a piece with neither. */ function ruleMapOfCarried(p: LinkableTable | IRPiece): Array<[string, Combinator]> | undefined { if (isIRPiece(p)) return evalRuleMapIR(p.ir) // IR FIRST: re-evaluating it yields FRESH combinators, so seeding composing trivia // onto their `_meta` cannot leak back into the artifact this piece came from. The live // map is the fallback for a runtime-built grammar that has no serializable IR. if (p.ir !== null) return evalRuleMapIR(p.ir) return p.ruleMap.length > 0 ? p.ruleMap.map(([k, v]) => [k, v] as [string, Combinator]) : undefined } /** Fold ordered rule maps into the composed map: a later piece's name WINS. * * A REFERENCE IS NOT A DEFINITION. A `rules(g => …)` cache also holds every `g.X` that * was merely ACCESSED, as an unresolvable lazy. Merged in order, such an entry lands * last and SHADOWS the piece that actually defines the name — the encoder then finds a * hole where the winner should be and refuses the whole grammar. The reference is not * lost: it stays inside the referring piece's rule bodies, where `enc.winners` binds it * by name to whichever piece supplies the definition. */ function mergeCarriedRuleMaps( maps: ReadonlyArray]>>, ): Array<[string, Combinator]> { const winners = new Map>() for (const map of maps) { for (const [name, rule] of map) { if (rule._def.tag === 'lazy') { try { rule._def.thunk() } catch { continue } } winners.set(name, rule) } } return [...winners] } export { FUSED_HOST_MODE, FUSED_HOST_ELIDED } from '../cst/host-mode.ts' /** The host mode a fused/composed rule map was built for. Defaults to 'ast'. */ export function fusedHostModeOf(registry: object): HostMode { const m = (registry as Record)[FUSED_HOST_MODE] return m === 'cst' ? 'cst' : 'ast' } /** Whether a fused/composed rule map dropped any direct builder's CST branch. */ export function fusedHostElidedOf(registry: object): boolean { return (registry as Record)[FUSED_HOST_ELIDED] === true } /** * Compose grammars/artifacts into a runnable parser map — the ONLY public * composition entry point. `compose([base, ext, …])`: later entries override * earlier ones by rule name, and because fusion re-binds every reference in one * shared scope, an override reroutes the base's OWN calls too (open recursion). * * Each entry may be a **grammar** (a `rules()` result — a map of combinators, * linkable-ified here) OR an already-compiled **linkable artifact** (what the * macro emits and a package ships). So a package needs no opt-in wrapper to be * composable — `compose([importedGrammar, myRules])` just works. * * The macro compiles `compose([...])` to STATIC fused source (no `new Function`). * Called at runtime (no macro, like `compile()`) it fuses via `new Function`. */ /** A composed parser carries its flattened source pieces (non-enumerable) so it * can be composed AGAIN — `compose([lessGrammar, delta])` where `lessGrammar` is * itself a `compose([...])` result. */ export const COMPOSED_PIECES = Symbol.for('parseman.composedPieces') /** The carried pieces a `compose()`/`composeLeaf()` result holds, or `undefined` when * the value is not a composed grammar. This is what makes a fused grammar analysable: * the pieces are re-lowerable IR even though the fused map itself is only functions. */ export function composedPiecesOf( grammar: Record, ): ReadonlyArray | undefined { const pieces = (grammar as unknown as Record)[COMPOSED_PIECES] return Array.isArray(pieces) ? pieces as ReadonlyArray : undefined } /** * A terminal fused grammar may be used to run a parser, but not as an input to * another composition. Macro `composeLeaf()` uses this for a local semantic * reduction over imported recognition-only IR: the local reductions stay in * their lexical module and therefore never become carried IR. */ const LEAF_COMPOSED = Symbol.for('parseman.leafComposed') /** The composing (outermost) trivia a runtime `compose()` applied — stored so a * LATER `compose([thisResult, …])` that declares no trivia of its own still re-lowers * these rules under the SAME trivia (composing-wins survives re-composition). The * carried IR pieces hold no trivia of their own, so it must be remembered separately. */ const COMPOSED_TRIVIA = Symbol.for('parseman.composedTrivia') /** Final winner map for semantic-coverage tooling. It exists only when every * carried compose piece is re-lowerable IR; opaque precompiled artifacts have no * combinator graph to inspect and therefore deliberately expose no fake map. */ const COMPOSED_COVERAGE_RULES = Symbol.for('parseman.composedCoverageRules') /** The compact IR form a grammar carries instead of its lowered rule source: the * combinator-construction expression, re-lowered here at fuse time. */ export type IRPiece = { ns: string; ir: string; trackLines?: true } function isIRPiece(p: unknown): p is IRPiece { return !!p && typeof p === 'object' && typeof (p as IRPiece).ir === 'string' && typeof (p as IRPiece).ns === 'string' && !('keys' in (p as object)) } /** A `linkable()` artifact — a TABLE piece. Distinguished from a bare IR piece by the * fields only a compiled artifact has (`keys`/`external`), and from a plain `rules()` * map by carrying `ns` at all. */ function isLinkableTable(p: unknown): p is LinkableTable { return !!p && typeof p === 'object' && typeof (p as LinkableTable).ns === 'string' && Array.isArray((p as LinkableTable).keys) } /** Memoize a zero-arg thunk, keeping it LAZY. Used where two diagnostic thunks want * the same carried-IR hydration: the work must not happen when the diagnostic is off, * and must not happen twice when it is on. */ export function once(fn: () => T): () => T { let done = false let value: T return () => { if (!done) { value = fn(); done = true } return value } } /** The re-lowerable carried pieces' rule maps, in compose order — the input to the * gating analysis (`diagnoseGrammar`). An opaque precompiled artifact contributes no * combinator graph, so it is skipped: a hole it would have bound stays unresolved * and its choice stays deferred, never falsely warned. * * Skipping is not the same as having nothing to say. Use `carriedRuleMapsDetailed` * where the skip must be REPORTED — a diagnostic that drops part of the grammar and * then returns a clean result is indistinguishable from one that verified it. */ export function carriedRuleMaps(carried: ReadonlyArray): Array]>> { return carriedRuleMapsDetailed(carried).maps } /** `carriedRuleMaps` plus the pieces it could NOT re-lower, named by namespace and * rule count, so a caller can report exactly how much of the grammar went unseen. */ export function carriedRuleMapsDetailed( carried: ReadonlyArray, ): { maps: Array]>>; opaque: Array<{ ns: string; ruleNames: string[] }> } { const maps: Array]>> = [] const opaque: Array<{ ns: string; ruleNames: string[] }> = [] for (const p of carried) { // A table artifact is recoverable far more often than a source one was: it carries // IR, and failing that the live combinators. Ask for the map before declaring the // piece opaque, or a perfectly analysable grammar is reported as unseen. const rules = ruleMapOfCarried(p) if (rules !== undefined) { maps.push(rules); continue } // NAMED, not anonymous. `keys` is the artifact's rule-name list; the field this // used to read (`ruleFns`) belonged to the source lowering, and reading a missing // one degrades every opaque piece to `` — "reported, but uselessly", // which is the exact failure this reporting exists to prevent. opaque.push({ ns: p.ns, ruleNames: isIRPiece(p) ? [] : [...p.keys] }) } return { maps, opaque } } /** * Recover the override-winner COMBINATOR map behind a `compose()` result, plus the * pieces that could not be recovered. * * A fused map holds rule functions, so any consumer that walks a combinator graph * (gating analysis, the spec/EBNF/railroad model) cannot read it directly. The graph * is not lost, though — `compose()` retains re-lowerable IR — so this is the single * shared recovery both consumers use. Sharing it is the point: two copies of this * logic is how one walker gets fixed and the other silently keeps failing. * * Returns `undefined` when `grammar` is not a composed result. */ export function recoverComposedRules( grammar: Record, ): { rules: Map>; opaque: Array<{ ns: string; ruleNames: string[] }> } | undefined { const carried = composedPiecesOf(grammar) if (carried === undefined) return undefined const { maps, opaque } = carriedRuleMapsDetailed(carried) const rules = new Map>() // Later wins, matching the linker's own fuse semantics. An accessed-but-undefined // `g.X` leaks in as an unresolved lazy — a REFERENCE, not a definition — and must // never shadow the artifact that really defines X. for (const map of maps) for (const [name, rule] of map) { if (rule._def.tag === 'lazy') { try { rule._def.thunk() } catch { continue } } rules.set(name, rule) } return { rules, opaque } } function coverageRulesOf(carried: Array): Record> | undefined { const winners: Record> = {} for (const piece of carried) { if (!isIRPiece(piece)) return undefined const map = evalRuleMapIR(piece.ir) for (const [name, rule] of map) { // An accessed-but-undefined `g.Name` is an external reference, never a // rule definition. Match compose's IR filtering rule exactly. if (rule._def.tag === 'lazy') { try { rule._def.thunk() } catch { continue } } winners[name] = rule } } return winners } /** Return the final override-winner combinator map carried by runtime * `compose()`, or `undefined` when a precompiled opaque artifact participated. * This is intentionally internal: callers must not treat it as a parser API. */ export function composedCoverageRules(grammar: Record): Record> | undefined { return (grammar as Record)[COMPOSED_COVERAGE_RULES] as Record> | undefined } /** Flatten one `compose()` item to its pieces: a prior composed result → its * carried list; an artifact → itself; a grammar (`rules()` map) → linkable-ified. */ function nextComposeNs(used: Set): string { let ns: string do { ns = `_lk${_nsCounter++}_` } while (used.has(ns)) used.add(ns) return ns } /** Flatten one `compose()` item to its RE-LOWERABLE carried items — the form stored * on the composed result so it can be composed AGAIN under a NEW composing trivia. * A grammar (`rules()` map) is carried as compact IR (`{ns, ir}`), NOT baked source, * so a later `compose([thisResult, delta])` re-lowers it with the delta's trivia * (multi-level composing-wins). A prior composed result contributes its OWN carried * items (already IR); a pre-compiled artifact has no source, so it stays baked. */ function itemCarried( item: LinkableTable | Record, used: Set, trivia?: Combinator, // Only reaches the non-serializable fallback below, where the grammar is baked // immediately instead of carried as re-lowerable IR. hostMode?: HostMode, ): Array { const carried = (item as Record)[COMPOSED_PIECES] // A prior composed result (runtime or macro-compiled): its carried list is already // re-lowerable (IR pieces, plus any pre-compiled artifacts). Pass it through so THIS // compose re-lowers it under its own composing trivia. Reserve its namespaces so a // sibling grammar map can't collide with them. if (Array.isArray(carried)) { const items = carried as Array for (const p of items) used.add(p.ns) return items } // A pre-compiled artifact (`linkable()`): a table piece. It ALWAYS carries its IR, // which is what makes table-to-table composition a rule-map merge rather than a // relocation of two encoded programs — so unlike a source artifact it stays // re-lowerable under a new composing trivia. if (isLinkableTable(item)) { used.add(item.ns) return [item] } // A grammar (`rules()` map): carry it as compact IR so a later compose re-lowers it // under ITS trivia. Unserializable → bake now with this compose's trivia (can't // re-lower later; acceptable fallback, mirrors the macro's full-pieces fallback). const map = item as Record> const ns = nextComposeNs(used) // Drop EXTERNAL entries first (same filter as compileLinkable): a `rules()` cache // also holds every ACCESSED-but-undefined `g.X` as an unresolved-lazy entry. Left in, // serializeRuleMap would emit `X: g["X"]` — a self-referential rule that shadows the // sibling artifact defining X and recurses forever. They resolve by name at fuse time. const entries = Object.entries(map).filter(([, val]) => { const d = val._def if (d.tag !== 'lazy') return true try { d.thunk(); return true } catch { return false } }) // Carry this grammar's ambient `scanSkip` (per-piece — opaque units are // dialect-specific, NOT composing-wins) into the IR so a re-lower stamps // `grammarScanSkip` back on. `linkable()`'s fallback reads it off `_meta` directly. const scanSkip = entries .map(([, val]) => (val._meta as { grammarScanSkip?: Combinator[] }).grammarScanSkip) .find(Boolean) const ir = serializeRuleMap(entries, scanSkip) return ir ? [{ ns, ir }] : [linkable(map, ns, trivia, hostMode)] } /** The composed grammar's ambient trivia = the LAST composed item that declares a * grammar-level trivia (via `rules({ trivia }, …)`, which tags `grammarTrivia` on its * rules). Outermost wins: the composing grammar's trivia applies to every fused rule, * including those inherited from a base — so e.g. an SCSS `rw` (which extends Less's) * governs the inherited Less/CSS rules too. `parser`/`noTrivia` still override locally. */ function composingTriviaOf(items: Array>): Combinator | undefined { for (let i = items.length - 1; i >= 0; i--) { const item = items[i] as Record | undefined if (!item || isLinkableTable(item)) continue // A COMPOSED item states its trivia on the stamp rather than on its values (its // rules live in carried IR, which does not carry `_meta`). Skipping such an item // entirely — as this did — loses the trivia of every grammar that reached this // compose through a PRIOR compose. const stamped = (item as Record)[COMPOSED_TRIVIA] as Combinator | undefined if (stamped) return stamped if ((item as Record)[COMPOSED_PIECES]) continue for (const v of Object.values(item)) { const t = (v as Combinator | undefined)?._meta?.grammarTrivia if (t) return t } } return undefined } export function compose( items: Array>, /** * Compile-time host mode for the fused artifact, same meaning as * `compile(g, { hostMode })`. Omit (or `'ast'`) for the eval driver — the fused rules * build through the grammar's own `build` callbacks and carry no positioned-CST * branch. Pass `'cst'` to fuse a SECOND artifact from the same pieces for the linter / * IDE / language-service driver. Two compilations of one grammar, decided here, rather * than one artifact deciding per node on every parse. */ opts?: { hostMode?: HostMode }, ): Record { if (items.some(item => (item as Record)[LEAF_COMPOSED] === true)) { throw new Error('compose: a composeLeaf() result is terminal and cannot be composed again') } const used = new Set() // The composed grammar's ambient trivia comes from the composing grammar itself — // whatever the last piece declared via rules({ trivia }, …). No separate option: // the trivia rides with the grammar that declared it. const trivia = composingTriviaOf(items) // Carried items are RE-LOWERABLE (IR); materialize them ONCE with this compose's // trivia for the now-fuse, but STORE the un-materialized carried list so a later // compose can re-lower it under a different trivia (multi-level composing-wins). const carried = items.flatMap(item => itemCarried(item, used, trivia, opts?.hostMode)) const map = fuseCarried(carried, trivia, opts?.hostMode) Object.defineProperty(map, COMPOSED_PIECES, { value: carried, enumerable: false }) if (trivia) Object.defineProperty(map, COMPOSED_TRIVIA, { value: trivia, enumerable: false }) const coverageRules = coverageRulesOf(carried) if (coverageRules) Object.defineProperty(map, COMPOSED_COVERAGE_RULES, { value: coverageRules, enumerable: false }) return map } /** * Compose a terminal grammar. This is for a leaf parser that overlays local * semantic reductions on reusable recognition rules. * * Under the macro this lowers to STATIC fused source (functions), exactly like * `compose()`. It is still macro-only as a *compiled* artifact: without macro * lowering there is no safe way to keep lexical builders out of carried IR, so it * never falls back to runtime CODEGEN composition. * * Called at runtime (no macro) it returns the INTERPRETED fuse of the same items — * a combinator map, not a map of compiled functions (`fuseInterpreted`), fused lazily * per rule name. * * THE RETURN TYPE IS `Runnable`, NOT `FusedRule`, BECAUSE BOTH PATHS ARE REAL. A macro * build yields fused functions; an un-macro'd call yields combinators. `Runnable` is * already the library's name for "either of those" — it is what `run()` and * `parseDoc()` take — so the declared type is TRUE on both paths and a caller needs no * narrowing to use the result. This used to declare `Record` and * launder the runtime path through an `as unknown as`, which let a caller hold a * combinator map while the type promised compiled functions. * * Do NOT "fix" this by deleting the runtime path. It is load-bearing: the `bench/jess` * harness family and two differential-gate legs (`emit-identity-one`, * `scan-shape-oracle-one`) import un-macro'd grammar modules and depend on this lazy * interpreted fuse, one dialect per process. Whether an un-macro'd `composeLeaf()` * should exist at all is a separate, open owner question — but while the gates depend * on it, it exists, and the type says so. */ export function composeLeaf( items: Array>, ): Record { const pieces = items.flatMap(interpretedPieces) let fused: Record> | undefined const map: Record = {} // LAZY on purpose. A grammar module typically builds SEVERAL leaf grammars over // one shared recognition piece (`cssGrammar`, `cssLineGrammar`, `cssCstGrammar`, // …). An interpreted fuse binds that shared piece IN PLACE, so only one of them // can exist at a time — fusing all of them at import would make merely importing // the module throw. Fusing on first ACCESS means the grammar you actually use // works, and reaching for a second, conflicting one fails loudly at that point. // (`trackLines`/`hostMode` are compile-time distinctions; the interpreter decides // both per parse, so those variants are the same interpreted grammar anyway.) for (const name of ruleNamesOf(pieces)) { Object.defineProperty(map, name, { enumerable: true, configurable: true, get: () => (fused ??= fusePieces(pieces))[name], }) } Object.defineProperty(map, LEAF_COMPOSED, { value: true, enumerable: false }) Object.defineProperty(map, INTERPRETED_PIECES, { value: pieces.filter(p => p.plain).map(p => p.entries), enumerable: false, }) return map as Record } /* ── Interpreted fuse ───────────────────────────────────────────────────────── * * `compose()` fuses by CODEGEN: every piece is compiled to `_r_` functions * dropped into one scope, so a reference resolves by NAME and an override reroutes * the base piece's own calls (open recursion). None of that exists interpreted — * the interpreter runs the combinator graph, and a cross-piece reference is an * ordinary `ref()` placeholder that nobody ever `.define()`d. That is why a * composed grammar could not be run interpreted at all, and why every diagnostic * that must NOT reach codegen (profiling, gating analysis, coverage) had to be * hand-fused in throwaway scripts. * * The interpreted fuse binds those placeholders directly, with the SAME semantics * the compiled fuse gets from name resolution: * - later piece wins per rule name (matching the compiled merge); * - an override REPOINTS the slot every call site already holds, so a base * piece's internal calls reroute too (open recursion); * - the composing grammar's trivia governs every fused rule (`composingTriviaOf`); * - a referenced-but-undefined rule is a fuse-time error, not a parse-time one. * * It is MUTATING by construction: a hole is a shared object, and binding it is the * only way its call sites can see the answer. `repointRef` therefore records what * it changed and refuses a CONFLICTING second bind, so two different fusions over * one shared piece fail loudly instead of silently rewriting each other's parser. * ───────────────────────────────────────────────────────────────────────────── */ /** * A rule slot produced by `ref()` — what `rules()` stores for every rule that is * referenced by name, and for every `g.X` hole a piece leaves for another piece to * fill. `parse`/`_def.thunk` are OWN properties of that object, which is what lets * the interpreted fuse repoint it in place. */ type RefSlot = Combinator & { _def: { tag: 'lazy'; thunk: () => Combinator } define(p: Combinator): void parse(input: string, pos: number, ctx: ParseContext): ParseResult } function isRefSlot(c: Combinator): c is RefSlot { return c._def.tag === 'lazy' && typeof (c as unknown as { define?: unknown }).define === 'function' } /** What a slot resolved to BEFORE any interpreted fuse touched it (`null` = it was * an unbound cross-piece hole). Recorded on first repoint so a LATER fuse computes * its winner from the grammar as authored, never from another fusion's binding. */ const FUSE_ORIGINAL = Symbol.for('parseman.interpretedFuseOriginal') /** What an interpreted fuse repointed this slot at. */ const FUSE_TARGET = Symbol.for('parseman.interpretedFuseTarget') /** The source rule maps behind a `fuseInterpreted()` result, so it can be fused * again — the interpreted mirror of `COMPOSED_PIECES`. */ const INTERPRETED_PIECES = Symbol.for('parseman.interpretedPieces') /** Whether `map` is an interpreted fuse (a combinator map) rather than a compiled * `compose()` result (a map of fused functions). INTERNAL — not re-exported from * `src/index.ts`. A consumer never has to ask this question: what it holds is * whatever the macro built. Diagnostics that fuse interpreted on purpose do. */ export function isInterpretedFuse(map: object): boolean { return Array.isArray((map as Record)[INTERPRETED_PIECES]) } type NamedEntries = ReadonlyArray<[string, Combinator]> /** `plain` = the item was an authored `rules()` map. Only plain maps declare the * composing trivia, mirroring `composingTriviaOf`, which skips artifacts and prior * composed results for exactly the same reason: their trivia was already applied. */ type FusePiece = { entries: NamedEntries; plain: boolean } /** The rule a map entry DEFINES, or `undefined` when it is an external reference * (an accessed-but-undefined `g.X`). Same local-vs-external test `compileLinkable` * and `itemCarried` apply, so the two fuses agree on what a piece contributes. */ function definitionOf(entry: Combinator): Combinator | undefined { if (isRefSlot(entry)) { const original = (entry as unknown as Record)[FUSE_ORIGINAL] if (original !== undefined) return (original as Combinator | null) ?? undefined try { return entry._def.thunk() } catch { return undefined } } if (entry._def.tag === 'lazy') { try { (entry._def as { thunk: () => Combinator }).thunk() } catch { return undefined } } return entry } /** Point `slot` at `target`, updating every call site that holds it. Mirrors * `ref().define()`'s metadata propagation; refuses to overwrite a binding a * DIFFERENT fusion already made (see the mutation note above). */ function repointRef(slot: RefSlot, target: Combinator, name: string): void { const tagged = slot as unknown as Record const bound = tagged[FUSE_TARGET] as Combinator | undefined const original = FUSE_ORIGINAL in tagged ? tagged[FUSE_ORIGINAL] as Combinator | null : (() => { try { return slot._def.thunk() } catch { return null } })() if ((bound ?? original) === target) return if (bound !== undefined) { throw new Error( `fuseInterpreted: rule "${name}" is already bound by a DIFFERENT interpreted fusion of the same grammar piece. ` + `An interpreted fuse binds the shared placeholder objects in place, so two fusions cannot share a piece — ` + `build a fresh instance of the piece (call its rules() factory again, or import the module under a distinct specifier) for the second fusion.`, ) } if (!(FUSE_ORIGINAL in tagged)) Object.defineProperty(slot, FUSE_ORIGINAL, { value: original, enumerable: false }) Object.defineProperty(slot, FUSE_TARGET, { value: target, enumerable: false }) slot._def.thunk = () => target slot.parse = (input, pos, ctx) => target.parse(input, pos, ctx) const meta = slot._meta meta.firstSet = target._meta.firstSet meta.canMatchNewline = target._meta.canMatchNewline meta.isTrivia = target._meta.isTrivia if (target._meta.triviaKindLabels !== undefined) meta.triviaKindLabels = target._meta.triviaKindLabels else (meta as { triviaKindLabels: readonly string[] | undefined }).triviaKindLabels = undefined if (target._meta.disjoint !== undefined) meta.disjoint = target._meta.disjoint else (meta as { disjoint: boolean | undefined }).disjoint = undefined } /** Flatten one `fuseInterpreted()` item to the rule maps it contributes, in order. */ function interpretedPieces(item: LinkableTable | Record): FusePiece[] { const fused = (item as Record)[INTERPRETED_PIECES] if (Array.isArray(fused)) return (fused as NamedEntries[]).map(entries => ({ entries, plain: false })) const carried = composedPiecesOf(item as Record) if (carried !== undefined) { // A compiled `compose()` result. Its carried IR re-lowers to combinators, but a // piece that arrived already COMPILED has no combinator graph at all — fusing // around it would silently drop its rules, which is the one failure mode a // diagnostic must never have. const { maps, opaque } = carriedRuleMapsDetailed(carried) if (opaque.length > 0) { throw new Error( `fuseInterpreted: cannot interpret a composed grammar containing precompiled artifact(s) ` + `${opaque.map(o => `"${o.ns}" (${o.ruleNames.length} rules)`).join(', ')} — they carry compiled functions, not a combinator graph. ` + `Pass the source grammars (the same items you passed to compose()) instead.`, ) } return maps.map(entries => ({ entries, plain: false })) } if (isLinkableTable(item)) { // A table piece DOES carry its IR, so this is recoverable rather than fatal: hydrate // the combinator graph back out of it and interpret that. const rules = ruleMapOfCarried(item) if (rules === undefined) { throw new Error('fuseInterpreted: a precompiled linkable artifact with no carried IR has no combinator graph to interpret; pass the source grammar (a rules() map) instead') } return [{ entries: rules, plain: false }] } return [{ entries: Object.entries(item as Record>), plain: true }] } /** * Materialize a composition as a RUNNABLE INTERPRETED rule map — the interpreted * counterpart of `compose()`, with identical fuse semantics (later piece wins, * override reroutes the base's own calls, composing trivia governs every rule). * No codegen, no `new Function`, no macro build step: the result is a plain map of * combinators that `run()` / `parseDoc()` accept exactly like a fused map. * * This is what diagnostics and profiling run against — they must stay in * interpreted mode, and before this they could not see a composed grammar at all. * * Items are the SAME items `compose()`/`composeLeaf()` take: `rules()` maps * (the intended input), a prior `fuseInterpreted()` result, or a runtime * `compose()` result (re-lowered from its carried IR — note that carried IR cannot * materialize direct `node()` builders, so prefer the source maps). A precompiled * `linkable()` artifact is rejected: it has no combinator graph. * * MUTATION: binding a cross-piece hole rewrites the shared placeholder object every * call site already holds — that IS how an override reaches a base piece's own * calls. A second, DIFFERENT fusion over the same piece objects therefore throws * rather than silently rewriting the first one's parser. */ export function fuseInterpreted( items: Array>, opts?: { hostMode?: HostMode }, ): Record> { return fusePieces(items.flatMap(interpretedPieces), opts) } /** The rule names a fusion of these pieces defines, in winner order — computable * WITHOUT binding anything, which is what lets `composeLeaf()` expose its key set * before it fuses. */ function ruleNamesOf(pieces: FusePiece[]): string[] { const names = new Set() for (const piece of pieces) { for (const [name, value] of piece.entries) if (definitionOf(value) !== undefined) names.add(name) } return [...names] } function fusePieces( pieces: FusePiece[], opts?: { hostMode?: HostMode }, ): Record> { // Composing-wins trivia, read exactly as `composingTriviaOf` reads it for compose(): // the LAST authored grammar that declared `rules({ trivia }, …)`. let trivia: Combinator | undefined for (let i = pieces.length - 1; i >= 0 && trivia === undefined; i--) { if (!pieces[i]!.plain) continue for (const [, rule] of pieces[i]!.entries) { const t = rule._meta.grammarTrivia if (t) { trivia = t; break } } } // Winner per rule name — later piece wins, matching `fuseRules`. The winner is the // DEFINITION, never the slot holding it: a slot can be repointed, and a chain // through one would make an override of X reroute into itself. const winner = new Map>() const entry = new Map>() for (const piece of pieces) { for (const [name, value] of piece.entries) { const def = definitionOf(value) if (def === undefined) continue winner.set(name, def) entry.set(name, value) } } // Bind every hole (and repoint every overridden slot) before anything runs. const missing = new Set() for (const piece of pieces) { for (const [name, value] of piece.entries) { if (!isRefSlot(value)) continue const target = winner.get(name) if (target === undefined) { missing.add(name); continue } repointRef(value, target, name) } } if (missing.size > 0) throw new Error(missingRuleMessage(pieces, missing)) const out: Record> = {} for (const [name, value] of entry) { out[name] = value const meta = value._meta as { isTrivia: boolean grammarTrivia?: Combinator grammarHostMode?: HostMode } // A trivia rule must never carry the ambient trivia (it would recursively skip // trivia within itself) — the same guard `compileLinkable` applies per rule. if (trivia !== undefined && !meta.isTrivia) meta.grammarTrivia = trivia // Host mode is PER PIECE, exactly as `compileLinkable` resolves it: an explicit // option wins, otherwise the owning piece's own `rules({ hostMode })` stamp. if (opts?.hostMode !== undefined && !meta.isTrivia) { if (opts.hostMode === 'cst') meta.grammarHostMode = 'cst' else (meta as { grammarHostMode: HostMode | undefined }).grammarHostMode = undefined } } Object.defineProperty(out, INTERPRETED_PIECES, { value: pieces.filter(p => p.plain).map(p => p.entries), enumerable: false, }) return out } /** Name-closure failure, reported the way the compiled fuse reports it: which rule * referenced the missing name. Computed only on the error path. */ function missingRuleMessage(pieces: FusePiece[], missing: Set): string { for (const piece of pieces) { const deps = ruleDependencies(piece.entries.filter(([, v]) => definitionOf(v) !== undefined)) for (const [name, ds] of deps) { for (const d of ds) if (missing.has(d)) return `fuseInterpreted: rule "${name}" references missing rule "${d}"` } } return `fuseInterpreted: missing rule(s) ${[...missing].map(n => `"${n}"`).join(', ')}` }