/** * Core rendering pipeline: parse tree → output string. * * The processor supports the full sequence engine from * docs/vision/02-sequences.md: * - attribute extraction (count, seed, inject, local) * - declarations, materialized * before the main loop; values looked up per iteration via inject * interpolation (`${{Name}}` resolves to the i-th cell of the * corresponding sequence; `${{_count}}` is the 1-based iteration index) * - fixture blocks inside : , , , * , , , , * * - iteration over 's lines for each of `count` cards * - raw-text emission with interpolation * - dispatch via the template * registry and data packs (person.*, date.*, and pack addresses like * russia.tax.inn_org, common.id.uuid — including the former "preset" * algorithmic generators, now editable pack data) * - dispatch * - `if` expressions on , , and inline * - sequence materialization for text, file, template, number, * regex, advanced_regex, symbol, date, increment, decrement, compute, and * compound sequence fields * - inline dispatch for file, template, number, regex, * advanced_regex without weighted choices, symbol, date, increment, and decrement * - presentation / inside and nested inside * , with Hamilton-exact percent distribution * * Ordering rules and newline placement match the 2022-2024 prototype * exactly to preserve byte-for-byte output on all pre-existing fixtures. */ import type { AssertSpec } from '../sequence/assert.js'; import { type EachInfo } from './each.js'; import type { DocumentContext, OpenCloseElementContext } from '../generated/TDCParser.js'; import type { PackRegistry } from '../data-pack/index.js'; import { type PoolTables } from '../sequence/pool.js'; import type { UniqArrangement, UniqPlan } from '../sequence/build.js'; import type { SequenceRegistry, SequenceSpec } from '../sequence/index.js'; import type { AttrMap } from './attrs.js'; export { describeEngineChoice, ENGINE_LABELS, type EngineChoice, explainRenderEngine, needsExactEngine, resolveRenderEngine, specsUseBareParent, specsUseDerivedColumn, specsUseHttp, } from './engine-route.js'; export interface RenderOptions { readonly seed?: string; readonly count?: number; readonly locale?: string; /** * The locale used when the config does not name one — the project's * `tdcv2.config.json`, not the command line. * * `locale` OVERRIDES what `` declares; this only fills in when * `` declares nothing. Folding a project default into `locale` made a * config that says `local="ru"` produce English, silently, which is the worst * shape a bug can take: the run succeeds and the data is wrong. */ readonly defaultLocale?: string; /** * Milliseconds since epoch, used by date-based templates as "now". * Defaults to real time; tests pass a fixed value for determinism. */ readonly now?: number; /** Legacy: force the streaming (Engine 2) engine. Prefer `mode`. */ readonly stream?: boolean | undefined; /** User mode: "memory" (Engine 1) or "disk" (Engine 2/3 auto). Overrides ``. */ readonly mode?: EngineMode | undefined; /** Advanced: force a specific engine (1/2/3). Highest precedence; used by tests/parallel. */ readonly engine?: EngineId | undefined; /** * A uniq arrangement worked out elsewhere, so this render does not work it * out again — see `SequenceBuildOptions.uniqPlan`. What lets one thread do * the analysis and several render ranges of the answer. */ readonly uniqPlan?: UniqPlan | undefined; /** Called with each uniq group's arrangement as it is worked out, so it can be passed on. */ readonly onUniqPlan?: ((group: string, arrangement: UniqArrangement) => void) | undefined; /** Called as the run advances — see `RenderProgress`. */ readonly onProgress?: ((progress: RenderProgress) => void) | undefined; /** Build the uniq members without making them unique — for the threads that compute the scan. */ readonly skipEnvUniq?: true; /** * Throw the uniq-repair refusal instead of taking the in-memory fallback. * * For the parallel coordinator's planning render. The fallback is right for * a single-threaded `mode="disk"` run — one table, one cost. Taken inside a * coordinator it repeats per worker: every one of them builds the whole * table to render its own slice. The coordinator sets this, catches the * typed refusal, and renders the run single-threaded instead. */ readonly refuseUniqFallback?: true; /** Fingerprint piles for the uniq hunt; auto-set for the disk engines. See `fingerprint.ts`. */ readonly uniqFingerprintBuckets?: number | undefined; /** Sorted fingerprint files computed elsewhere, per uniq group — the parallel coordinator's. */ readonly uniqFingerprintFiles?: Readonly> | undefined; /** Colliding rows already found elsewhere, per uniq group — Engine 4's pile workers. */ readonly uniqExcess?: Readonly> | undefined; /** Base directory for relative `src` paths. Defaults to process cwd. */ readonly baseDir?: string | undefined; /** Extra folders searched by `src="@data/..."` and relative file sources. */ readonly dataPaths?: readonly string[] | undefined; /** * Original DSL source text. When provided, render-time errors carry it * so the CLI/formatter can render a code snippet with a caret, matching * parser/validator diagnostics. Optional — without it, render errors * still carry line/column, just no source frame. */ readonly source?: string | undefined; /** * Loaded data packs. Their dotted addresses resolve as `template` * values (a uniform pick from the pack's list), in addition to the * builtin template paths. */ readonly packs?: PackRegistry | undefined; /** * Render only rows `[start, end)` instead of all `count` rows. Used by the * parallel CLI: each worker renders one contiguous range, and concatenating * the ranges in order reproduces the full output BYTE-FOR-BYTE. `before` * fixtures are emitted only when `start === 0`, `after` only when * `end === count`, and the block delimiter still keys off the GLOBAL last * row — so a range's bytes are exactly what a full render would emit there. * Only sound in stream mode with no inline content generators (the render * PRNG is sequential, not seekable); the parallel layer enforces that. */ readonly range?: { readonly start: number; readonly end: number; } | undefined; } interface EnvConfig { readonly count: number; readonly seed: string; readonly locale: string; readonly inject: string; readonly before: OpenCloseElementContext | undefined; readonly after: OpenCloseElementContext | undefined; readonly beforeBlock: OpenCloseElementContext | undefined; readonly afterBlock: OpenCloseElementContext | undefined; readonly delimiterBlock: OpenCloseElementContext | undefined; readonly beforeLine: OpenCloseElementContext | undefined; readonly afterLine: OpenCloseElementContext | undefined; readonly delimiterLine: OpenCloseElementContext | undefined; readonly regexMaxLength: number; /** How the engine was selected — a user mode ("memory"/"disk") or a forced id. */ readonly engineSelection: EngineSelection; /** * `` — rows computed strictly in order so `prev()` has * a previous row. Carried separately from `engineSelection` because that * records WHICH engine, and this records WHY: `engine="1"` picks the same one * without promising the ordering `prev()` depends on. */ readonly sequential: boolean; } /** * Internal engine id: * 1 = in-memory (Engine 1): exact everything, fast, bounded by RAM. * 2 = streaming (Engine 2): lazy, disk-friendly, multi-threaded; exact except * percent-weighted `uniq`, which it refuses. * 3 = exact-on-disk (Engine 3): everything Engine 1 does, at scale, on disk. * * Engines 2 and 3 hunt uniq duplicates by FINGERPRINT on large runs: 13-byte * hash records in piles, sorted as packed integers, verified against the true * tuples, and queried by binary search on disk — no in-memory structure over * the run. Grew up as experimental engines 4 and 5; measured at 20 GB under a * heap capped at 1 GB before it was made the standard path. */ export type EngineId = 1 | 2 | 3; /** * A moment of a running generation, for whoever is watching. * * A big run is silent for minutes at a time — the uniq scan alone was 40 * minutes of a 20 GB file — and a silent hour is indistinguishable from a hung * one. TDC Studio sits on exactly that question, and so does anyone driving * the engine from a service. The engine reports through a callback because it * cannot know who is asking: the CLI turns it into a status file, Studio into * a progress bar, a test into an array. * * `done` counts rows for the scans and the render, piles for the sort. Reports * are throttled at the source (about one per half-percent of a phase), so the * callback can be honest work without slowing the run. */ export interface RenderProgress { readonly phase: 'uniq-scan' | 'uniq-sort' | 'uniq-repair' | 'render'; readonly done: number; readonly total: number; } /** * User-facing mode. `memory` → Engine 1. `disk` → work off disk: TDC picks the * fastest disk engine the config allows — Engine 2 normally, Engine 3 when the * config needs exact percentages AND uniqueness together (which Engine 2 can't * do lazily). The user never has to know which; the choice is deterministic * (by config content, never by hardware), so it stays reproducible. */ export type EngineMode = 'memory' | 'disk'; /** * Either a user mode, or a forced engine id (advanced/`--engine`, for tests). * `why` says where the choice came from, for `check` and the CLI's notes. */ export type EngineSelection = { readonly mode: EngineMode; readonly why?: string; } | { readonly forced: EngineId; readonly why?: string; }; /** * Render a parsed TDC document to its output string. Deterministic on * (document, seed, now, locale, count). * * This is a thin wrapper over `renderStream` that collects every * yielded chunk into a single string. Callers that need a streaming * consumer (bounded memory regardless of `count`) should iterate * `renderStream` directly — see `TDC.toIterator()`. */ export declare function render(document: DocumentContext, options?: RenderOptions): string; /** * Build a {@link PreparedRender} on the async path: the sequence registry is * built synchronously (http generators leaving placeholder columns), then every * http column is filled from its service. Only configs with a `type="http"` * generator need this — everything else renders identically through the sync * `prepareRender`. */ export declare function prepareRenderAsync(document: DocumentContext, options?: RenderOptions): Promise; /** * Async counterpart of {@link render}. Required for a config containing a * `type="http"` generator; identical output to `render` for any config without * one. The result is non-deterministic when http is used — re-running does not * reproduce — which is why such a config never reaches the reproducible * streaming path. */ export declare function renderAsync(document: DocumentContext, options?: RenderOptions): Promise; /** * True if the config has any INLINE render-time `` or `` — one * placed directly in a ``/fixture line rather than inside a * ``. Those draw from the render loop's SEQUENTIAL prng (row 0, 1, 2… * in order), so they are not seekable and a row range can't be rendered * independently. Sequences (under ``) are seekable in stream mode and * don't count. Used by the parallel CLI to refuse configs it can't split * without changing the output. */ /** * Whether the config asks for uniqueness across the whole run. * * A `uniq="true"` sequence or a `` group is a promise about the finished * dataset, not about any one row. A worker rendering rows 0..99 can only make * its own hundred distinct, and four workers would each do that and duplicate * across the boundaries — silently, since every range looks correct on its own. * So `--jobs` refuses rather than split it. */ export declare function hasUniqueness(document: DocumentContext): boolean; /** * Uniqueness that a row range cannot be rendered under, even given the * arrangement worked out for the whole file. * * An env-level `` is not one: it rearranges whole columns, the answer is * a small map of which rows moved where, and a thread handed that map resolves * its own rows without knowing what the others hold. `uniq="true"` on a single * sequence is: it rearranges the gens INSIDE one compound column, which no * per-row resolver reproduces. */ /** * The env-level `` groups' members, as the config declares them. * * For the coordinator that hands the scan out: it has to name each group the * same way the engine does, since the engine looks its pre-computed records up * by that name. The engine drops a member that produced no sequence at all, so * a name can in principle differ — and then the records are simply not found * and the engine computes the scan itself. Slower, never wrong. */ export declare function envUniqGroupsOf(document: DocumentContext): readonly (readonly string[])[]; /** * Refuse a uniq group that cannot produce the rows asked of it — cheaply, and * without generating anything. * * The renderer does this before choosing an engine. The parallel coordinator * needs it before choosing to spawn threads: an impossible config used to set * every core computing tuples for an arrangement that could never exist, and * only refuse once they had finished. */ export declare function checkUniqFeasible(document: DocumentContext, count: number): void; export declare function hasUnsplittableUniqueness(document: DocumentContext): boolean; /** * Whether the config carries an ``. * * A per-row assertion names the FIRST failing row, and "first" is a statement * about the whole run. Workers each own a range and each stop at their own first * failure, so the row a reader is shown would be whichever thread got there — * a different number on the same config and the same seed. So a config with one * runs single-threaded, and the failure it reports is the true first row. */ export declare function hasPerRowAssertion(document: DocumentContext): boolean; export declare function hasInlineRenderGenerators(document: DocumentContext): boolean; /** Everything a consumer needs after the sequences are built. */ export interface PreparedRender { readonly tdc: OpenCloseElementContext; readonly blockEl: OpenCloseElementContext; readonly env: EnvConfig; readonly registry: SequenceRegistry; readonly now: number; /** The sequential PRNG, already advanced by sequence materialisation. */ readonly prng: () => number; /** Repeating sequences by name, for ``. */ readonly eachInfo: ReadonlyMap; /** The pools, for ``. */ readonly pools: PoolTables; /** The `` specs, in declaration order — what the object API reports. */ readonly sequenceSpecs: readonly SequenceSpec[]; /** `` — answered row by row, in the loop below. */ readonly perRowAsserts: readonly AssertSpec[]; } /** * Parse the document and build the sequence registry — everything that happens * before output is produced. Shared by the text renderer and the columnar * (typed/Parquet) writer so both see exactly the same data for a given seed. */ export declare function prepareRender(document: DocumentContext, options?: RenderOptions, deferHttp?: boolean): PreparedRender; export declare function renderStream(document: DocumentContext, options?: RenderOptions): Generator; /** * The output half of a render: turn an already-built {@link PreparedRender} into * text chunks. Split out so the sync (`renderStream`) and async (`renderAsync`) * paths share it — only the registry build differs between them, never the * assembly. */ export declare function streamFromPrepared(prepared: PreparedRender, options?: RenderOptions): Generator; /** * Resolve the engine SELECTION (mode or forced id), most-specific wins: * 1. `options.engine` (advanced / `--engine N`) → forced id * 2. `options.stream` (legacy `--stream`) → forced Engine 2 * 3. `options.mode` (`--mode`/`--disk`) → mode * 4. `` → forced id * 5. `` → forced Engine 2 (legacy alias) * 6. `` → mode * 7. default → disk * `disk` is the default: file generation always goes through the bounded-memory * engines (streaming / exact-on-disk), auto-picked from the config. The in-memory * engine (Engine 1) is no longer a default — it survives as an internal backstop * (the object API, the Engine-3 fallback, and an explicit `mode="memory"` / * `engine="1"` escape hatch). A user MODE (`disk`) is resolved to a concrete * engine later, from the config — deterministically, never from hardware. */ export declare function resolveEngineSelection(attrs: AttrMap, options: RenderOptions): EngineSelection; //# sourceMappingURL=render.d.ts.map