import { ByteSource } from "./byte_source.js"; import StepParser, { ParseResult, StepHeader, StepIndexEntry } from "./step_parser.js"; /** * Diagnostics from a streaming index build — how hard the moving window had * to work, so a pool-size sweep can compare cost against a resident parse. */ export interface StreamingIndexStats { /** Target window size in bytes (the pool). */ pool: number; /** Physical window buffer size actually used (grows past `pool` only if a * single record exceeds the pool). */ windowBytes: number; /** Number of times the window slid forward. */ slides: number; /** Largest single top-level record observed, in bytes. */ maxRecordLen: number; /** Total bytes read from the source (≈ file size + re-reads on growth). */ bytesRead: number; } /** * The output of a streaming index build: the same element index a resident * `parseDataBlock` produces, plus the header and window diagnostics. */ export interface StreamingIndexResult { header: StepHeader; elements: StepIndexEntry[]; result: ParseResult; stats: StreamingIndexStats; } /** * Build the entity index by streaming a `ByteSource` through a fixed-size * moving window, instead of parsing one resident buffer. The parse loop is * byte-for-byte the resident one (`StepParser.parseDataBlockStreamed` drives * the same generator); this coordinator only owns the window: it fills it, * parses the header from the first fill, then slides the window forward at * top-level record boundaries as the parser advances. * * Because the parser records file-absolute addresses and its rewind stack is * empty at every top-level boundary, sliding there is transparent: the * unconsumed tail is copied to the front, fresh bytes are appended, and the * buffer is rebased so `address` keeps its file-absolute value. Peak JS heap * for the index build is therefore `window + index columns`, independent of * file size. * * The window slides only once the cursor passes `pool / 2`, bounding the * memmove frequency; a record up to `pool / 2` bytes always fits after a * slide. If a single record exceeds that (never on the current corpus, whose * largest STEP record is ~25 KB), the whole parse restarts from the * beginning with the window doubled, and repeats until every record fits — * correctness over the pathological case, at the cost of a re-scan. (M1's * production path will instead grow in place / restart from the last * boundary; from-scratch keeps the spike simple.) * * @param source The byte source. * @param parser The STEP parser (typed to the schema). * @param pool Target window size in bytes. * @return {StreamingIndexResult} The index, header, result and diagnostics. */ export declare function buildIndexStreaming(source: ByteSource, parser: StepParser, pool: number): StreamingIndexResult; //# sourceMappingURL=streaming_index_builder.d.ts.map