import type { CachedIngest } from "../../src/index.js";
import { type TrainingItem } from "./items.js";
/** Turn ONE raw row into deposits, or null/[] when the row carries nothing
 *  usable. Pure: no I/O, no counters, no logging. */
export type RowAdapter = (row: unknown) => TrainingItem[] | null;
/** What a read produced. `skipped` and `unusable` are deliberately SEPARATE:
 *  a line that failed to parse is a defect in the file, while a row the adapter
 *  declined is a normal, expected outcome for a corpus being filtered (oasst2
 *  drops every single-turn tree by design). Collapsing them — as the two
 *  original line readers each did, in opposite directions — makes one of the
 *  two log lines a lie. */
export interface FileResult {
    examples: number;
    rowsUsed: number;
    skipped: number;
    unusable: number;
    stopped: boolean;
}
/** Everything a reader needs from the run: where to deposit, what to count,
 *  and how to be stopped. */
export interface ReadContext {
    ci: CachedIngest;
    /** Called once per deposit with its UTF-8 content size. Returns false to
     *  stop the read (the MAX_MB cap, or a pending shutdown). */
    onExample: (contentBytes: number) => Promise<boolean>;
    /** Feeds the reservoir behind the checkpoint recall box. */
    sample: (it: TrainingItem) => void;
    signal: AbortSignal;
    /** A stage-level budget. Checked per row and before each Parquet batch is
     *  decoded — a budget must STOP the read rather than reject rows: left to
     *  reject, a budgeted stage still DECODES every remaining row-group (143,346
     *  rows of one 86.7 MB SODA shard) and reports them as "unusable" when
     *  nothing was wrong with them, which is a lie in the run log.
     *
     *  Measured honestly: on that shard the wall time did NOT improve (2m 35s ->
     *  2m 37s), because a budgeted run is dominated by depositing the rows it DID
     *  take, not by scanning past the ones it did not. The win here is a truthful
     *  log and the CPU/allocation of ~143k skipped row decodes, not elapsed time.
     *  A larger shard past a small budget is where the decode cost would show. */
    shouldStop?: () => boolean;
    /** Rows to SKIP before depositing anything — the position a previous run
     *  reached, taken from the durable cursor (see runtime.ts). Resume used to
     *  mean "re-read this unit from the top", which was safe but re-deposited
     *  everything already stored and counted it a second time; the store then
     *  reported up to 77% more examples than it held.
     *
     *  Skipping is only sound because the cursor is written in the SAME COMMIT
     *  that flushes the deposits it counts, so a row before the cursor is
     *  necessarily durable. A skipped row is neither parsed nor counted, so a
     *  resumed read's log line describes what THIS read did and nothing else. */
    startRow?: number;
    /** "Row `rows` is FULLY dealt with" — every item it produced is deposited, or
     *  it produced none. Called at ROW BOUNDARIES ONLY, and never for a row the
     *  read stopped in the middle of.
     *
     *  That boundary is the whole point. A checkpoint fires per DEPOSIT, and a row
     *  can produce many (2Wiki emits ~5 facts per row, a dialogue one per turn),
     *  so a position recorded when a row STARTS would mark it consumed while some
     *  of its items were still unwritten — and the resume would skip them. Data
     *  loss, silently. Advancing only here means the worst case is re-depositing
     *  one row, which is idempotent and counted once. */
    onRowDone?: (rows: number) => void;
}
/** A reader: read `filePath`, deposit every row `toItems` accepts. */
export type Reader = (filePath: string, toItems: RowAdapter, rc: ReadContext) => Promise<FileResult>;
/** Deposit a row's items: an experience via ingest(text), an episode via
 *  ingest(context, continuation). After each, the per-example callback receives
 *  the item's UTF-8 content size — the quantity the scaling suite
 *  (14-scaling.test.mjs) reports as a constant KB/s — then gates the global
 *  example count and checkpointing (returns false to stop). */
export declare function ingestItems(ci: CachedIngest, items: TrainingItem[], onItem: (contentBytes: number) => Promise<boolean>, sample?: (it: TrainingItem) => void): Promise<boolean>;
/** Newline-delimited JSON, optionally gzipped.
 *
 *  ONE reader serves both the plain JSONL sources and the gzipped oasst2 tree
 *  dump: the only difference between them is a `DecompressionStream("gzip")` in
 *  the pipeline, and duplicating an 80-line splitter to express that was how
 *  the two copies drifted apart in the first place.
 *
 *  Lines are split without buffering the whole file OR an unbounded line: a
 *  record longer than `maxLineChars` is dropped (counted `skipped`) and the
 *  stream continues at the next newline, so a corrupt record can never exhaust
 *  memory or abort a good file. */
export declare const lines: (opts: {
    gzip?: boolean;
    maxLineChars: number;
}) => Reader;
/** A whole-file JSON ARRAY of rows. The arrays this reads are small enough
 *  (~16 MB) to parse whole; a huge file would be rejected by the cache ceiling
 *  long before this. */
export declare const jsonArray: () => Reader;
/** How many rows to materialise in one read from a row-group of `rgRows` rows
 *  occupying `groupBytes` uncompressed bytes, under a `budgetBytes` target.
 *
 *  The group's own footer statistics give the mean row width, so the batch
 *  follows the CORPUS's row size rather than the writer's layout: wide rows
 *  (SODA carries a whole dialogue per row) batch smaller than narrow ones at
 *  the same memory cost. Never exceeds the group — a batch is a subdivision of
 *  a group, never a span across two, because `parquetReadObjects` is given an
 *  absolute row range and column chunks are per-group. Never returns 0, or the
 *  read loop could not advance.
 *
 *  A writer that omits `total_byte_size` yields `groupBytes <= 0`; the batch is
 *  then the whole group, which is exactly the behaviour this replaced. That
 *  fallback is safe for every file we read today (all three report it) and
 *  degrades to the old memory profile rather than to a wrong result. */
export declare function parquetBatchRows(rgRows: number, groupBytes: number, budgetBytes: number): number;
/** Parquet, read in bounded row batches with hyparquet (+Snappy from
 *  hyparquet-compressors) over a web-standard Blob byte source. At most
 *  `batchBytes` of source rows are materialised at a time, so neither a
 *  multi-hundred-MB file nor a file written as ONE giant row-group loads whole
 *  into memory.
 *
 *  Batching also makes a single-group file INTERRUPTIBLE: the abort check runs
 *  per batch, where before a 1.19M-row group could not be cancelled at all.
 *
 *  `columns` PROJECTS the read down to the columns the adapter actually uses.
 *  That is not only a memory economy: 2Wiki's `context` column holds the
 *  Wikipedia prose the adapter exists to avoid depositing, and naming the
 *  columns makes that exclusion structural — the bytes are never decoded at
 *  all — in the same way reading only `utterances[].text` structurally excludes
 *  Taskmaster's `instructions` scaffolding. Absent ⇒ every column, as before. */
export declare const parquet: (opts?: {
    batchBytes?: number;
    columns?: string[];
}) => Reader;