import { type CachedIngest, type Mind, type Store } from "../../src/index.js"; import type { TrainingItem } from "./items.js"; import { type HttpOptions } from "./http.js"; import type { ReadContext } from "./readers.js"; import { type SavedProgress } from "./progress.js"; import { Progress, type ProgState } from "./ui.js"; /** A single process-wide abort signal. SIGINT/SIGTERM aborts it, which cancels * every in-flight fetch immediately (instead of waiting out a slow socket), so * Ctrl+C is responsive even mid-download. The deposit loop also polls it to * stop cleanly at the next item boundary, leaving the store consistent. */ export declare const shutdown: AbortController; /** The totals a run accumulates. Mutable BY DESIGN — see the file header. */ export interface Counters { depositCount: number; trainedContentBytes: number; totalBytesProcessed: number; totalCorpusBytes: number; langTally: Record; /** Rows a BUDGETED corpus has already taken, by corpus id, counting only * units it finished. A budget is a property of the STORE, not of a run, so * it has to survive a resume — see the budget notes in stage.ts. Absent from * an older store, which reads as 0 and reproduces the previous behaviour. */ rowsTaken: Record; } export interface TrainCtx { readonly store: Store; readonly mind: Mind; readonly ci: CachedIngest; readonly progress: Progress; readonly state: ProgState; readonly counters: Counters; /** Network options for a LISTING call: cancellable, and its throttle waits * are surfaced into the run log. Downloads and HEAD probes deliberately do * NOT carry the notifier — they wait silently, as they always have. */ readonly http: HttpOptions; /** Set once a stop has been requested (a signal, or the MAX_MB cap). Every * stage checks it at its file and item boundaries. */ stopRequested: boolean; stopReason: string; /** Repaint the panel. `force` bypasses the frame-rate limiter. */ tick(force?: boolean): void; /** Offer an item to the reservoir behind the checkpoint recall box. */ sample(it: TrainingItem): void; /** The per-deposit gate: counts, checkpoints, and returns false to stop. */ onDeposit(contentBytes: number): Promise; /** Bind a reader to this run, for the unit recorded as `unitId`, resuming at * `startRow`. `shouldStop` is a stage BUDGET; the MAX_MB cap and the * shutdown signal reach the reader by other routes. * * Naming the unit is what makes the cursor meaningful: a cursor is only ever * applied to the unit it was taken from. */ readCtx(opts: { unitId: string; /** Which corpus these deposits are tallied to. */ corpusId: string; startRow?: number; shouldStop?: () => boolean; /** Live budget count for this corpus, for the cursor snapshot only. */ rowsTakenNow?: () => number; }): ReadContext; /** The durable position a previous run reached inside a unit it did not * finish, or null. A stage applies it only to the matching unit. */ readonly resumeCursor: UnitCursor | null; /** Reuse a cached copy, else download into the cache. Null on failure. * `cached` says which happened — the stages disagree about whether a file * they did not download is theirs to delete afterwards, so the answer has * to reach the caller rather than being decided here. */ acquire(url: string, destName: string, label: string): Promise; /** Persist the resume record: completed units, counters, per-corpus tally, * and the in-flight unit's cursor. `unitDone` CLEARS the cursor — the unit * is recorded in `completedFiles` now, so a position inside it is meaningless * and must not be left behind for the next run to apply. */ persist(completedFiles: string[], unitDone?: boolean): Promise; /** Restore counters and the tally from the store, and announce the resume. */ restore(): Promise; /** Final checkpoint, summary line, and exit. */ finish(why: string): Promise; } /** A durable position inside a unit that is not finished, together with the * counters that describe exactly the data behind it. * * The counters travel WITH the position for one reason: they have to agree. A * resume that restored the store's running totals but re-read the unit from the * top counted the same deposits twice, and every interruption inflated the * figures again — measured at +77% (16,000 examples reported as 28,283) after * six interruptions of one corpus. Restoring the pair together makes the * reported numbers describe the store, however many times it was interrupted. * * `rows` is what the reader counts, so its unit is reader-specific — non-blank * lines for `lines`, array index for `jsonArray`, absolute file row for * `parquet`. That is safe because a corpus never changes reader, and it is why * a cursor names the unit it came from. */ export interface UnitCursor { unitId: string; rows: number; depositCount: number; trainedContentBytes: number; totalBytesProcessed: number; langTally: Record; rowsTaken: Record; } /** The outcome of {@link TrainCtx.acquire}. */ export interface AcquiredFile { path: string; /** True when the file was already in the cache and nothing was fetched. */ cached: boolean; } export interface RuntimeOptions { store: Store; mind: Mind; ci: CachedIngest; /** Names the curriculum in the panel header. */ title: string; } export declare function createRuntime(opts: RuntimeOptions): TrainCtx;