/**
 * @copyright Sister Software
 * @license AGPL-3.0
 * @author Teffen Ellis, et al.
 *
 *   Assembles a corpus manifest that keeps every file of a base corpus and adds new parquet files.
 */
import { type PathBuilderLike } from "path-ts";
import type { SplitName } from "#utils/split";
type ManifestFile = Record<string, unknown> & {
    path: string;
    source?: string;
};
/**
 * The legacy manifest keys for the file list and the rows per file.
 *
 * Earlier corpora are immutable and still use these keys, so readers accept both
 * spellings and writers emit only the current ones.
 * The keys are built by concatenation because a repository check bans the legacy word.
 */
declare const PRE_RENAME_FILES_KEY: "shards";
/**
 * Returns a manifest's file list from `slices` or from the legacy key.
 *
 * A non-empty `slices` wins, then a legacy list, then an empty `slices`.
 * An empty list is a valid answer, but a manifest with neither key throws, because an
 * empty result would misreport a missing list as a corpus without files.
 */
export declare function baseManifestFiles(manifest: {
    slices?: unknown;
    [PRE_RENAME_FILES_KEY]?: unknown;
}): ManifestFile[];
/**
 * One parquet file to add, with the source label in its rows.
 */
export interface OverlayFile {
    parquet: string;
    source: string;
    /**
     * The split that the file's rows belong to.
     * The default is `train`.
     *
     * `val` and `test` are allowed only for files that `splitOverlaySlice` returned,
     * so that the holdout policy chooses held-out rows.
     */
    split?: SplitName;
}
/**
 * Returns the split encoded in a `<stem>.<split>.parquet` filename, or `null` for any other name.
 *
 * Only `splitOverlaySlice` writes names with this suffix.
 */
export declare function splitFromFilename(parquet: string): SplitName | null;
/**
 * Options for {@link assembleOverlayManifest}.
 */
/**
 * The filename {@linkcode assembleOverlayManifest} writes its plan under, beside `MANIFEST.json`.
 */
export declare const OVERLAY_PLAN_FILE = "OVERLAY_PLAN.json";
/**
 * What one overlay assembly was asked to add, written beside the manifest it produced.
 *
 * The next assembly on the same base reads this rather than reconstructing the file list.
 * Reconstruction makes it possible for an overlay to contain a stale list of
 * files into a later corpus version.
 */
export interface OverlayPlan {
    corpusVersion: string;
    overlayBase: string | null;
    baseManifest: string;
    modalRoot: string;
    note: string;
    appliedAt: string;
    /**
     * Each added file with the route its split came from.
     *
     * `route` reads `split-slice` when the filename includes the split suffix written by the command.
     * `caller` reads the split supplied as an argument.
     *
     * A hand-placed split reached `v0.6.0-register-surface` through the second route
     * and put 770 of DE's 3,987 validation `source_id`s in train.
     * The filename was the only place that showed this (#2359).
     */
    files: Array<{
        parquet: string;
        source: string;
        split: SplitName;
        route: SplitRoute;
    }>;
}
/**
 * How one overlay file's split was decided.
 */
export declare const SplitRoute: {
    /**
     * `corpus split-slice` applied `splitForRow` to every row and wrote the split into the filename.
     */
    readonly SplitSlice: "split-slice";
    /**
     * The caller supplied the split.
     * The holdout policy never saw these rows.
     */
    readonly Caller: "caller";
};
export type SplitRoute = (typeof SplitRoute)[keyof typeof SplitRoute];
/**
 * Which route decided one file's split.
 */
export declare function splitRouteFor(parquet: string): SplitRoute;
export interface OverlayManifestOptions {
    base: string;
    newDir: PathBuilderLike;
    modalRoot: string;
    version: string;
    /**
     * The parquet files to add, in the order in which they follow the base files.
     */
    files: readonly OverlayFile[];
    note: string;
}
/**
 * Rewrites a base manifest's file path to its location in the corpus tree mounted on Modal at `/data`.
 */
export declare function rerootBaseFilePath(path: string, baseManifestPath: string): string;
/**
 * Maps a Modal path under `/data/` to the same file under the local data root
 * and returns any other path unchanged.
 */
export declare function localManifestFilePath(path: string): string;
/**
 * Writes a corpus manifest that keeps every file of `args.base` and appends `args.files`.
 *
 * A `val` or `test` file must use the `.<split>.parquet` suffix written by `splitOverlaySlice`.
 * The suffix shows that the holdout policy chose its rows.
 * A `train` file has no naming requirement.
 *
 * @throws When the base lists no files or a held-out file lacks the matching filename suffix.
 */
export declare function assembleOverlayManifest(args: OverlayManifestOptions): Promise<void>;
export {};
//# sourceMappingURL=manifest.d.ts.map