/** * Build the System One benchmark dataset from CLEO's own history (T12495). * * The control group is CLEO dogfooded: every row is a question a decision * site really asks, labelled from what the project's records already say. * * - `duplicateDetection` — task pairs. Positives: a `duplicates` relation, a * task cancelled with a reason that calls it a duplicate of a named task, * or a note that records a duplicate of a named task. Negatives: pairs of * tasks under the same parent with no relation between them. * - `observationType` — observations whose stored type can only have come * from a caller: it is one of the offered options and differs from BOTH * the keyword default the writer applies today and the substring default * it applied before T12494. A type that agrees with a keyword default may * have been defaulted, so it is not used as a label. * - `decisionContradiction` — decision pairs. Positives: a supersedes edge * (newer → older). Negatives: random pairs with no supersedes edge between * them. The newer decision's declared `supersedes` is never sent. * * Pair rows whose text gives the label away (one side names the other's id, * or says "duplicate of T123" / "supersedes D12") are dropped * ({@link leaksBenchLabel}). * * Every text field is redacted with the memory redaction patterns System One * uses (`redactContent`) and clipped before it is stored, so the dataset file * never holds a secret the provider would not have been sent anyway. * * Pure: the store is read through a {@link BenchSource}; randomness is seeded. * * @task T12495 * @epic T12486 */ import { type BenchRow, type BenchSite, type BenchSource } from './types.js'; /** Characters kept per text field in the dataset (the site builders clip further). */ export declare const BENCH_TEXT_MAX_CHARS = 1000; /** Default cap on rows per site. */ export declare const DEFAULT_BENCH_MAX_ROWS_PER_SITE = 400; /** Default negatives sampled per positive. */ export declare const DEFAULT_BENCH_NEGATIVES_PER_POSITIVE = 1; /** Negatives sampled even when a site has few or no positives. */ export declare const MIN_BENCH_NEGATIVES = 10; /** Options for {@link buildBenchDataset}. */ export interface BuildBenchDatasetOptions { /** Sites to build. Default: all of {@link BENCH_SITES}. */ readonly sites?: readonly BenchSite[]; /** PRNG seed. Default {@link DEFAULT_BENCH_SEED}. */ readonly seed?: number; /** Row cap per site. Default {@link DEFAULT_BENCH_MAX_ROWS_PER_SITE}. */ readonly maxRowsPerSite?: number; /** Negatives per positive. Default {@link DEFAULT_BENCH_NEGATIVES_PER_POSITIVE}. */ readonly negativesPerPositive?: number; /** Redaction. Default: the memory redaction patterns (`redactContent`). */ readonly redact?: (text: string) => string; } /** Row counts of a dataset, per site and label. */ export interface BenchDatasetSizes { /** Total rows. */ readonly total: number; /** Rows marked owner-verified. */ readonly ownerVerified: number; /** Site → label → rows. */ readonly bySite: Readonly>>>; } /** * The type the pre-T12494 writer assigned when the caller gave none: the * first keyword group with a SUBSTRING match, else `discovery`. * * @param text - Observation narrative. * @returns The legacy default type. */ export declare function legacyKeywordObservationType(text: string): string; /** * Whether a pair row's text gives its label away: either side names the * other's id, or says "duplicate of T123" / "supersedes D12". Such rows are * dropped, so a provider is never scored on reading the answer off the text. * * @param row - A dataset row. * @returns True when the text leaks the label. */ export declare function leaksBenchLabel(row: BenchRow): boolean; /** * Build the labelled dataset from a store. * * @param source - Store port (the accessors in production, records in tests). * @param opts - Sites, seed, caps and redaction. * @returns Rows, grouped by site in {@link BENCH_SITES} order. */ export declare function buildBenchDataset(source: BenchSource, opts?: BuildBenchDatasetOptions): Promise; /** * Count rows per site and label. * * @param rows - Dataset rows. * @returns The sizes. */ export declare function benchDatasetSizes(rows: readonly BenchRow[]): BenchDatasetSizes; /** * Serialise rows as JSONL (one row per line, trailing newline). * * @param rows - Dataset rows. * @returns The JSONL text. */ export declare function serializeBenchDataset(rows: readonly BenchRow[]): string; /** * Parse and validate JSONL rows. * * @param text - JSONL text. * @returns The rows. * @throws Error naming the first invalid line. */ export declare function parseBenchDataset(text: string): BenchRow[]; /** * Write rows to a JSONL file. * * @param path - Target file. * @param rows - Dataset rows. */ export declare function writeBenchDataset(path: string, rows: readonly BenchRow[]): Promise; /** * Read rows from a JSONL file. * * @param path - Source file. * @returns The rows. */ export declare function readBenchDataset(path: string): Promise; //# sourceMappingURL=dataset.d.ts.map