/** * Gold dataset loader and validator. * * The "gold" file is a YAML document containing human (or stronger-model) * annotations for a fixed set of sample_ids. omk loads it and compares the * annotations against the judge's scores from a finished evaluation report * to compute α / κ / Pearson agreement (see ./human-gold.ts). * * Schema is deliberately small: * * metadata: * annotator: # required, used for contamination check * annotatedAt: # required * version: # required * scale: { min: 1, max: 5 } # optional, default {1,5} * notes: # optional * * annotations: * - sample_id: * score: # required * reason: # optional, kept for audit only * * v0.21 supports single-score gold only. Multi-dimensional gold (per-rubric * scores) is a candidate for — * the schema is forwards-compatible (we can add a `dimensions` field next to * `score` later without breaking existing files). */ export interface GoldMetadata { /** Identifier of who/what produced the annotations — model id, person handle, or team name. */ annotator: string; annotatedAt: string; version: string; scale?: { min: number; max: number; }; notes?: string; } export interface GoldAnnotation { sample_id: string; score: number; reason?: string; } export interface GoldDataset { metadata: GoldMetadata; annotations: GoldAnnotation[]; /** Source file paths — surfaced for diagnostics (e.g. error messages). */ sourcePaths: string[]; } export interface ValidationIssue { /** File the issue came from — empty when issue is structural (e.g. missing metadata). */ path: string; /** Annotation index within the file, when applicable. */ index?: number; message: string; } export interface LoadResult { dataset?: GoldDataset; issues: ValidationIssue[]; } /** * Load a gold dataset from a directory. * * Convention: the directory contains exactly one `metadata.yaml` and one or * more `*.yaml` annotation files. Annotation files may declare top-level * `annotations: [...]` — multiple files are concatenated. This shape lets * users split annotations by topic (code.yaml, writing.yaml, ...) without * fighting one giant file in code review. */ export declare function loadGoldDataset(dir: string): LoadResult; /** Re-export YAML serializer so the `gold init` CLI can write a template. */ export declare function dumpYaml(value: unknown): string;