/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Golden eval-set validator (Phase 1 task #9 in the plan). * * The golden set is hand-labeled ground truth for the neural classifier. Each entry must carry * components whose surface forms actually occur in `raw` — otherwise the entry will silently rot * the eval signal. This module: * * - Defines `GoldenEntry` (schema check). * - Loads `.jsonl` files (one entry per line). * - Validates every entry: schema shape, ComponentTag membership, reachability of each component in * `raw` via the same `reconcileComponents` helper alignment uses. * - Returns a structured report of per-entry errors so the CLI / CI surface can act on it. * * The 1000-entry target (500 US + 500 FR) is a human task. This module catches the regressions that * creep in over time as new entries land. */ import { readdir } from "node:fs/promises" import { COMPONENT_TAGS, type ComponentTag } from "@mailwoman/core/types" import { reconcileComponents } from "@mailwoman/formatter" import { join } from "path-ts" import { TextSpliterator } from "spliterator" const TAG_SET = new Set(COMPONENT_TAGS as readonly string[]) /** * One entry in a golden `.jsonl` file. */ export interface GoldenEntry { raw: string components: Partial> country: string source: "golden" notes?: string } /** * Per-entry validation failure. */ export interface GoldenIssue { file: string line: number reason: string } /** * Aggregate report from `validateGoldenDir`. */ export interface GoldenReport { entries: number files: number issues: GoldenIssue[] } /** * Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations. */ export function parseGoldenLine(line: string): GoldenEntry { // The throw IS the result: `validateGoldenFile` catches it and records the message against the line // number, so a tolerant parse would report a corrupt row as valid. // oxlint-disable-next-line no-restricted-properties const obj = JSON.parse(line) as Partial & Record if (typeof obj.raw !== "string" || !obj.raw.length) { throw new Error("missing/empty raw") } if (typeof obj.country !== "string" || !/^[A-Z]{2}$/u.test(obj.country)) { throw new Error(`country must be ISO 3166-1 alpha-2 (got ${JSON.stringify(obj.country)})`) } if (obj.source !== "golden") { throw new Error(`source must be "golden" (got ${JSON.stringify(obj.source)})`) } const components = (obj.components ?? {}) as Record for (const [k, v] of Object.entries(components)) { if (!TAG_SET.has(k)) throw new Error(`unknown ComponentTag: ${k}`) if (typeof v !== "string" || !v.length) { throw new Error(`components.${k} must be a non-empty string`) } } return { raw: obj.raw, components: components as GoldenEntry["components"], country: obj.country, source: "golden", notes: typeof obj.notes === "string" ? obj.notes : undefined, } } /** * Check that every component in `entry` appears in `entry.raw` (reconciliation-equivalent). */ export function unreachableComponents(entry: GoldenEntry): ComponentTag[] { const reconciled = reconcileComponents(entry.components, entry.raw) const missing: ComponentTag[] = [] for (const tag of Object.keys(entry.components) as ComponentTag[]) { if (!(tag in reconciled)) { missing.push(tag) } } return missing } /** * Validate one `.jsonl` file end-to-end, returning a list of issues. * * Parses line by line over `TextSpliterator` rather than `JSONSpliterator`: every issue this returns carries the LINE * NUMBER it was found on, and a malformed line has to be REPORTED rather than thrown. `JSONSpliterator` parses each row * for you and throws on the first bad one — correct for consumers that want the rows, wrong for the validator whose * whole job is locating the bad ones. */ export async function validateGoldenFile(path: string): Promise { const issues: GoldenIssue[] = [] // Counted over EVERY row including blanks, so the number matches what an editor shows. let lineNumber = 0 for await (const raw of TextSpliterator.fromAsync(path, { skipEmpty: false })) { lineNumber++ const line = raw.trim() if (!line) continue const i = lineNumber - 1 try { const entry = parseGoldenLine(line) const unreachable = unreachableComponents(entry) if (unreachable.length) { issues.push({ file: path, line: i + 1, reason: `components not reachable in raw: ${unreachable.join(", ")}`, }) } } catch (error) { issues.push({ file: path, line: i + 1, reason: (error as Error).message }) } } return issues } /** * Validate every `.jsonl` in a golden directory. */ export async function validateGoldenDir(dir: string): Promise { const files = (await readdir(dir)).filter((n) => n.endsWith(".jsonl")).toSorted() const issues: GoldenIssue[] = [] let entries = 0 for (const name of files) { const fullPath = join(dir, name) const fileIssues = await validateGoldenFile(fullPath) issues.push(...fileIssues) for await (const line of TextSpliterator.fromAsync(fullPath)) { if (line.trim()) { entries++ } } } return { entries, files: files.length, issues } }