/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Corpus linter. Compares a new shard against pre-computed corpus statistics and flags patterns * that would cause the class of failure we hit with v0.6.2's "5th Avenue Theatre" adversarial * venue templates. * * Per DeepSeek turn 9 design (2026-05-29). v1 checks: * * 1. **Token-label distribution outliers.** For each token in the new shard, compare the shard's * majority label to the corpus's majority label. Flag when the corpus has a * confidently-established majority (>66%) AND the shard's majority differs AND both have * non-trivial counts (shard ≥ 50, corpus ≥ 200). * 2. **Label-vacuum tokens.** Token labeled with a tag that has ZERO instances in the corpus for that * token, despite the token being well-represented in the corpus. Stronger signal than #1 — * we're introducing a novel association, not shifting a distribution. * 3. **Bigram-label collisions.** Identical (token_bigram, label_bigram) appears in shard while the * same token_bigram has a DIFFERENT majority label_bigram in the corpus. The "5th Avenue" * with [B-venue, I-venue] vs corpus's [B-house_number, I-street] case. * 4. **Common-form anti-pattern rules.** Applies `lint-rules.json` — token-regex → forbidden-labels * mappings — flagging matches. * 5. **Basic sanity.** Truncated rows (tokens.length !== labels.length), all-O rows >90% of shard. * * Output: markdown report on stdout, optional JSON sidecar via `outJSON`. The command exits 0 if * no errors, 1 if any errors (warnings don't gate). Per the design, the MANIFEST entry for a * flagged shard should require `lint_acknowledged: true` before training consumes it. * * Usage: mailwoman dev lint corpus-shard\ * --shard \ * --stats \ * [--rules ]\ * [--out-md /tmp/lint-report.md]\ * [--out-json /tmp/lint-report.json] */ /** * Options for {@linkcode lintCorpusShard}. */ export interface LintCorpusShardOptions { /** * The new shard parquet to lint. */ shardPath: string; /** * Pre-computed corpus stats JSON (see `corpus-stats.ts`). */ statsPath: string; /** * Anti-pattern rules JSON. Default: the `lint-rules.json` beside this module. */ rulesPath?: string; /** * Write the markdown report here as well as stdout. */ outMd?: string; /** * Write a JSON sidecar of the flags + summary here. */ outJSON?: string; } /** * One lint flag emitted by a check. */ export interface LintShardFlag { check: string; severity: "error" | "warn"; token?: string; bigram?: string; shardLabel?: string; corpusLabel?: string; shardCount?: number; corpusCount?: number; detail: string; ruleID?: string; } /** * Findings summary returned by {@linkcode lintCorpusShard}. */ export interface LintCorpusShardSummary { errors: number; warnings: number; findings: LintShardFlag[]; /** * The rendered markdown report (also printed to stdout). */ report: string; } /** * Lint a shard against corpus stats + the anti-pattern rules; print the markdown report to stdout. */ export declare function lintCorpusShard(options: LintCorpusShardOptions, report?: (line: string) => void): Promise; //# sourceMappingURL=lint-shard.d.ts.map