/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Alignment: turn a `CanonicalRow` (raw + components) into a `LabeledRow` (raw + tokens + BIO * labels) or a `QuarantinedRow` (raw + reason) per the Phase 1 plan. * * Pipeline: * * 1. For each `(tag, value)` in `components`, find the value's character span in `raw`. First try a * verbatim substring match (case-insensitive, whitespace-collapsed). If that fails, fall * back to fuzzy match via `fastest-levenshtein`, with a tunable edit distance threshold. * 2. If any component cannot be located, reject the row with a human-readable reason and send it to * the quarantine pile (`reason: "component-not-found:"` or * `"edit-distance-exceeded::"`). * 3. Tokenize `raw` with the supplied `Tokenizer` (defaults to the whitespace tokenizer). * 4. For each token: walk the list of component spans, pick the one whose span contains the token's * character range. First token in a component span → `B-`; subsequent tokens → * `I-`; no overlap → `O`. * 5. Emit the located char spans verbatim as `span_starts[]` / `span_ends[]` / `span_tags[]` (the * v0.5.0 char-offset format, #519). The token quantization in step 4 is the part the v0.5.0 * rebuild deletes; during the transition both representations ride on every labeled row. * * Structural invariants the function preserves (the span ones loudly — a violation throws rather * than quarantines, because it indicates a bug here, not bad source data): * * - `tokens.length === labels.length` always. * - Each component contributes at most one contiguous BIO run (no `B-tag … O … I-tag` gaps). This is * enforced by greedy first-match span assignment + ordered token iteration. * - The span triple is sorted ascending by start and non-overlapping. * - `raw` is NFC-normalized (asserted per row; a non-NFC raw makes char offsets ambiguous downstream * — NFD `é` occupies two code units where NFC `é` occupies one — and silently so). */ import type { ComponentTag } from "@mailwoman/core/types"; import type { CanonicalRow, LabeledRow, QuarantinedRow } from "@mailwoman/corpus/types"; import { type Tokenizer } from "./tokenize.ts"; /** * Options for `alignRow`. */ export interface AlignOptions { /** * Tokenizer to use. Defaults to `whitespaceTokenizer()`. */ tokenizer?: Tokenizer; /** * Max Levenshtein edit distance to accept when a verbatim substring match fails. Set `0` to require verbatim matches * only. Default `2`. * * Distance is computed against same-length windows in `raw`, so the threshold scales naturally with the component * value length. */ maxEditDistance?: number; /** * Case-insensitive comparison for substring search. Default `true`. The retained span in `raw` is the original case; * only matching is case-insensitive. */ caseInsensitive?: boolean; } /** * Either a successful labeled row or a quarantined one. */ export type AlignmentResult = { kind: "labeled"; row: LabeledRow; } | { kind: "quarantined"; row: QuarantinedRow; }; /** * One located char-offset label span over a row's `raw` ([start, end) in UTF-16 code units). The element type behind * the parallel `span_starts[]`/`span_ends[]`/`span_tags[]` triple on `LabeledRow` (#519). */ export interface ComponentSpan { tag: ComponentTag; start: number; end: number; } /** * Align a single row. */ export declare function alignRow(row: CanonicalRow, opts?: AlignOptions): AlignmentResult; /** * Enforce the #519 span-triple invariants — in-bounds, sorted ascending by start, non-overlapping — loudly. * * For `alignRow`: `claimed`-span bookkeeping in `locateSpan` already makes overlap impossible and the caller sorts, so * a violation here is a bug in this file, not bad source data: throw (naming the row) rather than quarantine, so the * corruption can't ride into a corpus. Exported for every OTHER span producer (`composeAdversarialRow`'s offset * arithmetic, future synthesis paths) — any code that emits the triple without going through `alignRow` must pass its * output through this. */ export declare function assertSpanInvariants(spans: readonly ComponentSpan[], row: Pick): void; //# sourceMappingURL=align.d.ts.map