/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * G-NAF (Australia) corpus adapter — the parser-teaching half of #208. * * The model mis-parses Australian addresses in their native postcode-first / house-number-last * order: it tags a leading 4-digit postcode as a house number (its US/EU prior) and swaps street * ↔ locality with it. `scripts/eval/au-order-probe.ts` proved this is a word-ORDER coverage gap, * not capability — the same addresses parse perfectly in canonical order (65% → 87% @25km if the * parse were order-robust). EU survives the same eval because its postcodes are * format-distinctive (a hyphenated `26-300` reads as a postcode anywhere); a bare AU `3053` only * disambiguates by position. * * So this adapter renders each assembled G-NAF tuple (from {@link ./assemble}) in one of three real * AU layouts — real-AU canonical (number-first, postcode-trailing), postcode-first, * locality-first — ROTATED by row index (`i % 3`), so the locality + postcode each land in every * position across the shard. This is the exact mechanism that fixed #148's v1.9.0 order-overfit * for the 16 EU locales (`scripts/rerender-overture-multiorder.mjs`, v1.9.1 → shipped v4.13.0); * AU was simply never in that train (`country_weights` had no AU, and `data_loader.py` excludes * unlisted countries). Rotating one order per row (rather than emitting all three) keeps this a * clean single-variable extension of the proven recipe + matches its source-mass structure. The * corpus aligner BIO-labels each (every component surface form occurs verbatim in `raw`, so * alignment lands). * * Input: the assembled component JSONL (one `{house_number,street,locality,region,postcode}` per * line). Open G-NAF licence — attribute "Geoscape Australia". */ import { tryParsingJSON } from "@mailwoman/core/objects" import { stableSourceID } from "@mailwoman/corpus/adapters/utils" import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types" import { reconcileComponents } from "@mailwoman/formatter" import { TextSpliterator } from "spliterator" /** * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset * it came from. */ export const GNAF_ADAPTER_ID = "gnaf" /** * Open G-NAF is freely redistributable with attribution to Geoscape Australia (CC-BY-style). */ export const GNAF_DEFAULT_LICENSE = "CC-BY-4.0" interface GNAFTuple { house_number: string street: string locality: string region?: string postcode: string } /** * The address layouts an AU address actually arrives in. The model already handles postcode-TRAILING (canonical); the * two postcode-LEADING forms are the ones it fails, so they carry the lever. We keep the canonical form too so the * retrain doesn't forget it. */ function renderOrders(c: GNAFTuple): string[] { const region = c.region ? ` ${c.region}` : "" return [ // real-AU canonical: number-first, street, suburb [state] postcode — "50 Barry Street, Carlton NSW 2000" `${c.house_number} ${c.street}, ${c.locality}${region} ${c.postcode}`, // postcode-first (the dominant failure): "2000 Carlton, Barry Street 50" `${c.postcode} ${c.locality}, ${c.street} ${c.house_number}`, // locality-first: "Carlton, 2000, Barry Street 50" `${c.locality}, ${c.postcode}, ${c.street} ${c.house_number}`, ] } /** * Build the G-NAF adapter. `inputPath` is the assembled component JSONL (see {@link ./assemble}); it is country-pinned * to AU regardless of `opts.country` (G-NAF is Australia-only). */ export function createGNAFAdapter(): CorpusAdapter { return { id: GNAF_ADAPTER_ID, defaultLicense: GNAF_DEFAULT_LICENSE, description: "G-NAF (Australia): assembled address tuples rendered in multiple word orders (canonical / postcode-first / locality-first) — teaches the model AU's postcode-first layout.", async *rows(opts: AdapterOptions): AsyncIterable { let emitted = 0 let idx = 0 // Input is the assembled component JSONL (one tuple per line). TextSpliterator auto-disposes on // loop completion and on an early `break` (abort / limit), so the old explicit handle teardown is // gone; the parse tolerates a trailing CR on CRLF sources and the `!line.trim()` guard skips blanks. // The render order rotates (i % 3), matching v1.9.1's rerender. for await (const line of TextSpliterator.fromAsync(opts.inputPath)) { if (opts.signal?.aborted) break if (opts.limit !== undefined && emitted >= opts.limit) break if (!line.trim()) continue const t = tryParsingJSON(line) if (t === null) continue if (!t.house_number || !t.street || !t.locality || !t.postcode) continue const orders = renderOrders(t) const order = idx % orders.length idx++ const raw = orders[order]! const components: CanonicalRow["components"] = { house_number: t.house_number, street: t.street, locality: t.locality, postcode: t.postcode, } // region (state) rides only the canonical render (order 0); the postcode-leading layouts // omit it (matching the eval's serialization) so it never breaks verbatim alignment. if (order === 0 && t.region) { components.region = t.region } const aligned = reconcileComponents(components, raw) if (!Object.keys(aligned).length) continue yield { raw, components: aligned, country: "AU", locale: "en-AU", source: GNAF_ADAPTER_ID, source_id: `${stableSourceID(GNAF_ADAPTER_ID, aligned)}-o${order}`, corpus_version: "", license: GNAF_DEFAULT_LICENSE, } emitted++ } }, } } /** * The configured adapter instance registered with the corpus builder. */ export const gnafAdapter = createGNAFAdapter()