/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * `locale` shard recipe — the multi-locale generalization of the `german` recipe. Reads REAL * OpenAddresses tuples for a `--country` (DE/FR/NL/IT/ES), renders each via * {@link synthesizeLocaleRow} in BOTH orders (`--intl-fraction`, default 0.4 international / the * rest country-native), aligns to BIO, and emits a labeled JSONL. Generate-mode: it STREAMS each * source CSV (a streamed zip member for cached zips, plain `createReadStream` for extracted CSVs) and * reservoir-samples to {@link RESERVOIR_CAP} (so FR/ES countrywide work in bounded memory), then * draws `--count` rows from the pool with the passed `random`. Ported from * scripts/build-locale-shard.mjs. * * The reservoir uses its OWN seeded PRNG ({@link makeMulberry32}, per part), independent of the * emit `random`, so the input sample is reproducible WITHOUT perturbing the synth/order draws. * * SURFACE DIVERSITY (#241): two per-country shape draws ride the emit loop, sized by the * 2026-07-02 format-diversity audit against the `openaddresses-{es,nl,it}-sample.jsonl` observed * forms. ES: the OpenCage template comma-joins the house number (`CALLE MAYOR, 12`) but all 3,000 * eval rows space-join (`CALLE MAYOR 12`) — {@link ES_SPACE_JOIN_FRACTION} of native rows collapse * the comma. NL: OA (and the eval, 3,000/3,000) glue the postcode (`1187LM`) while the national * convention spaces it (`1187 LM`) — {@link NL_GLUED_POSTCODE_FRACTION} of rows keep the glued * source shape, the rest the spaced conventional one. These draws are consumed ONLY for their * country, so DE/FR emit streams are unchanged for a given seed. */ import { type LocaleBaseTuple, type SynthesizedLocaleRow } from "@mailwoman/corpus/synthesizers/german"; import { type ShardRecipe } from "./scaffold.ts"; /** * One per-country OA source part: either a cached `zip` + `csv` member (streamed out of the archive) or an extracted * plain `path` (streamed via `createReadStream`). Both carry the standard OA header * (LON,LAT,NUMBER,STREET,UNIT,CITY,DISTRICT,REGION,POSTCODE,ID,HASH). An optional `region` fallback covers countries * whose REGION column is empty (DE — the Bundesland is implied by the per-state file). */ export interface LocalePart { zip?: string; csv?: string; path?: string; region?: string; /** * NZ — the OA DISTRICT column holds the city (`Auckland`) and CITY holds the suburb (`Birkenhead`). When set, map * DISTRICT→locality and CITY→dependent_locality (falling back to CITY→locality when DISTRICT is empty, ~18% of NZ * rows). Without this, the default CITY→locality mapping wrongly trains the suburb as the locality. */ districtAsLocality?: boolean; /** * ES pedanía part only — the header is the RAW (un-conformed) CNIG export schema (`numero`, `tipo_vial`, * `nombre_via`, `poblacion`, `municipio`, `comunidad_autonoma`, `cod_postal`), NOT the standard OA * NUMBER/STREET/CITY/DISTRICT/REGION/POSTCODE header every other part uses. Verified 2026-07-22 by exact- coordinate * cross-check: the OA-conformed `extracted/es/countrywide.csv` collapses CITY to `municipio` and drops `poblacion` * (Spain's below-municipio núcleo/pedanía name) entirely, so the pedanía signal survives ONLY in this raw export. * `street` is reconstructed as `tipo_vial + " " + nombre_via` (verified byte-identical to the conformed STREET column * for the same row). CITY-analog = `poblacion`, DISTRICT-analog = `municipio`. */ cnigRaw?: boolean; } export interface LocaleCountrySource { source: string; parts: LocalePart[]; /** * The `corpus_version` stamped on emitted rows. DE/FR keep the historical `0.4.0` (regenerating those shards must * stay lineage-identical); ES/IT/NL are the #241 staging lineage (`v0.9.9-es-it-nl`). */ corpusVersion: string; /** * An ALTERNATE part list, read instead of {@link parts} when the `--district-as-locality` override is explicitly * `true` for this invocation (see `run()`). ES-only for now — the standard `parts` entry can't supply real * dependent-locality signal (its OA-conformed CSV drops `poblacion`; see {@link LocalePart.cnigRaw}), so the pedanía * build reads a wholly different raw source instead of flipping the standard CITY/DISTRICT columns. `undefined` for * every other country — the override then just forces `districtAsLocality` on the normal `parts`, as GB/NZ already do * per-part. */ pedaniaParts?: LocalePart[]; } /** * OA CITY-noise normalization (#241) — the documented cleaning step, derived from the 2026-07-02 FULL-STREAM audit of * the ES (15.6M rows), IT (13.9M), and NL (9.1M) sources (not a hand-list). Returns the cleaned city, or `null` to drop * the tuple. * * Cleaned classes: * * 1. DROP pseudo-localities — the ES cadastral aggregates (`Comunidad de 09076, 09150 y 09578`, `Ledanía de …`; 0.06% of * ES rows): any CITY containing a comma or a ≥4-digit run is a land-register aggregate, not a renderable city. * Structural, locale-safe — NL's genuine `2e Valthermond` (one digit) survives; IT/NL have zero hits. * 2. STRIP a trailing parenthesized 1–3-letter admin code — the NL BAG province disambiguator (`Bergen (NH)`, `Rijswijk * (GLD)` → `Bergen`, `Rijswijk`; 0.13% of NL rows). The analogue of the German Kreis/region-suffix class (#241 names * `Rabenau Sachs` / `Weißwasser /O.L.`): an admin-region gloss glued onto the locality value that dirties locality * labels. * * Audit-verified NON-noise, deliberately NOT cleaned (a naive suffix rule would mangle real names): * * - ES/IT city-ends-with-province (`Alhama de Almería`, `GENZANO DI ROMA`; ~0.8% each): genuine toponyms whose linking * `de`/`di` makes them full names, unlike the German glued-abbreviation class. * - ES bilingual slash names (`Laudio/Llodio`; 2.16%): official co-names — the eval expects them verbatim. * - IT ALL-CAPS city casing (98.79% of the source, and the eval's observed form): casing is the #829 case-augmentation * lever, not this shard's. */ export declare function cleanCityNoise(city: string): string | null; /** * Stream real tuples out of an OA source part and reservoir-sample to {@link RESERVOIR_CAP}. Reads the CSV row-by-row — * `readZipEntry | CSVSpliterator` for zip parts, `createReadStream | CSVSpliterator` for extracted parts (both bounded * memory) — and keeps a uniform random sample (Algorithm R) seeded by `rng`, separate from the emit loop's PRNG. NO * global dedup (a 25M-key Set would OOM; OA rows are near-unique). The city passes through {@link cleanCityNoise}; the * region falls back to `part.region` when the row's REGION cell is empty (DE). * * Exported for {@link locale.test.ts} — the CSV read path (quote handling, CRLF, region fallback) has no other test. */ export declare function readTuples(part: LocalePart, rng: () => number): Promise; /** * Country-append fraction (the fr-admin-split #728 pattern, generalized to the locale recipe): mutates `synth` in * place, `countryFraction` of the time appending an explicit country surface form ("United Kingdom") to `raw` + a * `country` component — the model relearns to emit country WHEN the token is present without over-firing it on the * (still-majority) country-less rows. `countryFraction <= 0` (the default) short-circuits the `random()` draw away * entirely — no `synth` mutation and no RNG consumption — so every existing locale's emit stream stays byte-identical * to before this option existed. Exported for {@link locale.test.ts}. */ export declare function applyCountryAppend(synth: SynthesizedLocaleRow, country: string, countryFraction: number, random: () => number): void; /** * Merge the `--district-as-locality` CLI override onto one part. `undefined` (flag absent) returns `part` unchanged * (same object — no allocation, no behavior change); `true`/`false` returns a shallow copy with `districtAsLocality` * forced to that value for this invocation only. Exported for {@link locale.test.ts}. */ export declare function applyDistrictAsLocalityOverride(part: LocalePart, override: boolean | undefined): LocalePart; /** * Pick which part list a `--country` run reads: {@link LocaleCountrySource.pedaniaParts} when the override is explicitly * `true` AND the country registers one (ES only, so far), else the default `parts` — unchanged for every other * country/override combination. Exported for {@link locale.test.ts}. */ export declare function resolveLocaleParts(countrySource: LocaleCountrySource, override: boolean | undefined): LocalePart[]; /** * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise, * and `description` below for the surface form it generates. */ export declare const localeRecipe: ShardRecipe; //# sourceMappingURL=locale.d.ts.map