/** * @copyright Sister Software */ import { dataRootPath } from "@mailwoman/core/data-root" import { stringifyJSON } from "@mailwoman/core/json" import { sample } from "@mailwoman/core/random" import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils" import { renderAUPoBox, renderNZPoBox } from "#recipes/po/box/cedex/au-nz" import { renderCaEn, renderCaFr } from "#recipes/po/box/cedex/ca" import { renderBpFr, renderCedexFr } from "#recipes/po/box/cedex/fr" import { FR_SOURCE, GEONAMES_CA, GEONAMES_POSTAL_AU, GEONAMES_POSTAL_NZ, isHoldoutLocality, readCaLocalities, readFrTuples, readPostalTuples, readUsTuples, US_EVAL_SOURCE, US_TRAIN_SOURCES, } from "#recipes/po/box/cedex/sources" import type { AUTuple, NZTuple, Rendered, USTuple } from "#recipes/po/box/cedex/types" import { renderPmbUs, renderPoBoxUs } from "#recipes/po/box/cedex/us" import { CLASS_MIX } from "#recipes/po/box/cedex/vocabulary" import { recipeSourceID, type CanonicalRecipeRow, type CorpusRecipe } from "#recipes/scaffold" import { defaultRecipeSource } from "#recipes/sources" import { synthesizeMilitaryPoBoxRow } from "#synthesizers/po-box" import { SurfaceOrigin } from "#types" import { alignRow } from "#utils" const COMPONENT_ORDER = ["house_number", "street", "po_box", "venue", "locality", "postcode", "region", "cedex"] const orderComponents = (components: Record): Record => Object.fromEntries(COMPONENT_ORDER.filter((key) => components[key]).map((key) => [key, components[key]!])) const pickClass = (draw: number): string => { let accumulated = 0 for (const [name, weight] of CLASS_MIX) { accumulated += weight if (draw < accumulated) return name } return CLASS_MIX.at(-1)![0] } /** * Generate PO-box and cedex rows from source pools in the US, France, Canada, Australia and New Zealand. */ export const poBoxCedexRecipe: CorpusRecipe = { name: "po-box-cedex", description: "PO box / CEDEX coverage rows (US/FR/CA/AU/NZ) — self-generated from cached OA + GeoNames pools", mode: "generate", options: [{ flag: "--golden", description: "Emit the leakage-safe holdout variant ({raw, components, country})" }], async run(opts, write) { const random = makeMulberry32(opts.seed) const count = opts.count ?? 50_000 const source = opts.sourceName ?? defaultRecipeSource("synth-po-box-cedex") const usPool: USTuple[] = [] for (const sourceEntry of opts.golden ? [US_EVAL_SOURCE] : US_TRAIN_SOURCES) { const tuples = await readUsTuples(sourceEntry) console.error(` ${sourceEntry.csv}: ${tuples.length} tuples`) for (const tuple of tuples) { usPool.push(tuple) } } const frAll = await readFrTuples(80_000) const frPool = frAll.filter((tuple) => isHoldoutLocality(tuple.locality) === opts.golden) console.error(` ${FR_SOURCE.csv}: ${frAll.length} tuples (${frPool.length} after holdout split)`) const qcAll = await readCaLocalities("10"), onAll = await readCaLocalities("08") const qcPool = qcAll.filter((locality) => isHoldoutLocality(locality) === opts.golden), onPool = onAll.filter((locality) => isHoldoutLocality(locality) === opts.golden) console.error(` GeoNames CA: QC ${qcAll.length}→${qcPool.length}, ON ${onAll.length}→${onPool.length}`) const auAll = (await readPostalTuples(GEONAMES_POSTAL_AU, { withState: true })) as AUTuple[], nzAll = (await readPostalTuples(GEONAMES_POSTAL_NZ, { withState: false })) as NZTuple[] const auPool = auAll.filter((tuple) => isHoldoutLocality(tuple.locality) === opts.golden), nzPool = nzAll.filter((tuple) => isHoldoutLocality(tuple.locality) === opts.golden) console.error(` GeoNames postal: AU ${auAll.length}→${auPool.length}, NZ ${nzAll.length}→${nzPool.length}`) if (!usPool.length || !frPool.length || !qcPool.length || !onPool.length || !auPool.length || !nzPool.length) throw new Error( `A base pool is empty — check ${dataRootPath("oa-cache")}, ${GEONAMES_CA}, ${GEONAMES_POSTAL_AU.zip}, and ${GEONAMES_POSTAL_NZ.zip}.` ) let emitted = 0, skipped = 0, guard = 0 while (emitted < count && guard++ < count * 10) { const kind = pickClass(random()) let rendered: Rendered let country: string let locale: string if (kind === "po-box-us") { rendered = renderPoBoxUs(random, sample(usPool, random)) country = "US" locale = "en-US" } else if (kind === "pmb-us") { const tuple = sample(usPool, random) if (!tuple.postcode || !tuple.street || !tuple.house_number) continue rendered = renderPmbUs(random, tuple) country = "US" locale = "en-US" } else if (kind === "bp-fr") { rendered = renderBpFr(random, sample(frPool, random)) country = "FR" locale = "fr-FR" } else if (kind === "cedex-fr") { rendered = renderCedexFr(random, sample(frPool, random)) country = "FR" locale = "fr-FR" } else if (kind === "cp-ca-fr") { rendered = renderCaFr(random, sample(qcPool, random)) country = "CA" locale = "fr-CA" } else if (kind === "po-box-au") { rendered = renderAUPoBox(random, sample(auPool, random)) country = "AU" locale = "en-AU" } else if (kind === "po-box-nz") { rendered = renderNZPoBox(random, sample(nzPool, random)) country = "NZ" locale = "en-NZ" } else if (kind === "po-box-us-military") { const military = synthesizeMilitaryPoBoxRow({ random }) const { country: _country, ...components } = military.components rendered = { fmt: "po-box-military", raw: military.raw, components: components as Record } country = "US" locale = "en-US" } else { rendered = renderCaEn(random, sample(onPool, random)) country = "CA" locale = "en-CA" } const { raw, components } = rendered if (!Object.values(components).every((value) => raw.includes(value))) { skipped++ continue } if (opts.golden) { write(stringifyJSON({ raw, components: orderComponents(components), country })) emitted++ continue } const canonical: CanonicalRecipeRow = { raw, components: orderComponents(components), country, locale, source, source_id: recipeSourceID(source, components), corpus_version: "0.4.0", license: country === "CA" ? "GeoNames CA (CC-BY 4.0) locality skeletons + Canada Post box forms (corpus templates); postcodes synthesized to the codex CA pattern" : country === "FR" ? "OpenAddresses FR (BAN-derived) skeletons + La Poste BP/CEDEX forms (corpus templates, NF Z 10-011)" : country === "AU" ? "GeoNames AU postal dump (CC-BY 4.0) locality/state/postcode tails + Australia Post Postal Delivery Type designators (@mailwoman/codex/au)" : country === "NZ" ? "GeoNames NZ postal dump (CC-BY 4.0) locality/postcode tails + NZ Post ADV358 Delivery Service Types (@mailwoman/codex/nz)" : "OpenAddresses US (non-VT) skeletons + USPS Pub-28 §29 PO-box designators (codex/corpus templates)", } const aligned = alignRow(canonical as Parameters[0]) if (aligned.kind !== "labeled" || !aligned.row) { skipped++ continue } // The CEDEX number is drawn rather than read from La Poste's own file. write( stringifyJSON({ ...aligned.row, recipe: kind, base_source_id: null, register: null, surface: SurfaceOrigin.Invented, }) ) emitted++ } return { emitted, skipped } }, }