/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * @file Reads address context and derives street-side negatives for the sub-venue recipe. */ import { dataRootPath } from "@mailwoman/core/data-root" import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils" import type { PathBuilderLike } from "path-ts" import { readTuples as readLocaleTuples, type LocalePart } from "#international/recipes/locale" import { readTuples as readRecipeTuples } from "#recipes/scaffold" import type { LocaleBaseTuple } from "#surfaces/locale" // #region Address context /** * DE + ES address context. * * Both read through {@link readLocaleTuples}, the `locale` recipe's own streaming + reservoir reader, * so the CSV handling (quoted fields, crlf, city-noise cleaning, the DE per-part region fallback) * has exactly one implementation. * * The DE adapter reads two members from `europe.zip` instead of `oa-cache/de__*.zip`. * The cached per-state zips used by the `locale` recipe are absent on this host. * * The archive members match OA's current run byte for byte. * The note "a file's mtime is not its data's vintage" in `corpus/agents.md` records this comparison. */ const CONTEXT_PARTS: Readonly> = { DE: [ { zip: dataRootPath("openaddresses", "europe.zip"), csv: "de/berlin.csv", region: "Berlin" }, { zip: dataRootPath("openaddresses", "europe.zip"), csv: "de/sn/statewide.csv", region: "Sachsen" }, ], ES: [{ path: dataRootPath("openaddresses", "extracted", "es", "countrywide.csv") }], } /** * Load the address skeletons every leg renders onto: GB / US / FR from the house-venue v3 tuples * (the same 176,519 real rows the `invented-house-venue` recipe output is built from, so the two * recipes' address halves are drawn from one pool), DE and ES streamed out of OpenAddresses. */ export async function loadContextTuples( tuplesPath: PathBuilderLike, seed: number ): Promise> { const byCountry = new Map() for await (const tuple of readRecipeTuples(tuplesPath)) { const country = String(tuple.country ?? "") if (!country || !tuple.locality || !tuple.street) continue const mapped: LocaleBaseTuple = { house_number: String(tuple.houseNumber ?? tuple.house_number ?? ""), street: String(tuple.street), locality: String(tuple.locality), region: String(tuple.region ?? ""), postcode: String(tuple.postcode ?? ""), } const list = byCountry.get(country) if (list) { list.push(mapped) } else { byCountry.set(country, [mapped]) } } for (const [country, parts] of Object.entries(CONTEXT_PARTS)) { if (byCountry.has(country)) continue const pooled: LocaleBaseTuple[] = [] for (const [index, part] of parts.entries()) { // A dedicated stream prng, seeded per part, so the input sample is reproducible // without perturbing the emit loop's draws (the `locale` recipe's rule, kept). const streamRandom = makeMulberry32(seed + index) for (const tuple of await readLocaleTuples(part, streamRandom)) { pooled.push(tuple) } } byCountry.set(country, pooled) } return byCountry } /** * The street-side confound classes, mined from the leg's own address tuples. * * Real streets rather than invented ones. * The 176,519-row context pool includes 195 GB `hall` streets and 114 GB `gate` streets. * * It includes 134 distinct GB `-gate` tokens and a two-figure ` ` * population in both GB and US. * Each item comes from a real address where someone lives. * Each example comes from a real address tuple. */ export interface StreetNegatives { designator: LocaleBaseTuple[] modifierDesignator: LocaleBaseTuple[] gateSuffix: LocaleBaseTuple[] } /** * Shortest token that can take a `-gate` street suffix and still be a name rather than the bare word: * `gate` itself is four characters, so the class starts at five (`Highgate`, `Moorgate`, `Stonegate`). */ const MIN_GATE_SUFFIX_TOKEN_LENGTH = 5 export function buildStreetNegatives( context: readonly LocaleBaseTuple[], designatorPhrases: readonly string[], modifiers: readonly string[], country: string ): StreetNegatives { const designatorSet = new Set(designatorPhrases) const modifierSet = new Set(modifiers) const designator: LocaleBaseTuple[] = [] const modifierDesignator: LocaleBaseTuple[] = [] const gateSuffix: LocaleBaseTuple[] = [] for (const tuple of context) { const tokens = tuple.street.toLowerCase().match(/[\p{L}]+/gu) ?? [] let isDesignator = false let isPair = false for (const [index, token] of tokens.entries()) { if (designatorSet.has(token)) { isDesignator = true } if (index + 1 < tokens.length && modifierSet.has(token) && designatorSet.has(tokens[index + 1]!)) { isPair = true } if (country === "GB" && token.length >= MIN_GATE_SUFFIX_TOKEN_LENGTH && token.endsWith("gate")) { gateSuffix.push(tuple) } } if (isPair) { modifierDesignator.push(tuple) } else if (isDesignator) { designator.push(tuple) } } return { designator, modifierDesignator, gateSuffix } } // #endregion