/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * `no-street-led`: the Norwegian street-led recipe. * * Street-led forms ("Tangavegen 40, 5620 Tørvikbygd") have a larger residual than the pc-first form. * A diacritic street head (…vegen/…veien with ø/å/æ) creates a leading-name-before-comma boundary. * The recipe cycles each tuple through all three real orders to balance polarity. * * 1. canonical "«st» «n», «pc» «city»" (the larger residual class) * 2. city-first "«city», «pc», «st» «n»" * 3. pc-first "«pc» «city», «st» «n»" (the floor the others converge to) */ import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils" import { TextSpliterator } from "spliterator" import { alignAndWrite, foldNOSurface, readTuples, requireRegister, type CorpusRecipe, recipeSourceID, } from "#recipes/scaffold" import { defaultRecipeSource } from "#recipes/sources" import { SurfaceOrigin } from "#types" /** * Recipe registered with the corpus builder. */ export const noStreetLedRecipe: CorpusRecipe = { name: "no-street-led", description: "NO street-led boundary form (#901 family): '«st» «n», «pc» «city»' — diacritic street heads", mode: "tuples", options: [ { flag: "--exclude-surfaces ", description: "REQUIRED. The NO digit board's reserved surface list " + "(mailwoman/eval-harness/fixtures/no-digits.surfaces.txt); every listed street is skipped.", }, ], async run(opts, write) { makeMulberry32(opts.seed) // The split excludes every surface the digit board reserves, so a retrain // grades the boundary form on unseen streets. // There is no safe default, so the flag throws. const excludePath = opts.excludeSurfaces if (!excludePath) { throw new Error( "no-street-led: --exclude-surfaces is REQUIRED. Pass the NO digit board's reserved list " + "(mailwoman/eval-harness/fixtures/no-digits.surfaces.txt) or this recipe trains on its own eval set. " + "Source-disjoint by street SURFACE is the split discipline; there is no safe default." ) } const excluded = new Set() for await (const line of TextSpliterator.fromAsync(excludePath)) { if (!line.startsWith("#")) { excluded.add(line) } } if (!excluded.size) throw new Error(`no-street-led: --exclude-surfaces "${excludePath}" listed no surfaces`) let read = 0 let emitted = 0 let skipped = 0 let contaminated = 0 const NO_STREET_LED_PROVENANCE = { register: requireRegister(opts, "no-street-led"), surface: SurfaceOrigin.Composed, } for await (const t of readTuples(opts.input!)) { read++ const street = String(t.street ?? "").trim() const city = String(t.locality ?? "").trim() const number = String(t.number ?? "").trim() const postcode = String(t.postcode ?? "").trim() if (!street || !city || !number || !postcode) { skipped++ continue } // A surface on the digit board never enters training. if (excluded.has(foldNOSurface(street))) { contaminated++ continue } const order = read % 3 let raw: string const components: Record = { street, house_number: number, postcode, locality: city, } if (order === 0) { raw = `${street} ${number}, ${postcode} ${city}` } else if (order === 1) { raw = `${city}, ${postcode}, ${street} ${number}` } else { raw = `${postcode} ${city}, ${street} ${number}` } // One resolution for both, because `source_id` includes the source as its prefix // and a pair that disagreed would name a source no row of this output declares. const source = defaultRecipeSource("synth-no-street-led") const source_id = recipeSourceID(source, { ...components, o: String(order), v: String(read) }) const canonical = { raw, components, country: "NO", locale: "nb-NO", source, source_id, corpus_version: "0.10.0", license: "Synthetic — no-street-led; (street, number, postcode, city) from OpenAddresses NO (per-source attribution in the model card)", } // Per row, because the id names this tuple's record. // Tuple files created before `sourceID` have no identifier to forward. // A `null` base ID preserves that absence. if ( alignAndWrite(write, canonical, "no-street-led", { ...NO_STREET_LED_PROVENANCE, baseSourceID: t.sourceID ?? null, }) ) { emitted++ } else { skipped++ } } return { read, emitted, skipped, contaminated } }, }