/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * `wof-admin`: Who's On First admin GeoJSON-bundle adapter. * * **Phase 1.5.1 pivot.** The original Phase 1.5 SQLite adapter (formerly at * `packages/corpus/lib/adapters/wof-admin/`, removed in this same change) was replaced by this * one because the SQLite distribution path was unworkable for the real corpus build: * * 1. `dist.whosonfirst.org/sqlite/` is dead (nxdomain); the Geocode-Earth mirror is the only one. * 2. The Geocode-Earth-hosted postalcode DB tags every row `mz:is_current = -1` ("unknown but treated * as active"); the SQLite adapter's `is_current = 1` predicate emitted zero rows. * 3. The `names` table in the SQLite distribution is empty — localized `name:*` variants live in a * separate distribution. The St. Petersburg / Mt. Vernon / Ft. Lauderdale alternation cases * (the original Phase 1.5.1 motivator) cannot be solved on the SQLite path even with a * patched `is_current` predicate. * * Input: a directory containing one or more cloned `whosonfirst-data-admin-` GitHub repos. Each * repo has `data/XXX/YYY/ZZZ/.geojson` files; `**\/*.geojson` walks the tree recursively. * Alternate-geometry siblings (`-alt-*`) are skipped — they're separate exports of the same * record rather than new records. * * Per record, the adapter emits one row per `(name-variant, hierarchy-variant)` pair: * * - **Name variants**: the canonical `wof:name` (slot key `default`) plus every `name:*` localized * variant present on the feature (`name:eng_x_preferred`, `name:eng_x_colloquial`, * `name:rus_x_preferred`, ...). This is the Phase 1.5.1 fix for the St. Petersburg case: * `"Saint Petersburg"` (canonical) and `"St. Petersburg"` (eng_x_colloquial) both become * training rows for the same WOF id. * - **Hierarchy variants** (unchanged from the SQLite adapter): locality → 3 variants, region → 2, * country → 1, county → 1. * * `source_id` is `wof-admin---`. The previous SQLite adapter * used `wof-admin--` (no name slot); the new format adds a name-slot * segment so the colloquial / preferred / per-locale variants survive dedup independently. * * License: CC0. The adapter stamps every row with `CC0-1.0`. */ import type { ComponentTag } from "@mailwoman/codex/component" import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst" import { COUNTRY_DISPLAY_NAME, emitWOFJSONRows, LOCALE_BY_COUNTRY, nameSlotsFor as wofNameSlotsFor, type WOFVariantSpec, } from "#adapters/wof/json-rows" import { SourceRegister } from "#registers" import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types" import { buildAncestorNameIndex, walkFeatures, type AncestorNames, type WOFRecord } from "#utils" /** * Map a WOF placetype to a Mailwoman `ComponentTag`, or `undefined` to skip. * * Per-adapter deliberately (the postalcode adapter has its own): each table is a record * filter for its adapter's emission set rather than a shared vocabulary. */ function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag | undefined { switch (placetype) { case "country": case "nation": return "country" case "macroregion": case "region": return "region" case "macrocounty": case "county": case "localadmin": return "subregion" case "locality": return "locality" case "borough": case "macrohood": case "neighbourhood": case "microhood": return "dependent_locality" default: return undefined } } /** * Compute the hierarchy variants for a record given its ancestry chain and the chosen `selfName`. * * `selfName` is the surface form to use for the record's own component * (locality / region / country / subregion). * Callers pass the canonical `wof:name` for the `"default"` slot and a `name:*` * localized value for variant slots. * * Ancestor names always come from the ancestor's canonical `wof:name`. * * Country variants substitute `COUNTRY_DISPLAY_NAME` for the default slot so the * OpenCage template produces the canonicalized form (`"United States of America"`), * matching the legacy SQLite adapter's behavior. */ export function variantsFor(row: WOFRecord, ancestry: AncestorNames, selfName: string): WOFVariantSpec[] { const selfTag = placetypeToTag(row.placetype) if (!selfTag) return [] // Every variant here is a gazetteer hierarchy — `Paris`, `Paris, Île-de-France`, // `Paris, Île-de-France, France` — and several steps are not addresses at all. // See `WOFVariantSpec.hierarchy`. const hierarchy = true const region = ancestry.region const country = ancestry.country const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country ?? row.country const variants: WOFVariantSpec[] = [] switch (selfTag) { case "locality": case "dependent_locality": { variants.push({ hierarchy, suffix: "self", components: { [selfTag]: selfName } }) if (region) { variants.push({ hierarchy, suffix: "with-region", components: { [selfTag]: selfName, region }, }) } if (region && country) { variants.push({ hierarchy, suffix: "with-region-country", components: { [selfTag]: selfName, region, country: countryDisplay }, }) } else if (!region && country) { variants.push({ hierarchy, suffix: "with-country", components: { [selfTag]: selfName, country: countryDisplay }, }) } return variants } case "region": { variants.push({ hierarchy, suffix: "self", components: { region: selfName } }) if (country) { variants.push({ hierarchy, suffix: "with-country", components: { region: selfName, country: countryDisplay }, }) } return variants } case "country": { variants.push({ hierarchy, suffix: "self", components: { country: selfName } }) return variants } case "subregion": { variants.push({ hierarchy, suffix: "self", components: { subregion: selfName } }) return variants } default: return [] } } /** * Build the per-record name-slot list. * * The canonical `"default"` slot uses the OpenCage-canonical country form * when the record is itself a country (matches SQLite-adapter behavior); * every other placetype's default slot uses `wof:name` verbatim. */ export function nameSlotsFor(rec: WOFRecord): Array<{ key: string; value: string }> { return wofNameSlotsFor(rec, { canonicalName: (record) => placetypeToTag(record.placetype) === "country" ? (COUNTRY_DISPLAY_NAME[record.country] ?? record.name) : record.name, }) } /** * Registry id for this adapter. * * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from. */ export const WOF_ADMIN_ADAPTER_ID = "wof-admin" /** * Construct the wof-admin JSON-bundle adapter. * * The adapter is stateless across runs. * Two calls with the same input directory produce byte-identical `canonical.jsonl` * (records are emitted in sorted `wof:id` order to be insensitive to filesystem walk ordering). */ export function createWOFAdminAdapter(): CorpusAdapter { return { id: WOF_ADMIN_ADAPTER_ID, defaultLicense: "CC0-1.0", addressRole: AddressRole.Premise, register: SourceRegister.WhosOnFirst, surface: SurfaceOrigin.Rendered, description: "Who's On First admin GeoJSON bundles (countries, regions, counties, localities) — multi-name variants per record.", async *rows(opts: AdapterOptions): AsyncIterable { // Pass 1: scan every GeoJSON file once, build the in-memory record index. // We keep only records whose placetype maps to a ComponentTag — irrelevant // placetypes (campus, county-region hybrids on which Mailwoman has no opinion) // are dropped here so they don't inflate the ancestry index. // Country-filtered runs prune to the matching country code too. // The ancestors of a same-country record live in the same admin repo. const byID = new Map() for await (const rec of walkFeatures(opts.inputPath, { signal: opts.signal })) { if (opts.signal?.aborted) return if (opts.country && rec.country !== opts.country) continue if (!placetypeToTag(rec.placetype)) continue byID.set(rec.id, rec) } const ancestry = buildAncestorNameIndex(byID, placetypeToTag) // Pass 2: emit rows in sorted-id order for deterministic jsonl. yield* emitWOFJSONRows({ records: byID, ancestry, adapterOptions: opts, adapterID: WOF_ADMIN_ADAPTER_ID, localeByCountry: LOCALE_BY_COUNTRY, nameSlotsFor, variantsFor, }) }, } } /** * Single shared instance, suitable for `defaultAdapterRegistry`. */ export const wofAdminAdapter = createWOFAdminAdapter()