/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * `sub-venue` shard recipe (#35 step 4) — teach the `unit` tag the venue-INTERIOR shapes it was * never taught, so the modifier+designator class wins at the shipped `venueStructureBiasScale` of * 6.0 instead of needing ~11 nats. `docs/engineering/sub-venue-corpus-task.mdx` is the spec; the * vocabulary is `corpus/data/sub-venue-lexicon.json` (v0.2.0) and the curation ledger is * `corpus/src/tools/sub-venue-promotions.ts`. The READ half — promotions, identifier * distributions, name pools — is `sub-venue-sources.ts`; this file renders lines and emits rows. * * ── WHY THIS SYNTHESIZES RATHER THAN HARVESTS ──────────────────────────────────────────────────── * Wave 1's lesson, and the reason the spec's "get real data first" instruction is honoured in a * shape it did not anticipate: the attested SURFACE STRINGS are thin. 87 GB features attest * `terminal`, 29 attest `wing`, 4 attest `concourse` (3 of which are a street called CONCOURSE WAY). * You cannot train a tag on 29 strings. What the five extracts DO carry at volume is the three * things a generator needs — 45,000+ real venue names across four countries, a per-region * identifier DISTRIBUTION measured over 2,868 gate/terminal/campus refs, and the confound * population that becomes the negatives. So the bulk is `designator × per-region-identifier × * modifier` sampled per locale, and the attested strings ride along as seasoning * ({@link ATTESTED_FRACTION}) rather than as the corpus. * * ── THE PER-REGION IDENTIFIER RULE, AND WHY IT IS NOT COSMETIC ─────────────────────────────────── * `Gate A12` is a rendering, not a string anyone wrote down: all but 13 of Great Britain's 658 * `aeroway=gate` features are unnamed and carry only a `ref`. The lexicon therefore ships a * distribution, and it differs by country far more than the shared English vocabulary suggests — * GB gates are 71% bare digits and JP 89%, FR and DE are ~60% letter-digit (`A37`, `B05`), and ES * gives a THIRD of its gates a range (`B18-B20`), which no other country does at that rate. A * generator that samples Great Britain's shape into a Spanish line produces a plausible string that * is wrong about Spain, so every leg samples its own region. * * ── ONLY PROMOTED (designator, locale) PAIRS PRODUCE POSITIVES ─────────────────────────────────── * A promotion names a designator, a phrase AND a locale, because the same token is a designator in * one language and a disaster in another: `hall` is 0-of-3,273 in Great Britain and 35-of-40 in * France; `wing` is 23-of-29 in Great Britain and 4-of-3,358 in the United States. A REJECTED pair * generates NEGATIVES in that locale instead — en-US `wing` rows are Red Wing, not units. * * `shape: "identifier-required"` is honoured as the ledger's docstring demands: de-DE `halle` is * emitted only as `Halle `, never bare and never after a modifier, because its 168-hit * confound includes the CITY Halle (Saale) and only the identifier-bearing shape separates them. * {@link buildSubVenueForm} enforces it and `sub-venue.test.ts` pins it. * * ── LABELS ─────────────────────────────────────────────────────────────────────────────────────── * Sub-venue is `unit`; the container is `venue`. That is the spec's wording and it invents nothing: * `block` / `sub_block` exist in the `ComponentTag` union but are JP-char-model-only and outside * `ACTIVE_TAGS` (STAGE3), so they are not reachable from a Latin shard. * * ── WHAT IS DELIBERATELY NOT HERE ──────────────────────────────────────────────────────────────── * 1. **A modifier+designator form outside English.** `VENUE_STRUCTURE_MODIFIERS` is an English * list, and the extracts say the localized modifier surfaces do not exist to copy: `aile` in * France is 0 hits, `ala` in Spain 0, `flügel` in Germany 0. Generating `Terminal Sud` would * be inventing a vocabulary with no confound board behind it, which is the exact failure the * promotion ledger exists to prevent. Non-English legs get designator+identifier only. * 2. **A ja-JP leg.** `ターミナル` is promoted (1,213 real of 1,215) and it is the one non-Latin * surface the task named, but Japanese addresses train through `build_jp_shard.py` against the * `stage3-jp` 47-label head, where the interior tags are `block`/`sub_block`/`building_number` * — a different model, a different label set, and a different builder. A katakana `unit` row * in this (Latin) feed would in any case be dropped by `country_weights`, which carries no * `JP` key. The JP extract is harvested and ready; the leg belongs to the JP shard. */ import type { ComponentTag } from "@mailwoman/core/types"; import type { LocaleBaseTuple } from "@mailwoman/corpus/synthesizers/german"; import { type ShardRecipe } from "./scaffold.ts"; import { type IdentifierModel, type PromotedSurface } from "./sub-venue-sources.ts"; export * from "./sub-venue-sources.ts"; /** * One locale's leg of the shard. * * `positiveShare` / `negativeShare` are relative weights, normalized at run time — they do not have to sum to 1. */ export interface SubVenueLeg { locale: string; country: string; /** * ISO 3166-1 alpha-2 key into the lexicon's `identifierShapes` AND the extract filename. The two axes are the same * axis: a distribution is measured in a region's own extract. */ region: string; /** * Extract filename under `--extracts-dir`. Absent = no OSM extract for this leg (en-US), which then draws its venue * and confound pools from `poi.db` instead. */ extract?: string; /** * May this leg use the English ` ` grammar? See the module docstring's exclusion 1. */ english: boolean; positiveShare: number; negativeShare: number; /** * Ca-ES only — keep context tuples whose postcode starts with one of these. Catalan-language territories by postal * prefix (07 Illes Balears, 08 Barcelona, 17 Girona, 25 Lleida, 43 Tarragona) rather than by a REGION string, whose * spelling in the OA export is not something to guess at. */ postcodePrefixes?: readonly string[]; } /** * The legs, and the numbers behind the shares. * * En-GB and en-US carry the most because they are where the eval board lives (28 of the 30 confound rows are GB or US * addresses) and because the English shipped vocabulary is the only one with a modifier grammar — the target class. * fr-FR, de-DE and es-ES exist because the ledger promoted surfaces there (169, 19+32 and 190 real hits respectively) * and a shard that skipped them would leave every non-English promotion untrained. ca-ES is small on purpose: its * promotion is 15 hits and its line differs from es-ES only in the Catalan street vocabulary the postal-prefix filter * selects for. * * The negative shares invert that ordering where the confound mass does. en-US carries the largest negative share * because its confound population is the largest measured anywhere in the ledger — 3,354 `wing` (Red Wing boots 676, * chicken wings 759), 2,330 `pier` (Pier 1 Imports, which IS the designator+identifier shape), 27,081 `hall`. */ export declare const SUBVENUE_LEGS: readonly SubVenueLeg[]; /** * En-US has no OSM extract, so its identifier distribution has to be borrowed. GB is the borrow, and the leg's `region` * says so literally rather than in a comment: the two English-speaking aviation systems number their gates the same way * (GB 71% bare digit) and poi.db — the only US source in reach — carries names, not refs, so it cannot supply a * distribution of its own. Recorded here because it is the one place a leg's `region` is not its own country. */ export declare const US_IDENTIFIER_REGION_BORROWED_FROM = "GB"; /** * The row count this shard is built at, and the arithmetic behind it. `--count` overrides; this is the number to use * absent a reason. * * The training sampler (`corpus-python/src/mailwoman_train/data_loader.py`, `_raw_row_stream`) draws SOURCES from a * multinomial over `source_weights` and yields the next row from that source's iterator. Two consequences set the * size: * * 1. A source's share of an epoch is `weight / Σweights`, independent of how many rows it has. * 2. **A source that exhausts is DELETED from the multinomial** — there is no cycling. Under-size the shard and its * nominal dose is fiction for the rest of the epoch. * * Measured against the shipped `v4.1.0-gb-venue-l1e4-2k` weight table: 33 sources summing to 144.5. At the dose the B11 * GB-venue exercise settled on for a hard rare class — 12.0, the value `synth-fr-bare-street`, `synth-si-bare-village`, * `synth-cz-pcfirst-preposition`, `synth-fr-fragment` and `synth-no-fragment` all carry — the share is `12 / 156.5 = * 7.67%`, and `train_rows_per_epoch` is 1,000,000. So the epoch draws **76,677 rows** from this shard, and anything * smaller runs dry mid-epoch. 120,000 clears that with room for a config that drops a source or raises the dose. (For * contrast: `synth-fr-bare-street` is 10,803 rows at dose 12.0, so it exhausts 14% into its own nominal share every * epoch — a precedent for the dose, not for the size.) */ export declare const RECOMMENDED_ROW_COUNT = 120000; /** * Surfaces reserved by `mailwoman/eval-harness/fixtures/venue-structure-confounds.jsonl` — the 30-row board this shard * has to hold. A row containing any of these is DROPPED and counted in `contaminated`. * * The `--exclude-surfaces` precedent from `fr-fragment` / `no-fragment`, applied by hand rather than by file because * the board lives in `mailwoman/` and `@mailwoman/corpus` cannot reach across that workspace boundary at run time. Keep * it in sync when the board grows; a shard that trains on its own eval set measures memorization. * * Note what this costs and why it is still right: reserving `east gate` / `west gate` removes the two GB surfaces the * board uses for its `modifier-designator-street` class, so the shard teaches that class from the OTHER real ones its * sources carry (`North Gate`, `South Gate`, `East Hall`, `West Hall`, `Lower Hall`, `East Campus`, …). The class is * taught; the board's own strings are not. */ export declare const BOARD_RESERVED_SURFACES: readonly string[]; /** * Does this row's text collide with a board-reserved surface? */ export declare function isBoardReserved(raw: string): boolean; /** * One labelled piece of the line. Pieces inside a group are space-joined; groups are joined by the register's * separator. */ interface Piece { text: string; tag?: ComponentTag; } type Group = Piece[]; /** * Surface register. Every eval in this repo gets a lowercase leg because lowercase is the register users type — Google * Maps taught them — so every shard has to carry one. */ declare const Register: { readonly Canonical: "canonical"; readonly CommaFree: "comma-free"; readonly Lower: "lower"; readonly Upper: "upper"; }; type Register = (typeof Register)[keyof typeof Register]; /** * Join groups into `raw` + `components`, applying the register to BOTH so alignment still finds every value. */ export declare function renderGroups(groups: Group[], register: Register): { raw: string; components: Partial>; }; /** * The street + tail groups for a country, in that country's own order. * * DE/ES/FR put the postcode before the locality and DE/ES put the house number after the street; GB and US keep the * anglophone order and US carries a region. These are the same orders `synthesizers/german.ts` renders, restated here * because this recipe assembles its groups piece-by-piece (it has to, to place a sub-venue group in front of them). */ export declare function addressGroups(country: string, tuple: LocaleBaseTuple, withStreet: boolean): Group[]; /** * A sub-venue string plus how it was made, for the composition report. */ export interface SubVenueForm { text: string; form: "designator-identifier" | "modifier-designator" | "attested"; designatorID: string; } /** * Build one sub-venue string for a leg. * * The `identifier-required` guard is here and not at the call site on purpose: it is the one rule in this file that a * refactor must not be able to route around. A promotion carrying `shape: "identifier-required"` can only ever leave * this function as ` `, and returns `null` rather than a bare or modified form. */ export declare function buildSubVenueForm(leg: SubVenueLeg, promoted: readonly PromotedSurface[], model: IdentifierModel, modifiers: readonly string[], attested: readonly string[], random: () => number): SubVenueForm | null; /** * Alias kept for readers of the arc's earlier drafts. * * @deprecated Use {@link buildSubVenueForm}. */ export declare const buildPositiveForms: typeof buildSubVenueForm; /** * Negative classes, named so the composition report can count them and a failure can be attributed. */ export declare const NegativeClass: { /** * A locale-REJECTED surface in the venue slot: Red Wing Shoes, Village Hall, Porte de Champerret. */ readonly RejectedVenue: "rejected-venue"; /** * The designator inside a longer proper name, whole string tagged `venue`: Lochaline Ferry Terminal. */ readonly LongerName: "longer-name"; /** * A real street whose name carries a designator token: Pier Road, Egg Hall, Orchard Gate. */ readonly DesignatorStreet: "designator-street"; /** * A real street of the ` ` shape — the class that would otherwise be read as a sub-venue. */ readonly ModifierDesignatorStreet: "modifier-designator-street"; /** * A GB single-token `-gate` street: Eastgate, Southgate, Moorgate, Stonegate. */ readonly GateSuffixStreet: "gate-suffix-street"; /** * A PROMOTED phrase outside the shape its promotion covers — `Halle Rosengarten`, `PHOENIX Halle`. The other half of * an `identifier-required` ruling; see `LegPools.unpromotedShapes`. */ readonly UnpromotedShape: "unpromoted-shape"; }; export type NegativeClass = (typeof NegativeClass)[keyof typeof NegativeClass]; /** * Per-leg composition tallies the build prints and the report quotes. */ export interface SubVenueLegStats { locale: string; positives: number; negatives: number; byForm: Record; byDesignator: Record; byNegativeClass: Record; byRegister: Record; poolSizes: Record; } /** * Split `total` across `shares` (which need not sum to 1), largest-remainder so the parts sum exactly. */ export declare function allocate(total: number, shares: readonly number[]): number[]; /** * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise, * and `description` below for the surface form it generates. */ export declare const subVenueRecipe: ShardRecipe; //# sourceMappingURL=sub-venue.d.ts.map