/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * @file Convert a recipe output written before `recipe`/`register`/`surface` to the current row schema. * * `recipe` and `base_source_id` are renames of `synth_method` and `synth_base_id`, so they assert no new claim. * `register` is `mailwoman-derived-tuples` for every row, because naming a publisher would record inference as fact. */ import { openWriteStream } from "@mailwoman/core/fs/streams" import { parseJSONStrict, stringifyJSON } from "@mailwoman/core/json" import { once } from "@mailwoman/core/utils/events" import type { PathBuilderLike } from "path-ts" import { TextSpliterator } from "spliterator" import { retiredSourceName } from "#recipes/sources" import { SourceRegister } from "#registers" import { SurfaceOrigin } from "#types" /** * How each source id's recipe produced its surface. * * A source absent from this table refuses rather than taking a default. */ export const RECIPE_SURFACES: Record = { // Assembled from template tables. // No register asserts the address exists. "synth-intersection": SurfaceOrigin.Invented, "synth-po-box-cedex": SurfaceOrigin.Invented, "synth-house-venue": SurfaceOrigin.Invented, "deepseek-kryptonite": SurfaceOrigin.Invented, // A real record's fields, written in an order the recipe chose. "synth-affix": SurfaceOrigin.Composed, "synth-fr-order": SurfaceOrigin.Composed, "synth-fr-bare-street": SurfaceOrigin.Composed, "synth-fr-lieudit": SurfaceOrigin.Composed, "synth-suffix-boundary": SurfaceOrigin.Composed, "synth-unit": SurfaceOrigin.Composed, "synth-unit-v30": SurfaceOrigin.Composed, "synth-trailing-region": SurfaceOrigin.Composed, "synth-trailing-region-us": SurfaceOrigin.Composed, "synth-trailing-region-structured": SurfaceOrigin.Composed, "synth-trailing-region-ca-bare": SurfaceOrigin.Composed, "synth-trailing-region-es-v28": SurfaceOrigin.Composed, "synth-cz-pcfirst-preposition": SurfaceOrigin.Composed, "synth-cz-pcfirst-preposition-v21": SurfaceOrigin.Composed, "synth-fr-bare-street-v22": SurfaceOrigin.Composed, "synth-si-bare-village": SurfaceOrigin.Composed, "synth-no-street-led": SurfaceOrigin.Composed, "synth-no-fragment": SurfaceOrigin.Composed, "synth-fr-fragment": SurfaceOrigin.Composed, "synth-fragment": SurfaceOrigin.Composed, "synth-fr-admin-split": SurfaceOrigin.Composed, "synth-nl-postcode": SurfaceOrigin.Composed, "synth-sub-venue": SurfaceOrigin.Composed, "synth-german": SurfaceOrigin.Composed, "synth-nz": SurfaceOrigin.Composed, "synth-es": SurfaceOrigin.Composed, "synth-gb": SurfaceOrigin.Composed, "synth-sg-register": SurfaceOrigin.Composed, "synth-pk-register": SurfaceOrigin.Composed, "synth-bd-register": SurfaceOrigin.Composed, "synth-reviewed-postcode-tail": SurfaceOrigin.Composed, // The publisher's own string, preserved by an adapter rather than a recipe. overture: SurfaceOrigin.Attested, "overture-latam": SurfaceOrigin.Attested, gnaf: SurfaceOrigin.Attested, osm: SurfaceOrigin.Attested, // Written from this repository's own tables. // They publish the string verbatim. "synth-bare-country": SurfaceOrigin.Attested, "synth-bare-postcode": SurfaceOrigin.Attested, // Drawn rather than read: the box number, the anchor and the venue are the recipe's. "synth-po-box-military": SurfaceOrigin.Invented, "synth-anchor-absorption": SurfaceOrigin.Invented, } /** * What a migration pass observed. */ export interface MigrationSummary { rows: number bySource: Record alreadyMigrated: number } /** * The surface for one source id, refusing a source the table does not name. */ export function surfaceForSource(source: string): SurfaceOrigin { // The table keys on the retired spelling. // A corpus that uses the current spelling needs the retired key for this lookup. // Otherwise each row fails as though it had no surface. // That failure misstates a renamed source as an unrecorded one. const surface = RECIPE_SURFACES[retiredSourceName(source)] if (!surface) { throw new Error( `No surface recorded for source ${stringifyJSON(source)}. Read the recipe that writes it, ` + `add it to RECIPE_SURFACES, and say there whether it renders a real record or assembles one.` ) } return surface } /** * Rewrite one recipe output onto the current row schema, passing an already-migrated * row through untouched so a half-migrated file converges. */ export async function migrateRecipeOutput(input: PathBuilderLike, output: PathBuilderLike): Promise { const summary: MigrationSummary = { rows: 0, bySource: {}, alreadyMigrated: 0 } const sink = openWriteStream(output, { encoding: "utf8" }) for await (const line of TextSpliterator.fromAsync(input)) { const row = parseJSONStrict>(line) summary.rows++ const source = String(row["source"] ?? "") summary.bySource[source] = (summary.bySource[source] ?? 0) + 1 if (row["surface"]) { summary.alreadyMigrated++ if (!sink.write(`${line}\n`)) { await once(sink, "drain") } continue } const { synth_method: recipe, synth_base_id: baseSourceID, ...rest } = row const migrated = { ...rest, recipe: recipe ?? null, base_source_id: baseSourceID ?? null, register: SourceRegister.DerivedTuples, surface: surfaceForSource(source), } if (!sink.write(`${stringifyJSON(migrated)}\n`)) { await once(sink, "drain") } } sink.end() await once(sink, "finish") return summary }