/** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * Regenerate `core/resources/languages/types.gen.ts` — the ISO 639-1 / 639-2 language-code types + * label maps — from the committed `internal/languages.csv` resource dictionary. * * BOTH 639-2 standards are emitted, and the reason is data, not completeness. ISO 639-2 has two * three-letter forms: /B (bibliographic) and /T (terminological, identical to 639-3). They differ * for exactly 20 languages, and **Who's On First keys its name properties in /T** — so a /B-only * union excluded `deu`, `fra` and `nld`, three tier-1 locales, while admitting spellings the data * never uses. Measured over the built gazetteer's `names` table: 3,591,751 rows on the /T side of * those 20 pairs against 247 on the /B side, a ratio of 14,542:1. * * /B is NOT dropped: `packages/codex/country/official-languages.ts` lists both forms deliberately * (`DE: ["de","deu","ger"]`) and 247 real rows carry one, so the union accepts either and both map * to the same label and the same alpha-2. * * NAME DEBT, deliberately not paid here: `Alpha3bLanguageCode`, `Alpha3bLabelMap`, `Alpha3bToAlpha2` * and the CSV's `alpha3-b` header all still say "b" while holding both standards. Renaming them is a * published-API break and belongs in a major. * * Usage: mailwoman dev generate language-types */ import * as fs from "node:fs/promises" import { pascalCase } from "change-case" import { CSVSpliterator } from "spliterator" import { resourceDictionaryPath, workspacePath } from "../utils/index.ts" /** * Options for {@linkcode generateLanguageTypes}. */ export interface GenerateLanguageTypesOptions { /** * Output path override. Default: `core/resources/languages/types.gen.ts` (the committed types). */ out?: string } /** * Summary returned by {@linkcode generateLanguageTypes}. */ export interface GenerateLanguageTypesSummary { languages: number outPath: string } /** * Regenerate the committed language-code types from `internal/languages.csv`. */ export async function generateLanguageTypes( options: GenerateLanguageTypesOptions = {}, report?: (line: string) => void ): Promise { const outfile = options.out ?? workspacePath("core", "resources", "languages", "types.gen.ts") const dataSourcePath = resourceDictionaryPath("internal", "languages.csv") const alpha2Entries = new Map() const alpha3bEntries = new Map() const entryLines: [alpha2: string, alpha3b: string][] = [] // The 20 divergent /T spellings, kept apart so they widen the accepting direction only. const alpha3tPairs: [alpha2: string, alpha3t: string][] = [] report?.(`Reading ${dataSourcePath}`) // Quote handling is on and load-bearing: six labels wrap an embedded comma // (`gre,el,"Greek, Modern (1453-)"`), and the spliterator leaves quoting off by default. // `header` defaults true, which is what skips the `alpha3-b,alpha2,English` line. for await (const columns of CSVSpliterator.fromAsync(dataSourcePath, { enableQuoteHandling: true })) { const alpha3b = columns[0] as string const alpha2 = columns[1] as string const labelsConcatenated = columns[2] as string // Empty for the 163 languages whose two 639-2 forms agree; a distinct code for the 20 that diverge. const alpha3t = (columns[3] as string | undefined) ?? "" const labels = labelsConcatenated.split("; ") alpha2Entries.set(alpha2, labels) alpha3bEntries.set(alpha3b, labels) entryLines.push([alpha2, alpha3b]) // The /T spelling is a first-class member of the same union and maps to the same label and the // same alpha-2. It is the form WOF actually writes, so a lookup keyed on it must hit. // // It goes to `alpha3tPairs`, NOT to `entryLines`. Both lists build BOTH direction maps from a // `new Map([...])`, where the LAST entry for a key wins — so appending the /T form to // `entryLines` would silently flip `Alpha2ToAlpha3b.get("de")` from `ger` to `deu`. That map is // named for the /B standard and documented as returning it; changing what it answers is a // separate decision from widening what the union ACCEPTS, and it is not this one. if (alpha3t) { alpha3bEntries.set(alpha3t, labels) alpha3tPairs.push([alpha2, alpha3t]) } } const handle = await fs.open(outfile, "w") const writeLine = (line: string) => handle.writeFile(`${line}\n`) // Header. await writeLine(` /** * @copyright Sister Software * @license AGPL-3.0 * @author Teffen Ellis, et al. * * This file was generated by: mailwoman dev generate language-types */ `) // Types. await writeLine(` /** * A two-letter language code. */ export type Alpha2LanguageCode = `) for (const alpha2 of alpha2Entries.keys()) { await writeLine(` | "${alpha2}"`) } await writeLine(` /** * An enumeration of ISO 639-1 language codes. */ export const Alpha2LanguageCode = { `) for (const [alpha2, labels] of alpha2Entries) { for (const label of labels) { await writeLine(` ${pascalCase(label)}: "${alpha2}",`) } } await writeLine(`} as const satisfies Record`) await writeLine(` /** * A three-letter language code. */ export type Alpha3bLanguageCode = `) for (const alpha3b of alpha3bEntries.keys()) { await writeLine(` | "${alpha3b}"`) } await writeLine(` /** * A valid ISO 639-1 or ISO 639-2 language code. */ export type LanguageCode = Alpha2LanguageCode | Alpha3bLanguageCode | "all" `) // Maps. await writeLine(` /** * A map of two-letter language codes to their labels. */ export const Alpha2LabelMap: ReadonlyMap = new Map([ `) for (const [alpha2, labels] of alpha2Entries) { await writeLine(`["${alpha2}", ${JSON.stringify(labels)}],`) } await writeLine(`])`) await writeLine(` /** * A map of three-letter language codes to their labels. */ export const Alpha3bLabelMap: ReadonlyMap = new Map([ `) for (const [alpha3b, labels] of alpha3bEntries) { await writeLine(`["${alpha3b}", ${JSON.stringify(labels)}],`) } await writeLine(`])`) // Conversion await writeLine(` /** * Convert a two-letter language code to a three-letter language code. */ export const Alpha2ToAlpha3b: ReadonlyMap = new Map([ `) for (const [alpha2, alpha3b] of entryLines) { await writeLine(`["${alpha2}", "${alpha3b}"],`) } await writeLine(`])`) await writeLine(` /** * Convert a three-letter language code to a two-letter language code. */ export const Alpha3bToAlpha2: ReadonlyMap = new Map([ `) // This direction ACCEPTS a code, so it takes both spellings — the keys are distinct, nothing is // overwritten, and `deu` answers `de` exactly as `ger` does. for (const [alpha2, alpha3] of [...entryLines, ...alpha3tPairs]) { await writeLine(`["${alpha3}", "${alpha2}"],`) } await writeLine(`])`) // Cleanup. await handle.close() report?.(`Wrote ${outfile}`) return { languages: entryLines.length, outPath: outfile } }