/* Downloads the latest translations from Transifex */ import { setTimeout } from 'node:timers/promises'; import fs from 'fs'; import { load as loadYaml } from 'js-yaml'; import { transifexApi, type Collection } from '@transifex/api'; import { dereferencedTranslatableContent } from './references.ts'; import type { Options, References, ResourceInfo, SourceStrings, TStrings } from './types.def.ts'; export function expandTStrings(localeCode: string, tstrings: TStrings, sourceLanguageTranslations?: TStrings) { const isCommonwealthEnglish = localeCode.startsWith('en-') && localeCode !== 'en-US'; const presets = tstrings.presets || {}; for (const key of Object.keys(presets)) { let preset = presets[key]; if (preset.name) { let names = preset.name.split('\n').map(s => s.trim()).filter(Boolean); preset.name = names[0]; if (names.length > 1) { preset.aliases = names.slice(1); } } if (typeof preset.aliases === 'string') { preset.aliases = preset.aliases.split('\n'); } // remove duplicates const rawTerms = typeof preset.terms === 'string' // remove translation message if it was included somehow ? preset.terms.replace(/<.*>/, '') // convert to an array .split(',') : []; // for en-* translations, copy the american translations as alternative terms const americanPreset = sourceLanguageTranslations?.presets[key]; if (isCommonwealthEnglish && americanPreset) { const extaTerms = [ ...(Array.isArray(americanPreset.terms) ? americanPreset.terms : []), ...(Array.isArray(americanPreset.aliases) ? americanPreset.aliases : []), ...(americanPreset.name ? [americanPreset.name] : []), ]; const existingName = (preset.name || americanPreset.name)?.toLowerCase(); rawTerms.push(...extaTerms.filter(value => value.toLowerCase() !== existingName)); } if (!rawTerms.length) continue; preset.terms = Array.from(new Set( rawTerms // make everything lowercase and remove whitespace .map(s => s.toLowerCase().trim()) // remove empty strings .filter(Boolean) )); if (!preset.terms.length) { // no need to include empty terms delete preset.terms; if (!Object.keys(preset).length) { delete presets[key]; } } } const fields = tstrings.fields || {}; for (const key of Object.keys(fields)) { let field = fields[key]; // remove duplicates const rawTerms = typeof field.terms === 'string' // remove translation message if it was included somehow ? field.terms.replace(/\[.*\]/, '') // convert to an array .split(/[,،]/) : []; // for en-* translations, copy the american translations as alternative terms const americanField = sourceLanguageTranslations?.fields[key]; if (isCommonwealthEnglish && americanField) { const extaTerms = [ ...(Array.isArray(americanField.terms) ? americanField.terms : []), ...(Array.isArray(americanField.aliases) ? americanField.aliases : []), ...(americanField.label ? [americanField.label] : []), ] const existingLabel = (field.label || americanField.label)?.toLowerCase(); rawTerms.push(...extaTerms.filter(value => value.toLowerCase() !== existingLabel)); } if (!rawTerms.length) continue; field.terms = Array.from(new Set( rawTerms // make everything lowercase and remove whitespace .map(s => s.toLowerCase().trim()) // remove empty strings .filter(Boolean) )); if (!field.terms.length) { // no need to include empty terms delete field.terms; if (!Object.keys(field).length) { delete fields[key]; } } } } /** * Sorts the keys of an object alphabetically */ export function sortObject(original: T): T { const sorted = {} as T; for (const k of Object.keys(original).sort() as (keyof T)[]) { sorted[k] = original[k]; } return sorted; } async function fetchTranslations(_options: Partial, references: References, sourceLanguageTranslations: TStrings) { // Transifex doesn't allow anonymous downloading if (process.env.transifex_password) { // Deployment scripts may prefer environment variables transifexApi.setup({ auth: process.env.transifex_password }); } else { // Credentials can be stored in transifex.auth as a json object. This file is gitignored. // You must use an API token for authentication: You can generate one at https://www.transifex.com/user/settings/api/. // { // "password": "" // } transifexApi.setup({ auth: JSON.parse(fs.readFileSync('./transifex.auth', 'utf8')).password }); } const options: Options = { translOrgId: '', translProjectId: '', translResourceIds: ['presets'], translReviewedOnly: false, outDirectory: 'dist', sourceLocale: 'en', ..._options, }; const outDir = `./${options.outDirectory}/translations`; if (!fs.existsSync(outDir)) { fs.mkdirSync(outDir); } const translResourceIds = options.translResourceIds!; const resources = await Promise.all(translResourceIds.map(getResource)); const resourceInfos = await Promise.all(translResourceIds.map(getResourceInfo)); gotResource(resources); gotResourceInfo(resourceInfos); async function getResourceInfo(resourceId: string): Promise { try { const result: ResourceInfo[] = []; for await (const stat of transifexApi.ResourceLanguageStats.filter({ project: `o:${options.translOrgId}:p:${options.translProjectId}`, resource: `o:${options.translOrgId}:p:${options.translProjectId}:r:${resourceId}` }).all()) { result.push(stat as unknown as ResourceInfo); } process.stdout.write(`got resource language stats collection for ${resourceId}\n`); return result; } catch (err) { process.stderr.write(`error while getting resource language stats collection for ${resourceId}\n`); console.error(err); throw err; } } function gotResourceInfo(results: ResourceInfo[][]) { let coverageByLocaleCode: { [localeCode: string]: number } = {}; results.forEach(function(info) { info.forEach(stat => { let code = stat.relationships.language.data.id.substr(2).replace(/_/g, '-'); let type = 'translated_strings'; if (options.translReviewedOnly && ( !Array.isArray(options.translReviewedOnly) || options.translReviewedOnly.indexOf(code) !== -1)) { type = 'reviewed_strings'; } let coveragePart = (stat.attributes[type] / stat.attributes.total_strings) / results.length; if (coverageByLocaleCode[code] === undefined) coverageByLocaleCode[code] = 0; coverageByLocaleCode[code] += coveragePart; }); }); let dataLocales: { [localeCode: string]: { pct: number } } = {}; // explicitly list the source locale as having 100% coverage dataLocales[options.sourceLocale] = { pct: 1 }; for (let code in coverageByLocaleCode) { let coverage = coverageByLocaleCode[code]; // we don't need high precision here, but we need to know if it's exactly 100% or not coverage = Math.floor(coverage * 100) / 100; dataLocales[code] = { pct: coverage }; } const sortedLocales = sortObject(dataLocales); fs.writeFileSync(`${outDir}/index.json`, JSON.stringify(sortedLocales)); fs.writeFileSync(`${outDir}/index.min.json`, JSON.stringify(sortedLocales, null, 4)); } async function getResource(resourceId: string): Promise { const codes = await getLanguages(); const promises = []; for (const code of codes) { // wait 50ms between each request to avoid sending 100s // of requests at once to transifex. await setTimeout(50); promises.push(getLanguage(resourceId)(code)); } const results = await Promise.all(promises); let locale: SourceStrings = {}; results.forEach((result, i) => { expandTStrings(codes[i], result.presets || {}, sourceLanguageTranslations); locale[codes[i]] = result; }); return locale; } function gotResource(results: SourceStrings[]) { // merge in strings fetched from transifex let allStrings: SourceStrings = {}; results.forEach(resourceStrings => { Object.keys(resourceStrings).forEach(code => { if (!allStrings[code]) allStrings[code] = {}; let source = resourceStrings[code]; let target = allStrings[code]; Object.keys(source).forEach(k => target[k] = source[k]); }); }); for (let code in allStrings) { if (allStrings[code].presets) { dereferencedTranslatableContent(allStrings[code].presets, references, false); } let obj: { [localeCode: string]: { presets?: TStrings } } = {}; obj[code] = allStrings[code] || {}; fs.writeFileSync(`${outDir}/${code}.json`, JSON.stringify(obj, null, 4)); fs.writeFileSync(`${outDir}/${code}.min.json`, JSON.stringify(obj)); } } function getLanguage(resourceId: string) { return async (code: string): Promise => { try { code = code.replace(/-/g, '_'); let reviewedOnly = options.translReviewedOnly && ( !Array.isArray(options.translReviewedOnly) || options.translReviewedOnly.indexOf(code) !== -1); const url = await transifexApi.ResourceTranslationsAsyncDownload.download({ resource: {data:{type:'resources', id:`o:${options.translOrgId}:p:${options.translProjectId}:r:${resourceId}`}}, language: {data:{type:'languages', id:`l:${code}`}}, // fetch only reviewed strings for some languages mode: reviewedOnly ? 'reviewed' : 'default' }); const data = await fetch(url).then(d => d.text()); process.stdout.write(`got translations for ${resourceId}, language ${code}\n`); return (loadYaml(data) as SourceStrings)[code]; } catch (err) { process.stderr.write(`error while getting translations for ${resourceId}, language ${code}\n`); throw err; } }; } async function getLanguages() { try { const result: string[] = []; const project = await transifexApi.Project.get({ organization: `o:${options.translOrgId}`, slug: options.translProjectId }); const lngs = await project.fetch('languages', false) as Collection; for await (const lng of lngs.all() as { attributes: { code: string } }[]) { if (lng.attributes.code === 'en') continue; result.push(lng.attributes.code.replace(/_/g, '-')); } process.stdout.write('got project languages\n'); return result; } catch (err) { process.stderr.write('error while getting project languages\n'); throw err; } } } export default fetchTranslations;