/* eslint-disable max-lines */ import path from 'node:path'; import { execSync } from 'node:child_process'; import { SMARTLING_BASE_URL, SMARTLING_PROJECT_ID, SMARTLING_FILES_PATH, DEFAULT_LOCALE, SMARTLING_STRINGS_PATH, SMARTLING_CONTEXT_PATH, STRINGS_PAGE_SIZE, INJECT_PLACEHOLDER_WORKAROUND, INJECTABLE_WORKAROUND_VALUE, } from '../constants.js'; import { Pattern, TextElement, FluentParser, FluentSerializer, } from '@fluent/syntax'; import type { Expression, PatternElement, Resource, Message, Attribute, Entry, } from '@fluent/syntax'; import { readFileSync, writeFileSync } from 'node:fs'; import { doFetch, fileExtensionToUse, getAuthToken, getRepoRoot, useFTLFlow, } from '../base-client/base-client.js'; const SMARTLING_CSV_HEADER = [ 'Key', 'Instruction', 'Character limit', 'Source String', ]; type CSVRow = { [K in (typeof SMARTLING_CSV_HEADER)[number]]: string; }; const gitRoot = getRepoRoot(); /** * * Extracts text from the elements of a pattern by recursively traversing the elements. * */ export function extractTextFromElements(elements: PatternElement[]): string { return elements .map((element) => { if (element.type === 'TextElement') { return element.value; } return extractTextFromPlaceable(element.expression); }) .map((text) => text.trim()) .join(' '); } /* * Extracts text from a placeable expression by recursively traversing the expression. * Based on the expression type, it extracts the text accordingly. */ export function extractTextFromPlaceable(expression: Expression): string { switch (expression.type) { case 'SelectExpression': { const selectExpr = expression; const selectorText = extractTextFromPlaceable(selectExpr.selector); const variantsText = selectExpr.variants .map( (v) => `[${v.key.name}] ${extractTextFromElements(v.value.elements)}`, ) .join(' '); return `{ ${selectorText} -> ${variantsText} }`; } case 'VariableReference': { return `{ $${expression.id.name} }`; } case 'MessageReference': { return `{ ${expression.id.name} }`; } case 'TermReference': { return `{ -${expression.id.name} }`; } case 'StringLiteral': { return expression.value; } case 'NumberLiteral': { return expression.value.toString(); } default: return ''; } } /** * This function extracts the attributes from the message and returns them as a string. * */ export function extractAttributesAsString(attributes: Attribute[]): string { return attributes .map((attribute) => { const attrValue = extractTextFromElements(attribute.value.elements); return `.${attribute.id.name} = ${attrValue}`; }) .join('\n'); } /** * This function parses the FTL file and returns the CSV rows. * */ export function parseFTLToCSV(filePath: string): CSVRow[] { const resource = getResourceFromFtlFile(filePath); const rows: CSVRow[] = resource.body .filter((entry): entry is Message => entry.type === 'Message') .map((message: Message) => { const key = message.id.name; const sourceString = message.value ? extractTextFromElements(message.value.elements) : ''; const attributesString = extractAttributesAsString(message.attributes); const combinedSourceString = attributesString ? `${sourceString}\n${attributesString}` : sourceString; return { Key: key, Instruction: '', // Set the specific instruction for the translators if needed 'Character limit': '', // Set the character limit if any 'Source String': combinedSourceString, }; }); return rows; } export function saveToCSV(rows: CSVRow[], outputFilePath: string): void { const csvContent = [SMARTLING_CSV_HEADER, ...rows.map(toSmartlingRow)].join( '\n', ); writeFileSync(outputFilePath, csvContent, 'utf8'); } function toSmartlingRow(row: CSVRow): string { return SMARTLING_CSV_HEADER.map((header) => toCSVField(row[header])).join( ',', ); } function toCSVField(field: string): string { if (!field) return ''; if (field.includes('"')) { field = field.replace(/"/g, '""'); } if (field.includes(',') || field.includes('"') || field.includes('\n')) { return `"${field}"`; } return field; } /** * This function uploads the file to Smartling once is converted to a CSV file. * * @param filePath */ // eslint-disable-next-line max-statements export async function uploadTranslationsFileToSmartling( filePath: string, ): Promise { const fileName = path.basename(filePath); let fileType = fileExtensionToUse.replace('.', ''); let blob = new Blob([readFileSync(filePath)]); if (fileType === 'ftl') { fileType = 'fluent'; if (INJECT_PLACEHOLDER_WORKAROUND) { const serializer = new FluentSerializer(); blob = new Blob([serializer.serialize(getResourceFromFtlFile(filePath))]); } } const fileUri = path.relative(gitRoot, filePath).replace(/\\/g, '/'); const formData = new FormData(); formData.append('file', blob, fileName); formData.append('fileUri', fileUri); formData.append('fileType', fileType); try { const token = await getAuthToken(); await doFetch( `${SMARTLING_BASE_URL}${SMARTLING_FILES_PATH}/${SMARTLING_PROJECT_ID}/file`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, }, body: formData, }, ); console.log(`Successfully uploaded ${filePath} to Smartling.`); } catch (error) { console.error(`Failed to upload ${filePath} to Smartling:`, error); throw error; } } /** * This function converts the FTL file to a CSV file. * */ export function convertFtlToCsv(filePath: string): string { const csvRows = parseFTLToCSV(filePath); const outputDir = path.dirname(filePath); const outputFilePath = path.join( outputDir, `${path.basename(filePath, '.ftl')}.csv`, ); saveToCSV(csvRows, outputFilePath); return outputFilePath; } interface SmartlingString { hashcode: string; stringText: string | null; stringVariant: string | null; parsedStringText: string; maxLength: number | null; stringInstructions: string[]; contentFileStringInstructions: { fileUri: string; contentFileStringInstruction: string; }[]; keys: { fileUri: string; key: string; }[]; } type FileName = string; const smartlingStringsCache: Record< FileName, Record > = {}; async function findStringInSmartlingPagination( translationKey: string, fileUri: string, ): Promise { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (smartlingStringsCache[fileUri]?.[translationKey]) return smartlingStringsCache[fileUri][translationKey]; const token = await getAuthToken(); let endSearchLoop = false; let offset = 0; let totalItems = 0; while (!endSearchLoop) { const { response: translationStringsResponse } = await doFetch( `${SMARTLING_BASE_URL}${SMARTLING_STRINGS_PATH}/${SMARTLING_PROJECT_ID}/source-strings?fileUri=${encodeURIComponent( fileUri, )}&limit=${STRINGS_PAGE_SIZE}&offset=${offset}`, { headers: { Authorization: `Bearer ${token}`, }, }, ); if (totalItems === 0) totalItems = translationStringsResponse.data.totalCount; const translationStrings: SmartlingString[] = translationStringsResponse.data.items; smartlingStringsCache[fileUri] = {}; for (const translationString of translationStrings) { smartlingStringsCache[fileUri][translationString.parsedStringText] = translationString; } // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (smartlingStringsCache[fileUri][translationKey]) endSearchLoop = true; offset += STRINGS_PAGE_SIZE; if (totalItems <= offset) endSearchLoop = true; } return smartlingStringsCache[fileUri][translationKey] ?? null; } async function uploadContextBindingToSmartling({ filePath, contextUid, }: { filePath: string; contextUid: string; }) { const { ext: extension } = path.parse(filePath); const fileName = path.basename(filePath); const key = fileName.replace(extension, ''); const token = await getAuthToken(); try { const translationFileUri = path .relative(gitRoot, filePath) .replace( `/context-screenshots/${fileName}`, `/${DEFAULT_LOCALE}.${useFTLFlow ? 'ftl' : 'csv'}`, ); const string = await findStringInSmartlingPagination( key, translationFileUri, ); if (!string) { throw new Error( `String not found in Smartling for file ${filePath} and key ${key}`, ); } // To avoid having uploaded context screenshots without a string binding, // I chose to do one binding per request in order. // If the job gets too slow due to too much requests, we can batch the bindings. const { response: bindingResponse } = await doFetch( `${SMARTLING_BASE_URL}${SMARTLING_CONTEXT_PATH}/${SMARTLING_PROJECT_ID}/bindings`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', }, body: JSON.stringify({ bindings: [{ contextUid, stringHashcode: string.hashcode }], }), }, ); if ( bindingResponse.code === 'SUCCESS' && bindingResponse.data.created.totalCount > 0 ) { console.log( `Successfully bound ${filePath} to its corresponding string in Smartling.`, ); } else { throw new Error('Failed to bind context screenshot to its string'); } } catch (error) { console.error( `Error binding context file ${filePath} to its corresponding string:`, error, ); } } async function uploadScreenshotToSmartling(filePath: string) { const { ext: extension, base: fileName } = path.parse(filePath); const token = await getAuthToken(); let contextUid: string | undefined; try { const formData = new FormData(); const blob = new Blob([readFileSync(filePath)], { type: `image/${extension.replace('.', '')}`, }); formData.append('name', fileName); formData.append('content', blob, fileName); const { response: responseJson } = await doFetch( `${SMARTLING_BASE_URL}${SMARTLING_CONTEXT_PATH}/${SMARTLING_PROJECT_ID}/contexts`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, }, body: formData, }, ); contextUid = responseJson.data.contextUid; console.log(`Successfully uploaded ${filePath} to Smartling.`); } catch (error) { console.error(`Failed to upload ${filePath} to Smartling:`, error); } return { contextUid }; } async function uploadScreenshotAndContextBinding(filePath: string) { const { contextUid } = await uploadScreenshotToSmartling(filePath); if (!contextUid) return; await uploadContextBindingToSmartling({ filePath, contextUid }); } async function uploadContextScreenshots() { const changedContextScreenshotFiles = execSync( 'git diff --name-only --diff-filter=d HEAD~1 HEAD', ) .toString() .split('\n') .filter( (file) => file.includes('/context-screenshots/') && file.endsWith('.png'), ); for (const file of changedContextScreenshotFiles) { if (file) { const absolutePath = path.join(gitRoot, file); await uploadScreenshotAndContextBinding(absolutePath); } } } export async function uploadTranslationsFiles(): Promise { // Filters deleted files. const changedFTLFiles = execSync( 'git diff --name-only --diff-filter=d HEAD~1 HEAD', ) .toString() .split('\n') .filter((file) => file.endsWith(`${DEFAULT_LOCALE}.ftl`)); for (const file of changedFTLFiles) { if (file) { const absolutePath = path.join(gitRoot, file); const filePath = useFTLFlow ? absolutePath : convertFtlToCsv(absolutePath); await uploadTranslationsFileToSmartling(filePath); } } } /** * Reads a .ftl file and returns its content as a Fluent Resource. * * If `INJECT_PLACEHOLDER_WORKAROUND` is true, applies a workaround to the resource. * * @param absolutePath the absolute path of the .ftl file. * @returns the content of the .ftl file as a Fluent Resource. */ export function getResourceFromFtlFile(absolutePath: string): Resource { const parser = new FluentParser(); const data = readFileSync(absolutePath, 'utf8'); const resource: Resource = parser.parse(data); if (INJECT_PLACEHOLDER_WORKAROUND) { injectPlaceholderValueWorkaround(resource); } return resource; } /** * A workaround for a Smartling bug. * * Smartling doesn't support messages with attributes but without a value. * This function adds a placeholder value to such messages. * * The naming of the function is intentionally bad to remove it ASAP once bug is fixed on * Smartling side. * * @param {Resource} resource - The resource to be modified */ export function injectPlaceholderValueWorkaround(resource: Resource): void { resource.body .filter( (resource: Entry) => resource.type === 'Message' && resource.attributes.length > 0 && !resource.value, ) .forEach((resource: Entry) => { resource.value = new Pattern([ new TextElement(INJECTABLE_WORKAROUND_VALUE), ]); }); } export async function upload(): Promise { try { await uploadTranslationsFiles(); await uploadContextScreenshots(); } catch (error) { console.error('Error processing files:', error); throw error; } }