/* eslint-disable max-lines */ import { parse } from 'csv-parse/sync'; import { SMARTLING_BASE_URL, SMARTLING_PROJECT_ID, SMARTLING_FILES_PATH, SMARTLING_JOBS_PATH, INJECT_PLACEHOLDER_WORKAROUND, INJECTABLE_WORKAROUND_VALUE, } from '../constants.js'; import { existsSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { getAuthToken, getRepoRoot, useFTLFlow, skipUnavailableLocales, fileExtensionToUse, doFetch, } from '../base-client/base-client.js'; import { execSync } from 'node:child_process'; import { FluentParser, FluentSerializer } from '@fluent/syntax'; import { EmptyTranslationError } from './errors/empty-translation-error.js'; import { MalformedSmartlingResponseError } from './errors/malformed-smartling-response-error.js'; type SmartlingJobDetails = Readonly<{ translationJobUid: string; jobName: string; jobNumber: string; targetLocaleIds: string[]; description: string; dueDate: string; referenceNumber: string; callbackUrl: string | null; callbackMethod: string | null; createdDate: string; modifiedDate: string; createdByUserUid: string; modifiedByUserUid: string | null; firstCompletedDate: string; lastCompletedDate: string; firstAuthorizedDate: string; lastAuthorizedDate: string; jobStatus: SmartlingJobStatus; issues: { sourceIssuesCount: number; translationIssuesCount: number; }; customFields: unknown[]; sourceFiles: SmartlingSourceFile[]; priority: string | null; rushRequest: boolean; }>; type SmartlingSourceFile = Readonly<{ fileUid: string; uri: string; name: string; }>; export const enum SmartlingJobStatus { DRAFT = 'DRAFT', AWAITING_AUTHORIZATION = 'AWAITING_AUTHORIZATION', IN_PROGRESS = 'IN_PROGRESS', COMPLETED = 'COMPLETED', CANCELLED = 'CANCELLED', CLOSED = 'CLOSED', DELETED = 'DELETED', } type SmartlingJobInfo = Readonly<{ translationJobUid: string; jobName: string; jobNumber: string; dueDate: string; targetLocaleIds: string[]; createdDate: string; jobStatus: SmartlingJobStatus; referenceNumber: string; description: string; rushRequest: boolean; }>; const SOURCE_STRING_KEY = 'Source String'; /** * List all file Uris associated with the project. * @param {string} token Smartling API token * @returns {Promise} List of file Uris * @throws {Error} If failed to list files or if no files are found */ async function listAllFileUris(token: string): Promise { const fileUris: string[] = []; const res = await doFetch( `${SMARTLING_BASE_URL}${SMARTLING_FILES_PATH}/${SMARTLING_PROJECT_ID}/files/list`, { method: 'GET', headers: { Authorization: `Bearer ${token}` }, }, ); const items = res?.response?.data?.items; if (!items || items.length === 0) { throw new Error( `No files found for project ${SMARTLING_PROJECT_ID}. Check your project in Smartling to see if it's a Smartling API issue or your configuration`, ); } fileUris.push(...items.map((item: { fileUri: string }) => item.fileUri)); return fileUris; } /** * List all enabled target locales associated with the project. * @param {string} token Smartling API token * @returns {Promise} List of enabled target locales * @throws {Error} If failed to get project details or if no targetLocales are found */ async function listAllTargetLocales(token: string): Promise { const res = await doFetch( `${SMARTLING_BASE_URL}/projects-api/v2/projects/${SMARTLING_PROJECT_ID}`, { method: 'GET', headers: { Authorization: `Bearer ${token}` }, }, ); const locales = res?.response?.data?.targetLocales; if (!locales) { throw new Error( `No targetLocales found in project details for project ${SMARTLING_PROJECT_ID}. Check your project in Smartling to see if it's a Smartling API issue or your configuration`, ); } return locales .filter((loc: { enabled: boolean }) => loc.enabled) .map((loc: { localeId: string }) => loc.localeId); } export async function forcePullAllPublishedTranslations(): Promise { try { const token = await getAuthToken(); const [fileUris, locales] = await Promise.all([ listAllFileUris(token), listAllTargetLocales(token), ]); const sourceFiles = []; for (const fileUri of fileUris) { const sourceFile = { uri: fileUri, name: parseLocaleName(fileUri), fileUid: parseLocaleName(fileUri), }; sourceFiles.push(sourceFile); } await fetchAndSaveFiles([{ sourceFiles, locales }], token); } catch (error) { console.error('Error force pulling translations from Smartling:', error); throw error; } } /** * * This function pulls the translations from Smartling API * and saves them to the local file system. * */ export async function pullTranslationsFromSmartling(): Promise { try { const token = await getAuthToken(); const localeList = await Promise.all( await getLocalesThatNeedUpdates(token), ); await fetchAndSaveFiles(localeList, token); } catch (error) { console.error('Error pulling translations from Smartling:', error); throw error; } } /** * * This function fetches the list of CSV files from Smartling * based on the project ID and returns the list of items * */ export async function getJobList(token: string): Promise { const result = await doFetch( `${SMARTLING_BASE_URL}${SMARTLING_JOBS_PATH}/${SMARTLING_PROJECT_ID}/jobs`, { method: 'GET', headers: { Authorization: `Bearer ${token}`, }, }, ); const items = result?.response?.data?.items; if (!items) { throw new MalformedSmartlingResponseError( 'Job have no items (should at least have empty array)', ); } return items; } /** * * This function fetches the locale files from Smartling and saves them to the local file system with the correct format * */ async function fetchAndSaveFiles( relevantLocales: { locales: string[]; sourceFiles: SmartlingSourceFile[]; }[], token: string, ): Promise { const tasks: Promise[] = []; const failures: [string, string][] = []; for (const relevantLocale of relevantLocales) { for (const sourceFile of relevantLocale.sourceFiles) { for (const locale of relevantLocale.locales) { const task = doFetchAndSaveFile(locale, sourceFile, token); task.catch((error) => { failures.push([ locale, error instanceof Error ? error.message : String(error), ]); }); } } } await Promise.all(tasks); if (failures.length > 0) { const message = `the following locales failed:${failures.map( ([locale, reason]) => `\n ${locale}: ${reason}`, )}`; throw new Error(message); } } async function doFetchAndSaveFile( locale: string, sourceFile: SmartlingSourceFile, token: string, ) { try { const fileContent = await getFileContent(locale, sourceFile.uri, token); saveFile(locale, fileContent, sourceFile.uri); } catch (ex: unknown) { if (skipUnavailableLocales && ex instanceof EmptyTranslationError) { console.warn( `skipping locale "${locale}", error while fetching ${sourceFile.uri}: ${ex.message}`, ); } else { throw ex; } } } /** * * This function extracts the locale name from the file path * e.g. 'en-US.csv' -> 'en-US' or '/localization/en-US.csv' -> 'en-US' * */ export function parseLocaleName(fileName: string): string { const name = fileName.split('/').at(-1); const localeName = name?.split('.').at(0); if (!localeName) { throw new Error(`Locale could not be parsed from ${fileName}`); } return localeName; } export function convertCSVToFTLFromString(csvContent: string): string { const records = parse(csvContent, { columns: true }); let ftlContent = ''; records.forEach((record: { [x: string]: string; Key: string }) => { const key = record.Key.trim(); let sourceString = record[SOURCE_STRING_KEY].trim(); if (sourceString === INJECTABLE_WORKAROUND_VALUE) { sourceString = ''; } ftlContent += `${key} = ${sourceString}\n`; }); return ftlContent; } /** * * This function saves the FTL files to the local file system * */ export function saveFile( fileName: string, rawFileContent: string, outputDir: string, ): void { const preProcessedFileContent = INJECT_PLACEHOLDER_WORKAROUND && useFTLFlow ? cleanFTLContent(rawFileContent) : rawFileContent; const processedFileContent = useFTLFlow ? preProcessedFileContent : convertCSVToFTLFromString(preProcessedFileContent); const relativePath = getRelativePath(fileName, outputDir); writeFileSync(relativePath, processedFileContent, 'utf8'); console.log('Successfully saved file on:', relativePath); } /** * Cleans the FTL content by removing any messages which have a placeholder as * their only element. This is a workaround for a Smartling bug that * does not allow just attributes and no value for a given key * * @param {string} ftlContent - The FTL content to be cleaned * * @returns {string} The cleaned FTL content */ export function cleanFTLContent(ftlContent: string): string { const parser = new FluentParser(); const serializer = new FluentSerializer(); const resource = parser.parse(ftlContent); for (const entry of resource.body) { if ( entry.type === 'Message' && entry.value?.elements && entry.value.elements.length === 1 ) { const pattern = entry.value; if ( pattern.elements[0].type === 'Placeable' && pattern.elements[0].expression.type === 'MessageReference' && pattern.elements[0].expression.id.name === 'placeholder' ) { entry.value = null; } } } return serializer.serialize(resource); } /** * * This function returns the relative path of the FTL file * e.g. 'en-US' into something like -> '/localization/en-US.ftl' * */ export function getRelativePath(fileName: string, outputDir: string): string { const ftlFilePath = `/${outputDir.replace( `/en-US${fileExtensionToUse}`, '', )}/${fileName}.ftl`; return join(getRepoRoot(), ftlFilePath); } /** * This function fetches the content of the file from Smartling API * based on the file URI and token */ export async function getFileContent( localeId: string, fileUri: string, token: string, ): Promise { const response = await fetch(getSmartlingAPIFileUrl(localeId, fileUri), { method: 'GET', headers: { Authorization: `Bearer ${token}`, }, }); if (!response.ok) { const jsonResp = await response.json(); const errorMessages = jsonResp.response.errors; throw new Error( `Error! Status code ${ response.status } from Smartling when fetching zip blobs: ${JSON.stringify( errorMessages, )}`, ); } if (!response.body) { throw new MalformedSmartlingResponseError( `The response body is missing for file ${fileUri}`, ); } const responseText = await response.text(); if (!responseText || responseText === '') { /* * It's better to not pull it if the string is empty! Why so? Well... * I'ts very likely that empty text means either data is compromised or * We do not want to skip handling this cases. * * (Even in the corner case of... let's say an empty project... the file is completely empty, * that's very likely something we don't want to pull want to happen, or at least notify the user aggressively IMO * Because reaching this points means that you triggered an empty translation job, which is bad use of their product * or an early use of this feature which is not intended to) * */ const errorToThrow = new EmptyTranslationError(localeId); console.error(errorToThrow.message); throw errorToThrow; } return responseText; } /** * This function checks if a PR should be created * based on the local files and the Smartling API * */ export async function shouldCreatePR(): Promise { const token = await getAuthToken(); const jobDetailsList = await Promise.all(await getAllJobDetails(token)); return jobDetailsList.some((job) => anyLocalFileNeedsUpdate(job)); } /** * Retrieves a list of locales that need updates, along with their corresponding source files. * * This function uses the provided authentication token to retrieve a list of Smartling job details. * It then filters this list to obtain only the jobs that have local files that need updates. * Finally, it returns a list of objects containing the IDs of the locales that need updates and their corresponding source files. * * @param token Authentication token for accessing the Smartling API * @returns A list of objects containing the IDs of the locales that need updates and their corresponding source files */ export async function getLocalesThatNeedUpdates( token: string, ): Promise<{ locales: string[]; sourceFiles: SmartlingSourceFile[] }[]> { const jobDetailsList = await Promise.all(await getAllJobDetails(token)); const relevantJobDetails = jobDetailsList.filter((job) => anyLocalFileNeedsUpdate(job), ); const result = []; for (const jobDetail of relevantJobDetails) { result.push({ locales: jobDetail.targetLocaleIds, sourceFiles: jobDetail.sourceFiles, }); } return result; } /** * Retrieves a list of promises to Smartling job details. * * This function first retrieves a list of Smartling job details and then maps * each job to a promise of its corresponding job details. * * @param token Authentication token for accessing the Smartling API * @returns A list of promises to Smartling job details */ export async function getAllJobDetails( token: string, ): Promise[]> { const jobList = await getJobList(token); return jobList.map((job) => getJobDetails(job.translationJobUid, token)); } export async function getJobDetails( jobId: string, token: string, ): Promise { // By default it fetches only enabled locales unless query param supportedOnly is set to false. const result = await doFetch( `${SMARTLING_BASE_URL}${SMARTLING_JOBS_PATH}/${SMARTLING_PROJECT_ID}/jobs/${jobId}`, { method: 'GET', headers: { Authorization: `Bearer ${token}`, }, }, ); if (!result.response.data) { console.error( `This should not happen, but if you are seeing this, the job details response is missing for job ${jobId}. Please check if it is a Smartling API issue`, ); throw new Error( `This should not happen, but if you are seeing this, the job details response is missing for job ${jobId}. Please check if it is a Smartling API issue`, ); } return result.response.data; } /** * Determines if any local file associated with the given job details needs an update. * * This function checks if the job status is 'COMPLETED' and evaluates each source file * in the job details to determine if a local file update is needed based on the git last modified date and * the last completed date of the Smartling job. * * @param jobDetails The details of the Smartling job, including its status and source files. * @returns A boolean indicating whether any local file needs an update. */ export function anyLocalFileNeedsUpdate( jobDetails: SmartlingJobDetails, ): boolean { return ( jobDetails.jobStatus === SmartlingJobStatus.COMPLETED && // !!IMPORTANT!! if speed becomes an issue, consider building a shell command that can accept a list of filenames // and return only those whose last modified time in git is less than the job's lastCompletedDate jobDetails.sourceFiles.some((file) => localFileNeedsUpdate(file.uri, jobDetails.lastCompletedDate), ) ); } /** * Checks if a local file needs an update by comparing the last modification date on * the local git repository with the last completed date of the Smartling job, or if the file does not exist. * * @param fileUri The uri of the file to check. * @param lastCompletedDate The last completed date of the Smartling job in ISO 8601 format. * @returns A boolean indicating whether a local file update is needed. */ export function localFileNeedsUpdate( fileUri: string, lastCompletedDate: string, ): boolean { try { if (!existsSync(fileUri)) return true; const gitCommand = `git log -1 --format=%ct -- ${fileUri}`; const gitTimestamp = execSync(gitCommand, { encoding: 'utf8' }).trim(); const lastGitModTime = new Date(parseInt(gitTimestamp, 10) * 1000); const smartlingLastCompleted = new Date(lastCompletedDate); // if the last upload to smartling happened after the last modification on git, it needs an update return smartlingLastCompleted.getTime() > lastGitModTime.getTime(); } catch (error) { console.error(`Error checking local file ${fileUri}:`, error); return false; } } /** * returns a url to get a file's details from smartling */ function getSmartlingAPIFileUrl(localeId: string, fileUri: string): URL { const path = [ SMARTLING_FILES_PATH, SMARTLING_PROJECT_ID, 'locales', parseLocaleName(localeId), 'file', ].join('/'); const url = new URL(path, SMARTLING_BASE_URL); url.searchParams.set('fileUri', fileUri); url.searchParams.set('retrievalType', 'published'); return url; }