using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using UnityEditor; using UnityEditor.Localization; using UnityEngine.Localization.Tables; namespace Crowdin.Plugin.Operations { using Helpers; using Helpers.FileHelpers; using Helpers.PluginLocalization; using Helpers.UnityLocalization; using Models; using Models.Translations; using Models.Translations.Items; using Server_API.Helpers; using Server_API.Models.Labels; using Server_API.Models.Projects; using Server_API.Models.SourceStrings; using Server_API.Models.StringTranslations; using Server_API.Services; /// Pulls string translations from Crowdin into the local Unity Localization string tables. public static class PullStrings { public static async Task RunAsync( PullOptions options, IProgress progress, CancellationToken cancellationToken) { progress.Report(new OperationProgress(LocalizationHelper.GetLocalStringByKey(LocalKeys.pulling_from_server))); TranslationsHelper.RefreshAllStringsTranslation(); var projectsService = new ProjectsApiService(); var sourceStringService = new SourceStringsApiService(); var stringTranslationsService = new StringTranslationsApiService(); bool isProjectStringBased = SettingsHelper.GetProjectType() is ProjectType.StringBased; ProjectInfo projectInfo = await projectsService.GetProjectById(SettingsHelper.GetProjectId(), cancellationToken); int receivedCount = 0; foreach (string tableName in options.Tables) { StringTableCollection tableCollection = LocalizationEditorSettings.GetStringTableCollection(tableName); List sourceStrings; if (isProjectStringBased) { LabelData label = await StringBasedProjectsHelper.FindLabelByTitle($"{tableName}.csv", cancellationToken); if (label is null) { UnityEngine.Debug.LogError($"Label for table \"{tableName}\" not found"); continue; } sourceStrings = await sourceStringService.ListStrings(labelId: label.Id, branchId: options.BranchId, ct: cancellationToken); } else { var croQl = $"name of file = \"{tableName}.csv\""; sourceStrings = await sourceStringService.ListStrings(croQl, branchId: options.BranchId, ct: cancellationToken); } UnityEngine.Debug.Log($"API: downloaded {sourceStrings.Count} strings from table {tableCollection.TableCollectionName}"); int[] stringIds = sourceStrings.Select(sourceString => sourceString.Id).ToArray(); if (stringIds.Length is 0) { UnityEngine.Debug.Log($"No strings found. Skipped table \"{tableCollection.TableCollectionName}\""); continue; } var sourceStringIdToKeyDict = sourceStrings.ToDictionary( sourceString => sourceString.Id, sourceString => sourceString.Key); var sourceStringKeyToSourceText = sourceStrings .Where(sourceString => { bool validKey = sourceString.Key is not null; if (!validKey) { UnityEngine.Debug.LogWarningFormat( "Source string \"{0}\" key is null. Will be skipped", sourceString.SourceText); } return validKey; }) .ToDictionary( sourceString => sourceString.Key, sourceString => sourceString.SourceText); var translationsDict = new Dictionary<(string key, string stringTableCrowdinLangCode), string>(); foreach (StringTable stringTable in tableCollection.StringTables) { string stringTableCrowdinLangCode = UnityLocalizationHelper.GetLanguageCode(stringTable.LocaleIdentifier); UnityEngine.Debug.LogFormat( "Table language ({0}) -> Crowdin language ({1})", stringTable.LocaleIdentifier, stringTableCrowdinLangCode); if (string.IsNullOrWhiteSpace(stringTableCrowdinLangCode)) { throw new Exception($"Cannot get Crowdin language code from: {stringTable.LocaleIdentifier}"); } if (stringTable.LocaleIdentifier.CultureInfo is not null && stringTable.LocaleIdentifier.CultureInfo.Name == SettingsHelper.GetSourceLanguage()) { UnityEngine.Debug.LogFormat( "Skip locale \"{0}\". Reason: it's a source language", stringTable.LocaleIdentifier.CultureInfo.Name); continue; } if (stringTable.LocaleIdentifier.CultureInfo is not null && projectInfo.TargetLanguageIds.All(langCode => langCode != stringTableCrowdinLangCode)) { UnityEngine.Debug.LogWarningFormat( "Skip locale \"{0}\". Reason: not exists on the remote side", stringTable.LocaleIdentifier.CultureInfo.Name); continue; } List languageTranslations = await stringTranslationsService .ListLanguageTranslations(stringTableCrowdinLangCode, stringIds, branchId: options.BranchId, ct: cancellationToken); UnityEngine.Debug.Log($"API: downloaded {languageTranslations.Count} language translations from table {tableCollection.TableCollectionName}"); foreach (LanguageTranslation tr in languageTranslations) { if (!sourceStringIdToKeyDict.TryGetValue(tr.StringId, out string key)) { UnityEngine.Debug.Log($"Key \"{tr.StringId}\" not found. Skip"); continue; } if (string.IsNullOrWhiteSpace(key)) { UnityEngine.Debug.LogFormat( "Key is empty for StringId={0} & TranslationId={1}. Skip", tr.StringId, tr.TranslationId); continue; } if (translationsDict.ContainsKey((key, stringTableCrowdinLangCode))) { UnityEngine.Debug.LogWarningFormat( "Entry with TableKey={0} & Locale={1} is already exists. Translations may duplicate", key, stringTableCrowdinLangCode); } else { translationsDict.Add((key, stringTableCrowdinLangCode), tr.Text); receivedCount++; } } } progress.Report(new OperationProgress(LocalizationHelper.GetLocalStringByKey(LocalKeys.ensure_strings_synced_start))); // Only the in-memory TranslationsHelper cache is updated here. The local Unity tables // themselves are left untouched until ImportPulledStrings(...) runs below, once every // table has finished its network phase - that way, cancelling mid-pull can't leave a // table with newly-added keys and no translations. foreach (SourceString sourceString in sourceStrings) { string key = sourceString.Key; if (!sourceStringKeyToSourceText.ContainsKey(key)) { continue; } ContextEntryIdHelper.TryReadEntryIdFromContextString( sourceString.Context, out long? entryId, out string strippedContext); List translations = SettingsHelper.GetLanguageScheme() .Where(languageItem => translationsDict.ContainsKey((key, languageItem.langType))) .Select(languageItem => { string text = translationsDict[(key, languageItem.langType)]; if (string.IsNullOrWhiteSpace(text)) { UnityEngine.Debug.LogWarning($"No translation found for key \"{key}\" and locale \"{languageItem.langType}\""); } return new TranslationTextItem { language = languageItem.langType, text = text }; }) .ToList(); var translation = new TranslationTextObj( entryId, key, sourceString.SourceText, translations, strippedContext, tableCollection.TableCollectionName); TranslationsHelper.TryAddOrUpdateTranslation(translation, options.PreserveEntryIds); } progress.Report(new OperationProgress(LocalizationHelper.GetLocalStringByKey(LocalKeys.ensure_strings_synced_end))); } progress.Report(new OperationProgress(LocalizationHelper.GetLocalStringByKey(LocalKeys.pull_successful))); AssetDatabase.Refresh(); Dictionary importedPerTable = ImportPulledStrings(options.Tables, options.PreserveEntryIds, progress); progress.Report(new OperationProgress(string.Empty)); progress.Report(new OperationProgress(LocalizationHelper.GetLocalStringByKey( LocalKeys.stats_received_string_translations, receivedCount.ToString()))); return new PullResult(receivedCount, importedPerTable); } private static Dictionary ImportPulledStrings( IReadOnlyList tables, bool preserveEntryIds, IProgress progress) { progress.Report(new OperationProgress(LocalizationHelper.GetLocalStringByKey(LocalKeys.start_import_strings))); var perTable = new Dictionary(); IEnumerable translationStrings = TranslationsHelper.GetTranslationStrings(); if (translationStrings != null) { Dictionary> stringsDict = translationStrings .Where(trString => tables.Contains(trString.exportedFromTable)) .Where(trString => ImportDataHelper.ImportStringRow(trString, trString.exportedFromTable, preserveEntryIds)) .GroupBy(trString => trString.exportedFromTable) .ToDictionary( grouping => grouping.Key, grouping => grouping.Select(trString => trString.key).ToList()); foreach ((string tableName, List stringKeys) in stringsDict) { progress.Report(new OperationProgress(string.Empty)); progress.Report(new OperationProgress(LocalizationHelper.GetLocalStringByKey(LocalKeys.import_table, tableName))); foreach (string stringKey in stringKeys) { perTable[tableName] = perTable.GetValueOrDefault(tableName) + 1; progress.Report(new OperationProgress(LocalizationHelper.GetLocalStringByKey(LocalKeys.import_str, stringKey))); } } progress.Report(new OperationProgress(string.Empty)); } int totalTables = perTable.Count; int totalItems = perTable.Values.Sum(); progress.Report(new OperationProgress(LocalizationHelper.GetLocalStringByKey( LocalKeys.import_comp_string_translations, totalTables.ToString(), totalItems.ToString()))); foreach ((string table, int count) in perTable) { progress.Report(new OperationProgress(LocalizationHelper.GetLocalStringByKey( LocalKeys.import_item_string, table, count.ToString()))); } SaveChangesToAssets(); return perTable; } private static void SaveChangesToAssets() { foreach (var tableCollection in LocalizationEditorSettings.GetStringTableCollections()) { EditorUtility.SetDirty(tableCollection.SharedData); EditorUtility.SetDirty(tableCollection); foreach (var stringTable in tableCollection.StringTables) { EditorUtility.SetDirty(stringTable); } } AssetDatabase.SaveAssets(); AssetDatabase.Refresh(ImportAssetOptions.ForceUpdate); } } }