{"version":3,"file":"fetchDistantDictionaries.mjs","names":[],"sources":["../../src/fetchDistantDictionaries.ts"],"sourcesContent":["import { createIntlayerCMS } from '@intlayer/api';\nimport { dictionaryEndpoint } from '@intlayer/api/dictionary';\n// @ts-ignore @intlayer/backend is not build yet\nimport type { DictionaryAPI } from '@intlayer/backend';\nimport { getAppLogger, x } from '@intlayer/config/logger';\nimport { getConfiguration } from '@intlayer/config/node';\nimport { retryManager } from '@intlayer/config/utils';\nimport type { DictionariesStatus } from './loadDictionaries';\nimport { chunkArray } from './utils/chunkArray';\nimport { parallelize } from './utils/parallelize';\n\n/**\n * Number of dictionaries requested per HTTP call. Must stay under the batch\n * limit enforced by the backend (`MAX_DICTIONARY_KEYS_PER_REQUEST`).\n */\nconst DEFAULT_DICTIONARIES_PER_REQUEST = 3;\n\n/** Number of batch requests kept in flight at the same time. */\nconst DEFAULT_PARALLEL_REQUESTS = 5;\n\n/** Number of retries performed on a failing batch request. */\nconst DEFAULT_MAX_RETRY = 2;\n\n/** Delay between two attempts of a failing batch request, in milliseconds. */\nconst DEFAULT_RETRY_DELAY = 250;\n\n/** HTTP statuses worth another attempt despite being client errors. */\nconst RETRYABLE_CLIENT_ERROR_STATUSES = new Set([408, 425, 429]);\n\n/**\n * A request is worth retrying when it failed for a reason another attempt could\n * fix: a network error (no status at all) or a server-side / throttling status.\n * Replaying a `404` or a `401` only multiplies the noise.\n */\nconst isRetryableError = (error: unknown): boolean => {\n  const status = (error as { status?: number })?.status;\n\n  if (typeof status !== 'number') return true;\n\n  if (status >= 500) return true;\n\n  return RETRYABLE_CLIENT_ERROR_STATUSES.has(status);\n};\n\n/**\n * A backend that predates the batch endpoint routes `/dictionary/by-keys` to\n * its `/dictionary/:dictionaryKey` handler, which answers `404` - either as an\n * unknown dictionary or as an unknown route. Both mean the same thing here:\n * this backend cannot serve batches, so fall back to one request per key.\n */\nconst isBatchEndpointUnsupported = (error: unknown): boolean =>\n  (error as { status?: number })?.status === 404;\n\ntype FetchDistantDictionariesOptions = {\n  dictionaryKeys: string[];\n  newDictionariesPath?: string;\n  logPrefix?: string;\n  /**\n   * Number of dictionaries fetched by a single request.\n   * @default 3\n   */\n  dictionariesPerRequest?: number;\n  /**\n   * Number of requests kept in flight at the same time.\n   * @default 5\n   */\n  parallelRequests?: number;\n  /**\n   * Number of retries performed when a batch request fails.\n   * @default 2\n   */\n  maxRetry?: number;\n  /**\n   * Delay between two attempts of a failing batch request, in milliseconds.\n   * @default 250\n   */\n  retryDelay?: number;\n};\n\n/**\n * Fetch distant dictionaries and update the logger with their statuses.\n *\n * Keys are grouped into batches fetched by a single request each, and those\n * requests are kept in flight together — by default 5 concurrent requests of 3\n * dictionaries, so 15 dictionaries travel at any given time. A slot is refilled\n * as soon as its request settles, so no batch waits for the whole wave.\n * Failing requests are retried before their keys are reported as errors.\n */\nexport const fetchDistantDictionaries = async (\n  options: FetchDistantDictionariesOptions,\n  onStatusUpdate?: (status: DictionariesStatus[]) => void\n): Promise<DictionaryAPI[]> => {\n  const config = getConfiguration();\n  const appLogger = getAppLogger(config);\n  try {\n    const dictionary = dictionaryEndpoint(createIntlayerCMS(config));\n\n    const {\n      dictionariesPerRequest = DEFAULT_DICTIONARIES_PER_REQUEST,\n      parallelRequests = DEFAULT_PARALLEL_REQUESTS,\n      maxRetry = DEFAULT_MAX_RETRY,\n      retryDelay = DEFAULT_RETRY_DELAY,\n    } = options;\n\n    // A same key can be listed twice when several qualified dictionaries share\n    // it - one request per key is enough to retrieve all of its siblings.\n    const distantDictionariesKeys = [...new Set(options.dictionaryKeys)];\n\n    if (distantDictionariesKeys.length === 0) return [];\n\n    const batches = chunkArray(distantDictionariesKeys, dictionariesPerRequest);\n\n    // Flipped once for the whole run as soon as a backend proves unable to\n    // serve batches, so the remaining batches skip the doomed request.\n    let isBatchEndpointSupported = true;\n    let hasLoggedBatchFallback = false;\n\n    /**\n     * Fetch each key of a batch with its own request. Used against backends\n     * without the batch endpoint. A key missing on the remote only discards\n     * itself, never its neighbours.\n     */\n    const requestDictionariesOneByOne = async (\n      batchKeys: string[]\n    ): Promise<DictionaryAPI[]> => {\n      const results = await parallelize(\n        batchKeys,\n        async (dictionaryKey) => {\n          try {\n            const getDictionaryResult =\n              await dictionary.getDictionary(dictionaryKey);\n\n            return getDictionaryResult.data;\n          } catch (_error) {\n            return undefined;\n          }\n        },\n        batchKeys.length\n      );\n\n      return results.filter(\n        (distantDictionary): distantDictionary is DictionaryAPI =>\n          distantDictionary !== undefined\n      );\n    };\n\n    /**\n     * Request one batch of dictionaries, retrying the whole request when it\n     * fails for a transient reason - a batch carries several dictionaries, so\n     * a network hiccup would otherwise discard all of them at once.\n     */\n    const requestBatch = retryManager(\n      async (batchKeys: string[]): Promise<DictionaryAPI[]> => {\n        if (isBatchEndpointSupported) {\n          try {\n            const getDictionariesResult =\n              await dictionary.getDictionariesByKeys(batchKeys);\n\n            return getDictionariesResult.data ?? [];\n          } catch (error) {\n            if (!isBatchEndpointUnsupported(error)) throw error;\n\n            isBatchEndpointSupported = false;\n\n            if (!hasLoggedBatchFallback) {\n              hasLoggedBatchFallback = true;\n              appLogger(\n                'This backend does not provide the batch dictionary endpoint - falling back to one request per dictionary.',\n                { level: 'warn' }\n              );\n            }\n          }\n        }\n\n        return await requestDictionariesOneByOne(batchKeys);\n      },\n      {\n        maxRetry,\n        delay: retryDelay,\n        shouldRetry: isRetryableError,\n        onError: ({ error, attempt }) =>\n          appLogger(\n            `Retrying distant dictionaries request (attempt ${attempt + 1}/${maxRetry}): ${error}`,\n            { level: 'warn' }\n          ),\n      }\n    );\n\n    /**\n     * Fetch one batch of dictionaries within a single request, and report the\n     * resulting status of every key it contains.\n     */\n    const processBatch = async (\n      batchKeys: string[]\n    ): Promise<DictionaryAPI[]> => {\n      onStatusUpdate?.(\n        batchKeys.map((dictionaryKey) => ({\n          dictionaryKey,\n          type: 'remote',\n          status: 'fetching',\n        }))\n      );\n\n      try {\n        const distantDictionaries = await requestBatch(batchKeys);\n\n        const fetchedKeys = new Set(\n          distantDictionaries.map(\n            (distantDictionary: DictionaryAPI) => distantDictionary.key\n          )\n        );\n\n        onStatusUpdate?.(\n          batchKeys.map(\n            (dictionaryKey): DictionariesStatus =>\n              fetchedKeys.has(dictionaryKey)\n                ? {\n                    dictionaryKey,\n                    type: 'remote',\n                    status: 'fetched',\n                  }\n                : {\n                    dictionaryKey,\n                    type: 'remote',\n                    status: 'error',\n                    error: `Dictionary ${dictionaryKey} not found on remote`,\n                  }\n          )\n        );\n\n        return distantDictionaries;\n      } catch (error) {\n        onStatusUpdate?.(\n          batchKeys.map((dictionaryKey) => ({\n            dictionaryKey,\n            type: 'remote',\n            status: 'error',\n            error: `Error fetching dictionary ${dictionaryKey}: ${error}`,\n          }))\n        );\n        return [];\n      }\n    };\n\n    const batchResults = await parallelize(\n      batches,\n      processBatch,\n      parallelRequests\n    );\n\n    return batchResults.flat();\n  } catch (_error) {\n    appLogger(`${x} Failed to fetch distant dictionaries`, { level: 'error' });\n    return [];\n  }\n};\n"],"mappings":";;;;;;;;;;;;;AAeA,MAAM,mCAAmC;;AAGzC,MAAM,4BAA4B;;AAGlC,MAAM,oBAAoB;;AAG1B,MAAM,sBAAsB;;AAG5B,MAAM,kDAAkC,IAAI,IAAI;CAAC;CAAK;CAAK;AAAG,CAAC;;;;;;AAO/D,MAAM,oBAAoB,UAA4B;CACpD,MAAM,SAAU,OAA+B;CAE/C,IAAI,OAAO,WAAW,UAAU,OAAO;CAEvC,IAAI,UAAU,KAAK,OAAO;CAE1B,OAAO,gCAAgC,IAAI,MAAM;AACnD;;;;;;;AAQA,MAAM,8BAA8B,UACjC,OAA+B,WAAW;;;;;;;;;;AAqC7C,MAAa,2BAA2B,OACtC,SACA,mBAC6B;CAC7B,MAAM,SAAS,iBAAiB;CAChC,MAAM,YAAY,aAAa,MAAM;CACrC,IAAI;EACF,MAAM,aAAa,mBAAmB,kBAAkB,MAAM,CAAC;EAE/D,MAAM,EACJ,yBAAyB,kCACzB,mBAAmB,2BACnB,WAAW,mBACX,aAAa,wBACX;EAIJ,MAAM,0BAA0B,CAAC,GAAG,IAAI,IAAI,QAAQ,cAAc,CAAC;EAEnE,IAAI,wBAAwB,WAAW,GAAG,OAAO,CAAC;EAElD,MAAM,UAAU,WAAW,yBAAyB,sBAAsB;EAI1E,IAAI,2BAA2B;EAC/B,IAAI,yBAAyB;;;;;;EAO7B,MAAM,8BAA8B,OAClC,cAC6B;GAgB7B,QAAO,MAfe,YACpB,WACA,OAAO,kBAAkB;IACvB,IAAI;KAIF,QAAO,MAFC,WAAW,cAAc,aAAa,EAEpB,CAAC;IAC7B,SAAS,QAAQ;KACf;IACF;GACF,GACA,UAAU,MACZ,EAEc,CAAC,QACZ,sBACC,sBAAsB,MAC1B;EACF;;;;;;EAOA,MAAM,eAAe,aACnB,OAAO,cAAkD;GACvD,IAAI,0BACF,IAAI;IAIF,QAAO,MAFC,WAAW,sBAAsB,SAAS,EAEtB,CAAC,QAAQ,CAAC;GACxC,SAAS,OAAO;IACd,IAAI,CAAC,2BAA2B,KAAK,GAAG,MAAM;IAE9C,2BAA2B;IAE3B,IAAI,CAAC,wBAAwB;KAC3B,yBAAyB;KACzB,UACE,6GACA,EAAE,OAAO,OAAO,CAClB;IACF;GACF;GAGF,OAAO,MAAM,4BAA4B,SAAS;EACpD,GACA;GACE;GACA,OAAO;GACP,aAAa;GACb,UAAU,EAAE,OAAO,cACjB,UACE,kDAAkD,UAAU,EAAE,GAAG,SAAS,KAAK,SAC/E,EAAE,OAAO,OAAO,CAClB;EACJ,CACF;;;;;EAMA,MAAM,eAAe,OACnB,cAC6B;GAC7B,iBACE,UAAU,KAAK,mBAAmB;IAChC;IACA,MAAM;IACN,QAAQ;GACV,EAAE,CACJ;GAEA,IAAI;IACF,MAAM,sBAAsB,MAAM,aAAa,SAAS;IAExD,MAAM,cAAc,IAAI,IACtB,oBAAoB,KACjB,sBAAqC,kBAAkB,GAC1D,CACF;IAEA,iBACE,UAAU,KACP,kBACC,YAAY,IAAI,aAAa,IACzB;KACE;KACA,MAAM;KACN,QAAQ;IACV,IACA;KACE;KACA,MAAM;KACN,QAAQ;KACR,OAAO,cAAc,cAAc;IACrC,CACR,CACF;IAEA,OAAO;GACT,SAAS,OAAO;IACd,iBACE,UAAU,KAAK,mBAAmB;KAChC;KACA,MAAM;KACN,QAAQ;KACR,OAAO,6BAA6B,cAAc,IAAI;IACxD,EAAE,CACJ;IACA,OAAO,CAAC;GACV;EACF;EAQA,QAAO,MANoB,YACzB,SACA,cACA,gBACF,EAEmB,CAAC,KAAK;CAC3B,SAAS,QAAQ;EACf,UAAU,GAAG,EAAE,wCAAwC,EAAE,OAAO,QAAQ,CAAC;EACzE,OAAO,CAAC;CACV;AACF"}