{"version":3,"file":"downloadPrivacyRequestFiles-DFJHFxhc.mjs","names":[],"sources":["../src/lib/requests/getFileMetadataForPrivacyRequests.ts","../src/lib/requests/streamPrivacyRequestFiles.ts","../src/lib/requests/downloadPrivacyRequestFiles.ts"],"sourcesContent":["import { TableEncryptionType } from '@transcend-io/privacy-types';\nimport { decodeCodec, valuesOf } from '@transcend-io/type-utils';\nimport { map } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport type { Got } from 'got';\nimport * as t from 'io-ts';\n\nimport { logger } from '../../logger.js';\nimport { PrivacyRequest } from '../graphql/index.js';\n\nexport const IntlMessage = t.type({\n  /** The message key */\n  defaultMessage: t.string,\n  /** ID */\n  id: t.string,\n});\n\n/** Type */\nexport type IntlMessage = t.TypeOf<typeof IntlMessage>;\n\nexport const RequestFileMetadata = t.type({\n  /** The key to pass to download the file contents */\n  downloadKey: t.string,\n  /** Error message related to file */\n  error: t.union([t.null, t.string]),\n  /** Mimetype of file */\n  mimetype: t.string,\n  /** Size of file, stored as string as this can be a BigInt */\n  size: t.string,\n  /** Name of file based on datapoint names in Transcend */\n  fileName: t.string,\n  /** The metadata on the datapoint */\n  dataPoint: t.type({\n    /** ID of datapoint */\n    id: t.string,\n    /** The title of datapoint */\n    title: t.union([IntlMessage, t.null]),\n    /** Description of datapoint */\n    description: t.union([IntlMessage, t.null]),\n    /** Name of datapoint */\n    name: t.string,\n    /** Slug of datapoint */\n    slug: t.string,\n    /** Table level encryption information */\n    encryption: t.union([valuesOf(TableEncryptionType), t.null]),\n    /** The name of the data silo */\n    dataSilo: t.type({\n      /** ID of the data silo */\n      id: t.string,\n      /** The title of the data silo */\n      title: t.string,\n      /** The description of the data silo */\n      description: t.string,\n      /** The type of the data silo */\n      type: t.string,\n      /** The outer type of the data silo */\n      outerType: t.union([t.string, t.null]),\n    }),\n    /** The path to the datapoint if a database (e.g. name of schema) */\n    path: t.array(t.string),\n  }),\n});\n\n/** Type override */\nexport type RequestFileMetadata = t.TypeOf<typeof RequestFileMetadata>;\n\nexport const RequestFileMetadataResponse = t.type({\n  /** The list of file metadata */\n  nodes: t.array(RequestFileMetadata),\n  /** The total number of file metadata */\n  totalCount: t.number,\n  /** Links to next pages */\n  _links: t.partial({\n    /** The link to the next page of file metadata */\n    next: t.union([t.string, t.null]),\n    /** The link to the previous page of file metadata */\n    previous: t.union([t.string, t.null]),\n  }),\n});\n\n/** Type override */\nexport type RequestFileMetadataResponse = t.TypeOf<typeof RequestFileMetadataResponse>;\n\n/**\n * Given a list of privacy requests, download the file metadata\n * for these requests - this is useful to prepare the files in a\n * data access request for download.\n *\n * @param requests - The list of privacy requests to download files for\n * @param options - Options\n * @returns The number of requests canceled\n */\nexport async function getFileMetadataForPrivacyRequests(\n  requests: Pick<PrivacyRequest, 'id' | 'status'>[],\n  {\n    sombra,\n    concurrency = 5,\n    limit = 100,\n  }: {\n    /** Sombra instance */\n    sombra: Got;\n    /** Number of files to pull at once */\n    limit?: number;\n    /** Concurrency limit for approving */\n    concurrency?: number;\n  },\n): Promise<[Pick<PrivacyRequest, 'id' | 'status'>, RequestFileMetadata[]][]> {\n  logger.info(colors.magenta(`Pulling file metadata for ${requests.length} requests`));\n\n  // Time duration\n  const t0 = new Date().getTime();\n  // create a new progress bar instance and use shades_classic theme\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n\n  // Start timer\n  let total = 0;\n  progressBar.start(requests.length, 0);\n\n  // Loop over the requests\n  const results = await map(\n    requests,\n    async (\n      requestToDownload,\n    ): Promise<[Pick<PrivacyRequest, 'id' | 'status'>, RequestFileMetadata[]]> => {\n      const localResults: RequestFileMetadata[] = [];\n\n      // Paginate over the file metadata for this request\n      let shouldContinue = true;\n      let offset = 0;\n      while (shouldContinue) {\n        let response: RequestFileMetadataResponse;\n        try {\n          // Grab the file metadata for this request\n\n          const rawResponse = await sombra\n            .get(`v1/data-subject-request/${requestToDownload.id}/download-keys`, {\n              searchParams: {\n                limit,\n                offset,\n              },\n            })\n            .json();\n          response = decodeCodec(RequestFileMetadataResponse, rawResponse);\n          localResults.push(...response.nodes);\n\n          // Increase offset and break if no more pages\n          offset += limit;\n          shouldContinue =\n            // eslint-disable-next-line no-underscore-dangle\n            !!response._links.next && response.nodes.length === limit;\n        } catch (err) {\n          throw new Error(`Received an error from server: ${err?.response?.body || err?.message}`);\n        }\n      }\n\n      total += 1;\n      progressBar.update(total);\n      return [requestToDownload, localResults];\n    },\n    { concurrency },\n  );\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(\n      `Successfully downloaded file metadata ${requests.length} requests in \"${\n        totalTime / 1000\n      }\" seconds!`,\n    ),\n  );\n\n  return results;\n}\n","import { map } from '@transcend-io/utils';\nimport colors from 'colors';\nimport type { Got } from 'got';\n\nimport { logger } from '../../logger.js';\nimport { RequestFileMetadata } from './getFileMetadataForPrivacyRequests.js';\n\n/**\n * This function will take in a set of file metadata for privacy requests\n * call the Transcend API to stream the file metadata for these requests\n * and pass that through a callback function\n *\n * @param fileMetadata - Metadata to download\n * @param options - Options for the request\n */\nexport async function streamPrivacyRequestFiles(\n  fileMetadata: RequestFileMetadata[],\n  {\n    requestId,\n    sombra,\n    onFileDownloaded,\n    concurrency = 20,\n  }: {\n    /** Request ID for logging */\n    requestId: string;\n    /** Sombra got instance */\n    sombra: Got;\n    /** Handler on each file */\n    onFileDownloaded: (metadata: RequestFileMetadata, stream: Uint8Array) => void;\n    /** Concurrent downloads at once */\n    concurrency?: number;\n  },\n): Promise<void> {\n  // Loop over each file\n  await map(\n    fileMetadata,\n    async (metadata) => {\n      try {\n        // Construct the stream\n        await sombra\n          .get('v1/files', {\n            searchParams: {\n              downloadKey: metadata.downloadKey,\n            },\n          })\n          .buffer()\n          .then((fileResponse) => onFileDownloaded(metadata, fileResponse));\n      } catch (err) {\n        if (err?.response?.body?.includes('fileMetadata#verify')) {\n          logger.error(\n            colors.red(\n              `Failed to pull file for: ${metadata.fileName} (request:${requestId}) - JWT expired. ` +\n                'This likely means that the file is no longer available. ' +\n                'Try restarting the request from scratch in Transcend Admin Dashboard. ' +\n                'Skipping the download of this file.',\n            ),\n          );\n          return;\n        }\n        throw new Error(`Received an error from server: ${err?.response?.body || err?.message}`);\n      }\n    },\n    {\n      concurrency,\n    },\n  );\n}\n","import { existsSync, mkdirSync, writeFileSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\n\nimport { RequestAction, RequestStatus } from '@transcend-io/privacy-types';\nimport {\n  buildTranscendGraphQLClient,\n  createSombraGotInstance,\n  makeGraphQLRequest,\n} from '@transcend-io/sdk';\nimport { map } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { fetchAllRequests, APPROVE_PRIVACY_REQUEST } from '../graphql/index.js';\nimport { getFileMetadataForPrivacyRequests } from './getFileMetadataForPrivacyRequests.js';\nimport { streamPrivacyRequestFiles } from './streamPrivacyRequestFiles.js';\n\n/**\n * Download a set of privacy requests to disk\n *\n * @param options - Options\n * @returns The number of requests canceled\n */\nexport async function downloadPrivacyRequestFiles({\n  auth,\n  folderPath,\n  requestIds,\n  createdAtBefore,\n  sombraAuth,\n  createdAtAfter,\n  updatedAtBefore,\n  updatedAtAfter,\n  statuses = [RequestStatus.Approving, RequestStatus.Downloadable],\n  concurrency = 5,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n  approveAfterDownload = false,\n}: {\n  /** The folder path to download the files to */\n  folderPath: string;\n  /** Transcend API key authentication */\n  auth: string;\n  /** Sombra API key authentication */\n  sombraAuth?: string;\n  /** Concurrency limit for approving */\n  concurrency?: number;\n  /** The request statuses to cancel */\n  statuses?: RequestStatus[];\n  /** The set of privacy requests to cancel */\n  requestIds?: string[];\n  /** Filter for requests created before this date */\n  createdAtBefore?: Date;\n  /** Filter for requests created after this date */\n  createdAtAfter?: Date;\n  /** Filter for requests updated before this date */\n  updatedAtBefore?: Date;\n  /** Filter for requests updated after this date */\n  updatedAtAfter?: Date;\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n  /** When true, approve any requests in Transcend that are in status=APPROVING */\n  approveAfterDownload?: boolean;\n}): Promise<number> {\n  // Find all requests made before createdAt that are in a removing data state\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n\n  // Create sombra instance to communicate with\n  const sombra = await createSombraGotInstance(transcendUrl, auth, {\n    logger,\n    sombraApiKey: sombraAuth,\n    sombraUrl: process.env.SOMBRA_URL,\n  });\n\n  // Create the folder if it does not exist\n  if (!existsSync(folderPath)) {\n    mkdirSync(folderPath);\n  }\n\n  // Pull in the requests\n  const allRequests = await fetchAllRequests(client, {\n    actions: [RequestAction.Access],\n    createdAtBefore,\n    createdAtAfter,\n    updatedAtBefore,\n    updatedAtAfter,\n    statuses,\n    requestIds,\n  });\n\n  // Download the file metadata for each request\n  const requestFileMetadata = await getFileMetadataForPrivacyRequests(allRequests, {\n    sombra,\n    concurrency,\n  });\n\n  // Start timer for download process\n  const t0 = new Date().getTime();\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n  let total = 0;\n  let totalApproved = 0;\n  progressBar.start(allRequests.length, 0);\n\n  // Download the files for each request\n  await map(\n    requestFileMetadata,\n    async ([request, metadata]) => {\n      // Create a new folder to store request files\n      const requestFolder = join(folderPath, request.id);\n      if (!existsSync(requestFolder)) {\n        mkdirSync(requestFolder);\n      }\n\n      // Stream each file to disk\n      await streamPrivacyRequestFiles(metadata, {\n        sombra,\n        requestId: request.id,\n        onFileDownloaded: (fil, stream) => {\n          // Ensure a folder exists for the file\n          // filename looks like Health/heartbeat.csv\n          const filePath = join(requestFolder, fil.fileName);\n          const folder = dirname(filePath);\n          if (!existsSync(folder)) {\n            mkdirSync(folder, { recursive: true });\n          }\n\n          // Write to disk\n          writeFileSync(filePath, stream);\n        },\n      });\n\n      // Approve the request if requested\n      if (approveAfterDownload && request.status === RequestStatus.Approving) {\n        await makeGraphQLRequest(client, APPROVE_PRIVACY_REQUEST, {\n          variables: { input: { requestId: request.id } },\n          logger,\n        });\n        totalApproved += 1;\n      }\n\n      // Increment the progress bar\n      total += 1;\n      progressBar.update(total);\n    },\n    { concurrency },\n  );\n\n  progressBar.stop();\n  const t1 = new Date().getTime();\n  const totalTime = t1 - t0;\n\n  logger.info(\n    colors.green(`Successfully downloaded ${total} requests in \"${totalTime / 1000}\" seconds!`),\n  );\n  if (totalApproved > 0) {\n    logger.info(colors.green(`Approved ${totalApproved} requests in Transcend.`));\n  }\n  return allRequests.length;\n}\n"],"mappings":"urBAWA,MAAa,EAAc,EAAE,KAAK,CAEhC,eAAgB,EAAE,OAElB,GAAI,EAAE,OACP,CAAC,CAKW,EAAsB,EAAE,KAAK,CAExC,YAAa,EAAE,OAEf,MAAO,EAAE,MAAM,CAAC,EAAE,KAAM,EAAE,OAAO,CAAC,CAElC,SAAU,EAAE,OAEZ,KAAM,EAAE,OAER,SAAU,EAAE,OAEZ,UAAW,EAAE,KAAK,CAEhB,GAAI,EAAE,OAEN,MAAO,EAAE,MAAM,CAAC,EAAa,EAAE,KAAK,CAAC,CAErC,YAAa,EAAE,MAAM,CAAC,EAAa,EAAE,KAAK,CAAC,CAE3C,KAAM,EAAE,OAER,KAAM,EAAE,OAER,WAAY,EAAE,MAAM,CAAC,EAAS,EAAoB,CAAE,EAAE,KAAK,CAAC,CAE5D,SAAU,EAAE,KAAK,CAEf,GAAI,EAAE,OAEN,MAAO,EAAE,OAET,YAAa,EAAE,OAEf,KAAM,EAAE,OAER,UAAW,EAAE,MAAM,CAAC,EAAE,OAAQ,EAAE,KAAK,CAAC,CACvC,CAAC,CAEF,KAAM,EAAE,MAAM,EAAE,OAAO,CACxB,CAAC,CACH,CAAC,CAKW,EAA8B,EAAE,KAAK,CAEhD,MAAO,EAAE,MAAM,EAAoB,CAEnC,WAAY,EAAE,OAEd,OAAQ,EAAE,QAAQ,CAEhB,KAAM,EAAE,MAAM,CAAC,EAAE,OAAQ,EAAE,KAAK,CAAC,CAEjC,SAAU,EAAE,MAAM,CAAC,EAAE,OAAQ,EAAE,KAAK,CAAC,CACtC,CAAC,CACH,CAAC,CAcF,eAAsB,EACpB,EACA,CACE,SACA,cAAc,EACd,QAAQ,KASiE,CAC3E,EAAO,KAAK,EAAO,QAAQ,6BAA6B,EAAS,OAAO,WAAW,CAAC,CAGpF,IAAM,EAAK,IAAI,MAAM,CAAC,SAAS,CAEzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CAGjF,EAAQ,EACZ,EAAY,MAAM,EAAS,OAAQ,EAAE,CAGrC,IAAM,EAAU,MAAM,EACpB,EACA,KACE,IAC4E,CAC5E,IAAM,EAAsC,EAAE,CAG1C,EAAiB,GACjB,EAAS,EACb,KAAO,GAAgB,CACrB,IAAI,EACJ,GAAI,CAWF,EAAW,EAAY,EAA6B,MAR1B,EACvB,IAAI,2BAA2B,EAAkB,GAAG,gBAAiB,CACpE,aAAc,CACZ,QACA,SACD,CACF,CAAC,CACD,MAAM,CACuD,CAChE,EAAa,KAAK,GAAG,EAAS,MAAM,CAGpC,GAAU,EACV,EAEE,CAAC,CAAC,EAAS,OAAO,MAAQ,EAAS,MAAM,SAAW,QAC/C,EAAK,CACZ,MAAU,MAAM,kCAAkC,GAAK,UAAU,MAAQ,GAAK,UAAU,EAM5F,MAFA,IAAS,EACT,EAAY,OAAO,EAAM,CAClB,CAAC,EAAmB,EAAa,EAE1C,CAAE,cAAa,CAChB,CAED,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAUvB,OARA,EAAO,KACL,EAAO,MACL,yCAAyC,EAAS,OAAO,gBACvD,EAAY,IACb,YACF,CACF,CAEM,EChKT,eAAsB,EACpB,EACA,CACE,YACA,SACA,mBACA,cAAc,IAWD,CAEf,MAAM,EACJ,EACA,KAAO,IAAa,CAClB,GAAI,CAEF,MAAM,EACH,IAAI,WAAY,CACf,aAAc,CACZ,YAAa,EAAS,YACvB,CACF,CAAC,CACD,QAAQ,CACR,KAAM,GAAiB,EAAiB,EAAU,EAAa,CAAC,OAC5D,EAAK,CACZ,GAAI,GAAK,UAAU,MAAM,SAAS,sBAAsB,CAAE,CACxD,EAAO,MACL,EAAO,IACL,4BAA4B,EAAS,SAAS,YAAY,EAAU,oLAIrE,CACF,CACD,OAEF,MAAU,MAAM,kCAAkC,GAAK,UAAU,MAAQ,GAAK,UAAU,GAG5F,CACE,cACD,CACF,CCxCH,eAAsB,EAA4B,CAChD,OACA,aACA,aACA,kBACA,aACA,iBACA,kBACA,iBACA,WAAW,CAAC,EAAc,UAAW,EAAc,aAAa,CAChE,cAAc,EACd,eAAe,EACf,uBAAuB,IA0BL,CAElB,IAAM,EAAS,EAA4B,EAAc,EAAK,CAGxD,EAAS,MAAM,EAAwB,EAAc,EAAM,CAC/D,SACA,aAAc,EACd,UAAW,QAAQ,IAAI,WACxB,CAAC,CAGG,EAAW,EAAW,EACzB,EAAU,EAAW,CAIvB,IAAM,EAAc,MAAM,EAAiB,EAAQ,CACjD,QAAS,CAAC,EAAc,OAAO,CAC/B,kBACA,iBACA,kBACA,iBACA,WACA,aACD,CAAC,CAGI,EAAsB,MAAM,EAAkC,EAAa,CAC/E,SACA,cACD,CAAC,CAGI,EAAK,IAAI,MAAM,CAAC,SAAS,CACzB,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CACjF,EAAQ,EACR,EAAgB,EACpB,EAAY,MAAM,EAAY,OAAQ,EAAE,CAGxC,MAAM,EACJ,EACA,MAAO,CAAC,EAAS,KAAc,CAE7B,IAAM,EAAgB,EAAK,EAAY,EAAQ,GAAG,CAC7C,EAAW,EAAc,EAC5B,EAAU,EAAc,CAI1B,MAAM,EAA0B,EAAU,CACxC,SACA,UAAW,EAAQ,GACnB,kBAAmB,EAAK,IAAW,CAGjC,IAAM,EAAW,EAAK,EAAe,EAAI,SAAS,CAC5C,EAAS,EAAQ,EAAS,CAC3B,EAAW,EAAO,EACrB,EAAU,EAAQ,CAAE,UAAW,GAAM,CAAC,CAIxC,EAAc,EAAU,EAAO,EAElC,CAAC,CAGE,GAAwB,EAAQ,SAAW,EAAc,YAC3D,MAAM,EAAmB,EAAQ,EAAyB,CACxD,UAAW,CAAE,MAAO,CAAE,UAAW,EAAQ,GAAI,CAAE,CAC/C,SACD,CAAC,CACF,GAAiB,GAInB,GAAS,EACT,EAAY,OAAO,EAAM,EAE3B,CAAE,cAAa,CAChB,CAED,EAAY,MAAM,CAElB,IAAM,EADK,IAAI,MAAM,CAAC,SACF,CAAG,EAQvB,OANA,EAAO,KACL,EAAO,MAAM,2BAA2B,EAAM,gBAAgB,EAAY,IAAK,YAAY,CAC5F,CACG,EAAgB,GAClB,EAAO,KAAK,EAAO,MAAM,YAAY,EAAc,yBAAyB,CAAC,CAExE,EAAY"}