{"version":3,"file":"streamPrivacyRequestsToCsv-CFZkTtYF.mjs","names":[],"sources":["../src/lib/requests/formatRequestForCsv.ts","../src/lib/requests/splitDateRange.ts","../src/lib/requests/streamPrivacyRequestsToCsv.ts"],"sourcesContent":["import type { RequestIdentifier } from '@transcend-io/sdk';\nimport { groupBy } from 'lodash-es';\n\nimport type { PrivacyRequest } from '../graphql/index.js';\n\nexport interface ExportedPrivacyRequest extends PrivacyRequest {\n  /** Request identifiers */\n  requestIdentifiers: RequestIdentifier[];\n}\n\n/** A single CSV row */\nexport type CsvRow = { [k in string]: string | null | number | boolean };\n\n/**\n * Format a single privacy request (with optional identifiers) into a flat CSV row.\n *\n * @param request - The request with identifiers attached\n * @returns Flat object suitable for CSV output\n */\nexport function formatRequestForCsv({\n  attributeValues,\n  requestIdentifiers,\n  id,\n  email,\n  type,\n  status,\n  subjectType,\n  details,\n  createdAt,\n  successfullyCompletedAt,\n  country,\n  locale,\n  origin,\n  countrySubDivision,\n  isSilent,\n  isTest,\n  coreIdentifier,\n  purpose,\n  ...request\n}: ExportedPrivacyRequest): CsvRow {\n  return {\n    'Request ID': id,\n    'Created At': createdAt,\n    'Successfully Completed At': successfullyCompletedAt || '',\n    Email: email,\n    'Core Identifier': coreIdentifier,\n    'Request Type': type,\n    'Data Subject Type': subjectType,\n    Status: status,\n    Country: country,\n    'Country Sub Division': countrySubDivision,\n    Details: details,\n    Origin: origin,\n    'Silent Mode': isSilent,\n    'Is Test Request': isTest,\n    Language: locale,\n    'Purpose Trigger Name': purpose?.title || purpose?.name || '',\n    'Purpose Trigger Value': purpose?.consent?.toString() || '',\n    ...(purpose?.enrichedPreferences || []).reduce((acc: Record<string, string | boolean>, p) => {\n      const title = p.preferenceTopic?.title.defaultMessage || p.name;\n      return title\n        ? {\n            ...acc,\n            [title]: p.selectValues\n              ? p.selectValues.map((x) => x.name).join(';')\n              : p.selectValue?.name || p.booleanValue,\n          }\n        : acc;\n    }, {}),\n    ...request,\n    ...Object.entries(groupBy(attributeValues, 'attributeKey.name')).reduce(\n      (acc, [name, values]) =>\n        Object.assign(acc, {\n          [name]: values.map(({ name: n }) => n).join(','),\n        }),\n      {},\n    ),\n    ...Object.entries(groupBy(requestIdentifiers, 'name')).reduce(\n      (acc, [name, values]) =>\n        Object.assign(acc, {\n          [name]: values.map(({ value }) => value).join(','),\n        }),\n      {},\n    ),\n  };\n}\n","export interface ChunkedDateRange {\n  /** Chunk start */\n  createdAtAfter: Date;\n  /** Chunk end */\n  createdAtBefore: Date;\n}\n\n/**\n * Split a date range into N evenly-spaced chunks.\n *\n * @param after - Start of the date range\n * @param before - End of the date range\n * @param chunks - Number of chunks to split into\n * @returns Array of date range bounds\n */\nexport function splitDateRange(after: Date, before: Date, chunks: number): ChunkedDateRange[] {\n  const startMs = after.getTime();\n  const endMs = before.getTime();\n  const chunkSizeMs = (endMs - startMs) / chunks;\n  return Array.from({ length: chunks }, (_, i) => ({\n    createdAtAfter: new Date(startMs + chunkSizeMs * i),\n    createdAtBefore: new Date(i === chunks - 1 ? endMs : startMs + chunkSizeMs * (i + 1)),\n  }));\n}\n","import { RequestAction, RequestStatus } from '@transcend-io/privacy-types';\nimport {\n  buildTranscendGraphQLClient,\n  createSombraGotInstance,\n  fetchAllRequestIdentifiers,\n  validateSombraVersion,\n} from '@transcend-io/sdk';\nimport { map } from '@transcend-io/utils';\nimport cliProgress from 'cli-progress';\nimport colors from 'colors';\nimport { uniq } from 'lodash-es';\n\nimport { DEFAULT_TRANSCEND_API } from '../../constants.js';\nimport { logger } from '../../logger.js';\nimport { fetchAllRequests, fetchRequestsTotalCount } from '../graphql/index.js';\nimport { initCsvFile, appendCsvRowsOrdered, parseFilePath } from '../helpers/index.js';\nimport { formatRequestForCsv, ExportedPrivacyRequest } from './formatRequestForCsv.js';\nimport { splitDateRange } from './splitDateRange.js';\n\n/**\n * Stream privacy requests directly to CSV files, one file per date-range chunk.\n * Memory stays bounded to a single page of results at a time.\n * Supports both with and without request identifier enrichment.\n *\n * @param options - Options\n * @returns The list of written file paths and total row count\n */\nexport async function streamPrivacyRequestsToCsv({\n  auth,\n  sombraAuth,\n  actions = [],\n  statuses = [],\n  identifierSearch,\n  concurrency = 1,\n  pageLimit = 100,\n  transcendUrl = DEFAULT_TRANSCEND_API,\n  createdAtBefore,\n  createdAtAfter,\n  updatedAtBefore,\n  updatedAtAfter,\n  isTest,\n  skipRequestIdentifiers = false,\n  file,\n}: {\n  /** Transcend API key authentication */\n  auth: string;\n  /** Search for a specific identifier */\n  identifierSearch?: string;\n  /** Sombra API key authentication */\n  sombraAuth?: string;\n  /** API URL for Transcend backend */\n  transcendUrl?: string;\n  /** Statuses to filter on */\n  statuses?: RequestStatus[];\n  /** The request action to fetch */\n  actions?: RequestAction[];\n  /** Number of parallel date-range chunks */\n  concurrency?: number;\n  /** Concurrency for fetching identifiers per page */\n  pageLimit?: number;\n  /** Filter for requests created before this date */\n  createdAtBefore?: Date;\n  /** Filter for requests created after this date */\n  createdAtAfter?: Date;\n  /** Filter for requests updated before this date */\n  updatedAtBefore?: Date;\n  /** Filter for requests updated after this date */\n  updatedAtAfter?: Date;\n  /** Return test requests */\n  isTest?: boolean;\n  /** Skip fetching request identifiers */\n  skipRequestIdentifiers?: boolean;\n  /** Output CSV file path */\n  file: string;\n}): Promise<{\n  /** Paths to written CSV files */\n  filePaths: string[];\n  /** Total rows written */\n  totalCount: number;\n}> {\n  const client = buildTranscendGraphQLClient(transcendUrl, auth);\n  const sombra = skipRequestIdentifiers\n    ? undefined\n    : await createSombraGotInstance(transcendUrl, auth, {\n        logger,\n        sombraApiKey: sombraAuth,\n        sombraUrl: process.env.SOMBRA_URL,\n      });\n\n  // Log date range\n  let dateRange = '';\n  if (createdAtBefore) {\n    dateRange += ` before ${createdAtBefore.toISOString()}`;\n  }\n  if (createdAtAfter) {\n    dateRange += `${dateRange ? ', and' : ''} after ${createdAtAfter.toISOString()}`;\n  }\n  logger.info(\n    colors.magenta(\n      `${\n        actions.length > 0\n          ? `Pulling requests of type \"${actions.join('\" , \"')}\"`\n          : 'Pulling all requests'\n      }${dateRange}`,\n    ),\n  );\n\n  // Split into parallel date-range chunks when possible\n  const useChunks = concurrency > 1 && createdAtAfter && createdAtBefore;\n  const chunks = useChunks\n    ? splitDateRange(createdAtAfter, createdAtBefore, concurrency)\n    : [{ createdAtAfter, createdAtBefore }];\n\n  if (useChunks) {\n    logger.info(colors.magenta(`Splitting date range into ${concurrency} parallel chunks`));\n  }\n\n  // Fetch total count once for the shared progress bar\n  const filterBy = {\n    type: actions.length > 0 ? actions : undefined,\n    status: statuses.length > 0 ? statuses : undefined,\n    isTest,\n    createdAtBefore: createdAtBefore ? createdAtBefore.toISOString() : undefined,\n    createdAtAfter: createdAtAfter ? createdAtAfter.toISOString() : undefined,\n    updatedAtBefore: updatedAtBefore ? updatedAtBefore.toISOString() : undefined,\n    updatedAtAfter: updatedAtAfter ? updatedAtAfter.toISOString() : undefined,\n  };\n\n  const t0 = Date.now();\n\n  // Validate Sombra version once before bulk-fetching identifiers\n  if (!skipRequestIdentifiers) {\n    await validateSombraVersion(client, { logger });\n  }\n\n  const totalExpected = await fetchRequestsTotalCount(client, filterBy);\n  logger.info(colors.magenta(`Fetching ${totalExpected} requests`));\n\n  const progressBar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);\n  progressBar.start(totalExpected, 0);\n\n  let globalFetched = 0;\n\n  const { baseName, extension } = parseFilePath(file);\n\n  const filePaths = chunks.map((_, i) =>\n    chunks.length === 1 ? file : `${baseName}-${i}${extension}`,\n  );\n\n  interface FailedChunk {\n    /** Chunk index */\n    index: number;\n    /** Start of failed date range */\n    createdAtAfter?: Date;\n    /** End of failed date range */\n    createdAtBefore?: Date;\n    /** Error message */\n    error: string;\n  }\n\n  const failedChunks: FailedChunk[] = [];\n\n  const chunkCounts = await map(\n    chunks,\n    async (chunk, i) => {\n      const chunkFile = filePaths[i];\n      let headers: string[] | undefined;\n      let rowCount = 0;\n\n      try {\n        await fetchAllRequests(client, {\n          actions,\n          text: identifierSearch,\n          statuses,\n          createdAtBefore: chunk.createdAtBefore,\n          createdAtAfter: chunk.createdAtAfter,\n          updatedAtBefore,\n          updatedAtAfter,\n          isTest,\n          onPage: async (nodes) => {\n            if (nodes.length === 0) return;\n\n            // Optionally enrich each request with its identifiers\n            const enriched: ExportedPrivacyRequest[] = skipRequestIdentifiers\n              ? nodes.map((n) => ({ ...n, requestIdentifiers: [] }))\n              : await map(\n                  nodes,\n                  async (n) => ({\n                    ...n,\n                    requestIdentifiers: await fetchAllRequestIdentifiers(client, sombra!, {\n                      filterBy: { requestId: n.id },\n                      skipSombraCheck: true,\n                      logger,\n                    }),\n                  }),\n                  { concurrency: pageLimit },\n                );\n\n            const rows: Record<string, string | null | number | boolean>[] =\n              enriched.map(formatRequestForCsv);\n\n            if (!headers) {\n              headers = uniq(rows.map((r: Record<string, unknown>) => Object.keys(r)).flat());\n              initCsvFile(chunkFile, headers);\n            }\n\n            appendCsvRowsOrdered(chunkFile, rows, headers);\n            rowCount += rows.length;\n            globalFetched += rows.length;\n            progressBar.update(globalFetched);\n          },\n        });\n      } catch (err) {\n        const message = err instanceof Error ? err.message : String(err);\n        logger.error(\n          colors.red(\n            `Chunk ${i} failed (${\n              chunk.createdAtAfter?.toISOString() ?? 'start'\n            } → ${chunk.createdAtBefore?.toISOString() ?? 'end'}): ${message}`,\n          ),\n        );\n        failedChunks.push({\n          index: i,\n          createdAtAfter: chunk.createdAtAfter,\n          createdAtBefore: chunk.createdAtBefore,\n          error: message,\n        });\n      }\n\n      if (!headers) {\n        initCsvFile(chunkFile, []);\n      }\n\n      return rowCount;\n    },\n    { concurrency: useChunks ? concurrency : 1 },\n  );\n\n  progressBar.stop();\n  const totalCount = chunkCounts.reduce((a, b) => a + b, 0);\n  const elapsed = (Date.now() - t0) / 1000;\n\n  if (failedChunks.length > 0) {\n    logger.error(\n      colors.red(\n        `\\n${failedChunks.length} chunk(s) failed. ` +\n          'Re-run with these date ranges to fill the gaps:',\n      ),\n    );\n    for (const fc of failedChunks) {\n      logger.error(\n        colors.red(\n          `  Chunk ${fc.index}: --createdAtAfter=${\n            fc.createdAtAfter?.toISOString() ?? ''\n          } --createdAtBefore=${fc.createdAtBefore?.toISOString() ?? ''}`,\n        ),\n      );\n    }\n  }\n\n  logger.info(\n    colors.green(`Streamed ${totalCount} requests to ${filePaths.length} file(s) in ${elapsed}s`),\n  );\n\n  return { filePaths, totalCount };\n}\n"],"mappings":"+dAmBA,SAAgB,EAAoB,CAClC,kBACA,qBACA,KACA,QACA,OACA,SACA,cACA,UACA,YACA,0BACA,UACA,SACA,SACA,qBACA,WACA,SACA,iBACA,UACA,GAAG,GAC8B,CACjC,MAAO,CACL,aAAc,EACd,aAAc,EACd,4BAA6B,GAA2B,GACxD,MAAO,EACP,kBAAmB,EACnB,eAAgB,EAChB,oBAAqB,EACrB,OAAQ,EACR,QAAS,EACT,uBAAwB,EACxB,QAAS,EACT,OAAQ,EACR,cAAe,EACf,kBAAmB,EACnB,SAAU,EACV,uBAAwB,GAAS,OAAS,GAAS,MAAQ,GAC3D,wBAAyB,GAAS,SAAS,UAAU,EAAI,GACzD,IAAI,GAAS,qBAAuB,EAAE,EAAE,QAAQ,EAAuC,IAAM,CAC3F,IAAM,EAAQ,EAAE,iBAAiB,MAAM,gBAAkB,EAAE,KAC3D,OAAO,EACH,CACE,GAAG,GACF,GAAQ,EAAE,aACP,EAAE,aAAa,IAAK,GAAM,EAAE,KAAK,CAAC,KAAK,IAAI,CAC3C,EAAE,aAAa,MAAQ,EAAE,aAC9B,CACD,GACH,EAAE,CAAC,CACN,GAAG,EACH,GAAG,OAAO,QAAQ,EAAQ,EAAiB,oBAAoB,CAAC,CAAC,QAC9D,EAAK,CAAC,EAAM,KACX,OAAO,OAAO,EAAK,EAChB,GAAO,EAAO,KAAK,CAAE,KAAM,KAAQ,EAAE,CAAC,KAAK,IAAI,CACjD,CAAC,CACJ,EAAE,CACH,CACD,GAAG,OAAO,QAAQ,EAAQ,EAAoB,OAAO,CAAC,CAAC,QACpD,EAAK,CAAC,EAAM,KACX,OAAO,OAAO,EAAK,EAChB,GAAO,EAAO,KAAK,CAAE,WAAY,EAAM,CAAC,KAAK,IAAI,CACnD,CAAC,CACJ,EAAE,CACH,CACF,CCrEH,SAAgB,EAAe,EAAa,EAAc,EAAoC,CAC5F,IAAM,EAAU,EAAM,SAAS,CACzB,EAAQ,EAAO,SAAS,CACxB,GAAe,EAAQ,GAAW,EACxC,OAAO,MAAM,KAAK,CAAE,OAAQ,EAAQ,EAAG,EAAG,KAAO,CAC/C,eAAgB,IAAI,KAAK,EAAU,EAAc,EAAE,CACnD,gBAAiB,IAAI,KAAK,IAAM,EAAS,EAAI,EAAQ,EAAU,GAAe,EAAI,GAAG,CACtF,EAAE,CCKL,eAAsB,EAA2B,CAC/C,OACA,aACA,UAAU,EAAE,CACZ,WAAW,EAAE,CACb,mBACA,cAAc,EACd,YAAY,IACZ,eAAe,EACf,kBACA,iBACA,kBACA,iBACA,SACA,yBAAyB,GACzB,QAqCC,CACD,IAAM,EAAS,EAA4B,EAAc,EAAK,CACxD,EAAS,EACX,IAAA,GACA,MAAM,EAAwB,EAAc,EAAM,CAChD,SACA,aAAc,EACd,UAAW,QAAQ,IAAI,WACxB,CAAC,CAGF,EAAY,GACZ,IACF,GAAa,WAAW,EAAgB,aAAa,IAEnD,IACF,GAAa,GAAG,EAAY,QAAU,GAAG,SAAS,EAAe,aAAa,IAEhF,EAAO,KACL,EAAO,QACL,GACE,EAAQ,OAAS,EACb,6BAA6B,EAAQ,KAAK,QAAQ,CAAC,GACnD,yBACH,IACJ,CACF,CAGD,IAAM,EAAY,EAAc,GAAK,GAAkB,EACjD,EAAS,EACX,EAAe,EAAgB,EAAiB,EAAY,CAC5D,CAAC,CAAE,iBAAgB,kBAAiB,CAAC,CAErC,GACF,EAAO,KAAK,EAAO,QAAQ,6BAA6B,EAAY,kBAAkB,CAAC,CAIzF,IAAM,EAAW,CACf,KAAM,EAAQ,OAAS,EAAI,EAAU,IAAA,GACrC,OAAQ,EAAS,OAAS,EAAI,EAAW,IAAA,GACzC,SACA,gBAAiB,EAAkB,EAAgB,aAAa,CAAG,IAAA,GACnE,eAAgB,EAAiB,EAAe,aAAa,CAAG,IAAA,GAChE,gBAAiB,EAAkB,EAAgB,aAAa,CAAG,IAAA,GACnE,eAAgB,EAAiB,EAAe,aAAa,CAAG,IAAA,GACjE,CAEK,EAAK,KAAK,KAAK,CAGhB,GACH,MAAM,EAAsB,EAAQ,CAAE,SAAQ,CAAC,CAGjD,IAAM,EAAgB,MAAM,EAAwB,EAAQ,EAAS,CACrE,EAAO,KAAK,EAAO,QAAQ,YAAY,EAAc,WAAW,CAAC,CAEjE,IAAM,EAAc,IAAI,EAAY,UAAU,EAAE,CAAE,EAAY,QAAQ,eAAe,CACrF,EAAY,MAAM,EAAe,EAAE,CAEnC,IAAI,EAAgB,EAEd,CAAE,WAAU,aAAc,EAAc,EAAK,CAE7C,EAAY,EAAO,KAAK,EAAG,IAC/B,EAAO,SAAW,EAAI,EAAO,GAAG,EAAS,GAAG,IAAI,IACjD,CAaK,EAA8B,EAAE,CAEhC,EAAc,MAAM,EACxB,EACA,MAAO,EAAO,IAAM,CAClB,IAAM,EAAY,EAAU,GACxB,EACA,EAAW,EAEf,GAAI,CACF,MAAM,EAAiB,EAAQ,CAC7B,UACA,KAAM,EACN,WACA,gBAAiB,EAAM,gBACvB,eAAgB,EAAM,eACtB,kBACA,iBACA,SACA,OAAQ,KAAO,IAAU,CACvB,GAAI,EAAM,SAAW,EAAG,OAkBxB,IAAM,GAfqC,EACvC,EAAM,IAAK,IAAO,CAAE,GAAG,EAAG,mBAAoB,EAAE,CAAE,EAAE,CACpD,MAAM,EACJ,EACA,KAAO,KAAO,CACZ,GAAG,EACH,mBAAoB,MAAM,EAA2B,EAAQ,EAAS,CACpE,SAAU,CAAE,UAAW,EAAE,GAAI,CAC7B,gBAAiB,GACjB,SACD,CAAC,CACH,EACD,CAAE,YAAa,EAAW,CAC3B,EAGM,IAAI,EAAoB,CAE9B,IACH,EAAU,EAAK,EAAK,IAAK,GAA+B,OAAO,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,CAC/E,EAAY,EAAW,EAAQ,EAGjC,EAAqB,EAAW,EAAM,EAAQ,CAC9C,GAAY,EAAK,OACjB,GAAiB,EAAK,OACtB,EAAY,OAAO,EAAc,EAEpC,CAAC,OACK,EAAK,CACZ,IAAM,EAAU,aAAe,MAAQ,EAAI,QAAU,OAAO,EAAI,CAChE,EAAO,MACL,EAAO,IACL,SAAS,EAAE,WACT,EAAM,gBAAgB,aAAa,EAAI,QACxC,KAAK,EAAM,iBAAiB,aAAa,EAAI,MAAM,KAAK,IAC1D,CACF,CACD,EAAa,KAAK,CAChB,MAAO,EACP,eAAgB,EAAM,eACtB,gBAAiB,EAAM,gBACvB,MAAO,EACR,CAAC,CAOJ,OAJK,GACH,EAAY,EAAW,EAAE,CAAC,CAGrB,GAET,CAAE,YAAa,EAAY,EAAc,EAAG,CAC7C,CAED,EAAY,MAAM,CAClB,IAAM,EAAa,EAAY,QAAQ,EAAG,IAAM,EAAI,EAAG,EAAE,CACnD,GAAW,KAAK,KAAK,CAAG,GAAM,IAEpC,GAAI,EAAa,OAAS,EAAG,CAC3B,EAAO,MACL,EAAO,IACL,KAAK,EAAa,OAAO,mEAE1B,CACF,CACD,IAAK,IAAM,KAAM,EACf,EAAO,MACL,EAAO,IACL,WAAW,EAAG,MAAM,qBAClB,EAAG,gBAAgB,aAAa,EAAI,GACrC,qBAAqB,EAAG,iBAAiB,aAAa,EAAI,KAC5D,CACF,CAQL,OAJA,EAAO,KACL,EAAO,MAAM,YAAY,EAAW,eAAe,EAAU,OAAO,cAAc,EAAQ,GAAG,CAC9F,CAEM,CAAE,YAAW,aAAY"}