{"version":3,"file":"parquetToCsvOneFile-0O4iLz0t.mjs","names":[],"sources":["../src/lib/helpers/parquetToCsvOneFile.ts"],"sourcesContent":["import { mkdirSync, rmSync, existsSync } from 'node:fs';\nimport { dirname, join, parse } from 'node:path';\n\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport colors from 'colors';\n\nimport { logger } from '../../logger.js';\n\n/** Progress callback used by the parent runner to surface progress to the UI. */\ntype OnProgress = (processed: number, total?: number) => void;\n\n/**\n * Options for converting a single Parquet file into a single CSV file.\n */\nexport type ParquetToCsvOneFileOptions = {\n  /** Absolute or relative path to the input `.parquet` file. */\n  filePath: string;\n  /**\n   * Directory where the output CSV will be written.\n   * If omitted, the CSV is written next to the input file.\n   */\n  outputDir?: string;\n  /**\n   * When true, removes a pre-existing output file with the same name before writing.\n   * Useful for re-runs; ignored if the file does not exist.\n   */\n  clearOutputDir: boolean;\n  /**\n   * Optional progress hook. Called with the number of processed records.\n   * `total` is not computed here; it will be `undefined`.\n   */\n  onProgress?: OnProgress;\n};\n\n/**\n * Convert a single Parquet file to a single CSV file (1:1) using DuckDB.\n *\n * Output naming: `${basename}.csv` in `outputDir ?? dirname(filePath)`.\n *\n * Errors:\n *  - Throws on I/O failures or DuckDB execution errors.\n *\n *  Why DuckDB?\n * - Robust reader for many Parquet dialects (e.g., Spark output, nested types, timestamps).\n * - Streaming COPY handles large files without loading everything into JS memory.\n *\n * What this does:\n *  - Opens an in-memory DuckDB database (no `.db` file created).\n *  - Optionally disables temp spilling to disk (so only your CSV is written).\n *  - Executes a single `COPY (SELECT * FROM read_parquet(...)) TO ...` statement.\n *  - Produces exactly one CSV per input Parquet (no chunking or rotation).\n *\n * Notes & defaults:\n *  - DuckDBInstance: `:memory:` (ephemeral). No persistent DB file is created.\n *  - Temp files: disabled via `PRAGMA temp_directory=''` (best-effort; ignored if unsupported).\n *  - CSV format: header row, comma delimiter, double-quote quoting, empty string for NULL.\n *  - Progress: DuckDB COPY doesn't expose row-level progress via the JS API; we emit a\n *    best-effort final callback.\n *\n * Requirements:\n *  - `@duckdb/node-api` npm package installed and available at runtime.\n *  - Supported platform binary (mac arm64/x64, linux x64, windows x64).\n *\n * @param opts - Conversion options\n * @param DuckDb - DuckDB instance to use\n * @returns Promise<void> when the CSV has been written\n */\nexport async function parquetToCsvOneFile(\n  opts: ParquetToCsvOneFileOptions,\n  DuckDb: typeof DuckDBInstance,\n): Promise<void> {\n  const { filePath, outputDir, clearOutputDir, onProgress } = opts;\n\n  const baseDir = outputDir || dirname(filePath);\n  const { name: baseName } = parse(filePath);\n  const outPath = join(baseDir, `${baseName}.csv`);\n\n  // Ensure output directory exists\n  mkdirSync(baseDir, { recursive: true });\n\n  // Remove any pre-existing output file if requested\n  if (clearOutputDir && existsSync(outPath)) {\n    try {\n      rmSync(outPath, { force: true });\n    } catch (err) {\n      logger.warn(\n        colors.yellow(\n          `Could not remove existing output file ${outPath}: ${(err as Error).message}`,\n        ),\n      );\n    }\n  }\n\n  // In-memory DB: no .db file created on disk\n  const db = await DuckDb.create(':memory:');\n  const conn = await db.connect();\n\n  try {\n    // Optional: prevent DuckDB from creating temp files on disk (best-effort).\n    // Some versions may ignore or error; we ignore such errors safely.\n    await runIgnoreError(conn, \"PRAGMA temp_directory='';\");\n\n    // Optionally: cap memory to encourage in-memory execution or fail-fast\n    // (commented out by default; uncomment to enforce a limit)\n    // await runIgnoreError(conn, \"PRAGMA memory_limit='4GB';\");\n\n    // Ensure stable CSV settings: header, comma delimiter, double quotes, empty string for NULLs.\n    // Escape single quotes for SQL string literals\n    const q = (p: string): string => `'${p.replace(/'/g, \"''\")}'`;\n\n    // Use COPY with a subquery so DuckDB streams Parquet -> CSV efficiently.\n    const sql = `\n      COPY (SELECT * FROM read_parquet(${q(filePath)}))\n      TO ${q(outPath)}\n      (HEADER, DELIMITER ',', QUOTE '\"', ESCAPE '\"', NULL '');\n    `;\n\n    await run(conn, sql);\n\n    // Best-effort progress notification (DuckDB JS API doesn't expose progress for COPY)\n    onProgress?.(0, undefined);\n\n    logger.info(colors.green(`Wrote CSV → ${outPath}`));\n  } finally {\n    // Close connection + db handles gracefully\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    await disposeSafe(conn as any);\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    await disposeSafe(db as any);\n  }\n}\n\n/* =============================================================================\n * DuckDB helpers\n * =============================================================================\n */\n\n/**\n * Execute a SQL statement on a DuckDB connection and dispose the result.\n *\n * @param conn - DuckDB connection\n * @param sql  - SQL string to run\n * @returns Promise<void>\n */\nasync function run(conn: DuckDBConnection, sql: string): Promise<void> {\n  const result = await conn.run(sql);\n  // The high-level API returns a Result; ensure we dispose it to free buffers.\n  // eslint-disable-next-line @typescript-eslint/no-explicit-any\n  await disposeSafe(result as any);\n}\n\n/**\n * Execute a SQL statement but ignore any error that occurs.\n * Useful for best-effort PRAGMAs that may not be supported across versions.\n *\n * @param conn - DuckDB connection\n * @param sql  - SQL string to run\n * @returns Promise<void>\n */\nasync function runIgnoreError(conn: DuckDBConnection, sql: string): Promise<void> {\n  try {\n    await run(conn, sql);\n  } catch {\n    // ignore\n  }\n}\n\n/**\n * Dispose a DuckDB resource (connection or instance) if present.\n *\n * @param handle - Object exposing an async `dispose()` method\n * @returns Promise<void>\n */\nasync function disposeSafe(\n  handle:\n    | {\n        /** Dispose handler */\n        dispose: () => Promise<void>;\n      }\n    | null\n    | undefined,\n): Promise<void> {\n  if (!handle || typeof handle.dispose !== 'function') return;\n  try {\n    await handle.dispose();\n  } catch {\n    // ignore\n  }\n}\n"],"mappings":"yLAmEA,eAAsB,EACpB,EACA,EACe,CACf,GAAM,CAAE,WAAU,YAAW,iBAAgB,cAAe,EAEtD,EAAU,GAAa,EAAQ,EAAS,CACxC,CAAE,KAAM,GAAa,EAAM,EAAS,CACpC,EAAU,EAAK,EAAS,GAAG,EAAS,MAAM,CAMhD,GAHA,EAAU,EAAS,CAAE,UAAW,GAAM,CAAC,CAGnC,GAAkB,EAAW,EAAQ,CACvC,GAAI,CACF,EAAO,EAAS,CAAE,MAAO,GAAM,CAAC,OACzB,EAAK,CACZ,EAAO,KACL,EAAO,OACL,yCAAyC,EAAQ,IAAK,EAAc,UACrE,CACF,CAKL,IAAM,EAAK,MAAM,EAAO,OAAO,WAAW,CACpC,EAAO,MAAM,EAAG,SAAS,CAE/B,GAAI,CAGF,MAAM,EAAe,EAAM,4BAA4B,CAQvD,IAAM,EAAK,GAAsB,IAAI,EAAE,QAAQ,KAAM,KAAK,CAAC,GAS3D,MAAM,EAAI,EAAM;yCALqB,EAAE,EAAS,CAAC;WAC1C,EAAE,EAAQ,CAAC;;MAIE,CAGpB,IAAa,EAAG,IAAA,GAAU,CAE1B,EAAO,KAAK,EAAO,MAAM,eAAe,IAAU,CAAC,QAC3C,CAGR,MAAM,EAAY,EAAY,CAE9B,MAAM,EAAY,EAAU,EAgBhC,eAAe,EAAI,EAAwB,EAA4B,CAIrE,MAAM,EAAY,MAHG,EAAK,IAAI,EAAI,CAGF,CAWlC,eAAe,EAAe,EAAwB,EAA4B,CAChF,GAAI,CACF,MAAM,EAAI,EAAM,EAAI,MACd,GAWV,eAAe,EACb,EAOe,CACX,MAAC,GAAU,OAAO,EAAO,SAAY,YACzC,GAAI,CACF,MAAM,EAAO,SAAS,MAChB"}