{"version":3,"sources":["../src/index.ts"],"names":["api","FlatfileClient","WORKBOOK_CREATION_DELAY","Extractor","fileExt","extractorType","parseBuffer","options","listener","event","fileId","spaceId","file","jobs","Flatfile","jobId","chunkSize","parallel","debug","tick","progress","info","entitlements","headerSelectionEnabled","e","sourceEditorEnabled","customRecordsPageSize","buffer","getFileBuffer","getHeaders","data","headers","capture","workbook","createWorkbook","resolve","sheet","createAllRecords","normalizeRecordKeys","_progress","part","totalParts","updateSheetMetadata","environmentId","workbookCapture","workbookConfig","getWorkbookConfig","name","sheets","sheetName","getSheetConfig","descriptions","normalizeSheetConfig","slugify","keysToFields","record","normalizedRecord","key","normalizeKey","keys","metadata"],"mappings":";;;;;;AAUA,IAAMA,CAAM,CAAA,IAAIC,mBAEVC,CAA0B,CAAA,GAAA,CAEnBC,EAAY,CACvBC,CAAAA,CACAC,EACAC,CAIAC,CAAAA,CAAAA,GAEQC,GAA+B,CACrCA,CAAAA,CAAS,GAAG,cAAgB,CAAA,MAAOC,GAAU,CAC3C,GAAM,CAAE,MAAAC,CAAAA,CAAAA,CAAQ,OAAAC,CAAAA,CAAQ,EAAIF,CAAM,CAAA,OAAA,CAC5B,CAAE,IAAMG,CAAAA,CAAK,EAAI,MAAMZ,CAAAA,CAAI,KAAM,CAAA,GAAA,CAAIU,CAAM,CASjD,CAAA,GARIE,EAAK,IAAS,GAAA,QAAA,EAId,OAAOR,CAAY,EAAA,QAAA,EAAY,CAACQ,CAAAA,CAAK,KAAK,QAASR,CAAAA,CAAO,GAI1DA,CAAmB,YAAA,MAAA,EAAU,CAACA,CAAQ,CAAA,IAAA,CAAKQ,EAAK,IAAI,CAAA,CACtD,OAEF,IAAMC,CAAAA,CAAO,MAAMb,CAAI,CAAA,IAAA,CAAK,OAAO,CACjC,IAAA,CAAMc,YAAS,CAAA,OAAA,CAAQ,KACvB,SAAW,CAAA,CAAA,eAAA,EAAkBT,CAAa,CAC1C,CAAA,CAAA,MAAA,CAAQS,aAAS,SAAU,CAAA,KAAA,CAC3B,MAAQJ,CAAAA,CACV,CAAC,CACD,CAAA,MAAMV,EAAI,IAAK,CAAA,OAAA,CAAQa,EAAK,IAAK,CAAA,EAAE,EACrC,CAAC,EACDL,CAAS,CAAA,EAAA,CACP,YACA,CAAE,SAAA,CAAW,kBAAkBH,CAAa,CAAA,CAAG,EAC/C,MAAOI,CAAAA,EAAU,CACf,GAAM,CAAE,OAAAC,CAAQ,CAAA,KAAA,CAAAK,EAAO,OAAAJ,CAAAA,CAAQ,CAAIF,CAAAA,CAAAA,CAAM,QACnC,CAAE,SAAA,CAAAO,EAAW,QAAAC,CAAAA,CAAAA,CAAU,MAAAC,CAAM,CAAA,CAAI,CACrC,SAAA,CAAW,IACX,QAAU,CAAA,CAAA,CACV,MAAO,CACP,CAAA,CAAA,GAAGX,CACL,CAEMY,CAAAA,CAAAA,CAAO,MAAOC,CAAAA,CAAkBC,IAAkB,CACtD,MAAMrB,EAAI,IAAK,CAAA,GAAA,CAAIe,EAAO,CAAE,QAAA,CAAAK,EAAU,IAAAC,CAAAA,CAAK,CAAC,CACxCH,CAAAA,CAAAA,EACF,QAAQ,GAAI,CAAA,CAAA,cAAA,EAAiBE,CAAQ,CAAWC,QAAAA,EAAAA,CAAI,CAAE,CAAA,EAE1D,EAEA,GAAI,CACF,MAAMF,CAAK,CAAA,CAAA,CAAG,iCAAiC,CAC/C,CAAA,GAAM,CAAE,IAAA,CAAMP,CAAK,CAAI,CAAA,MAAMZ,EAAI,KAAM,CAAA,GAAA,CAAIU,CAAM,CAE3C,CAAA,CAAE,IAAMY,CAAAA,CAAa,EAAI,MAAMtB,CAAAA,CAAI,aAAa,IAAK,CAAA,CACzD,WAAYW,CACd,CAAC,EACKY,CAAyB,CAAA,CAAC,CAACD,CAAa,CAAA,IAAA,CAC3CE,GAAMA,CAAE,CAAA,GAAA,GAAQ,iBACnB,CAEMC,CAAAA,CAAAA,CAAsB,CAAC,CAACH,EAAa,IACxCE,CAAAA,CAAAA,EAAMA,EAAE,GAAQ,GAAA,cACnB,EAEME,CAAwBJ,CAAAA,CAAAA,CAAa,IACxCE,CAAAA,CAAAA,EAAMA,EAAE,GAAQ,GAAA,uBACnB,EAEMG,CAAS,CAAA,MAAMC,6BAAcnB,CAAK,CAAA,CAGlCoB,CAAa,CAAA,MAAOtB,EAAcuB,CAAqB,GAAA,CAC3D,GAAI,CACF,GAAM,CAAE,IAAMC,CAAAA,CAAQ,EAAI,MAAM/B,CAAAA,CAAI,MAAM,YAAa,CAAA,CACrD,QAAS,CACP,GAAGO,CACL,CACA,CAAA,IAAA,CAAAuB,CACF,CAAC,EACD,OAAOC,CACT,OAASP,CAAG,CAAA,CACV,eAAQ,GAAIA,CAAAA,CAAAA,CAAG,CAAE,KAAA,CAAO,IAAK,CAAC,CAAA,CACvB,CAAE,IAAM,CAAA,CAAA,CAAG,OAAQ,EAAG,CAC/B,CACF,EAEA,MAAML,CAAAA,CAAK,EAAG,gCAAgC,CAAA,CAC9C,IAAMa,CAAU,CAAA,MAAM1B,EAAYqB,CAAQ,CAAA,CACxC,GAAGpB,CACH,CAAA,MAAA,CAAAG,EACA,OAASE,CAAAA,CAAAA,CAAK,IACd,sBAAAW,CAAAA,CAAAA,CACA,UAAAM,CAAAA,CACF,CAAC,CAED,CAAA,MAAMV,EAAK,CAAG,CAAA,mCAAmC,EACjD,IAAMc,CAAAA,CAAW,MAAMC,CAAAA,CACrBzB,EAAM,OAAQ,CAAA,aAAA,CACdG,EACAoB,CACAP,CAAAA,CACF,EAQA,GAJA,MAAMzB,CAAI,CAAA,KAAA,CAAM,OAAOY,CAAK,CAAA,EAAA,CAAI,CAC9B,UAAYqB,CAAAA,CAAAA,CAAS,EACvB,CAAC,CAAA,CAEG,CAACA,CAAS,CAAA,MAAA,EAAUA,EAAS,MAAO,CAAA,MAAA,GAAW,EACjD,MAAM,IAAI,MAAM,6BAA6B,CAAA,CAG/C,MAAMd,CAAAA,CAAK,GAAI,kCAAkC,CAAA,CAEjD,MAAM,IAAI,OAAA,CAASgB,GAAY,CAC7B,UAAA,CAAWA,CAASjC,CAAAA,CAAuB,EAC7C,CAAC,CAAA,CAED,QAAWkC,CAASH,IAAAA,CAAAA,CAAS,OACtBD,CAAQI,CAAAA,CAAAA,CAAM,IAAI,CAAA,EAGvB,MAAMC,2BACJD,CAAAA,CAAAA,CAAM,GACNJ,CAAQI,CAAAA,CAAAA,CAAM,IAAI,CAAE,CAAA,IAAA,CAAK,IAAIE,CAAmB,CAAA,CAChD,MAAOC,CAAWC,CAAAA,CAAAA,CAAMC,IAAe,CACrC,MAAMtB,EACJ,IAAK,CAAA,GAAA,CAAI,EAAI,CAAA,IAAA,CAAK,MAAM,EAAK,CAAA,EAAA,EAAMqB,EAAOC,CAAW,CAAA,CAAC,EACtD,kCACF,EACF,CACA,CAAA,CACE,SAAUf,CAAuB,EAAA,QAAA,EAAU,QAC7C,CACF,CAAA,CAIEH,GACF,MAAMmB,CAAAA,CAAoBT,CAAUD,CAAAA,CAAO,EAG7C,MAAMhC,CAAAA,CAAI,MAAM,MAAOY,CAAAA,CAAAA,CAAK,GAAI,CAC9B,MAAA,CAAQ,UACV,CAAC,CAAA,CACD,MAAMZ,CAAI,CAAA,IAAA,CAAK,SAASe,CAAO,CAAA,CAC7B,KAAM,+CACN,CAAA,OAAA,CAAS,CACP,OAAA,CAAS,kCACX,CACF,CAAC,EACH,CAASS,MAAAA,CAAAA,CAAG,CACNN,CACF,EAAA,OAAA,CAAQ,GAAI,CAAA,CAAA,iBAAA,EAAoBM,EAAE,OAAO,CAAA,CAAE,EAE7C,MAAMxB,CAAAA,CAAI,MAAM,MAAOU,CAAAA,CAAAA,CAAQ,CAC7B,MAAA,CAAQ,QACV,CAAC,CAAA,CACD,MAAMV,CAAI,CAAA,IAAA,CAAK,KAAKe,CAAO,CAAA,CACzB,KAAM,qCACN,CAAA,OAAA,CAAS,CACP,OAASS,CAAAA,CAAAA,CAAE,OACb,CACF,CAAC,EACH,CACF,CACF,EACF,EAGF,eAAeU,CACbS,CAAAA,CAAAA,CACA/B,EACAgC,CACAnB,CAAAA,CAAAA,CAC4B,CAC5B,IAAMoB,CAAAA,CAAiBC,CACrBlC,CAAAA,CAAAA,CAAK,KACLA,CAAK,CAAA,OAAA,CACL+B,EACAC,CACAnB,CAAAA,CACF,EACM,CAAE,IAAA,CAAMQ,CAAS,CAAA,CAAI,MAAMjC,CAAI,CAAA,SAAA,CAAU,OAAO6C,CAAc,CAAA,CACpE,OAAOZ,CACT,CAEA,SAASa,CACPC,CAAAA,CAAAA,CACApC,EACAgC,CACAC,CAAAA,CAAAA,CACAnB,EAC+B,CAC/B,IAAMuB,EAAS,MAAO,CAAA,OAAA,CAAQJ,CAAe,CAAA,CAAE,IAAI,CAAC,CAACK,EAAWb,CAAK,CAAA,GAC5Dc,EAAeD,CAAWb,CAAAA,CAAAA,CAAOX,CAAmB,CAC5D,EAED,OAAO,CACL,KAAM,CAAUsB,OAAAA,EAAAA,CAAI,GACpB,MAAQ,CAAA,CAAC,MAAM,CAAA,CACf,QAAApC,CACA,CAAA,aAAA,CAAAgC,EACA,MAAAK,CAAAA,CAAAA,CACA,WAAY,CAAClC,YAAAA,CAAS,mBAAmB,mBAAmB,CAC9D,CACF,CAEA,SAASoC,EACPH,CACA,CAAA,CAAE,QAAAhB,CAAS,CAAA,YAAA,CAAAoB,CAAa,CAAA,CACxB1B,EACsB,CAEtB,OAAO2B,gCAAqB,CAC1B,IAAA,CAAAL,EACA,IAAMM,CAAAA,kBAAAA,CAAQN,CAAI,CAAA,CAClB,OAAQO,CAAa,CAAA,CAAE,KAAMvB,CAAS,CAAA,YAAA,CAAAoB,CAAa,CAAC,CAAA,CACpD,qBAAuB1B,CAAAA,CACzB,CAAC,CACH,CAEA,SAASa,CAAoBiB,CAAAA,CAAAA,CAAkD,CAC7E,IAAMC,CAAAA,CAAmB,EACzB,CAAA,IAAA,IAAWC,KAAOF,CACZ,CAAA,MAAA,CAAO,OAAOA,CAAQE,CAAAA,CAAG,IAC3BD,CAAiBE,CAAAA,uBAAAA,CAAaD,CAAG,CAAC,EAAIF,CAAOE,CAAAA,CAAG,GAGpD,OAAOD,CACT,CAEO,SAASF,CAAAA,CAAa,CAC3B,IAAA,CAAAK,EACA,YAAAR,CAAAA,CAAAA,CAAe,EACjB,CAAA,CAGwB,CACtB,OAAOQ,CAAAA,CAAK,GAAKF,CAAAA,CAAAA,GACVA,IAAKA,CAAM,CAAA,EAAA,CAAA,CACZ,OAAOA,CAAQ,EAAA,QAAA,GACjBA,EAAM,MAAOA,CAAAA,CAAG,GAElBA,CAAMA,CAAAA,CAAAA,CAAI,MAEH,CAAA,CACL,IAAAA,CACA,CAAA,KAAA,CAAOA,EACP,WAAaN,CAAAA,CAAAA,GAAeM,CAAG,CAAA,EAAK,GACpC,IAAM,CAAA,QACR,EACD,CACH,CAEA,eAAef,CACbT,CAAAA,CAAAA,CACAW,EACe,CACf,MAAM,QAAQ,GACZX,CAAAA,CAAAA,CAAS,OAAO,GAAI,CAAA,MAAOG,GAAU,CACnC,GAAM,CAAE,QAAA,CAAAwB,CAAS,CAAIhB,CAAAA,CAAAA,CAAgBR,EAAM,IAAI,CAAA,CAC/C,MAAMpC,CAAI,CAAA,MAAA,CAAO,YAAYoC,CAAM,CAAA,EAAA,CAAI,CACrC,QAAAwB,CAAAA,CACF,CAAC,EACH,CAAC,CACH,EACF","file":"index.cjs","sourcesContent":["import { Flatfile, FlatfileClient } from '@flatfile/api'\nimport type { FlatfileListener } from '@flatfile/listener'\nimport {\n createAllRecords,\n normalizeKey,\n normalizeSheetConfig,\n slugify,\n} from '@flatfile/util-common'\nimport { getFileBuffer } from '@flatfile/util-file-buffer'\n\nconst api = new FlatfileClient()\n\nconst WORKBOOK_CREATION_DELAY = 3_000\n\nexport const Extractor = (\n fileExt: string | RegExp,\n extractorType: string,\n parseBuffer: (\n buffer: Buffer,\n options: any\n ) => WorkbookCapture | Promise,\n options?: Record\n) => {\n return (listener: FlatfileListener) => {\n listener.on('file:created', async (event) => {\n const { fileId, spaceId } = event.context\n const { data: file } = await api.files.get(fileId)\n if (file.mode === 'export') {\n return\n }\n\n if (typeof fileExt === 'string' && !file.name.endsWith(fileExt)) {\n return\n }\n\n if (fileExt instanceof RegExp && !fileExt.test(file.name)) {\n return\n }\n const jobs = await api.jobs.create({\n type: Flatfile.JobType.File,\n operation: `extract-plugin-${extractorType}`,\n status: Flatfile.JobStatus.Ready,\n source: fileId,\n })\n await api.jobs.execute(jobs.data.id)\n })\n listener.on(\n 'job:ready',\n { operation: `extract-plugin-${extractorType}` },\n async (event) => {\n const { fileId, jobId, spaceId } = event.context\n const { chunkSize, parallel, debug } = {\n chunkSize: 5_000,\n parallel: 1,\n debug: false,\n ...options,\n }\n\n const tick = async (progress: number, info?: string) => {\n await api.jobs.ack(jobId, { progress, info })\n if (debug) {\n console.log(`Job progress: ${progress}, Info: ${info}`)\n }\n }\n\n try {\n await tick(1, 'plugins.extraction.retrieveFile')\n const { data: file } = await api.files.get(fileId)\n\n const { data: entitlements } = await api.entitlements.list({\n resourceId: spaceId,\n })\n const headerSelectionEnabled = !!entitlements.find(\n (e) => e.key === 'headerSelection'\n )\n\n const sourceEditorEnabled = !!entitlements.find(\n (e) => e.key === 'sourceEditor'\n )\n\n const customRecordsPageSize = entitlements.find(\n (e) => e.key === 'customRecordsPageSize'\n )\n\n const buffer = await getFileBuffer(event)\n\n // inject the getHeaders function into the options\n const getHeaders = async (options: any, data: string[][]) => {\n try {\n const { data: headers } = await api.files.detectHeader({\n options: {\n ...options,\n },\n data,\n })\n return headers\n } catch (e) {\n console.dir(e, { depth: null })\n return { skip: 0, header: [] }\n }\n }\n\n await tick(3, 'plugins.extraction.parseSheets')\n const capture = await parseBuffer(buffer, {\n ...options,\n fileId,\n fileExt: file.ext,\n headerSelectionEnabled,\n getHeaders,\n })\n\n await tick(5, 'plugins.extraction.createWorkbook')\n const workbook = await createWorkbook(\n event.context.environmentId,\n file,\n capture,\n sourceEditorEnabled\n )\n\n // Add workbook to file so if the extraction fails and the file is deleted, the workbook is also deleted\n // instead of being orphaned\n await api.files.update(file.id, {\n workbookId: workbook.id,\n })\n\n if (!workbook.sheets || workbook.sheets.length === 0) {\n throw new Error('plugins.extraction.noSheets')\n }\n\n await tick(10, 'plugins.extraction.addingRecords')\n\n await new Promise((resolve) => {\n setTimeout(resolve, WORKBOOK_CREATION_DELAY)\n })\n\n for (const sheet of workbook.sheets) {\n if (!capture[sheet.name]) {\n continue\n }\n await createAllRecords(\n sheet.id,\n capture[sheet.name].data.map(normalizeRecordKeys),\n async (_progress, part, totalParts) => {\n await tick(\n Math.min(99, Math.round(10 + 90 * (part / totalParts))),\n 'plugins.extraction.addingRecords'\n )\n },\n {\n pageSize: customRecordsPageSize?.metadata?.pageSize,\n }\n )\n }\n\n // After all records are added, update the sheet metadata\n if (headerSelectionEnabled) {\n await updateSheetMetadata(workbook, capture)\n }\n\n await api.files.update(file.id, {\n status: 'complete',\n })\n await api.jobs.complete(jobId, {\n info: 'files.uploadFile.popovers.extractionCompleted',\n outcome: {\n message: 'plugins.extraction.extractedFile',\n },\n })\n } catch (e) {\n if (debug) {\n console.log(`Extractor error: ${e.message}`)\n }\n await api.files.update(fileId, {\n status: 'failed',\n })\n await api.jobs.fail(jobId, {\n info: 'plugins.extraction.extractionFailed',\n outcome: {\n message: e.message,\n },\n })\n }\n }\n )\n }\n}\n\nasync function createWorkbook(\n environmentId: string,\n file: Flatfile.File_,\n workbookCapture: WorkbookCapture,\n sourceEditorEnabled: boolean\n): Promise {\n const workbookConfig = getWorkbookConfig(\n file.name,\n file.spaceId,\n environmentId,\n workbookCapture,\n sourceEditorEnabled\n )\n const { data: workbook } = await api.workbooks.create(workbookConfig)\n return workbook\n}\n\nfunction getWorkbookConfig(\n name: string,\n spaceId: string,\n environmentId: string,\n workbookCapture: WorkbookCapture,\n sourceEditorEnabled: boolean\n): Flatfile.CreateWorkbookConfig {\n const sheets = Object.entries(workbookCapture).map(([sheetName, sheet]) => {\n return getSheetConfig(sheetName, sheet, sourceEditorEnabled)\n })\n\n return {\n name: `[file] ${name}`,\n labels: ['file'], // we use this on the backend to add the EXTRACTED_FROM_SOURCE treatment to the workbook on previous versions of the plugin\n spaceId,\n environmentId,\n sheets,\n treatments: [Flatfile.WorkbookTreatments.ExtractedFromSource],\n }\n}\n\nfunction getSheetConfig(\n name: string,\n { headers, descriptions }: SheetCapture,\n sourceEditorEnabled: boolean\n): Flatfile.SheetConfig {\n // normalizeSheetConfig the keys to ensure they are unique and valid\n return normalizeSheetConfig({\n name,\n slug: slugify(name),\n fields: keysToFields({ keys: headers, descriptions }),\n allowAdditionalFields: sourceEditorEnabled,\n })\n}\n\nfunction normalizeRecordKeys(record: Flatfile.RecordData): Flatfile.RecordData {\n const normalizedRecord = {} as Flatfile.RecordData\n for (const key in record) {\n if (Object.hasOwn(record, key)) {\n normalizedRecord[normalizeKey(key)] = record[key]\n }\n }\n return normalizedRecord\n}\n\nexport function keysToFields({\n keys,\n descriptions = {},\n}: {\n keys: string[]\n descriptions?: Record\n}): Flatfile.Property[] {\n return keys.map((key) => {\n if (!key) key = ''\n if (typeof key !== 'string') {\n key = String(key)\n }\n key = key.trim()\n\n return {\n key,\n label: key,\n description: descriptions?.[key] || '',\n type: 'string',\n }\n })\n}\n\nasync function updateSheetMetadata(\n workbook: Flatfile.Workbook,\n workbookCapture: WorkbookCapture\n): Promise {\n await Promise.all(\n workbook.sheets.map(async (sheet) => {\n const { metadata } = workbookCapture[sheet.name]\n await api.sheets.updateSheet(sheet.id, {\n metadata,\n })\n })\n )\n}\n/**\n * Generic structure for capturing a workbook\n */\nexport type WorkbookCapture = Record\n\n/**\n * Generic structure for capturing a sheet\n */\nexport type SheetCapture = {\n headers: string[]\n descriptions?: Record | null\n data: Flatfile.RecordData[]\n metadata?: { rowHeaders: number[] }\n}\n"]}