import { dateFormat, isDate } from "sync-ql"; import { read, utils } from "xlsx"; import { getFromS3, putToS3 } from "./storage"; export interface ExcelFiles { [fileName: string]: ExcelFile } export interface ExcelFile { [sheetName: string]: ExcelSheet } export type ExcelSheet = any[]; export interface ExcelColumn { name: string, type: 'string' | 'number' | 'date' | 'boolean', index: number, } export async function readExcelFiles(files: FileList, sheetRows?: number, sheets?: number, delimiter?: string, callback?: (progress: number) => void): Promise { const result: ExcelFiles = {}; if (files && files.length > 0) { let filesSize = 0; let loadedFilesSize = 0; for (let f = 0; f < files.length; f++) { const file = files[f]; if (file) { filesSize += file.size } } for (let f = 0; f < files.length; f++) { const file = files[f]; if (file) { try { const excelFile = await readExcelFile(file, sheetRows, sheets, delimiter, loaded => { if (callback) { callback(Math.round(10000 * (loadedFilesSize + loaded) / filesSize) / 100) } }); loadedFilesSize += file.size; if (excelFile) { result[file.name] = excelFile; } } finally { if (callback) { callback(Math.round(10000 * loadedFilesSize / filesSize) / 100) } } } } } return result; } async function readExcelFile(file: File, sheetRows?: number, sheets?: number, delimiter?: string, callback?: (loadedSize: number) => void): Promise { if (file) { const result: ExcelFile = {}; return await new Promise((resolve, reject) => { const reader = new FileReader(); { reader.onprogress = ({ loaded }) => { if (callback) { callback(loaded); } } reader.onload = ({ target, loaded }) => { if (callback) { callback(loaded); } setTimeout(() => { try { const workBook = read(target.result, { type: 'binary', sheetRows, FS: delimiter }) const sheetsCount = sheets && sheets > 0 ? Math.min(sheets, workBook.SheetNames.length) : workBook.SheetNames.length; for (let s = 0; s < sheetsCount; s++) { const workSheetName = workBook.SheetNames[s]; const workSheet = workBook.Sheets[workSheetName]; result[workSheetName] = utils.sheet_to_json(workSheet, { header: 1 }); } resolve(result); } catch (err) { reject(err) } }, 1000) } } reader.readAsBinaryString(file); }); } return null; } export async function uploadCSVToS3(bucket: string, files: ExcelFiles, folder: string, callback?: (progress: number) => void, useStage: boolean = true) { if (files) { let remainingThreads = 0; { for (const fileName in files) { if (Object.prototype.hasOwnProperty.call(files, fileName)) { const file = files[fileName]; if (file) { remainingThreads += Object.keys(file).length; } } } } const filesLength = remainingThreads; if (filesLength > 0) { await new Promise((resolve) => { for (const fileName in files) { if (Object.prototype.hasOwnProperty.call(files, fileName)) { const file = files[fileName]; if (file) { const csvFileNameSplit = fileName.split('.'); { if (csvFileNameSplit.length > 1) { csvFileNameSplit.splice(csvFileNameSplit.length - 1, 1); } } const csvFileName = `${folder}/${csvFileNameSplit.join('.')}`; const workSheetCount = Object.keys(file).length; if (workSheetCount > 0) { for (const workSheetName in file) { if (Object.prototype.hasOwnProperty.call(file, workSheetName)) { const workSheetData = file[workSheetName]; if (workSheetData) { const lines: any[] = []; workSheetData.forEach(line => { lines.push(line.map((x: any) => typeof x === "string" && x.includes(',') ? `"${x}"` : x).join(',')) }) const csvFullFileName = workSheetCount > 1 ? `${csvFileName}.${workSheetName}.csv` : `${csvFileName}.csv`; putToS3(bucket, csvFullFileName, lines.join('\n'), { useStage }) .finally(() => { remainingThreads--; if (callback) { callback(Math.round(10000 * (filesLength - remainingThreads) / filesLength) / 100) } if (remainingThreads <= 0) resolve(undefined); }) } } } } } } } }); } } } export function getFirstSheet(files: ExcelFiles) { if (files) { for (const fileName in files) { if (Object.prototype.hasOwnProperty.call(files, fileName)) { const file = files[fileName]; if (file) { for (const sheetName in file) { if (Object.prototype.hasOwnProperty.call(file, sheetName)) { if (file[sheetName]) { return file[sheetName]; } } } } } } } return null; } export async function downloadCSVfromS3(bucket: string, sourceFile: string, progressCallback: (progress: ProgressEvent) => void, useStage: boolean = true, fileName?: string) { const result = await getFromS3(bucket, sourceFile, { progressCallback(progress: ProgressEvent) { if (progressCallback) { progressCallback(progress); } }, useStage }); const split = sourceFile.split('/') fileName = fileName ? fileName : split[split.length - 1]; if (result) { const url = URL.createObjectURL(result.Body); const link = document.createElement('a'); link.href = url; link.download = fileName; document.body.appendChild(link); link.click(); setTimeout(() => { document.body.removeChild(link); }) } } export async function downloadCSVfromMemory(sourceFile: string, obj: any[]) { if (obj) { const header: { [key: string]: true } = {} obj.forEach(x => { if (x) { Object.keys(x).forEach(key => { header[key] = true; }) } }) const file: string[][] = [Object.keys(header)] obj.forEach(x => { const line: string[] = [] file[0].forEach(key => { const value = x[key]; if (value !== undefined && value !== null) { if (typeof value === 'boolean') { line.push(value ? '1' : '0') } else if (typeof value === 'number') { line.push(`${value}`) } else if (isDate(value)) { line.push(dateFormat(value, 'yyyy-MM-dd')) } else if (typeof value === 'string') { line.push(`"${value}"`) } else line.push(''); } else line.push('') }) file.push(line) }) file[0].forEach((x, i) => file[0][i] = `"${x}"`) const csv = file.map(line => line.join(',')).join('\r\n') const url = window.URL.createObjectURL( new Blob([csv], { type: 'text/csv' }) ); const link = document.createElement('a'); link.href = url; link.download = sourceFile; document.body.appendChild(link); link.click(); setTimeout(() => { document.body.removeChild(link); }) } }