import fs from 'fs'; import path from 'path'; import microtime from 'microtime'; import { logger } from '../../utils/logger'; import { isJsonFile, removeEmptyDirsUpwards } from '../../utils/fs'; import { config } from '../../config'; import { store } from '../store'; import { diffManager } from '../diff/diffManager'; import { Diff } from '../diff/diff.types'; import { Dump, DumpManager } from './dump.types'; import { hookManager } from '../hook'; import { dumpMetadataHelper } from './dumpMetadataHelper'; import { dumpIndexHelper } from './dumpIndexHelper'; import { createHash } from '../../utils/string'; const storeUpdatedFiles = ( diff: Diff, dump: Dump, filesDumpDir: string, ): void => { diff.updated.forEach(relativeFilepath => { const storeFileContentData = store.data.get(relativeFilepath); if (storeFileContentData) { const absoluteFilepathInDumpDir = `${filesDumpDir}/${relativeFilepath}`; try { // If store data is an object, stringify it to execute its // queries and resolve its includes. const isJson = isJsonFile(relativeFilepath); const storeFileContentString = isJson ? JSON.stringify(storeFileContentData) : storeFileContentData; // Before overwriting a file, check if it has changed. let needsSave = true; let oldPublicUrl: string | null = null; const newPublicUrl: string | null = storeFileContentData.data?.content?.url?.path || null; const newHash = createHash(storeFileContentString); const indexEntry = dumpIndexHelper.getEntry(relativeFilepath); if (indexEntry) { if (newHash === indexEntry.hash) { // No need to save it, file has not changed. needsSave = false; } else if (isJson) { // Get old public URL before it changes on disk and index. oldPublicUrl = indexEntry.url; } } if (needsSave) { // Create parent directories if they are missing. const dir = path.dirname(absoluteFilepathInDumpDir); if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } // Save it on disk. fs.writeFileSync(absoluteFilepathInDumpDir, storeFileContentString); // Save it to index. dumpIndexHelper.addEntry(relativeFilepath, { hash: newHash, url: newPublicUrl, }); // Mark it as updated. dump.updated.set(relativeFilepath, { oldPublicUrl, newPublicUrl, }); } } catch (e) { logger.error( `Dump: error writing file "${absoluteFilepathInDumpDir}": ${e}`, ); } } else { logger.error( `Dump: store data for file "${relativeFilepath}" not found.`, ); } }); }; const removeDeletedFiles = ( diff: Diff, dump: Dump, filesDumpDir: string, ): void => { diff.deleted.forEach(relativeFilepath => { const absoluteFilepath = `${filesDumpDir}/${relativeFilepath}`; try { const indexEntry = dumpIndexHelper.getEntry(relativeFilepath); if (indexEntry) { // Get old public URLs before they change on disk and index. const oldPublicUrl = indexEntry.url; // Delete it. fs.unlinkSync(absoluteFilepath); // Delete any empty directory. removeEmptyDirsUpwards(path.dirname(absoluteFilepath)); // Delete it from index. dumpIndexHelper.removeEntry(relativeFilepath); // Mark it as deleted. dump.deleted.set(relativeFilepath, { oldPublicUrl, newPublicUrl: null, }); } } catch (e) { logger.error(`Dump: error deleting file "${absoluteFilepath}": ${e}`); } }); }; const removeStaleFiles = ( diff: Diff, dump: Dump, filesDumpDir: string, ): void => { const iterator = dumpIndexHelper.getKeys(); for (const relativeFilepath of iterator) { const absoluteFilepath = `${filesDumpDir}/${relativeFilepath}`; if (!store.data.has(relativeFilepath) && fs.existsSync(absoluteFilepath)) { logger.info(`Dump: removing stale file: ${relativeFilepath}`); try { // Get old public URLs before they change on disk. const dumpStaleFileString = fs .readFileSync(absoluteFilepath) .toString(); const isJson = isJsonFile(relativeFilepath); const dumpStaleFileContentData = isJson ? JSON.parse(dumpStaleFileString) : dumpStaleFileString; let oldPublicUrl: string | null = dumpStaleFileContentData.data?.content?.url?.path || null; // An stale file can be a manually copied file. // In that case, that stale file would have an oldPublicUrl from a valid file, // which would lead to deleting a valid URL. // To solve that case, check that oldPublicUrl is not a valid URL. if (oldPublicUrl && store.index.url.has(oldPublicUrl)) { logger.info( `Dump: skipping valid url from stale file: ${oldPublicUrl}`, ); oldPublicUrl = null; } // Delete it. fs.unlinkSync(absoluteFilepath); // Delete any empty directory. removeEmptyDirsUpwards(path.dirname(absoluteFilepath)); // Delete it from index. dumpIndexHelper.removeEntry(relativeFilepath); // Mark it as deleted. dump.deleted.set(relativeFilepath, { oldPublicUrl, newPublicUrl: null, }); } catch (e) { logger.error( `Dump: error deleting stale file "${absoluteFilepath}": ${e}`, ); } } } }; export const dumpManager: DumpManager = { dump(options = { incremental: true }): Dump { const startDate = microtime.now(); // Diff data is processed and transformed into a dump object. const diff = diffManager.getDiff({ incremental: options.incremental }); let dump: Dump = { execTimeMs: 0, fromUniqueId: diff.fromUniqueId, toUniqueId: diff.toUniqueId, updated: new Map(), deleted: new Map(), diff, }; if (config.dumpDir) { const filesDumpDir = `${config.dumpDir}/files`; if (diff.updated.size || diff.deleted.size) { // Store updated files. storeUpdatedFiles(diff, dump, filesDumpDir); // Remove deleted files. removeDeletedFiles(diff, dump, filesDumpDir); // Remove stale files. removeStaleFiles(diff, dump, filesDumpDir); // Save dump index dumpIndexHelper.saveDumpIndex(); // Invoke "onDumpCreate" hook. dump = hookManager.invokeOnDumpCreate(dump); const execTimeMs = (microtime.now() - startDate) / 1000; dump.execTimeMs = execTimeMs; // Merge and store dump metadata if any. if (dump.updated.size || dump.deleted.size) { const storeSuccessful = dumpMetadataHelper.storeDumpMetadata(dump); if (storeSuccessful) { // Resetting the diff must happen when dump metadata is successfully // stored into disk. diffManager.reset(diff.toUniqueId); } logger.info( `Dump created in ${execTimeMs} ms. Updated: ${dump.updated.size} / Deleted: ${dump.deleted.size}`, ); } else { // Resetting the diff must happen when no other operations are pending. diffManager.reset(diff.toUniqueId); logger.info( `Dump done in ${execTimeMs} ms without changes stored into disk.`, ); } } else { logger.info('Dump not stored into disk due to an empty diff.'); } } else { logger.error('"dumpDir" option not provided. Dump cannot be executed.'); } return dump; }, reset(uniqueId): void { if (config.dumpDir) { const resetMetadata = dumpMetadataHelper.removeDumpDataOlderThan(uniqueId); logger.debug(`Dump reset : "${JSON.stringify(resetMetadata)}"`); } else { logger.error( '"dumpDir" option not provided. Dump reset cannot be executed.', ); } }, };