import { FileSystemStorageAdapter, FileSystemStorageConfig } from '../../../../src/adapters/storage/FileSystemStorageAdapter'; import * as fs from 'fs-extra'; import * as path from 'path'; interface LargeTestItem { id?: string; name: string; content: string; size: number; createdAt?: number; updatedAt?: number; } // Test directory for large file storage const largeTestRootDir = path.join(process.cwd(), 'test-fs-large-storage'); describe('FileSystemStorageAdapter Large File Tests', () => { let adapter: FileSystemStorageAdapter; let testConfig: FileSystemStorageConfig; beforeEach(() => { adapter = new FileSystemStorageAdapter(); testConfig = { type: 'file', rootDir: largeTestRootDir, fileExtension: 'json', enableTransactions: false, prettyPrint: false, maxConcurrentOps: 100, }; }); // Clean up test directory after each test afterEach(async () => { await adapter.close(); if (await fs.pathExists(largeTestRootDir)) { await fs.remove(largeTestRootDir); } }); describe('Large File Handling', () => { beforeEach(async () => { await adapter.initialize(testConfig); }); test('should handle large content items (100MB+)', async () => { // Create a large string (approximately 10MB for faster testing) const largeContent = 'x'.repeat(10 * 1024 * 1024); // 10MB const testItem: Omit = { name: 'Large Content Item', content: largeContent, size: largeContent.length }; // Measure time for creation const createStart = process.hrtime.bigint(); const createdItem = await adapter.create('largeItems', testItem); const createEnd = process.hrtime.bigint(); const createTimeMs = Number((createEnd - createStart) / BigInt(1000000)); // Verify the item was created successfully expect(createdItem).toBeDefined(); expect(createdItem.id).toBeDefined(); expect(createdItem.name).toBe(testItem.name); expect(createdItem.size).toBe(testItem.size); // Measure time for retrieval const retrieveStart = process.hrtime.bigint(); const retrievedItem = await adapter.getById('largeItems', createdItem.id); const retrieveEnd = process.hrtime.bigint(); const retrieveTimeMs = Number((retrieveEnd - retrieveStart) / BigInt(1000000)); // Verify the retrieved item matches expect(retrievedItem).toBeDefined(); expect(retrievedItem?.id).toBe(createdItem.id); expect(retrievedItem?.content).toBeDefined(); expect(retrievedItem?.content.length).toBe(largeContent.length); // Log performance metrics console.log(`Large file (100MB) create time: ${createTimeMs}ms`); console.log(`Large file (100MB) retrieve time: ${retrieveTimeMs}ms`); }); test('should handle very large content items (200MB+)', async () => { // Create an even larger string (approximately 20MB for faster testing) const largeContent = 'x'.repeat(20 * 1024 * 1024); // 20MB const testItem: Omit = { name: 'Very Large Content Item', content: largeContent, size: largeContent.length }; // Create the item const createdItem = await adapter.create('largeItems', testItem); // Verify the item was created successfully expect(createdItem).toBeDefined(); expect(createdItem.id).toBeDefined(); expect(createdItem.size).toBe(testItem.size); // Verify the file exists on disk const itemFilePath = path.join(largeTestRootDir, 'largeItems', `${createdItem.id}.json`); expect(await fs.pathExists(itemFilePath)).toBe(true); // Verify the file size on disk is as expected (with some overhead for JSON formatting) const stats = await fs.stat(itemFilePath); expect(stats.size).toBeGreaterThanOrEqual(largeContent.length); }); test('should handle multiple large files efficiently', async () => { // Create multiple moderately large files (10MB each) const fileCount = 3; const contentSize = 10 * 1024 * 1024; // 10MB per file const largeContent = 'x'.repeat(contentSize); const testItems: Omit[] = []; for (let i = 0; i < fileCount; i++) { testItems.push({ name: `Large Item ${i + 1}`, content: largeContent, size: contentSize }); } // Measure time for bulk creation const bulkStart = process.hrtime.bigint(); const createdItems = await adapter.bulkCreate('largeItems', testItems); const bulkEnd = process.hrtime.bigint(); const bulkTimeMs = Number((bulkEnd - bulkStart) / BigInt(1000000)); // Verify all items were created expect(createdItems.length).toBe(fileCount); // Log performance metrics console.log(`Bulk create ${fileCount} files (${contentSize / (1024 * 1024)}MB each) time: ${bulkTimeMs}ms`); console.log(`Average per-file create time: ${bulkTimeMs / fileCount}ms`); }); test('should update large files efficiently', async () => { // Create a large file first const initialContent = 'x'.repeat(10 * 1024 * 1024); // 10MB const testItem: Omit = { name: 'Updatable Large Item', content: initialContent, size: initialContent.length }; const createdItem = await adapter.create('largeItems', testItem); expect(createdItem).toBeDefined(); // Update the large file with new content const newContent = 'y'.repeat(12 * 1024 * 1024); // 12MB const updateData = { content: newContent, size: newContent.length }; // Measure time for update const updateStart = process.hrtime.bigint(); const updatedItem = await adapter.update('largeItems', createdItem.id, updateData); const updateEnd = process.hrtime.bigint(); const updateTimeMs = Number((updateEnd - updateStart) / BigInt(1000000)); // Verify the update was successful expect(updatedItem).toBeDefined(); expect(updatedItem?.id).toBe(createdItem.id); expect(updatedItem?.size).toBe(newContent.length); // Verify the updated content can be retrieved const retrievedItem = await adapter.getById('largeItems', createdItem.id); expect(retrievedItem?.content.length).toBe(newContent.length); // Log performance metrics console.log(`Update large file (120MB) time: ${updateTimeMs}ms`); }); test('should delete large files efficiently', async () => { // Create a large file first const largeContent = 'x'.repeat(10 * 1024 * 1024); // 10MB const testItem: Omit = { name: 'Deletable Large Item', content: largeContent, size: largeContent.length }; const createdItem = await adapter.create('largeItems', testItem); expect(createdItem).toBeDefined(); // Verify the file exists on disk const itemFilePath = path.join(largeTestRootDir, 'largeItems', `${createdItem.id}.json`); expect(await fs.pathExists(itemFilePath)).toBe(true); // Measure time for deletion const deleteStart = process.hrtime.bigint(); const deleted = await adapter.delete('largeItems', createdItem.id); const deleteEnd = process.hrtime.bigint(); const deleteTimeMs = Number((deleteEnd - deleteStart) / BigInt(1000000)); // Verify the deletion was successful expect(deleted).toBe(true); // Verify the file no longer exists on disk expect(await fs.pathExists(itemFilePath)).toBe(false); // Log performance metrics console.log(`Delete large file (100MB) time: ${deleteTimeMs}ms`); }); }); describe('Performance Metrics', () => { beforeEach(async () => { await adapter.initialize(testConfig); }); test('should maintain reasonable memory usage for large files', async () => { // Create a large file (10MB) const largeContent = 'x'.repeat(10 * 1024 * 1024); const testItem: Omit = { name: 'Memory Test Item', content: largeContent, size: largeContent.length }; // Get initial memory usage const initialMemory = process.memoryUsage().heapUsed; // Create the large item const createdItem = await adapter.create('largeItems', testItem); expect(createdItem).toBeDefined(); // Get memory usage after creation const afterCreateMemory = process.memoryUsage().heapUsed; // Retrieve the item const retrievedItem = await adapter.getById('largeItems', createdItem.id); expect(retrievedItem).toBeDefined(); // Get memory usage after retrieval const afterRetrieveMemory = process.memoryUsage().heapUsed; // Log memory usage console.log(`Initial memory usage: ${Math.round(initialMemory / (1024 * 1024))} MB`); console.log(`Memory usage after create: ${Math.round(afterCreateMemory / (1024 * 1024))} MB`); console.log(`Memory usage after retrieve: ${Math.round(afterRetrieveMemory / (1024 * 1024))} MB`); console.log(`Memory increase from create: ${Math.round((afterCreateMemory - initialMemory) / (1024 * 1024))} MB`); console.log(`Memory increase from retrieve: ${Math.round((afterRetrieveMemory - afterCreateMemory) / (1024 * 1024))} MB`); // Verify memory usage is reasonable (shouldn't be more than 5x the file size for Node.js overhead) // This is a rough check to ensure we're not leaking memory excessively const memoryIncrease = afterRetrieveMemory - initialMemory; expect(memoryIncrease).toBeLessThan(largeContent.length * 5); }); }); });