feat: implement comprehensive backup and restore system with S3 support

- Add S3/MinIO storage adapter with multipart upload support - Implement database backup service for SQLite and PostgreSQL - Create backup manifest generator for tracking backup contents - Enhance backup service with S3 integration and incremental backups - Add restore service with safety measures and rollback capability - Create comprehensive test suite for all backup functionality - Add admin API endpoints for backup/restore management - Implement frontend UI with dashboard, configuration, and restore wizard - Add roadmap section to README with implemented backup feature This implementation provides: - Multiple backup destinations (local, rsync, S3/MinIO) - Intelligent change detection to minimize backup frequency - Full database backups with compression - Manifest-based restore with integrity validation - Pre-restore safety backups with rollback - Comprehensive error handling and monitoring - User-friendly admin interface 🤖 Generated with Claude Code Co-Authored-By: Claude <[email protected]>
2025-07-22 09:05:52 +02:00
parent 977839156b
commit 20dd43c093
44 changed files with 15839 additions and 14 deletions
@@ -0,0 +1,311 @@
+const S3StorageAdapter = require('../s3Storage');
+const { S3Client } = require('@aws-sdk/client-s3');
+const { Upload } = require('@aws-sdk/lib-storage');
+const fs = require('fs');
+const stream = require('stream');
+
+// Mock AWS SDK
+jest.mock('@aws-sdk/client-s3');
+jest.mock('@aws-sdk/lib-storage');
+jest.mock('@aws-sdk/s3-request-presigner');
+
+describe('S3StorageAdapter', () => {
+  let mockS3Client;
+  let mockSend;
+  let s3Storage;
+  
+  beforeEach(() => {
+    // Reset mocks
+    jest.clearAllMocks();
+    
+    // Mock S3Client
+    mockSend = jest.fn();
+    mockS3Client = {
+      send: mockSend
+    };
+    S3Client.mockImplementation(() => mockS3Client);
+    
+    // Create adapter instance
+    s3Storage = new S3StorageAdapter({
+      bucket: 'test-bucket',
+      region: 'us-east-1',
+      accessKeyId: 'test-key',
+      secretAccessKey: 'test-secret'
+    });
+  });
+  
+  describe('constructor', () => {
+    it('should initialize with required config', () => {
+      expect(s3Storage.bucket).toBe('test-bucket');
+      expect(s3Storage.config.region).toBe('us-east-1');
+    });
+    
+    it('should throw error if bucket is not provided', () => {
+      expect(() => {
+        new S3StorageAdapter({ region: 'us-east-1' });
+      }).toThrow('S3 bucket name is required');
+    });
+    
+    it('should configure for MinIO with path style', () => {
+      const minioStorage = new S3StorageAdapter({
+        bucket: 'test-bucket',
+        endpoint: 'http://localhost:9000',
+        forcePathStyle: true,
+        sslEnabled: false
+      });
+      
+      expect(S3Client).toHaveBeenCalledWith(
+        expect.objectContaining({
+          endpoint: 'http://localhost:9000',
+          forcePathStyle: true
+        })
+      );
+    });
+  });
+  
+  describe('testConnection', () => {
+    it('should successfully test connection', async () => {
+      mockSend.mockResolvedValueOnce({});
+      
+      const result = await s3Storage.testConnection();
+      
+      expect(result).toBe(true);
+      expect(mockSend).toHaveBeenCalledWith(
+        expect.objectContaining({
+          input: { Bucket: 'test-bucket' }
+        })
+      );
+    });
+    
+    it('should throw error on connection failure', async () => {
+      mockSend.mockRejectedValueOnce(new Error('Access Denied'));
+      
+      await expect(s3Storage.testConnection()).rejects.toThrow('S3 connection test failed');
+    });
+  });
+  
+  describe('upload', () => {
+    let mockUpload;
+    let mockDone;
+    
+    beforeEach(() => {
+      mockDone = jest.fn().mockResolvedValue({
+        Location: 'https://test-bucket.s3.amazonaws.com/test-key',
+        ETag: '"test-etag"'
+      });
+      
+      mockUpload = {
+        on: jest.fn().mockReturnThis(),
+        done: mockDone
+      };
+      
+      Upload.mockImplementation(() => mockUpload);
+      
+      // Mock fs.stat
+      jest.spyOn(fs.promises, 'stat').mockResolvedValue({
+        size: 1024
+      });
+      
+      // Mock fs.createReadStream
+      jest.spyOn(fs, 'createReadStream').mockReturnValue(new stream.Readable());
+    });
+    
+    afterEach(() => {
+      jest.restoreAllMocks();
+    });
+    
+    it('should upload file successfully', async () => {
+      const result = await s3Storage.upload('/path/to/file.jpg', 'test-key');
+      
+      expect(result.Location).toBe('https://test-bucket.s3.amazonaws.com/test-key');
+      expect(Upload).toHaveBeenCalledWith(
+        expect.objectContaining({
+          client: mockS3Client,
+          params: expect.objectContaining({
+            Bucket: 'test-bucket',
+            Key: 'test-key',
+            ContentType: 'application/octet-stream'
+          })
+        })
+      );
+    });
+    
+    it('should track upload progress', async () => {
+      const onProgress = jest.fn();
+      let progressCallback;
+      
+      mockUpload.on.mockImplementation((event, callback) => {
+        if (event === 'httpUploadProgress') {
+          progressCallback = callback;
+        }
+        return mockUpload;
+      });
+      
+      const uploadPromise = s3Storage.upload('/path/to/file.jpg', 'test-key', {
+        onProgress
+      });
+      
+      // Simulate progress
+      progressCallback({ loaded: 512, total: 1024 });
+      
+      await uploadPromise;
+      
+      expect(onProgress).toHaveBeenCalledWith(512, 1024);
+    });
+    
+    it('should emit upload events', async () => {
+      const uploadStartSpy = jest.fn();
+      const uploadCompleteSpy = jest.fn();
+      
+      s3Storage.on('uploadStart', uploadStartSpy);
+      s3Storage.on('uploadComplete', uploadCompleteSpy);
+      
+      await s3Storage.upload('/path/to/file.jpg', 'test-key');
+      
+      expect(uploadStartSpy).toHaveBeenCalledWith({ key: 'test-key', size: 1024 });
+      expect(uploadCompleteSpy).toHaveBeenCalledWith({
+        key: 'test-key',
+        location: 'https://test-bucket.s3.amazonaws.com/test-key'
+      });
+    });
+  });
+  
+  describe('exists', () => {
+    it('should return true if object exists', async () => {
+      mockSend.mockResolvedValueOnce({});
+      
+      const result = await s3Storage.exists('test-key');
+      
+      expect(result).toBe(true);
+      expect(mockSend).toHaveBeenCalledWith(
+        expect.objectContaining({
+          input: { Bucket: 'test-bucket', Key: 'test-key' }
+        })
+      );
+    });
+    
+    it('should return false if object does not exist', async () => {
+      const error = new Error('Not Found');
+      error.name = 'NotFound';
+      error.$metadata = { httpStatusCode: 404 };
+      mockSend.mockRejectedValueOnce(error);
+      
+      const result = await s3Storage.exists('test-key');
+      
+      expect(result).toBe(false);
+    });
+  });
+  
+  describe('generateKey', () => {
+    it('should generate unique key with timestamp and random string', () => {
+      const key = s3Storage.generateKey('photo.jpg');
+      
+      expect(key).toMatch(/^\d+_[a-f0-9]{16}_photo\.jpg$/);
+    });
+    
+    it('should add prefix if provided', () => {
+      const key = s3Storage.generateKey('photo.jpg', 'events/wedding');
+      
+      expect(key).toMatch(/^events\/wedding\/\d+_[a-f0-9]{16}_photo\.jpg$/);
+    });
+    
+    it('should sanitize filename', () => {
+      const key = s3Storage.generateKey('my photo (1).jpg');
+      
+      expect(key).toMatch(/^\d+_[a-f0-9]{16}_my_photo__1_\.jpg$/);
+    });
+  });
+  
+  describe('retry logic', () => {
+    it('should retry on retryable errors', async () => {
+      const retryableError = new Error('Connection reset');
+      retryableError.code = 'ECONNRESET';
+      
+      // First attempt fails, second succeeds
+      mockSend
+        .mockRejectedValueOnce(retryableError)
+        .mockResolvedValueOnce({});
+      
+      // Mock setTimeout to speed up test
+      jest.useFakeTimers();
+      
+      const promise = s3Storage.exists('test-key');
+      
+      // Advance timers
+      jest.runAllTimers();
+      
+      const result = await promise;
+      
+      expect(result).toBe(true);
+      expect(mockSend).toHaveBeenCalledTimes(2);
+      
+      jest.useRealTimers();
+    });
+    
+    it('should not retry on non-retryable errors', async () => {
+      const nonRetryableError = new Error('Invalid credentials');
+      nonRetryableError.code = 'InvalidCredentials';
+      
+      mockSend.mockRejectedValueOnce(nonRetryableError);
+      
+      await expect(s3Storage.exists('test-key')).rejects.toThrow('Invalid credentials');
+      expect(mockSend).toHaveBeenCalledTimes(1);
+    });
+    
+    it('should stop retrying after max attempts', async () => {
+      const retryableError = new Error('Service unavailable');
+      retryableError.code = 'ServiceUnavailable';
+      
+      mockSend.mockRejectedValue(retryableError);
+      
+      // Mock setTimeout to speed up test
+      jest.useFakeTimers();
+      
+      const promise = s3Storage.exists('test-key');
+      
+      // Advance timers for all retries
+      for (let i = 0; i < 4; i++) {
+        jest.runAllTimers();
+      }
+      
+      await expect(promise).rejects.toThrow('Service unavailable');
+      expect(mockSend).toHaveBeenCalledTimes(4); // Initial + 3 retries
+      
+      jest.useRealTimers();
+    });
+  });
+  
+  describe('getStats', () => {
+    it('should calculate storage statistics', async () => {
+      mockSend.mockResolvedValueOnce({
+        Contents: [
+          { Key: 'file1.jpg', Size: 1024 },
+          { Key: 'file2.jpg', Size: 2048 }
+        ],
+        NextContinuationToken: 'token123'
+      }).mockResolvedValueOnce({
+        Contents: [
+          { Key: 'file3.jpg', Size: 3072 }
+        ]
+      });
+      
+      const stats = await s3Storage.getStats('events/');
+      
+      expect(stats).toEqual({
+        totalSize: 6144,
+        totalCount: 3,
+        totalSizeFormatted: '6 KB'
+      });
+    });
+  });
+  
+  describe('_formatBytes', () => {
+    it('should format bytes correctly', () => {
+      expect(s3Storage._formatBytes(0)).toBe('0 Bytes');
+      expect(s3Storage._formatBytes(1024)).toBe('1 KB');
+      expect(s3Storage._formatBytes(1048576)).toBe('1 MB');
+      expect(s3Storage._formatBytes(1073741824)).toBe('1 GB');
+      expect(s3Storage._formatBytes(1536, 1)).toBe('1.5 KB');
+    });
+  });
+});
@@ -0,0 +1,221 @@
+/**
+ * Example usage of the S3StorageAdapter
+ * 
+ * This file demonstrates how to use the S3 storage adapter for various operations
+ */
+
+const S3StorageAdapter = require('./s3Storage');
+
+// Example 1: Basic AWS S3 Configuration
+const s3Storage = new S3StorageAdapter({
+  bucket: 'my-photo-bucket',
+  region: 'us-east-1',
+  accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+  secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
+});
+
+// Example 2: MinIO Configuration (S3-compatible)
+const minioStorage = new S3StorageAdapter({
+  bucket: 'photo-storage',
+  endpoint: 'http://localhost:9000', // MinIO endpoint
+  accessKeyId: 'minioadmin',
+  secretAccessKey: 'minioadmin',
+  forcePathStyle: true, // Required for MinIO
+  sslEnabled: false // For local development
+});
+
+// Example 3: DigitalOcean Spaces Configuration
+const spacesStorage = new S3StorageAdapter({
+  bucket: 'my-space-name',
+  endpoint: 'https://nyc3.digitaloceanspaces.com',
+  region: 'nyc3',
+  accessKeyId: process.env.DO_SPACES_KEY,
+  secretAccessKey: process.env.DO_SPACES_SECRET
+});
+
+// Usage Examples
+async function examples() {
+  try {
+    // Test connection
+    await s3Storage.testConnection();
+    console.log('Connection successful!');
+    
+    // Upload a file with progress tracking
+    const uploadResult = await s3Storage.upload(
+      '/path/to/local/photo.jpg',
+      'events/wedding-2024/photo.jpg',
+      {
+        contentType: 'image/jpeg',
+        metadata: {
+          event: 'wedding-2024',
+          photographer: 'John Doe'
+        },
+        onProgress: (loaded, total) => {
+          const percentage = Math.round((loaded / total) * 100);
+          console.log(`Upload progress: ${percentage}%`);
+        }
+      }
+    );
+    console.log('Upload complete:', uploadResult.Location);
+    
+    // Upload from stream
+    const readStream = fs.createReadStream('/path/to/large-video.mp4');
+    await s3Storage.uploadStream(readStream, 'events/wedding-2024/video.mp4', {
+      contentType: 'video/mp4',
+      onProgress: (loaded, total) => {
+        console.log(`Streamed ${loaded} of ${total} bytes`);
+      }
+    });
+    
+    // Download a file
+    await s3Storage.download(
+      'events/wedding-2024/photo.jpg',
+      '/path/to/downloaded/photo.jpg',
+      {
+        onProgress: (loaded, total) => {
+          const percentage = Math.round((loaded / total) * 100);
+          console.log(`Download progress: ${percentage}%`);
+        }
+      }
+    );
+    
+    // Get a download stream
+    const downloadStream = await s3Storage.downloadStream('events/wedding-2024/photo.jpg');
+    downloadStream.pipe(fs.createWriteStream('/path/to/output.jpg'));
+    
+    // List files
+    const listing = await s3Storage.list('events/wedding-2024/');
+    console.log(`Found ${listing.Contents.length} files`);
+    listing.Contents.forEach(file => {
+      console.log(`- ${file.Key} (${file.Size} bytes)`);
+    });
+    
+    // Generate pre-signed URL for temporary access
+    const downloadUrl = await s3Storage.getSignedUrl('getObject', 'events/wedding-2024/photo.jpg', {
+      expiresIn: 3600 // 1 hour
+    });
+    console.log('Pre-signed download URL:', downloadUrl);
+    
+    // Generate pre-signed upload URL
+    const uploadUrl = await s3Storage.getSignedUrl('putObject', 'events/wedding-2024/new-photo.jpg', {
+      expiresIn: 1800, // 30 minutes
+      params: {
+        ContentType: 'image/jpeg'
+      }
+    });
+    console.log('Pre-signed upload URL:', uploadUrl);
+    
+    // Check if file exists
+    const exists = await s3Storage.exists('events/wedding-2024/photo.jpg');
+    console.log('File exists:', exists);
+    
+    // Get metadata
+    const metadata = await s3Storage.getMetadata('events/wedding-2024/photo.jpg');
+    console.log('File metadata:', metadata);
+    
+    // Copy file
+    await s3Storage.copy(
+      'events/wedding-2024/photo.jpg',
+      'events/wedding-2024/photo-copy.jpg'
+    );
+    
+    // Move file
+    await s3Storage.move(
+      'events/wedding-2024/photo-copy.jpg',
+      'events/wedding-2024/archived/photo.jpg'
+    );
+    
+    // Delete file
+    await s3Storage.delete('events/wedding-2024/temp-photo.jpg');
+    
+    // Delete multiple files
+    const deleteResult = await s3Storage.deleteMany([
+      'events/wedding-2024/temp1.jpg',
+      'events/wedding-2024/temp2.jpg',
+      'events/wedding-2024/temp3.jpg'
+    ]);
+    console.log(`Deleted ${deleteResult.Deleted.length} files`);
+    
+    // Get storage statistics
+    const stats = await s3Storage.getStats('events/');
+    console.log(`Total files: ${stats.totalCount}`);
+    console.log(`Total size: ${stats.totalSizeFormatted}`);
+    
+    // Listen to events
+    s3Storage.on('uploadProgress', (data) => {
+      console.log(`Uploading ${data.key}: ${data.loaded}/${data.total}`);
+    });
+    
+    s3Storage.on('uploadComplete', (data) => {
+      console.log(`Upload completed: ${data.key}`);
+    });
+    
+    s3Storage.on('uploadError', (data) => {
+      console.error(`Upload failed for ${data.key}:`, data.error);
+    });
+    
+  } catch (error) {
+    console.error('Error:', error);
+  }
+}
+
+// Integration with existing photo upload workflow
+async function integrateWithPhotoUpload(eventId, files) {
+  const storage = new S3StorageAdapter({
+    bucket: process.env.S3_BUCKET,
+    region: process.env.AWS_REGION,
+    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
+  });
+  
+  const uploadedPhotos = [];
+  
+  for (const file of files) {
+    try {
+      // Generate unique S3 key
+      const s3Key = storage.generateKey(file.originalname, `events/${eventId}`);
+      
+      // Upload to S3
+      const result = await storage.upload(file.path, s3Key, {
+        contentType: file.mimetype,
+        metadata: {
+          eventId: eventId,
+          originalName: file.originalname,
+          uploadedAt: new Date().toISOString()
+        }
+      });
+      
+      uploadedPhotos.push({
+        filename: s3Key,
+        originalName: file.originalname,
+        size: file.size,
+        mimeType: file.mimetype,
+        s3Location: result.Location,
+        s3Key: s3Key
+      });
+      
+      // Clean up local temp file
+      await fs.promises.unlink(file.path);
+      
+    } catch (error) {
+      console.error(`Failed to upload ${file.originalname}:`, error);
+      throw error;
+    }
+  }
+  
+  return uploadedPhotos;
+}
+
+// Environment variables needed:
+// AWS_ACCESS_KEY_ID=your-access-key
+// AWS_SECRET_ACCESS_KEY=your-secret-key
+// AWS_REGION=us-east-1
+// S3_BUCKET=your-bucket-name
+
+// For MinIO:
+// MINIO_ENDPOINT=http://localhost:9000
+// MINIO_ACCESS_KEY=minioadmin
+// MINIO_SECRET_KEY=minioadmin
+// MINIO_BUCKET=photo-storage
+
+module.exports = { examples, integrateWithPhotoUpload };
@@ -0,0 +1,733 @@
+const { S3Client, HeadBucketCommand, HeadObjectCommand, GetObjectCommand, PutObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, ListObjectsV2Command, CopyObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand } = require('@aws-sdk/client-s3');
+const { Upload } = require('@aws-sdk/lib-storage');
+const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
+const fs = require('fs');
+const fsPromises = require('fs').promises;
+const path = require('path');
+const stream = require('stream');
+const crypto = require('crypto');
+const logger = require('../../utils/logger');
+
+/**
+ * S3 Storage Adapter for handling file uploads to S3 and S3-compatible services
+ * 
+ * Features:
+ * - Support for S3 and S3-compatible services (MinIO, DigitalOcean Spaces, etc.)
+ * - Multipart upload for large files (>100MB)
+ * - Progress tracking with event emitter
+ * - Retry logic with exponential backoff
+ * - Stream support for memory efficiency
+ * - Path-style URL support for MinIO
+ * - Connection testing
+ * - Comprehensive error handling
+ * 
+ * @class S3StorageAdapter
+ */
+class S3StorageAdapter extends stream.EventEmitter {
+  /**
+   * Creates an instance of S3StorageAdapter
+   * 
+   * @param {Object} config - Configuration object
+   * @param {string} config.bucket - S3 bucket name
+   * @param {string} [config.region='us-east-1'] - AWS region
+   * @param {string} [config.endpoint] - Custom endpoint URL for S3-compatible services
+   * @param {string} [config.accessKeyId] - AWS access key ID
+   * @param {string} [config.secretAccessKey] - AWS secret access key
+   * @param {boolean} [config.forcePathStyle=false] - Force path-style URLs (required for MinIO)
+   * @param {boolean} [config.sslEnabled=true] - Enable SSL for connections
+   * @param {number} [config.multipartThreshold=104857600] - Threshold for multipart upload (default 100MB)
+   * @param {number} [config.partSize=10485760] - Part size for multipart upload (default 10MB)
+   * @param {number} [config.maxRetries=3] - Maximum number of retry attempts
+   * @param {number} [config.retryDelay=1000] - Initial retry delay in milliseconds
+   */
+  constructor(config) {
+    super();
+    
+    // Validate required config
+    if (!config.bucket) {
+      throw new Error('S3 bucket name is required');
+    }
+    
+    // Set defaults
+    this.config = {
+      region: 'us-east-1',
+      forcePathStyle: false,
+      sslEnabled: true,
+      multipartThreshold: 100 * 1024 * 1024, // 100MB
+      partSize: 10 * 1024 * 1024, // 10MB
+      maxRetries: 3,
+      retryDelay: 1000,
+      ...config
+    };
+    
+    // Initialize S3 client
+    const s3Config = {
+      region: this.config.region,
+      forcePathStyle: this.config.forcePathStyle
+    };
+    
+    // Add credentials if provided
+    if (this.config.accessKeyId && this.config.secretAccessKey) {
+      s3Config.credentials = {
+        accessKeyId: this.config.accessKeyId,
+        secretAccessKey: this.config.secretAccessKey
+      };
+    }
+    
+    // Add custom endpoint if provided (for S3-compatible services)
+    if (this.config.endpoint) {
+      s3Config.endpoint = this.config.endpoint;
+      // For MinIO and other S3-compatible services
+      if (!this.config.endpoint.startsWith('https://') && this.config.sslEnabled) {
+        s3Config.endpoint = `https://${this.config.endpoint}`;
+      } else if (!this.config.endpoint.startsWith('http://') && !this.config.sslEnabled) {
+        s3Config.endpoint = `http://${this.config.endpoint}`;
+      }
+    }
+    
+    this.s3Client = new S3Client(s3Config);
+    this.bucket = this.config.bucket;
+    
+    // Bind methods to preserve context
+    this.upload = this.upload.bind(this);
+    this.uploadStream = this.uploadStream.bind(this);
+    this.download = this.download.bind(this);
+    this.downloadStream = this.downloadStream.bind(this);
+    this.delete = this.delete.bind(this);
+    this.exists = this.exists.bind(this);
+    this.list = this.list.bind(this);
+    this.copy = this.copy.bind(this);
+    this.move = this.move.bind(this);
+    this.getSignedUrl = this.getSignedUrl.bind(this);
+    this.testConnection = this.testConnection.bind(this);
+  }
+  
+  /**
+   * Test connection to S3 bucket
+   * 
+   * @returns {Promise<boolean>} - True if connection successful
+   * @throws {Error} - If connection fails
+   */
+  async testConnection() {
+    try {
+      await this.s3Client.send(new HeadBucketCommand({ Bucket: this.bucket }));
+      logger.info(`Successfully connected to S3 bucket: ${this.bucket}`);
+      return true;
+    } catch (error) {
+      logger.error(`Failed to connect to S3 bucket ${this.bucket}:`, error);
+      throw new Error(`S3 connection test failed: ${error.message}`);
+    }
+  }
+  
+  /**
+   * Upload a file to S3 with automatic multipart for large files
+   * 
+   * @param {string} localPath - Local file path to upload
+   * @param {string} s3Key - S3 object key (path in bucket)
+   * @param {Object} [options={}] - Additional options
+   * @param {Object} [options.metadata] - Object metadata
+   * @param {string} [options.contentType] - Content type
+   * @param {string} [options.cacheControl] - Cache control header
+   * @param {Function} [options.onProgress] - Progress callback function(loaded, total)
+   * @returns {Promise<Object>} - Upload result with Location, ETag, etc.
+   */
+  async upload(localPath, s3Key, options = {}) {
+    try {
+      const stats = await fsPromises.stat(localPath);
+      const fileSize = stats.size;
+      
+      // Emit upload start event
+      this.emit('uploadStart', { key: s3Key, size: fileSize });
+      
+      // Create file stream
+      const fileStream = fs.createReadStream(localPath);
+      
+      // Prepare upload parameters
+      const uploadParams = {
+        Bucket: this.bucket,
+        Key: s3Key,
+        Body: fileStream,
+        ContentType: options.contentType || 'application/octet-stream',
+        Metadata: options.metadata || {},
+        CacheControl: options.cacheControl
+      };
+      
+      // Remove undefined values
+      Object.keys(uploadParams).forEach(key => uploadParams[key] === undefined && delete uploadParams[key]);
+      
+      // Use AWS SDK v3 Upload for automatic multipart handling
+      const parallelUploads3 = new Upload({
+        client: this.s3Client,
+        params: uploadParams,
+        queueSize: 4, // Optional: concurrent uploads
+        partSize: this.config.partSize,
+        leavePartsOnError: false
+      });
+      
+      // Track upload progress
+      parallelUploads3.on('httpUploadProgress', (progress) => {
+        if (options.onProgress) {
+          options.onProgress(progress.loaded, progress.total);
+        }
+        this.emit('uploadProgress', { key: s3Key, loaded: progress.loaded, total: progress.total });
+      });
+      
+      // Perform upload with retry
+      const result = await this._retryOperation(() => parallelUploads3.done());
+      
+      this.emit('uploadComplete', { key: s3Key, location: result.Location });
+      return result;
+    } catch (error) {
+      logger.error(`Failed to upload file ${localPath} to S3:`, error);
+      this.emit('uploadError', { key: s3Key, error });
+      throw error;
+    }
+  }
+  
+  /**
+   * Upload a stream to S3
+   * 
+   * @param {stream.Readable} readStream - Readable stream to upload
+   * @param {string} s3Key - S3 object key
+   * @param {Object} [options={}] - Additional options
+   * @returns {Promise<Object>} - Upload result
+   */
+  async uploadStream(readStream, s3Key, options = {}) {
+    const uploadParams = {
+      Bucket: this.bucket,
+      Key: s3Key,
+      Body: readStream,
+      ContentType: options.contentType || 'application/octet-stream',
+      Metadata: options.metadata || {},
+      CacheControl: options.cacheControl
+    };
+    
+    // Remove undefined values
+    Object.keys(uploadParams).forEach(key => uploadParams[key] === undefined && delete uploadParams[key]);
+    
+    const parallelUploads3 = new Upload({
+      client: this.s3Client,
+      params: uploadParams,
+      queueSize: 4,
+      partSize: this.config.partSize,
+      leavePartsOnError: false
+    });
+    
+    // Track progress if callback provided
+    if (options.onProgress) {
+      parallelUploads3.on('httpUploadProgress', (progress) => {
+        options.onProgress(progress.loaded, progress.total);
+        this.emit('uploadProgress', { key: s3Key, ...progress });
+      });
+    }
+    
+    return await this._retryOperation(() => parallelUploads3.done());
+  }
+  
+  /**
+   * Download a file from S3
+   * 
+   * @param {string} s3Key - S3 object key
+   * @param {string} localPath - Local file path to save to
+   * @param {Object} [options={}] - Additional options
+   * @param {Function} [options.onProgress] - Progress callback
+   * @returns {Promise<void>}
+   */
+  async download(s3Key, localPath, options = {}) {
+    try {
+      // Ensure directory exists
+      await fsPromises.mkdir(path.dirname(localPath), { recursive: true });
+      
+      // Get object metadata first for progress tracking
+      const headResult = await this._retryOperation(() => 
+        this.s3Client.send(new HeadObjectCommand({ 
+          Bucket: this.bucket, 
+          Key: s3Key 
+        }))
+      );
+      
+      const fileSize = headResult.ContentLength;
+      this.emit('downloadStart', { key: s3Key, size: fileSize });
+      
+      // Get object
+      const getObjectResult = await this._retryOperation(() =>
+        this.s3Client.send(new GetObjectCommand({
+          Bucket: this.bucket,
+          Key: s3Key
+        }))
+      );
+      
+      // Create write stream
+      const writeStream = fs.createWriteStream(localPath);
+      
+      // Download with progress tracking
+      await new Promise((resolve, reject) => {
+        let downloaded = 0;
+        
+        const bodyStream = getObjectResult.Body;
+        
+        bodyStream.on('data', (chunk) => {
+          downloaded += chunk.length;
+          if (options.onProgress) {
+            options.onProgress(downloaded, fileSize);
+          }
+          this.emit('downloadProgress', { key: s3Key, loaded: downloaded, total: fileSize });
+        });
+        
+        bodyStream.on('error', reject);
+        writeStream.on('error', reject);
+        writeStream.on('finish', resolve);
+        
+        bodyStream.pipe(writeStream);
+      });
+      
+      this.emit('downloadComplete', { key: s3Key });
+    } catch (error) {
+      logger.error(`Failed to download file ${s3Key} from S3:`, error);
+      this.emit('downloadError', { key: s3Key, error });
+      throw error;
+    }
+  }
+  
+  /**
+   * Get a download stream from S3
+   * 
+   * @param {string} s3Key - S3 object key
+   * @param {Object} [options={}] - Additional options
+   * @param {string} [options.range] - Byte range to download (e.g., 'bytes=0-1023')
+   * @returns {Promise<stream.Readable>} - Readable stream
+   */
+  async downloadStream(s3Key, options = {}) {
+    const params = {
+      Bucket: this.bucket,
+      Key: s3Key,
+      Range: options.range
+    };
+    
+    // Remove undefined values
+    Object.keys(params).forEach(key => params[key] === undefined && delete params[key]);
+    
+    const result = await this.s3Client.send(new GetObjectCommand(params));
+    return result.Body;
+  }
+  
+  /**
+   * Delete a file from S3
+   * 
+   * @param {string} s3Key - S3 object key
+   * @returns {Promise<void>}
+   */
+  async delete(s3Key) {
+    return await this._retryOperation(() =>
+      this.s3Client.send(new DeleteObjectCommand({
+        Bucket: this.bucket,
+        Key: s3Key
+      }))
+    );
+  }
+  
+  /**
+   * Delete multiple files from S3
+   * 
+   * @param {string[]} s3Keys - Array of S3 object keys
+   * @returns {Promise<Object>} - Delete result
+   */
+  async deleteMany(s3Keys) {
+    if (!s3Keys || s3Keys.length === 0) {
+      return { Deleted: [], Errors: [] };
+    }
+    
+    // S3 deleteObjects has a limit of 1000 keys per request
+    const chunks = [];
+    for (let i = 0; i < s3Keys.length; i += 1000) {
+      chunks.push(s3Keys.slice(i, i + 1000));
+    }
+    
+    const results = await Promise.all(
+      chunks.map(chunk => 
+        this._retryOperation(() =>
+          this.s3Client.send(new DeleteObjectsCommand({
+            Bucket: this.bucket,
+            Delete: {
+              Objects: chunk.map(key => ({ Key: key })),
+              Quiet: false
+            }
+          }))
+        )
+      )
+    );
+    
+    // Combine results
+    return {
+      Deleted: results.flatMap(r => r.Deleted || []),
+      Errors: results.flatMap(r => r.Errors || [])
+    };
+  }
+  
+  /**
+   * Check if a file exists in S3
+   * 
+   * @param {string} s3Key - S3 object key
+   * @returns {Promise<boolean>} - True if exists
+   */
+  async exists(s3Key) {
+    try {
+      await this.s3Client.send(new HeadObjectCommand({
+        Bucket: this.bucket,
+        Key: s3Key
+      }));
+      return true;
+    } catch (error) {
+      if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
+        return false;
+      }
+      throw error;
+    }
+  }
+  
+  /**
+   * List files in S3
+   * 
+   * @param {string} prefix - S3 prefix to list
+   * @param {Object} [options={}] - Additional options
+   * @param {number} [options.maxKeys=1000] - Maximum number of keys to return
+   * @param {string} [options.continuationToken] - Continuation token for pagination
+   * @returns {Promise<Object>} - List result with Contents array and NextContinuationToken
+   */
+  async list(prefix, options = {}) {
+    const params = {
+      Bucket: this.bucket,
+      Prefix: prefix,
+      MaxKeys: options.maxKeys || 1000,
+      ContinuationToken: options.continuationToken
+    };
+    
+    return await this._retryOperation(() =>
+      this.s3Client.send(new ListObjectsV2Command(params))
+    );
+  }
+  
+  /**
+   * Copy a file within S3
+   * 
+   * @param {string} sourceKey - Source S3 object key
+   * @param {string} targetKey - Target S3 object key
+   * @param {Object} [options={}] - Additional options
+   * @returns {Promise<Object>} - Copy result
+   */
+  async copy(sourceKey, targetKey, options = {}) {
+    const copySource = `${this.bucket}/${sourceKey}`;
+    
+    const params = {
+      Bucket: this.bucket,
+      CopySource: copySource,
+      Key: targetKey,
+      MetadataDirective: options.metadata ? 'REPLACE' : 'COPY',
+      Metadata: options.metadata,
+      ContentType: options.contentType,
+      CacheControl: options.cacheControl
+    };
+    
+    // Remove undefined values
+    Object.keys(params).forEach(key => params[key] === undefined && delete params[key]);
+    
+    return await this._retryOperation(() =>
+      this.s3Client.send(new CopyObjectCommand(params))
+    );
+  }
+  
+  /**
+   * Move a file within S3 (copy then delete)
+   * 
+   * @param {string} sourceKey - Source S3 object key
+   * @param {string} targetKey - Target S3 object key
+   * @param {Object} [options={}] - Additional options
+   * @returns {Promise<Object>} - Move result
+   */
+  async move(sourceKey, targetKey, options = {}) {
+    // First copy the object
+    const copyResult = await this.copy(sourceKey, targetKey, options);
+    
+    // Then delete the original
+    await this.delete(sourceKey);
+    
+    return copyResult;
+  }
+  
+  /**
+   * Get a pre-signed URL for downloading or uploading
+   * 
+   * @param {string} operation - Operation type ('getObject' or 'putObject')
+   * @param {string} s3Key - S3 object key
+   * @param {Object} [options={}] - Additional options
+   * @param {number} [options.expiresIn=3600] - URL expiration in seconds
+   * @param {Object} [options.params] - Additional parameters for the operation
+   * @returns {Promise<string>} - Pre-signed URL
+   */
+  async getSignedUrl(operation, s3Key, options = {}) {
+    const params = {
+      Bucket: this.bucket,
+      Key: s3Key,
+      ...options.params
+    };
+    
+    let command;
+    switch (operation.toLowerCase()) {
+      case 'getobject':
+        command = new GetObjectCommand(params);
+        break;
+      case 'putobject':
+        command = new PutObjectCommand(params);
+        break;
+      default:
+        throw new Error(`Unsupported operation: ${operation}`);
+    }
+    
+    return await getSignedUrl(this.s3Client, command, {
+      expiresIn: options.expiresIn || 3600
+    });
+  }
+  
+  /**
+   * Get object metadata
+   * 
+   * @param {string} s3Key - S3 object key
+   * @returns {Promise<Object>} - Object metadata
+   */
+  async getMetadata(s3Key) {
+    return await this._retryOperation(() =>
+      this.s3Client.send(new HeadObjectCommand({
+        Bucket: this.bucket,
+        Key: s3Key
+      }))
+    );
+  }
+  
+  /**
+   * Update object metadata
+   * 
+   * @param {string} s3Key - S3 object key
+   * @param {Object} metadata - New metadata
+   * @returns {Promise<Object>} - Update result
+   */
+  async updateMetadata(s3Key, metadata) {
+    // S3 requires copying the object to itself to update metadata
+    return await this.copy(s3Key, s3Key, { metadata });
+  }
+  
+  /**
+   * Manual multipart upload for advanced use cases
+   * 
+   * @param {string} localPath - Local file path
+   * @param {string} s3Key - S3 object key
+   * @param {number} fileSize - File size in bytes
+   * @param {Object} options - Upload options
+   * @returns {Promise<Object>} - Upload result
+   * @private
+   */
+  async _manualMultipartUpload(localPath, s3Key, fileSize, options) {
+    logger.info(`Starting manual multipart upload for ${s3Key} (${fileSize} bytes)`);
+    
+    // Initiate multipart upload
+    const multipartParams = {
+      Bucket: this.bucket,
+      Key: s3Key,
+      ContentType: options.contentType || 'application/octet-stream',
+      Metadata: options.metadata || {},
+      CacheControl: options.cacheControl
+    };
+    
+    // Remove undefined values
+    Object.keys(multipartParams).forEach(key => multipartParams[key] === undefined && delete multipartParams[key]);
+    
+    const multipart = await this._retryOperation(() =>
+      this.s3Client.send(new CreateMultipartUploadCommand(multipartParams))
+    );
+    
+    const uploadId = multipart.UploadId;
+    const partSize = this.config.partSize;
+    const numParts = Math.ceil(fileSize / partSize);
+    
+    let uploaded = 0;
+    const parts = [];
+    
+    try {
+      // Upload parts
+      for (let partNum = 1; partNum <= numParts; partNum++) {
+        const start = (partNum - 1) * partSize;
+        const end = Math.min(start + partSize, fileSize);
+        
+        const partStream = fs.createReadStream(localPath, {
+          start,
+          end: end - 1
+        });
+        
+        const partParams = {
+          Bucket: this.bucket,
+          Key: s3Key,
+          PartNumber: partNum,
+          UploadId: uploadId,
+          Body: partStream
+        };
+        
+        // Upload part with retry
+        const partResult = await this._retryOperation(() =>
+          this.s3Client.send(new UploadPartCommand(partParams))
+        );
+        
+        parts.push({
+          ETag: partResult.ETag,
+          PartNumber: partNum
+        });
+        
+        uploaded += (end - start);
+        
+        if (options.onProgress) {
+          options.onProgress(uploaded, fileSize);
+        }
+        this.emit('uploadProgress', { key: s3Key, loaded: uploaded, total: fileSize });
+        
+        logger.info(`Uploaded part ${partNum}/${numParts} for ${s3Key}`);
+      }
+      
+      // Complete multipart upload
+      const completeParams = {
+        Bucket: this.bucket,
+        Key: s3Key,
+        UploadId: uploadId,
+        MultipartUpload: { Parts: parts }
+      };
+      
+      const result = await this._retryOperation(() =>
+        this.s3Client.send(new CompleteMultipartUploadCommand(completeParams))
+      );
+      
+      this.emit('uploadComplete', { key: s3Key, location: result.Location });
+      logger.info(`Completed multipart upload for ${s3Key}`);
+      
+      return result;
+    } catch (error) {
+      // Abort multipart upload on error
+      logger.error(`Multipart upload failed for ${s3Key}, aborting:`, error);
+      
+      try {
+        await this.s3Client.send(new AbortMultipartUploadCommand({
+          Bucket: this.bucket,
+          Key: s3Key,
+          UploadId: uploadId
+        }));
+      } catch (abortError) {
+        logger.error(`Failed to abort multipart upload:`, abortError);
+      }
+      
+      throw error;
+    }
+  }
+  
+  /**
+   * Retry operation with exponential backoff
+   * @private
+   */
+  async _retryOperation(operation, retryCount = 0) {
+    try {
+      return await operation();
+    } catch (error) {
+      if (retryCount >= this.config.maxRetries) {
+        throw error;
+      }
+      
+      // Check if error is retryable
+      const retryableErrors = ['ECONNRESET', 'ETIMEDOUT', 'ENOTFOUND', 'ESOCKETTIMEDOUT', 'RequestTimeout', 'SlowDown', 'ServiceUnavailable', 'InternalError'];
+      const isRetryable = retryableErrors.some(code => 
+        error.code === code || 
+        error.name === code || 
+        error.$metadata?.httpStatusCode === 503 ||
+        error.$metadata?.httpStatusCode === 500
+      );
+      
+      if (!isRetryable) {
+        throw error;
+      }
+      
+      // Calculate delay with exponential backoff and jitter
+      const delay = Math.min(
+        this.config.retryDelay * Math.pow(2, retryCount) + Math.random() * 1000,
+        30000 // Max 30 seconds
+      );
+      
+      logger.warn(`Retrying operation after ${delay}ms (attempt ${retryCount + 1}/${this.config.maxRetries}):`, error.message);
+      
+      await new Promise(resolve => setTimeout(resolve, delay));
+      
+      return this._retryOperation(operation, retryCount + 1);
+    }
+  }
+  
+  /**
+   * Generate a unique S3 key for a file
+   * 
+   * @param {string} originalName - Original filename
+   * @param {string} [prefix=''] - Optional prefix for the key
+   * @returns {string} - Generated S3 key
+   */
+  generateKey(originalName, prefix = '') {
+    const timestamp = Date.now();
+    const randomStr = crypto.randomBytes(8).toString('hex');
+    const ext = path.extname(originalName);
+    const basename = path.basename(originalName, ext);
+    
+    // Sanitize basename
+    const safeName = basename.replace(/[^a-zA-Z0-9-_]/g, '_');
+    
+    const key = `${timestamp}_${randomStr}_${safeName}${ext}`;
+    
+    return prefix ? path.posix.join(prefix, key) : key;
+  }
+  
+  /**
+   * Get storage statistics
+   * 
+   * @param {string} [prefix=''] - Optional prefix to filter
+   * @returns {Promise<Object>} - Storage statistics
+   */
+  async getStats(prefix = '') {
+    let totalSize = 0;
+    let totalCount = 0;
+    let continuationToken;
+    
+    do {
+      const result = await this.list(prefix, { continuationToken });
+      
+      if (result.Contents) {
+        totalCount += result.Contents.length;
+        totalSize += result.Contents.reduce((sum, obj) => sum + (obj.Size || 0), 0);
+      }
+      
+      continuationToken = result.NextContinuationToken;
+    } while (continuationToken);
+    
+    return {
+      totalSize,
+      totalCount,
+      totalSizeFormatted: this._formatBytes(totalSize)
+    };
+  }
+  
+  /**
+   * Format bytes to human readable format
+   * @private
+   */
+  _formatBytes(bytes, decimals = 2) {
+    if (bytes === 0) return '0 Bytes';
+    
+    const k = 1024;
+    const dm = decimals < 0 ? 0 : decimals;
+    const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
+    
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
+  }
+}
+
+module.exports = S3StorageAdapter;