feat: increase file upload limit from 20 to 500 with performance optimizations

Backend changes:
- Update multer configuration to accept up to 500 files per upload
- Implement batch processing (10 files per transaction) for better performance
- Add memory-efficient Sharp configuration for thumbnail generation
- Increase Express body parser limits to handle large payloads
- Add proper error handling and reporting for partial upload failures

Frontend changes:
- Update validation to allow 500 files maximum
- Implement chunked uploads (50 files per chunk) to prevent timeouts
- Add progress tracking with chunk information display
- Update error messages and translations (EN/DE)

Performance optimizations:
- Disable Sharp cache to prevent memory buildup
- Limit Sharp concurrency to 2 operations
- Use sequential read for large images
- Process files in database transaction batches
- Return detailed upload results including success/failure counts

This implementation ensures the application can handle large photo uploads
efficiently without running into memory or timeout issues.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

backend/server.js

@@ -125,9 +125,9 @@ const authLimiter = rateLimit({
 app.use('/api/', limiter);
 app.use('/api/auth', authLimiter);
 
-// Body parsing middleware
-app.use(express.json());
-app.use(express.urlencoded({ extended: true }));
+// Body parsing middleware with increased limits for large uploads
+app.use(express.json({ limit: '100mb' }));
+app.use(express.urlencoded({ extended: true, limit: '100mb' }));
 
 // Maintenance mode middleware - add after body parsing but before routes
 app.use(maintenanceMiddleware);

backend/src/routes/adminPhotos.js

@@ -61,7 +61,10 @@ const { validateFileType } = require('../utils/fileSecurityUtils');
 const upload = multer({
   storage: storage,
   limits: {
-    fileSize: 50 * 1024 * 1024, // 50MB limit
+    fileSize: 50 * 1024 * 1024, // 50MB limit per file
+    files: 500, // Maximum 500 files
+    // Set a reasonable field size limit to prevent memory issues
+    fieldSize: 10 * 1024 * 1024, // 10MB for non-file fields
   },
   fileFilter: (req, file, cb) => {
     // Accept images only with proper validation
@@ -85,13 +88,17 @@ const validateUploadContent = createFileUploadValidator({
 });
 
 // Upload photos for an event
+// Increased limit to 500 files, but recommend chunked uploads for better performance
 router.post('/:eventId/upload', adminAuth, (req, res, next) => {
-  upload.array('photos', 20)(req, res, (err) => {
+  upload.array('photos', 500)(req, res, (err) => {
     if (err) {
       console.error('Multer error:', err);
       if (err instanceof multer.MulterError) {
         if (err.code === 'LIMIT_FILE_SIZE') {
-          return res.status(400).json({ error: 'File too large. Maximum size is 50MB.' });
+          return res.status(400).json({ error: 'File too large. Maximum size is 50MB per file.' });
+        }
+        if (err.code === 'LIMIT_FILE_COUNT') {
+          return res.status(400).json({ error: 'Too many files. Maximum 500 files per upload.' });
         }
         return res.status(400).json({ error: `Upload error: ${err.message}` });
       }
@@ -136,90 +143,122 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
     }
     
     const uploadedPhotos = [];
+    const errors = [];
     
-    // Process each uploaded file
-    for (const file of req.files) {
-      let trx;
+    // Process files in batches to optimize database operations
+    const BATCH_SIZE = 10; // Process 10 files at a time for database operations
+    
+    for (let i = 0; i < req.files.length; i += BATCH_SIZE) {
+      const batch = req.files.slice(i, i + BATCH_SIZE);
+      
+      // Start a single transaction for the batch
+      const trx = await db.transaction();
+      
       try {
-        // Start transaction for atomic counter update
-        trx = await db.transaction();
-        
-        // Get and increment the counter for this category
-        let counter = 1;
+        // Get initial counter for this batch
+        let batchCounter = 1;
         if (category) {
-          // Lock the category row and get current counter
           const categoryData = await trx('photo_categories')
             .where({ id: parsedCategoryId })
             .forUpdate()
             .first();
-          
-          counter = (categoryData.photo_counter || 0) + 1;
-          
-          // Update counter
-          await trx('photo_categories')
-            .where({ id: parsedCategoryId })
-            .update({ photo_counter: counter });
+          batchCounter = (categoryData.photo_counter || 0) + 1;
         } else {
-          // For uncategorized photos, count existing uncategorized photos
           const uncategorizedCount = await trx('photos')
             .where({ event_id: eventId })
             .whereNull('category_id')
             .count('id as count')
             .first();
-          
-          counter = (uncategorizedCount.count || 0) + 1;
+          batchCounter = (uncategorizedCount.count || 0) + 1;
         }
         
-        // Generate new filename
-        const extension = path.extname(file.originalname);
-        const newFilename = generatePhotoFilename(
-          event.event_name,
-          category ? category.name : 'uncategorized',
-          counter,
-          extension
-        );
+        const batchPhotos = [];
         
-        // Rename the file
-        const oldPath = file.path;
-        const newPath = path.join(path.dirname(oldPath), newFilename);
-        await fs.rename(oldPath, newPath);
+        for (let fileIndex = 0; fileIndex < batch.length; fileIndex++) {
+          const file = batch[fileIndex];
+          const counter = batchCounter + fileIndex;
+          
+          try {
+            // Generate new filename
+            const extension = path.extname(file.originalname);
+            const newFilename = generatePhotoFilename(
+              event.event_name,
+              category ? category.name : 'uncategorized',
+              counter,
+              extension
+            );
+            
+            // Rename the file
+            const oldPath = file.path;
+            const newPath = path.join(path.dirname(oldPath), newFilename);
+            await fs.rename(oldPath, newPath);
+            
+            // Update file object
+            file.filename = newFilename;
+            file.path = newPath;
+            
+            // Generate thumbnail with new filename
+            const thumbnailPath = await generateThumbnail(file.path);
+            
+            // Calculate relative paths
+            const storagePath = getStoragePath();
+            const relativePath = path.relative(path.join(storagePath, 'events/active'), file.path);
+            const relativeThumbPath = thumbnailPath;
+            
+            // Prepare photo data for batch insert
+            batchPhotos.push({
+              event_id: eventId,
+              filename: file.filename,
+              path: relativePath,
+              thumbnail_path: relativeThumbPath,
+              category_id: parsedCategoryId || null,
+              type: 'individual',
+              size_bytes: file.size
+            });
+          } catch (error) {
+            console.error(`Error processing file ${file.originalname}:`, error);
+            errors.push({ filename: file.originalname, error: error.message });
+            // Delete the file if it was partially processed
+            if (file.path) {
+              try { await fs.unlink(file.path); } catch (e) {}
+            }
+          }
+        }
         
-        // Update file object
-        file.filename = newFilename;
-        file.path = newPath;
+        // Batch insert all photos from this batch
+        if (batchPhotos.length > 0) {
+          const insertedIds = await trx('photos').insert(batchPhotos).returning('id');
+          
+          // Update category counter if needed
+          if (category) {
+            await trx('photo_categories')
+              .where({ id: parsedCategoryId })
+              .update({ photo_counter: batchCounter + batchPhotos.length - 1 });
+          }
+          
+          // Add to uploaded photos array
+          batchPhotos.forEach((photo, index) => {
+            uploadedPhotos.push({
+              id: insertedIds[index]?.id || insertedIds[index],
+              filename: photo.filename,
+              size: photo.size_bytes,
+              category_id: photo.category_id
+            });
+          });
+        }
         
-        // Generate thumbnail with new filename
-        const thumbnailPath = await generateThumbnail(file.path);
-        
-        // Calculate relative paths
-        const storagePath = getStoragePath();
-        const relativePath = path.relative(path.join(storagePath, 'events/active'), file.path);
-        const relativeThumbPath = thumbnailPath; // thumbnailPath is already relative to storage root
-        
-        // Add to database
-        const [photoId] = await trx('photos').insert({
-          event_id: eventId,
-          filename: file.filename,
-          path: relativePath,
-          thumbnail_path: relativeThumbPath,
-          category_id: parsedCategoryId || null,
-          type: 'individual', // Keep for backwards compatibility
-          size_bytes: file.size
-        });
-        
-        // Commit transaction
+        // Commit the batch transaction
         await trx.commit();
-        
-        uploadedPhotos.push({
-          id: photoId,
-          filename: file.filename,
-          size: file.size,
-          category_id: parsedCategoryId || null
-        });
       } catch (error) {
-        console.error(`Error processing file ${file.filename}:`, error);
-        if (trx) await trx.rollback();
-        // Continue with other files
+        console.error(`Error processing batch starting at index ${i}:`, error);
+        await trx.rollback();
+        
+        // Try to clean up files from failed batch
+        for (const file of batch) {
+          if (file.path) {
+            try { await fs.unlink(file.path); } catch (e) {}
+          }
+        }
       }
     }
     
@@ -230,10 +269,22 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
       { type: 'admin', id: req.admin.id, name: req.admin.username }
     );
     
-    res.json({
+    // Prepare response
+    const response = {
       message: `Successfully uploaded ${uploadedPhotos.length} photos`,
-      photos: uploadedPhotos
-    });
+      photos: uploadedPhotos,
+      totalFiles: req.files.length,
+      successCount: uploadedPhotos.length,
+      failureCount: errors.length
+    };
+    
+    // Include error details if any files failed
+    if (errors.length > 0) {
+      response.errors = errors;
+      response.message = `Uploaded ${uploadedPhotos.length} of ${req.files.length} photos. ${errors.length} failed.`;
+    }
+    
+    res.json(response);
   } catch (error) {
     console.error('Error uploading photos:', error);
     res.status(500).json({ error: 'Failed to upload photos' });

backend/src/services/imageProcessor.js

@@ -2,6 +2,10 @@ const sharp = require('sharp');
 const path = require('path');
 const fs = require('fs').promises;
 
+// Configure sharp for better memory management with large batches
+sharp.cache(false); // Disable cache to prevent memory buildup
+sharp.concurrency(2); // Limit concurrent operations
+
 const THUMBNAIL_WIDTH = 300;
 const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
 const getThumbnailPath = () => path.join(getStoragePath(), 'thumbnails');
@@ -15,16 +19,29 @@ async function generateThumbnail(imagePath) {
   // Ensure thumbnail directory exists
   await fs.mkdir(thumbnailDir, { recursive: true });
   
-  // Generate thumbnail
-  await sharp(imagePath)
-    .resize(THUMBNAIL_WIDTH, null, {
-      withoutEnlargement: true,
-      fit: 'inside'
+  try {
+    // Generate thumbnail with memory-efficient settings
+    await sharp(imagePath, { 
+      limitInputPixels: 268402689, // ~16k x 16k max
+      sequentialRead: true // More memory efficient for large images
     })
-    .jpeg({ quality: 80 })
-    .toFile(thumbnailPath);
-  
-  return path.relative(getStoragePath(), thumbnailPath);
+      .resize(THUMBNAIL_WIDTH, null, {
+        withoutEnlargement: true,
+        fit: 'inside'
+      })
+      .jpeg({ 
+        quality: 80,
+        progressive: true, // Progressive JPEG for better loading
+        mozjpeg: true // Better compression
+      })
+      .toFile(thumbnailPath);
+    
+    return path.relative(getStoragePath(), thumbnailPath);
+  } catch (error) {
+    console.error(`Failed to generate thumbnail for ${filename}:`, error);
+    // Return null if thumbnail generation fails, don't fail the whole upload
+    return null;
+  }
 }
 
 module.exports = { generateThumbnail };

frontend/src/components/admin/PhotoUpload.tsx

@@ -18,6 +18,8 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
   const [isUploading, setIsUploading] = useState(false);
   const [selectedFiles, setSelectedFiles] = useState<File[]>([]);
   const [uploadProgress, setUploadProgress] = useState(0);
+  const [currentChunk, setCurrentChunk] = useState(0);
+  const [totalChunks, setTotalChunks] = useState(0);
   const [selectedCategoryId, setSelectedCategoryId] = useState<number | null>(null);
   const fileInputRef = useRef<HTMLInputElement>(null);
   
@@ -35,13 +37,13 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
     
     // Check total file count with existing files
     const totalFiles = selectedFiles.length + imageFiles.length;
-    if (totalFiles > 20) {
-      const allowedNewFiles = 20 - selectedFiles.length;
+    if (totalFiles > 500) {
+      const allowedNewFiles = 500 - selectedFiles.length;
       if (allowedNewFiles <= 0) {
-        toast.error(t('upload.maxFilesReached') || 'Maximum 20 files allowed');
+        toast.error(t('upload.maxFilesReached') || 'Maximum 500 files allowed');
         return;
       }
-      toast.warning(t('upload.someFilesSkipped') || `Only ${allowedNewFiles} more files can be added (20 max)`);
+      toast.warning(t('upload.someFilesSkipped') || `Only ${allowedNewFiles} more files can be added (500 max)`);
       setSelectedFiles(prev => [...prev, ...imageFiles.slice(0, allowedNewFiles)]);
       return;
     }
@@ -57,42 +59,62 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
     if (selectedFiles.length === 0) return;
     
     // Validate file count
-    if (selectedFiles.length > 20) {
-      toast.error(t('upload.tooManyFiles') || 'Maximum 20 files can be uploaded at once');
+    if (selectedFiles.length > 500) {
+      toast.error(t('upload.tooManyFiles') || 'Maximum 500 files can be uploaded at once');
       return;
     }
 
     setIsUploading(true);
     setUploadProgress(0);
 
-    const formData = new FormData();
-    selectedFiles.forEach((file, index) => {
-      console.log(`Adding file ${index}: ${file.name}, size: ${file.size}`);
-      formData.append('photos', file);
-    });
+    // For large uploads, chunk the files to prevent memory issues
+    const CHUNK_SIZE = 50; // Upload 50 files at a time
+    const chunks = [];
     
-    if (selectedCategoryId) {
-      formData.append('category_id', selectedCategoryId.toString());
-    }
-    
-    // Debug: Log FormData contents
-    console.log('FormData entries:');
-    for (let pair of formData.entries()) {
-      console.log(pair[0], pair[1]);
+    for (let i = 0; i < selectedFiles.length; i += CHUNK_SIZE) {
+      chunks.push(selectedFiles.slice(i, i + CHUNK_SIZE));
     }
 
+    setTotalChunks(chunks.length);
+    let totalUploaded = 0;
+    let failedFiles = [];
+
     try {
-      const response = await api.post(`/admin/events/${eventId}/upload`, formData, {
-        // Don't set Content-Type header - axios will set it with the boundary
-        onUploadProgress: (progressEvent) => {
-          if (progressEvent.total) {
-            const progress = Math.round((progressEvent.loaded * 100) / progressEvent.total);
-            setUploadProgress(progress);
-          }
-        },
-      });
+      for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
+        setCurrentChunk(chunkIndex + 1);
+        const chunk = chunks[chunkIndex];
+        const formData = new FormData();
+        
+        chunk.forEach((file) => {
+          formData.append('photos', file);
+        });
+        
+        if (selectedCategoryId) {
+          formData.append('category_id', selectedCategoryId.toString());
+        }
 
-      console.log('Upload result:', response.data);
+        try {
+          const response = await api.post(`/admin/events/${eventId}/upload`, formData, {
+            onUploadProgress: (progressEvent) => {
+              if (progressEvent.total) {
+                // Calculate overall progress across all chunks
+                const chunkProgress = progressEvent.loaded / progressEvent.total;
+                const overallProgress = ((chunkIndex + chunkProgress) / chunks.length) * 100;
+                setUploadProgress(Math.round(overallProgress));
+              }
+            },
+          });
+
+          totalUploaded += chunk.length;
+          console.log(`Chunk ${chunkIndex + 1}/${chunks.length} uploaded:`, response.data);
+        } catch (error: any) {
+          console.error(`Error uploading chunk ${chunkIndex + 1}:`, error);
+          failedFiles.push(...chunk.map(f => f.name));
+          
+          // Continue with next chunk even if one fails
+          continue;
+        }
+      }
 
       // Clear selected files
       setSelectedFiles([]);
@@ -100,8 +122,15 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
         fileInputRef.current.value = '';
       }
 
-      // Show success message
-      toast.success(t('toast.uploadSuccess'));
+      // Show appropriate message
+      if (failedFiles.length === 0) {
+        toast.success(t('upload.uploadComplete') || `Successfully uploaded ${totalUploaded} files`);
+      } else {
+        toast.warning(
+          t('upload.someFilesFailed') || 
+          `Uploaded ${totalUploaded} files. ${failedFiles.length} files failed.`
+        );
+      }
       
       // Call callback
       if (onUploadComplete) {
@@ -113,6 +142,8 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
     } finally {
       setIsUploading(false);
       setUploadProgress(0);
+      setCurrentChunk(0);
+      setTotalChunks(0);
     }
   };
 
@@ -223,7 +254,10 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
       {isUploading && (
         <div className="mt-4">
           <div className="flex justify-between text-sm text-neutral-600 mb-1">
-            <span>{t('upload.uploading')}</span>
+            <span>
+              {t('upload.uploading')}
+              {totalChunks > 1 && ` (${t('common.chunk') || 'Chunk'} ${currentChunk}/${totalChunks})`}
+            </span>
             <span>{uploadProgress}%</span>
           </div>
           <div className="w-full bg-neutral-200 rounded-full h-2">
@@ -232,6 +266,11 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
               style={{ width: `${uploadProgress}%` }}
             />
           </div>
+          {totalChunks > 1 && (
+            <p className="text-xs text-neutral-500 mt-1">
+              {t('upload.uploadingChunks') || `Uploading ${selectedFiles.length} files in ${totalChunks} batches...`}
+            </p>
+          )}
         </div>
       )}
     </div>

frontend/src/i18n/locales/de.json

@@ -47,7 +47,10 @@
     "uploadComplete": "Upload abgeschlossen!",
     "uploadFailed": "Upload fehlgeschlagen",
     "someFilesFailed": "Einige Dateien konnten nicht hochgeladen werden",
-    "uploadPhotos": "Fotos hochladen"
+    "uploadPhotos": "Fotos hochladen",
+    "maxFilesReached": "Maximal 500 Dateien erlaubt",
+    "someFilesSkipped": "Einige Dateien wurden übersprungen (500 Dateien Limit)",
+    "tooManyFiles": "Maximal 500 Dateien können gleichzeitig hochgeladen werden"
   },
   "navigation": {
     "dashboard": "Dashboard",

frontend/src/i18n/locales/en.json

@@ -48,9 +48,9 @@
     "uploadFailed": "Upload failed",
     "someFilesFailed": "Some files failed to upload",
     "uploadPhotos": "Upload Photos",
-    "maxFilesReached": "Maximum 20 files allowed",
-    "someFilesSkipped": "Some files were skipped (20 file limit)",
-    "tooManyFiles": "Maximum 20 files can be uploaded at once"
+    "maxFilesReached": "Maximum 500 files allowed",
+    "someFilesSkipped": "Some files were skipped (500 file limit)",
+    "tooManyFiles": "Maximum 500 files can be uploaded at once"
   },
   "navigation": {
     "dashboard": "Dashboard",