feat: increase file upload limit from 20 to 500 with performance optimizations

Backend changes:
- Update multer configuration to accept up to 500 files per upload
- Implement batch processing (10 files per transaction) for better performance
- Add memory-efficient Sharp configuration for thumbnail generation
- Increase Express body parser limits to handle large payloads
- Add proper error handling and reporting for partial upload failures

Frontend changes:
- Update validation to allow 500 files maximum
- Implement chunked uploads (50 files per chunk) to prevent timeouts
- Add progress tracking with chunk information display
- Update error messages and translations (EN/DE)

Performance optimizations:
- Disable Sharp cache to prevent memory buildup
- Limit Sharp concurrency to 2 operations
- Use sequential read for large images
- Process files in database transaction batches
- Return detailed upload results including success/failure counts

This implementation ensures the application can handle large photo uploads
efficiently without running into memory or timeout issues.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

backend/src/routes/adminPhotos.js

@@ -61,7 +61,10 @@ const { validateFileType } = require('../utils/fileSecurityUtils');
 const upload = multer({
   storage: storage,
   limits: {
-    fileSize: 50 * 1024 * 1024, // 50MB limit
+    fileSize: 50 * 1024 * 1024, // 50MB limit per file
+    files: 500, // Maximum 500 files
+    // Set a reasonable field size limit to prevent memory issues
+    fieldSize: 10 * 1024 * 1024, // 10MB for non-file fields
   },
   fileFilter: (req, file, cb) => {
     // Accept images only with proper validation
@@ -85,13 +88,17 @@ const validateUploadContent = createFileUploadValidator({
 });
 
 // Upload photos for an event
+// Increased limit to 500 files, but recommend chunked uploads for better performance
 router.post('/:eventId/upload', adminAuth, (req, res, next) => {
-  upload.array('photos', 20)(req, res, (err) => {
+  upload.array('photos', 500)(req, res, (err) => {
     if (err) {
       console.error('Multer error:', err);
       if (err instanceof multer.MulterError) {
         if (err.code === 'LIMIT_FILE_SIZE') {
-          return res.status(400).json({ error: 'File too large. Maximum size is 50MB.' });
+          return res.status(400).json({ error: 'File too large. Maximum size is 50MB per file.' });
+        }
+        if (err.code === 'LIMIT_FILE_COUNT') {
+          return res.status(400).json({ error: 'Too many files. Maximum 500 files per upload.' });
         }
         return res.status(400).json({ error: `Upload error: ${err.message}` });
       }
@@ -136,90 +143,122 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
     }
     
     const uploadedPhotos = [];
+    const errors = [];
     
-    // Process each uploaded file
-    for (const file of req.files) {
-      let trx;
+    // Process files in batches to optimize database operations
+    const BATCH_SIZE = 10; // Process 10 files at a time for database operations
+    
+    for (let i = 0; i < req.files.length; i += BATCH_SIZE) {
+      const batch = req.files.slice(i, i + BATCH_SIZE);
+      
+      // Start a single transaction for the batch
+      const trx = await db.transaction();
+      
       try {
-        // Start transaction for atomic counter update
-        trx = await db.transaction();
-        
-        // Get and increment the counter for this category
-        let counter = 1;
+        // Get initial counter for this batch
+        let batchCounter = 1;
         if (category) {
-          // Lock the category row and get current counter
           const categoryData = await trx('photo_categories')
             .where({ id: parsedCategoryId })
             .forUpdate()
             .first();
-          
-          counter = (categoryData.photo_counter || 0) + 1;
-          
-          // Update counter
-          await trx('photo_categories')
-            .where({ id: parsedCategoryId })
-            .update({ photo_counter: counter });
+          batchCounter = (categoryData.photo_counter || 0) + 1;
         } else {
-          // For uncategorized photos, count existing uncategorized photos
           const uncategorizedCount = await trx('photos')
             .where({ event_id: eventId })
             .whereNull('category_id')
             .count('id as count')
             .first();
-          
-          counter = (uncategorizedCount.count || 0) + 1;
+          batchCounter = (uncategorizedCount.count || 0) + 1;
         }
         
-        // Generate new filename
-        const extension = path.extname(file.originalname);
-        const newFilename = generatePhotoFilename(
-          event.event_name,
-          category ? category.name : 'uncategorized',
-          counter,
-          extension
-        );
+        const batchPhotos = [];
         
-        // Rename the file
-        const oldPath = file.path;
-        const newPath = path.join(path.dirname(oldPath), newFilename);
-        await fs.rename(oldPath, newPath);
+        for (let fileIndex = 0; fileIndex < batch.length; fileIndex++) {
+          const file = batch[fileIndex];
+          const counter = batchCounter + fileIndex;
+          
+          try {
+            // Generate new filename
+            const extension = path.extname(file.originalname);
+            const newFilename = generatePhotoFilename(
+              event.event_name,
+              category ? category.name : 'uncategorized',
+              counter,
+              extension
+            );
+            
+            // Rename the file
+            const oldPath = file.path;
+            const newPath = path.join(path.dirname(oldPath), newFilename);
+            await fs.rename(oldPath, newPath);
+            
+            // Update file object
+            file.filename = newFilename;
+            file.path = newPath;
+            
+            // Generate thumbnail with new filename
+            const thumbnailPath = await generateThumbnail(file.path);
+            
+            // Calculate relative paths
+            const storagePath = getStoragePath();
+            const relativePath = path.relative(path.join(storagePath, 'events/active'), file.path);
+            const relativeThumbPath = thumbnailPath;
+            
+            // Prepare photo data for batch insert
+            batchPhotos.push({
+              event_id: eventId,
+              filename: file.filename,
+              path: relativePath,
+              thumbnail_path: relativeThumbPath,
+              category_id: parsedCategoryId || null,
+              type: 'individual',
+              size_bytes: file.size
+            });
+          } catch (error) {
+            console.error(`Error processing file ${file.originalname}:`, error);
+            errors.push({ filename: file.originalname, error: error.message });
+            // Delete the file if it was partially processed
+            if (file.path) {
+              try { await fs.unlink(file.path); } catch (e) {}
+            }
+          }
+        }
         
-        // Update file object
-        file.filename = newFilename;
-        file.path = newPath;
+        // Batch insert all photos from this batch
+        if (batchPhotos.length > 0) {
+          const insertedIds = await trx('photos').insert(batchPhotos).returning('id');
+          
+          // Update category counter if needed
+          if (category) {
+            await trx('photo_categories')
+              .where({ id: parsedCategoryId })
+              .update({ photo_counter: batchCounter + batchPhotos.length - 1 });
+          }
+          
+          // Add to uploaded photos array
+          batchPhotos.forEach((photo, index) => {
+            uploadedPhotos.push({
+              id: insertedIds[index]?.id || insertedIds[index],
+              filename: photo.filename,
+              size: photo.size_bytes,
+              category_id: photo.category_id
+            });
+          });
+        }
         
-        // Generate thumbnail with new filename
-        const thumbnailPath = await generateThumbnail(file.path);
-        
-        // Calculate relative paths
-        const storagePath = getStoragePath();
-        const relativePath = path.relative(path.join(storagePath, 'events/active'), file.path);
-        const relativeThumbPath = thumbnailPath; // thumbnailPath is already relative to storage root
-        
-        // Add to database
-        const [photoId] = await trx('photos').insert({
-          event_id: eventId,
-          filename: file.filename,
-          path: relativePath,
-          thumbnail_path: relativeThumbPath,
-          category_id: parsedCategoryId || null,
-          type: 'individual', // Keep for backwards compatibility
-          size_bytes: file.size
-        });
-        
-        // Commit transaction
+        // Commit the batch transaction
         await trx.commit();
-        
-        uploadedPhotos.push({
-          id: photoId,
-          filename: file.filename,
-          size: file.size,
-          category_id: parsedCategoryId || null
-        });
       } catch (error) {
-        console.error(`Error processing file ${file.filename}:`, error);
-        if (trx) await trx.rollback();
-        // Continue with other files
+        console.error(`Error processing batch starting at index ${i}:`, error);
+        await trx.rollback();
+        
+        // Try to clean up files from failed batch
+        for (const file of batch) {
+          if (file.path) {
+            try { await fs.unlink(file.path); } catch (e) {}
+          }
+        }
       }
     }
     
@@ -230,10 +269,22 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
       { type: 'admin', id: req.admin.id, name: req.admin.username }
     );
     
-    res.json({
+    // Prepare response
+    const response = {
       message: `Successfully uploaded ${uploadedPhotos.length} photos`,
-      photos: uploadedPhotos
-    });
+      photos: uploadedPhotos,
+      totalFiles: req.files.length,
+      successCount: uploadedPhotos.length,
+      failureCount: errors.length
+    };
+    
+    // Include error details if any files failed
+    if (errors.length > 0) {
+      response.errors = errors;
+      response.message = `Uploaded ${uploadedPhotos.length} of ${req.files.length} photos. ${errors.length} failed.`;
+    }
+    
+    res.json(response);
   } catch (error) {
     console.error('Error uploading photos:', error);
     res.status(500).json({ error: 'Failed to upload photos' });

backend/src/services/imageProcessor.js

@@ -2,6 +2,10 @@ const sharp = require('sharp');
 const path = require('path');
 const fs = require('fs').promises;
 
+// Configure sharp for better memory management with large batches
+sharp.cache(false); // Disable cache to prevent memory buildup
+sharp.concurrency(2); // Limit concurrent operations
+
 const THUMBNAIL_WIDTH = 300;
 const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
 const getThumbnailPath = () => path.join(getStoragePath(), 'thumbnails');
@@ -15,16 +19,29 @@ async function generateThumbnail(imagePath) {
   // Ensure thumbnail directory exists
   await fs.mkdir(thumbnailDir, { recursive: true });
   
-  // Generate thumbnail
-  await sharp(imagePath)
-    .resize(THUMBNAIL_WIDTH, null, {
-      withoutEnlargement: true,
-      fit: 'inside'
+  try {
+    // Generate thumbnail with memory-efficient settings
+    await sharp(imagePath, { 
+      limitInputPixels: 268402689, // ~16k x 16k max
+      sequentialRead: true // More memory efficient for large images
     })
-    .jpeg({ quality: 80 })
-    .toFile(thumbnailPath);
-  
-  return path.relative(getStoragePath(), thumbnailPath);
+      .resize(THUMBNAIL_WIDTH, null, {
+        withoutEnlargement: true,
+        fit: 'inside'
+      })
+      .jpeg({ 
+        quality: 80,
+        progressive: true, // Progressive JPEG for better loading
+        mozjpeg: true // Better compression
+      })
+      .toFile(thumbnailPath);
+    
+    return path.relative(getStoragePath(), thumbnailPath);
+  } catch (error) {
+    console.error(`Failed to generate thumbnail for ${filename}:`, error);
+    // Return null if thumbnail generation fails, don't fail the whole upload
+    return null;
+  }
 }
 
 module.exports = { generateThumbnail };