diff --git a/backend/server.js b/backend/server.js index c066eb9..dc62277 100644 --- a/backend/server.js +++ b/backend/server.js @@ -125,9 +125,9 @@ const authLimiter = rateLimit({ app.use('/api/', limiter); app.use('/api/auth', authLimiter); -// Body parsing middleware -app.use(express.json()); -app.use(express.urlencoded({ extended: true })); +// Body parsing middleware with increased limits for large uploads +app.use(express.json({ limit: '100mb' })); +app.use(express.urlencoded({ extended: true, limit: '100mb' })); // Maintenance mode middleware - add after body parsing but before routes app.use(maintenanceMiddleware); diff --git a/backend/src/routes/adminPhotos.js b/backend/src/routes/adminPhotos.js index 61c7ce4..7f39d1c 100644 --- a/backend/src/routes/adminPhotos.js +++ b/backend/src/routes/adminPhotos.js @@ -61,7 +61,10 @@ const { validateFileType } = require('../utils/fileSecurityUtils'); const upload = multer({ storage: storage, limits: { - fileSize: 50 * 1024 * 1024, // 50MB limit + fileSize: 50 * 1024 * 1024, // 50MB limit per file + files: 500, // Maximum 500 files + // Set a reasonable field size limit to prevent memory issues + fieldSize: 10 * 1024 * 1024, // 10MB for non-file fields }, fileFilter: (req, file, cb) => { // Accept images only with proper validation @@ -85,13 +88,17 @@ const validateUploadContent = createFileUploadValidator({ }); // Upload photos for an event +// Increased limit to 500 files, but recommend chunked uploads for better performance router.post('/:eventId/upload', adminAuth, (req, res, next) => { - upload.array('photos', 20)(req, res, (err) => { + upload.array('photos', 500)(req, res, (err) => { if (err) { console.error('Multer error:', err); if (err instanceof multer.MulterError) { if (err.code === 'LIMIT_FILE_SIZE') { - return res.status(400).json({ error: 'File too large. Maximum size is 50MB.' }); + return res.status(400).json({ error: 'File too large. Maximum size is 50MB per file.' }); + } + if (err.code === 'LIMIT_FILE_COUNT') { + return res.status(400).json({ error: 'Too many files. Maximum 500 files per upload.' }); } return res.status(400).json({ error: `Upload error: ${err.message}` }); } @@ -136,90 +143,122 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => { } const uploadedPhotos = []; + const errors = []; - // Process each uploaded file - for (const file of req.files) { - let trx; + // Process files in batches to optimize database operations + const BATCH_SIZE = 10; // Process 10 files at a time for database operations + + for (let i = 0; i < req.files.length; i += BATCH_SIZE) { + const batch = req.files.slice(i, i + BATCH_SIZE); + + // Start a single transaction for the batch + const trx = await db.transaction(); + try { - // Start transaction for atomic counter update - trx = await db.transaction(); - - // Get and increment the counter for this category - let counter = 1; + // Get initial counter for this batch + let batchCounter = 1; if (category) { - // Lock the category row and get current counter const categoryData = await trx('photo_categories') .where({ id: parsedCategoryId }) .forUpdate() .first(); - - counter = (categoryData.photo_counter || 0) + 1; - - // Update counter - await trx('photo_categories') - .where({ id: parsedCategoryId }) - .update({ photo_counter: counter }); + batchCounter = (categoryData.photo_counter || 0) + 1; } else { - // For uncategorized photos, count existing uncategorized photos const uncategorizedCount = await trx('photos') .where({ event_id: eventId }) .whereNull('category_id') .count('id as count') .first(); - - counter = (uncategorizedCount.count || 0) + 1; + batchCounter = (uncategorizedCount.count || 0) + 1; } - // Generate new filename - const extension = path.extname(file.originalname); - const newFilename = generatePhotoFilename( - event.event_name, - category ? category.name : 'uncategorized', - counter, - extension - ); + const batchPhotos = []; - // Rename the file - const oldPath = file.path; - const newPath = path.join(path.dirname(oldPath), newFilename); - await fs.rename(oldPath, newPath); + for (let fileIndex = 0; fileIndex < batch.length; fileIndex++) { + const file = batch[fileIndex]; + const counter = batchCounter + fileIndex; + + try { + // Generate new filename + const extension = path.extname(file.originalname); + const newFilename = generatePhotoFilename( + event.event_name, + category ? category.name : 'uncategorized', + counter, + extension + ); + + // Rename the file + const oldPath = file.path; + const newPath = path.join(path.dirname(oldPath), newFilename); + await fs.rename(oldPath, newPath); + + // Update file object + file.filename = newFilename; + file.path = newPath; + + // Generate thumbnail with new filename + const thumbnailPath = await generateThumbnail(file.path); + + // Calculate relative paths + const storagePath = getStoragePath(); + const relativePath = path.relative(path.join(storagePath, 'events/active'), file.path); + const relativeThumbPath = thumbnailPath; + + // Prepare photo data for batch insert + batchPhotos.push({ + event_id: eventId, + filename: file.filename, + path: relativePath, + thumbnail_path: relativeThumbPath, + category_id: parsedCategoryId || null, + type: 'individual', + size_bytes: file.size + }); + } catch (error) { + console.error(`Error processing file ${file.originalname}:`, error); + errors.push({ filename: file.originalname, error: error.message }); + // Delete the file if it was partially processed + if (file.path) { + try { await fs.unlink(file.path); } catch (e) {} + } + } + } - // Update file object - file.filename = newFilename; - file.path = newPath; + // Batch insert all photos from this batch + if (batchPhotos.length > 0) { + const insertedIds = await trx('photos').insert(batchPhotos).returning('id'); + + // Update category counter if needed + if (category) { + await trx('photo_categories') + .where({ id: parsedCategoryId }) + .update({ photo_counter: batchCounter + batchPhotos.length - 1 }); + } + + // Add to uploaded photos array + batchPhotos.forEach((photo, index) => { + uploadedPhotos.push({ + id: insertedIds[index]?.id || insertedIds[index], + filename: photo.filename, + size: photo.size_bytes, + category_id: photo.category_id + }); + }); + } - // Generate thumbnail with new filename - const thumbnailPath = await generateThumbnail(file.path); - - // Calculate relative paths - const storagePath = getStoragePath(); - const relativePath = path.relative(path.join(storagePath, 'events/active'), file.path); - const relativeThumbPath = thumbnailPath; // thumbnailPath is already relative to storage root - - // Add to database - const [photoId] = await trx('photos').insert({ - event_id: eventId, - filename: file.filename, - path: relativePath, - thumbnail_path: relativeThumbPath, - category_id: parsedCategoryId || null, - type: 'individual', // Keep for backwards compatibility - size_bytes: file.size - }); - - // Commit transaction + // Commit the batch transaction await trx.commit(); - - uploadedPhotos.push({ - id: photoId, - filename: file.filename, - size: file.size, - category_id: parsedCategoryId || null - }); } catch (error) { - console.error(`Error processing file ${file.filename}:`, error); - if (trx) await trx.rollback(); - // Continue with other files + console.error(`Error processing batch starting at index ${i}:`, error); + await trx.rollback(); + + // Try to clean up files from failed batch + for (const file of batch) { + if (file.path) { + try { await fs.unlink(file.path); } catch (e) {} + } + } } } @@ -230,10 +269,22 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => { { type: 'admin', id: req.admin.id, name: req.admin.username } ); - res.json({ + // Prepare response + const response = { message: `Successfully uploaded ${uploadedPhotos.length} photos`, - photos: uploadedPhotos - }); + photos: uploadedPhotos, + totalFiles: req.files.length, + successCount: uploadedPhotos.length, + failureCount: errors.length + }; + + // Include error details if any files failed + if (errors.length > 0) { + response.errors = errors; + response.message = `Uploaded ${uploadedPhotos.length} of ${req.files.length} photos. ${errors.length} failed.`; + } + + res.json(response); } catch (error) { console.error('Error uploading photos:', error); res.status(500).json({ error: 'Failed to upload photos' }); diff --git a/backend/src/services/imageProcessor.js b/backend/src/services/imageProcessor.js index 7045d32..f27d3ec 100644 --- a/backend/src/services/imageProcessor.js +++ b/backend/src/services/imageProcessor.js @@ -2,6 +2,10 @@ const sharp = require('sharp'); const path = require('path'); const fs = require('fs').promises; +// Configure sharp for better memory management with large batches +sharp.cache(false); // Disable cache to prevent memory buildup +sharp.concurrency(2); // Limit concurrent operations + const THUMBNAIL_WIDTH = 300; const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage'); const getThumbnailPath = () => path.join(getStoragePath(), 'thumbnails'); @@ -15,16 +19,29 @@ async function generateThumbnail(imagePath) { // Ensure thumbnail directory exists await fs.mkdir(thumbnailDir, { recursive: true }); - // Generate thumbnail - await sharp(imagePath) - .resize(THUMBNAIL_WIDTH, null, { - withoutEnlargement: true, - fit: 'inside' + try { + // Generate thumbnail with memory-efficient settings + await sharp(imagePath, { + limitInputPixels: 268402689, // ~16k x 16k max + sequentialRead: true // More memory efficient for large images }) - .jpeg({ quality: 80 }) - .toFile(thumbnailPath); - - return path.relative(getStoragePath(), thumbnailPath); + .resize(THUMBNAIL_WIDTH, null, { + withoutEnlargement: true, + fit: 'inside' + }) + .jpeg({ + quality: 80, + progressive: true, // Progressive JPEG for better loading + mozjpeg: true // Better compression + }) + .toFile(thumbnailPath); + + return path.relative(getStoragePath(), thumbnailPath); + } catch (error) { + console.error(`Failed to generate thumbnail for ${filename}:`, error); + // Return null if thumbnail generation fails, don't fail the whole upload + return null; + } } module.exports = { generateThumbnail }; diff --git a/frontend/src/components/admin/PhotoUpload.tsx b/frontend/src/components/admin/PhotoUpload.tsx index b117357..d82f26a 100644 --- a/frontend/src/components/admin/PhotoUpload.tsx +++ b/frontend/src/components/admin/PhotoUpload.tsx @@ -18,6 +18,8 @@ export const PhotoUpload: React.FC = ({ eventId, onUploadCompl const [isUploading, setIsUploading] = useState(false); const [selectedFiles, setSelectedFiles] = useState([]); const [uploadProgress, setUploadProgress] = useState(0); + const [currentChunk, setCurrentChunk] = useState(0); + const [totalChunks, setTotalChunks] = useState(0); const [selectedCategoryId, setSelectedCategoryId] = useState(null); const fileInputRef = useRef(null); @@ -35,13 +37,13 @@ export const PhotoUpload: React.FC = ({ eventId, onUploadCompl // Check total file count with existing files const totalFiles = selectedFiles.length + imageFiles.length; - if (totalFiles > 20) { - const allowedNewFiles = 20 - selectedFiles.length; + if (totalFiles > 500) { + const allowedNewFiles = 500 - selectedFiles.length; if (allowedNewFiles <= 0) { - toast.error(t('upload.maxFilesReached') || 'Maximum 20 files allowed'); + toast.error(t('upload.maxFilesReached') || 'Maximum 500 files allowed'); return; } - toast.warning(t('upload.someFilesSkipped') || `Only ${allowedNewFiles} more files can be added (20 max)`); + toast.warning(t('upload.someFilesSkipped') || `Only ${allowedNewFiles} more files can be added (500 max)`); setSelectedFiles(prev => [...prev, ...imageFiles.slice(0, allowedNewFiles)]); return; } @@ -57,42 +59,62 @@ export const PhotoUpload: React.FC = ({ eventId, onUploadCompl if (selectedFiles.length === 0) return; // Validate file count - if (selectedFiles.length > 20) { - toast.error(t('upload.tooManyFiles') || 'Maximum 20 files can be uploaded at once'); + if (selectedFiles.length > 500) { + toast.error(t('upload.tooManyFiles') || 'Maximum 500 files can be uploaded at once'); return; } setIsUploading(true); setUploadProgress(0); - const formData = new FormData(); - selectedFiles.forEach((file, index) => { - console.log(`Adding file ${index}: ${file.name}, size: ${file.size}`); - formData.append('photos', file); - }); + // For large uploads, chunk the files to prevent memory issues + const CHUNK_SIZE = 50; // Upload 50 files at a time + const chunks = []; - if (selectedCategoryId) { - formData.append('category_id', selectedCategoryId.toString()); - } - - // Debug: Log FormData contents - console.log('FormData entries:'); - for (let pair of formData.entries()) { - console.log(pair[0], pair[1]); + for (let i = 0; i < selectedFiles.length; i += CHUNK_SIZE) { + chunks.push(selectedFiles.slice(i, i + CHUNK_SIZE)); } + setTotalChunks(chunks.length); + let totalUploaded = 0; + let failedFiles = []; + try { - const response = await api.post(`/admin/events/${eventId}/upload`, formData, { - // Don't set Content-Type header - axios will set it with the boundary - onUploadProgress: (progressEvent) => { - if (progressEvent.total) { - const progress = Math.round((progressEvent.loaded * 100) / progressEvent.total); - setUploadProgress(progress); - } - }, - }); + for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) { + setCurrentChunk(chunkIndex + 1); + const chunk = chunks[chunkIndex]; + const formData = new FormData(); + + chunk.forEach((file) => { + formData.append('photos', file); + }); + + if (selectedCategoryId) { + formData.append('category_id', selectedCategoryId.toString()); + } - console.log('Upload result:', response.data); + try { + const response = await api.post(`/admin/events/${eventId}/upload`, formData, { + onUploadProgress: (progressEvent) => { + if (progressEvent.total) { + // Calculate overall progress across all chunks + const chunkProgress = progressEvent.loaded / progressEvent.total; + const overallProgress = ((chunkIndex + chunkProgress) / chunks.length) * 100; + setUploadProgress(Math.round(overallProgress)); + } + }, + }); + + totalUploaded += chunk.length; + console.log(`Chunk ${chunkIndex + 1}/${chunks.length} uploaded:`, response.data); + } catch (error: any) { + console.error(`Error uploading chunk ${chunkIndex + 1}:`, error); + failedFiles.push(...chunk.map(f => f.name)); + + // Continue with next chunk even if one fails + continue; + } + } // Clear selected files setSelectedFiles([]); @@ -100,8 +122,15 @@ export const PhotoUpload: React.FC = ({ eventId, onUploadCompl fileInputRef.current.value = ''; } - // Show success message - toast.success(t('toast.uploadSuccess')); + // Show appropriate message + if (failedFiles.length === 0) { + toast.success(t('upload.uploadComplete') || `Successfully uploaded ${totalUploaded} files`); + } else { + toast.warning( + t('upload.someFilesFailed') || + `Uploaded ${totalUploaded} files. ${failedFiles.length} files failed.` + ); + } // Call callback if (onUploadComplete) { @@ -113,6 +142,8 @@ export const PhotoUpload: React.FC = ({ eventId, onUploadCompl } finally { setIsUploading(false); setUploadProgress(0); + setCurrentChunk(0); + setTotalChunks(0); } }; @@ -223,7 +254,10 @@ export const PhotoUpload: React.FC = ({ eventId, onUploadCompl {isUploading && (
- {t('upload.uploading')} + + {t('upload.uploading')} + {totalChunks > 1 && ` (${t('common.chunk') || 'Chunk'} ${currentChunk}/${totalChunks})`} + {uploadProgress}%
@@ -232,6 +266,11 @@ export const PhotoUpload: React.FC = ({ eventId, onUploadCompl style={{ width: `${uploadProgress}%` }} />
+ {totalChunks > 1 && ( +

+ {t('upload.uploadingChunks') || `Uploading ${selectedFiles.length} files in ${totalChunks} batches...`} +

+ )}
)} diff --git a/frontend/src/i18n/locales/de.json b/frontend/src/i18n/locales/de.json index ece35b6..0b2b458 100644 --- a/frontend/src/i18n/locales/de.json +++ b/frontend/src/i18n/locales/de.json @@ -47,7 +47,10 @@ "uploadComplete": "Upload abgeschlossen!", "uploadFailed": "Upload fehlgeschlagen", "someFilesFailed": "Einige Dateien konnten nicht hochgeladen werden", - "uploadPhotos": "Fotos hochladen" + "uploadPhotos": "Fotos hochladen", + "maxFilesReached": "Maximal 500 Dateien erlaubt", + "someFilesSkipped": "Einige Dateien wurden übersprungen (500 Dateien Limit)", + "tooManyFiles": "Maximal 500 Dateien können gleichzeitig hochgeladen werden" }, "navigation": { "dashboard": "Dashboard", diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index fc8a0dc..b4edb70 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -48,9 +48,9 @@ "uploadFailed": "Upload failed", "someFilesFailed": "Some files failed to upload", "uploadPhotos": "Upload Photos", - "maxFilesReached": "Maximum 20 files allowed", - "someFilesSkipped": "Some files were skipped (20 file limit)", - "tooManyFiles": "Maximum 20 files can be uploaded at once" + "maxFilesReached": "Maximum 500 files allowed", + "someFilesSkipped": "Some files were skipped (500 file limit)", + "tooManyFiles": "Maximum 500 files can be uploaded at once" }, "navigation": { "dashboard": "Dashboard",