feat: increase file upload limit from 20 to 500 with performance optimizations
Test and Lint / backend-test (push) Successful in 1m10s
Test and Lint / frontend-test (push) Successful in 2m22s
continuous-integration/drone/push Build is passing
Version and Release / version-bump (push) Successful in 37s
Version and Release / trigger-drone (push) Successful in 3s

Backend changes:
- Update multer configuration to accept up to 500 files per upload
- Implement batch processing (10 files per transaction) for better performance
- Add memory-efficient Sharp configuration for thumbnail generation
- Increase Express body parser limits to handle large payloads
- Add proper error handling and reporting for partial upload failures

Frontend changes:
- Update validation to allow 500 files maximum
- Implement chunked uploads (50 files per chunk) to prevent timeouts
- Add progress tracking with chunk information display
- Update error messages and translations (EN/DE)

Performance optimizations:
- Disable Sharp cache to prevent memory buildup
- Limit Sharp concurrency to 2 operations
- Use sequential read for large images
- Process files in database transaction batches
- Return detailed upload results including success/failure counts

This implementation ensures the application can handle large photo uploads
efficiently without running into memory or timeout issues.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-07-14 21:11:59 +02:00
parent ac48bfdd0d
commit 6906c8bcf7
6 changed files with 228 additions and 118 deletions
+71 -32
View File
@@ -18,6 +18,8 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
const [isUploading, setIsUploading] = useState(false);
const [selectedFiles, setSelectedFiles] = useState<File[]>([]);
const [uploadProgress, setUploadProgress] = useState(0);
const [currentChunk, setCurrentChunk] = useState(0);
const [totalChunks, setTotalChunks] = useState(0);
const [selectedCategoryId, setSelectedCategoryId] = useState<number | null>(null);
const fileInputRef = useRef<HTMLInputElement>(null);
@@ -35,13 +37,13 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
// Check total file count with existing files
const totalFiles = selectedFiles.length + imageFiles.length;
if (totalFiles > 20) {
const allowedNewFiles = 20 - selectedFiles.length;
if (totalFiles > 500) {
const allowedNewFiles = 500 - selectedFiles.length;
if (allowedNewFiles <= 0) {
toast.error(t('upload.maxFilesReached') || 'Maximum 20 files allowed');
toast.error(t('upload.maxFilesReached') || 'Maximum 500 files allowed');
return;
}
toast.warning(t('upload.someFilesSkipped') || `Only ${allowedNewFiles} more files can be added (20 max)`);
toast.warning(t('upload.someFilesSkipped') || `Only ${allowedNewFiles} more files can be added (500 max)`);
setSelectedFiles(prev => [...prev, ...imageFiles.slice(0, allowedNewFiles)]);
return;
}
@@ -57,42 +59,62 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
if (selectedFiles.length === 0) return;
// Validate file count
if (selectedFiles.length > 20) {
toast.error(t('upload.tooManyFiles') || 'Maximum 20 files can be uploaded at once');
if (selectedFiles.length > 500) {
toast.error(t('upload.tooManyFiles') || 'Maximum 500 files can be uploaded at once');
return;
}
setIsUploading(true);
setUploadProgress(0);
const formData = new FormData();
selectedFiles.forEach((file, index) => {
console.log(`Adding file ${index}: ${file.name}, size: ${file.size}`);
formData.append('photos', file);
});
// For large uploads, chunk the files to prevent memory issues
const CHUNK_SIZE = 50; // Upload 50 files at a time
const chunks = [];
if (selectedCategoryId) {
formData.append('category_id', selectedCategoryId.toString());
}
// Debug: Log FormData contents
console.log('FormData entries:');
for (let pair of formData.entries()) {
console.log(pair[0], pair[1]);
for (let i = 0; i < selectedFiles.length; i += CHUNK_SIZE) {
chunks.push(selectedFiles.slice(i, i + CHUNK_SIZE));
}
setTotalChunks(chunks.length);
let totalUploaded = 0;
let failedFiles = [];
try {
const response = await api.post(`/admin/events/${eventId}/upload`, formData, {
// Don't set Content-Type header - axios will set it with the boundary
onUploadProgress: (progressEvent) => {
if (progressEvent.total) {
const progress = Math.round((progressEvent.loaded * 100) / progressEvent.total);
setUploadProgress(progress);
}
},
});
for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
setCurrentChunk(chunkIndex + 1);
const chunk = chunks[chunkIndex];
const formData = new FormData();
chunk.forEach((file) => {
formData.append('photos', file);
});
if (selectedCategoryId) {
formData.append('category_id', selectedCategoryId.toString());
}
console.log('Upload result:', response.data);
try {
const response = await api.post(`/admin/events/${eventId}/upload`, formData, {
onUploadProgress: (progressEvent) => {
if (progressEvent.total) {
// Calculate overall progress across all chunks
const chunkProgress = progressEvent.loaded / progressEvent.total;
const overallProgress = ((chunkIndex + chunkProgress) / chunks.length) * 100;
setUploadProgress(Math.round(overallProgress));
}
},
});
totalUploaded += chunk.length;
console.log(`Chunk ${chunkIndex + 1}/${chunks.length} uploaded:`, response.data);
} catch (error: any) {
console.error(`Error uploading chunk ${chunkIndex + 1}:`, error);
failedFiles.push(...chunk.map(f => f.name));
// Continue with next chunk even if one fails
continue;
}
}
// Clear selected files
setSelectedFiles([]);
@@ -100,8 +122,15 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
fileInputRef.current.value = '';
}
// Show success message
toast.success(t('toast.uploadSuccess'));
// Show appropriate message
if (failedFiles.length === 0) {
toast.success(t('upload.uploadComplete') || `Successfully uploaded ${totalUploaded} files`);
} else {
toast.warning(
t('upload.someFilesFailed') ||
`Uploaded ${totalUploaded} files. ${failedFiles.length} files failed.`
);
}
// Call callback
if (onUploadComplete) {
@@ -113,6 +142,8 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
} finally {
setIsUploading(false);
setUploadProgress(0);
setCurrentChunk(0);
setTotalChunks(0);
}
};
@@ -223,7 +254,10 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
{isUploading && (
<div className="mt-4">
<div className="flex justify-between text-sm text-neutral-600 mb-1">
<span>{t('upload.uploading')}</span>
<span>
{t('upload.uploading')}
{totalChunks > 1 && ` (${t('common.chunk') || 'Chunk'} ${currentChunk}/${totalChunks})`}
</span>
<span>{uploadProgress}%</span>
</div>
<div className="w-full bg-neutral-200 rounded-full h-2">
@@ -232,6 +266,11 @@ export const PhotoUpload: React.FC<PhotoUploadProps> = ({ eventId, onUploadCompl
style={{ width: `${uploadProgress}%` }}
/>
</div>
{totalChunks > 1 && (
<p className="text-xs text-neutral-500 mt-1">
{t('upload.uploadingChunks') || `Uploading ${selectedFiles.length} files in ${totalChunks} batches...`}
</p>
)}
</div>
)}
</div>
+4 -1
View File
@@ -47,7 +47,10 @@
"uploadComplete": "Upload abgeschlossen!",
"uploadFailed": "Upload fehlgeschlagen",
"someFilesFailed": "Einige Dateien konnten nicht hochgeladen werden",
"uploadPhotos": "Fotos hochladen"
"uploadPhotos": "Fotos hochladen",
"maxFilesReached": "Maximal 500 Dateien erlaubt",
"someFilesSkipped": "Einige Dateien wurden übersprungen (500 Dateien Limit)",
"tooManyFiles": "Maximal 500 Dateien können gleichzeitig hochgeladen werden"
},
"navigation": {
"dashboard": "Dashboard",
+3 -3
View File
@@ -48,9 +48,9 @@
"uploadFailed": "Upload failed",
"someFilesFailed": "Some files failed to upload",
"uploadPhotos": "Upload Photos",
"maxFilesReached": "Maximum 20 files allowed",
"someFilesSkipped": "Some files were skipped (20 file limit)",
"tooManyFiles": "Maximum 20 files can be uploaded at once"
"maxFilesReached": "Maximum 500 files allowed",
"someFilesSkipped": "Some files were skipped (500 file limit)",
"tooManyFiles": "Maximum 500 files can be uploaded at once"
},
"navigation": {
"dashboard": "Dashboard",