fix: prevent photo corruption during upload and add batch processing improvements
Mirror to GitHub / mirror (push) Successful in 20s
Test and Lint / backend-test (push) Successful in 1m9s
continuous-integration/drone/push Build is passing
Test and Lint / frontend-test (push) Successful in 2m9s
Version and Release / version-bump (push) Successful in 34s
Version and Release / trigger-drone (push) Successful in 3s

- Remove async functions from multer callbacks (primary corruption cause)
- Implement temp directory upload approach with proper cleanup
- Add comprehensive file integrity validation before processing
- Fix batch upload category assignment and photo naming
- Add automatic cleanup service for orphaned temp uploads
- Enhance error handling with better corruption detection
- Add fix-temp-photos script to repair existing temporary files
- Update file watcher to ignore temp upload files

Fixes issues with:
- Corrupted photos showing only partial images
- Photos retaining temp_ names after upload
- Category assignments lost during batch uploads
- Incomplete file uploads causing "Premature end of input" errors

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-07-17 10:02:23 +02:00
parent a896fa66c0
commit e636cf5d56
7 changed files with 498 additions and 95 deletions
+2 -1
View File
@@ -1,6 +1,6 @@
{
"name": "picpeak-backend",
"version": "1.0.62",
"version": "1.0.63",
"description": "Backend for PicPeak event photo sharing platform",
"main": "server.js",
"scripts": {
@@ -8,6 +8,7 @@
"dev": "nodemon server.js",
"migrate": "node migrations/run-migrations.js",
"migrate:safe": "node migrations/run-migrations-safe.js",
"fix-temp-photos": "node scripts/fix-temp-photos.js",
"test": "jest",
"lint": "eslint src/"
},
+171
View File
@@ -0,0 +1,171 @@
require('dotenv').config({ path: '../.env' });
const path = require('path');
const fs = require('fs').promises;
const { db } = require('../src/database/db');
const { generatePhotoFilename } = require('../src/utils/filenameSanitizer');
async function fixTempPhotos() {
console.log('Starting to fix temporary photo files...\n');
try {
// Find all photos with temp_ filenames
const tempPhotos = await db('photos')
.where('filename', 'like', 'temp_%')
.orderBy('event_id', 'asc')
.orderBy('category_id', 'asc')
.orderBy('id', 'asc');
console.log(`Found ${tempPhotos.length} photos with temporary filenames\n`);
if (tempPhotos.length === 0) {
console.log('No temporary photos found. Exiting.');
return;
}
// Group photos by event and category
const grouped = {};
for (const photo of tempPhotos) {
const key = `${photo.event_id}_${photo.category_id || 'null'}`;
if (!grouped[key]) {
grouped[key] = [];
}
grouped[key].push(photo);
}
console.log(`Processing ${Object.keys(grouped).length} event/category groups...\n`);
// Process each group
for (const [key, photos] of Object.entries(grouped)) {
const [eventId, categoryIdStr] = key.split('_');
const categoryId = categoryIdStr === 'null' ? null : parseInt(categoryIdStr);
console.log(`\nProcessing Event ID: ${eventId}, Category ID: ${categoryId || 'uncategorized'}`);
console.log(`Photos in group: ${photos.length}`);
// Get event details
const event = await db('events').where({ id: eventId }).first();
if (!event) {
console.error(`Event ${eventId} not found! Skipping...`);
continue;
}
// Get category details if applicable
let category = null;
let startCounter = 1;
if (categoryId) {
category = await db('photo_categories').where({ id: categoryId }).first();
if (!category) {
console.error(`Category ${categoryId} not found! Treating as uncategorized...`);
} else {
// Get the highest counter for this category
const maxPhoto = await db('photos')
.where({ event_id: eventId, category_id: categoryId })
.whereNot('filename', 'like', 'temp_%')
.orderBy('id', 'desc')
.first();
if (maxPhoto && maxPhoto.filename) {
// Extract counter from filename
const match = maxPhoto.filename.match(/_(\d+)\.[^.]+$/);
if (match) {
startCounter = parseInt(match[1]) + 1;
}
}
}
} else {
// For uncategorized, get the highest counter
const maxPhoto = await db('photos')
.where({ event_id: eventId })
.whereNull('category_id')
.whereNot('filename', 'like', 'temp_%')
.orderBy('id', 'desc')
.first();
if (maxPhoto && maxPhoto.filename) {
const match = maxPhoto.filename.match(/_(\d+)\.[^.]+$/);
if (match) {
startCounter = parseInt(match[1]) + 1;
}
}
}
console.log(`Starting counter: ${startCounter}`);
// Process each photo in the group
let successCount = 0;
let errorCount = 0;
for (let i = 0; i < photos.length; i++) {
const photo = photos[i];
const counter = startCounter + i;
try {
// Generate new filename
const extension = path.extname(photo.filename);
const newFilename = generatePhotoFilename(
event.event_name,
category ? category.name : 'uncategorized',
counter,
extension
);
// Build full paths
const storagePath = process.env.STORAGE_PATH || path.join(__dirname, '../../storage');
const oldPath = path.join(storagePath, 'events/active', photo.path);
const newPath = path.join(path.dirname(oldPath), newFilename);
// Check if old file exists
try {
await fs.access(oldPath);
} catch (e) {
console.error(`File not found: ${oldPath}`);
errorCount++;
continue;
}
// Rename the file
await fs.rename(oldPath, newPath);
// Update database
const newRelativePath = path.relative(path.join(storagePath, 'events/active'), newPath);
await db('photos')
.where({ id: photo.id })
.update({
filename: newFilename,
path: newRelativePath
});
console.log(`✓ Renamed: ${photo.filename}${newFilename}`);
successCount++;
} catch (error) {
console.error(`✗ Failed to process photo ${photo.id}: ${error.message}`);
errorCount++;
}
}
// Update category counter if needed
if (category && successCount > 0) {
const newCounter = startCounter + photos.length - 1;
await db('photo_categories')
.where({ id: categoryId })
.update({ photo_counter: newCounter });
console.log(`Updated category counter to ${newCounter}`);
}
console.log(`\nGroup summary: ${successCount} successful, ${errorCount} errors`);
}
console.log('\n=== COMPLETE ===');
console.log('All temporary photos have been processed.');
} catch (error) {
console.error('Fatal error:', error);
} finally {
await db.destroy();
}
}
// Run the script
fixTempPhotos().catch(console.error);
+8
View File
@@ -218,6 +218,14 @@ async function startServer() {
const { initializeCleanupJob } = require('./src/utils/authSecurity');
initializeCleanupJob();
// Initialize temp upload cleanup job
const { cleanupTempUploads } = require('./src/utils/cleanupTempUploads');
// Run cleanup on startup
cleanupTempUploads();
// Schedule periodic cleanup every hour
setInterval(cleanupTempUploads, 60 * 60 * 1000);
logger.info('Temp upload cleanup scheduled');
// Start file watcher
startFileWatcher();
+24 -3
View File
@@ -20,10 +20,19 @@ async function validateUploadedFile(filePath) {
if (imageExtensions.includes(ext)) {
// Try to read metadata - this will fail if image is corrupted
const metadata = await sharp(filePath).metadata();
let metadata;
try {
metadata = await sharp(filePath, {
failOnError: false, // Don't fail on recoverable errors
limitInputPixels: 268402689 // ~16k x 16k max
}).metadata();
} catch (metadataError) {
// If metadata reading fails, the file is likely incomplete
throw new Error(`Invalid image file: ${metadataError.message}`);
}
if (!metadata.width || !metadata.height) {
throw new Error('Invalid image dimensions');
if (!metadata || !metadata.width || !metadata.height) {
throw new Error('Invalid image dimensions - file may be incomplete');
}
// Check for reasonable dimensions
@@ -31,6 +40,18 @@ async function validateUploadedFile(filePath) {
throw new Error('Image dimensions too small');
}
// Additional check: verify we can actually decode a small portion of the image
try {
await sharp(filePath, {
failOnError: false,
limitInputPixels: 268402689
})
.resize(10, 10) // Try to resize to very small size
.toBuffer();
} catch (decodeError) {
throw new Error(`Image decode failed - file may be corrupted: ${decodeError.message}`);
}
return true;
}
+208 -91
View File
@@ -14,46 +14,31 @@ const router = express.Router();
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
// Configure multer for file uploads
// IMPORTANT: Using synchronous functions to prevent file corruption
const storage = multer.diskStorage({
destination: async (req, file, cb) => {
destination: (req, file, cb) => {
console.log('Multer destination called for file:', file.originalname);
const { eventId } = req.params;
try {
// Get event details
const event = await db('events').where({ id: eventId }).first();
if (!event) {
console.error('Event not found in multer destination:', eventId);
return cb(new Error('Event not found'));
}
// Store event in request for use in filename generation
req.eventData = event;
// Create destination path - now just event folder, no type subfolder
const destPath = path.join(getStoragePath(), 'events/active', event.slug);
console.log('Destination path:', destPath);
// Ensure directory exists
await fs.mkdir(destPath, { recursive: true });
cb(null, destPath);
} catch (error) {
console.error('Error in multer destination:', error);
cb(error);
}
// We'll validate the event exists in the route handler
// For now, just create a temp destination
const tempPath = path.join(getStoragePath(), 'temp', `upload_${Date.now()}_${Math.random().toString(36).substring(7)}`);
// Create directory synchronously
require('fs').mkdirSync(tempPath, { recursive: true });
console.log('Temp destination path:', tempPath);
// Store temp path for cleanup
req.tempUploadPath = tempPath;
cb(null, tempPath);
},
filename: async (req, file, cb) => {
filename: (req, file, cb) => {
console.log('Multer filename called for file:', file.originalname);
try {
// Use temporary filename for now, will rename after getting category info
const tempName = `temp_${Date.now()}_${Math.round(Math.random() * 1E9)}${path.extname(file.originalname)}`;
console.log('Temp filename:', tempName);
cb(null, tempName);
} catch (error) {
console.error('Error in multer filename:', error);
cb(error);
}
// Use a simple temporary filename
const tempName = `temp_${Date.now()}_${Math.round(Math.random() * 1E9)}${path.extname(file.originalname)}`;
console.log('Temp filename:', tempName);
cb(null, tempName);
}
});
@@ -66,6 +51,9 @@ const upload = multer({
files: 500, // Maximum 500 files
// Set a reasonable field size limit to prevent memory issues
fieldSize: 10 * 1024 * 1024, // 10MB for non-file fields
// Add part size limits to prevent incomplete uploads
parts: 10000, // Maximum number of parts (fields + files)
headerPairs: 2000 // Maximum number of header key-value pairs
},
fileFilter: (req, file, cb) => {
// Accept images only with proper validation
@@ -76,7 +64,9 @@ const upload = multer({
} else {
cb(new Error('Only JPEG, PNG and WebP images are allowed'));
}
}
},
// Add abort on limit to stop processing when limits are exceeded
abortOnLimit: true
});
const { createFileUploadValidator } = require('../utils/fileSecurityUtils');
@@ -88,9 +78,29 @@ const validateUploadContent = createFileUploadValidator({
validateContent: true
});
// Request timeout middleware for uploads
const uploadTimeout = (timeout = 300000) => { // 5 minutes default
return (req, res, next) => {
// Set timeout for the request
req.setTimeout(timeout, () => {
console.error('Upload request timed out');
if (!res.headersSent) {
res.status(408).json({ error: 'Upload request timed out' });
}
});
// Set response timeout as well
res.setTimeout(timeout, () => {
console.error('Upload response timed out');
});
next();
};
};
// Upload photos for an event
// Increased limit to 500 files, but recommend chunked uploads for better performance
router.post('/:eventId/upload', adminAuth, (req, res, next) => {
router.post('/:eventId/upload', adminAuth, uploadTimeout(600000), (req, res, next) => { // 10 minute timeout
upload.array('photos', 500)(req, res, (err) => {
if (err) {
console.error('Multer error:', err);
@@ -122,12 +132,28 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
const event = await db('events').where({ id: eventId }).first();
if (!event) {
console.error('Event not found:', eventId);
// Clean up temp files
if (req.tempUploadPath) {
try {
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
} catch (e) {
console.error('Failed to clean up temp path:', e);
}
}
return res.status(404).json({ error: 'Event not found' });
}
if (!req.files || req.files.length === 0) {
console.error('No files in request. req.files:', req.files);
console.error('Request body keys:', Object.keys(req.body));
// Clean up temp files
if (req.tempUploadPath) {
try {
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
} catch (e) {
console.error('Failed to clean up temp path:', e);
}
}
return res.status(400).json({ error: 'No files uploaded' });
}
@@ -139,15 +165,27 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
if (parsedCategoryId) {
category = await db('photo_categories').where({ id: parsedCategoryId }).first();
if (!category) {
// Clean up temp files
if (req.tempUploadPath) {
try {
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
} catch (e) {
console.error('Failed to clean up temp path:', e);
}
}
return res.status(400).json({ error: 'Invalid category' });
}
}
// Create final destination directory
const finalDestPath = path.join(getStoragePath(), 'events/active', event.slug);
await fs.mkdir(finalDestPath, { recursive: true });
const uploadedPhotos = [];
const errors = [];
// Process files in batches to optimize database operations
const BATCH_SIZE = 10; // Process 10 files at a time for database operations
const BATCH_SIZE = 25; // Increased batch size for better performance with large uploads
for (let i = 0; i < req.files.length; i += BATCH_SIZE) {
const batch = req.files.slice(i, i + BATCH_SIZE);
@@ -170,16 +208,25 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
.whereNull('category_id')
.count('id as count')
.first();
batchCounter = (uncategorizedCount.count || 0) + 1;
batchCounter = (parseInt(uncategorizedCount.count) || 0) + 1;
}
const batchPhotos = [];
const fileRenameOperations = []; // Store rename operations to do after commit
// First pass: prepare data and move files from temp to final location
for (let fileIndex = 0; fileIndex < batch.length; fileIndex++) {
const file = batch[fileIndex];
const counter = batchCounter + fileIndex;
const tempPath = file.path; // Original temp path
try {
// Verify file is complete before processing
const tempStats = await fs.stat(tempPath);
if (tempStats.size === 0) {
throw new Error('File is empty - upload may have been interrupted');
}
// Generate new filename
const extension = path.extname(file.originalname);
const newFilename = generatePhotoFilename(
@@ -189,83 +236,142 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
extension
);
// Rename the file
const oldPath = file.path;
const newPath = path.join(path.dirname(oldPath), newFilename);
await fs.rename(oldPath, newPath);
// Update file object
file.filename = newFilename;
file.path = newPath;
// Generate thumbnail with new filename (with better error handling)
let thumbnailPath = null;
try {
thumbnailPath = await generateThumbnail(file.path);
} catch (thumbError) {
console.error(`Thumbnail generation failed for ${file.filename}:`, thumbError.message);
// Continue without thumbnail rather than failing the whole upload
}
// Calculate relative paths
// Calculate final path
const finalPath = path.join(finalDestPath, newFilename);
const storagePath = getStoragePath();
const relativePath = path.relative(path.join(storagePath, 'events/active'), file.path);
const relativeThumbPath = thumbnailPath;
const relativePath = path.relative(path.join(storagePath, 'events/active'), finalPath);
// Prepare photo data for batch insert
batchPhotos.push({
event_id: eventId,
filename: file.filename,
const photoData = {
event_id: parseInt(eventId),
filename: newFilename,
path: relativePath,
thumbnail_path: relativeThumbPath,
category_id: parsedCategoryId || null,
thumbnail_path: null, // Will generate after successful commit
category_id: parsedCategoryId ? parseInt(parsedCategoryId) : null,
type: 'individual',
size_bytes: file.size
size_bytes: tempStats.size // Use actual file size from stat
};
batchPhotos.push(photoData);
// Store move operation for later
fileRenameOperations.push({
tempPath: tempPath,
finalPath: finalPath,
filename: newFilename,
photoData: photoData
});
} catch (error) {
console.error(`Error processing file ${file.originalname}:`, error);
console.error(`Error preparing file ${file.originalname}:`, error);
errors.push({ filename: file.originalname, error: error.message });
// Delete the file if it was partially processed
if (file.path) {
try { await fs.unlink(file.path); } catch (e) {}
}
}
}
// Batch insert all photos from this batch
// Insert all photos in this batch
if (batchPhotos.length > 0) {
console.log(`Inserting batch of ${batchPhotos.length} photos with category_id: ${parsedCategoryId}`);
const insertedIds = await trx('photos').insert(batchPhotos).returning('id');
// Update category counter if needed
if (category) {
if (category && parsedCategoryId) {
const newCounter = batchCounter + batchPhotos.length - 1;
await trx('photo_categories')
.where({ id: parsedCategoryId })
.update({ photo_counter: batchCounter + batchPhotos.length - 1 });
.update({ photo_counter: newCounter });
console.log(`Updated category ${parsedCategoryId} counter to ${newCounter}`);
}
// Add to uploaded photos array
batchPhotos.forEach((photo, index) => {
uploadedPhotos.push({
id: insertedIds[index]?.id || insertedIds[index],
filename: photo.filename,
size: photo.size_bytes,
category_id: photo.category_id
});
});
// Commit the transaction first
await trx.commit();
console.log(`Successfully committed batch of ${batchPhotos.length} photos`);
// Now move files from temp to final location after successful commit
for (let idx = 0; idx < fileRenameOperations.length; idx++) {
const operation = fileRenameOperations[idx];
try {
// Move the file from temp to final location
await fs.rename(operation.tempPath, operation.finalPath);
console.log(`Moved file from ${operation.tempPath} to ${operation.finalPath}`);
// Verify the file was moved successfully
const finalStats = await fs.stat(operation.finalPath);
if (finalStats.size !== operation.photoData.size_bytes) {
throw new Error(`File size mismatch after move: expected ${operation.photoData.size_bytes}, got ${finalStats.size}`);
}
// Generate thumbnail with final path
let thumbnailPath = null;
try {
thumbnailPath = await generateThumbnail(operation.finalPath);
// Update the database with thumbnail path
if (thumbnailPath && insertedIds[idx]) {
const photoId = insertedIds[idx]?.id || insertedIds[idx];
await db('photos')
.where({ id: photoId })
.update({ thumbnail_path: thumbnailPath });
}
} catch (thumbError) {
console.error(`Thumbnail generation failed for ${operation.filename}:`, thumbError.message);
}
// Add to successful uploads
uploadedPhotos.push({
id: insertedIds[idx]?.id || insertedIds[idx],
filename: operation.filename,
size: operation.photoData.size_bytes,
category_id: operation.photoData.category_id
});
} catch (moveError) {
console.error(`Failed to move file ${operation.tempPath} to ${operation.finalPath}:`, moveError);
errors.push({
filename: operation.filename,
error: `File move failed: ${moveError.message}`
});
// Try to clean up the database entry if file move failed
if (insertedIds[idx]) {
const photoId = insertedIds[idx]?.id || insertedIds[idx];
try {
await db('photos').where({ id: photoId }).delete();
console.log(`Cleaned up database entry for failed photo ${photoId}`);
} catch (cleanupError) {
console.error(`Failed to clean up database entry:`, cleanupError);
}
}
}
}
} else {
// No photos to insert, just rollback
await trx.rollback();
}
// Commit the batch transaction
await trx.commit();
} catch (error) {
console.error(`Error processing batch starting at index ${i}:`, error);
await trx.rollback();
console.error('Stack trace:', error.stack);
// Try to clean up files from failed batch
for (const file of batch) {
if (file.path) {
try { await fs.unlink(file.path); } catch (e) {}
}
// Rollback if not already committed
if (!trx.isCompleted()) {
await trx.rollback();
}
// Add all files in this batch to errors
for (const file of batch) {
errors.push({
filename: file.originalname,
error: `Batch processing failed: ${error.message}`
});
}
}
}
// Clean up temp upload directory
if (req.tempUploadPath) {
try {
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
console.log(`Cleaned up temp upload directory: ${req.tempUploadPath}`);
} catch (e) {
console.error('Failed to clean up temp upload directory:', e);
}
}
@@ -298,6 +404,17 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
res.json(response);
} catch (error) {
console.error('Error uploading photos:', error);
// Clean up temp upload directory on error
if (req.tempUploadPath) {
try {
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
console.log(`Cleaned up temp upload directory after error: ${req.tempUploadPath}`);
} catch (e) {
console.error('Failed to clean up temp upload directory:', e);
}
}
res.status(500).json({ error: 'Failed to upload photos' });
}
});
+7
View File
@@ -51,6 +51,13 @@ async function processNewPhoto(filePath) {
const ext = path.extname(filePath).toLowerCase();
if (!['.jpg', '.jpeg', '.png', '.webp'].includes(ext)) return;
// Skip temporary upload files
const filename = path.basename(filePath);
if (filename.startsWith('temp_')) {
logger.debug(`Skipping temporary upload file: ${filename}`);
return;
}
// Find the event
const event = await db('events').where({ slug: eventSlug, is_active: formatBoolean(true) }).first();
if (!event) return;
+78
View File
@@ -0,0 +1,78 @@
const path = require('path');
const fs = require('fs').promises;
const logger = require('./logger');
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
/**
* Clean up old temporary upload directories
* Removes temp directories older than 1 hour
*/
async function cleanupTempUploads() {
const tempPath = path.join(getStoragePath(), 'temp');
try {
// Ensure temp directory exists
await fs.mkdir(tempPath, { recursive: true });
// Read all items in temp directory
const items = await fs.readdir(tempPath);
let cleanedCount = 0;
const oneHourAgo = Date.now() - (60 * 60 * 1000); // 1 hour
for (const item of items) {
const itemPath = path.join(tempPath, item);
try {
const stats = await fs.stat(itemPath);
// Only process directories that match our upload pattern
if (stats.isDirectory() && item.startsWith('upload_')) {
// Extract timestamp from directory name
const parts = item.split('_');
if (parts.length >= 2) {
const timestamp = parseInt(parts[1]);
// Remove if older than 1 hour
if (!isNaN(timestamp) && timestamp < oneHourAgo) {
logger.info(`Cleaning up old temp upload directory: ${item}`);
await fs.rm(itemPath, { recursive: true, force: true });
cleanedCount++;
}
}
}
} catch (error) {
logger.error(`Error processing temp item ${item}:`, error.message);
}
}
if (cleanedCount > 0) {
logger.info(`Cleaned up ${cleanedCount} old temp upload directories`);
}
} catch (error) {
logger.error('Error during temp upload cleanup:', error);
}
}
/**
* Start periodic cleanup of temp uploads
* Runs every hour
*/
function startTempUploadCleanup() {
// Run immediately on startup
cleanupTempUploads();
// Then run every hour
setInterval(() => {
cleanupTempUploads();
}, 60 * 60 * 1000); // 1 hour
logger.info('Temp upload cleanup service started');
}
module.exports = {
cleanupTempUploads,
startTempUploadCleanup
};