fix: prevent photo corruption during upload and add batch processing improvements
Mirror to GitHub / mirror (push) Successful in 20s
Test and Lint / backend-test (push) Successful in 1m9s
continuous-integration/drone/push Build is passing
Test and Lint / frontend-test (push) Successful in 2m9s
Version and Release / version-bump (push) Successful in 34s
Version and Release / trigger-drone (push) Successful in 3s
Mirror to GitHub / mirror (push) Successful in 20s
Test and Lint / backend-test (push) Successful in 1m9s
continuous-integration/drone/push Build is passing
Test and Lint / frontend-test (push) Successful in 2m9s
Version and Release / version-bump (push) Successful in 34s
Version and Release / trigger-drone (push) Successful in 3s
- Remove async functions from multer callbacks (primary corruption cause) - Implement temp directory upload approach with proper cleanup - Add comprehensive file integrity validation before processing - Fix batch upload category assignment and photo naming - Add automatic cleanup service for orphaned temp uploads - Enhance error handling with better corruption detection - Add fix-temp-photos script to repair existing temporary files - Update file watcher to ignore temp upload files Fixes issues with: - Corrupted photos showing only partial images - Photos retaining temp_ names after upload - Category assignments lost during batch uploads - Incomplete file uploads causing "Premature end of input" errors 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "picpeak-backend",
|
||||
"version": "1.0.62",
|
||||
"version": "1.0.63",
|
||||
"description": "Backend for PicPeak event photo sharing platform",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
@@ -8,6 +8,7 @@
|
||||
"dev": "nodemon server.js",
|
||||
"migrate": "node migrations/run-migrations.js",
|
||||
"migrate:safe": "node migrations/run-migrations-safe.js",
|
||||
"fix-temp-photos": "node scripts/fix-temp-photos.js",
|
||||
"test": "jest",
|
||||
"lint": "eslint src/"
|
||||
},
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
require('dotenv').config({ path: '../.env' });
|
||||
const path = require('path');
|
||||
const fs = require('fs').promises;
|
||||
const { db } = require('../src/database/db');
|
||||
const { generatePhotoFilename } = require('../src/utils/filenameSanitizer');
|
||||
|
||||
async function fixTempPhotos() {
|
||||
console.log('Starting to fix temporary photo files...\n');
|
||||
|
||||
try {
|
||||
// Find all photos with temp_ filenames
|
||||
const tempPhotos = await db('photos')
|
||||
.where('filename', 'like', 'temp_%')
|
||||
.orderBy('event_id', 'asc')
|
||||
.orderBy('category_id', 'asc')
|
||||
.orderBy('id', 'asc');
|
||||
|
||||
console.log(`Found ${tempPhotos.length} photos with temporary filenames\n`);
|
||||
|
||||
if (tempPhotos.length === 0) {
|
||||
console.log('No temporary photos found. Exiting.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Group photos by event and category
|
||||
const grouped = {};
|
||||
for (const photo of tempPhotos) {
|
||||
const key = `${photo.event_id}_${photo.category_id || 'null'}`;
|
||||
if (!grouped[key]) {
|
||||
grouped[key] = [];
|
||||
}
|
||||
grouped[key].push(photo);
|
||||
}
|
||||
|
||||
console.log(`Processing ${Object.keys(grouped).length} event/category groups...\n`);
|
||||
|
||||
// Process each group
|
||||
for (const [key, photos] of Object.entries(grouped)) {
|
||||
const [eventId, categoryIdStr] = key.split('_');
|
||||
const categoryId = categoryIdStr === 'null' ? null : parseInt(categoryIdStr);
|
||||
|
||||
console.log(`\nProcessing Event ID: ${eventId}, Category ID: ${categoryId || 'uncategorized'}`);
|
||||
console.log(`Photos in group: ${photos.length}`);
|
||||
|
||||
// Get event details
|
||||
const event = await db('events').where({ id: eventId }).first();
|
||||
if (!event) {
|
||||
console.error(`Event ${eventId} not found! Skipping...`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get category details if applicable
|
||||
let category = null;
|
||||
let startCounter = 1;
|
||||
|
||||
if (categoryId) {
|
||||
category = await db('photo_categories').where({ id: categoryId }).first();
|
||||
if (!category) {
|
||||
console.error(`Category ${categoryId} not found! Treating as uncategorized...`);
|
||||
} else {
|
||||
// Get the highest counter for this category
|
||||
const maxPhoto = await db('photos')
|
||||
.where({ event_id: eventId, category_id: categoryId })
|
||||
.whereNot('filename', 'like', 'temp_%')
|
||||
.orderBy('id', 'desc')
|
||||
.first();
|
||||
|
||||
if (maxPhoto && maxPhoto.filename) {
|
||||
// Extract counter from filename
|
||||
const match = maxPhoto.filename.match(/_(\d+)\.[^.]+$/);
|
||||
if (match) {
|
||||
startCounter = parseInt(match[1]) + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// For uncategorized, get the highest counter
|
||||
const maxPhoto = await db('photos')
|
||||
.where({ event_id: eventId })
|
||||
.whereNull('category_id')
|
||||
.whereNot('filename', 'like', 'temp_%')
|
||||
.orderBy('id', 'desc')
|
||||
.first();
|
||||
|
||||
if (maxPhoto && maxPhoto.filename) {
|
||||
const match = maxPhoto.filename.match(/_(\d+)\.[^.]+$/);
|
||||
if (match) {
|
||||
startCounter = parseInt(match[1]) + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Starting counter: ${startCounter}`);
|
||||
|
||||
// Process each photo in the group
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (let i = 0; i < photos.length; i++) {
|
||||
const photo = photos[i];
|
||||
const counter = startCounter + i;
|
||||
|
||||
try {
|
||||
// Generate new filename
|
||||
const extension = path.extname(photo.filename);
|
||||
const newFilename = generatePhotoFilename(
|
||||
event.event_name,
|
||||
category ? category.name : 'uncategorized',
|
||||
counter,
|
||||
extension
|
||||
);
|
||||
|
||||
// Build full paths
|
||||
const storagePath = process.env.STORAGE_PATH || path.join(__dirname, '../../storage');
|
||||
const oldPath = path.join(storagePath, 'events/active', photo.path);
|
||||
const newPath = path.join(path.dirname(oldPath), newFilename);
|
||||
|
||||
// Check if old file exists
|
||||
try {
|
||||
await fs.access(oldPath);
|
||||
} catch (e) {
|
||||
console.error(`File not found: ${oldPath}`);
|
||||
errorCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Rename the file
|
||||
await fs.rename(oldPath, newPath);
|
||||
|
||||
// Update database
|
||||
const newRelativePath = path.relative(path.join(storagePath, 'events/active'), newPath);
|
||||
await db('photos')
|
||||
.where({ id: photo.id })
|
||||
.update({
|
||||
filename: newFilename,
|
||||
path: newRelativePath
|
||||
});
|
||||
|
||||
console.log(`✓ Renamed: ${photo.filename} → ${newFilename}`);
|
||||
successCount++;
|
||||
|
||||
} catch (error) {
|
||||
console.error(`✗ Failed to process photo ${photo.id}: ${error.message}`);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Update category counter if needed
|
||||
if (category && successCount > 0) {
|
||||
const newCounter = startCounter + photos.length - 1;
|
||||
await db('photo_categories')
|
||||
.where({ id: categoryId })
|
||||
.update({ photo_counter: newCounter });
|
||||
console.log(`Updated category counter to ${newCounter}`);
|
||||
}
|
||||
|
||||
console.log(`\nGroup summary: ${successCount} successful, ${errorCount} errors`);
|
||||
}
|
||||
|
||||
console.log('\n=== COMPLETE ===');
|
||||
console.log('All temporary photos have been processed.');
|
||||
|
||||
} catch (error) {
|
||||
console.error('Fatal error:', error);
|
||||
} finally {
|
||||
await db.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
// Run the script
|
||||
fixTempPhotos().catch(console.error);
|
||||
@@ -218,6 +218,14 @@ async function startServer() {
|
||||
const { initializeCleanupJob } = require('./src/utils/authSecurity');
|
||||
initializeCleanupJob();
|
||||
|
||||
// Initialize temp upload cleanup job
|
||||
const { cleanupTempUploads } = require('./src/utils/cleanupTempUploads');
|
||||
// Run cleanup on startup
|
||||
cleanupTempUploads();
|
||||
// Schedule periodic cleanup every hour
|
||||
setInterval(cleanupTempUploads, 60 * 60 * 1000);
|
||||
logger.info('Temp upload cleanup scheduled');
|
||||
|
||||
// Start file watcher
|
||||
startFileWatcher();
|
||||
|
||||
|
||||
@@ -20,10 +20,19 @@ async function validateUploadedFile(filePath) {
|
||||
|
||||
if (imageExtensions.includes(ext)) {
|
||||
// Try to read metadata - this will fail if image is corrupted
|
||||
const metadata = await sharp(filePath).metadata();
|
||||
let metadata;
|
||||
try {
|
||||
metadata = await sharp(filePath, {
|
||||
failOnError: false, // Don't fail on recoverable errors
|
||||
limitInputPixels: 268402689 // ~16k x 16k max
|
||||
}).metadata();
|
||||
} catch (metadataError) {
|
||||
// If metadata reading fails, the file is likely incomplete
|
||||
throw new Error(`Invalid image file: ${metadataError.message}`);
|
||||
}
|
||||
|
||||
if (!metadata.width || !metadata.height) {
|
||||
throw new Error('Invalid image dimensions');
|
||||
if (!metadata || !metadata.width || !metadata.height) {
|
||||
throw new Error('Invalid image dimensions - file may be incomplete');
|
||||
}
|
||||
|
||||
// Check for reasonable dimensions
|
||||
@@ -31,6 +40,18 @@ async function validateUploadedFile(filePath) {
|
||||
throw new Error('Image dimensions too small');
|
||||
}
|
||||
|
||||
// Additional check: verify we can actually decode a small portion of the image
|
||||
try {
|
||||
await sharp(filePath, {
|
||||
failOnError: false,
|
||||
limitInputPixels: 268402689
|
||||
})
|
||||
.resize(10, 10) // Try to resize to very small size
|
||||
.toBuffer();
|
||||
} catch (decodeError) {
|
||||
throw new Error(`Image decode failed - file may be corrupted: ${decodeError.message}`);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -14,46 +14,31 @@ const router = express.Router();
|
||||
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
||||
|
||||
// Configure multer for file uploads
|
||||
// IMPORTANT: Using synchronous functions to prevent file corruption
|
||||
const storage = multer.diskStorage({
|
||||
destination: async (req, file, cb) => {
|
||||
destination: (req, file, cb) => {
|
||||
console.log('Multer destination called for file:', file.originalname);
|
||||
const { eventId } = req.params;
|
||||
|
||||
try {
|
||||
// Get event details
|
||||
const event = await db('events').where({ id: eventId }).first();
|
||||
if (!event) {
|
||||
console.error('Event not found in multer destination:', eventId);
|
||||
return cb(new Error('Event not found'));
|
||||
}
|
||||
|
||||
// Store event in request for use in filename generation
|
||||
req.eventData = event;
|
||||
|
||||
// Create destination path - now just event folder, no type subfolder
|
||||
const destPath = path.join(getStoragePath(), 'events/active', event.slug);
|
||||
console.log('Destination path:', destPath);
|
||||
|
||||
// Ensure directory exists
|
||||
await fs.mkdir(destPath, { recursive: true });
|
||||
|
||||
cb(null, destPath);
|
||||
} catch (error) {
|
||||
console.error('Error in multer destination:', error);
|
||||
cb(error);
|
||||
}
|
||||
// We'll validate the event exists in the route handler
|
||||
// For now, just create a temp destination
|
||||
const tempPath = path.join(getStoragePath(), 'temp', `upload_${Date.now()}_${Math.random().toString(36).substring(7)}`);
|
||||
|
||||
// Create directory synchronously
|
||||
require('fs').mkdirSync(tempPath, { recursive: true });
|
||||
console.log('Temp destination path:', tempPath);
|
||||
|
||||
// Store temp path for cleanup
|
||||
req.tempUploadPath = tempPath;
|
||||
|
||||
cb(null, tempPath);
|
||||
},
|
||||
filename: async (req, file, cb) => {
|
||||
filename: (req, file, cb) => {
|
||||
console.log('Multer filename called for file:', file.originalname);
|
||||
try {
|
||||
// Use temporary filename for now, will rename after getting category info
|
||||
const tempName = `temp_${Date.now()}_${Math.round(Math.random() * 1E9)}${path.extname(file.originalname)}`;
|
||||
console.log('Temp filename:', tempName);
|
||||
cb(null, tempName);
|
||||
} catch (error) {
|
||||
console.error('Error in multer filename:', error);
|
||||
cb(error);
|
||||
}
|
||||
// Use a simple temporary filename
|
||||
const tempName = `temp_${Date.now()}_${Math.round(Math.random() * 1E9)}${path.extname(file.originalname)}`;
|
||||
console.log('Temp filename:', tempName);
|
||||
cb(null, tempName);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -66,6 +51,9 @@ const upload = multer({
|
||||
files: 500, // Maximum 500 files
|
||||
// Set a reasonable field size limit to prevent memory issues
|
||||
fieldSize: 10 * 1024 * 1024, // 10MB for non-file fields
|
||||
// Add part size limits to prevent incomplete uploads
|
||||
parts: 10000, // Maximum number of parts (fields + files)
|
||||
headerPairs: 2000 // Maximum number of header key-value pairs
|
||||
},
|
||||
fileFilter: (req, file, cb) => {
|
||||
// Accept images only with proper validation
|
||||
@@ -76,7 +64,9 @@ const upload = multer({
|
||||
} else {
|
||||
cb(new Error('Only JPEG, PNG and WebP images are allowed'));
|
||||
}
|
||||
}
|
||||
},
|
||||
// Add abort on limit to stop processing when limits are exceeded
|
||||
abortOnLimit: true
|
||||
});
|
||||
|
||||
const { createFileUploadValidator } = require('../utils/fileSecurityUtils');
|
||||
@@ -88,9 +78,29 @@ const validateUploadContent = createFileUploadValidator({
|
||||
validateContent: true
|
||||
});
|
||||
|
||||
// Request timeout middleware for uploads
|
||||
const uploadTimeout = (timeout = 300000) => { // 5 minutes default
|
||||
return (req, res, next) => {
|
||||
// Set timeout for the request
|
||||
req.setTimeout(timeout, () => {
|
||||
console.error('Upload request timed out');
|
||||
if (!res.headersSent) {
|
||||
res.status(408).json({ error: 'Upload request timed out' });
|
||||
}
|
||||
});
|
||||
|
||||
// Set response timeout as well
|
||||
res.setTimeout(timeout, () => {
|
||||
console.error('Upload response timed out');
|
||||
});
|
||||
|
||||
next();
|
||||
};
|
||||
};
|
||||
|
||||
// Upload photos for an event
|
||||
// Increased limit to 500 files, but recommend chunked uploads for better performance
|
||||
router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
||||
router.post('/:eventId/upload', adminAuth, uploadTimeout(600000), (req, res, next) => { // 10 minute timeout
|
||||
upload.array('photos', 500)(req, res, (err) => {
|
||||
if (err) {
|
||||
console.error('Multer error:', err);
|
||||
@@ -122,12 +132,28 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
||||
const event = await db('events').where({ id: eventId }).first();
|
||||
if (!event) {
|
||||
console.error('Event not found:', eventId);
|
||||
// Clean up temp files
|
||||
if (req.tempUploadPath) {
|
||||
try {
|
||||
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
|
||||
} catch (e) {
|
||||
console.error('Failed to clean up temp path:', e);
|
||||
}
|
||||
}
|
||||
return res.status(404).json({ error: 'Event not found' });
|
||||
}
|
||||
|
||||
if (!req.files || req.files.length === 0) {
|
||||
console.error('No files in request. req.files:', req.files);
|
||||
console.error('Request body keys:', Object.keys(req.body));
|
||||
// Clean up temp files
|
||||
if (req.tempUploadPath) {
|
||||
try {
|
||||
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
|
||||
} catch (e) {
|
||||
console.error('Failed to clean up temp path:', e);
|
||||
}
|
||||
}
|
||||
return res.status(400).json({ error: 'No files uploaded' });
|
||||
}
|
||||
|
||||
@@ -139,15 +165,27 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
||||
if (parsedCategoryId) {
|
||||
category = await db('photo_categories').where({ id: parsedCategoryId }).first();
|
||||
if (!category) {
|
||||
// Clean up temp files
|
||||
if (req.tempUploadPath) {
|
||||
try {
|
||||
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
|
||||
} catch (e) {
|
||||
console.error('Failed to clean up temp path:', e);
|
||||
}
|
||||
}
|
||||
return res.status(400).json({ error: 'Invalid category' });
|
||||
}
|
||||
}
|
||||
|
||||
// Create final destination directory
|
||||
const finalDestPath = path.join(getStoragePath(), 'events/active', event.slug);
|
||||
await fs.mkdir(finalDestPath, { recursive: true });
|
||||
|
||||
const uploadedPhotos = [];
|
||||
const errors = [];
|
||||
|
||||
// Process files in batches to optimize database operations
|
||||
const BATCH_SIZE = 10; // Process 10 files at a time for database operations
|
||||
const BATCH_SIZE = 25; // Increased batch size for better performance with large uploads
|
||||
|
||||
for (let i = 0; i < req.files.length; i += BATCH_SIZE) {
|
||||
const batch = req.files.slice(i, i + BATCH_SIZE);
|
||||
@@ -170,16 +208,25 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
||||
.whereNull('category_id')
|
||||
.count('id as count')
|
||||
.first();
|
||||
batchCounter = (uncategorizedCount.count || 0) + 1;
|
||||
batchCounter = (parseInt(uncategorizedCount.count) || 0) + 1;
|
||||
}
|
||||
|
||||
const batchPhotos = [];
|
||||
const fileRenameOperations = []; // Store rename operations to do after commit
|
||||
|
||||
// First pass: prepare data and move files from temp to final location
|
||||
for (let fileIndex = 0; fileIndex < batch.length; fileIndex++) {
|
||||
const file = batch[fileIndex];
|
||||
const counter = batchCounter + fileIndex;
|
||||
const tempPath = file.path; // Original temp path
|
||||
|
||||
try {
|
||||
// Verify file is complete before processing
|
||||
const tempStats = await fs.stat(tempPath);
|
||||
if (tempStats.size === 0) {
|
||||
throw new Error('File is empty - upload may have been interrupted');
|
||||
}
|
||||
|
||||
// Generate new filename
|
||||
const extension = path.extname(file.originalname);
|
||||
const newFilename = generatePhotoFilename(
|
||||
@@ -189,83 +236,142 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
||||
extension
|
||||
);
|
||||
|
||||
// Rename the file
|
||||
const oldPath = file.path;
|
||||
const newPath = path.join(path.dirname(oldPath), newFilename);
|
||||
await fs.rename(oldPath, newPath);
|
||||
|
||||
// Update file object
|
||||
file.filename = newFilename;
|
||||
file.path = newPath;
|
||||
|
||||
// Generate thumbnail with new filename (with better error handling)
|
||||
let thumbnailPath = null;
|
||||
try {
|
||||
thumbnailPath = await generateThumbnail(file.path);
|
||||
} catch (thumbError) {
|
||||
console.error(`Thumbnail generation failed for ${file.filename}:`, thumbError.message);
|
||||
// Continue without thumbnail rather than failing the whole upload
|
||||
}
|
||||
|
||||
// Calculate relative paths
|
||||
// Calculate final path
|
||||
const finalPath = path.join(finalDestPath, newFilename);
|
||||
const storagePath = getStoragePath();
|
||||
const relativePath = path.relative(path.join(storagePath, 'events/active'), file.path);
|
||||
const relativeThumbPath = thumbnailPath;
|
||||
const relativePath = path.relative(path.join(storagePath, 'events/active'), finalPath);
|
||||
|
||||
// Prepare photo data for batch insert
|
||||
batchPhotos.push({
|
||||
event_id: eventId,
|
||||
filename: file.filename,
|
||||
const photoData = {
|
||||
event_id: parseInt(eventId),
|
||||
filename: newFilename,
|
||||
path: relativePath,
|
||||
thumbnail_path: relativeThumbPath,
|
||||
category_id: parsedCategoryId || null,
|
||||
thumbnail_path: null, // Will generate after successful commit
|
||||
category_id: parsedCategoryId ? parseInt(parsedCategoryId) : null,
|
||||
type: 'individual',
|
||||
size_bytes: file.size
|
||||
size_bytes: tempStats.size // Use actual file size from stat
|
||||
};
|
||||
|
||||
batchPhotos.push(photoData);
|
||||
|
||||
// Store move operation for later
|
||||
fileRenameOperations.push({
|
||||
tempPath: tempPath,
|
||||
finalPath: finalPath,
|
||||
filename: newFilename,
|
||||
photoData: photoData
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`Error processing file ${file.originalname}:`, error);
|
||||
console.error(`Error preparing file ${file.originalname}:`, error);
|
||||
errors.push({ filename: file.originalname, error: error.message });
|
||||
// Delete the file if it was partially processed
|
||||
if (file.path) {
|
||||
try { await fs.unlink(file.path); } catch (e) {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Batch insert all photos from this batch
|
||||
// Insert all photos in this batch
|
||||
if (batchPhotos.length > 0) {
|
||||
console.log(`Inserting batch of ${batchPhotos.length} photos with category_id: ${parsedCategoryId}`);
|
||||
|
||||
const insertedIds = await trx('photos').insert(batchPhotos).returning('id');
|
||||
|
||||
// Update category counter if needed
|
||||
if (category) {
|
||||
if (category && parsedCategoryId) {
|
||||
const newCounter = batchCounter + batchPhotos.length - 1;
|
||||
await trx('photo_categories')
|
||||
.where({ id: parsedCategoryId })
|
||||
.update({ photo_counter: batchCounter + batchPhotos.length - 1 });
|
||||
.update({ photo_counter: newCounter });
|
||||
console.log(`Updated category ${parsedCategoryId} counter to ${newCounter}`);
|
||||
}
|
||||
|
||||
// Add to uploaded photos array
|
||||
batchPhotos.forEach((photo, index) => {
|
||||
uploadedPhotos.push({
|
||||
id: insertedIds[index]?.id || insertedIds[index],
|
||||
filename: photo.filename,
|
||||
size: photo.size_bytes,
|
||||
category_id: photo.category_id
|
||||
});
|
||||
});
|
||||
// Commit the transaction first
|
||||
await trx.commit();
|
||||
console.log(`Successfully committed batch of ${batchPhotos.length} photos`);
|
||||
|
||||
// Now move files from temp to final location after successful commit
|
||||
for (let idx = 0; idx < fileRenameOperations.length; idx++) {
|
||||
const operation = fileRenameOperations[idx];
|
||||
try {
|
||||
// Move the file from temp to final location
|
||||
await fs.rename(operation.tempPath, operation.finalPath);
|
||||
console.log(`Moved file from ${operation.tempPath} to ${operation.finalPath}`);
|
||||
|
||||
// Verify the file was moved successfully
|
||||
const finalStats = await fs.stat(operation.finalPath);
|
||||
if (finalStats.size !== operation.photoData.size_bytes) {
|
||||
throw new Error(`File size mismatch after move: expected ${operation.photoData.size_bytes}, got ${finalStats.size}`);
|
||||
}
|
||||
|
||||
// Generate thumbnail with final path
|
||||
let thumbnailPath = null;
|
||||
try {
|
||||
thumbnailPath = await generateThumbnail(operation.finalPath);
|
||||
|
||||
// Update the database with thumbnail path
|
||||
if (thumbnailPath && insertedIds[idx]) {
|
||||
const photoId = insertedIds[idx]?.id || insertedIds[idx];
|
||||
await db('photos')
|
||||
.where({ id: photoId })
|
||||
.update({ thumbnail_path: thumbnailPath });
|
||||
}
|
||||
} catch (thumbError) {
|
||||
console.error(`Thumbnail generation failed for ${operation.filename}:`, thumbError.message);
|
||||
}
|
||||
|
||||
// Add to successful uploads
|
||||
uploadedPhotos.push({
|
||||
id: insertedIds[idx]?.id || insertedIds[idx],
|
||||
filename: operation.filename,
|
||||
size: operation.photoData.size_bytes,
|
||||
category_id: operation.photoData.category_id
|
||||
});
|
||||
} catch (moveError) {
|
||||
console.error(`Failed to move file ${operation.tempPath} to ${operation.finalPath}:`, moveError);
|
||||
errors.push({
|
||||
filename: operation.filename,
|
||||
error: `File move failed: ${moveError.message}`
|
||||
});
|
||||
|
||||
// Try to clean up the database entry if file move failed
|
||||
if (insertedIds[idx]) {
|
||||
const photoId = insertedIds[idx]?.id || insertedIds[idx];
|
||||
try {
|
||||
await db('photos').where({ id: photoId }).delete();
|
||||
console.log(`Cleaned up database entry for failed photo ${photoId}`);
|
||||
} catch (cleanupError) {
|
||||
console.error(`Failed to clean up database entry:`, cleanupError);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No photos to insert, just rollback
|
||||
await trx.rollback();
|
||||
}
|
||||
|
||||
// Commit the batch transaction
|
||||
await trx.commit();
|
||||
} catch (error) {
|
||||
console.error(`Error processing batch starting at index ${i}:`, error);
|
||||
await trx.rollback();
|
||||
console.error('Stack trace:', error.stack);
|
||||
|
||||
// Try to clean up files from failed batch
|
||||
for (const file of batch) {
|
||||
if (file.path) {
|
||||
try { await fs.unlink(file.path); } catch (e) {}
|
||||
}
|
||||
// Rollback if not already committed
|
||||
if (!trx.isCompleted()) {
|
||||
await trx.rollback();
|
||||
}
|
||||
|
||||
// Add all files in this batch to errors
|
||||
for (const file of batch) {
|
||||
errors.push({
|
||||
filename: file.originalname,
|
||||
error: `Batch processing failed: ${error.message}`
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up temp upload directory
|
||||
if (req.tempUploadPath) {
|
||||
try {
|
||||
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
|
||||
console.log(`Cleaned up temp upload directory: ${req.tempUploadPath}`);
|
||||
} catch (e) {
|
||||
console.error('Failed to clean up temp upload directory:', e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -298,6 +404,17 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
console.error('Error uploading photos:', error);
|
||||
|
||||
// Clean up temp upload directory on error
|
||||
if (req.tempUploadPath) {
|
||||
try {
|
||||
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
|
||||
console.log(`Cleaned up temp upload directory after error: ${req.tempUploadPath}`);
|
||||
} catch (e) {
|
||||
console.error('Failed to clean up temp upload directory:', e);
|
||||
}
|
||||
}
|
||||
|
||||
res.status(500).json({ error: 'Failed to upload photos' });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -51,6 +51,13 @@ async function processNewPhoto(filePath) {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
if (!['.jpg', '.jpeg', '.png', '.webp'].includes(ext)) return;
|
||||
|
||||
// Skip temporary upload files
|
||||
const filename = path.basename(filePath);
|
||||
if (filename.startsWith('temp_')) {
|
||||
logger.debug(`Skipping temporary upload file: ${filename}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Find the event
|
||||
const event = await db('events').where({ slug: eventSlug, is_active: formatBoolean(true) }).first();
|
||||
if (!event) return;
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
const path = require('path');
|
||||
const fs = require('fs').promises;
|
||||
const logger = require('./logger');
|
||||
|
||||
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
||||
|
||||
/**
|
||||
* Clean up old temporary upload directories
|
||||
* Removes temp directories older than 1 hour
|
||||
*/
|
||||
async function cleanupTempUploads() {
|
||||
const tempPath = path.join(getStoragePath(), 'temp');
|
||||
|
||||
try {
|
||||
// Ensure temp directory exists
|
||||
await fs.mkdir(tempPath, { recursive: true });
|
||||
|
||||
// Read all items in temp directory
|
||||
const items = await fs.readdir(tempPath);
|
||||
|
||||
let cleanedCount = 0;
|
||||
const oneHourAgo = Date.now() - (60 * 60 * 1000); // 1 hour
|
||||
|
||||
for (const item of items) {
|
||||
const itemPath = path.join(tempPath, item);
|
||||
|
||||
try {
|
||||
const stats = await fs.stat(itemPath);
|
||||
|
||||
// Only process directories that match our upload pattern
|
||||
if (stats.isDirectory() && item.startsWith('upload_')) {
|
||||
// Extract timestamp from directory name
|
||||
const parts = item.split('_');
|
||||
if (parts.length >= 2) {
|
||||
const timestamp = parseInt(parts[1]);
|
||||
|
||||
// Remove if older than 1 hour
|
||||
if (!isNaN(timestamp) && timestamp < oneHourAgo) {
|
||||
logger.info(`Cleaning up old temp upload directory: ${item}`);
|
||||
await fs.rm(itemPath, { recursive: true, force: true });
|
||||
cleanedCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Error processing temp item ${item}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
if (cleanedCount > 0) {
|
||||
logger.info(`Cleaned up ${cleanedCount} old temp upload directories`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Error during temp upload cleanup:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start periodic cleanup of temp uploads
|
||||
* Runs every hour
|
||||
*/
|
||||
function startTempUploadCleanup() {
|
||||
// Run immediately on startup
|
||||
cleanupTempUploads();
|
||||
|
||||
// Then run every hour
|
||||
setInterval(() => {
|
||||
cleanupTempUploads();
|
||||
}, 60 * 60 * 1000); // 1 hour
|
||||
|
||||
logger.info('Temp upload cleanup service started');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
cleanupTempUploads,
|
||||
startTempUploadCleanup
|
||||
};
|
||||
Reference in New Issue
Block a user