fix: prevent photo corruption during upload and add batch processing improvements
Mirror to GitHub / mirror (push) Successful in 20s
Test and Lint / backend-test (push) Successful in 1m9s
continuous-integration/drone/push Build is passing
Test and Lint / frontend-test (push) Successful in 2m9s
Version and Release / version-bump (push) Successful in 34s
Version and Release / trigger-drone (push) Successful in 3s
Mirror to GitHub / mirror (push) Successful in 20s
Test and Lint / backend-test (push) Successful in 1m9s
continuous-integration/drone/push Build is passing
Test and Lint / frontend-test (push) Successful in 2m9s
Version and Release / version-bump (push) Successful in 34s
Version and Release / trigger-drone (push) Successful in 3s
- Remove async functions from multer callbacks (primary corruption cause) - Implement temp directory upload approach with proper cleanup - Add comprehensive file integrity validation before processing - Fix batch upload category assignment and photo naming - Add automatic cleanup service for orphaned temp uploads - Enhance error handling with better corruption detection - Add fix-temp-photos script to repair existing temporary files - Update file watcher to ignore temp upload files Fixes issues with: - Corrupted photos showing only partial images - Photos retaining temp_ names after upload - Category assignments lost during batch uploads - Incomplete file uploads causing "Premature end of input" errors 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "picpeak-backend",
|
"name": "picpeak-backend",
|
||||||
"version": "1.0.62",
|
"version": "1.0.63",
|
||||||
"description": "Backend for PicPeak event photo sharing platform",
|
"description": "Backend for PicPeak event photo sharing platform",
|
||||||
"main": "server.js",
|
"main": "server.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
@@ -8,6 +8,7 @@
|
|||||||
"dev": "nodemon server.js",
|
"dev": "nodemon server.js",
|
||||||
"migrate": "node migrations/run-migrations.js",
|
"migrate": "node migrations/run-migrations.js",
|
||||||
"migrate:safe": "node migrations/run-migrations-safe.js",
|
"migrate:safe": "node migrations/run-migrations-safe.js",
|
||||||
|
"fix-temp-photos": "node scripts/fix-temp-photos.js",
|
||||||
"test": "jest",
|
"test": "jest",
|
||||||
"lint": "eslint src/"
|
"lint": "eslint src/"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -0,0 +1,171 @@
|
|||||||
|
require('dotenv').config({ path: '../.env' });
|
||||||
|
const path = require('path');
|
||||||
|
const fs = require('fs').promises;
|
||||||
|
const { db } = require('../src/database/db');
|
||||||
|
const { generatePhotoFilename } = require('../src/utils/filenameSanitizer');
|
||||||
|
|
||||||
|
async function fixTempPhotos() {
|
||||||
|
console.log('Starting to fix temporary photo files...\n');
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Find all photos with temp_ filenames
|
||||||
|
const tempPhotos = await db('photos')
|
||||||
|
.where('filename', 'like', 'temp_%')
|
||||||
|
.orderBy('event_id', 'asc')
|
||||||
|
.orderBy('category_id', 'asc')
|
||||||
|
.orderBy('id', 'asc');
|
||||||
|
|
||||||
|
console.log(`Found ${tempPhotos.length} photos with temporary filenames\n`);
|
||||||
|
|
||||||
|
if (tempPhotos.length === 0) {
|
||||||
|
console.log('No temporary photos found. Exiting.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Group photos by event and category
|
||||||
|
const grouped = {};
|
||||||
|
for (const photo of tempPhotos) {
|
||||||
|
const key = `${photo.event_id}_${photo.category_id || 'null'}`;
|
||||||
|
if (!grouped[key]) {
|
||||||
|
grouped[key] = [];
|
||||||
|
}
|
||||||
|
grouped[key].push(photo);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Processing ${Object.keys(grouped).length} event/category groups...\n`);
|
||||||
|
|
||||||
|
// Process each group
|
||||||
|
for (const [key, photos] of Object.entries(grouped)) {
|
||||||
|
const [eventId, categoryIdStr] = key.split('_');
|
||||||
|
const categoryId = categoryIdStr === 'null' ? null : parseInt(categoryIdStr);
|
||||||
|
|
||||||
|
console.log(`\nProcessing Event ID: ${eventId}, Category ID: ${categoryId || 'uncategorized'}`);
|
||||||
|
console.log(`Photos in group: ${photos.length}`);
|
||||||
|
|
||||||
|
// Get event details
|
||||||
|
const event = await db('events').where({ id: eventId }).first();
|
||||||
|
if (!event) {
|
||||||
|
console.error(`Event ${eventId} not found! Skipping...`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get category details if applicable
|
||||||
|
let category = null;
|
||||||
|
let startCounter = 1;
|
||||||
|
|
||||||
|
if (categoryId) {
|
||||||
|
category = await db('photo_categories').where({ id: categoryId }).first();
|
||||||
|
if (!category) {
|
||||||
|
console.error(`Category ${categoryId} not found! Treating as uncategorized...`);
|
||||||
|
} else {
|
||||||
|
// Get the highest counter for this category
|
||||||
|
const maxPhoto = await db('photos')
|
||||||
|
.where({ event_id: eventId, category_id: categoryId })
|
||||||
|
.whereNot('filename', 'like', 'temp_%')
|
||||||
|
.orderBy('id', 'desc')
|
||||||
|
.first();
|
||||||
|
|
||||||
|
if (maxPhoto && maxPhoto.filename) {
|
||||||
|
// Extract counter from filename
|
||||||
|
const match = maxPhoto.filename.match(/_(\d+)\.[^.]+$/);
|
||||||
|
if (match) {
|
||||||
|
startCounter = parseInt(match[1]) + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// For uncategorized, get the highest counter
|
||||||
|
const maxPhoto = await db('photos')
|
||||||
|
.where({ event_id: eventId })
|
||||||
|
.whereNull('category_id')
|
||||||
|
.whereNot('filename', 'like', 'temp_%')
|
||||||
|
.orderBy('id', 'desc')
|
||||||
|
.first();
|
||||||
|
|
||||||
|
if (maxPhoto && maxPhoto.filename) {
|
||||||
|
const match = maxPhoto.filename.match(/_(\d+)\.[^.]+$/);
|
||||||
|
if (match) {
|
||||||
|
startCounter = parseInt(match[1]) + 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`Starting counter: ${startCounter}`);
|
||||||
|
|
||||||
|
// Process each photo in the group
|
||||||
|
let successCount = 0;
|
||||||
|
let errorCount = 0;
|
||||||
|
|
||||||
|
for (let i = 0; i < photos.length; i++) {
|
||||||
|
const photo = photos[i];
|
||||||
|
const counter = startCounter + i;
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Generate new filename
|
||||||
|
const extension = path.extname(photo.filename);
|
||||||
|
const newFilename = generatePhotoFilename(
|
||||||
|
event.event_name,
|
||||||
|
category ? category.name : 'uncategorized',
|
||||||
|
counter,
|
||||||
|
extension
|
||||||
|
);
|
||||||
|
|
||||||
|
// Build full paths
|
||||||
|
const storagePath = process.env.STORAGE_PATH || path.join(__dirname, '../../storage');
|
||||||
|
const oldPath = path.join(storagePath, 'events/active', photo.path);
|
||||||
|
const newPath = path.join(path.dirname(oldPath), newFilename);
|
||||||
|
|
||||||
|
// Check if old file exists
|
||||||
|
try {
|
||||||
|
await fs.access(oldPath);
|
||||||
|
} catch (e) {
|
||||||
|
console.error(`File not found: ${oldPath}`);
|
||||||
|
errorCount++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rename the file
|
||||||
|
await fs.rename(oldPath, newPath);
|
||||||
|
|
||||||
|
// Update database
|
||||||
|
const newRelativePath = path.relative(path.join(storagePath, 'events/active'), newPath);
|
||||||
|
await db('photos')
|
||||||
|
.where({ id: photo.id })
|
||||||
|
.update({
|
||||||
|
filename: newFilename,
|
||||||
|
path: newRelativePath
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`✓ Renamed: ${photo.filename} → ${newFilename}`);
|
||||||
|
successCount++;
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`✗ Failed to process photo ${photo.id}: ${error.message}`);
|
||||||
|
errorCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update category counter if needed
|
||||||
|
if (category && successCount > 0) {
|
||||||
|
const newCounter = startCounter + photos.length - 1;
|
||||||
|
await db('photo_categories')
|
||||||
|
.where({ id: categoryId })
|
||||||
|
.update({ photo_counter: newCounter });
|
||||||
|
console.log(`Updated category counter to ${newCounter}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`\nGroup summary: ${successCount} successful, ${errorCount} errors`);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('\n=== COMPLETE ===');
|
||||||
|
console.log('All temporary photos have been processed.');
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Fatal error:', error);
|
||||||
|
} finally {
|
||||||
|
await db.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the script
|
||||||
|
fixTempPhotos().catch(console.error);
|
||||||
@@ -218,6 +218,14 @@ async function startServer() {
|
|||||||
const { initializeCleanupJob } = require('./src/utils/authSecurity');
|
const { initializeCleanupJob } = require('./src/utils/authSecurity');
|
||||||
initializeCleanupJob();
|
initializeCleanupJob();
|
||||||
|
|
||||||
|
// Initialize temp upload cleanup job
|
||||||
|
const { cleanupTempUploads } = require('./src/utils/cleanupTempUploads');
|
||||||
|
// Run cleanup on startup
|
||||||
|
cleanupTempUploads();
|
||||||
|
// Schedule periodic cleanup every hour
|
||||||
|
setInterval(cleanupTempUploads, 60 * 60 * 1000);
|
||||||
|
logger.info('Temp upload cleanup scheduled');
|
||||||
|
|
||||||
// Start file watcher
|
// Start file watcher
|
||||||
startFileWatcher();
|
startFileWatcher();
|
||||||
|
|
||||||
|
|||||||
@@ -20,10 +20,19 @@ async function validateUploadedFile(filePath) {
|
|||||||
|
|
||||||
if (imageExtensions.includes(ext)) {
|
if (imageExtensions.includes(ext)) {
|
||||||
// Try to read metadata - this will fail if image is corrupted
|
// Try to read metadata - this will fail if image is corrupted
|
||||||
const metadata = await sharp(filePath).metadata();
|
let metadata;
|
||||||
|
try {
|
||||||
|
metadata = await sharp(filePath, {
|
||||||
|
failOnError: false, // Don't fail on recoverable errors
|
||||||
|
limitInputPixels: 268402689 // ~16k x 16k max
|
||||||
|
}).metadata();
|
||||||
|
} catch (metadataError) {
|
||||||
|
// If metadata reading fails, the file is likely incomplete
|
||||||
|
throw new Error(`Invalid image file: ${metadataError.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
if (!metadata.width || !metadata.height) {
|
if (!metadata || !metadata.width || !metadata.height) {
|
||||||
throw new Error('Invalid image dimensions');
|
throw new Error('Invalid image dimensions - file may be incomplete');
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for reasonable dimensions
|
// Check for reasonable dimensions
|
||||||
@@ -31,6 +40,18 @@ async function validateUploadedFile(filePath) {
|
|||||||
throw new Error('Image dimensions too small');
|
throw new Error('Image dimensions too small');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Additional check: verify we can actually decode a small portion of the image
|
||||||
|
try {
|
||||||
|
await sharp(filePath, {
|
||||||
|
failOnError: false,
|
||||||
|
limitInputPixels: 268402689
|
||||||
|
})
|
||||||
|
.resize(10, 10) // Try to resize to very small size
|
||||||
|
.toBuffer();
|
||||||
|
} catch (decodeError) {
|
||||||
|
throw new Error(`Image decode failed - file may be corrupted: ${decodeError.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -14,46 +14,31 @@ const router = express.Router();
|
|||||||
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
||||||
|
|
||||||
// Configure multer for file uploads
|
// Configure multer for file uploads
|
||||||
|
// IMPORTANT: Using synchronous functions to prevent file corruption
|
||||||
const storage = multer.diskStorage({
|
const storage = multer.diskStorage({
|
||||||
destination: async (req, file, cb) => {
|
destination: (req, file, cb) => {
|
||||||
console.log('Multer destination called for file:', file.originalname);
|
console.log('Multer destination called for file:', file.originalname);
|
||||||
const { eventId } = req.params;
|
const { eventId } = req.params;
|
||||||
|
|
||||||
try {
|
// We'll validate the event exists in the route handler
|
||||||
// Get event details
|
// For now, just create a temp destination
|
||||||
const event = await db('events').where({ id: eventId }).first();
|
const tempPath = path.join(getStoragePath(), 'temp', `upload_${Date.now()}_${Math.random().toString(36).substring(7)}`);
|
||||||
if (!event) {
|
|
||||||
console.error('Event not found in multer destination:', eventId);
|
// Create directory synchronously
|
||||||
return cb(new Error('Event not found'));
|
require('fs').mkdirSync(tempPath, { recursive: true });
|
||||||
}
|
console.log('Temp destination path:', tempPath);
|
||||||
|
|
||||||
// Store event in request for use in filename generation
|
// Store temp path for cleanup
|
||||||
req.eventData = event;
|
req.tempUploadPath = tempPath;
|
||||||
|
|
||||||
// Create destination path - now just event folder, no type subfolder
|
cb(null, tempPath);
|
||||||
const destPath = path.join(getStoragePath(), 'events/active', event.slug);
|
|
||||||
console.log('Destination path:', destPath);
|
|
||||||
|
|
||||||
// Ensure directory exists
|
|
||||||
await fs.mkdir(destPath, { recursive: true });
|
|
||||||
|
|
||||||
cb(null, destPath);
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error in multer destination:', error);
|
|
||||||
cb(error);
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
filename: async (req, file, cb) => {
|
filename: (req, file, cb) => {
|
||||||
console.log('Multer filename called for file:', file.originalname);
|
console.log('Multer filename called for file:', file.originalname);
|
||||||
try {
|
// Use a simple temporary filename
|
||||||
// Use temporary filename for now, will rename after getting category info
|
const tempName = `temp_${Date.now()}_${Math.round(Math.random() * 1E9)}${path.extname(file.originalname)}`;
|
||||||
const tempName = `temp_${Date.now()}_${Math.round(Math.random() * 1E9)}${path.extname(file.originalname)}`;
|
console.log('Temp filename:', tempName);
|
||||||
console.log('Temp filename:', tempName);
|
cb(null, tempName);
|
||||||
cb(null, tempName);
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error in multer filename:', error);
|
|
||||||
cb(error);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -66,6 +51,9 @@ const upload = multer({
|
|||||||
files: 500, // Maximum 500 files
|
files: 500, // Maximum 500 files
|
||||||
// Set a reasonable field size limit to prevent memory issues
|
// Set a reasonable field size limit to prevent memory issues
|
||||||
fieldSize: 10 * 1024 * 1024, // 10MB for non-file fields
|
fieldSize: 10 * 1024 * 1024, // 10MB for non-file fields
|
||||||
|
// Add part size limits to prevent incomplete uploads
|
||||||
|
parts: 10000, // Maximum number of parts (fields + files)
|
||||||
|
headerPairs: 2000 // Maximum number of header key-value pairs
|
||||||
},
|
},
|
||||||
fileFilter: (req, file, cb) => {
|
fileFilter: (req, file, cb) => {
|
||||||
// Accept images only with proper validation
|
// Accept images only with proper validation
|
||||||
@@ -76,7 +64,9 @@ const upload = multer({
|
|||||||
} else {
|
} else {
|
||||||
cb(new Error('Only JPEG, PNG and WebP images are allowed'));
|
cb(new Error('Only JPEG, PNG and WebP images are allowed'));
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
|
// Add abort on limit to stop processing when limits are exceeded
|
||||||
|
abortOnLimit: true
|
||||||
});
|
});
|
||||||
|
|
||||||
const { createFileUploadValidator } = require('../utils/fileSecurityUtils');
|
const { createFileUploadValidator } = require('../utils/fileSecurityUtils');
|
||||||
@@ -88,9 +78,29 @@ const validateUploadContent = createFileUploadValidator({
|
|||||||
validateContent: true
|
validateContent: true
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Request timeout middleware for uploads
|
||||||
|
const uploadTimeout = (timeout = 300000) => { // 5 minutes default
|
||||||
|
return (req, res, next) => {
|
||||||
|
// Set timeout for the request
|
||||||
|
req.setTimeout(timeout, () => {
|
||||||
|
console.error('Upload request timed out');
|
||||||
|
if (!res.headersSent) {
|
||||||
|
res.status(408).json({ error: 'Upload request timed out' });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Set response timeout as well
|
||||||
|
res.setTimeout(timeout, () => {
|
||||||
|
console.error('Upload response timed out');
|
||||||
|
});
|
||||||
|
|
||||||
|
next();
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
// Upload photos for an event
|
// Upload photos for an event
|
||||||
// Increased limit to 500 files, but recommend chunked uploads for better performance
|
// Increased limit to 500 files, but recommend chunked uploads for better performance
|
||||||
router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
router.post('/:eventId/upload', adminAuth, uploadTimeout(600000), (req, res, next) => { // 10 minute timeout
|
||||||
upload.array('photos', 500)(req, res, (err) => {
|
upload.array('photos', 500)(req, res, (err) => {
|
||||||
if (err) {
|
if (err) {
|
||||||
console.error('Multer error:', err);
|
console.error('Multer error:', err);
|
||||||
@@ -122,12 +132,28 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
|||||||
const event = await db('events').where({ id: eventId }).first();
|
const event = await db('events').where({ id: eventId }).first();
|
||||||
if (!event) {
|
if (!event) {
|
||||||
console.error('Event not found:', eventId);
|
console.error('Event not found:', eventId);
|
||||||
|
// Clean up temp files
|
||||||
|
if (req.tempUploadPath) {
|
||||||
|
try {
|
||||||
|
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Failed to clean up temp path:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
return res.status(404).json({ error: 'Event not found' });
|
return res.status(404).json({ error: 'Event not found' });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!req.files || req.files.length === 0) {
|
if (!req.files || req.files.length === 0) {
|
||||||
console.error('No files in request. req.files:', req.files);
|
console.error('No files in request. req.files:', req.files);
|
||||||
console.error('Request body keys:', Object.keys(req.body));
|
console.error('Request body keys:', Object.keys(req.body));
|
||||||
|
// Clean up temp files
|
||||||
|
if (req.tempUploadPath) {
|
||||||
|
try {
|
||||||
|
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Failed to clean up temp path:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
return res.status(400).json({ error: 'No files uploaded' });
|
return res.status(400).json({ error: 'No files uploaded' });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -139,15 +165,27 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
|||||||
if (parsedCategoryId) {
|
if (parsedCategoryId) {
|
||||||
category = await db('photo_categories').where({ id: parsedCategoryId }).first();
|
category = await db('photo_categories').where({ id: parsedCategoryId }).first();
|
||||||
if (!category) {
|
if (!category) {
|
||||||
|
// Clean up temp files
|
||||||
|
if (req.tempUploadPath) {
|
||||||
|
try {
|
||||||
|
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Failed to clean up temp path:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
return res.status(400).json({ error: 'Invalid category' });
|
return res.status(400).json({ error: 'Invalid category' });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Create final destination directory
|
||||||
|
const finalDestPath = path.join(getStoragePath(), 'events/active', event.slug);
|
||||||
|
await fs.mkdir(finalDestPath, { recursive: true });
|
||||||
|
|
||||||
const uploadedPhotos = [];
|
const uploadedPhotos = [];
|
||||||
const errors = [];
|
const errors = [];
|
||||||
|
|
||||||
// Process files in batches to optimize database operations
|
// Process files in batches to optimize database operations
|
||||||
const BATCH_SIZE = 10; // Process 10 files at a time for database operations
|
const BATCH_SIZE = 25; // Increased batch size for better performance with large uploads
|
||||||
|
|
||||||
for (let i = 0; i < req.files.length; i += BATCH_SIZE) {
|
for (let i = 0; i < req.files.length; i += BATCH_SIZE) {
|
||||||
const batch = req.files.slice(i, i + BATCH_SIZE);
|
const batch = req.files.slice(i, i + BATCH_SIZE);
|
||||||
@@ -170,16 +208,25 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
|||||||
.whereNull('category_id')
|
.whereNull('category_id')
|
||||||
.count('id as count')
|
.count('id as count')
|
||||||
.first();
|
.first();
|
||||||
batchCounter = (uncategorizedCount.count || 0) + 1;
|
batchCounter = (parseInt(uncategorizedCount.count) || 0) + 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
const batchPhotos = [];
|
const batchPhotos = [];
|
||||||
|
const fileRenameOperations = []; // Store rename operations to do after commit
|
||||||
|
|
||||||
|
// First pass: prepare data and move files from temp to final location
|
||||||
for (let fileIndex = 0; fileIndex < batch.length; fileIndex++) {
|
for (let fileIndex = 0; fileIndex < batch.length; fileIndex++) {
|
||||||
const file = batch[fileIndex];
|
const file = batch[fileIndex];
|
||||||
const counter = batchCounter + fileIndex;
|
const counter = batchCounter + fileIndex;
|
||||||
|
const tempPath = file.path; // Original temp path
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
// Verify file is complete before processing
|
||||||
|
const tempStats = await fs.stat(tempPath);
|
||||||
|
if (tempStats.size === 0) {
|
||||||
|
throw new Error('File is empty - upload may have been interrupted');
|
||||||
|
}
|
||||||
|
|
||||||
// Generate new filename
|
// Generate new filename
|
||||||
const extension = path.extname(file.originalname);
|
const extension = path.extname(file.originalname);
|
||||||
const newFilename = generatePhotoFilename(
|
const newFilename = generatePhotoFilename(
|
||||||
@@ -189,83 +236,142 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
|||||||
extension
|
extension
|
||||||
);
|
);
|
||||||
|
|
||||||
// Rename the file
|
// Calculate final path
|
||||||
const oldPath = file.path;
|
const finalPath = path.join(finalDestPath, newFilename);
|
||||||
const newPath = path.join(path.dirname(oldPath), newFilename);
|
|
||||||
await fs.rename(oldPath, newPath);
|
|
||||||
|
|
||||||
// Update file object
|
|
||||||
file.filename = newFilename;
|
|
||||||
file.path = newPath;
|
|
||||||
|
|
||||||
// Generate thumbnail with new filename (with better error handling)
|
|
||||||
let thumbnailPath = null;
|
|
||||||
try {
|
|
||||||
thumbnailPath = await generateThumbnail(file.path);
|
|
||||||
} catch (thumbError) {
|
|
||||||
console.error(`Thumbnail generation failed for ${file.filename}:`, thumbError.message);
|
|
||||||
// Continue without thumbnail rather than failing the whole upload
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calculate relative paths
|
|
||||||
const storagePath = getStoragePath();
|
const storagePath = getStoragePath();
|
||||||
const relativePath = path.relative(path.join(storagePath, 'events/active'), file.path);
|
const relativePath = path.relative(path.join(storagePath, 'events/active'), finalPath);
|
||||||
const relativeThumbPath = thumbnailPath;
|
|
||||||
|
|
||||||
// Prepare photo data for batch insert
|
// Prepare photo data for batch insert
|
||||||
batchPhotos.push({
|
const photoData = {
|
||||||
event_id: eventId,
|
event_id: parseInt(eventId),
|
||||||
filename: file.filename,
|
filename: newFilename,
|
||||||
path: relativePath,
|
path: relativePath,
|
||||||
thumbnail_path: relativeThumbPath,
|
thumbnail_path: null, // Will generate after successful commit
|
||||||
category_id: parsedCategoryId || null,
|
category_id: parsedCategoryId ? parseInt(parsedCategoryId) : null,
|
||||||
type: 'individual',
|
type: 'individual',
|
||||||
size_bytes: file.size
|
size_bytes: tempStats.size // Use actual file size from stat
|
||||||
|
};
|
||||||
|
|
||||||
|
batchPhotos.push(photoData);
|
||||||
|
|
||||||
|
// Store move operation for later
|
||||||
|
fileRenameOperations.push({
|
||||||
|
tempPath: tempPath,
|
||||||
|
finalPath: finalPath,
|
||||||
|
filename: newFilename,
|
||||||
|
photoData: photoData
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error processing file ${file.originalname}:`, error);
|
console.error(`Error preparing file ${file.originalname}:`, error);
|
||||||
errors.push({ filename: file.originalname, error: error.message });
|
errors.push({ filename: file.originalname, error: error.message });
|
||||||
// Delete the file if it was partially processed
|
|
||||||
if (file.path) {
|
|
||||||
try { await fs.unlink(file.path); } catch (e) {}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Batch insert all photos from this batch
|
// Insert all photos in this batch
|
||||||
if (batchPhotos.length > 0) {
|
if (batchPhotos.length > 0) {
|
||||||
|
console.log(`Inserting batch of ${batchPhotos.length} photos with category_id: ${parsedCategoryId}`);
|
||||||
|
|
||||||
const insertedIds = await trx('photos').insert(batchPhotos).returning('id');
|
const insertedIds = await trx('photos').insert(batchPhotos).returning('id');
|
||||||
|
|
||||||
// Update category counter if needed
|
// Update category counter if needed
|
||||||
if (category) {
|
if (category && parsedCategoryId) {
|
||||||
|
const newCounter = batchCounter + batchPhotos.length - 1;
|
||||||
await trx('photo_categories')
|
await trx('photo_categories')
|
||||||
.where({ id: parsedCategoryId })
|
.where({ id: parsedCategoryId })
|
||||||
.update({ photo_counter: batchCounter + batchPhotos.length - 1 });
|
.update({ photo_counter: newCounter });
|
||||||
|
console.log(`Updated category ${parsedCategoryId} counter to ${newCounter}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add to uploaded photos array
|
// Commit the transaction first
|
||||||
batchPhotos.forEach((photo, index) => {
|
await trx.commit();
|
||||||
uploadedPhotos.push({
|
console.log(`Successfully committed batch of ${batchPhotos.length} photos`);
|
||||||
id: insertedIds[index]?.id || insertedIds[index],
|
|
||||||
filename: photo.filename,
|
// Now move files from temp to final location after successful commit
|
||||||
size: photo.size_bytes,
|
for (let idx = 0; idx < fileRenameOperations.length; idx++) {
|
||||||
category_id: photo.category_id
|
const operation = fileRenameOperations[idx];
|
||||||
});
|
try {
|
||||||
});
|
// Move the file from temp to final location
|
||||||
|
await fs.rename(operation.tempPath, operation.finalPath);
|
||||||
|
console.log(`Moved file from ${operation.tempPath} to ${operation.finalPath}`);
|
||||||
|
|
||||||
|
// Verify the file was moved successfully
|
||||||
|
const finalStats = await fs.stat(operation.finalPath);
|
||||||
|
if (finalStats.size !== operation.photoData.size_bytes) {
|
||||||
|
throw new Error(`File size mismatch after move: expected ${operation.photoData.size_bytes}, got ${finalStats.size}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate thumbnail with final path
|
||||||
|
let thumbnailPath = null;
|
||||||
|
try {
|
||||||
|
thumbnailPath = await generateThumbnail(operation.finalPath);
|
||||||
|
|
||||||
|
// Update the database with thumbnail path
|
||||||
|
if (thumbnailPath && insertedIds[idx]) {
|
||||||
|
const photoId = insertedIds[idx]?.id || insertedIds[idx];
|
||||||
|
await db('photos')
|
||||||
|
.where({ id: photoId })
|
||||||
|
.update({ thumbnail_path: thumbnailPath });
|
||||||
|
}
|
||||||
|
} catch (thumbError) {
|
||||||
|
console.error(`Thumbnail generation failed for ${operation.filename}:`, thumbError.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add to successful uploads
|
||||||
|
uploadedPhotos.push({
|
||||||
|
id: insertedIds[idx]?.id || insertedIds[idx],
|
||||||
|
filename: operation.filename,
|
||||||
|
size: operation.photoData.size_bytes,
|
||||||
|
category_id: operation.photoData.category_id
|
||||||
|
});
|
||||||
|
} catch (moveError) {
|
||||||
|
console.error(`Failed to move file ${operation.tempPath} to ${operation.finalPath}:`, moveError);
|
||||||
|
errors.push({
|
||||||
|
filename: operation.filename,
|
||||||
|
error: `File move failed: ${moveError.message}`
|
||||||
|
});
|
||||||
|
|
||||||
|
// Try to clean up the database entry if file move failed
|
||||||
|
if (insertedIds[idx]) {
|
||||||
|
const photoId = insertedIds[idx]?.id || insertedIds[idx];
|
||||||
|
try {
|
||||||
|
await db('photos').where({ id: photoId }).delete();
|
||||||
|
console.log(`Cleaned up database entry for failed photo ${photoId}`);
|
||||||
|
} catch (cleanupError) {
|
||||||
|
console.error(`Failed to clean up database entry:`, cleanupError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// No photos to insert, just rollback
|
||||||
|
await trx.rollback();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Commit the batch transaction
|
|
||||||
await trx.commit();
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Error processing batch starting at index ${i}:`, error);
|
console.error(`Error processing batch starting at index ${i}:`, error);
|
||||||
await trx.rollback();
|
console.error('Stack trace:', error.stack);
|
||||||
|
|
||||||
// Try to clean up files from failed batch
|
// Rollback if not already committed
|
||||||
for (const file of batch) {
|
if (!trx.isCompleted()) {
|
||||||
if (file.path) {
|
await trx.rollback();
|
||||||
try { await fs.unlink(file.path); } catch (e) {}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add all files in this batch to errors
|
||||||
|
for (const file of batch) {
|
||||||
|
errors.push({
|
||||||
|
filename: file.originalname,
|
||||||
|
error: `Batch processing failed: ${error.message}`
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up temp upload directory
|
||||||
|
if (req.tempUploadPath) {
|
||||||
|
try {
|
||||||
|
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
|
||||||
|
console.log(`Cleaned up temp upload directory: ${req.tempUploadPath}`);
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Failed to clean up temp upload directory:', e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -298,6 +404,17 @@ router.post('/:eventId/upload', adminAuth, (req, res, next) => {
|
|||||||
res.json(response);
|
res.json(response);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error uploading photos:', error);
|
console.error('Error uploading photos:', error);
|
||||||
|
|
||||||
|
// Clean up temp upload directory on error
|
||||||
|
if (req.tempUploadPath) {
|
||||||
|
try {
|
||||||
|
await fs.rm(req.tempUploadPath, { recursive: true, force: true });
|
||||||
|
console.log(`Cleaned up temp upload directory after error: ${req.tempUploadPath}`);
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Failed to clean up temp upload directory:', e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
res.status(500).json({ error: 'Failed to upload photos' });
|
res.status(500).json({ error: 'Failed to upload photos' });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -51,6 +51,13 @@ async function processNewPhoto(filePath) {
|
|||||||
const ext = path.extname(filePath).toLowerCase();
|
const ext = path.extname(filePath).toLowerCase();
|
||||||
if (!['.jpg', '.jpeg', '.png', '.webp'].includes(ext)) return;
|
if (!['.jpg', '.jpeg', '.png', '.webp'].includes(ext)) return;
|
||||||
|
|
||||||
|
// Skip temporary upload files
|
||||||
|
const filename = path.basename(filePath);
|
||||||
|
if (filename.startsWith('temp_')) {
|
||||||
|
logger.debug(`Skipping temporary upload file: ${filename}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Find the event
|
// Find the event
|
||||||
const event = await db('events').where({ slug: eventSlug, is_active: formatBoolean(true) }).first();
|
const event = await db('events').where({ slug: eventSlug, is_active: formatBoolean(true) }).first();
|
||||||
if (!event) return;
|
if (!event) return;
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
const path = require('path');
|
||||||
|
const fs = require('fs').promises;
|
||||||
|
const logger = require('./logger');
|
||||||
|
|
||||||
|
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clean up old temporary upload directories
|
||||||
|
* Removes temp directories older than 1 hour
|
||||||
|
*/
|
||||||
|
async function cleanupTempUploads() {
|
||||||
|
const tempPath = path.join(getStoragePath(), 'temp');
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Ensure temp directory exists
|
||||||
|
await fs.mkdir(tempPath, { recursive: true });
|
||||||
|
|
||||||
|
// Read all items in temp directory
|
||||||
|
const items = await fs.readdir(tempPath);
|
||||||
|
|
||||||
|
let cleanedCount = 0;
|
||||||
|
const oneHourAgo = Date.now() - (60 * 60 * 1000); // 1 hour
|
||||||
|
|
||||||
|
for (const item of items) {
|
||||||
|
const itemPath = path.join(tempPath, item);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const stats = await fs.stat(itemPath);
|
||||||
|
|
||||||
|
// Only process directories that match our upload pattern
|
||||||
|
if (stats.isDirectory() && item.startsWith('upload_')) {
|
||||||
|
// Extract timestamp from directory name
|
||||||
|
const parts = item.split('_');
|
||||||
|
if (parts.length >= 2) {
|
||||||
|
const timestamp = parseInt(parts[1]);
|
||||||
|
|
||||||
|
// Remove if older than 1 hour
|
||||||
|
if (!isNaN(timestamp) && timestamp < oneHourAgo) {
|
||||||
|
logger.info(`Cleaning up old temp upload directory: ${item}`);
|
||||||
|
await fs.rm(itemPath, { recursive: true, force: true });
|
||||||
|
cleanedCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
logger.error(`Error processing temp item ${item}:`, error.message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cleanedCount > 0) {
|
||||||
|
logger.info(`Cleaned up ${cleanedCount} old temp upload directories`);
|
||||||
|
}
|
||||||
|
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Error during temp upload cleanup:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Start periodic cleanup of temp uploads
|
||||||
|
* Runs every hour
|
||||||
|
*/
|
||||||
|
function startTempUploadCleanup() {
|
||||||
|
// Run immediately on startup
|
||||||
|
cleanupTempUploads();
|
||||||
|
|
||||||
|
// Then run every hour
|
||||||
|
setInterval(() => {
|
||||||
|
cleanupTempUploads();
|
||||||
|
}, 60 * 60 * 1000); // 1 hour
|
||||||
|
|
||||||
|
logger.info('Temp upload cleanup service started');
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
cleanupTempUploads,
|
||||||
|
startTempUploadCleanup
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user