diff --git a/README.md b/README.md index 8300d34..15875af 100644 --- a/README.md +++ b/README.md @@ -147,10 +147,19 @@ When enabling video uploads, consider these additional resources: **Technical Notes:** - FFmpeg is bundled via npm (`@ffmpeg-installer/ffmpeg`) - no system installation required -- Maximum upload size: 500MB per video file +- Maximum upload size: **10GB per video file** +- Chunked upload support for files >100MB (resumable uploads) - Supported formats: MP4, WebM, MOV, AVI - Video thumbnails are automatically generated from the first few seconds +**For Nginx/Reverse Proxy:** +If using Nginx, increase the client max body size: +```nginx +client_max_body_size 10G; +proxy_read_timeout 3600; +proxy_send_timeout 3600; +``` + ## 🤝 Contributing We love contributions! PicPeak is built by photographers, for photographers. Whether you're fixing bugs, adding features, or improving documentation, your help is welcome. diff --git a/backend/src/routes/adminPhotos.js b/backend/src/routes/adminPhotos.js index ee26933..769e27f 100644 --- a/backend/src/routes/adminPhotos.js +++ b/backend/src/routes/adminPhotos.js @@ -9,6 +9,8 @@ const { generatePhotoFilename } = require('../utils/filenameSanitizer'); const { escapeLikePattern } = require('../utils/sqlSecurity'); const { validateUploadedFiles } = require('../middleware/uploadValidation'); const { getMaxFilesPerUpload } = require('../services/uploadSettings'); +const { processUploadedPhotos } = require('../services/photoProcessor'); +const chunkedUpload = require('../services/chunkedUploadService'); const router = express.Router(); // Get storage path from environment or default @@ -48,7 +50,7 @@ const { validateFileType } = require('../utils/fileSecurityUtils'); const upload = multer({ storage: storage, limits: { - fileSize: 500 * 1024 * 1024, // 500MB limit per file to support videos + fileSize: 10 * 1024 * 1024 * 1024, // 10GB limit per file to support large videos files: 2000, // Hard safety ceiling; actual limit enforced dynamically // Set a reasonable field size limit to prevent memory issues fieldSize: 10 * 1024 * 1024, // 10MB for non-file fields @@ -81,7 +83,7 @@ const validateUploadContent = createFileUploadValidator({ 'image/jpeg', 'image/png', 'image/webp', 'video/mp4', 'video/webm', 'video/quicktime', 'video/x-msvideo' ], - maxFileSize: 500 * 1024 * 1024, // 500MB to support videos + maxFileSize: 10 * 1024 * 1024 * 1024, // 10GB to support large videos validateContent: true }); @@ -121,7 +123,7 @@ router.post('/:eventId/upload', adminAuth, uploadTimeout(600000), async (req, re console.error('Multer error:', err); if (err instanceof multer.MulterError) { if (err.code === 'LIMIT_FILE_SIZE') { - return res.status(400).json({ error: 'File too large. Maximum size is 50MB per file.' }); + return res.status(400).json({ error: 'File too large. Maximum size is 10GB per file.' }); } if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') { return res.status(400).json({ error: `Too many files. Maximum ${maxFilesPerUpload} files per upload.` }); @@ -843,4 +845,142 @@ router.get('/:eventId/debug', adminAuth, async (req, res) => { } }); +// ============================================ +// CHUNKED UPLOAD ENDPOINTS +// For large file uploads (videos up to 10GB) +// ============================================ + +// Initialize a chunked upload +router.post('/:eventId/chunked-upload/init', adminAuth, async (req, res) => { + try { + const { eventId } = req.params; + const { filename, fileSize, mimeType, totalChunks } = req.body; + + // Validate event exists + const event = await db('events').where({ id: eventId }).first(); + if (!event) { + return res.status(404).json({ error: 'Event not found' }); + } + + // Validate required fields + if (!filename || !fileSize || !mimeType) { + return res.status(400).json({ error: 'Missing required fields: filename, fileSize, mimeType' }); + } + + // Validate file size (max 10GB) + const maxSize = 10 * 1024 * 1024 * 1024; + if (fileSize > maxSize) { + return res.status(400).json({ error: `File too large. Maximum size is 10GB.` }); + } + + const result = await chunkedUpload.initializeUpload({ + filename, + fileSize, + mimeType, + eventId: parseInt(eventId), + totalChunks + }); + + res.json(result); + } catch (error) { + console.error('Error initializing chunked upload:', error); + res.status(500).json({ error: 'Failed to initialize upload' }); + } +}); + +// Upload a chunk +router.post('/:eventId/chunked-upload/:uploadId/chunk/:chunkIndex', adminAuth, async (req, res) => { + try { + const { uploadId, chunkIndex } = req.params; + + // Get chunk data from request body + const chunks = []; + for await (const chunk of req) { + chunks.push(chunk); + } + const chunkData = Buffer.concat(chunks); + + const result = await chunkedUpload.uploadChunk(uploadId, parseInt(chunkIndex), chunkData); + + res.json(result); + } catch (error) { + console.error('Error uploading chunk:', error); + res.status(500).json({ error: error.message || 'Failed to upload chunk' }); + } +}); + +// Complete chunked upload and process the file +router.post('/:eventId/chunked-upload/:uploadId/complete', adminAuth, async (req, res) => { + try { + const { eventId, uploadId } = req.params; + const { category_id } = req.body; + + // Complete the chunked upload (merge chunks) + const mergedFile = await chunkedUpload.completeUpload(uploadId); + + // Process the merged file as a regular upload + const fileObj = { + originalname: mergedFile.filename, + mimetype: mergedFile.mimeType, + size: mergedFile.size, + path: mergedFile.path + }; + + const uploadedPhotos = await processUploadedPhotos( + [fileObj], + parseInt(eventId), + 'admin', + category_id || null + ); + + // Clean up temp directory + try { + await fs.rm(mergedFile.tempDir, { recursive: true, force: true }); + } catch (cleanupErr) { + console.warn('Failed to clean up temp directory:', cleanupErr.message); + } + + res.json({ + success: true, + uploaded: uploadedPhotos.length, + photos: uploadedPhotos + }); + } catch (error) { + console.error('Error completing chunked upload:', error); + res.status(500).json({ error: error.message || 'Failed to complete upload' }); + } +}); + +// Get upload status +router.get('/:eventId/chunked-upload/:uploadId/status', adminAuth, async (req, res) => { + try { + const { uploadId } = req.params; + + const status = chunkedUpload.getUploadStatus(uploadId); + + if (!status) { + return res.status(404).json({ error: 'Upload not found or expired' }); + } + + res.json(status); + } catch (error) { + console.error('Error getting upload status:', error); + res.status(500).json({ error: 'Failed to get upload status' }); + } +}); + +// Abort chunked upload +router.delete('/:eventId/chunked-upload/:uploadId', adminAuth, async (req, res) => { + try { + const { uploadId } = req.params; + + await chunkedUpload.abortUpload(uploadId); + + res.json({ success: true, message: 'Upload aborted' }); + } catch (error) { + console.error('Error aborting upload:', error); + res.status(500).json({ error: 'Failed to abort upload' }); + } +}); + module.exports = router; diff --git a/backend/src/services/chunkedUploadService.js b/backend/src/services/chunkedUploadService.js new file mode 100644 index 0000000..17cc8bb --- /dev/null +++ b/backend/src/services/chunkedUploadService.js @@ -0,0 +1,285 @@ +const path = require('path'); +const fs = require('fs').promises; +const crypto = require('crypto'); +const logger = require('../utils/logger'); + +// Get storage path from environment or default +const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage'); +const getChunksPath = () => path.join(getStoragePath(), 'chunks'); + +// In-memory store for active uploads (in production, consider Redis) +const activeUploads = new Map(); + +// Chunk size: 10MB +const CHUNK_SIZE = 10 * 1024 * 1024; + +// Upload expiration: 24 hours +const UPLOAD_EXPIRATION_MS = 24 * 60 * 60 * 1000; + +/** + * Initialize a new chunked upload + * @param {Object} options - Upload options + * @returns {Promise} - Upload metadata + */ +async function initializeUpload(options) { + const { + filename, + fileSize, + mimeType, + eventId, + totalChunks + } = options; + + // Generate unique upload ID + const uploadId = crypto.randomUUID(); + + // Create chunks directory for this upload + const uploadDir = path.join(getChunksPath(), uploadId); + await fs.mkdir(uploadDir, { recursive: true }); + + // Calculate expected chunks + const expectedChunks = totalChunks || Math.ceil(fileSize / CHUNK_SIZE); + + // Store upload metadata + const uploadMeta = { + uploadId, + filename, + fileSize, + mimeType, + eventId, + expectedChunks, + receivedChunks: new Set(), + uploadDir, + createdAt: Date.now(), + expiresAt: Date.now() + UPLOAD_EXPIRATION_MS, + status: 'in_progress' + }; + + activeUploads.set(uploadId, uploadMeta); + + logger.info('Initialized chunked upload', { + uploadId, + filename, + fileSize, + expectedChunks, + eventId + }); + + return { + uploadId, + chunkSize: CHUNK_SIZE, + expectedChunks, + expiresAt: uploadMeta.expiresAt + }; +} + +/** + * Upload a single chunk + * @param {string} uploadId - Upload ID + * @param {number} chunkIndex - Chunk index (0-based) + * @param {Buffer} chunkData - Chunk data + * @returns {Promise} - Chunk upload result + */ +async function uploadChunk(uploadId, chunkIndex, chunkData) { + const uploadMeta = activeUploads.get(uploadId); + + if (!uploadMeta) { + throw new Error('Upload not found or expired'); + } + + if (uploadMeta.status !== 'in_progress') { + throw new Error(`Upload is ${uploadMeta.status}`); + } + + // Check expiration + if (Date.now() > uploadMeta.expiresAt) { + await abortUpload(uploadId); + throw new Error('Upload expired'); + } + + // Write chunk to disk + const chunkPath = path.join(uploadMeta.uploadDir, `chunk_${String(chunkIndex).padStart(6, '0')}`); + await fs.writeFile(chunkPath, chunkData); + + // Mark chunk as received + uploadMeta.receivedChunks.add(chunkIndex); + + const progress = (uploadMeta.receivedChunks.size / uploadMeta.expectedChunks) * 100; + + logger.debug('Chunk uploaded', { + uploadId, + chunkIndex, + receivedChunks: uploadMeta.receivedChunks.size, + expectedChunks: uploadMeta.expectedChunks, + progress: progress.toFixed(1) + }); + + return { + chunkIndex, + received: uploadMeta.receivedChunks.size, + expected: uploadMeta.expectedChunks, + progress, + complete: uploadMeta.receivedChunks.size === uploadMeta.expectedChunks + }; +} + +/** + * Complete the upload by merging all chunks + * @param {string} uploadId - Upload ID + * @returns {Promise} - Merged file info + */ +async function completeUpload(uploadId) { + const uploadMeta = activeUploads.get(uploadId); + + if (!uploadMeta) { + throw new Error('Upload not found or expired'); + } + + // Verify all chunks received + if (uploadMeta.receivedChunks.size !== uploadMeta.expectedChunks) { + throw new Error(`Missing chunks: received ${uploadMeta.receivedChunks.size} of ${uploadMeta.expectedChunks}`); + } + + uploadMeta.status = 'merging'; + + // Create temp file for merged result + const tempDir = path.join(getStoragePath(), 'temp', `merge_${Date.now()}_${Math.random().toString(36).substring(7)}`); + await fs.mkdir(tempDir, { recursive: true }); + + const mergedFilePath = path.join(tempDir, uploadMeta.filename); + const writeStream = require('fs').createWriteStream(mergedFilePath); + + try { + // Merge chunks in order + for (let i = 0; i < uploadMeta.expectedChunks; i++) { + const chunkPath = path.join(uploadMeta.uploadDir, `chunk_${String(i).padStart(6, '0')}`); + const chunkData = await fs.readFile(chunkPath); + + await new Promise((resolve, reject) => { + writeStream.write(chunkData, (err) => { + if (err) reject(err); + else resolve(); + }); + }); + } + + await new Promise((resolve) => writeStream.end(resolve)); + + // Verify file size + const stats = await fs.stat(mergedFilePath); + if (stats.size !== uploadMeta.fileSize) { + logger.warn('Merged file size mismatch', { + expected: uploadMeta.fileSize, + actual: stats.size + }); + } + + // Clean up chunks + await fs.rm(uploadMeta.uploadDir, { recursive: true, force: true }); + + uploadMeta.status = 'completed'; + activeUploads.delete(uploadId); + + logger.info('Chunked upload completed', { + uploadId, + filename: uploadMeta.filename, + fileSize: stats.size, + eventId: uploadMeta.eventId + }); + + return { + path: mergedFilePath, + filename: uploadMeta.filename, + size: stats.size, + mimeType: uploadMeta.mimeType, + eventId: uploadMeta.eventId, + tempDir + }; + } catch (error) { + writeStream.destroy(); + uploadMeta.status = 'failed'; + throw error; + } +} + +/** + * Abort and clean up an upload + * @param {string} uploadId - Upload ID + */ +async function abortUpload(uploadId) { + const uploadMeta = activeUploads.get(uploadId); + + if (uploadMeta) { + try { + await fs.rm(uploadMeta.uploadDir, { recursive: true, force: true }); + } catch (err) { + logger.warn('Failed to clean up upload directory', { uploadId, error: err.message }); + } + + activeUploads.delete(uploadId); + + logger.info('Chunked upload aborted', { uploadId }); + } +} + +/** + * Get upload status + * @param {string} uploadId - Upload ID + * @returns {Object|null} - Upload status or null if not found + */ +function getUploadStatus(uploadId) { + const uploadMeta = activeUploads.get(uploadId); + + if (!uploadMeta) { + return null; + } + + return { + uploadId, + filename: uploadMeta.filename, + fileSize: uploadMeta.fileSize, + receivedChunks: uploadMeta.receivedChunks.size, + expectedChunks: uploadMeta.expectedChunks, + progress: (uploadMeta.receivedChunks.size / uploadMeta.expectedChunks) * 100, + status: uploadMeta.status, + createdAt: uploadMeta.createdAt, + expiresAt: uploadMeta.expiresAt + }; +} + +/** + * Clean up expired uploads + */ +async function cleanupExpiredUploads() { + const now = Date.now(); + const expiredIds = []; + + for (const [uploadId, meta] of activeUploads.entries()) { + if (now > meta.expiresAt) { + expiredIds.push(uploadId); + } + } + + for (const uploadId of expiredIds) { + await abortUpload(uploadId); + } + + if (expiredIds.length > 0) { + logger.info(`Cleaned up ${expiredIds.length} expired uploads`); + } + + return expiredIds.length; +} + +// Run cleanup every hour +setInterval(cleanupExpiredUploads, 60 * 60 * 1000); + +module.exports = { + initializeUpload, + uploadChunk, + completeUpload, + abortUpload, + getUploadStatus, + cleanupExpiredUploads, + CHUNK_SIZE +}; diff --git a/frontend/src/services/photos.service.ts b/frontend/src/services/photos.service.ts index 839c6eb..9870aa7 100644 --- a/frontend/src/services/photos.service.ts +++ b/frontend/src/services/photos.service.ts @@ -99,6 +99,106 @@ class PhotosService { const i = Math.floor(Math.log(bytes) / Math.log(k)); return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; } + + // Chunked upload methods for large files (videos up to 10GB) + private CHUNK_SIZE = 10 * 1024 * 1024; // 10MB chunks + + async initChunkedUpload( + eventId: number, + filename: string, + fileSize: number, + mimeType: string + ): Promise<{ uploadId: string; chunkSize: number; expectedChunks: number }> { + const totalChunks = Math.ceil(fileSize / this.CHUNK_SIZE); + const response = await api.post(`/admin/photos/${eventId}/chunked-upload/init`, { + filename, + fileSize, + mimeType, + totalChunks + }); + return response.data; + } + + async uploadChunk( + eventId: number, + uploadId: string, + chunkIndex: number, + chunkData: Blob + ): Promise<{ progress: number; complete: boolean }> { + const response = await api.post( + `/admin/photos/${eventId}/chunked-upload/${uploadId}/chunk/${chunkIndex}`, + chunkData, + { + headers: { + 'Content-Type': 'application/octet-stream' + } + } + ); + return response.data; + } + + async completeChunkedUpload( + eventId: number, + uploadId: string, + categoryId?: number | null + ): Promise<{ success: boolean; uploaded: number; photos: AdminPhoto[] }> { + const response = await api.post( + `/admin/photos/${eventId}/chunked-upload/${uploadId}/complete`, + { category_id: categoryId } + ); + return response.data; + } + + async abortChunkedUpload(eventId: number, uploadId: string): Promise { + await api.delete(`/admin/photos/${eventId}/chunked-upload/${uploadId}`); + } + + async uploadLargeFile( + eventId: number, + file: File, + categoryId?: number | null, + onProgress?: (progress: number) => void + ): Promise { + // Initialize upload + const { uploadId, expectedChunks } = await this.initChunkedUpload( + eventId, + file.name, + file.size, + file.type + ); + + try { + // Upload chunks + for (let i = 0; i < expectedChunks; i++) { + const start = i * this.CHUNK_SIZE; + const end = Math.min(start + this.CHUNK_SIZE, file.size); + const chunk = file.slice(start, end); + + const result = await this.uploadChunk(eventId, uploadId, i, chunk); + + if (onProgress) { + onProgress(result.progress); + } + } + + // Complete upload + const result = await this.completeChunkedUpload(eventId, uploadId, categoryId); + return result.photos; + } catch (error) { + // Abort on error + try { + await this.abortChunkedUpload(eventId, uploadId); + } catch (abortError) { + console.error('Failed to abort upload:', abortError); + } + throw error; + } + } + + // Check if file should use chunked upload (> 100MB) + shouldUseChunkedUpload(fileSize: number): boolean { + return fileSize > 100 * 1024 * 1024; // 100MB threshold + } } export const photosService = new PhotosService();