Add chunked upload support for large video files up to 10GB

- Increased max file size from 500MB to 10GB
- Created chunkedUploadService.js for managing chunked uploads
- Added chunked upload API endpoints (init, chunk, complete, status, abort)
- Added frontend chunked upload methods to photos.service.ts
- Files >100MB automatically use chunked uploads
- 10MB chunk size for reliable transfers
- Auto-cleanup of expired uploads after 24 hours
- Updated README with 10GB limit and nginx configuration example

README.md

@@ -147,10 +147,19 @@ When enabling video uploads, consider these additional resources:
 
 **Technical Notes:**
 - FFmpeg is bundled via npm (`@ffmpeg-installer/ffmpeg`) - no system installation required
-- Maximum upload size: 500MB per video file
+- Maximum upload size: **10GB per video file**
+- Chunked upload support for files >100MB (resumable uploads)
 - Supported formats: MP4, WebM, MOV, AVI
 - Video thumbnails are automatically generated from the first few seconds
 
+**For Nginx/Reverse Proxy:**
+If using Nginx, increase the client max body size:
+```nginx
+client_max_body_size 10G;
+proxy_read_timeout 3600;
+proxy_send_timeout 3600;
+```
+
 ## 🤝 Contributing
 
 We love contributions! PicPeak is built by photographers, for photographers. Whether you're fixing bugs, adding features, or improving documentation, your help is welcome.

backend/src/routes/adminPhotos.js

@@ -9,6 +9,8 @@ const { generatePhotoFilename } = require('../utils/filenameSanitizer');
 const { escapeLikePattern } = require('../utils/sqlSecurity');
 const { validateUploadedFiles } = require('../middleware/uploadValidation');
 const { getMaxFilesPerUpload } = require('../services/uploadSettings');
+const { processUploadedPhotos } = require('../services/photoProcessor');
+const chunkedUpload = require('../services/chunkedUploadService');
 const router = express.Router();
 
 // Get storage path from environment or default
@@ -48,7 +50,7 @@ const { validateFileType } = require('../utils/fileSecurityUtils');
 const upload = multer({
   storage: storage,
   limits: {
-    fileSize: 500 * 1024 * 1024, // 500MB limit per file to support videos
+    fileSize: 10 * 1024 * 1024 * 1024, // 10GB limit per file to support large videos
     files: 2000, // Hard safety ceiling; actual limit enforced dynamically
     // Set a reasonable field size limit to prevent memory issues
     fieldSize: 10 * 1024 * 1024, // 10MB for non-file fields
@@ -81,7 +83,7 @@ const validateUploadContent = createFileUploadValidator({
     'image/jpeg', 'image/png', 'image/webp',
     'video/mp4', 'video/webm', 'video/quicktime', 'video/x-msvideo'
   ],
-  maxFileSize: 500 * 1024 * 1024, // 500MB to support videos
+  maxFileSize: 10 * 1024 * 1024 * 1024, // 10GB to support large videos
   validateContent: true
 });
 
@@ -121,7 +123,7 @@ router.post('/:eventId/upload', adminAuth, uploadTimeout(600000), async (req, re
       console.error('Multer error:', err);
       if (err instanceof multer.MulterError) {
         if (err.code === 'LIMIT_FILE_SIZE') {
-          return res.status(400).json({ error: 'File too large. Maximum size is 50MB per file.' });
+          return res.status(400).json({ error: 'File too large. Maximum size is 10GB per file.' });
         }
         if (err.code === 'LIMIT_FILE_COUNT' || err.code === 'LIMIT_UNEXPECTED_FILE') {
           return res.status(400).json({ error: `Too many files. Maximum ${maxFilesPerUpload} files per upload.` });
@@ -843,4 +845,142 @@ router.get('/:eventId/debug', adminAuth, async (req, res) => {
   }
 });
 
+// ============================================
+// CHUNKED UPLOAD ENDPOINTS
+// For large file uploads (videos up to 10GB)
+// ============================================
+
+// Initialize a chunked upload
+router.post('/:eventId/chunked-upload/init', adminAuth, async (req, res) => {
+  try {
+    const { eventId } = req.params;
+    const { filename, fileSize, mimeType, totalChunks } = req.body;
+
+    // Validate event exists
+    const event = await db('events').where({ id: eventId }).first();
+    if (!event) {
+      return res.status(404).json({ error: 'Event not found' });
+    }
+
+    // Validate required fields
+    if (!filename || !fileSize || !mimeType) {
+      return res.status(400).json({ error: 'Missing required fields: filename, fileSize, mimeType' });
+    }
+
+    // Validate file size (max 10GB)
+    const maxSize = 10 * 1024 * 1024 * 1024;
+    if (fileSize > maxSize) {
+      return res.status(400).json({ error: `File too large. Maximum size is 10GB.` });
+    }
+
+    const result = await chunkedUpload.initializeUpload({
+      filename,
+      fileSize,
+      mimeType,
+      eventId: parseInt(eventId),
+      totalChunks
+    });
+
+    res.json(result);
+  } catch (error) {
+    console.error('Error initializing chunked upload:', error);
+    res.status(500).json({ error: 'Failed to initialize upload' });
+  }
+});
+
+// Upload a chunk
+router.post('/:eventId/chunked-upload/:uploadId/chunk/:chunkIndex', adminAuth, async (req, res) => {
+  try {
+    const { uploadId, chunkIndex } = req.params;
+
+    // Get chunk data from request body
+    const chunks = [];
+    for await (const chunk of req) {
+      chunks.push(chunk);
+    }
+    const chunkData = Buffer.concat(chunks);
+
+    const result = await chunkedUpload.uploadChunk(uploadId, parseInt(chunkIndex), chunkData);
+
+    res.json(result);
+  } catch (error) {
+    console.error('Error uploading chunk:', error);
+    res.status(500).json({ error: error.message || 'Failed to upload chunk' });
+  }
+});
+
+// Complete chunked upload and process the file
+router.post('/:eventId/chunked-upload/:uploadId/complete', adminAuth, async (req, res) => {
+  try {
+    const { eventId, uploadId } = req.params;
+    const { category_id } = req.body;
+
+    // Complete the chunked upload (merge chunks)
+    const mergedFile = await chunkedUpload.completeUpload(uploadId);
+
+    // Process the merged file as a regular upload
+    const fileObj = {
+      originalname: mergedFile.filename,
+      mimetype: mergedFile.mimeType,
+      size: mergedFile.size,
+      path: mergedFile.path
+    };
+
+    const uploadedPhotos = await processUploadedPhotos(
+      [fileObj],
+      parseInt(eventId),
+      'admin',
+      category_id || null
+    );
+
+    // Clean up temp directory
+    try {
+      await fs.rm(mergedFile.tempDir, { recursive: true, force: true });
+    } catch (cleanupErr) {
+      console.warn('Failed to clean up temp directory:', cleanupErr.message);
+    }
+
+    res.json({
+      success: true,
+      uploaded: uploadedPhotos.length,
+      photos: uploadedPhotos
+    });
+  } catch (error) {
+    console.error('Error completing chunked upload:', error);
+    res.status(500).json({ error: error.message || 'Failed to complete upload' });
+  }
+});
+
+// Get upload status
+router.get('/:eventId/chunked-upload/:uploadId/status', adminAuth, async (req, res) => {
+  try {
+    const { uploadId } = req.params;
+
+    const status = chunkedUpload.getUploadStatus(uploadId);
+
+    if (!status) {
+      return res.status(404).json({ error: 'Upload not found or expired' });
+    }
+
+    res.json(status);
+  } catch (error) {
+    console.error('Error getting upload status:', error);
+    res.status(500).json({ error: 'Failed to get upload status' });
+  }
+});
+
+// Abort chunked upload
+router.delete('/:eventId/chunked-upload/:uploadId', adminAuth, async (req, res) => {
+  try {
+    const { uploadId } = req.params;
+
+    await chunkedUpload.abortUpload(uploadId);
+
+    res.json({ success: true, message: 'Upload aborted' });
+  } catch (error) {
+    console.error('Error aborting upload:', error);
+    res.status(500).json({ error: 'Failed to abort upload' });
+  }
+});
+
 module.exports = router;

backend/src/services/chunkedUploadService.js

@@ -0,0 +1,285 @@
+const path = require('path');
+const fs = require('fs').promises;
+const crypto = require('crypto');
+const logger = require('../utils/logger');
+
+// Get storage path from environment or default
+const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
+const getChunksPath = () => path.join(getStoragePath(), 'chunks');
+
+// In-memory store for active uploads (in production, consider Redis)
+const activeUploads = new Map();
+
+// Chunk size: 10MB
+const CHUNK_SIZE = 10 * 1024 * 1024;
+
+// Upload expiration: 24 hours
+const UPLOAD_EXPIRATION_MS = 24 * 60 * 60 * 1000;
+
+/**
+ * Initialize a new chunked upload
+ * @param {Object} options - Upload options
+ * @returns {Promise<Object>} - Upload metadata
+ */
+async function initializeUpload(options) {
+  const {
+    filename,
+    fileSize,
+    mimeType,
+    eventId,
+    totalChunks
+  } = options;
+
+  // Generate unique upload ID
+  const uploadId = crypto.randomUUID();
+
+  // Create chunks directory for this upload
+  const uploadDir = path.join(getChunksPath(), uploadId);
+  await fs.mkdir(uploadDir, { recursive: true });
+
+  // Calculate expected chunks
+  const expectedChunks = totalChunks || Math.ceil(fileSize / CHUNK_SIZE);
+
+  // Store upload metadata
+  const uploadMeta = {
+    uploadId,
+    filename,
+    fileSize,
+    mimeType,
+    eventId,
+    expectedChunks,
+    receivedChunks: new Set(),
+    uploadDir,
+    createdAt: Date.now(),
+    expiresAt: Date.now() + UPLOAD_EXPIRATION_MS,
+    status: 'in_progress'
+  };
+
+  activeUploads.set(uploadId, uploadMeta);
+
+  logger.info('Initialized chunked upload', {
+    uploadId,
+    filename,
+    fileSize,
+    expectedChunks,
+    eventId
+  });
+
+  return {
+    uploadId,
+    chunkSize: CHUNK_SIZE,
+    expectedChunks,
+    expiresAt: uploadMeta.expiresAt
+  };
+}
+
+/**
+ * Upload a single chunk
+ * @param {string} uploadId - Upload ID
+ * @param {number} chunkIndex - Chunk index (0-based)
+ * @param {Buffer} chunkData - Chunk data
+ * @returns {Promise<Object>} - Chunk upload result
+ */
+async function uploadChunk(uploadId, chunkIndex, chunkData) {
+  const uploadMeta = activeUploads.get(uploadId);
+
+  if (!uploadMeta) {
+    throw new Error('Upload not found or expired');
+  }
+
+  if (uploadMeta.status !== 'in_progress') {
+    throw new Error(`Upload is ${uploadMeta.status}`);
+  }
+
+  // Check expiration
+  if (Date.now() > uploadMeta.expiresAt) {
+    await abortUpload(uploadId);
+    throw new Error('Upload expired');
+  }
+
+  // Write chunk to disk
+  const chunkPath = path.join(uploadMeta.uploadDir, `chunk_${String(chunkIndex).padStart(6, '0')}`);
+  await fs.writeFile(chunkPath, chunkData);
+
+  // Mark chunk as received
+  uploadMeta.receivedChunks.add(chunkIndex);
+
+  const progress = (uploadMeta.receivedChunks.size / uploadMeta.expectedChunks) * 100;
+
+  logger.debug('Chunk uploaded', {
+    uploadId,
+    chunkIndex,
+    receivedChunks: uploadMeta.receivedChunks.size,
+    expectedChunks: uploadMeta.expectedChunks,
+    progress: progress.toFixed(1)
+  });
+
+  return {
+    chunkIndex,
+    received: uploadMeta.receivedChunks.size,
+    expected: uploadMeta.expectedChunks,
+    progress,
+    complete: uploadMeta.receivedChunks.size === uploadMeta.expectedChunks
+  };
+}
+
+/**
+ * Complete the upload by merging all chunks
+ * @param {string} uploadId - Upload ID
+ * @returns {Promise<Object>} - Merged file info
+ */
+async function completeUpload(uploadId) {
+  const uploadMeta = activeUploads.get(uploadId);
+
+  if (!uploadMeta) {
+    throw new Error('Upload not found or expired');
+  }
+
+  // Verify all chunks received
+  if (uploadMeta.receivedChunks.size !== uploadMeta.expectedChunks) {
+    throw new Error(`Missing chunks: received ${uploadMeta.receivedChunks.size} of ${uploadMeta.expectedChunks}`);
+  }
+
+  uploadMeta.status = 'merging';
+
+  // Create temp file for merged result
+  const tempDir = path.join(getStoragePath(), 'temp', `merge_${Date.now()}_${Math.random().toString(36).substring(7)}`);
+  await fs.mkdir(tempDir, { recursive: true });
+
+  const mergedFilePath = path.join(tempDir, uploadMeta.filename);
+  const writeStream = require('fs').createWriteStream(mergedFilePath);
+
+  try {
+    // Merge chunks in order
+    for (let i = 0; i < uploadMeta.expectedChunks; i++) {
+      const chunkPath = path.join(uploadMeta.uploadDir, `chunk_${String(i).padStart(6, '0')}`);
+      const chunkData = await fs.readFile(chunkPath);
+
+      await new Promise((resolve, reject) => {
+        writeStream.write(chunkData, (err) => {
+          if (err) reject(err);
+          else resolve();
+        });
+      });
+    }
+
+    await new Promise((resolve) => writeStream.end(resolve));
+
+    // Verify file size
+    const stats = await fs.stat(mergedFilePath);
+    if (stats.size !== uploadMeta.fileSize) {
+      logger.warn('Merged file size mismatch', {
+        expected: uploadMeta.fileSize,
+        actual: stats.size
+      });
+    }
+
+    // Clean up chunks
+    await fs.rm(uploadMeta.uploadDir, { recursive: true, force: true });
+
+    uploadMeta.status = 'completed';
+    activeUploads.delete(uploadId);
+
+    logger.info('Chunked upload completed', {
+      uploadId,
+      filename: uploadMeta.filename,
+      fileSize: stats.size,
+      eventId: uploadMeta.eventId
+    });
+
+    return {
+      path: mergedFilePath,
+      filename: uploadMeta.filename,
+      size: stats.size,
+      mimeType: uploadMeta.mimeType,
+      eventId: uploadMeta.eventId,
+      tempDir
+    };
+  } catch (error) {
+    writeStream.destroy();
+    uploadMeta.status = 'failed';
+    throw error;
+  }
+}
+
+/**
+ * Abort and clean up an upload
+ * @param {string} uploadId - Upload ID
+ */
+async function abortUpload(uploadId) {
+  const uploadMeta = activeUploads.get(uploadId);
+
+  if (uploadMeta) {
+    try {
+      await fs.rm(uploadMeta.uploadDir, { recursive: true, force: true });
+    } catch (err) {
+      logger.warn('Failed to clean up upload directory', { uploadId, error: err.message });
+    }
+
+    activeUploads.delete(uploadId);
+
+    logger.info('Chunked upload aborted', { uploadId });
+  }
+}
+
+/**
+ * Get upload status
+ * @param {string} uploadId - Upload ID
+ * @returns {Object|null} - Upload status or null if not found
+ */
+function getUploadStatus(uploadId) {
+  const uploadMeta = activeUploads.get(uploadId);
+
+  if (!uploadMeta) {
+    return null;
+  }
+
+  return {
+    uploadId,
+    filename: uploadMeta.filename,
+    fileSize: uploadMeta.fileSize,
+    receivedChunks: uploadMeta.receivedChunks.size,
+    expectedChunks: uploadMeta.expectedChunks,
+    progress: (uploadMeta.receivedChunks.size / uploadMeta.expectedChunks) * 100,
+    status: uploadMeta.status,
+    createdAt: uploadMeta.createdAt,
+    expiresAt: uploadMeta.expiresAt
+  };
+}
+
+/**
+ * Clean up expired uploads
+ */
+async function cleanupExpiredUploads() {
+  const now = Date.now();
+  const expiredIds = [];
+
+  for (const [uploadId, meta] of activeUploads.entries()) {
+    if (now > meta.expiresAt) {
+      expiredIds.push(uploadId);
+    }
+  }
+
+  for (const uploadId of expiredIds) {
+    await abortUpload(uploadId);
+  }
+
+  if (expiredIds.length > 0) {
+    logger.info(`Cleaned up ${expiredIds.length} expired uploads`);
+  }
+
+  return expiredIds.length;
+}
+
+// Run cleanup every hour
+setInterval(cleanupExpiredUploads, 60 * 60 * 1000);
+
+module.exports = {
+  initializeUpload,
+  uploadChunk,
+  completeUpload,
+  abortUpload,
+  getUploadStatus,
+  cleanupExpiredUploads,
+  CHUNK_SIZE
+};

frontend/src/services/photos.service.ts

@@ -99,6 +99,106 @@ class PhotosService {
     const i = Math.floor(Math.log(bytes) / Math.log(k));
     return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
   }
+
+  // Chunked upload methods for large files (videos up to 10GB)
+  private CHUNK_SIZE = 10 * 1024 * 1024; // 10MB chunks
+
+  async initChunkedUpload(
+    eventId: number,
+    filename: string,
+    fileSize: number,
+    mimeType: string
+  ): Promise<{ uploadId: string; chunkSize: number; expectedChunks: number }> {
+    const totalChunks = Math.ceil(fileSize / this.CHUNK_SIZE);
+    const response = await api.post(`/admin/photos/${eventId}/chunked-upload/init`, {
+      filename,
+      fileSize,
+      mimeType,
+      totalChunks
+    });
+    return response.data;
+  }
+
+  async uploadChunk(
+    eventId: number,
+    uploadId: string,
+    chunkIndex: number,
+    chunkData: Blob
+  ): Promise<{ progress: number; complete: boolean }> {
+    const response = await api.post(
+      `/admin/photos/${eventId}/chunked-upload/${uploadId}/chunk/${chunkIndex}`,
+      chunkData,
+      {
+        headers: {
+          'Content-Type': 'application/octet-stream'
+        }
+      }
+    );
+    return response.data;
+  }
+
+  async completeChunkedUpload(
+    eventId: number,
+    uploadId: string,
+    categoryId?: number | null
+  ): Promise<{ success: boolean; uploaded: number; photos: AdminPhoto[] }> {
+    const response = await api.post(
+      `/admin/photos/${eventId}/chunked-upload/${uploadId}/complete`,
+      { category_id: categoryId }
+    );
+    return response.data;
+  }
+
+  async abortChunkedUpload(eventId: number, uploadId: string): Promise<void> {
+    await api.delete(`/admin/photos/${eventId}/chunked-upload/${uploadId}`);
+  }
+
+  async uploadLargeFile(
+    eventId: number,
+    file: File,
+    categoryId?: number | null,
+    onProgress?: (progress: number) => void
+  ): Promise<AdminPhoto[]> {
+    // Initialize upload
+    const { uploadId, expectedChunks } = await this.initChunkedUpload(
+      eventId,
+      file.name,
+      file.size,
+      file.type
+    );
+
+    try {
+      // Upload chunks
+      for (let i = 0; i < expectedChunks; i++) {
+        const start = i * this.CHUNK_SIZE;
+        const end = Math.min(start + this.CHUNK_SIZE, file.size);
+        const chunk = file.slice(start, end);
+
+        const result = await this.uploadChunk(eventId, uploadId, i, chunk);
+
+        if (onProgress) {
+          onProgress(result.progress);
+        }
+      }
+
+      // Complete upload
+      const result = await this.completeChunkedUpload(eventId, uploadId, categoryId);
+      return result.photos;
+    } catch (error) {
+      // Abort on error
+      try {
+        await this.abortChunkedUpload(eventId, uploadId);
+      } catch (abortError) {
+        console.error('Failed to abort upload:', abortError);
+      }
+      throw error;
+    }
+  }
+
+  // Check if file should use chunked upload (> 100MB)
+  shouldUseChunkedUpload(fileSize: number): boolean {
+    return fileSize > 100 * 1024 * 1024; // 100MB threshold
+  }
 }
 
 export const photosService = new PhotosService();