feat: pre-generate watermarks for instant lightbox loading
Previously watermarks were applied on-the-fly when viewing photos in the lightbox, causing 1+ minute load times for high-resolution images. This change pre-generates watermarked versions during upload and when watermark settings change, enabling instant image loading (~50-100ms). - Add database migration for watermark_path tracking (061) - Add watermarkGeneratorService for batch operations - Extend watermarkService with save-to-disk capability - Modify gallery endpoint to serve pre-generated files - Add background regeneration when branding settings change - Add npm script for migrating existing photos Closes #112
This commit is contained in:
Paul Nothaft
@@ -0,0 +1,433 @@
|
||||
/**
|
||||
* WatermarkGeneratorService
|
||||
*
|
||||
* Handles batch generation of pre-watermarked images for fast serving.
|
||||
* This service is responsible for:
|
||||
* - Generating watermarks for newly uploaded photos
|
||||
* - Regenerating all watermarks when settings change
|
||||
* - Tracking regeneration progress
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
const { db } = require('../database/db');
|
||||
const watermarkService = require('./watermarkService');
|
||||
const { getStoragePath } = require('../config/storage');
|
||||
|
||||
class WatermarkGeneratorService {
|
||||
constructor() {
|
||||
// Track active regeneration jobs
|
||||
this.activeJobs = new Map();
|
||||
// Batch size for processing (to manage memory)
|
||||
this.batchSize = 10;
|
||||
// Concurrent processing limit
|
||||
this.concurrentLimit = 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate watermark for a single photo
|
||||
* @param {number} photoId - The photo ID
|
||||
* @returns {Object} Result with success status and watermark path
|
||||
*/
|
||||
async generateForPhoto(photoId) {
|
||||
try {
|
||||
// Get photo with event info
|
||||
const photo = await db('photos')
|
||||
.join('events', 'photos.event_id', 'events.id')
|
||||
.where('photos.id', photoId)
|
||||
.select(
|
||||
'photos.*',
|
||||
'events.slug',
|
||||
'events.source_mode',
|
||||
'events.external_path'
|
||||
)
|
||||
.first();
|
||||
|
||||
if (!photo) {
|
||||
return { success: false, error: 'Photo not found' };
|
||||
}
|
||||
|
||||
// Skip video files
|
||||
if (photo.media_type === 'video' || (photo.mime_type && photo.mime_type.startsWith('video/'))) {
|
||||
return { success: false, error: 'Videos do not support watermarks' };
|
||||
}
|
||||
|
||||
// Get watermark settings
|
||||
const settings = await watermarkService.getWatermarkSettings();
|
||||
if (!settings || !settings.enabled) {
|
||||
return { success: false, error: 'Watermarking is disabled' };
|
||||
}
|
||||
|
||||
// Resolve the original file path
|
||||
const originalPath = this.resolvePhotoPath(photo);
|
||||
if (!originalPath) {
|
||||
return { success: false, error: 'Could not resolve photo path' };
|
||||
}
|
||||
|
||||
// Generate and save watermark
|
||||
const result = await watermarkService.generateAndSaveWatermark(photo, originalPath, settings);
|
||||
|
||||
if (result.success) {
|
||||
// Update database with watermark path
|
||||
await db('photos')
|
||||
.where({ id: photoId })
|
||||
.update({
|
||||
watermark_path: result.watermarkPath,
|
||||
watermark_generated_at: db.fn.now()
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error(`Error generating watermark for photo ${photoId}:`, error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the full file path for a photo
|
||||
*/
|
||||
resolvePhotoPath(photo) {
|
||||
const storagePath = getStoragePath();
|
||||
|
||||
// Handle external/reference mode
|
||||
if (photo.source_mode === 'reference' && photo.external_relpath) {
|
||||
const externalRoot = process.env.EXTERNAL_MEDIA_PATH || path.join(storagePath, 'external');
|
||||
return path.join(externalRoot, photo.external_path || '', photo.external_relpath);
|
||||
}
|
||||
|
||||
// Standard managed mode
|
||||
if (photo.file_path) {
|
||||
// file_path might be absolute or relative
|
||||
if (path.isAbsolute(photo.file_path)) {
|
||||
return photo.file_path;
|
||||
}
|
||||
return path.join(storagePath, photo.file_path);
|
||||
}
|
||||
|
||||
// Fallback to constructing path from slug and filename
|
||||
return path.join(storagePath, 'events', 'active', photo.slug, photo.filename);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate watermarks for all photos in an event
|
||||
* @param {number} eventId - The event ID
|
||||
* @param {Function} onProgress - Optional callback for progress updates
|
||||
* @returns {Object} Result with success count and errors
|
||||
*/
|
||||
async generateForEvent(eventId, onProgress = null) {
|
||||
const results = { total: 0, success: 0, failed: 0, errors: [] };
|
||||
|
||||
try {
|
||||
// Get all photos for the event (excluding videos)
|
||||
const photos = await db('photos')
|
||||
.join('events', 'photos.event_id', 'events.id')
|
||||
.where('photos.event_id', eventId)
|
||||
.whereNot(function() {
|
||||
this.where('photos.media_type', 'video')
|
||||
.orWhere('photos.mime_type', 'like', 'video/%');
|
||||
})
|
||||
.select(
|
||||
'photos.*',
|
||||
'events.slug',
|
||||
'events.source_mode',
|
||||
'events.external_path'
|
||||
);
|
||||
|
||||
results.total = photos.length;
|
||||
|
||||
if (photos.length === 0) {
|
||||
return results;
|
||||
}
|
||||
|
||||
// Get watermark settings once
|
||||
const settings = await watermarkService.getWatermarkSettings();
|
||||
if (!settings || !settings.enabled) {
|
||||
return { ...results, errors: ['Watermarking is disabled'] };
|
||||
}
|
||||
|
||||
// Process in batches
|
||||
for (let i = 0; i < photos.length; i += this.batchSize) {
|
||||
const batch = photos.slice(i, i + this.batchSize);
|
||||
|
||||
// Process batch with limited concurrency
|
||||
const batchResults = await Promise.all(
|
||||
batch.map(photo => this.processPhotoWatermark(photo, settings))
|
||||
);
|
||||
|
||||
// Collect results
|
||||
for (const result of batchResults) {
|
||||
if (result.success) {
|
||||
results.success++;
|
||||
} else {
|
||||
results.failed++;
|
||||
if (result.error) {
|
||||
results.errors.push(`Photo ${result.photoId}: ${result.error}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Progress callback
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
total: results.total,
|
||||
processed: results.success + results.failed,
|
||||
success: results.success,
|
||||
failed: results.failed
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
} catch (error) {
|
||||
console.error(`Error generating watermarks for event ${eventId}:`, error);
|
||||
return { ...results, errors: [...results.errors, error.message] };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process watermark for a single photo (internal helper)
|
||||
*/
|
||||
async processPhotoWatermark(photo, settings) {
|
||||
try {
|
||||
const originalPath = this.resolvePhotoPath(photo);
|
||||
if (!originalPath) {
|
||||
return { success: false, photoId: photo.id, error: 'Could not resolve path' };
|
||||
}
|
||||
|
||||
const result = await watermarkService.generateAndSaveWatermark(photo, originalPath, settings);
|
||||
|
||||
if (result.success) {
|
||||
await db('photos')
|
||||
.where({ id: photo.id })
|
||||
.update({
|
||||
watermark_path: result.watermarkPath,
|
||||
watermark_generated_at: db.fn.now()
|
||||
});
|
||||
}
|
||||
|
||||
return { ...result, photoId: photo.id };
|
||||
} catch (error) {
|
||||
return { success: false, photoId: photo.id, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Regenerate watermarks for all photos in the system
|
||||
* @param {Function} onProgress - Optional callback for progress updates
|
||||
* @returns {Object} Result with success count and errors
|
||||
*/
|
||||
async regenerateAll(onProgress = null) {
|
||||
const jobId = Date.now().toString();
|
||||
const results = { jobId, total: 0, success: 0, failed: 0, errors: [], status: 'running' };
|
||||
|
||||
try {
|
||||
this.activeJobs.set(jobId, results);
|
||||
|
||||
// Get watermark settings
|
||||
const settings = await watermarkService.getWatermarkSettings();
|
||||
if (!settings || !settings.enabled) {
|
||||
results.status = 'completed';
|
||||
results.errors.push('Watermarking is disabled');
|
||||
return results;
|
||||
}
|
||||
|
||||
// First, clear existing watermarks from DB (the files will be overwritten)
|
||||
// This ensures stale paths don't persist if regeneration fails
|
||||
|
||||
// Get all image photos (exclude videos)
|
||||
const photos = await db('photos')
|
||||
.join('events', 'photos.event_id', 'events.id')
|
||||
.whereNot(function() {
|
||||
this.where('photos.media_type', 'video')
|
||||
.orWhere('photos.mime_type', 'like', 'video/%');
|
||||
})
|
||||
.select(
|
||||
'photos.*',
|
||||
'events.slug',
|
||||
'events.source_mode',
|
||||
'events.external_path'
|
||||
);
|
||||
|
||||
results.total = photos.length;
|
||||
|
||||
if (photos.length === 0) {
|
||||
results.status = 'completed';
|
||||
return results;
|
||||
}
|
||||
|
||||
console.log(`Starting watermark regeneration for ${photos.length} photos`);
|
||||
|
||||
// Process in batches
|
||||
for (let i = 0; i < photos.length; i += this.batchSize) {
|
||||
// Check if job was cancelled
|
||||
if (!this.activeJobs.has(jobId)) {
|
||||
results.status = 'cancelled';
|
||||
return results;
|
||||
}
|
||||
|
||||
const batch = photos.slice(i, i + this.batchSize);
|
||||
|
||||
// Process batch with limited concurrency
|
||||
const batchResults = await Promise.all(
|
||||
batch.map(photo => this.processPhotoWatermark(photo, settings))
|
||||
);
|
||||
|
||||
// Collect results
|
||||
for (const result of batchResults) {
|
||||
if (result.success) {
|
||||
results.success++;
|
||||
} else {
|
||||
results.failed++;
|
||||
if (result.error && results.errors.length < 50) {
|
||||
results.errors.push(`Photo ${result.photoId}: ${result.error}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update job status
|
||||
this.activeJobs.set(jobId, { ...results });
|
||||
|
||||
// Progress callback
|
||||
if (onProgress) {
|
||||
onProgress({
|
||||
jobId,
|
||||
total: results.total,
|
||||
processed: results.success + results.failed,
|
||||
success: results.success,
|
||||
failed: results.failed,
|
||||
percentComplete: Math.round(((results.success + results.failed) / results.total) * 100)
|
||||
});
|
||||
}
|
||||
|
||||
// Small delay between batches to prevent CPU saturation
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
|
||||
results.status = 'completed';
|
||||
console.log(`Watermark regeneration completed: ${results.success}/${results.total} successful`);
|
||||
|
||||
return results;
|
||||
} catch (error) {
|
||||
console.error('Error during watermark regeneration:', error);
|
||||
results.status = 'failed';
|
||||
results.errors.push(error.message);
|
||||
return results;
|
||||
} finally {
|
||||
// Clean up job tracking after a delay
|
||||
setTimeout(() => {
|
||||
this.activeJobs.delete(jobId);
|
||||
}, 60000); // Keep for 1 minute for status queries
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all watermarks (when watermarking is disabled)
|
||||
*/
|
||||
async clearAllWatermarks() {
|
||||
try {
|
||||
// Get all photos with watermarks
|
||||
const photos = await db('photos')
|
||||
.whereNotNull('watermark_path')
|
||||
.select('id', 'watermark_path');
|
||||
|
||||
// Delete watermark files
|
||||
for (const photo of photos) {
|
||||
await watermarkService.deleteWatermarkFile(photo.watermark_path);
|
||||
}
|
||||
|
||||
// Clear database paths
|
||||
await db('photos')
|
||||
.whereNotNull('watermark_path')
|
||||
.update({
|
||||
watermark_path: null,
|
||||
watermark_generated_at: null
|
||||
});
|
||||
|
||||
console.log(`Cleared ${photos.length} watermarks`);
|
||||
return { success: true, cleared: photos.length };
|
||||
} catch (error) {
|
||||
console.error('Error clearing watermarks:', error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete watermark for a specific photo
|
||||
*/
|
||||
async deleteForPhoto(photoId) {
|
||||
try {
|
||||
const photo = await db('photos')
|
||||
.where({ id: photoId })
|
||||
.select('watermark_path')
|
||||
.first();
|
||||
|
||||
if (photo && photo.watermark_path) {
|
||||
await watermarkService.deleteWatermarkFile(photo.watermark_path);
|
||||
await db('photos')
|
||||
.where({ id: photoId })
|
||||
.update({
|
||||
watermark_path: null,
|
||||
watermark_generated_at: null
|
||||
});
|
||||
}
|
||||
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
console.error(`Error deleting watermark for photo ${photoId}:`, error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get status of an active regeneration job
|
||||
*/
|
||||
getJobStatus(jobId) {
|
||||
return this.activeJobs.get(jobId) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel an active regeneration job
|
||||
*/
|
||||
cancelJob(jobId) {
|
||||
if (this.activeJobs.has(jobId)) {
|
||||
this.activeJobs.delete(jobId);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if there's an active regeneration job
|
||||
*/
|
||||
hasActiveJob() {
|
||||
for (const [, job] of this.activeJobs) {
|
||||
if (job.status === 'running') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get count of photos needing watermark generation
|
||||
*/
|
||||
async getPendingCount() {
|
||||
const settings = await watermarkService.getWatermarkSettings();
|
||||
if (!settings || !settings.enabled) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const result = await db('photos')
|
||||
.whereNull('watermark_path')
|
||||
.whereNot(function() {
|
||||
this.where('media_type', 'video')
|
||||
.orWhere('mime_type', 'like', 'video/%');
|
||||
})
|
||||
.count('id as count')
|
||||
.first();
|
||||
|
||||
return parseInt(result.count) || 0;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = new WatermarkGeneratorService();
|
||||
@@ -2,6 +2,7 @@ const sharp = require('sharp');
|
||||
const path = require('path');
|
||||
const fs = require('fs').promises;
|
||||
const { db } = require('../database/db');
|
||||
const { getStoragePath } = require('../config/storage');
|
||||
|
||||
class WatermarkService {
|
||||
constructor() {
|
||||
@@ -232,6 +233,125 @@ class WatermarkService {
|
||||
clearCache() {
|
||||
this.cache.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the watermarks directory path, creating it if needed
|
||||
*/
|
||||
async getWatermarksDir() {
|
||||
const watermarksDir = path.join(getStoragePath(), 'watermarks');
|
||||
try {
|
||||
await fs.access(watermarksDir);
|
||||
} catch {
|
||||
await fs.mkdir(watermarksDir, { recursive: true });
|
||||
}
|
||||
return watermarksDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the file extension from a filename
|
||||
*/
|
||||
getFileExtension(filename) {
|
||||
const ext = path.extname(filename).toLowerCase();
|
||||
// Map common extensions
|
||||
if (ext === '.jpeg') return '.jpg';
|
||||
return ext || '.jpg';
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate watermarked version of a photo and save to disk
|
||||
* @param {Object} photo - Photo object with id, filename, and path info
|
||||
* @param {string} originalPath - Full path to the original image file
|
||||
* @param {Object} settings - Watermark settings (optional, will fetch if not provided)
|
||||
* @returns {Object} { success, watermarkPath, error }
|
||||
*/
|
||||
async generateAndSaveWatermark(photo, originalPath, settings = null) {
|
||||
try {
|
||||
// Get settings if not provided
|
||||
if (!settings) {
|
||||
settings = await this.getWatermarkSettings();
|
||||
}
|
||||
|
||||
// If watermarking is disabled, return early
|
||||
if (!settings || !settings.enabled) {
|
||||
return { success: false, watermarkPath: null, error: 'Watermarking is disabled' };
|
||||
}
|
||||
|
||||
// Verify original file exists
|
||||
try {
|
||||
await fs.access(originalPath);
|
||||
} catch {
|
||||
return { success: false, watermarkPath: null, error: 'Original file not found' };
|
||||
}
|
||||
|
||||
// Generate watermarked buffer using existing method
|
||||
const watermarkedBuffer = await this.applyWatermark(originalPath, settings);
|
||||
|
||||
// Determine output path
|
||||
const watermarksDir = await this.getWatermarksDir();
|
||||
const ext = this.getFileExtension(photo.filename);
|
||||
const outputFilename = `${photo.id}_watermarked${ext}`;
|
||||
const outputPath = path.join(watermarksDir, outputFilename);
|
||||
|
||||
// Write the watermarked image to disk
|
||||
await fs.writeFile(outputPath, watermarkedBuffer);
|
||||
|
||||
// Return relative path for database storage
|
||||
const relativePath = `watermarks/${outputFilename}`;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
watermarkPath: relativePath,
|
||||
error: null
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error generating watermark for photo ${photo.id}:`, error);
|
||||
return {
|
||||
success: false,
|
||||
watermarkPath: null,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a pre-generated watermark file
|
||||
* @param {string} watermarkPath - Relative path to the watermark file
|
||||
* @returns {boolean} - True if deleted successfully
|
||||
*/
|
||||
async deleteWatermarkFile(watermarkPath) {
|
||||
if (!watermarkPath) return false;
|
||||
|
||||
try {
|
||||
const fullPath = path.join(getStoragePath(), watermarkPath);
|
||||
await fs.unlink(fullPath);
|
||||
return true;
|
||||
} catch (error) {
|
||||
// File might not exist, which is fine
|
||||
if (error.code !== 'ENOENT') {
|
||||
console.error('Error deleting watermark file:', error);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a hash of current watermark settings for change detection
|
||||
* @returns {string} - Hash string of settings
|
||||
*/
|
||||
async getSettingsHash() {
|
||||
const settings = await this.getWatermarkSettings();
|
||||
if (!settings) return '';
|
||||
|
||||
const hashData = `${settings.enabled}-${settings.logoPath || ''}-${settings.position}-${settings.opacity}-${settings.size}`;
|
||||
// Simple hash for change detection (not cryptographic)
|
||||
let hash = 0;
|
||||
for (let i = 0; i < hashData.length; i++) {
|
||||
const char = hashData.charCodeAt(i);
|
||||
hash = ((hash << 5) - hash) + char;
|
||||
hash = hash & hash; // Convert to 32bit integer
|
||||
}
|
||||
return hash.toString(16);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = new WatermarkService();
|
||||
Reference in New Issue
Block a user