Lets PicPeak write photos, thumbnails, hero images, watermarks, and archive zips to any S3-compatible bucket (AWS S3, MinIO, Cloudflare R2, Backblaze B2, Wasabi, DigitalOcean Spaces) instead of the local filesystem. Selected via STORAGE_BACKEND=local|s3. Architecture - backend/src/services/storage/StorageBackend.js — abstract interface (put/get/exists/stat/delete/list/copy/rename/signedUrl/putFromFile/ getToFile) — typedef-only, documents the contract. - LocalFsStorage.js — wraps fs with atomic-write-via-tmp-rename, path traversal protection, list-as-walker. - S3StorageBackend.js — thin wrapper around the existing S3StorageAdapter (used by backupService) mapping it onto the canonical interface; supports optional STORAGE_S3_PREFIX namespace. - index.js — factory selected by STORAGE_BACKEND with startup ping (HEADs sentinel key on S3, fs.stat on local) so misconfig fails fast before the first request. Consumer refactors (~12 services + routes), each parametrized over the abstraction: - imageProcessor / videoProcessor — pipe Sharp/ffmpeg output through storage.put; expose withLocalCopy() helper for S3-mode regeneration paths that need a local file for sharp/ffmpeg. - archiveService / downloadZipService — finalize zip in tmp dir, then storage.putFromFile. Atomic-rename pattern preserved on local; S3 emulates via copy + delete (worker prunes orphaned .tmp.* on startup). - photoProcessor / photoReplacementService / adminPhotos upload+delete / routes/v1/events.js POST /events/:id/photos / routes/events.js — every upload path now goes storage.putFromFile(temp) → unlink temp. - gallery.js bulk-download (cached + on-the-fly + selected) — managed photos via storage.get, external-mode unchanged. - protectedImages / secureImages / photoResolver — read via storage.get; resolvePhotoStorageKey returns the canonical key. - watermarkService / watermarkGeneratorService — persistent watermarks via storage.put. - fileWatcher — bails out with a clear log warning when STORAGE_BACKEND=s3 (chokidar can't watch S3); auto-import lands via the S3 prefix walker introduced in the follow-up commit. - expirationChecker — small touch (event.expired webhook fire from #327 shipping in the next commit). Migration tooling - backend/scripts/migrate-storage.js — one-shot --dry-run capable script that walks photos.path, thumbnail_path, hero_path, watermark_path and events.archive_path/download_zip_path; streams local → S3; sha256 size-match skip for idempotent re-run; failures CSV. Presigned-URL "Download All" (#328 follow-up shipped in this commit) - routes/gallery.js — when STORAGE_BACKEND=s3 + event.allow_presigned_download + downloads enabled + watermark NOT enabled, /download-all returns a 302 redirect to a 5-minute presigned S3 URL. Per-event opt-in surface ships in the next commit's UI. Tests - backend/__tests__/integration/storageBackend.test.js — parametrized contract suite running against BOTH LocalFs AND MinIO (18 tests, both backends — 36 cases total). - backend/__tests__/integration/imageProcessor.storage.test.js — same parametrized pattern for the image processor (10 tests × 2 backends). - backend/__tests__/integration/backup-s3.test.js — bootstrap fix: drop the redundant initDb() (001_init handles it) and remove schema-drift in configureS3Backup (app_settings has no created_at anymore and the unique constraint is on setting_key alone, not composite). 0/12 → 7/12 (5 remaining are unrelated assertion drift). - backend/src/services/photoResolver.js — mixed-source events (reference mode with managed-uploaded photos) now fall back to managed when external_relpath is missing instead of throwing. - tests/e2e/s3-storage-roundtrip.spec.ts — Playwright spec that auto-skips against local backend; full upload → serve → delete round-trip when run against an S3-mode backend. Server wiring (server.js) - initStorage() called after database init, before rate limiters. - This commit's diff also includes the webhook delivery worker startup and the S3 auto-importer startup. Those features ship in the next two commits — co-located here for one bisectable diff per file. Docs + ops - README §"Storage Backends" — capability matrix, switching playbook, IAM policy snippet, MinIO/R2/B2 examples. - README §"Webhooks" — also added here (full diff bundled). - .env.example — STORAGE_BACKEND + STORAGE_S3_* + STORAGE_AUTO_IMPORT documented; WEBHOOK_* added in the same diff. - .gitignore — re-anchor the existing `storage/` rule to `/storage/` so backend/src/services/storage/ (the new abstraction code) is trackable. The runtime ./storage/ data dir stays ignored. Out of scope for v1 (per the issue): presigned URLs for individual photo display (always streamed for protection middleware), CDN integration, hybrid hot/cold tiers, S3 → local migration, multi-bucket per-event.
411 lines
12 KiB
JavaScript
411 lines
12 KiB
JavaScript
/**
|
|
* WatermarkGeneratorService
|
|
*
|
|
* Handles batch generation of pre-watermarked images for fast serving.
|
|
* This service is responsible for:
|
|
* - Generating watermarks for newly uploaded photos
|
|
* - Regenerating all watermarks when settings change
|
|
* - Tracking regeneration progress
|
|
*/
|
|
|
|
const { db } = require('../database/db');
|
|
const watermarkService = require('./watermarkService');
|
|
const { resolvePhotoStorageKey, resolvePhotoFilePath } = require('./photoResolver');
|
|
const { withLocalCopy } = require('./imageProcessor');
|
|
|
|
class WatermarkGeneratorService {
|
|
constructor() {
|
|
// Track active regeneration jobs
|
|
this.activeJobs = new Map();
|
|
// Batch size for processing (to manage memory)
|
|
this.batchSize = 10;
|
|
// Concurrent processing limit
|
|
this.concurrentLimit = 2;
|
|
}
|
|
|
|
/**
|
|
* Generate watermark for a single photo
|
|
* @param {number} photoId - The photo ID
|
|
* @returns {Object} Result with success status and watermark path
|
|
*/
|
|
async generateForPhoto(photoId) {
|
|
try {
|
|
// Get photo with event info
|
|
const photo = await db('photos')
|
|
.join('events', 'photos.event_id', 'events.id')
|
|
.where('photos.id', photoId)
|
|
.select(
|
|
'photos.*',
|
|
'events.slug',
|
|
'events.source_mode',
|
|
'events.external_path'
|
|
)
|
|
.first();
|
|
|
|
if (!photo) {
|
|
return { success: false, error: 'Photo not found' };
|
|
}
|
|
|
|
// Skip video files
|
|
if (photo.media_type === 'video' || (photo.mime_type && photo.mime_type.startsWith('video/'))) {
|
|
return { success: false, error: 'Videos do not support watermarks' };
|
|
}
|
|
|
|
// Get watermark settings
|
|
const settings = await watermarkService.getWatermarkSettings();
|
|
if (!settings || !settings.enabled) {
|
|
return { success: false, error: 'Watermarking is disabled' };
|
|
}
|
|
|
|
// Resolve the source via the storage backend (managed) or local disk
|
|
// (external reference mode). watermarkService needs a local file path.
|
|
const event = { slug: photo.slug, source_mode: photo.source_mode, external_path: photo.external_path };
|
|
const storageKey = resolvePhotoStorageKey(event, photo);
|
|
const result = storageKey
|
|
? await withLocalCopy(storageKey, (lp) =>
|
|
watermarkService.generateAndSaveWatermark(photo, lp, settings)
|
|
)
|
|
: await watermarkService.generateAndSaveWatermark(photo, resolvePhotoFilePath(event, photo), settings);
|
|
|
|
if (result.success) {
|
|
// Update database with watermark path
|
|
await db('photos')
|
|
.where({ id: photoId })
|
|
.update({
|
|
watermark_path: result.watermarkPath,
|
|
watermark_generated_at: db.fn.now()
|
|
});
|
|
}
|
|
|
|
return result;
|
|
} catch (error) {
|
|
console.error(`Error generating watermark for photo ${photoId}:`, error);
|
|
return { success: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Generate watermarks for all photos in an event
|
|
* @param {number} eventId - The event ID
|
|
* @param {Function} onProgress - Optional callback for progress updates
|
|
* @returns {Object} Result with success count and errors
|
|
*/
|
|
async generateForEvent(eventId, onProgress = null) {
|
|
const results = { total: 0, success: 0, failed: 0, errors: [] };
|
|
|
|
try {
|
|
// Get all photos for the event (excluding videos)
|
|
const photos = await db('photos')
|
|
.join('events', 'photos.event_id', 'events.id')
|
|
.where('photos.event_id', eventId)
|
|
.whereNot(function() {
|
|
this.where('photos.media_type', 'video')
|
|
.orWhere('photos.mime_type', 'like', 'video/%');
|
|
})
|
|
.select(
|
|
'photos.*',
|
|
'events.slug',
|
|
'events.source_mode',
|
|
'events.external_path'
|
|
);
|
|
|
|
results.total = photos.length;
|
|
|
|
if (photos.length === 0) {
|
|
return results;
|
|
}
|
|
|
|
// Get watermark settings once
|
|
const settings = await watermarkService.getWatermarkSettings();
|
|
if (!settings || !settings.enabled) {
|
|
return { ...results, errors: ['Watermarking is disabled'] };
|
|
}
|
|
|
|
// Process in batches
|
|
for (let i = 0; i < photos.length; i += this.batchSize) {
|
|
const batch = photos.slice(i, i + this.batchSize);
|
|
|
|
// Process batch with limited concurrency
|
|
const batchResults = await Promise.all(
|
|
batch.map(photo => this.processPhotoWatermark(photo, settings))
|
|
);
|
|
|
|
// Collect results
|
|
for (const result of batchResults) {
|
|
if (result.success) {
|
|
results.success++;
|
|
} else {
|
|
results.failed++;
|
|
if (result.error) {
|
|
results.errors.push(`Photo ${result.photoId}: ${result.error}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Progress callback
|
|
if (onProgress) {
|
|
onProgress({
|
|
total: results.total,
|
|
processed: results.success + results.failed,
|
|
success: results.success,
|
|
failed: results.failed
|
|
});
|
|
}
|
|
}
|
|
|
|
return results;
|
|
} catch (error) {
|
|
console.error(`Error generating watermarks for event ${eventId}:`, error);
|
|
return { ...results, errors: [...results.errors, error.message] };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Process watermark for a single photo (internal helper)
|
|
*/
|
|
async processPhotoWatermark(photo, settings) {
|
|
try {
|
|
const event = { slug: photo.slug, source_mode: photo.source_mode, external_path: photo.external_path };
|
|
const storageKey = resolvePhotoStorageKey(event, photo);
|
|
const result = storageKey
|
|
? await withLocalCopy(storageKey, (lp) =>
|
|
watermarkService.generateAndSaveWatermark(photo, lp, settings)
|
|
)
|
|
: await watermarkService.generateAndSaveWatermark(photo, resolvePhotoFilePath(event, photo), settings);
|
|
|
|
if (result.success) {
|
|
await db('photos')
|
|
.where({ id: photo.id })
|
|
.update({
|
|
watermark_path: result.watermarkPath,
|
|
watermark_generated_at: db.fn.now()
|
|
});
|
|
}
|
|
|
|
return { ...result, photoId: photo.id };
|
|
} catch (error) {
|
|
return { success: false, photoId: photo.id, error: error.message };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Regenerate watermarks for all photos in the system
|
|
* @param {Function} onProgress - Optional callback for progress updates
|
|
* @returns {Object} Result with success count and errors
|
|
*/
|
|
async regenerateAll(onProgress = null) {
|
|
const jobId = Date.now().toString();
|
|
const results = { jobId, total: 0, success: 0, failed: 0, errors: [], status: 'running' };
|
|
|
|
try {
|
|
this.activeJobs.set(jobId, results);
|
|
|
|
// Get watermark settings
|
|
const settings = await watermarkService.getWatermarkSettings();
|
|
if (!settings || !settings.enabled) {
|
|
results.status = 'completed';
|
|
results.errors.push('Watermarking is disabled');
|
|
return results;
|
|
}
|
|
|
|
// First, clear existing watermarks from DB (the files will be overwritten)
|
|
// This ensures stale paths don't persist if regeneration fails
|
|
|
|
// Get all image photos (exclude videos)
|
|
const photos = await db('photos')
|
|
.join('events', 'photos.event_id', 'events.id')
|
|
.whereNot(function() {
|
|
this.where('photos.media_type', 'video')
|
|
.orWhere('photos.mime_type', 'like', 'video/%');
|
|
})
|
|
.select(
|
|
'photos.*',
|
|
'events.slug',
|
|
'events.source_mode',
|
|
'events.external_path'
|
|
);
|
|
|
|
results.total = photos.length;
|
|
|
|
if (photos.length === 0) {
|
|
results.status = 'completed';
|
|
return results;
|
|
}
|
|
|
|
console.log(`Starting watermark regeneration for ${photos.length} photos`);
|
|
|
|
// Process in batches
|
|
for (let i = 0; i < photos.length; i += this.batchSize) {
|
|
// Check if job was cancelled
|
|
if (!this.activeJobs.has(jobId)) {
|
|
results.status = 'cancelled';
|
|
return results;
|
|
}
|
|
|
|
const batch = photos.slice(i, i + this.batchSize);
|
|
|
|
// Process batch with limited concurrency
|
|
const batchResults = await Promise.all(
|
|
batch.map(photo => this.processPhotoWatermark(photo, settings))
|
|
);
|
|
|
|
// Collect results
|
|
for (const result of batchResults) {
|
|
if (result.success) {
|
|
results.success++;
|
|
} else {
|
|
results.failed++;
|
|
if (result.error && results.errors.length < 50) {
|
|
results.errors.push(`Photo ${result.photoId}: ${result.error}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Update job status
|
|
this.activeJobs.set(jobId, { ...results });
|
|
|
|
// Progress callback
|
|
if (onProgress) {
|
|
onProgress({
|
|
jobId,
|
|
total: results.total,
|
|
processed: results.success + results.failed,
|
|
success: results.success,
|
|
failed: results.failed,
|
|
percentComplete: Math.round(((results.success + results.failed) / results.total) * 100)
|
|
});
|
|
}
|
|
|
|
// Small delay between batches to prevent CPU saturation
|
|
await new Promise(resolve => setTimeout(resolve, 100));
|
|
}
|
|
|
|
results.status = 'completed';
|
|
console.log(`Watermark regeneration completed: ${results.success}/${results.total} successful`);
|
|
|
|
return results;
|
|
} catch (error) {
|
|
console.error('Error during watermark regeneration:', error);
|
|
results.status = 'failed';
|
|
results.errors.push(error.message);
|
|
return results;
|
|
} finally {
|
|
// Clean up job tracking after a delay
|
|
setTimeout(() => {
|
|
this.activeJobs.delete(jobId);
|
|
}, 60000); // Keep for 1 minute for status queries
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Clear all watermarks (when watermarking is disabled)
|
|
*/
|
|
async clearAllWatermarks() {
|
|
try {
|
|
// Get all photos with watermarks
|
|
const photos = await db('photos')
|
|
.whereNotNull('watermark_path')
|
|
.select('id', 'watermark_path');
|
|
|
|
// Delete watermark files
|
|
for (const photo of photos) {
|
|
await watermarkService.deleteWatermarkFile(photo.watermark_path);
|
|
}
|
|
|
|
// Clear database paths
|
|
await db('photos')
|
|
.whereNotNull('watermark_path')
|
|
.update({
|
|
watermark_path: null,
|
|
watermark_generated_at: null
|
|
});
|
|
|
|
console.log(`Cleared ${photos.length} watermarks`);
|
|
return { success: true, cleared: photos.length };
|
|
} catch (error) {
|
|
console.error('Error clearing watermarks:', error);
|
|
return { success: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Delete watermark for a specific photo
|
|
*/
|
|
async deleteForPhoto(photoId) {
|
|
try {
|
|
const photo = await db('photos')
|
|
.where({ id: photoId })
|
|
.select('watermark_path')
|
|
.first();
|
|
|
|
if (photo && photo.watermark_path) {
|
|
await watermarkService.deleteWatermarkFile(photo.watermark_path);
|
|
await db('photos')
|
|
.where({ id: photoId })
|
|
.update({
|
|
watermark_path: null,
|
|
watermark_generated_at: null
|
|
});
|
|
}
|
|
|
|
return { success: true };
|
|
} catch (error) {
|
|
console.error(`Error deleting watermark for photo ${photoId}:`, error);
|
|
return { success: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get status of an active regeneration job
|
|
*/
|
|
getJobStatus(jobId) {
|
|
return this.activeJobs.get(jobId) || null;
|
|
}
|
|
|
|
/**
|
|
* Cancel an active regeneration job
|
|
*/
|
|
cancelJob(jobId) {
|
|
if (this.activeJobs.has(jobId)) {
|
|
this.activeJobs.delete(jobId);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Check if there's an active regeneration job
|
|
*/
|
|
hasActiveJob() {
|
|
for (const [, job] of this.activeJobs) {
|
|
if (job.status === 'running') {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/**
|
|
* Get count of photos needing watermark generation
|
|
*/
|
|
async getPendingCount() {
|
|
const settings = await watermarkService.getWatermarkSettings();
|
|
if (!settings || !settings.enabled) {
|
|
return 0;
|
|
}
|
|
|
|
const result = await db('photos')
|
|
.whereNull('watermark_path')
|
|
.whereNot(function() {
|
|
this.where('media_type', 'video')
|
|
.orWhere('mime_type', 'like', 'video/%');
|
|
})
|
|
.count('id as count')
|
|
.first();
|
|
|
|
return parseInt(result.count) || 0;
|
|
}
|
|
}
|
|
|
|
module.exports = new WatermarkGeneratorService();
|