1b717ce5ed
Lets PicPeak write photos, thumbnails, hero images, watermarks, and archive zips to any S3-compatible bucket (AWS S3, MinIO, Cloudflare R2, Backblaze B2, Wasabi, DigitalOcean Spaces) instead of the local filesystem. Selected via STORAGE_BACKEND=local|s3. Architecture - backend/src/services/storage/StorageBackend.js — abstract interface (put/get/exists/stat/delete/list/copy/rename/signedUrl/putFromFile/ getToFile) — typedef-only, documents the contract. - LocalFsStorage.js — wraps fs with atomic-write-via-tmp-rename, path traversal protection, list-as-walker. - S3StorageBackend.js — thin wrapper around the existing S3StorageAdapter (used by backupService) mapping it onto the canonical interface; supports optional STORAGE_S3_PREFIX namespace. - index.js — factory selected by STORAGE_BACKEND with startup ping (HEADs sentinel key on S3, fs.stat on local) so misconfig fails fast before the first request. Consumer refactors (~12 services + routes), each parametrized over the abstraction: - imageProcessor / videoProcessor — pipe Sharp/ffmpeg output through storage.put; expose withLocalCopy() helper for S3-mode regeneration paths that need a local file for sharp/ffmpeg. - archiveService / downloadZipService — finalize zip in tmp dir, then storage.putFromFile. Atomic-rename pattern preserved on local; S3 emulates via copy + delete (worker prunes orphaned .tmp.* on startup). - photoProcessor / photoReplacementService / adminPhotos upload+delete / routes/v1/events.js POST /events/:id/photos / routes/events.js — every upload path now goes storage.putFromFile(temp) → unlink temp. - gallery.js bulk-download (cached + on-the-fly + selected) — managed photos via storage.get, external-mode unchanged. - protectedImages / secureImages / photoResolver — read via storage.get; resolvePhotoStorageKey returns the canonical key. - watermarkService / watermarkGeneratorService — persistent watermarks via storage.put. - fileWatcher — bails out with a clear log warning when STORAGE_BACKEND=s3 (chokidar can't watch S3); auto-import lands via the S3 prefix walker introduced in the follow-up commit. - expirationChecker — small touch (event.expired webhook fire from #327 shipping in the next commit). Migration tooling - backend/scripts/migrate-storage.js — one-shot --dry-run capable script that walks photos.path, thumbnail_path, hero_path, watermark_path and events.archive_path/download_zip_path; streams local → S3; sha256 size-match skip for idempotent re-run; failures CSV. Presigned-URL "Download All" (#328 follow-up shipped in this commit) - routes/gallery.js — when STORAGE_BACKEND=s3 + event.allow_presigned_download + downloads enabled + watermark NOT enabled, /download-all returns a 302 redirect to a 5-minute presigned S3 URL. Per-event opt-in surface ships in the next commit's UI. Tests - backend/__tests__/integration/storageBackend.test.js — parametrized contract suite running against BOTH LocalFs AND MinIO (18 tests, both backends — 36 cases total). - backend/__tests__/integration/imageProcessor.storage.test.js — same parametrized pattern for the image processor (10 tests × 2 backends). - backend/__tests__/integration/backup-s3.test.js — bootstrap fix: drop the redundant initDb() (001_init handles it) and remove schema-drift in configureS3Backup (app_settings has no created_at anymore and the unique constraint is on setting_key alone, not composite). 0/12 → 7/12 (5 remaining are unrelated assertion drift). - backend/src/services/photoResolver.js — mixed-source events (reference mode with managed-uploaded photos) now fall back to managed when external_relpath is missing instead of throwing. - tests/e2e/s3-storage-roundtrip.spec.ts — Playwright spec that auto-skips against local backend; full upload → serve → delete round-trip when run against an S3-mode backend. Server wiring (server.js) - initStorage() called after database init, before rate limiters. - This commit's diff also includes the webhook delivery worker startup and the S3 auto-importer startup. Those features ship in the next two commits — co-located here for one bisectable diff per file. Docs + ops - README §"Storage Backends" — capability matrix, switching playbook, IAM policy snippet, MinIO/R2/B2 examples. - README §"Webhooks" — also added here (full diff bundled). - .env.example — STORAGE_BACKEND + STORAGE_S3_* + STORAGE_AUTO_IMPORT documented; WEBHOOK_* added in the same diff. - .gitignore — re-anchor the existing `storage/` rule to `/storage/` so backend/src/services/storage/ (the new abstraction code) is trackable. The runtime ./storage/ data dir stays ignored. Out of scope for v1 (per the issue): presigned URLs for individual photo display (always streamed for protection middleware), CDN integration, hybrid hot/cold tiers, S3 → local migration, multi-bucket per-event.
161 lines
5.6 KiB
JavaScript
161 lines
5.6 KiB
JavaScript
const chokidar = require('chokidar');
|
|
const path = require('path');
|
|
const fs = require('fs').promises;
|
|
const { db } = require('../database/db');
|
|
const { formatBoolean } = require('../utils/dbCompat');
|
|
const { generateThumbnail, generateVideoPlaceholder } = require('./imageProcessor');
|
|
const logger = require('../utils/logger');
|
|
const { isVideoMimeType } = require('./videoProcessor');
|
|
const mime = require('mime-types');
|
|
const downloadZipService = require('./downloadZipService');
|
|
|
|
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
|
const WATCH_PATH = () => path.join(getStoragePath(), 'events/active');
|
|
|
|
function startFileWatcher() {
|
|
// Auto-import via filesystem watching only works with the local storage
|
|
// backend. In S3 mode there is no local directory to watch — every photo
|
|
// must enter through the admin upload API. Skip cleanly with a clear log
|
|
// so operators aren't surprised by the missing feature.
|
|
const backend = (process.env.STORAGE_BACKEND || 'local').toLowerCase();
|
|
if (backend !== 'local') {
|
|
logger.warn(`[fileWatcher] auto-import disabled — STORAGE_BACKEND=${backend}. Use the admin upload API instead.`);
|
|
return null;
|
|
}
|
|
|
|
const watcher = chokidar.watch(WATCH_PATH(), {
|
|
ignored: /(^|[\/\\])\../, // ignore dotfiles
|
|
persistent: true,
|
|
awaitWriteFinish: {
|
|
stabilityThreshold: 2000,
|
|
pollInterval: 100
|
|
}
|
|
});
|
|
|
|
watcher
|
|
.on('add', async (filePath) => {
|
|
try {
|
|
await processNewPhoto(filePath);
|
|
} catch (error) {
|
|
logger.error('Error processing new photo:', error);
|
|
}
|
|
})
|
|
.on('unlink', async (filePath) => {
|
|
try {
|
|
await removePhoto(filePath);
|
|
} catch (error) {
|
|
logger.error('Error removing photo:', error);
|
|
}
|
|
});
|
|
|
|
logger.info('File watcher started');
|
|
}
|
|
|
|
async function processNewPhoto(filePath) {
|
|
const relativePath = path.relative(WATCH_PATH(), filePath);
|
|
const pathParts = relativePath.split(path.sep);
|
|
|
|
if (pathParts.length < 2) return; // Not in correct folder structure
|
|
|
|
const eventSlug = pathParts[0];
|
|
const photoType = pathParts[1] === 'collages' ? 'collage' : 'individual';
|
|
|
|
// Check if this is an image or video file
|
|
const ext = path.extname(filePath).toLowerCase();
|
|
const detectedMime = mime.lookup(filePath) || '';
|
|
const isVideo = isVideoMimeType(detectedMime, filePath) || ['.mp4', '.mov', '.webm'].includes(ext);
|
|
if (!isVideo && !['.jpg', '.jpeg', '.png', '.webp'].includes(ext)) return;
|
|
|
|
// Skip temporary upload files
|
|
const filename = path.basename(filePath);
|
|
if (filename.startsWith('temp_')) {
|
|
logger.debug(`Skipping temporary upload file: ${filename}`);
|
|
return;
|
|
}
|
|
|
|
// Find the event
|
|
const event = await db('events').where({ slug: eventSlug, is_active: formatBoolean(true) }).first();
|
|
if (!event) return;
|
|
|
|
// Get file stats
|
|
const stats = await fs.stat(filePath);
|
|
|
|
// Generate thumbnail or placeholder
|
|
let thumbnailPath = null;
|
|
if (isVideo) {
|
|
thumbnailPath = await generateVideoPlaceholder(filename);
|
|
} else {
|
|
thumbnailPath = await generateThumbnail(filePath);
|
|
}
|
|
|
|
// Calculate relative thumbnail path
|
|
const relativeThumbPath = thumbnailPath; // thumbnailPath is already relative to storage root
|
|
const mimeType = detectedMime || (isVideo ? 'video/mp4' : 'image/jpeg');
|
|
|
|
// Check if photo already exists (by filename or path, to handle replacements)
|
|
const existingPhoto = await db('photos')
|
|
.where({ event_id: event.id })
|
|
.where(function() {
|
|
this.where('filename', path.basename(filePath))
|
|
.orWhere('path', relativePath);
|
|
})
|
|
.first();
|
|
|
|
if (!existingPhoto) {
|
|
// Add to database
|
|
const insertResult = await db('photos').insert({
|
|
event_id: event.id,
|
|
filename: path.basename(filePath),
|
|
path: relativePath,
|
|
thumbnail_path: relativeThumbPath,
|
|
type: isVideo ? 'video' : photoType,
|
|
size_bytes: stats.size,
|
|
mime_type: mimeType
|
|
}).returning('id');
|
|
const photoId = insertResult[0]?.id || insertResult[0];
|
|
|
|
logger.info(`Added new photo: ${relativePath}`);
|
|
downloadZipService.invalidate(event.id);
|
|
|
|
// Webhook (#327) — auto-import path. Only fires in local mode since
|
|
// the watcher is disabled in S3 mode.
|
|
try {
|
|
const webhookService = require('./webhookService');
|
|
await webhookService.fire('photo.uploaded', {
|
|
event: { id: event.id, slug: event.slug, event_name: event.event_name },
|
|
photo: { id: photoId, filename: path.basename(filePath), size_bytes: stats.size, source: 'auto-import' },
|
|
});
|
|
} catch (e) { /* non-fatal */ }
|
|
} else {
|
|
logger.debug(`Photo already exists: ${relativePath}`);
|
|
}
|
|
}
|
|
|
|
async function removePhoto(filePath) {
|
|
const relativePath = path.relative(WATCH_PATH(), filePath);
|
|
|
|
// Look up event before deleting to invalidate zip cache
|
|
const photo = await db('photos').where({ path: relativePath }).first();
|
|
|
|
// Remove from database
|
|
await db('photos').where({ path: relativePath }).delete();
|
|
|
|
if (photo) {
|
|
downloadZipService.invalidate(photo.event_id);
|
|
|
|
// Webhook (#327) — fire only if the row actually existed.
|
|
try {
|
|
const event = await db('events').where({ id: photo.event_id }).first();
|
|
const webhookService = require('./webhookService');
|
|
await webhookService.fire('photo.deleted', {
|
|
event: { id: photo.event_id, slug: event?.slug, event_name: event?.event_name },
|
|
photo: { id: photo.id, filename: photo.filename, source: 'auto-import' },
|
|
});
|
|
} catch (e) { /* non-fatal */ }
|
|
}
|
|
|
|
logger.info(`Removed photo: ${relativePath}`);
|
|
}
|
|
|
|
module.exports = { startFileWatcher };
|