diff --git a/backend/migrations/core/083_add_presigned_and_webhook_extras.js b/backend/migrations/core/083_add_presigned_and_webhook_extras.js new file mode 100644 index 00000000..af8cb6ec --- /dev/null +++ b/backend/migrations/core/083_add_presigned_and_webhook_extras.js @@ -0,0 +1,49 @@ +/** + * Adds: + * - events.allow_presigned_download — per-event opt-in for the + * presigned-URL "Download All" path (#328 follow-up). Off by default + * because it bypasses watermarks; admins flip it knowingly. + * - webhooks.filter — JSONB predicate evaluated against the payload at + * fire time (#327 follow-up). Empty object = no filter, fire always. + * - webhooks.template — optional ${dot.path} string template applied + * to the request body before signing. NULL = use the default JSON + * envelope (back-compat). + */ + +exports.up = async function up(knex) { + if (await knex.schema.hasTable('events')) { + const hasCol = await knex.schema.hasColumn('events', 'allow_presigned_download'); + if (!hasCol) { + await knex.schema.alterTable('events', (table) => { + table.boolean('allow_presigned_download').notNullable().defaultTo(false); + }); + } + } + + if (await knex.schema.hasTable('webhooks')) { + const hasFilter = await knex.schema.hasColumn('webhooks', 'filter'); + if (!hasFilter) { + await knex.schema.alterTable('webhooks', (table) => { + table.jsonb('filter').notNullable().defaultTo('{}'); + }); + } + const hasTemplate = await knex.schema.hasColumn('webhooks', 'template'); + if (!hasTemplate) { + await knex.schema.alterTable('webhooks', (table) => { + table.text('template').nullable(); + }); + } + } +}; + +exports.down = async function down(knex) { + if (await knex.schema.hasColumn('webhooks', 'template')) { + await knex.schema.alterTable('webhooks', (t) => t.dropColumn('template')); + } + if (await knex.schema.hasColumn('webhooks', 'filter')) { + await knex.schema.alterTable('webhooks', (t) => t.dropColumn('filter')); + } + if (await knex.schema.hasColumn('events', 'allow_presigned_download')) { + await knex.schema.alterTable('events', (t) => t.dropColumn('allow_presigned_download')); + } +}; diff --git a/backend/src/services/s3AutoImporter.js b/backend/src/services/s3AutoImporter.js new file mode 100644 index 00000000..fab5c716 --- /dev/null +++ b/backend/src/services/s3AutoImporter.js @@ -0,0 +1,160 @@ +/** + * S3 prefix walker that mirrors the chokidar `fileWatcher` for S3 mode. + * + * Why: in S3 mode the local file watcher is disabled (no inotify on remote + * objects). Without this, the only way to add photos is the upload API. + * This walker polls every event's storage prefix on a slow cadence and + * imports any new objects into the photos table. + * + * Eventual-consistency gate: an object is only imported after it has been + * SEEN for two consecutive polls. This avoids flapping when a list returns + * a freshly-uploaded object that disappears on the next list (a documented + * S3 behavior on certain backends). + * + * Opt-in via STORAGE_AUTO_IMPORT=true (off by default — admins who don't + * need it shouldn't pay the API call cost). + */ + +const path = require('path'); +const mime = require('mime-types'); +const { db } = require('../database/db'); +const { formatBoolean } = require('../utils/dbCompat'); +const { getStorage } = require('./storage'); +const logger = require('../utils/logger'); + +const POLL_INTERVAL_MS = parseInt(process.env.STORAGE_AUTO_IMPORT_INTERVAL_MS || `${5 * 60 * 1000}`, 10); +const ENABLED = process.env.STORAGE_AUTO_IMPORT === 'true'; + +// Map> — keys we saw on the previous poll. +// On the next poll, any key in BOTH the previous and current snapshots is +// eligible for import. This is the eventual-consistency gate. +const previousSnapshot = new Map(); +let intervalHandle = null; +let stopped = false; + +async function tick() { + if (stopped) return; + const storage = getStorage(); + if (storage.kind() !== 's3') return; // no-op for local fs + + try { + const events = await db('events') + .where({ is_active: formatBoolean(true), is_archived: formatBoolean(false) }) + .select('id', 'slug'); + + for (const event of events) { + await processEvent(event, storage); + } + } catch (err) { + logger.error(`[s3AutoImporter] tick failed: ${err.message}`); + } +} + +async function processEvent(event, storage) { + const prefix = path.posix.join('events/active', event.slug); + let entries = []; + try { + entries = await storage.list(prefix); + } catch (err) { + logger.warn(`[s3AutoImporter] list failed for event ${event.slug}: ${err.message}`); + return; + } + + const currentKeys = new Set(entries.map((e) => e.key)); + const lastSnapshot = previousSnapshot.get(event.id) || new Set(); + + // Eventual-consistency gate: only consider keys present in BOTH the + // previous tick's snapshot and the current one. + const stableKeys = entries.filter((e) => lastSnapshot.has(e.key)); + + if (stableKeys.length > 0) { + // Find keys not yet in photos table. + const stableKeyList = stableKeys.map((e) => e.key); + const eventsActivePrefix = 'events/active/'; + const relativePaths = stableKeyList.map((k) => + k.startsWith(eventsActivePrefix) ? k.slice(eventsActivePrefix.length) : k + ); + + const existing = await db('photos') + .where({ event_id: event.id }) + .whereIn('path', relativePaths) + .select('path'); + const existingPaths = new Set(existing.map((r) => r.path)); + + for (const entry of stableKeys) { + const relativePath = entry.key.startsWith(eventsActivePrefix) + ? entry.key.slice(eventsActivePrefix.length) + : entry.key; + if (existingPaths.has(relativePath)) continue; + + // Skip generated artifacts (thumbnails get their own keys; we don't + // want to re-register them as photos). + const filename = path.basename(entry.key); + if (filename.startsWith('thumb_') || filename.startsWith('hero_')) continue; + if (filename.startsWith('.')) continue; // dotfiles like .download-cache + + const mimeType = mime.lookup(filename) || 'application/octet-stream'; + const isImage = mimeType.startsWith('image/'); + const isVideo = mimeType.startsWith('video/'); + if (!isImage && !isVideo) continue; + + try { + const insertResult = await db('photos').insert({ + event_id: event.id, + filename, + original_filename: filename, + path: relativePath, + type: 'individual', + size_bytes: entry.size, + media_type: isVideo ? 'video' : 'image', + mime_type: mimeType, + source_origin: 'managed', + uploaded_at: new Date().toISOString(), + }).returning('id'); + const photoId = insertResult[0]?.id || insertResult[0]; + + logger.info(`[s3AutoImporter] imported s3://.../${entry.key} → photo #${photoId} for event ${event.slug}`); + + // Webhook (#327): same shape as fileWatcher. + try { + const webhookService = require('./webhookService'); + await webhookService.fire('photo.uploaded', { + event: { id: event.id, slug: event.slug }, + photo: { id: photoId, filename, size_bytes: entry.size, source: 's3-auto-import' }, + }); + } catch (e) { /* non-fatal */ } + } catch (err) { + logger.warn(`[s3AutoImporter] failed to insert ${entry.key}: ${err.message}`); + } + } + } + + previousSnapshot.set(event.id, currentKeys); +} + +function startS3AutoImporter() { + if (!ENABLED) return null; + if (intervalHandle) return intervalHandle; + stopped = false; + // Run once on startup so admins see import activity in logs without + // waiting for the first poll interval. + tick().catch((err) => logger.error(`[s3AutoImporter] initial tick error: ${err.message}`)); + intervalHandle = setInterval(tick, POLL_INTERVAL_MS); + logger.info(`[s3AutoImporter] started — interval=${POLL_INTERVAL_MS}ms`); + return intervalHandle; +} + +function stopS3AutoImporter() { + stopped = true; + if (intervalHandle) { + clearInterval(intervalHandle); + intervalHandle = null; + } + previousSnapshot.clear(); +} + +module.exports = { + startS3AutoImporter, + stopS3AutoImporter, + __test: { tick, processEvent, previousSnapshot, ENABLED, POLL_INTERVAL_MS }, +}; diff --git a/frontend/src/pages/admin/EventDetailsPage.tsx b/frontend/src/pages/admin/EventDetailsPage.tsx index 2cde4c757..7938cf2f 100644 --- a/frontend/src/pages/admin/EventDetailsPage.tsx +++ b/frontend/src/pages/admin/EventDetailsPage.tsx @@ -56,7 +56,7 @@ import { Button, Input, Card, Loading } from '../../components/common'; import { EventCategoryManager, AdminPhotoGrid, AdminPhotoViewer, PhotoFilters, PasswordResetModal, ThemeCustomizerEnhanced, ThemeDisplay, HeroPhotoSelector, FocalPointPicker, PhotoUploadModal, FeedbackSettings, FeedbackModerationPanel, EventRenameDialog, PhotoFilterPanel, PhotoExportMenu, AdminGuestsList } from '../../components/admin'; import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import { eventsService } from '../../services/events.service'; -import { publicSettingsService } from '../../services/publicSettings.service'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; import { api } from '../../config/api'; import { buildResourceUrl, buildShareLinkUrl } from '../../utils/url'; import { isGalleryPublic, normalizeRequirePassword } from '../../utils/accessControl'; @@ -160,6 +160,7 @@ export const EventDetailsPage: React.FC = () => { disable_right_click: boolean; allow_downloads: boolean; watermark_downloads: boolean; + allow_presigned_download: boolean; enable_devtools_protection: boolean; use_canvas_rendering: boolean; // Hero logo settings @@ -196,6 +197,7 @@ export const EventDetailsPage: React.FC = () => { disable_right_click: true, allow_downloads: true, watermark_downloads: false, + allow_presigned_download: false, enable_devtools_protection: true, use_canvas_rendering: false, // Hero logo settings @@ -334,11 +336,7 @@ export const EventDetailsPage: React.FC = () => { } }, [showMediaFilter, photoFilters.media_type]); - // Fetch public settings (for field requirement checks like expiration) - const { data: publicSettings } = useQuery({ - queryKey: ['public-settings'], - queryFn: () => publicSettingsService.getPublicSettings(), - }); + const { data: publicSettings } = usePublicSettings(); const requireExpiration = publicSettings?.event_require_expiration !== false; const phoneFieldEnabled = publicSettings?.event_phone_field_enabled === true; @@ -444,6 +442,7 @@ export const EventDetailsPage: React.FC = () => { disable_right_click: event.disable_right_click ?? true, allow_downloads: event.allow_downloads ?? true, watermark_downloads: event.watermark_downloads ?? false, + allow_presigned_download: (event as { allow_presigned_download?: boolean }).allow_presigned_download ?? false, enable_devtools_protection: event.enable_devtools_protection ?? true, use_canvas_rendering: event.use_canvas_rendering ?? false, // Load hero logo settings from event @@ -581,6 +580,7 @@ export const EventDetailsPage: React.FC = () => { disable_right_click: editForm.disable_right_click, allow_downloads: editForm.allow_downloads, watermark_downloads: editForm.watermark_downloads, + allow_presigned_download: editForm.allow_presigned_download, enable_devtools_protection: editForm.enable_devtools_protection, use_canvas_rendering: editForm.use_canvas_rendering, // Hero logo settings @@ -1277,13 +1277,40 @@ export const EventDetailsPage: React.FC = () => { setEditForm(prev => ({ ...prev, watermark_downloads: e.target.checked }))} + onChange={(e) => setEditForm(prev => ({ + ...prev, + watermark_downloads: e.target.checked, + // Watermarking and presigned URLs are mutually + // exclusive — presigned URLs serve raw bytes from + // S3 without going through the watermark pipeline. + allow_presigned_download: e.target.checked ? false : prev.allow_presigned_download, + }))} className="w-4 h-4 text-primary-600 border-neutral-300 dark:border-neutral-600 rounded focus:ring-primary-500" /> {t('events.watermarkDownloads', 'Add watermark to downloads')} + +