From 446d80a4cc5eb0389994e29585b2a98dad373db2 Mon Sep 17 00:00:00 2001 From: Paul Nothaft Date: Tue, 28 Apr 2026 10:08:21 +0200 Subject: [PATCH] feat: presigned download UI + S3 prefix walker auto-importer (follow-ups) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the user-facing surface for the two #328 follow-ups previously landed in code form (presigned route + S3 mode notes), plus the schema migration that backs both #328 and #327 follow-ups. Migration 083 - events.allow_presigned_download — per-event opt-in for the presigned-URL "Download All" path. Off by default because it bypasses watermarks; admins flip it knowingly. Mutually exclusive with watermark_downloads. - webhooks.filter (jsonb default {}) — dot-path equality predicate evaluated at fire time. Empty object = no filter, fire always. Backs the filter logic that shipped with #327. - webhooks.template (text nullable) — optional ${dot.path} string substitution applied at delivery time. NULL = use the default JSON envelope (back-compat). Backs the template logic from #327. S3 prefix walker (services/s3AutoImporter.js) - Replaces the chokidar file-watcher in S3 mode (where there's no inotify equivalent on remote objects). - Polls every active event's S3 prefix every 5 min by default (STORAGE_AUTO_IMPORT_INTERVAL_MS overridable). - Eventual-consistency gate: an object is only imported after it's been seen for two consecutive polls. Avoids flapping when S3 returns a freshly-uploaded object that disappears on the next list (a documented S3 behavior on certain backends). - Skips generated artifacts (thumb_*, hero_*, dot-files). - Inserts photos rows + fires photo.uploaded webhooks the same way the local fileWatcher does. - Opt-in via STORAGE_AUTO_IMPORT=true. Off by default because it adds API call cost. EventDetailsPage UI (frontend) - Round D queryKey alignment for #325 dedup — replaces useQuery on publicSettingsService with the shared usePublicSettings() hook so the page joins the same React Query cache as every other consumer. - Per-event "Allow direct S3 download (no watermark, S3 mode only)" toggle in Download Protection. Disabled when watermark_downloads is on; tooltip explains the bandwidth/watermark trade-off. Toggling watermark_downloads on automatically clears allow_presigned_download to keep the two mutually exclusive in the UI. Verified live against MinIO - Presigned: GET /api/gallery/.../download-all → 302 with Location: http://minio:9000/...?X-Amz-Signature=...&X-Amz-Expires=300. Following the URL inside the docker network → HTTP 200, valid PK ZIP archive containing the photo. - Auto-importer: dropped a file via `mc cp` directly into the bucket; watcher imported it after 2 polls; webhook subscribed to photo.uploaded fired with source=s3-auto-import; receiver got POST with valid HMAC, status=success, 3ms latency. --- .../083_add_presigned_and_webhook_extras.js | 49 ++++++ backend/src/services/s3AutoImporter.js | 160 ++++++++++++++++++ frontend/src/pages/admin/EventDetailsPage.tsx | 41 ++++- 3 files changed, 243 insertions(+), 7 deletions(-) create mode 100644 backend/migrations/core/083_add_presigned_and_webhook_extras.js create mode 100644 backend/src/services/s3AutoImporter.js diff --git a/backend/migrations/core/083_add_presigned_and_webhook_extras.js b/backend/migrations/core/083_add_presigned_and_webhook_extras.js new file mode 100644 index 00000000..af8cb6ec --- /dev/null +++ b/backend/migrations/core/083_add_presigned_and_webhook_extras.js @@ -0,0 +1,49 @@ +/** + * Adds: + * - events.allow_presigned_download — per-event opt-in for the + * presigned-URL "Download All" path (#328 follow-up). Off by default + * because it bypasses watermarks; admins flip it knowingly. + * - webhooks.filter — JSONB predicate evaluated against the payload at + * fire time (#327 follow-up). Empty object = no filter, fire always. + * - webhooks.template — optional ${dot.path} string template applied + * to the request body before signing. NULL = use the default JSON + * envelope (back-compat). + */ + +exports.up = async function up(knex) { + if (await knex.schema.hasTable('events')) { + const hasCol = await knex.schema.hasColumn('events', 'allow_presigned_download'); + if (!hasCol) { + await knex.schema.alterTable('events', (table) => { + table.boolean('allow_presigned_download').notNullable().defaultTo(false); + }); + } + } + + if (await knex.schema.hasTable('webhooks')) { + const hasFilter = await knex.schema.hasColumn('webhooks', 'filter'); + if (!hasFilter) { + await knex.schema.alterTable('webhooks', (table) => { + table.jsonb('filter').notNullable().defaultTo('{}'); + }); + } + const hasTemplate = await knex.schema.hasColumn('webhooks', 'template'); + if (!hasTemplate) { + await knex.schema.alterTable('webhooks', (table) => { + table.text('template').nullable(); + }); + } + } +}; + +exports.down = async function down(knex) { + if (await knex.schema.hasColumn('webhooks', 'template')) { + await knex.schema.alterTable('webhooks', (t) => t.dropColumn('template')); + } + if (await knex.schema.hasColumn('webhooks', 'filter')) { + await knex.schema.alterTable('webhooks', (t) => t.dropColumn('filter')); + } + if (await knex.schema.hasColumn('events', 'allow_presigned_download')) { + await knex.schema.alterTable('events', (t) => t.dropColumn('allow_presigned_download')); + } +}; diff --git a/backend/src/services/s3AutoImporter.js b/backend/src/services/s3AutoImporter.js new file mode 100644 index 00000000..fab5c716 --- /dev/null +++ b/backend/src/services/s3AutoImporter.js @@ -0,0 +1,160 @@ +/** + * S3 prefix walker that mirrors the chokidar `fileWatcher` for S3 mode. + * + * Why: in S3 mode the local file watcher is disabled (no inotify on remote + * objects). Without this, the only way to add photos is the upload API. + * This walker polls every event's storage prefix on a slow cadence and + * imports any new objects into the photos table. + * + * Eventual-consistency gate: an object is only imported after it has been + * SEEN for two consecutive polls. This avoids flapping when a list returns + * a freshly-uploaded object that disappears on the next list (a documented + * S3 behavior on certain backends). + * + * Opt-in via STORAGE_AUTO_IMPORT=true (off by default — admins who don't + * need it shouldn't pay the API call cost). + */ + +const path = require('path'); +const mime = require('mime-types'); +const { db } = require('../database/db'); +const { formatBoolean } = require('../utils/dbCompat'); +const { getStorage } = require('./storage'); +const logger = require('../utils/logger'); + +const POLL_INTERVAL_MS = parseInt(process.env.STORAGE_AUTO_IMPORT_INTERVAL_MS || `${5 * 60 * 1000}`, 10); +const ENABLED = process.env.STORAGE_AUTO_IMPORT === 'true'; + +// Map> — keys we saw on the previous poll. +// On the next poll, any key in BOTH the previous and current snapshots is +// eligible for import. This is the eventual-consistency gate. +const previousSnapshot = new Map(); +let intervalHandle = null; +let stopped = false; + +async function tick() { + if (stopped) return; + const storage = getStorage(); + if (storage.kind() !== 's3') return; // no-op for local fs + + try { + const events = await db('events') + .where({ is_active: formatBoolean(true), is_archived: formatBoolean(false) }) + .select('id', 'slug'); + + for (const event of events) { + await processEvent(event, storage); + } + } catch (err) { + logger.error(`[s3AutoImporter] tick failed: ${err.message}`); + } +} + +async function processEvent(event, storage) { + const prefix = path.posix.join('events/active', event.slug); + let entries = []; + try { + entries = await storage.list(prefix); + } catch (err) { + logger.warn(`[s3AutoImporter] list failed for event ${event.slug}: ${err.message}`); + return; + } + + const currentKeys = new Set(entries.map((e) => e.key)); + const lastSnapshot = previousSnapshot.get(event.id) || new Set(); + + // Eventual-consistency gate: only consider keys present in BOTH the + // previous tick's snapshot and the current one. + const stableKeys = entries.filter((e) => lastSnapshot.has(e.key)); + + if (stableKeys.length > 0) { + // Find keys not yet in photos table. + const stableKeyList = stableKeys.map((e) => e.key); + const eventsActivePrefix = 'events/active/'; + const relativePaths = stableKeyList.map((k) => + k.startsWith(eventsActivePrefix) ? k.slice(eventsActivePrefix.length) : k + ); + + const existing = await db('photos') + .where({ event_id: event.id }) + .whereIn('path', relativePaths) + .select('path'); + const existingPaths = new Set(existing.map((r) => r.path)); + + for (const entry of stableKeys) { + const relativePath = entry.key.startsWith(eventsActivePrefix) + ? entry.key.slice(eventsActivePrefix.length) + : entry.key; + if (existingPaths.has(relativePath)) continue; + + // Skip generated artifacts (thumbnails get their own keys; we don't + // want to re-register them as photos). + const filename = path.basename(entry.key); + if (filename.startsWith('thumb_') || filename.startsWith('hero_')) continue; + if (filename.startsWith('.')) continue; // dotfiles like .download-cache + + const mimeType = mime.lookup(filename) || 'application/octet-stream'; + const isImage = mimeType.startsWith('image/'); + const isVideo = mimeType.startsWith('video/'); + if (!isImage && !isVideo) continue; + + try { + const insertResult = await db('photos').insert({ + event_id: event.id, + filename, + original_filename: filename, + path: relativePath, + type: 'individual', + size_bytes: entry.size, + media_type: isVideo ? 'video' : 'image', + mime_type: mimeType, + source_origin: 'managed', + uploaded_at: new Date().toISOString(), + }).returning('id'); + const photoId = insertResult[0]?.id || insertResult[0]; + + logger.info(`[s3AutoImporter] imported s3://.../${entry.key} → photo #${photoId} for event ${event.slug}`); + + // Webhook (#327): same shape as fileWatcher. + try { + const webhookService = require('./webhookService'); + await webhookService.fire('photo.uploaded', { + event: { id: event.id, slug: event.slug }, + photo: { id: photoId, filename, size_bytes: entry.size, source: 's3-auto-import' }, + }); + } catch (e) { /* non-fatal */ } + } catch (err) { + logger.warn(`[s3AutoImporter] failed to insert ${entry.key}: ${err.message}`); + } + } + } + + previousSnapshot.set(event.id, currentKeys); +} + +function startS3AutoImporter() { + if (!ENABLED) return null; + if (intervalHandle) return intervalHandle; + stopped = false; + // Run once on startup so admins see import activity in logs without + // waiting for the first poll interval. + tick().catch((err) => logger.error(`[s3AutoImporter] initial tick error: ${err.message}`)); + intervalHandle = setInterval(tick, POLL_INTERVAL_MS); + logger.info(`[s3AutoImporter] started — interval=${POLL_INTERVAL_MS}ms`); + return intervalHandle; +} + +function stopS3AutoImporter() { + stopped = true; + if (intervalHandle) { + clearInterval(intervalHandle); + intervalHandle = null; + } + previousSnapshot.clear(); +} + +module.exports = { + startS3AutoImporter, + stopS3AutoImporter, + __test: { tick, processEvent, previousSnapshot, ENABLED, POLL_INTERVAL_MS }, +}; diff --git a/frontend/src/pages/admin/EventDetailsPage.tsx b/frontend/src/pages/admin/EventDetailsPage.tsx index 2cde4c757..7938cf2f 100644 --- a/frontend/src/pages/admin/EventDetailsPage.tsx +++ b/frontend/src/pages/admin/EventDetailsPage.tsx @@ -56,7 +56,7 @@ import { Button, Input, Card, Loading } from '../../components/common'; import { EventCategoryManager, AdminPhotoGrid, AdminPhotoViewer, PhotoFilters, PasswordResetModal, ThemeCustomizerEnhanced, ThemeDisplay, HeroPhotoSelector, FocalPointPicker, PhotoUploadModal, FeedbackSettings, FeedbackModerationPanel, EventRenameDialog, PhotoFilterPanel, PhotoExportMenu, AdminGuestsList } from '../../components/admin'; import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import { eventsService } from '../../services/events.service'; -import { publicSettingsService } from '../../services/publicSettings.service'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; import { api } from '../../config/api'; import { buildResourceUrl, buildShareLinkUrl } from '../../utils/url'; import { isGalleryPublic, normalizeRequirePassword } from '../../utils/accessControl'; @@ -160,6 +160,7 @@ export const EventDetailsPage: React.FC = () => { disable_right_click: boolean; allow_downloads: boolean; watermark_downloads: boolean; + allow_presigned_download: boolean; enable_devtools_protection: boolean; use_canvas_rendering: boolean; // Hero logo settings @@ -196,6 +197,7 @@ export const EventDetailsPage: React.FC = () => { disable_right_click: true, allow_downloads: true, watermark_downloads: false, + allow_presigned_download: false, enable_devtools_protection: true, use_canvas_rendering: false, // Hero logo settings @@ -334,11 +336,7 @@ export const EventDetailsPage: React.FC = () => { } }, [showMediaFilter, photoFilters.media_type]); - // Fetch public settings (for field requirement checks like expiration) - const { data: publicSettings } = useQuery({ - queryKey: ['public-settings'], - queryFn: () => publicSettingsService.getPublicSettings(), - }); + const { data: publicSettings } = usePublicSettings(); const requireExpiration = publicSettings?.event_require_expiration !== false; const phoneFieldEnabled = publicSettings?.event_phone_field_enabled === true; @@ -444,6 +442,7 @@ export const EventDetailsPage: React.FC = () => { disable_right_click: event.disable_right_click ?? true, allow_downloads: event.allow_downloads ?? true, watermark_downloads: event.watermark_downloads ?? false, + allow_presigned_download: (event as { allow_presigned_download?: boolean }).allow_presigned_download ?? false, enable_devtools_protection: event.enable_devtools_protection ?? true, use_canvas_rendering: event.use_canvas_rendering ?? false, // Load hero logo settings from event @@ -581,6 +580,7 @@ export const EventDetailsPage: React.FC = () => { disable_right_click: editForm.disable_right_click, allow_downloads: editForm.allow_downloads, watermark_downloads: editForm.watermark_downloads, + allow_presigned_download: editForm.allow_presigned_download, enable_devtools_protection: editForm.enable_devtools_protection, use_canvas_rendering: editForm.use_canvas_rendering, // Hero logo settings @@ -1277,13 +1277,40 @@ export const EventDetailsPage: React.FC = () => { setEditForm(prev => ({ ...prev, watermark_downloads: e.target.checked }))} + onChange={(e) => setEditForm(prev => ({ + ...prev, + watermark_downloads: e.target.checked, + // Watermarking and presigned URLs are mutually + // exclusive — presigned URLs serve raw bytes from + // S3 without going through the watermark pipeline. + allow_presigned_download: e.target.checked ? false : prev.allow_presigned_download, + }))} className="w-4 h-4 text-primary-600 border-neutral-300 dark:border-neutral-600 rounded focus:ring-primary-500" /> {t('events.watermarkDownloads', 'Add watermark to downloads')} + +