feat: presigned download UI + S3 prefix walker auto-importer (follow-ups)
Closes the user-facing surface for the two #328 follow-ups previously landed in code form (presigned route + S3 mode notes), plus the schema migration that backs both #328 and #327 follow-ups. Migration 083 - events.allow_presigned_download — per-event opt-in for the presigned-URL "Download All" path. Off by default because it bypasses watermarks; admins flip it knowingly. Mutually exclusive with watermark_downloads. - webhooks.filter (jsonb default {}) — dot-path equality predicate evaluated at fire time. Empty object = no filter, fire always. Backs the filter logic that shipped with #327. - webhooks.template (text nullable) — optional ${dot.path} string substitution applied at delivery time. NULL = use the default JSON envelope (back-compat). Backs the template logic from #327. S3 prefix walker (services/s3AutoImporter.js) - Replaces the chokidar file-watcher in S3 mode (where there's no inotify equivalent on remote objects). - Polls every active event's S3 prefix every 5 min by default (STORAGE_AUTO_IMPORT_INTERVAL_MS overridable). - Eventual-consistency gate: an object is only imported after it's been seen for two consecutive polls. Avoids flapping when S3 returns a freshly-uploaded object that disappears on the next list (a documented S3 behavior on certain backends). - Skips generated artifacts (thumb_*, hero_*, dot-files). - Inserts photos rows + fires photo.uploaded webhooks the same way the local fileWatcher does. - Opt-in via STORAGE_AUTO_IMPORT=true. Off by default because it adds API call cost. EventDetailsPage UI (frontend) - Round D queryKey alignment for #325 dedup — replaces useQuery on publicSettingsService with the shared usePublicSettings() hook so the page joins the same React Query cache as every other consumer. - Per-event "Allow direct S3 download (no watermark, S3 mode only)" toggle in Download Protection. Disabled when watermark_downloads is on; tooltip explains the bandwidth/watermark trade-off. Toggling watermark_downloads on automatically clears allow_presigned_download to keep the two mutually exclusive in the UI. Verified live against MinIO - Presigned: GET /api/gallery/.../download-all → 302 with Location: http://minio:9000/...?X-Amz-Signature=...&X-Amz-Expires=300. Following the URL inside the docker network → HTTP 200, valid PK ZIP archive containing the photo. - Auto-importer: dropped a file via `mc cp` directly into the bucket; watcher imported it after 2 polls; webhook subscribed to photo.uploaded fired with source=s3-auto-import; receiver got POST with valid HMAC, status=success, 3ms latency.
This commit is contained in:
Paul Nothaft
@@ -0,0 +1,49 @@
|
||||
/**
|
||||
* Adds:
|
||||
* - events.allow_presigned_download — per-event opt-in for the
|
||||
* presigned-URL "Download All" path (#328 follow-up). Off by default
|
||||
* because it bypasses watermarks; admins flip it knowingly.
|
||||
* - webhooks.filter — JSONB predicate evaluated against the payload at
|
||||
* fire time (#327 follow-up). Empty object = no filter, fire always.
|
||||
* - webhooks.template — optional ${dot.path} string template applied
|
||||
* to the request body before signing. NULL = use the default JSON
|
||||
* envelope (back-compat).
|
||||
*/
|
||||
|
||||
exports.up = async function up(knex) {
|
||||
if (await knex.schema.hasTable('events')) {
|
||||
const hasCol = await knex.schema.hasColumn('events', 'allow_presigned_download');
|
||||
if (!hasCol) {
|
||||
await knex.schema.alterTable('events', (table) => {
|
||||
table.boolean('allow_presigned_download').notNullable().defaultTo(false);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (await knex.schema.hasTable('webhooks')) {
|
||||
const hasFilter = await knex.schema.hasColumn('webhooks', 'filter');
|
||||
if (!hasFilter) {
|
||||
await knex.schema.alterTable('webhooks', (table) => {
|
||||
table.jsonb('filter').notNullable().defaultTo('{}');
|
||||
});
|
||||
}
|
||||
const hasTemplate = await knex.schema.hasColumn('webhooks', 'template');
|
||||
if (!hasTemplate) {
|
||||
await knex.schema.alterTable('webhooks', (table) => {
|
||||
table.text('template').nullable();
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
exports.down = async function down(knex) {
|
||||
if (await knex.schema.hasColumn('webhooks', 'template')) {
|
||||
await knex.schema.alterTable('webhooks', (t) => t.dropColumn('template'));
|
||||
}
|
||||
if (await knex.schema.hasColumn('webhooks', 'filter')) {
|
||||
await knex.schema.alterTable('webhooks', (t) => t.dropColumn('filter'));
|
||||
}
|
||||
if (await knex.schema.hasColumn('events', 'allow_presigned_download')) {
|
||||
await knex.schema.alterTable('events', (t) => t.dropColumn('allow_presigned_download'));
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* S3 prefix walker that mirrors the chokidar `fileWatcher` for S3 mode.
|
||||
*
|
||||
* Why: in S3 mode the local file watcher is disabled (no inotify on remote
|
||||
* objects). Without this, the only way to add photos is the upload API.
|
||||
* This walker polls every event's storage prefix on a slow cadence and
|
||||
* imports any new objects into the photos table.
|
||||
*
|
||||
* Eventual-consistency gate: an object is only imported after it has been
|
||||
* SEEN for two consecutive polls. This avoids flapping when a list returns
|
||||
* a freshly-uploaded object that disappears on the next list (a documented
|
||||
* S3 behavior on certain backends).
|
||||
*
|
||||
* Opt-in via STORAGE_AUTO_IMPORT=true (off by default — admins who don't
|
||||
* need it shouldn't pay the API call cost).
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
const mime = require('mime-types');
|
||||
const { db } = require('../database/db');
|
||||
const { formatBoolean } = require('../utils/dbCompat');
|
||||
const { getStorage } = require('./storage');
|
||||
const logger = require('../utils/logger');
|
||||
|
||||
const POLL_INTERVAL_MS = parseInt(process.env.STORAGE_AUTO_IMPORT_INTERVAL_MS || `${5 * 60 * 1000}`, 10);
|
||||
const ENABLED = process.env.STORAGE_AUTO_IMPORT === 'true';
|
||||
|
||||
// Map<eventId, Set<storageKey>> — keys we saw on the previous poll.
|
||||
// On the next poll, any key in BOTH the previous and current snapshots is
|
||||
// eligible for import. This is the eventual-consistency gate.
|
||||
const previousSnapshot = new Map();
|
||||
let intervalHandle = null;
|
||||
let stopped = false;
|
||||
|
||||
async function tick() {
|
||||
if (stopped) return;
|
||||
const storage = getStorage();
|
||||
if (storage.kind() !== 's3') return; // no-op for local fs
|
||||
|
||||
try {
|
||||
const events = await db('events')
|
||||
.where({ is_active: formatBoolean(true), is_archived: formatBoolean(false) })
|
||||
.select('id', 'slug');
|
||||
|
||||
for (const event of events) {
|
||||
await processEvent(event, storage);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error(`[s3AutoImporter] tick failed: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function processEvent(event, storage) {
|
||||
const prefix = path.posix.join('events/active', event.slug);
|
||||
let entries = [];
|
||||
try {
|
||||
entries = await storage.list(prefix);
|
||||
} catch (err) {
|
||||
logger.warn(`[s3AutoImporter] list failed for event ${event.slug}: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const currentKeys = new Set(entries.map((e) => e.key));
|
||||
const lastSnapshot = previousSnapshot.get(event.id) || new Set();
|
||||
|
||||
// Eventual-consistency gate: only consider keys present in BOTH the
|
||||
// previous tick's snapshot and the current one.
|
||||
const stableKeys = entries.filter((e) => lastSnapshot.has(e.key));
|
||||
|
||||
if (stableKeys.length > 0) {
|
||||
// Find keys not yet in photos table.
|
||||
const stableKeyList = stableKeys.map((e) => e.key);
|
||||
const eventsActivePrefix = 'events/active/';
|
||||
const relativePaths = stableKeyList.map((k) =>
|
||||
k.startsWith(eventsActivePrefix) ? k.slice(eventsActivePrefix.length) : k
|
||||
);
|
||||
|
||||
const existing = await db('photos')
|
||||
.where({ event_id: event.id })
|
||||
.whereIn('path', relativePaths)
|
||||
.select('path');
|
||||
const existingPaths = new Set(existing.map((r) => r.path));
|
||||
|
||||
for (const entry of stableKeys) {
|
||||
const relativePath = entry.key.startsWith(eventsActivePrefix)
|
||||
? entry.key.slice(eventsActivePrefix.length)
|
||||
: entry.key;
|
||||
if (existingPaths.has(relativePath)) continue;
|
||||
|
||||
// Skip generated artifacts (thumbnails get their own keys; we don't
|
||||
// want to re-register them as photos).
|
||||
const filename = path.basename(entry.key);
|
||||
if (filename.startsWith('thumb_') || filename.startsWith('hero_')) continue;
|
||||
if (filename.startsWith('.')) continue; // dotfiles like .download-cache
|
||||
|
||||
const mimeType = mime.lookup(filename) || 'application/octet-stream';
|
||||
const isImage = mimeType.startsWith('image/');
|
||||
const isVideo = mimeType.startsWith('video/');
|
||||
if (!isImage && !isVideo) continue;
|
||||
|
||||
try {
|
||||
const insertResult = await db('photos').insert({
|
||||
event_id: event.id,
|
||||
filename,
|
||||
original_filename: filename,
|
||||
path: relativePath,
|
||||
type: 'individual',
|
||||
size_bytes: entry.size,
|
||||
media_type: isVideo ? 'video' : 'image',
|
||||
mime_type: mimeType,
|
||||
source_origin: 'managed',
|
||||
uploaded_at: new Date().toISOString(),
|
||||
}).returning('id');
|
||||
const photoId = insertResult[0]?.id || insertResult[0];
|
||||
|
||||
logger.info(`[s3AutoImporter] imported s3://.../${entry.key} → photo #${photoId} for event ${event.slug}`);
|
||||
|
||||
// Webhook (#327): same shape as fileWatcher.
|
||||
try {
|
||||
const webhookService = require('./webhookService');
|
||||
await webhookService.fire('photo.uploaded', {
|
||||
event: { id: event.id, slug: event.slug },
|
||||
photo: { id: photoId, filename, size_bytes: entry.size, source: 's3-auto-import' },
|
||||
});
|
||||
} catch (e) { /* non-fatal */ }
|
||||
} catch (err) {
|
||||
logger.warn(`[s3AutoImporter] failed to insert ${entry.key}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
previousSnapshot.set(event.id, currentKeys);
|
||||
}
|
||||
|
||||
function startS3AutoImporter() {
|
||||
if (!ENABLED) return null;
|
||||
if (intervalHandle) return intervalHandle;
|
||||
stopped = false;
|
||||
// Run once on startup so admins see import activity in logs without
|
||||
// waiting for the first poll interval.
|
||||
tick().catch((err) => logger.error(`[s3AutoImporter] initial tick error: ${err.message}`));
|
||||
intervalHandle = setInterval(tick, POLL_INTERVAL_MS);
|
||||
logger.info(`[s3AutoImporter] started — interval=${POLL_INTERVAL_MS}ms`);
|
||||
return intervalHandle;
|
||||
}
|
||||
|
||||
function stopS3AutoImporter() {
|
||||
stopped = true;
|
||||
if (intervalHandle) {
|
||||
clearInterval(intervalHandle);
|
||||
intervalHandle = null;
|
||||
}
|
||||
previousSnapshot.clear();
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
startS3AutoImporter,
|
||||
stopS3AutoImporter,
|
||||
__test: { tick, processEvent, previousSnapshot, ENABLED, POLL_INTERVAL_MS },
|
||||
};
|
||||
@@ -56,7 +56,7 @@ import { Button, Input, Card, Loading } from '../../components/common';
|
||||
import { EventCategoryManager, AdminPhotoGrid, AdminPhotoViewer, PhotoFilters, PasswordResetModal, ThemeCustomizerEnhanced, ThemeDisplay, HeroPhotoSelector, FocalPointPicker, PhotoUploadModal, FeedbackSettings, FeedbackModerationPanel, EventRenameDialog, PhotoFilterPanel, PhotoExportMenu, AdminGuestsList } from '../../components/admin';
|
||||
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
import { eventsService } from '../../services/events.service';
|
||||
import { publicSettingsService } from '../../services/publicSettings.service';
|
||||
import { usePublicSettings } from '../../hooks/usePublicSettings';
|
||||
import { api } from '../../config/api';
|
||||
import { buildResourceUrl, buildShareLinkUrl } from '../../utils/url';
|
||||
import { isGalleryPublic, normalizeRequirePassword } from '../../utils/accessControl';
|
||||
@@ -160,6 +160,7 @@ export const EventDetailsPage: React.FC = () => {
|
||||
disable_right_click: boolean;
|
||||
allow_downloads: boolean;
|
||||
watermark_downloads: boolean;
|
||||
allow_presigned_download: boolean;
|
||||
enable_devtools_protection: boolean;
|
||||
use_canvas_rendering: boolean;
|
||||
// Hero logo settings
|
||||
@@ -196,6 +197,7 @@ export const EventDetailsPage: React.FC = () => {
|
||||
disable_right_click: true,
|
||||
allow_downloads: true,
|
||||
watermark_downloads: false,
|
||||
allow_presigned_download: false,
|
||||
enable_devtools_protection: true,
|
||||
use_canvas_rendering: false,
|
||||
// Hero logo settings
|
||||
@@ -334,11 +336,7 @@ export const EventDetailsPage: React.FC = () => {
|
||||
}
|
||||
}, [showMediaFilter, photoFilters.media_type]);
|
||||
|
||||
// Fetch public settings (for field requirement checks like expiration)
|
||||
const { data: publicSettings } = useQuery({
|
||||
queryKey: ['public-settings'],
|
||||
queryFn: () => publicSettingsService.getPublicSettings(),
|
||||
});
|
||||
const { data: publicSettings } = usePublicSettings();
|
||||
const requireExpiration = publicSettings?.event_require_expiration !== false;
|
||||
const phoneFieldEnabled = publicSettings?.event_phone_field_enabled === true;
|
||||
|
||||
@@ -444,6 +442,7 @@ export const EventDetailsPage: React.FC = () => {
|
||||
disable_right_click: event.disable_right_click ?? true,
|
||||
allow_downloads: event.allow_downloads ?? true,
|
||||
watermark_downloads: event.watermark_downloads ?? false,
|
||||
allow_presigned_download: (event as { allow_presigned_download?: boolean }).allow_presigned_download ?? false,
|
||||
enable_devtools_protection: event.enable_devtools_protection ?? true,
|
||||
use_canvas_rendering: event.use_canvas_rendering ?? false,
|
||||
// Load hero logo settings from event
|
||||
@@ -581,6 +580,7 @@ export const EventDetailsPage: React.FC = () => {
|
||||
disable_right_click: editForm.disable_right_click,
|
||||
allow_downloads: editForm.allow_downloads,
|
||||
watermark_downloads: editForm.watermark_downloads,
|
||||
allow_presigned_download: editForm.allow_presigned_download,
|
||||
enable_devtools_protection: editForm.enable_devtools_protection,
|
||||
use_canvas_rendering: editForm.use_canvas_rendering,
|
||||
// Hero logo settings
|
||||
@@ -1277,13 +1277,40 @@ export const EventDetailsPage: React.FC = () => {
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={editForm.watermark_downloads}
|
||||
onChange={(e) => setEditForm(prev => ({ ...prev, watermark_downloads: e.target.checked }))}
|
||||
onChange={(e) => setEditForm(prev => ({
|
||||
...prev,
|
||||
watermark_downloads: e.target.checked,
|
||||
// Watermarking and presigned URLs are mutually
|
||||
// exclusive — presigned URLs serve raw bytes from
|
||||
// S3 without going through the watermark pipeline.
|
||||
allow_presigned_download: e.target.checked ? false : prev.allow_presigned_download,
|
||||
}))}
|
||||
className="w-4 h-4 text-primary-600 border-neutral-300 dark:border-neutral-600 rounded focus:ring-primary-500"
|
||||
/>
|
||||
<Droplets className="w-4 h-4 ml-2 mr-1 text-neutral-500 dark:text-neutral-400" />
|
||||
<span className="text-sm text-neutral-700 dark:text-neutral-300">{t('events.watermarkDownloads', 'Add watermark to downloads')}</span>
|
||||
</label>
|
||||
|
||||
<label
|
||||
className={`flex items-center ${editForm.watermark_downloads ? 'opacity-50 cursor-not-allowed' : ''}`}
|
||||
title={editForm.watermark_downloads
|
||||
? 'Disabled while watermarks are on — presigned URLs bypass the watermark pipeline.'
|
||||
: 'When the backend uses STORAGE_BACKEND=s3, "Download All" returns a 5-minute presigned S3 URL instead of streaming through the backend. Saves bandwidth on huge galleries; bypasses watermarking.'
|
||||
}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={!!editForm.allow_presigned_download}
|
||||
disabled={editForm.watermark_downloads}
|
||||
onChange={(e) => setEditForm(prev => ({ ...prev, allow_presigned_download: e.target.checked }))}
|
||||
className="w-4 h-4 text-primary-600 border-neutral-300 dark:border-neutral-600 rounded focus:ring-primary-500"
|
||||
/>
|
||||
<Download className="w-4 h-4 ml-2 mr-1 text-neutral-500 dark:text-neutral-400" />
|
||||
<span className="text-sm text-neutral-700 dark:text-neutral-300">
|
||||
{t('events.allowPresignedDownload', 'Allow direct S3 download (no watermark, S3 mode only)')}
|
||||
</span>
|
||||
</label>
|
||||
|
||||
<label className="flex items-center">
|
||||
<input
|
||||
type="checkbox"
|
||||
|
||||
Reference in New Issue
Block a user