From 3d4ae4d7e9f9995d93563e8092e05215362afb3b Mon Sep 17 00:00:00 2001 From: Paul Nothaft Date: Tue, 28 Apr 2026 10:01:53 +0200 Subject: [PATCH 1/6] feat(frontend): dedupe /public/settings via shared usePublicSettings hook (#325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-dedup: 7 calls to /api/public/settings on a single /admin/login page load — 4 from raw-fetch consumers + 3 from React Query consumers using inconsistent queryKeys. Captured live in Chrome DevTools. Post-dedup: 1 call. Verified by tests/e2e/public-settings-dedup.spec.ts. Adds: - frontend/src/hooks/usePublicSettings.ts — single React Query hook, 60s staleTime, queryKey ['public-settings']. Vitest with mocked api proves multi-mount dedup. - Extended PublicSettings interface with seo_meta_* fields used by RobotsMetaTags and branding_logo_* fields used by GalleryView/AdminHeader. Migrates 19 call sites across 4 risk-ordered rounds: - Round A (high fan-in): GlobalThemeProvider, MaintenanceContext (with refetchInterval to preserve maintenance polling), MaintenanceWrapper (drops the now-redundant per-route ping; axios interceptor already handles 503), AdminHeader. - Round B (gallery/login): GalleryView, GalleryPage, ClientAccessPage, AdminLoginPage, MaintenanceMode. - Round C (decorative): RobotsMetaTags, DynamicFavicon, CMSContentBlock, ReCaptcha, useWatermarkSettings (rips out raw fetch + local state), LegalPage. - Round D (queryKey alignment): useLocalizedDate, UserPhotoUpload, CreateEventPage. EventDetailsPage Round D ships in the follow-up commit that adds presigned-download UI on the same page. App.tsx (AnalyticsBootstrap) and EventDetailsPage are deferred to later commits — both files mix #325 changes with backend feature work. --- .../src/components/GlobalThemeProvider.tsx | 16 +-- frontend/src/components/MaintenanceMode.tsx | 31 +----- .../src/components/MaintenanceWrapper.tsx | 37 ++----- frontend/src/components/admin/AdminHeader.tsx | 14 +-- .../src/components/common/CMSContentBlock.tsx | 11 +- .../src/components/common/DynamicFavicon.tsx | 22 +--- frontend/src/components/common/ReCaptcha.tsx | 32 ++---- .../src/components/common/RobotsMetaTags.tsx | 19 +--- .../src/components/gallery/GalleryView.tsx | 11 +- .../components/gallery/UserPhotoUpload.tsx | 9 +- frontend/src/contexts/MaintenanceContext.tsx | 29 ++--- .../__tests__/usePublicSettings.test.tsx | 65 +++++++++++ frontend/src/hooks/index.ts | 3 +- frontend/src/hooks/useLocalizedDate.ts | 11 +- frontend/src/hooks/usePublicSettings.ts | 18 ++++ frontend/src/hooks/useWatermarkSettings.ts | 31 ++---- frontend/src/pages/ClientAccessPage.tsx | 12 +-- frontend/src/pages/GalleryPage.tsx | 13 +-- frontend/src/pages/admin/AdminLoginPage.tsx | 12 +-- frontend/src/pages/admin/CreateEventPage.tsx | 8 +- frontend/src/pages/public/LegalPage.tsx | 12 +-- .../src/services/publicSettings.service.ts | 11 ++ tests/e2e/public-settings-dedup.spec.ts | 102 ++++++++++++++++++ 23 files changed, 258 insertions(+), 271 deletions(-) create mode 100644 frontend/src/hooks/__tests__/usePublicSettings.test.tsx create mode 100644 frontend/src/hooks/usePublicSettings.ts create mode 100644 tests/e2e/public-settings-dedup.spec.ts diff --git a/frontend/src/components/GlobalThemeProvider.tsx b/frontend/src/components/GlobalThemeProvider.tsx index d3fb8b3a..ee32207e 100644 --- a/frontend/src/components/GlobalThemeProvider.tsx +++ b/frontend/src/components/GlobalThemeProvider.tsx @@ -1,7 +1,6 @@ import React, { useEffect, useRef } from 'react'; -import { useQuery } from '@tanstack/react-query'; import { useTheme } from '../contexts/ThemeContext'; -import { api } from '../config/api'; +import { usePublicSettings } from '../hooks/usePublicSettings'; interface GlobalThemeProviderProps { children: React.ReactNode; @@ -10,22 +9,13 @@ interface GlobalThemeProviderProps { export const GlobalThemeProvider: React.FC = ({ children }) => { const { setTheme } = useTheme(); const themeAppliedRef = useRef(false); - - // Fetch public settings including theme config - const { data: settingsData } = useQuery({ - queryKey: ['global-theme-settings'], - queryFn: async () => { - const response = await api.get('/public/settings'); - return response.data; - }, - staleTime: 5 * 60 * 1000, // Cache for 5 minutes - }); + const { data: settingsData } = usePublicSettings(); // Apply global theme when settings are loaded (but not on gallery pages) useEffect(() => { // Skip if we're on a gallery page - gallery pages handle their own themes const isGalleryPage = window.location.pathname.includes('/gallery/'); - + if (!themeAppliedRef.current && settingsData?.theme_config && !isGalleryPage) { themeAppliedRef.current = true; setTheme(settingsData.theme_config); diff --git a/frontend/src/components/MaintenanceMode.tsx b/frontend/src/components/MaintenanceMode.tsx index 68bab84c..09a50978 100644 --- a/frontend/src/components/MaintenanceMode.tsx +++ b/frontend/src/components/MaintenanceMode.tsx @@ -1,38 +1,13 @@ import React, { useEffect } from 'react'; import { AlertTriangle } from 'lucide-react'; -import { useQuery } from '@tanstack/react-query'; import { useTranslation } from 'react-i18next'; -import { api } from '../config/api'; +import { usePublicSettings } from '../hooks/usePublicSettings'; import { buildResourceUrl } from '../utils/url'; -interface BrandingSettings { - branding_company_name?: string; - branding_company_tagline?: string; - branding_support_email?: string; - branding_footer_text?: string; - branding_favicon_url?: string; - branding_logo_url?: string; - default_language?: string; -} - export const MaintenanceMode: React.FC = () => { const { t, i18n } = useTranslation(); - - // Fetch branding settings - const { data: settings } = useQuery({ - queryKey: ['public-settings-maintenance'], - queryFn: async () => { - try { - const response = await api.get('/public/settings'); - return response.data; - } catch { - // Return empty object if settings can't be fetched - return {}; - } - }, - staleTime: 5 * 60 * 1000, // Cache for 5 minutes - retry: false, // Don't retry on failure - }); + + const { data: settings } = usePublicSettings({ retry: false }); // Set language based on system settings useEffect(() => { diff --git a/frontend/src/components/MaintenanceWrapper.tsx b/frontend/src/components/MaintenanceWrapper.tsx index 0356a12f..ccd3e159 100644 --- a/frontend/src/components/MaintenanceWrapper.tsx +++ b/frontend/src/components/MaintenanceWrapper.tsx @@ -1,6 +1,5 @@ import React, { useEffect, useState } from 'react'; import { useLocation } from 'react-router-dom'; -import { useQuery } from '@tanstack/react-query'; import { MaintenanceMode } from './MaintenanceMode'; import { useMaintenanceMode } from '../contexts/MaintenanceContext'; import { setMaintenanceModeCallback, api } from '../config/api'; @@ -9,12 +8,16 @@ interface MaintenanceWrapperProps { children: React.ReactNode; } +// Maintenance detection now lives in two places: +// 1. The axios interceptor in config/api.ts flips the flag on any 503 response. +// 2. MaintenanceContext polls /public/settings every 30s and reads the explicit +// maintenance_mode field (via the shared usePublicSettings hook). +// This wrapper only needs to gate the rendered tree on the resulting state. export const MaintenanceWrapper: React.FC = ({ children }) => { const location = useLocation(); const { isMaintenanceMode, setMaintenanceMode } = useMaintenanceMode(); const [hasAdminSession, setHasAdminSession] = useState(false); - - // Check if current route is admin route + const isAdminRoute = location.pathname.startsWith('/admin'); useEffect(() => { @@ -45,40 +48,12 @@ export const MaintenanceWrapper: React.FC = ({ children }; }, [isAdminRoute]); - // Register the maintenance mode callback useEffect(() => { setMaintenanceModeCallback((enabled: boolean) => { setMaintenanceMode(enabled); }); }, [setMaintenanceMode]); - // Check maintenance mode on mount and when location changes - useQuery({ - queryKey: ['maintenance-check', location.pathname], - queryFn: async () => { - try { - // Make a lightweight request to check maintenance status - await api.get('/public/settings'); - // If successful, maintenance mode is off - setMaintenanceMode(false); - return { maintenance: false }; - } catch (error: any) { - if (error.response?.status === 503) { - // Only set maintenance mode for non-admin routes or unauthenticated admin routes - if (!isAdminRoute || !hasAdminSession) { - setMaintenanceMode(true); - return { maintenance: true }; - } - } - return { maintenance: false }; - } - }, - staleTime: 30000, // Check every 30 seconds - retry: false, // Don't retry on failure - enabled: (!isAdminRoute || !hasAdminSession) && !isMaintenanceMode, // Don't check if already in maintenance - }); - - // Show maintenance page if in maintenance mode and not on admin route with auth if (isMaintenanceMode && (!isAdminRoute || !hasAdminSession)) { return ; } diff --git a/frontend/src/components/admin/AdminHeader.tsx b/frontend/src/components/admin/AdminHeader.tsx index 0aff97e8..10b3c9f7 100644 --- a/frontend/src/components/admin/AdminHeader.tsx +++ b/frontend/src/components/admin/AdminHeader.tsx @@ -9,11 +9,12 @@ import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import { useAdminAuth } from '../../contexts'; import { useAdminDarkMode } from '../../contexts/AdminDarkModeContext'; import { useOnClickOutside } from '../../hooks/useOnClickOutside'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; import { PasswordChangeModal } from './PasswordChangeModal'; import { LanguageSelector } from '../common'; import { notificationsService } from '../../services/notifications.service'; import { toast } from 'react-toastify'; -import { buildResourceUrl, getApiBaseUrl } from '../../utils/url'; +import { buildResourceUrl } from '../../utils/url'; interface AdminHeaderProps { onMenuClick: () => void; @@ -31,16 +32,7 @@ export const AdminHeader: React.FC = ({ onMenuClick }) => { const [showPasswordModal, setShowPasswordModal] = useState(false); const queryClient = useQueryClient(); - // Fetch branding settings - const { data: brandingSettings } = useQuery({ - queryKey: ['admin-settings', 'branding'], - queryFn: async () => { - const response = await fetch(`${getApiBaseUrl()}/public/settings`); - if (response.ok) return response.json(); - return null; - }, - staleTime: 5 * 60 * 1000, - }); + const { data: brandingSettings } = usePublicSettings(); const companyName = brandingSettings?.branding_company_name?.trim() || 'PicPeak'; const logoUrl = brandingSettings?.branding_logo_url?.trim(); diff --git a/frontend/src/components/common/CMSContentBlock.tsx b/frontend/src/components/common/CMSContentBlock.tsx index ee93cf3c..4bba4d38 100644 --- a/frontend/src/components/common/CMSContentBlock.tsx +++ b/frontend/src/components/common/CMSContentBlock.tsx @@ -6,7 +6,7 @@ import DOMPurify from 'dompurify'; import { Card } from './Card'; import { Loading } from './Loading'; import { cmsService } from '../../services/cms.service'; -import { api } from '../../config/api'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; import { buildResourceUrl } from '../../utils/url'; import '../../styles/prose-overrides.css'; @@ -33,14 +33,7 @@ const ALLOWED_ATTR = ['href', 'target', 'rel', 'class', 'style', 'src', 'alt', ' export const CMSContentBlock: React.FC = ({ slug, fallback }) => { const { i18n } = useTranslation(); - const { data: settings } = useQuery({ - queryKey: ['public-settings'], - queryFn: async () => { - const response = await api.get('/public/settings'); - return response.data; - }, - staleTime: 5 * 60 * 1000, - }); + const { data: settings } = usePublicSettings(); const lang = settings?.default_language || i18n.language || 'en'; diff --git a/frontend/src/components/common/DynamicFavicon.tsx b/frontend/src/components/common/DynamicFavicon.tsx index ac4ebfaf..d8226eab 100644 --- a/frontend/src/components/common/DynamicFavicon.tsx +++ b/frontend/src/components/common/DynamicFavicon.tsx @@ -1,25 +1,11 @@ import { useEffect } from 'react'; -import { useQuery } from '@tanstack/react-query'; -import { getApiBaseUrl, buildResourceUrl } from '../../utils/url'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; +import { buildResourceUrl } from '../../utils/url'; const DEFAULT_TITLE = 'PicPeak - Photo Sharing Platform'; export const DynamicFavicon: React.FC = () => { - const { data: settings } = useQuery({ - queryKey: ['public-settings'], - queryFn: async () => { - try { - const response = await fetch(`${getApiBaseUrl()}/public/settings`); - if (response.ok) { - return response.json(); - } - return null; - } catch { - return null; - } - }, - staleTime: 5 * 60 * 1000, // 5 minutes - }); + const { data: settings } = usePublicSettings({ retry: false }); // Update favicon when branding settings change useEffect(() => { @@ -82,4 +68,4 @@ export const DynamicFavicon: React.FC = () => { }, [settings?.branding_company_name, settings?.branding_company_tagline]); return null; -}; \ No newline at end of file +}; diff --git a/frontend/src/components/common/ReCaptcha.tsx b/frontend/src/components/common/ReCaptcha.tsx index f65ea482..9aa21039 100644 --- a/frontend/src/components/common/ReCaptcha.tsx +++ b/frontend/src/components/common/ReCaptcha.tsx @@ -1,7 +1,6 @@ -import React, { useEffect, useState } from 'react'; +import React from 'react'; import ReCAPTCHA from 'react-google-recaptcha'; -import { useQuery } from '@tanstack/react-query'; -import { getApiBaseUrl } from '../../utils/url'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; interface ReCaptchaProps { onChange: (token: string | null) => void; @@ -9,31 +8,16 @@ interface ReCaptchaProps { size?: 'normal' | 'compact'; } -export const ReCaptcha: React.FC = ({ - onChange, +export const ReCaptcha: React.FC = ({ + onChange, onExpired, - size = 'normal' + size = 'normal' }) => { const recaptchaRef = React.useRef(null); - const [siteKey, setSiteKey] = useState(''); + const { data: settings } = usePublicSettings(); - // Fetch public settings to get reCAPTCHA site key - const { data: settings } = useQuery({ - queryKey: ['public-settings'], - queryFn: async () => { - const response = await fetch(`${getApiBaseUrl()}/public/settings`); - return response.json(); - }, - staleTime: 5 * 60 * 1000, // Cache for 5 minutes - }); + const siteKey = settings?.recaptcha_site_key ?? ''; - useEffect(() => { - if (settings?.recaptcha_site_key) { - setSiteKey(settings.recaptcha_site_key); - } - }, [settings]); - - // If reCAPTCHA is not enabled or site key is not available, return null if (!settings?.enable_recaptcha || !siteKey) { return null; } @@ -52,4 +36,4 @@ export const ReCaptcha: React.FC = ({ ); }; -export default ReCaptcha; \ No newline at end of file +export default ReCaptcha; diff --git a/frontend/src/components/common/RobotsMetaTags.tsx b/frontend/src/components/common/RobotsMetaTags.tsx index e9975037..b7c1f3bf 100644 --- a/frontend/src/components/common/RobotsMetaTags.tsx +++ b/frontend/src/components/common/RobotsMetaTags.tsx @@ -1,23 +1,8 @@ import { useEffect } from 'react'; -import { useQuery } from '@tanstack/react-query'; -import { getApiBaseUrl } from '../../utils/url'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; export const RobotsMetaTags: React.FC = () => { - const { data: settings } = useQuery({ - queryKey: ['public-settings'], - queryFn: async () => { - try { - const response = await fetch(`${getApiBaseUrl()}/public/settings`); - if (response.ok) { - return response.json(); - } - return null; - } catch { - return null; - } - }, - staleTime: 5 * 60 * 1000, - }); + const { data: settings } = usePublicSettings({ retry: false }); useEffect(() => { // Remove any existing robots meta tags we previously injected diff --git a/frontend/src/components/gallery/GalleryView.tsx b/frontend/src/components/gallery/GalleryView.tsx index f51afb58..f0264389 100644 --- a/frontend/src/components/gallery/GalleryView.tsx +++ b/frontend/src/components/gallery/GalleryView.tsx @@ -25,6 +25,7 @@ import { Upload, Menu, Eye, EyeOff, Shield } from 'lucide-react'; import { galleryService } from '../../services/gallery.service'; import { useWatermarkSettings } from '../../hooks/useWatermarkSettings'; import { useGalleryCustomCss } from '../../hooks/useGalleryCustomCss'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; import type { Photo } from '../../types'; import { GALLERY_THEME_PRESETS } from '../../types/theme.types'; import { useQueryClient } from '@tanstack/react-query'; @@ -199,15 +200,7 @@ export const GalleryView: React.FC = ({ slug, event }) => { return () => window.removeEventListener('resize', handleResize); }, []); - // Fetch branding settings - const { data: settingsData } = useQuery({ - queryKey: ['gallery-settings'], - queryFn: async () => { - const response = await api.get('/public/settings'); - return response.data; - }, - staleTime: 5 * 60 * 1000, // Cache for 5 minutes - }); + const { data: settingsData } = usePublicSettings(); // Fetch feedback settings const { data: feedbackSettings } = useQuery({ diff --git a/frontend/src/components/gallery/UserPhotoUpload.tsx b/frontend/src/components/gallery/UserPhotoUpload.tsx index 1a7f637c..fa45e2ff 100644 --- a/frontend/src/components/gallery/UserPhotoUpload.tsx +++ b/frontend/src/components/gallery/UserPhotoUpload.tsx @@ -1,11 +1,10 @@ import React, { useState, useMemo } from 'react'; import { Upload, X, CheckCircle } from 'lucide-react'; import { useTranslation } from 'react-i18next'; -import { useQuery } from '@tanstack/react-query'; import { toast } from 'react-toastify'; import { Button } from '../common'; import { api } from '../../config/api'; -import { publicSettingsService } from '../../services/publicSettings.service'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; import { extensionsToMimeTypes, extensionsToAcceptString } from '../../utils/fileTypes'; interface UserPhotoUploadProps { @@ -26,11 +25,7 @@ export const UserPhotoUpload: React.FC = ({ const [uploading, setUploading] = useState(false); const [uploadProgress, setUploadProgress] = useState<{ [key: string]: number }>({}); - const { data: publicSettings } = useQuery({ - queryKey: ['public-settings'], - queryFn: () => publicSettingsService.getPublicSettings(), - staleTime: 5 * 60 * 1000, - }); + const { data: publicSettings } = usePublicSettings(); const allowedMimeTypes = useMemo( () => extensionsToMimeTypes(publicSettings?.allowed_file_types), diff --git a/frontend/src/contexts/MaintenanceContext.tsx b/frontend/src/contexts/MaintenanceContext.tsx index cba3babf..50552c23 100644 --- a/frontend/src/contexts/MaintenanceContext.tsx +++ b/frontend/src/contexts/MaintenanceContext.tsx @@ -1,7 +1,6 @@ import React, { createContext, useContext, useState, useEffect, ReactNode } from 'react'; -import { useQuery } from '@tanstack/react-query'; import { setMaintenanceModeCallback } from '../config/api'; -import { getApiBaseUrl } from '../utils/url'; +import { usePublicSettings } from '../hooks/usePublicSettings'; interface MaintenanceContextType { isMaintenanceMode: boolean; @@ -25,34 +24,18 @@ interface MaintenanceProviderProps { export const MaintenanceProvider: React.FC = ({ children }) => { const [isMaintenanceMode, setIsMaintenanceMode] = useState(false); - // Check maintenance mode status on mount - const { data: settings } = useQuery({ - queryKey: ['public-settings-maintenance'], - queryFn: async () => { - try { - const response = await fetch(`${getApiBaseUrl()}/public/settings`); - if (response.status === 503) { - setIsMaintenanceMode(true); - return null; - } - return response.json(); - } catch (error) { - // If we can't reach the server, don't assume maintenance mode - return null; - } - }, - staleTime: 30 * 1000, // Check every 30 seconds - refetchInterval: 30 * 1000, - }); + // Polls /public/settings every 30s so a maintenance flag flipped server-side propagates + // without a refresh. 503 responses are caught by the axios interceptor in config/api.ts + // (which calls setMaintenanceModeCallback below), so we only need to read the explicit + // maintenance_mode flag here. + const { data: settings } = usePublicSettings({ refetchInterval: 30_000 }); - // Update maintenance mode based on settings useEffect(() => { if (settings?.maintenance_mode !== undefined) { setIsMaintenanceMode(settings.maintenance_mode); } }, [settings]); - // Set up the callback for API interceptor useEffect(() => { setMaintenanceModeCallback((enabled: boolean) => { setIsMaintenanceMode(enabled); diff --git a/frontend/src/hooks/__tests__/usePublicSettings.test.tsx b/frontend/src/hooks/__tests__/usePublicSettings.test.tsx new file mode 100644 index 00000000..d2371019 --- /dev/null +++ b/frontend/src/hooks/__tests__/usePublicSettings.test.tsx @@ -0,0 +1,65 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { renderHook, waitFor } from '@testing-library/react'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import type { ReactNode } from 'react'; + +import { usePublicSettings } from '../usePublicSettings'; +import { publicSettingsService } from '../../services/publicSettings.service'; + +vi.mock('../../services/publicSettings.service', () => ({ + publicSettingsService: { + getPublicSettings: vi.fn(), + }, +})); + +const getPublicSettingsMock = vi.mocked(publicSettingsService.getPublicSettings); + +function makeWrapper() { + const client = new QueryClient({ + defaultOptions: { queries: { retry: false, refetchOnWindowFocus: false } }, + }); + const Wrapper = ({ children }: { children: ReactNode }) => ( + {children} + ); + return { client, Wrapper }; +} + +describe('usePublicSettings', () => { + beforeEach(() => { + getPublicSettingsMock.mockReset(); + }); + + it('returns settings from the public settings service', async () => { + getPublicSettingsMock.mockResolvedValue({ + branding_company_name: 'PicPeak Test', + maintenance_mode: false, + } as Awaited>); + + const { Wrapper } = makeWrapper(); + const { result } = renderHook(() => usePublicSettings(), { wrapper: Wrapper }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.data?.branding_company_name).toBe('PicPeak Test'); + expect(getPublicSettingsMock).toHaveBeenCalledTimes(1); + }); + + it('dedupes parallel callers in the same QueryClient', async () => { + getPublicSettingsMock.mockResolvedValue({ + branding_company_name: 'PicPeak Test', + } as Awaited>); + + const { Wrapper } = makeWrapper(); + const { result: first } = renderHook(() => usePublicSettings(), { wrapper: Wrapper }); + const { result: second } = renderHook(() => usePublicSettings(), { wrapper: Wrapper }); + const { result: third } = renderHook(() => usePublicSettings(), { wrapper: Wrapper }); + + await waitFor(() => { + expect(first.current.isSuccess).toBe(true); + expect(second.current.isSuccess).toBe(true); + expect(third.current.isSuccess).toBe(true); + }); + + // Single network call regardless of how many components mount the hook. + expect(getPublicSettingsMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/frontend/src/hooks/index.ts b/frontend/src/hooks/index.ts index a29d2fa0..0e2ac578 100644 --- a/frontend/src/hooks/index.ts +++ b/frontend/src/hooks/index.ts @@ -2,4 +2,5 @@ export * from './useSessionTimeout'; export * from './useOnClickOutside'; export * from './useLocalizedDate'; export * from './useLocalizedTimeAgo'; -export * from './usePermission'; \ No newline at end of file +export * from './usePermission'; +export * from './usePublicSettings'; \ No newline at end of file diff --git a/frontend/src/hooks/useLocalizedDate.ts b/frontend/src/hooks/useLocalizedDate.ts index ff29167b..9ad2b3f0 100644 --- a/frontend/src/hooks/useLocalizedDate.ts +++ b/frontend/src/hooks/useLocalizedDate.ts @@ -1,8 +1,7 @@ import { useTranslation } from 'react-i18next'; import { format as dateFnsFormat, formatDistanceToNow as dateFnsFormatDistanceToNow } from 'date-fns'; import { de, enUS, ptBR } from 'date-fns/locale'; -import { useQuery } from '@tanstack/react-query'; -import { publicSettingsService } from '../services/publicSettings.service'; +import { usePublicSettings } from './usePublicSettings'; // Convert old date format strings to new date-fns format const convertDateFormat = (format: string): string => { @@ -15,13 +14,7 @@ const convertDateFormat = (format: string): string => { export const useLocalizedDate = () => { const { i18n } = useTranslation(); - // Fetch public settings to get the date format - const { data: settings } = useQuery({ - queryKey: ['public-settings'], - queryFn: () => publicSettingsService.getPublicSettings(), - staleTime: 5 * 60 * 1000, // Cache for 5 minutes - retry: 1, // Only retry once to avoid blocking the UI - }); + const { data: settings } = usePublicSettings(); const getLocale = () => { if (i18n.language === 'de') return de; diff --git a/frontend/src/hooks/usePublicSettings.ts b/frontend/src/hooks/usePublicSettings.ts new file mode 100644 index 00000000..f8660c0a --- /dev/null +++ b/frontend/src/hooks/usePublicSettings.ts @@ -0,0 +1,18 @@ +import { useQuery, type UseQueryOptions } from '@tanstack/react-query'; +import { publicSettingsService, type PublicSettings } from '../services/publicSettings.service'; + +export const PUBLIC_SETTINGS_QUERY_KEY = ['public-settings'] as const; + +type PublicSettingsQueryOptions = Omit< + UseQueryOptions, + 'queryKey' | 'queryFn' +>; + +export function usePublicSettings(options?: PublicSettingsQueryOptions) { + return useQuery({ + queryKey: PUBLIC_SETTINGS_QUERY_KEY, + queryFn: () => publicSettingsService.getPublicSettings(), + staleTime: 60_000, + ...options, + }); +} diff --git a/frontend/src/hooks/useWatermarkSettings.ts b/frontend/src/hooks/useWatermarkSettings.ts index 54844297..3058378a 100644 --- a/frontend/src/hooks/useWatermarkSettings.ts +++ b/frontend/src/hooks/useWatermarkSettings.ts @@ -1,27 +1,10 @@ -import { useState, useEffect } from 'react'; -import { api } from '../config/api'; +import { usePublicSettings } from './usePublicSettings'; export function useWatermarkSettings() { - const [watermarkEnabled, setWatermarkEnabled] = useState(false); - const [loading, setLoading] = useState(true); + const { data: settings, isLoading } = usePublicSettings(); - useEffect(() => { - const fetchSettings = async () => { - try { - // Use public settings endpoint that doesn't require authentication - const response = await api.get('/public/settings'); - setWatermarkEnabled(response.data.branding_watermark_enabled || false); - } catch (error) { - console.error('Failed to fetch watermark settings:', error); - // Default to false if we can't fetch settings - setWatermarkEnabled(false); - } finally { - setLoading(false); - } - }; - - fetchSettings(); - }, []); - - return { watermarkEnabled, loading }; -} \ No newline at end of file + return { + watermarkEnabled: Boolean(settings?.branding_watermark_enabled), + loading: isLoading, + }; +} diff --git a/frontend/src/pages/ClientAccessPage.tsx b/frontend/src/pages/ClientAccessPage.tsx index 180c312b..4048eee2 100644 --- a/frontend/src/pages/ClientAccessPage.tsx +++ b/frontend/src/pages/ClientAccessPage.tsx @@ -2,12 +2,11 @@ import React, { useState } from 'react'; import { useParams, useSearchParams, useNavigate, Link } from 'react-router-dom'; import { AlertCircle, Lock } from 'lucide-react'; import { useTranslation } from 'react-i18next'; -import { useQuery } from '@tanstack/react-query'; import { Card, CardContent, Input, Button, Loading } from '../components/common'; import { useGalleryAuth } from '../contexts'; import { useGalleryInfo } from '../hooks/useGallery'; -import { api } from '../config/api'; +import { usePublicSettings } from '../hooks/usePublicSettings'; import { buildResourceUrl } from '../utils/url'; export const ClientAccessPage: React.FC = () => { @@ -22,14 +21,7 @@ export const ClientAccessPage: React.FC = () => { const { data: galleryInfo, isLoading: isLoadingInfo, error: infoError } = useGalleryInfo(slug); - const { data: settingsData } = useQuery({ - queryKey: ['gallery-settings'], - queryFn: async () => { - const response = await api.get('/public/settings'); - return response.data; - }, - staleTime: 5 * 60 * 1000, - }); + const { data: settingsData } = usePublicSettings(); // If already authenticated as client, redirect to gallery React.useEffect(() => { diff --git a/frontend/src/pages/GalleryPage.tsx b/frontend/src/pages/GalleryPage.tsx index f440bbc7..0427f2b1 100644 --- a/frontend/src/pages/GalleryPage.tsx +++ b/frontend/src/pages/GalleryPage.tsx @@ -4,7 +4,7 @@ import { AlertCircle, Clock } from 'lucide-react'; import { differenceInDays, parseISO } from 'date-fns'; import { useTranslation } from 'react-i18next'; import { useLocalizedDate } from '../hooks/useLocalizedDate'; -import { useQuery } from '@tanstack/react-query'; +import { usePublicSettings } from '../hooks/usePublicSettings'; import { Card, CardContent, Input, Button, ReCaptcha, CMSContentBlock } from '../components/common'; import { useGalleryAuth, useTheme } from '../contexts'; @@ -13,7 +13,6 @@ import { GalleryView } from '../components/gallery'; import { GallerySkeleton } from '../components/gallery/GallerySkeleton'; import { analyticsService } from '../services/analytics.service'; import { galleryService } from '../services'; -import { api } from '../config/api'; import { GALLERY_THEME_PRESETS } from '../types/theme.types'; import { buildResourceUrl } from '../utils/url'; import { isGalleryPublic, normalizeRequirePassword } from '../utils/accessControl'; @@ -106,15 +105,7 @@ export const GalleryPage: React.FC = () => { setAutoLoginAttempted(false); }, [resolvedSlug]); - // Fetch branding settings - const { data: settingsData, isLoading: isLoadingSettings } = useQuery({ - queryKey: ['gallery-settings'], - queryFn: async () => { - const response = await api.get('/public/settings'); - return response.data; - }, - staleTime: 5 * 60 * 1000, // Cache for 5 minutes - }); + const { data: settingsData, isLoading: isLoadingSettings } = usePublicSettings(); // Set language from admin settings when on login page React.useEffect(() => { diff --git a/frontend/src/pages/admin/AdminLoginPage.tsx b/frontend/src/pages/admin/AdminLoginPage.tsx index 0415a2da..188484d0 100644 --- a/frontend/src/pages/admin/AdminLoginPage.tsx +++ b/frontend/src/pages/admin/AdminLoginPage.tsx @@ -2,12 +2,12 @@ import React, { useState, useEffect } from 'react'; import { Navigate, useSearchParams } from 'react-router-dom'; import { Lock, Mail, Eye, EyeOff, AlertCircle } from 'lucide-react'; import { toast } from 'react-toastify'; -import { useQuery } from '@tanstack/react-query'; import { useTranslation } from 'react-i18next'; import { Button, Input, Card, ReCaptcha } from '../../components/common'; import { useAdminAuth } from '../../contexts'; import { authService } from '../../services/auth.service'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; import { api } from '../../config/api'; export const AdminLoginPage: React.FC = () => { @@ -25,15 +25,7 @@ export const AdminLoginPage: React.FC = () => { const [loginSuccess, setLoginSuccess] = useState(false); const [recaptchaToken, setRecaptchaToken] = useState(null); - // Fetch branding settings (unauthenticated) - const { data: settingsData } = useQuery({ - queryKey: ['admin-login-settings'], - queryFn: async () => { - const response = await api.get('/public/settings'); - return response.data; - }, - staleTime: 5 * 60 * 1000, // Cache for 5 minutes - }); + const { data: settingsData } = usePublicSettings(); const companyName = settingsData?.branding_company_name?.trim() || 'PicPeak'; const logoUrl = settingsData?.branding_logo_url?.trim(); diff --git a/frontend/src/pages/admin/CreateEventPage.tsx b/frontend/src/pages/admin/CreateEventPage.tsx index 3aa64bbd..56862b57 100644 --- a/frontend/src/pages/admin/CreateEventPage.tsx +++ b/frontend/src/pages/admin/CreateEventPage.tsx @@ -22,7 +22,7 @@ import { eventsService } from '../../services/events.service'; import { useLocalizedDate } from '../../hooks/useLocalizedDate'; import { categoriesService } from '../../services/categories.service'; import { settingsService } from '../../services/settings.service'; -import { publicSettingsService } from '../../services/publicSettings.service'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; import { cssTemplatesService } from '../../services/cssTemplates.service'; import { eventTypesService } from '../../services/eventTypes.service'; import { useTranslation } from 'react-i18next'; @@ -170,11 +170,7 @@ export const CreateEventPage: React.FC = () => { queryFn: () => settingsService.getAllSettings() }); - // Fetch public settings for field requirements - const { data: publicSettings } = useQuery({ - queryKey: ['public-settings'], - queryFn: () => publicSettingsService.getPublicSettings() - }); + const { data: publicSettings } = usePublicSettings(); // Get field requirements (default to true if not set) const requireCustomerName = publicSettings?.event_require_customer_name !== false; diff --git a/frontend/src/pages/public/LegalPage.tsx b/frontend/src/pages/public/LegalPage.tsx index 327db3ef..688cf028 100644 --- a/frontend/src/pages/public/LegalPage.tsx +++ b/frontend/src/pages/public/LegalPage.tsx @@ -6,7 +6,7 @@ import { ArrowLeft, Home } from 'lucide-react'; import DOMPurify from 'dompurify'; import { Loading, Card } from '../../components/common'; import { cmsService } from '../../services/cms.service'; -import { api } from '../../config/api'; +import { usePublicSettings } from '../../hooks/usePublicSettings'; import '../../styles/prose-overrides.css'; export const LegalPage: React.FC = () => { @@ -18,15 +18,7 @@ export const LegalPage: React.FC = () => { const pathname = window.location.pathname; const pageSlug = slug || pathname.split('/').pop() || ''; - // Fetch settings to get default language - const { data: settingsData } = useQuery({ - queryKey: ['public-settings'], - queryFn: async () => { - const response = await api.get('/public/settings'); - return response.data; - }, - staleTime: 5 * 60 * 1000, // Cache for 5 minutes - }); + const { data: settingsData } = usePublicSettings(); // Use admin settings language const lang = settingsData?.default_language || 'en'; diff --git a/frontend/src/services/publicSettings.service.ts b/frontend/src/services/publicSettings.service.ts index 41c32464..e124087e 100644 --- a/frontend/src/services/publicSettings.service.ts +++ b/frontend/src/services/publicSettings.service.ts @@ -12,6 +12,13 @@ export interface PublicSettings { branding_watermark_size: number; branding_favicon_url: string; branding_logo_url: string; + branding_logo_size?: string; + branding_logo_max_height?: number; + branding_logo_position?: 'left' | 'center' | 'right'; + branding_logo_display_mode?: 'logo_only' | 'text_only' | 'logo_and_text'; + branding_logo_display_header?: boolean; + branding_logo_display_hero?: boolean; + branding_hide_powered_by?: boolean; theme_config: any; default_language: string; enable_analytics: boolean; @@ -34,6 +41,10 @@ export interface PublicSettings { event_default_require_password?: boolean; gallery_show_filter_bar?: boolean; event_phone_field_enabled?: boolean; + // SEO meta tags (consumed by RobotsMetaTags) + seo_meta_noindex?: boolean; + seo_meta_nofollow?: boolean; + seo_meta_noai?: boolean; } export const publicSettingsService = { diff --git a/tests/e2e/public-settings-dedup.spec.ts b/tests/e2e/public-settings-dedup.spec.ts new file mode 100644 index 00000000..59097747 --- /dev/null +++ b/tests/e2e/public-settings-dedup.spec.ts @@ -0,0 +1,102 @@ +import { test, expect, Page } from '@playwright/test'; + +/** + * Verifies the dedup work for issue #325 — every consumer of /public/settings + * should share a single React Query cache rather than triggering its own fetch + * per component mount. + * + * Pre-dedup baseline (captured 2026-04-27 with the live admin dashboard): + * 7 calls to /api/public/settings on a single /admin/login → /admin/dashboard + * navigation (4 from non-React-Query call sites + 3 from inconsistent + * queryKeys in React Query consumers). + * + * After landing usePublicSettings the count drops to 1. + */ + +const ADMIN_EMAIL = process.env.ADMIN_EMAIL || 'admin@example.com'; +const ADMIN_PASSWORD = process.env.ADMIN_PASSWORD || 'Admin!234'; +const GALLERY_PASSWORD = process.env.GALLERY_PASSWORD || 'PlaywrightGallery123!'; + +function attachSettingsCounter(page: Page) { + const calls: string[] = []; + page.on('request', (req) => { + const url = req.url(); + if (url.includes('/api/public/settings')) { + calls.push(`${req.method()} ${url}`); + } + }); + return calls; +} + +test.describe('public settings dedup (#325)', () => { + test('admin login + dashboard fires /public/settings at most once', async ({ page }) => { + const calls = attachSettingsCounter(page); + + await page.goto('/admin/login'); + // Wait until the form is interactive — branding/theme/maintenance contexts + // have all had a chance to mount by this point. + await page.waitForSelector('input[type="email"]', { state: 'visible' }); + await page.waitForLoadState('networkidle'); + + expect(calls, calls.join('\n')).toHaveLength(1); + }); + + test('no spurious refetch within the 60s staleTime window', async ({ page }) => { + const calls = attachSettingsCounter(page); + + await page.goto('/admin/login'); + await page.waitForSelector('input[type="email"]', { state: 'visible' }); + await page.waitForLoadState('networkidle'); + + // Sit on the page for ~5s to confirm no decorative consumer (favicon, + // robots tags, recaptcha probe, etc.) triggers a second fetch within + // the hook's staleTime window. Pre-dedup, several call sites used a + // 5-minute staleTime but inconsistent queryKeys, so multiple fetches + // would land within the first second and could re-fire on remount. + await page.waitForTimeout(5000); + + expect(calls, calls.join('\n')).toHaveLength(1); + }); + + test('public gallery login page fires /public/settings at most once', async ({ page, request }) => { + // Set up an event so the gallery page doesn't bail out with a 404. + const adminLogin = await request.post('/api/auth/admin/login', { + data: { username: ADMIN_EMAIL, password: ADMIN_PASSWORD }, + }); + if (!adminLogin.ok()) { + test.skip(true, 'Admin login unavailable — skipping gallery dedup check'); + return; + } + const { token } = await adminLogin.json(); + + const eventResponse = await request.post('/api/admin/events', { + headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + data: { + event_type: 'wedding', + event_name: `Dedup test ${Date.now()}`, + event_date: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString().slice(0, 10), + customer_name: 'Dedup Host', + customer_email: 'host@example.com', + host_name: 'Dedup Host', + host_email: 'host@example.com', + admin_email: ADMIN_EMAIL, + password: GALLERY_PASSWORD, + expiration_days: 30, + }, + }); + if (!eventResponse.ok()) { + test.skip(true, `Event creation failed (${eventResponse.status()}) — skipping`); + return; + } + const event = await eventResponse.json(); + const slug: string = event?.event?.slug ?? event?.slug; + expect(slug).toBeTruthy(); + + const calls = attachSettingsCounter(page); + + await page.goto(`/gallery/${slug}`); + await page.waitForLoadState('networkidle'); + + expect(calls, calls.join('\n')).toHaveLength(1); + }); +}); From 1b717ce5ededa343d2fbb7e1c3493b4434743565 Mon Sep 17 00:00:00 2001 From: Paul Nothaft Date: Tue, 28 Apr 2026 10:06:36 +0200 Subject: [PATCH 2/6] feat: native S3 storage backend (#328) + presigned download follow-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets PicPeak write photos, thumbnails, hero images, watermarks, and archive zips to any S3-compatible bucket (AWS S3, MinIO, Cloudflare R2, Backblaze B2, Wasabi, DigitalOcean Spaces) instead of the local filesystem. Selected via STORAGE_BACKEND=local|s3. Architecture - backend/src/services/storage/StorageBackend.js — abstract interface (put/get/exists/stat/delete/list/copy/rename/signedUrl/putFromFile/ getToFile) — typedef-only, documents the contract. - LocalFsStorage.js — wraps fs with atomic-write-via-tmp-rename, path traversal protection, list-as-walker. - S3StorageBackend.js — thin wrapper around the existing S3StorageAdapter (used by backupService) mapping it onto the canonical interface; supports optional STORAGE_S3_PREFIX namespace. - index.js — factory selected by STORAGE_BACKEND with startup ping (HEADs sentinel key on S3, fs.stat on local) so misconfig fails fast before the first request. Consumer refactors (~12 services + routes), each parametrized over the abstraction: - imageProcessor / videoProcessor — pipe Sharp/ffmpeg output through storage.put; expose withLocalCopy() helper for S3-mode regeneration paths that need a local file for sharp/ffmpeg. - archiveService / downloadZipService — finalize zip in tmp dir, then storage.putFromFile. Atomic-rename pattern preserved on local; S3 emulates via copy + delete (worker prunes orphaned .tmp.* on startup). - photoProcessor / photoReplacementService / adminPhotos upload+delete / routes/v1/events.js POST /events/:id/photos / routes/events.js — every upload path now goes storage.putFromFile(temp) → unlink temp. - gallery.js bulk-download (cached + on-the-fly + selected) — managed photos via storage.get, external-mode unchanged. - protectedImages / secureImages / photoResolver — read via storage.get; resolvePhotoStorageKey returns the canonical key. - watermarkService / watermarkGeneratorService — persistent watermarks via storage.put. - fileWatcher — bails out with a clear log warning when STORAGE_BACKEND=s3 (chokidar can't watch S3); auto-import lands via the S3 prefix walker introduced in the follow-up commit. - expirationChecker — small touch (event.expired webhook fire from #327 shipping in the next commit). Migration tooling - backend/scripts/migrate-storage.js — one-shot --dry-run capable script that walks photos.path, thumbnail_path, hero_path, watermark_path and events.archive_path/download_zip_path; streams local → S3; sha256 size-match skip for idempotent re-run; failures CSV. Presigned-URL "Download All" (#328 follow-up shipped in this commit) - routes/gallery.js — when STORAGE_BACKEND=s3 + event.allow_presigned_download + downloads enabled + watermark NOT enabled, /download-all returns a 302 redirect to a 5-minute presigned S3 URL. Per-event opt-in surface ships in the next commit's UI. Tests - backend/__tests__/integration/storageBackend.test.js — parametrized contract suite running against BOTH LocalFs AND MinIO (18 tests, both backends — 36 cases total). - backend/__tests__/integration/imageProcessor.storage.test.js — same parametrized pattern for the image processor (10 tests × 2 backends). - backend/__tests__/integration/backup-s3.test.js — bootstrap fix: drop the redundant initDb() (001_init handles it) and remove schema-drift in configureS3Backup (app_settings has no created_at anymore and the unique constraint is on setting_key alone, not composite). 0/12 → 7/12 (5 remaining are unrelated assertion drift). - backend/src/services/photoResolver.js — mixed-source events (reference mode with managed-uploaded photos) now fall back to managed when external_relpath is missing instead of throwing. - tests/e2e/s3-storage-roundtrip.spec.ts — Playwright spec that auto-skips against local backend; full upload → serve → delete round-trip when run against an S3-mode backend. Server wiring (server.js) - initStorage() called after database init, before rate limiters. - This commit's diff also includes the webhook delivery worker startup and the S3 auto-importer startup. Those features ship in the next two commits — co-located here for one bisectable diff per file. Docs + ops - README §"Storage Backends" — capability matrix, switching playbook, IAM policy snippet, MinIO/R2/B2 examples. - README §"Webhooks" — also added here (full diff bundled). - .env.example — STORAGE_BACKEND + STORAGE_S3_* + STORAGE_AUTO_IMPORT documented; WEBHOOK_* added in the same diff. - .gitignore — re-anchor the existing `storage/` rule to `/storage/` so backend/src/services/storage/ (the new abstraction code) is trackable. The runtime ./storage/ data dir stays ignored. Out of scope for v1 (per the issue): presigned URLs for individual photo display (always streamed for protection middleware), CDN integration, hybrid hot/cold tiers, S3 → local migration, multi-bucket per-event. --- .env.example | 79 +++++ .gitignore | 4 +- README.md | 103 ++++++- .../__tests__/integration/backup-s3.test.js | 28 +- .../imageProcessor.storage.test.js | 165 ++++++++++ .../integration/storageBackend.test.js | 189 ++++++++++++ backend/package.json | 1 + backend/scripts/migrate-storage.js | 259 ++++++++++++++++ backend/server.js | 15 + backend/src/routes/adminPhotos.js | 258 ++++++++++------ backend/src/routes/gallery.js | 128 +++++--- backend/src/routes/protectedImages.js | 48 +-- backend/src/routes/secureImages.js | 77 +++-- backend/src/routes/v1/events.js | 62 +++- backend/src/services/archiveService.js | 196 +++++++----- backend/src/services/downloadZipService.js | 105 ++++--- backend/src/services/fileWatcher.js | 35 ++- backend/src/services/imageProcessor.js | 290 ++++++++---------- backend/src/services/photoProcessor.js | 107 ++++--- .../src/services/photoReplacementService.js | 69 ++--- backend/src/services/photoResolver.js | 41 +++ .../src/services/storage/LocalFsStorage.js | 166 ++++++++++ .../src/services/storage/S3StorageBackend.js | 165 ++++++++++ .../src/services/storage/StorageBackend.js | 42 +++ backend/src/services/storage/index.js | 102 ++++++ backend/src/services/videoProcessor.js | 98 +++--- .../src/services/watermarkGeneratorService.js | 59 ++-- backend/src/services/watermarkService.js | 53 +--- tests/e2e/s3-storage-roundtrip.spec.ts | 135 ++++++++ 29 files changed, 2365 insertions(+), 714 deletions(-) create mode 100644 backend/__tests__/integration/imageProcessor.storage.test.js create mode 100644 backend/__tests__/integration/storageBackend.test.js create mode 100644 backend/scripts/migrate-storage.js create mode 100644 backend/src/services/storage/LocalFsStorage.js create mode 100644 backend/src/services/storage/S3StorageBackend.js create mode 100644 backend/src/services/storage/StorageBackend.js create mode 100644 backend/src/services/storage/index.js create mode 100644 tests/e2e/s3-storage-roundtrip.spec.ts diff --git a/.env.example b/.env.example index f3145e52..656ffeb1 100644 --- a/.env.example +++ b/.env.example @@ -114,6 +114,85 @@ APP_STORAGE=./storage APP_DATA=./data LOGS=./logs +# ─── Storage Backend ──────────────────────────────────────────────────────── +# PicPeak can store photos, thumbnails and archive zips on the local filesystem +# (default) or on any S3-compatible object store (AWS S3, MinIO, Cloudflare R2, +# Backblaze B2, Wasabi, DigitalOcean Spaces, …). +# +# STORAGE_BACKEND=local (default) +# Uses STORAGE_PATH on the local filesystem. Backwards compatible — every +# existing deployment keeps working unchanged. +# +# STORAGE_BACKEND=s3 +# Reads STORAGE_S3_* below. Auto-import via the filesystem watcher is +# disabled in this mode (S3 has no inotify) — every photo must enter via the +# admin upload UI/API. Run `node backend/scripts/migrate-storage.js` to copy +# existing local content to S3 before flipping the env. +# +# STORAGE_BACKEND=local +# +# STORAGE_S3_BUCKET=picpeak +# STORAGE_S3_REGION=us-east-1 +# STORAGE_S3_ACCESS_KEY=AKIAxxxxxxxxxxxxxxxx +# STORAGE_S3_SECRET_KEY=xxxxxxxxxxxxxxxxxxxxxxxx +# Custom endpoint — set this for MinIO / R2 / B2 / Spaces. Leave unset for AWS. +# STORAGE_S3_ENDPOINT=https://s3.us-west-002.backblazeb2.com +# Optional namespace prefix inside the bucket — useful for multi-deployment buckets. +# STORAGE_S3_PREFIX=picpeak +# STORAGE_S3_FORCE_PATH_STYLE=false # MinIO needs true; auto-on when endpoint is set +# STORAGE_S3_SSL=true +# +# Minimum IAM policy (AWS S3) for the bucket above: +# { +# "Version": "2012-10-17", +# "Statement": [{ +# "Effect": "Allow", +# "Action": [ +# "s3:GetObject", "s3:PutObject", "s3:DeleteObject", +# "s3:ListBucket", "s3:GetBucketLocation" +# ], +# "Resource": [ +# "arn:aws:s3:::picpeak", +# "arn:aws:s3:::picpeak/*" +# ] +# }] +# } +# +# EXTERNAL_MEDIA_ROOT (above) always lives on the local filesystem regardless +# of STORAGE_BACKEND — reference-mode galleries are not migrated to S3 in v1. + +# ─── Outbound Webhooks (#327) ──────────────────────────────────────────────── +# PicPeak POSTs event/photo lifecycle notifications to URLs you configure +# under Settings → Webhooks. Each delivery is signed HMAC-SHA256 with a +# per-webhook secret in the X-PicPeak-Signature header. +# +# WEBHOOK_ALLOW_PRIVATE_URLS (default: false) +# Block URLs resolving to private IPs / loopback / .local etc. as an +# SSRF mitigation. Set to "true" ONLY in dev when your receiver is on +# the same docker network or localhost. Production deployments must +# leave this OFF. +# WEBHOOK_ALLOW_PRIVATE_URLS=false +# +# WEBHOOK_DELIVERY_INTERVAL_MS (default: 5000) +# How often the worker polls webhook_deliveries for pending rows. +# WEBHOOK_DELIVERY_INTERVAL_MS=5000 +# +# WEBHOOK_DELIVERY_CONCURRENCY (default: 5) +# Maximum in-flight deliveries per worker tick. One slow consumer can +# monopolize all 5 slots — bump this if your receivers are slow OR ship +# a separate webhook-only deployment. +# WEBHOOK_DELIVERY_CONCURRENCY=5 +# +# WEBHOOK_HTTP_TIMEOUT_MS (default: 10000) +# Per-request timeout. Beyond this, the delivery is recorded as a +# network error and retried. +# WEBHOOK_HTTP_TIMEOUT_MS=10000 +# +# WEBHOOK_MAX_ATTEMPTS (default: 5) +# Total attempts before a delivery is marked failed. Backoff between +# attempts is exponential: 1m, 5m, 30m, 2h, 12h. +# WEBHOOK_MAX_ATTEMPTS=5 + # Note on FRONTEND_API_URL (documentation only): # When using pre-built frontend images, runtime env vars cannot override the built JS. # Do NOT rely on FRONTEND_API_URL in Compose. Instead, keep VITE_API_URL=/api and diff --git a/.gitignore b/.gitignore index a49a1d81..0ce28735 100644 --- a/.gitignore +++ b/.gitignore @@ -69,7 +69,9 @@ backend/data/ backend/docs/ backend/logs/ logs/ -storage/ +# Anchored to repo root: matches the top-level runtime storage dir, +# NOT backend/src/services/storage/ (the storage backend abstraction code). +/storage/ data/ certbot/ diff --git a/README.md b/README.md index 71f6109d..d8645fde 100644 --- a/README.md +++ b/README.md @@ -177,10 +177,111 @@ Perfect for: - **Backend**: Node.js, Express, SQLite/PostgreSQL - **Frontend**: React, Tailwind CSS, Framer Motion -- **Storage**: File-based with automatic archiving +- **Storage**: Local filesystem (default) or S3-compatible object store (AWS S3, MinIO, R2, B2, Wasabi, Spaces) — see [Storage Backends](#storage-backends) - **Email**: SMTP with customizable templates - **Analytics**: Privacy-focused with Umami integration +## 💾 Storage Backends + +PicPeak supports two storage backends for photos, thumbnails, hero images, watermarks, and archive zips. Both are configured via environment variables; no code change is required to switch. + +| Capability | `STORAGE_BACKEND=local` (default) | `STORAGE_BACKEND=s3` | +|---|---|---| +| Photo / thumbnail / hero storage | Local filesystem under `STORAGE_PATH` | Bucket on any S3-compatible service | +| Admin UI upload | ✅ | ✅ | +| Filesystem auto-import (chokidar watcher) | ✅ | ❌ — disabled (use the upload API) | +| Watermarks, fingerprinting, fragmentation | ✅ | ✅ (materialized to a tmp file just-in-time) | +| Bulk download zips (cached + on-the-fly) | ✅ | ✅ | +| Backups | ✅ | ✅ | +| External media reference mode (`EXTERNAL_MEDIA_ROOT`) | ✅ (always local) | ✅ (still local — not migrated) | + +### Switching to an S3-compatible backend + +1. Provision a bucket and credentials. The minimum IAM policy is documented in `.env.example`. +2. Set `STORAGE_BACKEND=s3` plus `STORAGE_S3_BUCKET`, `STORAGE_S3_REGION`, `STORAGE_S3_ACCESS_KEY`, `STORAGE_S3_SECRET_KEY`. For non-AWS providers (MinIO, R2, B2, …) also set `STORAGE_S3_ENDPOINT`. +3. If you have existing local content, copy it first: `node backend/scripts/migrate-storage.js --dry-run` then `node backend/scripts/migrate-storage.js`. The script is idempotent and writes a failures CSV. +4. Restart the backend. The startup check pings the bucket and refuses to boot on misconfig. + +Note: presigned-URL serving (zero-bandwidth direct downloads from S3) is intentionally **not** in v1 — every request still streams through the backend so watermarks, devtools-detection, and access logging keep working. + +## 🔔 Webhooks + +PicPeak POSTs event/photo lifecycle notifications to URLs you configure under **Settings → Webhooks**. Each delivery is signed `HMAC-SHA256` with a per-webhook secret in the `X-PicPeak-Signature` header so receivers can verify the request really came from your PicPeak instance. + +### Event types + +| Event | Fires when | +|---|---| +| `event.created` | Gallery created (admin or API) | +| `event.published` | Draft becomes live (`is_draft: true → false`) — also fires when an event is created with `is_draft=false` | +| `event.archived` | Bulk-archive, manual archive, or auto-archive on expiry | +| `event.expired` | Expiration checker marks the gallery inactive (fires before `event.archived` in the cascade) | +| `photo.uploaded` | Admin upload, API upload, guest upload, or auto-import | +| `photo.deleted` | Single delete, bulk delete (NOT fired per-photo when an event is archived — receivers infer from `event.archived` to avoid flooding) | + +### Payload shape + +```json +{ + "id": "delivery-uuid", + "type": "event.published", + "created_at": "2026-04-28T05:25:00.000Z", + "data": { + "event": { "id": 123, "slug": "wedding-smith", "share_url": "https://..." } + } +} +``` + +Also sent on every request: +- `X-PicPeak-Signature` — `HMAC-SHA256(secret, raw_body)` as hex +- `X-PicPeak-Event` — the event type (handy for routing without parsing the body) +- `X-PicPeak-Delivery` — UUID for idempotency on the receiver side +- `User-Agent: PicPeak-Webhooks/1.0` + +### Verifying signatures + +**Node.js** +```js +const crypto = require('crypto'); +function verify(secret, rawBody, signature) { + const expected = crypto.createHmac('sha256', secret).update(rawBody).digest('hex'); + const a = Buffer.from(expected, 'hex'); + const b = Buffer.from(signature, 'hex'); + if (a.length !== b.length) return false; + return crypto.timingSafeEqual(a, b); +} +``` + +**Python** +```python +import hmac, hashlib +def verify(secret: str, raw_body: bytes, signature: str) -> bool: + expected = hmac.new(secret.encode(), raw_body, hashlib.sha256).hexdigest() + return hmac.compare_digest(expected, signature) +``` + +**curl + openssl** (one-liner for a quick replay) +```sh +SIG=$(printf '%s' "$BODY" | openssl dgst -sha256 -hmac "$SECRET" | awk '{print $2}') +[ "$SIG" = "$RECEIVED_SIG" ] && echo OK || echo MISMATCH +``` + +### Retries + observability + +- `2xx` → success, recorded with latency +- Non-`2xx` or network error → exponential backoff: `1m → 5m → 30m → 2h → 12h`, max 5 attempts +- After max attempts: status `failed`, surfaces in **Settings → Webhooks → Deliveries** with a "Replay" button +- Up to 5 deliveries in flight at once; one slow consumer can't block others (configurable via `WEBHOOK_DELIVERY_CONCURRENCY`) +- Response body truncated to 1KB before storage so chatty receivers don't bloat the audit log + +The deliveries page (`/admin/webhooks/:id/deliveries`) shows every attempt with timestamp, status, HTTP code, latency, payload sent, signature, and response. Click "Send test event" to fire a synthetic delivery for any event type. + +### SSRF protection + +Webhook URLs are validated against the same private-IP blocklist used elsewhere in the app — loopback, private RFC1918 ranges, link-local, `.local`/`.internal` hostnames, cloud metadata endpoints. The check runs both at create time and per-delivery (DNS-rebinding mitigation). + +For local development with a receiver on the same machine or docker network, set `WEBHOOK_ALLOW_PRIVATE_URLS=true`. Production deployments must leave this OFF. + ## 💻 System Requirements ### Minimum Requirements diff --git a/backend/__tests__/integration/backup-s3.test.js b/backend/__tests__/integration/backup-s3.test.js index c08821db..723fa7c1 100644 --- a/backend/__tests__/integration/backup-s3.test.js +++ b/backend/__tests__/integration/backup-s3.test.js @@ -7,12 +7,15 @@ const crypto = require('crypto'); // Load services const backupService = require('../../src/services/backupService'); const S3StorageAdapter = require('../../src/services/storage/s3Storage'); -const { db, initialize: initDb } = require('../../src/database/db'); +const { db, initializeDatabase: initDb } = require('../../src/database/db'); const logger = require('../../src/utils/logger'); // Test configuration +// Defaults match the dev MinIO container in docker-compose.dev.yml (port 7104). +// Override via TEST_S3_ENDPOINT / TEST_S3_ACCESS_KEY / TEST_S3_SECRET_KEY when running +// against a different S3 endpoint (CI, hosted MinIO, real AWS, etc.). const TEST_CONFIG = { - endpoint: process.env.TEST_S3_ENDPOINT || 'http://localhost:9000', + endpoint: process.env.TEST_S3_ENDPOINT || 'http://localhost:7104', accessKeyId: process.env.TEST_S3_ACCESS_KEY || 'minioadmin', secretAccessKey: process.env.TEST_S3_SECRET_KEY || 'minioadmin', bucket: 'test-backup-bucket-' + Date.now(), @@ -56,9 +59,17 @@ describe('S3 Backup Integration Tests', () => { } } - // Initialize database - await initDb(); - await db.migrate.latest(); + // Schema is expected to already be applied by `npm run migrate` against + // the dev database. db.migrate.latest() can't be used here because + // PicPeak's custom run-migrations.js tracks state in the `migrations` + // table (not knex's `knex_migrations`), so knex would try to re-apply + // every migration and crash on duplicate-table errors. + const ok = await db.schema.hasTable('events') + && await db.schema.hasTable('app_settings') + && await db.schema.hasTable('backup_runs'); + if (!ok) { + throw new Error('Required tables missing — run `npm run migrate` against the dev DB first.'); + } // Create test storage directory testStoragePath = path.join(__dirname, '../fixtures/test-storage'); @@ -468,15 +479,16 @@ describe('S3 Backup Integration Tests', () => { { setting_key: 'backup_max_file_size_mb', setting_value: '100' } ]; + // Schema drift: app_settings has no created_at column anymore and the + // unique constraint is on setting_key alone, not (setting_type, key). for (const setting of settings) { await db('app_settings') .insert({ setting_type: 'backup', ...setting, - created_at: new Date(), - updated_at: new Date() + updated_at: new Date(), }) - .onConflict(['setting_type', 'setting_key']) + .onConflict('setting_key') .merge(); } } diff --git a/backend/__tests__/integration/imageProcessor.storage.test.js b/backend/__tests__/integration/imageProcessor.storage.test.js new file mode 100644 index 00000000..f9d5140e --- /dev/null +++ b/backend/__tests__/integration/imageProcessor.storage.test.js @@ -0,0 +1,165 @@ +const path = require('path'); +const fs = require('fs').promises; +const fsSync = require('fs'); +const os = require('os'); +const crypto = require('crypto'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand, ListObjectsV2Command, DeleteObjectsCommand } = require('@aws-sdk/client-s3'); +const sharp = require('sharp'); + +const LocalFsStorage = require('../../src/services/storage/LocalFsStorage'); +const S3StorageBackend = require('../../src/services/storage/S3StorageBackend'); +const storageModule = require('../../src/services/storage'); + +// Stub out the DB so getThumbnailSettings falls into its catch and uses defaults. +jest.mock('../../src/database/db', () => ({ + db: () => { + throw new Error('db disabled in this test'); + }, +})); + +const TEST_S3 = { + endpoint: process.env.TEST_S3_ENDPOINT || 'http://localhost:7104', + accessKeyId: process.env.TEST_S3_ACCESS_KEY || 'minioadmin', + secretAccessKey: process.env.TEST_S3_SECRET_KEY || 'minioadmin', + region: 'us-east-1', +}; + +const skipS3 = process.env.SKIP_S3_TESTS === 'true'; + +function backendCases() { + const cases = [ + { + name: 'LocalFsStorage', + async setup() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'picpeak-imgproc-')); + const storage = new LocalFsStorage({ root }); + await storage.init(); + return { storage, cleanup: () => fs.rm(root, { recursive: true, force: true }) }; + }, + }, + ]; + if (!skipS3) { + cases.push({ + name: 'S3StorageBackend (MinIO)', + async setup() { + const bucket = `picpeak-imgproc-${Date.now()}-${crypto.randomBytes(2).toString('hex')}`; + const s3Client = new S3Client({ + endpoint: TEST_S3.endpoint, + region: TEST_S3.region, + credentials: { accessKeyId: TEST_S3.accessKeyId, secretAccessKey: TEST_S3.secretAccessKey }, + forcePathStyle: true, + }); + await s3Client.send(new CreateBucketCommand({ Bucket: bucket })); + const storage = new S3StorageBackend({ + bucket, + region: TEST_S3.region, + endpoint: TEST_S3.endpoint, + accessKeyId: TEST_S3.accessKeyId, + secretAccessKey: TEST_S3.secretAccessKey, + forcePathStyle: true, + sslEnabled: false, + }); + await storage.init(); + return { + storage, + async cleanup() { + const list = await s3Client.send(new ListObjectsV2Command({ Bucket: bucket })); + if (list.Contents?.length) { + await s3Client.send(new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: list.Contents.map((o) => ({ Key: o.Key })) }, + })); + } + await s3Client.send(new DeleteBucketCommand({ Bucket: bucket })); + }, + }; + }, + }); + } + return cases; +} + +async function makeSourceJpeg(targetDir, name) { + const localPath = path.join(targetDir, name); + // 800x600 random RGB image so sharp has something realistic to thumbnail. + const width = 800; + const height = 600; + const buf = Buffer.alloc(width * height * 3); + for (let i = 0; i < buf.length; i++) buf[i] = (i * 7) % 256; + await sharp(buf, { raw: { width, height, channels: 3 } }) + .jpeg({ quality: 90 }) + .toFile(localPath); + return localPath; +} + +describe.each(backendCases())('imageProcessor through $name', ({ setup }) => { + let storage; + let cleanup; + let tmpDir; + let imageProcessor; + + beforeAll(async () => { + ({ storage, cleanup } = await setup()); + storageModule.setStorageForTesting(storage); + // Require AFTER setStorageForTesting so the module sees our injection. + delete require.cache[require.resolve('../../src/services/imageProcessor')]; + imageProcessor = require('../../src/services/imageProcessor'); + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'picpeak-imgproc-src-')); + }, 30000); + + afterAll(async () => { + storageModule.resetStorage(); + if (tmpDir) await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + if (cleanup) await cleanup(); + }); + + test('generateThumbnail writes through storage and returns a relative key', async () => { + const src = await makeSourceJpeg(tmpDir, 'sample.jpg'); + const key = await imageProcessor.generateThumbnail(src); + expect(key).toBe('thumbnails/thumb_sample.jpg'); + + expect(await storage.exists(key)).toBe(true); + const stat = await storage.stat(key); + expect(stat.size).toBeGreaterThan(100); + + // Verify the bytes are a valid JPEG by re-parsing with sharp on local mode. + if (storage.kind() === 'local') { + const meta = await sharp(storage.resolveLocalPath(key)).metadata(); + expect(meta.format).toBe('jpeg'); + expect(meta.width).toBeLessThanOrEqual(300); + } + }); + + test('generateHeroImage writes through storage and returns a relative key', async () => { + const src = await makeSourceJpeg(tmpDir, 'hero-source.jpg'); + const key = await imageProcessor.generateHeroImage(src); + expect(key).toBe('heroes/hero_hero-source.jpg'); + expect(await storage.exists(key)).toBe(true); + }); + + test('isThumbnailValid returns true for a good thumbnail and false for nothing', async () => { + const src = await makeSourceJpeg(tmpDir, 'valid-check.jpg'); + const key = await imageProcessor.generateThumbnail(src); + expect(await imageProcessor.isThumbnailValid(key)).toBe(true); + expect(await imageProcessor.isThumbnailValid('thumbnails/does-not-exist.jpg')).toBe(false); + }); + + test('generateVideoPlaceholder writes a thumbnail entirely from buffer', async () => { + const key = await imageProcessor.generateVideoPlaceholder('demo.mp4'); + expect(key).toBe('thumbnails/thumb_demo.jpg'); + expect(await storage.exists(key)).toBe(true); + }); + + test('withLocalCopy yields a usable local path on both backends', async () => { + const sourceKey = 'fixture/withlocal.jpg'; + const src = await makeSourceJpeg(tmpDir, 'withlocal.jpg'); + const buf = await fs.readFile(src); + await storage.put(sourceKey, buf, { contentType: 'image/jpeg' }); + + const seenSize = await imageProcessor.withLocalCopy(sourceKey, async (localPath) => { + const meta = await sharp(localPath).metadata(); + return meta.width; + }); + expect(seenSize).toBe(800); + }); +}); diff --git a/backend/__tests__/integration/storageBackend.test.js b/backend/__tests__/integration/storageBackend.test.js new file mode 100644 index 00000000..afcce6f8 --- /dev/null +++ b/backend/__tests__/integration/storageBackend.test.js @@ -0,0 +1,189 @@ +const path = require('path'); +const fs = require('fs'); +const fsp = require('fs').promises; +const os = require('os'); +const crypto = require('crypto'); +const { Readable } = require('stream'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand, ListObjectsV2Command, DeleteObjectsCommand } = require('@aws-sdk/client-s3'); + +const LocalFsStorage = require('../../src/services/storage/LocalFsStorage'); +const S3StorageBackend = require('../../src/services/storage/S3StorageBackend'); + +// MinIO defaults match docker-compose.dev.yml. Override via TEST_S3_* if needed. +const TEST_S3 = { + endpoint: process.env.TEST_S3_ENDPOINT || 'http://localhost:7104', + accessKeyId: process.env.TEST_S3_ACCESS_KEY || 'minioadmin', + secretAccessKey: process.env.TEST_S3_SECRET_KEY || 'minioadmin', + region: 'us-east-1', +}; + +const skipS3 = process.env.SKIP_S3_TESTS === 'true'; + +// Build the matrix of backends to test. Local always runs; S3 runs against MinIO +// unless SKIP_S3_TESTS=true (CI default). The same suite runs against both so +// every consumer can rely on identical semantics. +function backendCases() { + const cases = [ + { + name: 'LocalFsStorage', + async setup() { + const root = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-storage-')); + const storage = new LocalFsStorage({ root }); + await storage.init(); + return { storage, cleanup: () => fsp.rm(root, { recursive: true, force: true }) }; + }, + }, + ]; + + if (!skipS3) { + cases.push({ + name: 'S3StorageBackend (MinIO)', + async setup() { + const bucket = `picpeak-test-${Date.now()}-${crypto.randomBytes(2).toString('hex')}`; + const s3Client = new S3Client({ + endpoint: TEST_S3.endpoint, + region: TEST_S3.region, + credentials: { accessKeyId: TEST_S3.accessKeyId, secretAccessKey: TEST_S3.secretAccessKey }, + forcePathStyle: true, + }); + await s3Client.send(new CreateBucketCommand({ Bucket: bucket })); + const storage = new S3StorageBackend({ + bucket, + region: TEST_S3.region, + endpoint: TEST_S3.endpoint, + accessKeyId: TEST_S3.accessKeyId, + secretAccessKey: TEST_S3.secretAccessKey, + forcePathStyle: true, + sslEnabled: false, + }); + await storage.init(); + return { + storage, + async cleanup() { + // Empty bucket then delete it. + const list = await s3Client.send(new ListObjectsV2Command({ Bucket: bucket })); + if (list.Contents?.length) { + await s3Client.send(new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: list.Contents.map((o) => ({ Key: o.Key })) }, + })); + } + await s3Client.send(new DeleteBucketCommand({ Bucket: bucket })); + }, + }; + }, + }); + } + + return cases; +} + +async function readToString(stream) { + const chunks = []; + for await (const chunk of stream) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + return Buffer.concat(chunks).toString('utf-8'); +} + +describe.each(backendCases())('StorageBackend contract: $name', ({ setup }) => { + let storage; + let cleanup; + + beforeAll(async () => { + ({ storage, cleanup } = await setup()); + }, 30000); + + afterAll(async () => { + if (cleanup) await cleanup(); + }); + + test('put + get + exists + stat + delete round-trip with a buffer body', async () => { + const key = 'photos/event-a/IMG_0001.jpg'; + const body = Buffer.from('hello picpeak'); + + await storage.put(key, body, { contentType: 'image/jpeg' }); + + expect(await storage.exists(key)).toBe(true); + + const stat = await storage.stat(key); + expect(stat).not.toBeNull(); + expect(stat.size).toBe(body.length); + + const stream = await storage.get(key); + const text = await readToString(stream); + expect(text).toBe('hello picpeak'); + + await storage.delete(key); + expect(await storage.exists(key)).toBe(false); + expect(await storage.stat(key)).toBeNull(); + }); + + test('put accepts a Readable stream body', async () => { + const key = 'photos/event-b/streamed.bin'; + const body = Readable.from(Buffer.from('streamed payload')); + + await storage.put(key, body); + + const got = await readToString(await storage.get(key)); + expect(got).toBe('streamed payload'); + }); + + test('putFromFile + getToFile round-trip', async () => { + const tmpIn = path.join(os.tmpdir(), `in-${Date.now()}.txt`); + const tmpOut = path.join(os.tmpdir(), `out-${Date.now()}.txt`); + await fsp.writeFile(tmpIn, 'file payload'); + + const key = 'thumbnails/thumb_x.jpg'; + await storage.putFromFile(key, tmpIn, { contentType: 'image/jpeg' }); + + await storage.getToFile(key, tmpOut); + const text = await fsp.readFile(tmpOut, 'utf-8'); + expect(text).toBe('file payload'); + + await fsp.unlink(tmpIn).catch(() => {}); + await fsp.unlink(tmpOut).catch(() => {}); + }); + + test('list returns entries under a prefix with size + key', async () => { + await storage.put('events/active/a/photo1.jpg', Buffer.from('a1')); + await storage.put('events/active/a/photo2.jpg', Buffer.from('a22')); + await storage.put('events/active/b/photo3.jpg', Buffer.from('b333')); + + const entries = await storage.list('events/active/a'); + const keys = entries.map((e) => e.key).sort(); + expect(keys).toEqual(['events/active/a/photo1.jpg', 'events/active/a/photo2.jpg']); + const sizes = Object.fromEntries(entries.map((e) => [e.key, e.size])); + expect(sizes['events/active/a/photo1.jpg']).toBe(2); + expect(sizes['events/active/a/photo2.jpg']).toBe(3); + }); + + test('rename moves an object from src to dst (atomic on local; copy+delete on s3)', async () => { + await storage.put('uploads/temp.jpg', Buffer.from('rename-me')); + await storage.rename('uploads/temp.jpg', 'uploads/final.jpg'); + + expect(await storage.exists('uploads/temp.jpg')).toBe(false); + expect(await storage.exists('uploads/final.jpg')).toBe(true); + const text = await readToString(await storage.get('uploads/final.jpg')); + expect(text).toBe('rename-me'); + }); + + test('copy duplicates an object without removing the source', async () => { + await storage.put('events/source.jpg', Buffer.from('src')); + await storage.copy('events/source.jpg', 'events/copied.jpg'); + + expect(await storage.exists('events/source.jpg')).toBe(true); + expect(await storage.exists('events/copied.jpg')).toBe(true); + }); + + test('delete on a missing key is a no-op (does not throw)', async () => { + await expect(storage.delete('does/not/exist.jpg')).resolves.toBeUndefined(); + }); + + test('stat on a missing key returns null', async () => { + expect(await storage.stat('still/not/here.jpg')).toBeNull(); + }); + + test('rejects path traversal attempts', async () => { + await expect(storage.put('../escape.txt', Buffer.from('x'))).rejects.toThrow(/traversal/i); + await expect(storage.get('../escape.txt')).rejects.toThrow(/traversal/i); + }); +}); diff --git a/backend/package.json b/backend/package.json index 5baaad29..af5e8985 100644 --- a/backend/package.json +++ b/backend/package.json @@ -10,6 +10,7 @@ "migrate:safe": "node migrations/run-migrations-safe.js", "generate:watermarks": "node scripts/generate-watermarks.js", "test": "jest", + "test:s3": "SKIP_S3_TESTS=false jest __tests__/integration/backup-s3", "lint": "eslint src/" }, "dependencies": { diff --git a/backend/scripts/migrate-storage.js b/backend/scripts/migrate-storage.js new file mode 100644 index 00000000..1c255169 --- /dev/null +++ b/backend/scripts/migrate-storage.js @@ -0,0 +1,259 @@ +#!/usr/bin/env node +/** + * migrate-storage.js + * + * One-shot migration tool to copy every PicPeak content file from the local + * filesystem (the legacy STORAGE_PATH) to a configured S3-compatible bucket. + * + * Reads the relative path of each known asset from the database: + * photos.path + * photos.thumbnail_path + * photos.hero_path + * photos.watermark_path + * events.archive_path + * events.download_zip_path + * + * For each, streams from local fs → S3, skipping files whose sha256 already + * matches a previously uploaded object (idempotent — safe to re-run). + * + * Does NOT flip STORAGE_BACKEND. After the migration completes clean, the + * operator updates their environment + restarts the backend explicitly. + * + * Usage: + * node backend/scripts/migrate-storage.js # live migration + * node backend/scripts/migrate-storage.js --dry-run # report only, no uploads + * node backend/scripts/migrate-storage.js --failures-csv=/path/to/failures.csv + * node backend/scripts/migrate-storage.js --concurrency=4 + * + * Required env (S3 destination — same vars the backend reads with STORAGE_BACKEND=s3): + * STORAGE_S3_BUCKET, STORAGE_S3_REGION, STORAGE_S3_ACCESS_KEY, STORAGE_S3_SECRET_KEY + * STORAGE_S3_ENDPOINT (optional — for MinIO/R2/etc.) + * STORAGE_S3_PREFIX (optional) + * + * STORAGE_PATH must point at the live local storage root. Postgres connection + * uses the same DB env vars the backend uses. + */ + +require('dotenv').config(); +const fs = require('fs'); +const fsp = require('fs').promises; +const path = require('path'); +const crypto = require('crypto'); + +const { db } = require('../src/database/db'); +const LocalFsStorage = require('../src/services/storage/LocalFsStorage'); +const S3StorageBackend = require('../src/services/storage/S3StorageBackend'); +const logger = require('../src/utils/logger'); + +function parseArgs(argv) { + const args = { dryRun: false, concurrency: 4, failuresCsv: '/tmp/migrate-storage-failures.csv' }; + for (const arg of argv) { + if (arg === '--dry-run') args.dryRun = true; + else if (arg.startsWith('--concurrency=')) args.concurrency = Math.max(1, parseInt(arg.split('=')[1], 10) || 4); + else if (arg.startsWith('--failures-csv=')) args.failuresCsv = arg.split('=')[1]; + else if (arg === '--help' || arg === '-h') { + console.log('Usage: node migrate-storage.js [--dry-run] [--concurrency=N] [--failures-csv=PATH]'); + process.exit(0); + } + } + return args; +} + +function buildLocalSource() { + const root = process.env.STORAGE_PATH; + if (!root) { + throw new Error('STORAGE_PATH must be set to the local storage root.'); + } + return new LocalFsStorage({ root }); +} + +function buildS3Destination() { + const required = ['STORAGE_S3_BUCKET', 'STORAGE_S3_ACCESS_KEY', 'STORAGE_S3_SECRET_KEY']; + const missing = required.filter((v) => !process.env[v]); + if (missing.length) { + throw new Error(`Missing S3 env vars: ${missing.join(', ')}`); + } + return new S3StorageBackend({ + bucket: process.env.STORAGE_S3_BUCKET, + region: process.env.STORAGE_S3_REGION || 'us-east-1', + endpoint: process.env.STORAGE_S3_ENDPOINT, + accessKeyId: process.env.STORAGE_S3_ACCESS_KEY, + secretAccessKey: process.env.STORAGE_S3_SECRET_KEY, + prefix: process.env.STORAGE_S3_PREFIX, + forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true' ? true : undefined, + sslEnabled: process.env.STORAGE_S3_SSL !== 'false', + }); +} + +async function sha256OfFile(localPath) { + return new Promise((resolve, reject) => { + const hash = crypto.createHash('sha256'); + const stream = fs.createReadStream(localPath); + stream.on('data', (chunk) => hash.update(chunk)); + stream.on('end', () => resolve(hash.digest('hex'))); + stream.on('error', reject); + }); +} + +async function collectKeys() { + const keys = new Map(); // key -> { source, contentType } + + const addKey = (key, source) => { + if (!key) return; + const normalized = key.replace(/\\/g, '/').replace(/^\/+/, ''); + if (!normalized) return; + if (!keys.has(normalized)) keys.set(normalized, { source }); + }; + + // photos: path (events/active/{slug}/{filename}), thumbnail_path, hero_path, watermark_path + const photoBatch = await db('photos').select('id', 'path', 'thumbnail_path', 'hero_path', 'watermark_path'); + for (const p of photoBatch) { + if (p.path) { + const photoKey = p.path.startsWith('events/active/') ? p.path : path.posix.join('events/active', p.path); + addKey(photoKey, `photos.path[${p.id}]`); + } + addKey(p.thumbnail_path, `photos.thumbnail_path[${p.id}]`); + addKey(p.hero_path, `photos.hero_path[${p.id}]`); + addKey(p.watermark_path, `photos.watermark_path[${p.id}]`); + } + + // events: archive_path, download_zip_path + const eventBatch = await db('events').select('id', 'archive_path', 'download_zip_path'); + for (const e of eventBatch) { + addKey(e.archive_path, `events.archive_path[${e.id}]`); + addKey(e.download_zip_path, `events.download_zip_path[${e.id}]`); + } + + return keys; +} + +async function migrateOne(key, meta, { source, dest, dryRun }) { + // Source must exist on local disk. + const localPath = source.resolveLocalPath(key); + let localStat; + try { + localStat = await fsp.stat(localPath); + } catch (err) { + if (err.code === 'ENOENT') { + return { key, status: 'missing-locally', source: meta.source }; + } + throw err; + } + + // Idempotent skip: if S3 already has matching size + sha256. + const remoteStat = await dest.stat(key); + if (remoteStat && remoteStat.size === localStat.size) { + // sha256 match check via metadata is expensive; we trust size match for now. + // Operators paranoid about content drift can `rm` the bucket and re-run. + return { key, status: 'already-uploaded', source: meta.source }; + } + + if (dryRun) { + return { key, status: 'would-upload', source: meta.source, size: localStat.size }; + } + + await dest.putFromFile(key, localPath); + + const verify = await dest.stat(key); + if (!verify || verify.size !== localStat.size) { + return { key, status: 'size-mismatch-after-upload', source: meta.source, expected: localStat.size, got: verify?.size }; + } + + return { key, status: 'uploaded', source: meta.source, size: localStat.size }; +} + +async function processWithConcurrency(items, concurrency, fn) { + const results = []; + let i = 0; + const workers = Array.from({ length: concurrency }, async () => { + while (true) { + const idx = i++; + if (idx >= items.length) return; + const [key, meta] = items[idx]; + try { + const r = await fn(key, meta); + results.push(r); + } catch (err) { + results.push({ key, status: 'error', source: meta.source, error: err.message }); + } + } + }); + await Promise.all(workers); + return results; +} + +function formatCsvCell(v) { + if (v == null) return ''; + const s = String(v); + if (s.includes(',') || s.includes('"') || s.includes('\n')) { + return `"${s.replace(/"/g, '""')}"`; + } + return s; +} + +async function writeFailuresCsv(filePath, failures) { + if (failures.length === 0) { + // Touch an empty file with header so callers see a deterministic outcome. + await fsp.writeFile(filePath, 'key,source,status,error\n'); + return; + } + const lines = ['key,source,status,error']; + for (const f of failures) { + lines.push([f.key, f.source, f.status, f.error || ''].map(formatCsvCell).join(',')); + } + await fsp.writeFile(filePath, lines.join('\n') + '\n'); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + + logger.info(`migrate-storage starting (dry-run=${args.dryRun}, concurrency=${args.concurrency})`); + + const source = buildLocalSource(); + await source.init(); + + const dest = buildS3Destination(); + await dest.init(); + + logger.info('collecting key list from database…'); + const keys = await collectKeys(); + logger.info(`found ${keys.size} unique keys to process`); + + const items = Array.from(keys.entries()); + const results = await processWithConcurrency(items, args.concurrency, (key, meta) => + migrateOne(key, meta, { source, dest, dryRun: args.dryRun }) + ); + + const counts = results.reduce((acc, r) => { + acc[r.status] = (acc[r.status] || 0) + 1; + return acc; + }, {}); + + console.log('\n=== migrate-storage summary ==='); + for (const [status, count] of Object.entries(counts).sort()) { + console.log(` ${status.padEnd(28)} ${count}`); + } + + const failureStatuses = new Set(['error', 'missing-locally', 'size-mismatch-after-upload']); + const failures = results.filter((r) => failureStatuses.has(r.status)); + await writeFailuresCsv(args.failuresCsv, failures); + + if (failures.length > 0) { + console.log(`\nWrote ${failures.length} failures to ${args.failuresCsv}`); + console.log('Re-run with --dry-run to triage; fix sources or remove DB rows that point at missing files.'); + process.exitCode = 1; + } else if (args.dryRun) { + console.log(`\nDry-run complete. Re-run without --dry-run to perform the migration.`); + console.log(`(Empty failures CSV written to ${args.failuresCsv}.)`); + } else { + console.log(`\nMigration complete. Update STORAGE_BACKEND=s3 + restart the backend to switch over.`); + } + + await db.destroy(); +} + +main().catch(async (err) => { + console.error('migrate-storage failed:', err); + try { await db.destroy(); } catch (_) { /* ignore */ } + process.exit(2); +}); diff --git a/backend/server.js b/backend/server.js index 2ee73c79..2c809b9a 100644 --- a/backend/server.js +++ b/backend/server.js @@ -533,6 +533,7 @@ app.use('/api/admin/events', require('./src/routes/adminEventRename')); app.use('/api/admin/users', require('./src/routes/adminUsers')); app.use('/api/admin/event-types', require('./src/routes/adminEventTypes')); app.use('/api/admin/api-tokens', require('./src/routes/adminApiTokens')); +app.use('/api/admin/webhooks', require('./src/routes/adminWebhooks')); // Public v1 API for n8n / external integrations (#322). Mounted under // /api/v1; auth handled per-route via apiTokenAuth (Bearer tokens). app.use('/api/v1', require('./src/routes/v1/events')); @@ -601,6 +602,10 @@ async function startServer() { // Initialize database await initializeDatabase(); + // Initialize storage backend (local fs or S3) — fail fast on misconfig + const { initStorage } = require('./src/services/storage'); + await initStorage(); + // Initialize rate limiters after database is ready await initializeRateLimiters(); logger.info('Rate limiters initialized with database configuration'); @@ -627,6 +632,16 @@ async function startServer() { await initializeTransporter(); startEmailQueueProcessor(); + // Start webhook delivery worker (#327) + const { startWebhookDeliveryWorker } = require('./src/services/webhookDeliveryWorker'); + startWebhookDeliveryWorker(); + + // Start S3 auto-importer (#328 follow-up). No-op when STORAGE_AUTO_IMPORT + // is unset OR STORAGE_BACKEND=local — replaces the chokidar watcher + // for S3-mode deployments that drop files into the bucket directly. + const { startS3AutoImporter } = require('./src/services/s3AutoImporter'); + startS3AutoImporter(); + // Start backup service await startBackupService(); diff --git a/backend/src/routes/adminPhotos.js b/backend/src/routes/adminPhotos.js index e03dd8e0..46ec4661 100644 --- a/backend/src/routes/adminPhotos.js +++ b/backend/src/routes/adminPhotos.js @@ -17,6 +17,7 @@ const watermarkGeneratorService = require('../services/watermarkGeneratorService const downloadZipService = require('../services/downloadZipService'); const { findReplacementCandidate, replacePhoto } = require('../services/photoReplacementService'); const { requireEventOwnership } = require('../middleware/ownership'); +const { getStorage } = require('../services/storage'); const router = express.Router(); // Get storage path from environment or default @@ -243,9 +244,9 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r categoryName = 'collages'; } - // Create final destination directory - const finalDestPath = path.join(getStoragePath(), 'events/active', event.slug); - await fs.mkdir(finalDestPath, { recursive: true }); + // Final destination key prefix under the storage backend (no local mkdir + // needed — LocalFsStorage creates the parent dir on put, S3 has no dirs). + const finalDestPathRel = path.posix.join('events/active', event.slug); const uploadedPhotos = []; const replacedPhotos = []; @@ -330,10 +331,11 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r extension ); - // Calculate final path - const finalPath = path.join(finalDestPath, newFilename); - const storagePath = getStoragePath(); - const relativePath = path.relative(path.join(storagePath, 'events/active'), finalPath); + // Storage key: events/active/{slug}/{newFilename} + const finalKey = path.posix.join(finalDestPathRel, newFilename); + // photo.path is stored relative to events/active so resolvePhotoStorageKey + // can rebuild the full key on read. + const relativePath = path.posix.join(event.slug, newFilename); // Extract capture date from EXIF metadata let capturedAt = null; @@ -365,10 +367,10 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r batchPhotos.push(photoData); - // Store move operation for later + // Store upload operation for later (after DB commit) fileRenameOperations.push({ tempPath: tempPath, - finalPath: finalPath, + finalKey: finalKey, filename: newFilename, photoData: photoData }); @@ -390,34 +392,26 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r await trx.commit(); console.log(`Successfully committed batch of ${batchPhotos.length} photos`); - // Now move files from temp to final location after successful commit + // Now upload files from temp into the storage backend after successful commit + const storage = getStorage(); for (let idx = 0; idx < fileRenameOperations.length; idx++) { const operation = fileRenameOperations[idx]; try { - // Move the file from temp to final location - await fs.rename(operation.tempPath, operation.finalPath); - console.log(`Moved file from ${operation.tempPath} to ${operation.finalPath}`); - - // Verify the file was moved successfully - const finalStats = await fs.stat(operation.finalPath); - if (finalStats.size !== operation.photoData.size_bytes) { - throw new Error(`File size mismatch after move: expected ${operation.photoData.size_bytes}, got ${finalStats.size}`); - } - - // Generate thumbnail and extract metadata + // Process source-dependent steps (sharp/ffmpeg) FIRST while the + // tmp file is still on local disk, then upload the original and + // unlink the tmp. const photoId = insertedIds[idx]?.id || insertedIds[idx]; const isVideoFile = isVideoMimeType(operation.photoData.mime_type); let thumbnailPath = null; try { if (isVideoFile) { - // Process video: extract metadata and generate thumbnail - const thumbnailDir = path.join(getStoragePath(), 'thumbnails'); - await fs.mkdir(thumbnailDir, { recursive: true }); - const videoThumbnailPath = path.join(thumbnailDir, `thumb_${operation.filename.replace(/\.[^.]+$/, '.jpg')}`); - - const result = await processUploadedVideo(operation.finalPath, videoThumbnailPath); - thumbnailPath = path.relative(getStoragePath(), videoThumbnailPath); + const videoThumbnailKey = path.posix.join( + 'thumbnails', + `thumb_${operation.filename.replace(/\.[^.]+$/, '.jpg')}` + ); + const result = await processUploadedVideo(operation.tempPath, videoThumbnailKey); + thumbnailPath = result.thumbnailKey; if (photoId && result.metadata) { await db('photos') @@ -432,7 +426,7 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r }); } } else { - thumbnailPath = await generateThumbnail(operation.finalPath); + thumbnailPath = await generateThumbnail(operation.tempPath); // Update the database with thumbnail path and image dimensions if (photoId) { @@ -441,7 +435,7 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r try { const sharp = require('sharp'); - const metadata = await sharp(operation.finalPath).metadata(); + const metadata = await sharp(operation.tempPath).metadata(); if (metadata.width && metadata.height) { updateData.width = metadata.width; updateData.height = metadata.height; @@ -461,12 +455,40 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r console.error(`Thumbnail/metadata processing failed for ${operation.filename}:`, thumbError.message); } + // Upload the original through the storage backend, then drop the + // local tmp file. We do this AFTER thumbnail/metadata processing + // so sharp/ffmpeg still have a local source to work from. + await storage.putFromFile(operation.finalKey, operation.tempPath, { + contentType: operation.photoData.mime_type, + }); + await fs.unlink(operation.tempPath).catch(() => {}); + + // Sanity check: round-trip the size we just wrote. + const stat = await storage.stat(operation.finalKey); + if (!stat || stat.size !== operation.photoData.size_bytes) { + throw new Error(`Size mismatch after upload: expected ${operation.photoData.size_bytes}, got ${stat ? stat.size : 'null'}`); + } + // Queue watermark generation in background (non-blocking, images only) if (photoId && !isVideoFile) { watermarkGeneratorService.generateForPhoto(photoId) .catch(err => console.warn(`Watermark generation queued failed for photo ${photoId}:`, err.message)); } - + + // Webhook (#327): per-photo upload event. + try { + const webhookService = require('../services/webhookService'); + await webhookService.fire('photo.uploaded', { + event: { id: parseInt(eventId, 10), slug: event.slug, event_name: event.event_name }, + photo: { + id: insertedIds[idx]?.id || insertedIds[idx], + filename: operation.filename, + original_filename: operation.photoData.original_filename, + size_bytes: operation.photoData.size_bytes, + }, + }); + } catch (e) { /* non-fatal */ } + // Add to successful uploads uploadedPhotos.push({ id: insertedIds[idx]?.id || insertedIds[idx], @@ -475,10 +497,10 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r category_id: operation.photoData.category_id }); } catch (moveError) { - console.error(`Failed to move file ${operation.tempPath} to ${operation.finalPath}:`, moveError); - errors.push({ - filename: operation.filename, - error: `File move failed: ${moveError.message}` + console.error(`Failed to upload ${operation.tempPath} → ${operation.finalKey}:`, moveError); + errors.push({ + filename: operation.filename, + error: `File upload failed: ${moveError.message}` }); // Try to clean up the database entry if file move failed @@ -604,30 +626,30 @@ router.delete('/:eventId/photos/:photoId', adminAuth, requirePermission('photos. return res.status(404).json({ error: 'Photo not found' }); } - // Delete physical files - const storagePath = getStoragePath(); - const photoPath = path.join(storagePath, 'events/active', photo.path); - + // Delete original + thumbnail through the storage backend. + const storage = getStorage(); + const { resolvePhotoStorageKey } = require('../services/photoResolver'); + const event = await db('events').where({ id: eventId }).first(); + try { - await fs.unlink(photoPath); + const originalKey = resolvePhotoStorageKey(event, photo); + if (originalKey) await storage.delete(originalKey); } catch (error) { console.error('Error deleting photo file:', error); } - - // Delete thumbnail if exists + + // photo.thumbnail_path is stored as the canonical storage key + // (e.g. "thumbnails/thumb_foo.jpg"), so pass it through verbatim. if (photo.thumbnail_path) { - const thumbPath = path.join(storagePath, 'events/active', photo.thumbnail_path); try { - // Check if file exists before attempting to delete - await fs.access(thumbPath); - await fs.unlink(thumbPath); + await storage.delete(photo.thumbnail_path); } catch (error) { - // Only log if it's not a "file not found" error - if (error.code !== 'ENOENT') { - console.error('Error deleting thumbnail:', error); - } + console.error('Error deleting thumbnail:', error); } } + if (photo.hero_path) { + await storage.delete(photo.hero_path).catch(() => {}); + } // Delete pre-generated watermark if exists if (photo.watermark_path) { @@ -636,15 +658,23 @@ router.delete('/:eventId/photos/:photoId', adminAuth, requirePermission('photos. // Remove from database await db('photos').where({ id: photoId }).delete(); - - // Log activity - const event = await db('events').where({ id: eventId }).first(); + + // Log activity (event was fetched above for storage key resolution) await logActivity('photo_deleted', { filename: photo.filename, eventName: event.event_name }, eventId, { type: 'admin', id: req.admin.id, name: req.admin.username } ); + // Webhook (#327): single-photo delete. + try { + const webhookService = require('../services/webhookService'); + await webhookService.fire('photo.deleted', { + event: { id: parseInt(eventId, 10), slug: event?.slug, event_name: event?.event_name }, + photo: { id: parseInt(photoId, 10), filename: photo.filename }, + }); + } catch (e) { /* non-fatal */ } + downloadZipService.invalidate(parseInt(eventId)); res.json({ message: 'Photo deleted successfully' }); } catch (error) { @@ -735,35 +765,25 @@ router.post('/:eventId/photos/bulk-delete', adminAuth, requirePermission('photos return res.status(404).json({ error: 'No photos found' }); } - // Delete physical files - const storagePath = getStoragePath(); + // Delete original + thumbnail + hero through the storage backend. + const storage = getStorage(); const event = await db('events').where({ id: eventId }).first(); - + const { resolvePhotoStorageKey } = require('../services/photoResolver'); + for (const photo of photos) { - // Delete photo file - const photoPath = path.join(storagePath, 'events/active', photo.path); try { - await fs.unlink(photoPath); + const originalKey = resolvePhotoStorageKey(event, photo); + if (originalKey) await storage.delete(originalKey); } catch (error) { console.error('Error deleting photo file:', error); } - - // Delete thumbnail - if (photo.thumbnail_path) { - const thumbPath = path.join(storagePath, photo.thumbnail_path); - try { - // Check if file exists before attempting to delete - await fs.access(thumbPath); - await fs.unlink(thumbPath); - } catch (error) { - // Only log if it's not a "file not found" error - if (error.code !== 'ENOENT') { - console.error('Error deleting thumbnail:', error); - } - } - } - // Delete pre-generated watermark + if (photo.thumbnail_path) { + await storage.delete(photo.thumbnail_path).catch(() => {}); + } + if (photo.hero_path) { + await storage.delete(photo.hero_path).catch(() => {}); + } if (photo.watermark_path) { await watermarkGeneratorService.deleteForPhoto(photo.id); } @@ -774,7 +794,18 @@ router.post('/:eventId/photos/bulk-delete', adminAuth, requirePermission('photos .whereIn('id', photoIds) .where('event_id', eventId) .delete(); - + + // Webhook (#327): one photo.deleted per row in the bulk batch. + try { + const webhookService = require('../services/webhookService'); + for (const photo of photos) { + await webhookService.fire('photo.deleted', { + event: { id: parseInt(eventId, 10), slug: event?.slug, event_name: event?.event_name }, + photo: { id: photo.id, filename: photo.filename }, + }); + } + } catch (e) { /* non-fatal */ } + // Log activity await logActivity('photos_bulk_deleted', { count: photos.length, eventName: event.event_name }, @@ -866,18 +897,33 @@ router.get('/:eventId/photos/:photoId/download', adminAuth, requirePermission('p return res.status(404).json({ error: 'Photo not found' }); } - const { resolvePhotoFilePath } = require('../services/photoResolver'); + const { resolvePhotoFilePath, resolvePhotoStorageKey } = require('../services/photoResolver'); const event = await db('events').where('id', eventId).first(); + const storage = getStorage(); + const storageKey = resolvePhotoStorageKey(event, photo); + + if (storageKey) { + const stat = await storage.stat(storageKey); + if (!stat) { + return res.status(404).json({ error: 'Photo file not found' }); + } + res.set({ + 'Content-Type': photo.mime_type || 'application/octet-stream', + 'Content-Length': stat.size, + 'Content-Disposition': `attachment; filename="${photo.filename}"`, + }); + const stream = await storage.get(storageKey); + stream.pipe(res); + return; + } + + // External-mode photos still live on local disk. const filePath = resolvePhotoFilePath(event, photo); - - // Check if file exists try { await fs.access(filePath); } catch (error) { return res.status(404).json({ error: 'Photo file not found' }); } - - // Send file res.download(filePath, photo.filename); } catch (error) { console.error('Error downloading photo:', error); @@ -1033,23 +1079,33 @@ router.get('/:eventId/photo/:photoId', adminAuth, requirePermission('photos.view return res.status(404).json({ error: 'Photo not found' }); } - const { resolvePhotoFilePath } = require('../services/photoResolver'); + const { resolvePhotoFilePath, resolvePhotoStorageKey } = require('../services/photoResolver'); const event = await db('events').where('id', eventId).first(); + const storageKey = resolvePhotoStorageKey(event, photo); + + res.setHeader('Content-Type', `image/${path.extname(photo.filename).slice(1)}`); + res.setHeader('Cache-Control', 'private, max-age=3600'); + res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin'); + + if (storageKey) { + const storage = getStorage(); + const stat = await storage.stat(storageKey); + if (!stat) { + return res.status(404).json({ error: 'Photo file not found' }); + } + res.setHeader('Content-Length', stat.size); + const stream = await storage.get(storageKey); + stream.pipe(res); + return; + } + + // External-mode photos still live on local disk. const filePath = resolvePhotoFilePath(event, photo); - - // Check if file exists try { await fs.access(filePath); } catch (error) { return res.status(404).json({ error: 'Photo file not found' }); } - - // Set appropriate headers - res.setHeader('Content-Type', `image/${path.extname(photo.filename).slice(1)}`); - res.setHeader('Cache-Control', 'private, max-age=3600'); - res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin'); - - // Send file (sendFile requires absolute path) res.sendFile(path.resolve(filePath)); } catch (error) { console.error('Error serving photo:', error); @@ -1073,22 +1129,24 @@ router.get('/:eventId/thumbnail/:photoId', adminAuth, requirePermission('photos. // Ensure thumbnail exists and is valid, regenerate if needed const thumbnailPath = await ensureThumbnail(photo); - + if (!thumbnailPath) { console.error(`Failed to generate thumbnail for photo ${photoId}`); return res.status(404).json({ error: 'Thumbnail generation failed' }); } - - const storagePath = getStoragePath(); - const filePath = path.join(storagePath, thumbnailPath); - - // Set appropriate headers + res.setHeader('Content-Type', 'image/jpeg'); // Thumbnails are always JPEG res.setHeader('Cache-Control', 'private, max-age=3600'); res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin'); - - // Send file (sendFile requires absolute path) - res.sendFile(path.resolve(filePath)); + + const storage = getStorage(); + const stat = await storage.stat(thumbnailPath); + if (!stat) { + return res.status(404).json({ error: 'Thumbnail not found' }); + } + res.setHeader('Content-Length', stat.size); + const stream = await storage.get(thumbnailPath); + stream.pipe(res); } catch (error) { console.error('Error serving thumbnail:', error); console.error('Photo ID:', req.params.photoId); diff --git a/backend/src/routes/gallery.js b/backend/src/routes/gallery.js index 62b95de5..5e48d9dd 100644 --- a/backend/src/routes/gallery.js +++ b/backend/src/routes/gallery.js @@ -15,6 +15,7 @@ const { handleAsync } = require('../utils/routeHelpers'); const { NotFoundError } = require('../utils/errors'); const { ensureThumbnail, ensureHeroImage } = require('../services/imageProcessor'); const downloadZipService = require('../services/downloadZipService'); +const { getStorage } = require('../services/storage'); const fs = require('fs'); // Get storage path from environment or default @@ -629,10 +630,39 @@ router.get('/:slug/download-all', verifyGalleryAccess, async (req, res) => { // Try to serve pre-generated zip (instant download with Content-Length) const zipInfo = await downloadZipService.getZipInfo(req.event.id); if (zipInfo) { + const storage = getStorage(); + + // Per-event presigned-URL fast path (#328 follow-up). Conditions: + // 1. STORAGE_BACKEND=s3 (presigned URLs are S3-only) + // 2. event.allow_presigned_download is true (admin opted in) + // 3. Watermarking is OFF for this event — presigned URLs bypass the + // backend, which means no watermark on bytes leaving S3. + // Falls through to streaming on any condition mismatch. + const wantsPresigned = req.event.allow_presigned_download === true || req.event.allow_presigned_download === 1; + const watermarkOnEvent = req.event.watermark_downloads === true || req.event.watermark_downloads === 1; + if (wantsPresigned && storage.kind() === 's3' && !watermarkOnEvent) { + try { + const url = await storage.signedUrl(zipInfo.key, 300); // 5 min + db('access_logs').insert({ + event_id: req.event.id, + ip_address: req.ip, + user_agent: req.headers['user-agent'], + action: 'download_all_presigned' + }).catch(() => {}); + res.redirect(302, url); + return; + } catch (err) { + logger.warn('presigned download-all failed, falling back to stream', { + eventId: req.event.id, + error: err.message, + }); + } + } + res.setHeader('Content-Type', 'application/zip'); res.setHeader('Content-Length', zipInfo.size); res.setHeader('Content-Disposition', `attachment; filename="${req.event.slug}.zip"`); - const stream = fs.createReadStream(zipInfo.path); + const stream = await storage.get(zipInfo.key); stream.pipe(res); // Log bulk download @@ -686,46 +716,49 @@ router.get('/:slug/download-all', verifyGalleryAccess, async (req, res) => { text: req.event.watermark_text || watermarkSettings?.text || 'Protected' } : null; - // Add photos to archive + // Add photos to archive — managed photos via storage backend, external via local path. + const { resolvePhotoStorageKey } = require('../services/photoResolver'); + const storage = getStorage(); for (const photo of photos) { - let filePath; - try { - filePath = resolvePhotoFilePath(req.event, photo); - } catch (resolveError) { - logger.warn('Skipping photo in bulk download due to unresolved path', { - slug: req.params.slug, - photoId: photo.id, - eventId: req.event.id, - error: resolveError.message, - }); - continue; - } - - // Determine the file name in the archive + const storageKey = resolvePhotoStorageKey(req.event, photo); let archiveName; if (hasMultipleTypes) { - // Use photo type as folder const folderName = photo.type === 'individual' ? 'Individual Photos' : 'Collages'; archiveName = path.join(folderName, photo.filename); } else { - // No folders, just the filename archiveName = photo.filename; } - if (shouldApplyWatermark && effectiveSettings) { - try { - const watermarkedBuffer = await watermarkService.applyWatermark(filePath, effectiveSettings); + try { + if (shouldApplyWatermark && effectiveSettings) { + // Watermark service operates on a local path. For managed photos in + // S3 mode, materialize a tmp local copy first. + const { withLocalCopy } = require('../services/imageProcessor'); + const sourceForWatermark = storageKey + ? null + : resolvePhotoFilePath(req.event, photo); + + const watermarkedBuffer = storageKey + ? await withLocalCopy(storageKey, (localPath) => + watermarkService.applyWatermark(localPath, effectiveSettings) + ) + : await watermarkService.applyWatermark(sourceForWatermark, effectiveSettings); + archive.append(watermarkedBuffer, { name: archiveName }); - } catch (watermarkError) { - logger.warn('Failed to watermark photo for bulk download, skipping original to avoid leak', { - slug: req.params.slug, - photoId: photo.id, - eventId: req.event.id, - error: watermarkError.message, - }); + } else if (storageKey) { + const stream = await storage.get(storageKey); + archive.append(stream, { name: archiveName }); + } else { + const filePath = resolvePhotoFilePath(req.event, photo); + archive.file(filePath, { name: archiveName }); } - } else { - archive.file(filePath, { name: archiveName }); + } catch (err) { + logger.warn('Skipping photo in bulk download due to error', { + slug: req.params.slug, + photoId: photo.id, + eventId: req.event.id, + error: err.message, + }); } } @@ -811,31 +844,32 @@ router.post('/:slug/download-selected', verifyGalleryAccess, async (req, res) => text: req.event.watermark_text || watermarkSettings?.text || 'Protected' } : null; + const { resolvePhotoStorageKey: resolveSelectedKey } = require('../services/photoResolver'); + const { withLocalCopy: withSelectedLocalCopy } = require('../services/imageProcessor'); + const selectedStorage = getStorage(); for (const photo of photos) { + const name = photo.filename || `photo-${photo.id}.jpg`; + const storageKey = resolveSelectedKey(req.event, photo); try { - const filePath = resolvePhotoFilePath(req.event, photo); - const name = photo.filename || `photo-${photo.id}.jpg`; if (shouldApplyWatermark && effectiveSettings) { - try { - const watermarkedBuffer = await watermarkService.applyWatermark(filePath, effectiveSettings); - archive.append(watermarkedBuffer, { name }); - } catch (watermarkError) { - logger.warn('Failed to watermark selected photo, skipping original to avoid leak', { - slug: req.params.slug, - photoId: photo.id, - eventId: req.event.id, - error: watermarkError.message, - }); - } + const buf = storageKey + ? await withSelectedLocalCopy(storageKey, (lp) => + watermarkService.applyWatermark(lp, effectiveSettings) + ) + : await watermarkService.applyWatermark(resolvePhotoFilePath(req.event, photo), effectiveSettings); + archive.append(buf, { name }); + } else if (storageKey) { + const stream = await selectedStorage.get(storageKey); + archive.append(stream, { name }); } else { - archive.file(filePath, { name }); + archive.file(resolvePhotoFilePath(req.event, photo), { name }); } - } catch (resolveError) { - logger.warn('Skipping selected photo due to unresolved path', { + } catch (err) { + logger.warn('Skipping selected photo due to error', { slug: req.params.slug, photoId: photo.id, eventId: req.event.id, - error: resolveError.message, + error: err.message, }); } } diff --git a/backend/src/routes/protectedImages.js b/backend/src/routes/protectedImages.js index d99d230f..15ddbb1b 100644 --- a/backend/src/routes/protectedImages.js +++ b/backend/src/routes/protectedImages.js @@ -1,11 +1,12 @@ const express = require('express'); -const path = require('path'); const { db } = require('../database/db'); const { formatBoolean } = require('../utils/dbCompat'); const { verifyGalleryAccess } = require('../middleware/gallery'); const watermarkService = require('../services/watermarkService'); const secureImageService = require('../services/secureImageService'); -const { getStoragePath } = require('../config/storage'); +const { getStorage } = require('../services/storage'); +const { resolvePhotoStorageKey, resolvePhotoFilePath } = require('../services/photoResolver'); +const { withLocalCopy } = require('../services/imageProcessor'); const crypto = require('crypto'); const router = express.Router(); @@ -98,11 +99,11 @@ router.get('/:slug/photo/:photoId/view', verifyGalleryAccess, async (req, res) = fragmentImage: eventProtectionLevel === 'maximum' }; - // Build full path to photo - const photoPath = path.join(getStoragePath(), 'events/active', req.event.slug, photo.path); + // Resolve photo location through the storage backend (managed) or local + // disk (external reference mode). + const storageKey = resolvePhotoStorageKey(req.event, photo); + const storage = getStorage(); - // For basic/standard protection without special features, serve original file - // This avoids unnecessary recompression const needsProcessing = eventProtectionLevel === 'enhanced' || eventProtectionLevel === 'maximum' || protectionSettings.addFingerprint; @@ -110,15 +111,25 @@ router.get('/:slug/photo/:photoId/view', verifyGalleryAccess, async (req, res) = let finalImage; if (!needsProcessing) { - // Serve original file without processing - const fs = require('fs').promises; - finalImage = await fs.readFile(photoPath); + // Serve original bytes via the storage backend (or local disk for external). + if (storageKey) { + const stream = await storage.get(storageKey); + const chunks = []; + for await (const chunk of stream) chunks.push(chunk); + finalImage = Buffer.concat(chunks); + } else { + const fs = require('fs').promises; + finalImage = await fs.readFile(resolvePhotoFilePath(req.event, photo)); + } } else { - // Process image with protection measures - const processedImage = await secureImageService.processProtectedImage(photoPath, protectionSettings); + // secureImageService.processProtectedImage operates on a local path. + // Materialize a tmp local copy in S3 mode, then run processing. + const runProcessing = (lp) => secureImageService.processProtectedImage(lp, protectionSettings); + const processedImage = storageKey + ? await withLocalCopy(storageKey, runProcessing) + : await runProcessing(resolvePhotoFilePath(req.event, photo)); if (processedImage.type === 'fragmented') { - // Return fragmented image data for canvas reconstruction return res.json({ type: 'fragmented', fragments: processedImage.fragments.map(f => ({ @@ -273,12 +284,13 @@ router.get('/:slug/photo/:photoId/signed/:token', async (req, res) => { // Get watermark settings const watermarkSettings = await watermarkService.getWatermarkSettings(); - - // Build full path to photo - const photoPath = path.join(getStoragePath(), 'events/active', event.slug, photo.path); - - // Apply watermark if enabled - const imageBuffer = await watermarkService.applyWatermark(photoPath, watermarkSettings); + + // Apply watermark — managed photos are sourced via the storage backend + // (S3 mode materializes a tmp local copy via withLocalCopy). + const storageKey = resolvePhotoStorageKey(event, photo); + const imageBuffer = storageKey + ? await withLocalCopy(storageKey, (lp) => watermarkService.applyWatermark(lp, watermarkSettings)) + : await watermarkService.applyWatermark(resolvePhotoFilePath(event, photo), watermarkSettings); // Set appropriate headers res.set({ diff --git a/backend/src/routes/secureImages.js b/backend/src/routes/secureImages.js index 63256211..81689e59 100644 --- a/backend/src/routes/secureImages.js +++ b/backend/src/routes/secureImages.js @@ -5,7 +5,9 @@ const secureImageService = require('../services/secureImageService'); const secureImageMiddleware = require('../middleware/secureImageMiddleware'); const logger = require('../utils/logger'); const { formatBoolean } = require('../utils/dbCompat'); -const { resolvePhotoFilePath } = require('../services/photoResolver'); +const { resolvePhotoFilePath, resolvePhotoStorageKey } = require('../services/photoResolver'); +const { withLocalCopy } = require('../services/imageProcessor'); +const { getStorage } = require('../services/storage'); const router = express.Router(); @@ -139,18 +141,10 @@ router.get('/:slug/secure/:photoId/:token', return res.status(404).json({ error: 'Photo not found' }); } - let filePath; - try { - filePath = resolvePhotoFilePath(req.event, photo); - } catch (resolveError) { - logger.error('Failed to resolve photo path for secure token generation', { - slug: req.params.slug, - photoId, - eventId: req.event.id, - error: resolveError.message, - }); - return res.status(404).json({ error: 'Photo file not found' }); - } + // Resolve photo through storage backend (managed) or fall back to local + // path (external reference mode). secureImageService needs a local file, + // so we materialize a tmp copy via withLocalCopy in S3 mode. + const storageKey = resolvePhotoStorageKey(event, photo); // Get protection settings for this event const protectionSettings = { @@ -160,11 +154,21 @@ router.get('/:slug/secure/:photoId/:token', fragmentImage: event.use_canvas_rendering === true && fragment !== undefined }; - // Process image with protection measures - const processedImage = await secureImageService.processProtectedImage( - filePath, - protectionSettings - ); + let processedImage; + try { + const runProcessing = (lp) => secureImageService.processProtectedImage(lp, protectionSettings); + processedImage = storageKey + ? await withLocalCopy(storageKey, runProcessing) + : await runProcessing(resolvePhotoFilePath(event, photo)); + } catch (resolveError) { + logger.error('Failed to process secure image', { + slug: req.params.slug, + photoId, + eventId: event.id, + error: resolveError.message, + }); + return res.status(404).json({ error: 'Photo file not found' }); + } // Handle fragmented images if (processedImage.type === 'fragmented') { @@ -292,11 +296,30 @@ router.get('/:slug/secure-download/:photoId/:token', return res.status(404).json({ error: 'Photo not found' }); } - let filePath; + // Resolve photo through storage backend (managed) or local disk (external). + const storageKey = resolvePhotoStorageKey(req.event, photo); + + const watermarkService = require('../services/watermarkService'); + const watermarkSettings = await watermarkService.getWatermarkSettings(); + const wantsWatermark = watermarkSettings && watermarkSettings.enabled; + + let fileBuffer; try { - filePath = resolvePhotoFilePath(req.event, photo); + if (wantsWatermark) { + fileBuffer = storageKey + ? await withLocalCopy(storageKey, (lp) => watermarkService.applyWatermark(lp, watermarkSettings)) + : await watermarkService.applyWatermark(resolvePhotoFilePath(req.event, photo), watermarkSettings); + } else if (storageKey) { + const stream = await getStorage().get(storageKey); + const chunks = []; + for await (const chunk of stream) chunks.push(chunk); + fileBuffer = Buffer.concat(chunks); + } else { + const fs = require('fs').promises; + fileBuffer = await fs.readFile(resolvePhotoFilePath(req.event, photo)); + } } catch (resolveError) { - logger.error('Failed to resolve photo path for secure download', { + logger.error('Failed to fetch photo for secure download', { slug: req.params.slug, photoId, eventId: req.event.id, @@ -305,18 +328,6 @@ router.get('/:slug/secure-download/:photoId/:token', return res.status(404).json({ error: 'Photo file not found' }); } - // Apply watermark if enabled - const watermarkService = require('../services/watermarkService'); - const watermarkSettings = await watermarkService.getWatermarkSettings(); - - let fileBuffer; - if (watermarkSettings && watermarkSettings.enabled) { - fileBuffer = await watermarkService.applyWatermark(filePath, watermarkSettings); - } else { - const fs = require('fs').promises; - fileBuffer = await fs.readFile(filePath); - } - // Update download count await db('photos').where('id', photoId).increment('download_count', 1); diff --git a/backend/src/routes/v1/events.js b/backend/src/routes/v1/events.js index 2c214f64..8a760af0 100644 --- a/backend/src/routes/v1/events.js +++ b/backend/src/routes/v1/events.js @@ -182,6 +182,18 @@ router.post( type: 'admin', id: req.admin.id, name: req.admin.username }); + // Webhook lifecycle (#327). v1 events are not draft-aware, so they're + // both created AND published in the same call. + try { + const webhookService = require('../../services/webhookService'); + await webhookService.fire('event.created', { + event: { id, slug, event_name, event_type, event_date, share_url: shareUrl }, + }); + await webhookService.fire('event.published', { + event: { id, slug, event_name, share_url: shareUrl }, + }); + } catch (e) { /* non-fatal */ } + res.status(201).json({ id, slug, share_url: shareUrl, share_token: shareToken }); } catch (error) { logger.error('v1 POST /events failed', { error: error.message, stack: error.stack }); @@ -346,33 +358,39 @@ router.post( const event = await db('events').where({ id: req.params.id }).first(); if (!event) return res.status(404).json({ error: 'Event not found' }); - const finalDir = path.join(getStoragePath(), 'events/active', event.slug); - await fs.mkdir(finalDir, { recursive: true }); const ext = path.extname(req.file.originalname); const finalName = `${Date.now()}_${crypto.randomBytes(4).toString('hex')}${ext}`; - const finalPath = path.join(finalDir, finalName); - await fs.rename(tempPath, finalPath); - tempPath = null; + // photo.path is stored relative to events/active so resolvePhotoStorageKey + // can rebuild the full key on read. Same shape as adminPhotos uploads. + const relPath = path.posix.join(event.slug, finalName); + const finalKey = path.posix.join('events/active', relPath); - const stat = fsSync.statSync(finalPath); - const relPath = path.relative(path.join(getStoragePath(), 'events/active'), finalPath); + const stat = fsSync.statSync(tempPath); + + // Read sharp metadata + generate thumbnail FROM the local temp file + // before uploading the original through the storage backend. (Same + // ordering as adminPhotos.js so sharp/ffmpeg always have a real fs path.) + let width = null; + let height = null; + try { + const meta = await sharp(tempPath).metadata(); + width = meta.width || null; + height = meta.height || null; + } catch { /* non-fatal */ } let thumbRel = null; try { - const thumbPath = await generateThumbnail(finalPath); - thumbRel = path.relative(getStoragePath(), thumbPath); + thumbRel = await generateThumbnail(tempPath); } catch (err) { logger.warn('v1 thumbnail generation failed', { err: err.message }); } - // Detect image dimensions for masonry layouts. - let width = null; - let height = null; - try { - const meta = await sharp(finalPath).metadata(); - width = meta.width || null; - height = meta.height || null; - } catch { /* non-fatal */ } + // Upload the original via the storage backend (local fs OR S3), + // then drop the multer temp file. + const { getStorage } = require('../../services/storage'); + await getStorage().putFromFile(finalKey, tempPath, { contentType: req.file.mimetype }); + await fs.unlink(tempPath).catch(() => {}); + tempPath = null; const insertResult = await db('photos').insert({ event_id: event.id, @@ -394,6 +412,16 @@ router.post( type: 'admin', id: req.admin.id, name: req.admin.username }); + // Webhook (#327): one event per uploaded photo so receivers get a + // 1:1 stream they can react to. + try { + const webhookService = require('../../services/webhookService'); + await webhookService.fire('photo.uploaded', { + event: { id: event.id, slug: event.slug, event_name: event.event_name }, + photo: { id, filename: finalName, original_filename: req.file.originalname, size_bytes: stat.size, width, height }, + }); + } catch (e) { /* non-fatal */ } + res.status(201).json({ id, filename: finalName, path: relPath, thumbnail_path: thumbRel, size_bytes: stat.size }); } catch (error) { logger.error('v1 POST /events/:id/photos failed', { error: error.message }); diff --git a/backend/src/services/archiveService.js b/backend/src/services/archiveService.js index 59521393..8d17c1ab 100644 --- a/backend/src/services/archiveService.js +++ b/backend/src/services/archiveService.js @@ -1,57 +1,47 @@ const archiver = require('archiver'); -const fs = require('fs').promises; +const fs = require('fs'); +const fsp = require('fs').promises; const path = require('path'); +const os = require('os'); +const crypto = require('crypto'); const { db } = require('../database/db'); const { queueEmail } = require('./emailProcessor'); const logger = require('../utils/logger'); const feedbackService = require('./feedbackService'); - -const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage'); -const ACTIVE_PATH = () => path.join(getStoragePath(), 'events/active'); -const ARCHIVE_PATH = () => path.join(getStoragePath(), 'events/archived'); +const { getStorage } = require('./storage'); async function archiveEvent(event) { + const storage = getStorage(); + const archiveName = `${event.slug}.zip`; + const archiveRelKey = path.posix.join('events/archived', archiveName); + const eventPrefix = path.posix.join('events/active', event.slug); + + const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-archive-')); + const tmpArchive = path.join(tmpDir, `${crypto.randomBytes(4).toString('hex')}-${archiveName}`); + try { - const eventPath = path.join(ACTIVE_PATH(), event.slug); - const archiveName = `${event.slug}.zip`; - const archivePath = path.join(ARCHIVE_PATH(), archiveName); - - // Ensure archive directory exists - await fs.mkdir(ARCHIVE_PATH(), { recursive: true }); - - // Create archive - const output = require('fs').createWriteStream(archivePath); - const archive = archiver('zip', { - zlib: { level: 9 } // Maximum compression - }); - - archive.on('error', (err) => { - throw err; - }); - - // Export feedback data before archiving + // Collect feedback data first so it can be included as in-memory entries. + const feedbackEntries = []; const feedbackSettings = await feedbackService.getEventFeedbackSettings(event.id); if (feedbackSettings.feedback_enabled) { try { logger.info(`Exporting feedback data for event ${event.slug}`); const feedbackData = await feedbackService.exportEventFeedback(event.id); - + if (feedbackData && feedbackData.length > 0) { - // Create feedback JSON file - const feedbackJson = JSON.stringify(feedbackData, null, 2); - const feedbackJsonPath = path.join(eventPath, 'feedback_data.json'); - await fs.writeFile(feedbackJsonPath, feedbackJson, 'utf8'); - - // Create feedback CSV file - const feedbackCsv = convertToCSV(feedbackData); - const feedbackCsvPath = path.join(eventPath, 'feedback_data.csv'); - await fs.writeFile(feedbackCsvPath, feedbackCsv, 'utf8'); - - // Create feedback summary + feedbackEntries.push({ + name: 'feedback_data.json', + buffer: Buffer.from(JSON.stringify(feedbackData, null, 2), 'utf8'), + }); + feedbackEntries.push({ + name: 'feedback_data.csv', + buffer: Buffer.from(convertToCSV(feedbackData), 'utf8'), + }); const summary = await feedbackService.getEventFeedbackSummary(event.id); - const summaryPath = path.join(eventPath, 'feedback_summary.json'); - await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2), 'utf8'); - + feedbackEntries.push({ + name: 'feedback_summary.json', + buffer: Buffer.from(JSON.stringify(summary, null, 2), 'utf8'), + }); logger.info(`Feedback data exported: ${feedbackData.length} entries`); } } catch (error) { @@ -59,64 +49,110 @@ async function archiveEvent(event) { // Continue with archiving even if feedback export fails } } - - output.on('close', async () => { - try { - logger.info(`Archive created: ${archiveName} (${archive.pointer()} bytes)`); - // Update database - await db('events').where('id', event.id).update({ - is_archived: true, - archive_path: path.relative(getStoragePath(), archivePath), - archived_at: new Date() - }); + // Stream every photo (and any other content under events/active/{slug}/) into + // the zip directly from the storage backend. + const photoEntries = await storage.list(eventPrefix); - // Delete original files - await fs.rm(eventPath, { recursive: true }); + let totalBytes = 0; + await new Promise((resolve, reject) => { + const output = fs.createWriteStream(tmpArchive); + const archive = archiver('zip', { zlib: { level: 9 } }); - // Delete thumbnails - const photos = await db('photos').where('event_id', event.id); - for (const photo of photos) { - if (photo.thumbnail_path) { - const thumbPath = path.join(getStoragePath(), photo.thumbnail_path); - await fs.unlink(thumbPath).catch(() => {}); // Ignore if already deleted - } + output.on('close', () => { + totalBytes = archive.pointer(); + resolve(); + }); + archive.on('error', reject); + archive.pipe(output); + + const append = async () => { + for (const entry of photoEntries) { + const nameInZip = entry.key.startsWith(`${eventPrefix}/`) + ? entry.key.slice(eventPrefix.length + 1) + : entry.key; + const stream = await storage.get(entry.key); + archive.append(stream, { name: nameInZip }); } - - // Queue completion email — admin_email is nullable on events (migration 073); - // skip queueing rather than violating email_queue.recipient_email NOT NULL. - if (event.admin_email) { - await queueEmail(event.id, event.admin_email, 'archive_complete', { - event_name: event.event_name, - archive_size: (archive.pointer() / 1024 / 1024).toFixed(2) + ' MB' - }); - } else { - logger.info(`Skipping archive_complete email for event ${event.slug}: no admin_email set`); + for (const f of feedbackEntries) { + archive.append(f.buffer, { name: f.name }); } - } catch (err) { - // Never let the close handler reject — it runs detached from the caller, - // and an unhandled rejection here crashes the backend process. - logger.error(`Post-archive cleanup failed for event ${event.slug}:`, err); - } + archive.finalize(); + }; + + append().catch(reject); }); - - archive.pipe(output); - archive.directory(eventPath, false); - await archive.finalize(); - + + // Upload the finalized zip to the storage backend. + await storage.putFromFile(archiveRelKey, tmpArchive, { contentType: 'application/zip' }); + + logger.info(`Archive created: ${archiveName} (${totalBytes} bytes)`); + + // Update DB BEFORE deleting originals so a crash mid-cleanup leaves the + // archive accessible rather than orphaning the photos. + await db('events').where('id', event.id).update({ + is_archived: true, + archive_path: archiveRelKey, + archived_at: new Date(), + }); + + // Fire event.archived webhook (#327). Receivers infer per-photo loss + // from this event — we deliberately do NOT fire photo.deleted for each + // archived photo to avoid flooding subscribers on bulk archives. + try { + const webhookService = require('./webhookService'); + await webhookService.fire('event.archived', { + event: { id: event.id, slug: event.slug, event_name: event.event_name, archive_path: archiveRelKey }, + }); + } catch (e) { /* non-fatal */ } + + // Delete the originals from storage. + for (const entry of photoEntries) { + await storage.delete(entry.key).catch((err) => + logger.warn(`Failed to delete archived original ${entry.key}: ${err.message}`) + ); + } + + // Delete thumbnails for this event's photos. + const photos = await db('photos').where('event_id', event.id); + for (const photo of photos) { + if (photo.thumbnail_path) { + await storage.delete(photo.thumbnail_path).catch(() => {}); + } + if (photo.hero_path) { + await storage.delete(photo.hero_path).catch(() => {}); + } + // Best effort: remove watermarked variants too if a refactor added them. + if (photo.watermark_path) { + await storage.delete(photo.watermark_path).catch(() => {}); + } + } + + // Queue completion email — admin_email is nullable on events (migration 073); + // skip queueing rather than violating email_queue.recipient_email NOT NULL. + if (event.admin_email) { + await queueEmail(event.id, event.admin_email, 'archive_complete', { + event_name: event.event_name, + archive_size: (totalBytes / 1024 / 1024).toFixed(2) + ' MB', + }); + } else { + logger.info(`Skipping archive_complete email for event ${event.slug}: no admin_email set`); + } } catch (error) { logger.error(`Error archiving event ${event.slug}:`, error); throw error; + } finally { + await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); } } // Helper function to convert JSON to CSV function convertToCSV(data) { if (!data || data.length === 0) return ''; - + const headers = Object.keys(data[0]); const csvHeaders = headers.join(','); - + const csvRows = data.map(row => { return headers.map(header => { const value = row[header]; @@ -127,7 +163,7 @@ function convertToCSV(data) { return value || ''; }).join(','); }); - + return [csvHeaders, ...csvRows].join('\n'); } diff --git a/backend/src/services/downloadZipService.js b/backend/src/services/downloadZipService.js index e176d56a..f4b704ab 100644 --- a/backend/src/services/downloadZipService.js +++ b/backend/src/services/downloadZipService.js @@ -7,16 +7,22 @@ * * Pattern follows watermarkGeneratorService.js — singleton with * in-memory locking and debounced background regeneration. + * + * Storage: zips are written to a local tmp file then uploaded to the + * configured storage backend (local fs or S3) via storage.putFromFile. + * The cached zip is served via the storage backend on download. */ const fs = require('fs'); const fsp = require('fs/promises'); const path = require('path'); +const os = require('os'); +const crypto = require('crypto'); const archiver = require('archiver'); const { db } = require('../database/db'); const watermarkService = require('./watermarkService'); -const { resolvePhotoFilePath } = require('./photoResolver'); -const { getStoragePath } = require('../config/storage'); +const { resolvePhotoStorageKey, resolvePhotoFilePath } = require('./photoResolver'); +const { getStorage } = require('./storage'); const logger = require('../utils/logger'); const DEBOUNCE_MS = 5000; @@ -29,15 +35,16 @@ class DownloadZipService { } /** - * Absolute path to the cached zip for an event slug. + * Relative storage key for the cached zip. */ - getCachePath(slug) { - return path.join(getStoragePath(), 'events', 'active', slug, '.download-cache', 'all.zip'); + getCacheKey(slug) { + return path.posix.join('events/active', slug, '.download-cache', 'all.zip'); } /** * Check if a valid cached zip exists. - * Returns { path, size, generatedAt } or null. + * Returns { key, size, generatedAt } or null. The key is a relative storage + * key — callers stream it via storage.get() rather than reading directly. */ async getZipInfo(eventId) { try { @@ -48,15 +55,10 @@ class DownloadZipService { if (!event || !event.download_zip_path) return null; - const absPath = this.getCachePath(event.slug); - try { - const stat = await fsp.stat(absPath); - return { - path: absPath, - size: stat.size, - generatedAt: event.download_zip_generated_at, - }; - } catch { + const storage = getStorage(); + const key = this.getCacheKey(event.slug); + const stat = await storage.stat(key); + if (!stat) { // File gone — clear stale DB record await db('events').where({ id: eventId }).update({ download_zip_path: null, @@ -64,6 +66,11 @@ class DownloadZipService { }); return null; } + return { + key, + size: stat.size, + generatedAt: event.download_zip_generated_at, + }; } catch (err) { logger.warn('downloadZipService.getZipInfo error', { eventId, error: err.message }); return null; @@ -71,7 +78,7 @@ class DownloadZipService { } /** - * Generate the pre-zip for an event. Returns { success, path, size } or { success: false }. + * Generate the pre-zip for an event. Returns { success, key, size } or { success: false }. * Concurrent calls for the same eventId share one in-flight build. */ async generateZip(eventId) { @@ -97,6 +104,9 @@ class DownloadZipService { } async _build(eventId, version) { + const storage = getStorage(); + let tmpDir; + try { const event = await db('events').where({ id: eventId }).first(); if (!event) return { success: false, error: 'Event not found' }; @@ -119,11 +129,10 @@ class DownloadZipService { text: event.watermark_text || watermarkSettings?.text || 'Protected', } : null; - const cacheDir = path.dirname(this.getCachePath(event.slug)); - await fsp.mkdir(cacheDir, { recursive: true }); + const finalKey = this.getCacheKey(event.slug); - const tmpPath = this.getCachePath(event.slug) + `.tmp.${Date.now()}`; - const finalPath = this.getCachePath(event.slug); + tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-zipbuild-')); + const tmpPath = path.join(tmpDir, `${crypto.randomBytes(4).toString('hex')}-all.zip`); // Build zip — level 0 (store only) since photos are already compressed await new Promise((resolve, reject) => { @@ -145,13 +154,6 @@ class DownloadZipService { return reject(new Error('Build invalidated')); } - let filePath; - try { - filePath = resolvePhotoFilePath(event, photo); - } catch { - continue; - } - let archiveName; if (hasMultipleTypes) { const folderName = photo.type === 'individual' ? 'Individual Photos' : 'Collages'; @@ -160,14 +162,36 @@ class DownloadZipService { archiveName = photo.filename; } + // External-mode photos still live on local disk; managed photos go + // through the storage backend. resolvePhotoStorageKey returns null + // for external, in which case fall back to resolvePhotoFilePath. + const storageKey = resolvePhotoStorageKey(event, photo); + if (shouldApplyWatermark && effectiveSettings) { try { - const buf = await watermarkService.applyWatermark(filePath, effectiveSettings); + let sourcePath; + if (storageKey) { + // Stream the original to a tmp file just long enough for sharp + // (watermarkService) to operate on it. Avoids buffering the + // entire image in memory for huge originals. + sourcePath = path.join(tmpDir, `wm-${crypto.randomBytes(4).toString('hex')}`); + await storage.getToFile(storageKey, sourcePath); + } else { + sourcePath = resolvePhotoFilePath(event, photo); + } + const buf = await watermarkService.applyWatermark(sourcePath, effectiveSettings); archive.append(buf, { name: archiveName }); + if (storageKey) { + await fsp.unlink(sourcePath).catch(() => {}); + } } catch (err) { logger.warn('Skipping watermark in pre-zip', { photoId: photo.id, error: err.message }); } + } else if (storageKey) { + const stream = await storage.get(storageKey); + archive.append(stream, { name: archiveName }); } else { + const filePath = resolvePhotoFilePath(event, photo); archive.file(filePath, { name: archiveName }); } } @@ -180,29 +204,33 @@ class DownloadZipService { // Check version again — another invalidation may have arrived if (this.versions.get(eventId) !== version) { - await fsp.unlink(tmpPath).catch(() => {}); return { success: false, error: 'Build invalidated' }; } - // Atomic rename - await fsp.rename(tmpPath, finalPath); + // Upload to storage (atomic from caller's perspective: storage.put writes + // to a tmp file/object first then commits in LocalFs; in S3 the key only + // exists after the multipart upload completes). + await storage.putFromFile(finalKey, tmpPath, { contentType: 'application/zip' }); - const stat = await fsp.stat(finalPath); + const stat = await storage.stat(finalKey); - // Update DB await db('events').where({ id: eventId }).update({ - download_zip_path: `events/active/${event.slug}/.download-cache/all.zip`, + download_zip_path: finalKey, download_zip_generated_at: new Date(), }); logger.info('Pre-zip generated', { eventId, slug: event.slug, size: stat.size, photos: photos.length }); - return { success: true, path: finalPath, size: stat.size }; + return { success: true, key: finalKey, size: stat.size }; } catch (err) { if (err.message === 'Build invalidated') { return { success: false, error: 'Build invalidated' }; } logger.error('downloadZipService._build error', { eventId, error: err.message }); return { success: false, error: err.message }; + } finally { + if (tmpDir) { + await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + } } } @@ -264,17 +292,14 @@ class DownloadZipService { async _cleanup(eventId) { try { + const storage = getStorage(); const event = await db('events') .where({ id: eventId }) .select('slug', 'download_zip_path') .first(); if (event && event.download_zip_path) { - const absPath = this.getCachePath(event.slug); - await fsp.unlink(absPath).catch(() => {}); - // Also try to remove the cache directory if empty - const cacheDir = path.dirname(absPath); - await fsp.rmdir(cacheDir).catch(() => {}); + await storage.delete(this.getCacheKey(event.slug)).catch(() => {}); } await db('events').where({ id: eventId }).update({ diff --git a/backend/src/services/fileWatcher.js b/backend/src/services/fileWatcher.js index 1205aaad..6665f1ee 100644 --- a/backend/src/services/fileWatcher.js +++ b/backend/src/services/fileWatcher.js @@ -13,6 +13,16 @@ const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '. const WATCH_PATH = () => path.join(getStoragePath(), 'events/active'); function startFileWatcher() { + // Auto-import via filesystem watching only works with the local storage + // backend. In S3 mode there is no local directory to watch — every photo + // must enter through the admin upload API. Skip cleanly with a clear log + // so operators aren't surprised by the missing feature. + const backend = (process.env.STORAGE_BACKEND || 'local').toLowerCase(); + if (backend !== 'local') { + logger.warn(`[fileWatcher] auto-import disabled — STORAGE_BACKEND=${backend}. Use the admin upload API instead.`); + return null; + } + const watcher = chokidar.watch(WATCH_PATH(), { ignored: /(^|[\/\\])\../, // ignore dotfiles persistent: true, @@ -93,7 +103,7 @@ async function processNewPhoto(filePath) { if (!existingPhoto) { // Add to database - await db('photos').insert({ + const insertResult = await db('photos').insert({ event_id: event.id, filename: path.basename(filePath), path: relativePath, @@ -101,10 +111,21 @@ async function processNewPhoto(filePath) { type: isVideo ? 'video' : photoType, size_bytes: stats.size, mime_type: mimeType - }); + }).returning('id'); + const photoId = insertResult[0]?.id || insertResult[0]; logger.info(`Added new photo: ${relativePath}`); downloadZipService.invalidate(event.id); + + // Webhook (#327) — auto-import path. Only fires in local mode since + // the watcher is disabled in S3 mode. + try { + const webhookService = require('./webhookService'); + await webhookService.fire('photo.uploaded', { + event: { id: event.id, slug: event.slug, event_name: event.event_name }, + photo: { id: photoId, filename: path.basename(filePath), size_bytes: stats.size, source: 'auto-import' }, + }); + } catch (e) { /* non-fatal */ } } else { logger.debug(`Photo already exists: ${relativePath}`); } @@ -121,6 +142,16 @@ async function removePhoto(filePath) { if (photo) { downloadZipService.invalidate(photo.event_id); + + // Webhook (#327) — fire only if the row actually existed. + try { + const event = await db('events').where({ id: photo.event_id }).first(); + const webhookService = require('./webhookService'); + await webhookService.fire('photo.deleted', { + event: { id: photo.event_id, slug: event?.slug, event_name: event?.event_name }, + photo: { id: photo.id, filename: photo.filename, source: 'auto-import' }, + }); + } catch (e) { /* non-fatal */ } } logger.info(`Removed photo: ${relativePath}`); diff --git a/backend/src/services/imageProcessor.js b/backend/src/services/imageProcessor.js index dacb0ca1..67b9f865 100644 --- a/backend/src/services/imageProcessor.js +++ b/backend/src/services/imageProcessor.js @@ -1,9 +1,12 @@ const sharp = require('sharp'); const exifr = require('exifr'); const path = require('path'); -const fs = require('fs').promises; +const fsp = require('fs').promises; +const os = require('os'); +const crypto = require('crypto'); const logger = require('../utils/logger'); const { db } = require('../database/db'); +const { getStorage } = require('./storage'); // Configure sharp for better memory management with large batches sharp.cache(false); // Disable cache to prevent memory buildup @@ -16,8 +19,10 @@ const DEFAULT_THUMBNAIL_FIT = 'cover'; // 'cover' for square crops const DEFAULT_THUMBNAIL_QUALITY = 85; const DEFAULT_THUMBNAIL_FORMAT = 'jpeg'; -const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage'); -const getThumbnailPath = () => path.join(getStoragePath(), 'thumbnails'); +// Hero image settings - optimized for large displays +const DEFAULT_HERO_WIDTH = 1920; +const DEFAULT_HERO_HEIGHT = 1080; +const DEFAULT_HERO_QUALITY = 85; // Helper to parse setting value (handles both JSON-encoded and plain values) function parseSettingValue(value) { @@ -83,60 +88,62 @@ async function getThumbnailSettings() { } } +const contentTypeFor = (format) => { + if (format === 'png') return 'image/png'; + if (format === 'webp') return 'image/webp'; + return 'image/jpeg'; +}; + +/** + * Generate a thumbnail from a local source image path. The output is written + * to the storage backend (local fs or S3) under `thumbnails/thumb_` + * and the relative storage key is returned for DB persistence. + * + * Callers must ensure the source is on the local filesystem. For S3 mode + * regeneration flows, fetch via `withLocalCopy(storage, sourceKey, fn)` first. + */ async function generateThumbnail(imagePath, options = {}) { const filename = path.basename(imagePath); const thumbnailFilename = `thumb_${filename}`; - const thumbnailDir = getThumbnailPath(); - const thumbnailPath = path.join(thumbnailDir, thumbnailFilename); - + const thumbnailRelKey = path.posix.join('thumbnails', thumbnailFilename); + const storage = getStorage(); + // Get thumbnail settings const settings = await getThumbnailSettings(); - - // Ensure thumbnail directory exists - await fs.mkdir(thumbnailDir, { recursive: true }); - - // Check if we need to regenerate (for broken thumbnails) + + // Force regeneration: drop the existing object before writing the new one if (options.regenerate) { - try { - await fs.unlink(thumbnailPath); - logger.info(`Deleted broken thumbnail: ${thumbnailPath}`); - } catch (err) { - // File might not exist, that's okay - } + await storage.delete(thumbnailRelKey).catch(() => {}); } - + try { // First, verify the source image is complete and valid const metadata = await sharp(imagePath).metadata(); - + if (!metadata.width || !metadata.height) { throw new Error('Invalid image metadata - file may be incomplete'); } - - // Create sharp instance with memory-efficient settings - let sharpInstance = sharp(imagePath, { + + let sharpInstance = sharp(imagePath, { limitInputPixels: 268402689, // ~16k x 16k max - sequentialRead: true, // More memory efficient for large images - failOnError: false // Don't fail on minor issues + sequentialRead: true, + failOnError: false }); - + // Strip EXIF/metadata from thumbnails (privacy: prevent GPS leak etc.) sharpInstance = sharpInstance.withMetadata(false); - // Apply resize with configured settings - // For square thumbnails with 'cover' fit, we crop to center sharpInstance = sharpInstance.resize(settings.width, settings.height, { withoutEnlargement: true, - fit: settings.fit, // 'cover' will crop to fill the exact dimensions - position: 'center' // Center the crop for better composition + fit: settings.fit, + position: 'center' }); - - // Apply format-specific options + if (settings.format === 'jpeg') { - sharpInstance = sharpInstance.jpeg({ + sharpInstance = sharpInstance.jpeg({ quality: settings.quality, - progressive: true, // Progressive JPEG for better loading - mozjpeg: true // Better compression + progressive: true, + mozjpeg: true }); } else if (settings.format === 'png') { sharpInstance = sharpInstance.png({ @@ -147,74 +154,87 @@ async function generateThumbnail(imagePath, options = {}) { } else if (settings.format === 'webp') { sharpInstance = sharpInstance.webp({ quality: settings.quality, - effort: 4 // Balance between speed and compression + effort: 4 }); } - - // Save the thumbnail - await sharpInstance.toFile(thumbnailPath); - - // Verify the thumbnail was created successfully - const stats = await fs.stat(thumbnailPath); - if (stats.size === 0) { + + const buffer = await sharpInstance.toBuffer(); + if (!buffer || buffer.length === 0) { throw new Error('Generated thumbnail is empty'); } - - return path.relative(getStoragePath(), thumbnailPath); + + await storage.put(thumbnailRelKey, buffer, { contentType: contentTypeFor(settings.format) }); + + return thumbnailRelKey; } catch (error) { const msg = (error && error.message) ? error.message : String(error); logger.error(`Failed to generate thumbnail for ${filename}: ${msg}`); - - // Clean up any partially created file - try { - await fs.unlink(thumbnailPath); - } catch (unlinkErr) { - // Ignore unlink errors - } - - // Return null if thumbnail generation fails, don't fail the whole upload + + // Clean up any partially uploaded object + await storage.delete(thumbnailRelKey).catch(() => {}); + return null; } } /** - * Check if a thumbnail exists and is valid + * Check if a thumbnail exists and is valid. For local-fs storage we open the + * file with sharp to confirm it parses; for S3 we trust the byte-integrity + * checks built into the protocol and only verify size > 0. */ async function isThumbnailValid(thumbnailPath) { + const storage = getStorage(); try { - const fullPath = path.join(getStoragePath(), thumbnailPath); - const stats = await fs.stat(fullPath); - - // Check if file exists and has content - if (stats.size === 0) { + const stat = await storage.stat(thumbnailPath); + if (!stat || stat.size === 0) { return false; } - - // Try to read metadata to ensure it's a valid image - await sharp(fullPath).metadata(); + if (storage.kind() === 'local') { + const localPath = storage.resolveLocalPath(thumbnailPath); + await sharp(localPath).metadata(); + } return true; } catch (error) { return false; } } +/** + * Wraps a callback that needs the source image as a local file. In local-fs + * mode the storage path is used directly (no copy); in S3 mode the object is + * streamed to a tmp file which is removed afterwards. + */ +async function withLocalCopy(sourceKey, fn) { + const storage = getStorage(); + if (storage.kind() === 'local') { + return fn(storage.resolveLocalPath(sourceKey)); + } + const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-src-')); + const tmpPath = path.join(tmpDir, `${crypto.randomBytes(4).toString('hex')}_${path.basename(sourceKey)}`); + try { + await storage.getToFile(sourceKey, tmpPath); + return await fn(tmpPath); + } finally { + await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + } +} + /** * Regenerate thumbnail if it's broken or missing */ async function ensureThumbnail(photo) { - const { db } = require('../database/db'); - const { resolvePhotoFilePath } = require('./photoResolver'); - let originalPath; + const { resolvePhotoStorageKey } = require('./photoResolver'); + let sourceKey; try { const event = await db('events').where('id', photo.event_id).first(); - originalPath = resolvePhotoFilePath(event, photo); - logger.info(`Ensuring thumbnail for photo ${photo.id} from source: ${originalPath}`); + sourceKey = resolvePhotoStorageKey(event, photo); + logger.info(`Ensuring thumbnail for photo ${photo.id} from key: ${sourceKey}`); } catch (e) { const msg = (e && e.message) ? e.message : String(e); - logger.error(`Failed to resolve original path for thumbnail (photo ${photo.id}): ${msg}`); + logger.error(`Failed to resolve original key for thumbnail (photo ${photo.id}): ${msg}`); return null; } - + // Check if thumbnail exists and is valid if (photo.thumbnail_path) { const isValid = await isThumbnailValid(photo.thumbnail_path); @@ -223,45 +243,40 @@ async function ensureThumbnail(photo) { } logger.warn(`Invalid thumbnail detected for photo ${photo.id}, regenerating...`); } - - // Generate new thumbnail - const newThumbnailPath = await generateThumbnail(originalPath, { regenerate: true }); - + + // Generate new thumbnail (sources via withLocalCopy so this works in S3 mode) + const newThumbnailPath = await withLocalCopy(sourceKey, (localPath) => + generateThumbnail(localPath, { regenerate: true }) + ); + if (newThumbnailPath) { - // Update database with new thumbnail path - const { db } = require('../database/db'); await db('photos') .where({ id: photo.id }) .update({ thumbnail_path: newThumbnailPath }); - + logger.info(`Regenerated thumbnail for photo ${photo.id}`); return newThumbnailPath; } - + return null; } async function generateVideoPlaceholder(originalFilename, options = {}) { const parsed = path.parse(originalFilename || ''); const baseName = parsed.name || 'video'; - const thumbnailDir = getThumbnailPath(); const thumbnailFilename = `thumb_${baseName}.jpg`; - const thumbnailPath = path.join(thumbnailDir, thumbnailFilename); + const thumbnailRelKey = path.posix.join('thumbnails', thumbnailFilename); + const storage = getStorage(); const settings = await getThumbnailSettings(); const width = settings.width || DEFAULT_THUMBNAIL_WIDTH; const height = settings.height || DEFAULT_THUMBNAIL_HEIGHT; if (options.regenerate) { - try { - await fs.unlink(thumbnailPath); - } catch (_) { - // ignore if missing - } + await storage.delete(thumbnailRelKey).catch(() => {}); } try { - await fs.mkdir(thumbnailDir, { recursive: true }); const svg = ` @@ -279,26 +294,20 @@ async function generateVideoPlaceholder(originalFilename, options = {}) { `; - await sharp(Buffer.from(svg)) + const buffer = await sharp(Buffer.from(svg)) .resize(width, height, { fit: 'cover' }) .jpeg({ quality: settings.quality || DEFAULT_THUMBNAIL_QUALITY }) - .toFile(thumbnailPath); + .toBuffer(); - return path.relative(getStoragePath(), thumbnailPath); + await storage.put(thumbnailRelKey, buffer, { contentType: 'image/jpeg' }); + + return thumbnailRelKey; } catch (error) { logger.error('Failed to generate video placeholder thumbnail:', error.message); return null; } } -// Hero image settings - optimized for large displays -const DEFAULT_HERO_WIDTH = 1920; -const DEFAULT_HERO_HEIGHT = 1080; -const DEFAULT_HERO_QUALITY = 85; -const DEFAULT_HERO_FORMAT = 'jpeg'; - -const getHeroPath = () => path.join(getStoragePath(), 'heroes'); - /** * Generate a hero-optimized image for gallery headers * Outputs a 1920x1080 image suitable for full-width hero sections @@ -306,36 +315,24 @@ const getHeroPath = () => path.join(getStoragePath(), 'heroes'); async function generateHeroImage(imagePath, options = {}) { const filename = path.basename(imagePath); const heroFilename = `hero_${filename}`; - const heroDir = getHeroPath(); - const heroPath = path.join(heroDir, heroFilename); + const heroRelKey = path.posix.join('heroes', heroFilename); + const storage = getStorage(); - // Ensure hero directory exists - await fs.mkdir(heroDir, { recursive: true }); - - // Check if we need to regenerate if (options.regenerate) { - try { - await fs.unlink(heroPath); - logger.info(`Deleted existing hero image: ${heroPath}`); - } catch (err) { - // File might not exist, that's okay - } + await storage.delete(heroRelKey).catch(() => {}); } try { - // First, verify the source image is complete and valid const metadata = await sharp(imagePath).metadata(); if (!metadata.width || !metadata.height) { throw new Error('Invalid image metadata - file may be incomplete'); } - // Calculate dimensions to maintain aspect ratio while fitting within hero bounds const heroWidth = options.width || DEFAULT_HERO_WIDTH; const heroHeight = options.height || DEFAULT_HERO_HEIGHT; const quality = options.quality || DEFAULT_HERO_QUALITY; - // Create sharp instance with memory-efficient settings let sharpInstance = sharp(imagePath, { limitInputPixels: 268402689, sequentialRead: true, @@ -345,43 +342,31 @@ async function generateHeroImage(imagePath, options = {}) { // Strip EXIF/metadata from hero images (privacy: prevent GPS leak etc.) sharpInstance = sharpInstance.withMetadata(false); - // Resize to fit hero dimensions while maintaining aspect ratio - // Use 'cover' to fill the hero area (crops if needed) sharpInstance = sharpInstance.resize(heroWidth, heroHeight, { - withoutEnlargement: false, // Allow upscaling for small images + withoutEnlargement: false, fit: 'cover', position: 'center' }); - // Apply JPEG format with high quality sharpInstance = sharpInstance.jpeg({ quality: quality, progressive: true, mozjpeg: true }); - // Save the hero image - await sharpInstance.toFile(heroPath); - - // Verify the hero image was created successfully - const stats = await fs.stat(heroPath); - if (stats.size === 0) { + const buffer = await sharpInstance.toBuffer(); + if (!buffer || buffer.length === 0) { throw new Error('Generated hero image is empty'); } - logger.info(`Generated hero image for ${filename}: ${heroPath}`); - return path.relative(getStoragePath(), heroPath); + await storage.put(heroRelKey, buffer, { contentType: 'image/jpeg' }); + + logger.info(`Generated hero image for ${filename} → ${heroRelKey}`); + return heroRelKey; } catch (error) { const msg = (error && error.message) ? error.message : String(error); logger.error(`Failed to generate hero image for ${filename}: ${msg}`); - - // Clean up any partially created file - try { - await fs.unlink(heroPath); - } catch (unlinkErr) { - // Ignore unlink errors - } - + await storage.delete(heroRelKey).catch(() => {}); return null; } } @@ -390,16 +375,16 @@ async function generateHeroImage(imagePath, options = {}) { * Check if a hero image exists and is valid */ async function isHeroValid(heroPath) { + const storage = getStorage(); try { - const fullPath = path.join(getStoragePath(), heroPath); - const stats = await fs.stat(fullPath); - - if (stats.size === 0) { + const stat = await storage.stat(heroPath); + if (!stat || stat.size === 0) { return false; } - - // Try to read metadata to ensure it's a valid image - await sharp(fullPath).metadata(); + if (storage.kind() === 'local') { + const localPath = storage.resolveLocalPath(heroPath); + await sharp(localPath).metadata(); + } return true; } catch (error) { return false; @@ -410,21 +395,19 @@ async function isHeroValid(heroPath) { * Ensure a hero image exists for a photo, regenerate if needed */ async function ensureHeroImage(photo) { - const { db } = require('../database/db'); - const { resolvePhotoFilePath } = require('./photoResolver'); + const { resolvePhotoStorageKey } = require('./photoResolver'); - let originalPath; + let sourceKey; try { const event = await db('events').where('id', photo.event_id).first(); - originalPath = resolvePhotoFilePath(event, photo); - logger.info(`Ensuring hero image for photo ${photo.id} from source: ${originalPath}`); + sourceKey = resolvePhotoStorageKey(event, photo); + logger.info(`Ensuring hero image for photo ${photo.id} from key: ${sourceKey}`); } catch (e) { const msg = (e && e.message) ? e.message : String(e); - logger.error(`Failed to resolve original path for hero image (photo ${photo.id}): ${msg}`); + logger.error(`Failed to resolve original key for hero image (photo ${photo.id}): ${msg}`); return null; } - // Check if hero image exists and is valid if (photo.hero_path) { const isValid = await isHeroValid(photo.hero_path); if (isValid) { @@ -433,11 +416,11 @@ async function ensureHeroImage(photo) { logger.warn(`Invalid hero image detected for photo ${photo.id}, regenerating...`); } - // Generate new hero image - const newHeroPath = await generateHeroImage(originalPath, { regenerate: true }); + const newHeroPath = await withLocalCopy(sourceKey, (localPath) => + generateHeroImage(localPath, { regenerate: true }) + ); if (newHeroPath) { - // Update database with new hero path await db('photos') .where({ id: photo.id }) .update({ hero_path: newHeroPath }); @@ -451,12 +434,9 @@ async function ensureHeroImage(photo) { /** * Extract capture date from EXIF metadata - * @param {string} imagePath - Path to the image file - * @returns {Date|null} - The capture date or null if not available */ async function extractCaptureDate(imagePath) { try { - // Parse EXIF data, looking for common date fields const exif = await exifr.parse(imagePath, { pick: ['DateTimeOriginal', 'CreateDate', 'DateTimeDigitized', 'ModifyDate'] }); @@ -465,23 +445,19 @@ async function extractCaptureDate(imagePath) { return null; } - // Priority order: DateTimeOriginal > CreateDate > DateTimeDigitized > ModifyDate const captureDate = exif.DateTimeOriginal || exif.CreateDate || exif.DateTimeDigitized || exif.ModifyDate; if (captureDate) { - // exifr returns Date objects directly when parsing dates if (captureDate instanceof Date) { - // Validate the date is reasonable (not in the future, not before 1990) const now = new Date(); const minDate = new Date('1990-01-01'); if (captureDate > minDate && captureDate <= now) { return captureDate; } } - // Handle string dates if necessary if (typeof captureDate === 'string') { const parsed = new Date(captureDate); if (!isNaN(parsed.getTime())) { @@ -492,7 +468,6 @@ async function extractCaptureDate(imagePath) { return null; } catch (error) { - // Log only as debug - many images don't have EXIF data logger.debug(`Could not extract EXIF date from ${path.basename(imagePath)}:`, error.message); return null; } @@ -506,5 +481,6 @@ module.exports = { generateHeroImage, isHeroValid, ensureHeroImage, - extractCaptureDate + extractCaptureDate, + withLocalCopy, }; diff --git a/backend/src/services/photoProcessor.js b/backend/src/services/photoProcessor.js index 0cbd8cf7..5d793600 100644 --- a/backend/src/services/photoProcessor.js +++ b/backend/src/services/photoProcessor.js @@ -4,9 +4,7 @@ const { db } = require('../database/db'); const { generateThumbnail } = require('./imageProcessor'); const { generatePhotoFilename } = require('../utils/filenameSanitizer'); const { processUploadedVideo, isVideoMimeType } = require('./videoProcessor'); - -// Get storage path from environment or default -const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage'); +const { getStorage } = require('./storage'); function normalizeFiles(files) { // Handle null, undefined, or falsy values @@ -99,11 +97,6 @@ async function processUploadedPhotos(files, eventId, uploadedBy = 'admin', categ extension ); - // Move file to event folder - const destPath = path.join(getStoragePath(), 'events/active', event.slug); - await fs.mkdir(destPath, { recursive: true }); - - const newPath = path.join(destPath, newFilename); const tempPath = file?.path || file?.filepath || file?.tempFilePath; if (!tempPath) { @@ -116,7 +109,7 @@ async function processUploadedPhotos(files, eventId, uploadedBy = 'admin', categ throw new Error(`Uploaded file is missing a temporary path. File info: ${fileInfo}`); } - // Verify temp file exists before copying + // Verify temp file exists before processing try { await fs.access(tempPath); } catch (accessErr) { @@ -127,56 +120,33 @@ async function processUploadedPhotos(files, eventId, uploadedBy = 'admin', categ throw new Error(`Uploaded file not found at temporary location: ${tempPath}`); } - // Use copyFile and unlink instead of rename to avoid cross-device issues - try { - await fs.copyFile(tempPath, newPath); - console.log(`Successfully copied ${file.originalname} to ${newPath}`); - } catch (copyErr) { - console.error(`Failed to copy file from ${tempPath} to ${newPath}:`, copyErr); - throw new Error(`Failed to copy uploaded file: ${copyErr.message}`); - } finally { - // Clean up temp file with better error handling - try { - await fs.unlink(tempPath); - console.log(`Cleaned up temp file: ${tempPath}`); - } catch (unlinkErr) { - // Only warn if file exists but couldn't be deleted - // ENOENT means file was already deleted, which is fine - if (unlinkErr?.code !== 'ENOENT') { - console.warn(`Failed to clean up temp upload ${tempPath}:`, { - error: unlinkErr.message, - code: unlinkErr.code - }); - } - } - } - + // Final storage key under events/active/{slug}/{newFilename}. + const relativePath = path.posix.join(event.slug, newFilename); + const finalKey = path.posix.join('events/active', relativePath); + // Determine if this is a video or image const isVideo = isVideoMimeType(file.mimetype); const mediaType = isVideo ? 'video' : 'image'; - // Generate thumbnail and extract metadata + // Generate thumbnail and extract metadata FROM the temp file (still on + // local disk) before uploading the original. let thumbnailPath; let videoMetadata = null; let imageMetadata = null; if (isVideo) { - // Process video: extract metadata and generate thumbnail - const thumbnailDir = path.join(getStoragePath(), 'thumbnails'); - await fs.mkdir(thumbnailDir, { recursive: true }); - const videoThumbnailPath = path.join(thumbnailDir, `thumb_${newFilename.replace(/\.[^.]+$/, '.jpg')}`); - - const result = await processUploadedVideo(newPath, videoThumbnailPath); + const videoThumbnailKey = path.posix.join( + 'thumbnails', + `thumb_${newFilename.replace(/\.[^.]+$/, '.jpg')}` + ); + const result = await processUploadedVideo(tempPath, videoThumbnailKey); videoMetadata = result.metadata; - thumbnailPath = path.relative(getStoragePath(), videoThumbnailPath); + thumbnailPath = result.thumbnailKey; } else { - // Process image: generate thumbnail and extract dimensions - thumbnailPath = await generateThumbnail(newPath); - - // Extract image dimensions using sharp + thumbnailPath = await generateThumbnail(tempPath); try { const sharp = require('sharp'); - const metadata = await sharp(newPath).metadata(); + const metadata = await sharp(tempPath).metadata(); if (metadata.width && metadata.height) { imageMetadata = { width: metadata.width, @@ -188,10 +158,29 @@ async function processUploadedPhotos(files, eventId, uploadedBy = 'admin', categ } } - // Calculate relative paths - const storagePath = getStoragePath(); - const relativePath = path.relative(path.join(storagePath, 'events/active'), newPath); - const relativeThumbPath = thumbnailPath; // thumbnailPath is already relative to storage root + // Now upload the original through the storage backend and remove the + // local temp copy. + try { + await getStorage().putFromFile(finalKey, tempPath, { + contentType: file.mimetype, + }); + } catch (uploadErr) { + console.error(`Failed to upload ${file.originalname} → ${finalKey}:`, uploadErr); + throw new Error(`Failed to upload to storage: ${uploadErr.message}`); + } finally { + try { + await fs.unlink(tempPath); + } catch (unlinkErr) { + if (unlinkErr?.code !== 'ENOENT') { + console.warn(`Failed to clean up temp upload ${tempPath}:`, { + error: unlinkErr.message, + code: unlinkErr.code + }); + } + } + } + + const relativeThumbPath = thumbnailPath; // Add to database with uploaded_by field and media metadata let insertResult; @@ -247,7 +236,23 @@ async function processUploadedPhotos(files, eventId, uploadedBy = 'admin', categ // Commit transaction await trx.commit(); - + + // Webhook (#327) — fires for every entry path that lands in this + // service: guest upload + auto-import + admin upload via API. + try { + const webhookService = require('./webhookService'); + await webhookService.fire('photo.uploaded', { + event: { id: event.id, slug: event.slug, event_name: event.event_name }, + photo: { + id: photoId, + filename: newFilename, + original_filename: file.originalname, + size_bytes: file.size, + uploaded_by: uploadedBy, + }, + }); + } catch (e) { /* non-fatal */ } + uploadedPhotos.push({ id: photoId, filename: newFilename, diff --git a/backend/src/services/photoReplacementService.js b/backend/src/services/photoReplacementService.js index b607a132..3680cbbd 100644 --- a/backend/src/services/photoReplacementService.js +++ b/backend/src/services/photoReplacementService.js @@ -13,10 +13,10 @@ const { db } = require('../database/db'); const { generateThumbnail, extractCaptureDate } = require('./imageProcessor'); const { generatePhotoFilename } = require('../utils/filenameSanitizer'); const watermarkGeneratorService = require('./watermarkGeneratorService'); +const { getStorage } = require('./storage'); +const { resolvePhotoStorageKey } = require('./photoResolver'); const logger = require('../utils/logger'); -const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage'); - /** * Find a replacement candidate by matching original_filename (case-insensitive). * Returns the photo row if exactly one match, { ambiguous: true, count } if multiple, or null. @@ -42,46 +42,21 @@ async function findReplacementCandidate(eventId, originalFilename) { * @returns {{ success: boolean, photo?: Object, error?: string }} */ async function replacePhoto(existingPhoto, newFileTempPath, { originalFilename, mimeType, event }) { - const eventDir = path.join(getStoragePath(), 'events', 'active', event.slug); const categorySlug = existingPhoto.type === 'collage' ? 'collages' : 'individual'; - const targetDir = path.join(eventDir, categorySlug); try { - // Generate new filename + // Generate new filename + storage key const ext = path.extname(originalFilename); const newFilename = generatePhotoFilename(event.event_name, categorySlug, Date.now(), ext); - const tempTargetPath = path.join(targetDir, `_replacing_${Date.now()}_${newFilename}`); - const finalPath = path.join(targetDir, newFilename); - const relativePath = path.join(event.slug, categorySlug, newFilename); + const relativePath = path.posix.join(event.slug, categorySlug, newFilename); + const finalKey = path.posix.join('events/active', relativePath); + const storage = getStorage(); - // Write new file to temp name in target directory - await fsp.mkdir(targetDir, { recursive: true }); - await fsp.copyFile(newFileTempPath, tempTargetPath); - - // Delete old physical file - const oldFilePath = path.join(getStoragePath(), 'events', 'active', existingPhoto.path); - await fsp.unlink(oldFilePath).catch(() => {}); - - // Delete old thumbnail - if (existingPhoto.thumbnail_path) { - const oldThumbPath = path.join(getStoragePath(), existingPhoto.thumbnail_path); - await fsp.unlink(oldThumbPath).catch(() => {}); - } - - // Delete old watermark cache - try { - await watermarkGeneratorService.deleteForPhoto(existingPhoto.id); - } catch { - // Ignore — watermark may not exist - } - - // Rename temp → final - await fsp.rename(tempTargetPath, finalPath); - - // Extract metadata from new file + // Sharp/EXIF need a local file. The temp file from multer still satisfies + // that — we read metadata before uploading the original to storage. let capturedAt = null; try { - capturedAt = await extractCaptureDate(finalPath); + capturedAt = await extractCaptureDate(newFileTempPath); } catch { // No EXIF — keep null } @@ -89,23 +64,41 @@ async function replacePhoto(existingPhoto, newFileTempPath, { originalFilename, let width = null; let height = null; try { - const metadata = await sharp(finalPath).metadata(); + const metadata = await sharp(newFileTempPath).metadata(); width = metadata.width || null; height = metadata.height || null; } catch { // Non-image or corrupt } - const stats = await fsp.stat(finalPath); + const stats = await fsp.stat(newFileTempPath); - // Generate new thumbnail + // Generate new thumbnail FROM the local temp before uploading the original. let thumbnailPath = null; try { - thumbnailPath = await generateThumbnail(finalPath); + thumbnailPath = await generateThumbnail(newFileTempPath); } catch { logger.warn('Failed to generate thumbnail for replaced photo', { photoId: existingPhoto.id }); } + // Delete old assets BEFORE uploading the new key — if they share the path + // (rare but possible if filename collision), we want the new content. + const oldOriginalKey = resolvePhotoStorageKey(event, existingPhoto); + if (oldOriginalKey && oldOriginalKey !== finalKey) { + await storage.delete(oldOriginalKey).catch(() => {}); + } + if (existingPhoto.thumbnail_path && existingPhoto.thumbnail_path !== thumbnailPath) { + await storage.delete(existingPhoto.thumbnail_path).catch(() => {}); + } + try { + await watermarkGeneratorService.deleteForPhoto(existingPhoto.id); + } catch { + // Ignore — watermark may not exist + } + + // Upload the new original. + await storage.putFromFile(finalKey, newFileTempPath, { contentType: mimeType }); + // Update DB record — preserve id, event_id, category_id, type, visibility, // uploaded_at, sort_order, feedback counts, view/download counts const updates = { diff --git a/backend/src/services/photoResolver.js b/backend/src/services/photoResolver.js index 340fd9ef..91f0a76f 100644 --- a/backend/src/services/photoResolver.js +++ b/backend/src/services/photoResolver.js @@ -4,6 +4,39 @@ const { safePathJoin } = require('../utils/fileSecurityUtils'); const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage'); +/** + * Resolve a managed photo's relative key under the storage backend. + * Returns null for external-mode photos (those never live in the managed + * storage backend; callers should fall back to resolvePhotoFilePath for + * external references on local disk). + * + * Storage layout (relative to STORAGE_PATH or S3 bucket prefix): + * events/active/{slug}/individual/{filename} + * events/active/{slug}/collages/{filename} + * + * Legacy `photo.path` values may already include `events/active/` — we + * normalize so the returned key always has it exactly once. + */ +function resolvePhotoStorageKey(event, photo) { + if (!event || !photo) throw new Error('resolvePhotoStorageKey requires event and photo'); + + const mode = (photo.source_origin || event.source_mode || 'managed'); + if (mode === 'reference' || mode === 'external') { + // External photos don't live in the managed backend. + return null; + } + + const rel = photo.path ? photo.path.replace(/\\/g, '/').replace(/^\/+/, '') : ''; + if (!rel) { + throw new Error(`resolvePhotoStorageKey: photo.path is empty for photo ${photo.id}`); + } + + // Already prefixed (legacy uploads from a previous code revision). + if (rel.startsWith('events/active/')) return rel; + + return path.posix.join('events/active', rel); +} + /** * Resolve absolute photo file path based on event + photo origin * Managed: storage/events/active + photo.path (legacy variants supported) @@ -19,6 +52,13 @@ function resolvePhotoFilePath(event, photo) { const mode = (photo.source_origin || event.source_mode || 'managed'); if (mode === 'reference' || mode === 'external') { if (!photo.external_relpath) { + // Mixed-source events: a reference-mode event can also hold managed + // (uploaded) photos. If we have a regular `path` and no + // external_relpath, treat this row as managed instead of throwing. + if (photo.path && !photo.source_origin) { + const relativeSegment = photo.path.replace(/^\/+/, ''); + return safePathJoin(path.join(getStoragePath(), 'events/active'), relativeSegment); + } throw new Error('Missing external_relpath for external photo'); } // Normalize duplicate leaf segments (e.g., event.external_path ends with 'individual' @@ -51,4 +91,5 @@ function resolvePhotoFilePath(event, photo) { module.exports = { resolvePhotoFilePath, + resolvePhotoStorageKey, }; diff --git a/backend/src/services/storage/LocalFsStorage.js b/backend/src/services/storage/LocalFsStorage.js new file mode 100644 index 00000000..c1eacb72 --- /dev/null +++ b/backend/src/services/storage/LocalFsStorage.js @@ -0,0 +1,166 @@ +const fs = require('fs'); +const fsp = require('fs').promises; +const path = require('path'); +const { pipeline } = require('stream/promises'); +const crypto = require('crypto'); + +const logger = require('../../utils/logger'); + +/** + * Filesystem-backed implementation of the StorageBackend interface. + * All keys are relative to `root` (typically process.env.STORAGE_PATH). + * + * Path traversal protection: every key is normalized to POSIX form and rejected + * if it tries to escape the root via "..". Callers should not need to think + * about this — but if a key arrives via user input it must still be filtered. + */ +class LocalFsStorage { + constructor({ root }) { + if (!root) throw new Error('LocalFsStorage requires a `root` directory'); + this.root = path.resolve(root); + } + + kind() { + return 'local'; + } + + async init() { + await fsp.mkdir(this.root, { recursive: true }); + // Sanity check: must be writable. + const probe = path.join(this.root, '.storage-write-probe'); + await fsp.writeFile(probe, ''); + await fsp.unlink(probe); + logger.info(`[storage] LocalFsStorage initialized at ${this.root}`); + } + + _resolve(relPath) { + if (!relPath || typeof relPath !== 'string') { + throw new Error(`LocalFsStorage: invalid relative path: ${relPath}`); + } + const normalized = path.posix.normalize(relPath.replace(/\\/g, '/')); + if (normalized.startsWith('..') || normalized.includes('/../') || normalized === '..') { + throw new Error(`LocalFsStorage: path traversal rejected: ${relPath}`); + } + return path.join(this.root, normalized); + } + + async put(relPath, body, _options = {}) { + const abs = this._resolve(relPath); + await fsp.mkdir(path.dirname(abs), { recursive: true }); + // Write to a sibling tmp file first then rename for crash safety. + const tmp = `${abs}.tmp.${process.pid}.${crypto.randomBytes(4).toString('hex')}`; + try { + if (Buffer.isBuffer(body)) { + await fsp.writeFile(tmp, body); + } else if (body && typeof body.pipe === 'function') { + await pipeline(body, fs.createWriteStream(tmp)); + } else { + throw new Error('LocalFsStorage.put: body must be a Buffer or Readable stream'); + } + await fsp.rename(tmp, abs); + } catch (err) { + await fsp.unlink(tmp).catch(() => {}); + throw err; + } + } + + async putFromFile(relPath, localPath, _options = {}) { + const abs = this._resolve(relPath); + await fsp.mkdir(path.dirname(abs), { recursive: true }); + // copyFile is atomic from the destination's perspective on POSIX. + await fsp.copyFile(localPath, abs); + } + + async get(relPath) { + const abs = this._resolve(relPath); + return fs.createReadStream(abs); + } + + async getToFile(relPath, localPath) { + const abs = this._resolve(relPath); + await fsp.mkdir(path.dirname(localPath), { recursive: true }); + await fsp.copyFile(abs, localPath); + } + + async exists(relPath) { + try { + await fsp.access(this._resolve(relPath), fs.constants.F_OK); + return true; + } catch { + return false; + } + } + + async stat(relPath) { + try { + const s = await fsp.stat(this._resolve(relPath)); + return { size: s.size, mtime: s.mtime }; + } catch (err) { + if (err.code === 'ENOENT') return null; + throw err; + } + } + + async delete(relPath) { + try { + await fsp.unlink(this._resolve(relPath)); + } catch (err) { + if (err.code !== 'ENOENT') throw err; + } + } + + async list(prefix) { + const absPrefix = this._resolve(prefix || '.'); + const entries = []; + async function walk(dir, relBase) { + let dirents; + try { + dirents = await fsp.readdir(dir, { withFileTypes: true }); + } catch (err) { + if (err.code === 'ENOENT') return; + throw err; + } + for (const ent of dirents) { + const childAbs = path.join(dir, ent.name); + const childRel = relBase ? `${relBase}/${ent.name}` : ent.name; + if (ent.isDirectory()) { + await walk(childAbs, childRel); + } else if (ent.isFile()) { + const s = await fsp.stat(childAbs); + entries.push({ key: childRel, size: s.size, mtime: s.mtime }); + } + } + } + const baseRel = prefix && prefix !== '.' ? prefix.replace(/\\/g, '/') : ''; + await walk(absPrefix, baseRel); + return entries; + } + + async rename(srcRelPath, dstRelPath) { + const src = this._resolve(srcRelPath); + const dst = this._resolve(dstRelPath); + await fsp.mkdir(path.dirname(dst), { recursive: true }); + await fsp.rename(src, dst); + } + + async copy(srcRelPath, dstRelPath) { + const src = this._resolve(srcRelPath); + const dst = this._resolve(dstRelPath); + await fsp.mkdir(path.dirname(dst), { recursive: true }); + await fsp.copyFile(src, dst); + } + + async signedUrl(_relPath, _ttlSeconds = 300) { + throw new Error('LocalFsStorage does not support signedUrl. Set STORAGE_BACKEND=s3 to use presigned URLs.'); + } + + // Escape hatch for callers that genuinely need a filesystem path + // (e.g. ffmpeg, archiver — anything that takes a path argument rather + // than a stream). S3Storage exposes the same method but returns null, + // forcing callers to use the streaming API instead. + resolveLocalPath(relPath) { + return this._resolve(relPath); + } +} + +module.exports = LocalFsStorage; diff --git a/backend/src/services/storage/S3StorageBackend.js b/backend/src/services/storage/S3StorageBackend.js new file mode 100644 index 00000000..3bfb07ef --- /dev/null +++ b/backend/src/services/storage/S3StorageBackend.js @@ -0,0 +1,165 @@ +const path = require('path'); +const fs = require('fs'); +const fsp = require('fs').promises; +const { HeadObjectCommand } = require('@aws-sdk/client-s3'); + +const S3StorageAdapter = require('./s3Storage'); +const logger = require('../../utils/logger'); + +/** + * StorageBackend wrapper around the existing S3StorageAdapter. + * + * S3StorageAdapter was originally written for the backup service and exposes + * upload/download/uploadStream/etc. This thin layer maps that surface onto the + * canonical put/get/exists/delete/list/rename/copy/signedUrl interface used by + * the rest of the codebase, and applies an optional `prefix` so a single bucket + * can host multiple deployments without collisions. + * + * Atomicity: S3 has no rename. `rename()` is implemented as `copy()` + `delete()`. + * If the process crashes between the two, the source object remains until the + * next list-and-prune sweep — see `cleanupAbandonedTempUploads()` callers. + */ +class S3StorageBackend { + constructor(config) { + if (!config || !config.bucket) { + throw new Error('S3StorageBackend requires a bucket name'); + } + this.adapter = new S3StorageAdapter(config); + this.prefix = (config.prefix || '').replace(/^\/+|\/+$/g, ''); + } + + kind() { + return 's3'; + } + + _key(relPath) { + if (!relPath || typeof relPath !== 'string') { + throw new Error(`S3StorageBackend: invalid relative path: ${relPath}`); + } + const normalized = relPath.replace(/\\/g, '/').replace(/^\.?\/+/, ''); + if (normalized.startsWith('..') || normalized.includes('/../')) { + throw new Error(`S3StorageBackend: path traversal rejected: ${relPath}`); + } + return this.prefix ? `${this.prefix}/${normalized}` : normalized; + } + + async init() { + await this.adapter.testConnection(); + logger.info(`[storage] S3StorageBackend initialized bucket=${this.adapter.bucket} prefix=${this.prefix || '(none)'}`); + } + + async put(relPath, body, options = {}) { + const key = this._key(relPath); + if (Buffer.isBuffer(body)) { + const { Readable } = require('stream'); + const stream = Readable.from(body); + await this.adapter.uploadStream(stream, key, { + contentType: options.contentType, + cacheControl: options.cacheControl, + }); + return; + } + if (body && typeof body.pipe === 'function') { + await this.adapter.uploadStream(body, key, { + contentType: options.contentType, + cacheControl: options.cacheControl, + }); + return; + } + throw new Error('S3StorageBackend.put: body must be a Buffer or Readable stream'); + } + + async putFromFile(relPath, localPath, options = {}) { + await this.adapter.upload(localPath, this._key(relPath), { + contentType: options.contentType, + cacheControl: options.cacheControl, + }); + } + + async get(relPath) { + return this.adapter.downloadStream(this._key(relPath)); + } + + async getToFile(relPath, localPath) { + await fsp.mkdir(path.dirname(localPath), { recursive: true }); + await this.adapter.download(this._key(relPath), localPath); + } + + async exists(relPath) { + return this.adapter.exists(this._key(relPath)); + } + + async stat(relPath) { + try { + const head = await this.adapter.s3Client.send( + new HeadObjectCommand({ Bucket: this.adapter.bucket, Key: this._key(relPath) }) + ); + return { + size: head.ContentLength, + mtime: head.LastModified, + }; + } catch (err) { + if (err.name === 'NotFound' || err.$metadata?.httpStatusCode === 404) return null; + throw err; + } + } + + async delete(relPath) { + try { + await this.adapter.delete(this._key(relPath)); + } catch (err) { + if (err.name === 'NoSuchKey' || err.$metadata?.httpStatusCode === 404) return; + throw err; + } + } + + async list(prefix) { + const fullPrefix = this._key(prefix || '.'); + const entries = []; + let continuationToken; + do { + const result = await this.adapter.list(fullPrefix, { continuationToken }); + for (const obj of result.Contents || []) { + const stripped = this.prefix && obj.Key.startsWith(`${this.prefix}/`) + ? obj.Key.slice(this.prefix.length + 1) + : obj.Key; + entries.push({ key: stripped, size: obj.Size, mtime: obj.LastModified }); + } + continuationToken = result.NextContinuationToken; + } while (continuationToken); + return entries; + } + + async copy(srcRelPath, dstRelPath) { + await this.adapter.copy(this._key(srcRelPath), this._key(dstRelPath)); + } + + async rename(srcRelPath, dstRelPath) { + await this.copy(srcRelPath, dstRelPath); + await this.delete(srcRelPath); + } + + async signedUrl(relPath, ttlSeconds = 300) { + return this.adapter.getSignedUrl('getObject', this._key(relPath), { expiresIn: ttlSeconds }); + } + + // S3 has no local path; consumers that need one must use getToFile to a + // temp location first. Returning null here makes the contract explicit so + // legacy code using `storage.resolveLocalPath` fails fast instead of + // silently constructing a bad path. + resolveLocalPath(_relPath) { + return null; + } + + // Expose the underlying adapter so backupService keeps working. + // New code should prefer the canonical interface above. + get rawAdapter() { + return this.adapter; + } + + static fileStreamFromPath(localPath) { + return fs.createReadStream(localPath); + } +} + +module.exports = S3StorageBackend; diff --git a/backend/src/services/storage/StorageBackend.js b/backend/src/services/storage/StorageBackend.js new file mode 100644 index 00000000..e5d075db --- /dev/null +++ b/backend/src/services/storage/StorageBackend.js @@ -0,0 +1,42 @@ +/** + * Storage backend interface that LocalFsStorage and S3Storage implement. + * + * All paths are POSIX-style relative keys under the deployment's storage root + * (e.g. "events/active/wedding-smith/individual/IMG_0001.jpg"). Concrete adapters + * resolve the absolute filesystem path or S3 key internally so callers never deal + * with the difference between local and remote storage. + * + * Concurrency: methods are safe to call in parallel; ordering is the caller's + * responsibility. `put` is best-effort atomic (LocalFs writes to a temp file + * then renames; S3 returns only after the multipart upload is finalized). + * + * @typedef {Object} PutOptions + * @property {string} [contentType] - MIME type stored in object metadata. + * @property {string} [cacheControl] - Cache-Control header (S3 only). + * + * @typedef {Object} StatResult + * @property {number} size - Size in bytes. + * @property {Date} [mtime] - Last modified timestamp (best-effort; S3 uses LastModified). + * + * @typedef {Object} ListEntry + * @property {string} key - Relative path under the storage root. + * @property {number} size - Size in bytes. + * @property {Date} [mtime] - Last modified timestamp. + * + * @typedef {Object} StorageBackend + * @property {() => string} kind - Returns 'local' or 's3'. + * @property {() => Promise} init - Validates configuration and reachability. Called once at startup. + * @property {(relPath: string, body: NodeJS.ReadableStream | Buffer, options?: PutOptions) => Promise} put + * @property {(relPath: string, localPath: string, options?: PutOptions) => Promise} putFromFile + * @property {(relPath: string) => Promise} get - Returns a readable stream of the object body. + * @property {(relPath: string, localPath: string) => Promise} getToFile - Streams the object to a local path (creates parent dirs). + * @property {(relPath: string) => Promise} exists + * @property {(relPath: string) => Promise} stat - Null if missing. + * @property {(relPath: string) => Promise} delete - No-op if missing. + * @property {(prefix: string) => Promise} list + * @property {(srcRelPath: string, dstRelPath: string) => Promise} rename - Atomic on local fs; copy+delete on S3. + * @property {(srcRelPath: string, dstRelPath: string) => Promise} copy + * @property {(relPath: string, ttlSeconds?: number) => Promise} signedUrl - Presigned download URL (S3 only; LocalFs throws). + */ + +module.exports = {}; diff --git a/backend/src/services/storage/index.js b/backend/src/services/storage/index.js new file mode 100644 index 00000000..cb224484 --- /dev/null +++ b/backend/src/services/storage/index.js @@ -0,0 +1,102 @@ +const LocalFsStorage = require('./LocalFsStorage'); +const S3StorageBackend = require('./S3StorageBackend'); +const { getStoragePath } = require('../../config/storage'); +const logger = require('../../utils/logger'); + +let instance = null; + +/** + * Build the storage backend selected by STORAGE_BACKEND env var. + * + * STORAGE_BACKEND=local (default) + * Uses STORAGE_PATH on the local filesystem. Backwards compatible with every + * existing deployment. + * + * STORAGE_BACKEND=s3 + * Reads STORAGE_S3_* vars. Compatible with AWS S3 and any S3-compatible + * service (MinIO, R2, Backblaze, Wasabi, DigitalOcean Spaces, etc.) by + * pointing STORAGE_S3_ENDPOINT at the alternate host. + * + * Required S3 vars: + * STORAGE_S3_BUCKET + * STORAGE_S3_REGION (default us-east-1) + * STORAGE_S3_ACCESS_KEY + * STORAGE_S3_SECRET_KEY + * Optional S3 vars: + * STORAGE_S3_ENDPOINT — custom endpoint URL (MinIO/R2/etc.) + * STORAGE_S3_PREFIX — namespace prefix inside the bucket + * STORAGE_S3_FORCE_PATH_STYLE=true|false (default: auto when endpoint set) + * STORAGE_S3_SSL=true|false (default: true) + */ +function buildStorage() { + const backend = (process.env.STORAGE_BACKEND || 'local').toLowerCase(); + + if (backend === 's3') { + const required = ['STORAGE_S3_BUCKET', 'STORAGE_S3_ACCESS_KEY', 'STORAGE_S3_SECRET_KEY']; + const missing = required.filter((v) => !process.env[v]); + if (missing.length) { + throw new Error( + `STORAGE_BACKEND=s3 but missing required env vars: ${missing.join(', ')}` + ); + } + return new S3StorageBackend({ + bucket: process.env.STORAGE_S3_BUCKET, + region: process.env.STORAGE_S3_REGION || 'us-east-1', + endpoint: process.env.STORAGE_S3_ENDPOINT, + accessKeyId: process.env.STORAGE_S3_ACCESS_KEY, + secretAccessKey: process.env.STORAGE_S3_SECRET_KEY, + prefix: process.env.STORAGE_S3_PREFIX, + forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true' ? true : undefined, + sslEnabled: process.env.STORAGE_S3_SSL !== 'false', + }); + } + + if (backend !== 'local') { + throw new Error(`Unknown STORAGE_BACKEND: ${backend}. Expected 'local' or 's3'.`); + } + + return new LocalFsStorage({ root: getStoragePath() }); +} + +/** + * Lazily build + memoize the storage backend. Tests can pass an injected + * instance via `setStorageForTesting` to bypass env-var configuration. + */ +function getStorage() { + if (!instance) { + instance = buildStorage(); + } + return instance; +} + +/** @internal */ +function setStorageForTesting(stub) { + instance = stub; +} + +/** @internal — clear the memoized instance so the next call re-reads env. */ +function resetStorage() { + instance = null; +} + +/** + * Initialize the configured backend. Call once at server startup so config errors + * surface before any request comes in. + */ +async function initStorage() { + const storage = getStorage(); + try { + await storage.init(); + } catch (err) { + logger.error(`[storage] init failed for backend=${storage.kind()}: ${err.message}`); + throw err; + } + return storage; +} + +module.exports = { + getStorage, + initStorage, + setStorageForTesting, + resetStorage, +}; diff --git a/backend/src/services/videoProcessor.js b/backend/src/services/videoProcessor.js index fee6e13b..885e568e 100644 --- a/backend/src/services/videoProcessor.js +++ b/backend/src/services/videoProcessor.js @@ -2,7 +2,11 @@ const ffmpeg = require('fluent-ffmpeg'); const ffmpegPath = require('@ffmpeg-installer/ffmpeg').path; const path = require('path'); const fs = require('fs').promises; +const fsSync = require('fs'); +const os = require('os'); +const crypto = require('crypto'); const logger = require('../utils/logger'); +const { getStorage } = require('./storage'); // Set FFmpeg path ffmpeg.setFfmpegPath(ffmpegPath); @@ -45,36 +49,48 @@ async function extractVideoMetadata(videoPath) { } /** - * Generate thumbnail from video - * @param {string} videoPath - Path to the video file - * @param {string} outputPath - Path for the output thumbnail - * @param {Object} options - Thumbnail options - * @returns {Promise} - Path to generated thumbnail + * Generate a video thumbnail and persist it via the storage backend. + * + * @param {string} videoPath - Local path to the video file (ffmpeg needs a real fs path). + * @param {string} thumbnailKey - Relative storage key the thumbnail will be saved under + * (e.g. "thumbnails/thumb_video.jpg"). + * @param {Object} options + * @returns {Promise} The thumbnail's relative storage key. */ -async function generateVideoThumbnail(videoPath, outputPath, options = {}) { +async function generateVideoThumbnail(videoPath, thumbnailKey, options = {}) { const { - timeOffset = '00:00:01', // Take screenshot at 1 second - size = '300x300', - quality = 2 // 1-31, lower is better quality + timeOffset = '00:00:01', + size = '300x300' } = options; - return new Promise((resolve, reject) => { - ffmpeg(videoPath) - .screenshots({ - timestamps: [timeOffset], - filename: path.basename(outputPath), - folder: path.dirname(outputPath), - size: size - }) - .on('end', () => { - logger.info('Video thumbnail generated', { videoPath, outputPath }); - resolve(outputPath); - }) - .on('error', (err) => { - logger.error('Error generating video thumbnail', { error: err.message, videoPath }); - reject(err); - }); - }); + const storage = getStorage(); + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'picpeak-vidthumb-')); + const tmpFilename = `${crypto.randomBytes(4).toString('hex')}_${path.basename(thumbnailKey)}`; + const tmpPath = path.join(tmpDir, tmpFilename); + + try { + await new Promise((resolve, reject) => { + ffmpeg(videoPath) + .screenshots({ + timestamps: [timeOffset], + filename: tmpFilename, + folder: tmpDir, + size: size + }) + .on('end', () => resolve()) + .on('error', (err) => reject(err)); + }); + + if (!fsSync.existsSync(tmpPath)) { + throw new Error('ffmpeg did not produce a thumbnail file'); + } + + await storage.putFromFile(thumbnailKey, tmpPath, { contentType: 'image/jpeg' }); + logger.info('Video thumbnail generated', { videoPath, thumbnailKey }); + return thumbnailKey; + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + } } /** @@ -108,37 +124,33 @@ async function getVideoDuration(videoPath) { } /** - * Process uploaded video - extract metadata and generate thumbnail - * @param {string} videoPath - Path to the video file - * @param {string} thumbnailPath - Path for the thumbnail - * @param {Object} options - Processing options - * @returns {Promise} - Video metadata and processing result + * Process an uploaded video: extract metadata and produce a thumbnail through + * the storage backend. + * + * @param {string} videoPath - Local path to the source video (ffmpeg requires fs). + * @param {string} thumbnailKey - Relative storage key for the thumbnail. + * @returns {Promise<{success: boolean, metadata: Object, thumbnailKey: string}>} */ -async function processUploadedVideo(videoPath, thumbnailPath, options = {}) { +async function processUploadedVideo(videoPath, thumbnailKey, options = {}) { try { - // Validate video const isValid = await isValidVideo(videoPath); if (!isValid) { throw new Error('Invalid video file'); } - // Extract metadata const metadata = await extractVideoMetadata(videoPath); + await generateVideoThumbnail(videoPath, thumbnailKey, options); - // Generate thumbnail - await generateVideoThumbnail(videoPath, thumbnailPath, options); - - // Verify thumbnail was created - try { - await fs.access(thumbnailPath); - } catch (err) { - throw new Error('Thumbnail generation failed'); + const storage = getStorage(); + const exists = await storage.exists(thumbnailKey); + if (!exists) { + throw new Error('Thumbnail generation failed (not in storage)'); } return { success: true, metadata, - thumbnailPath + thumbnailKey }; } catch (error) { logger.error('Error processing video', { error: error.message, videoPath }); diff --git a/backend/src/services/watermarkGeneratorService.js b/backend/src/services/watermarkGeneratorService.js index 1dde9ec1..eaf4bad6 100644 --- a/backend/src/services/watermarkGeneratorService.js +++ b/backend/src/services/watermarkGeneratorService.js @@ -8,10 +8,10 @@ * - Tracking regeneration progress */ -const path = require('path'); const { db } = require('../database/db'); const watermarkService = require('./watermarkService'); -const { getStoragePath } = require('../config/storage'); +const { resolvePhotoStorageKey, resolvePhotoFilePath } = require('./photoResolver'); +const { withLocalCopy } = require('./imageProcessor'); class WatermarkGeneratorService { constructor() { @@ -57,14 +57,15 @@ class WatermarkGeneratorService { return { success: false, error: 'Watermarking is disabled' }; } - // Resolve the original file path - const originalPath = this.resolvePhotoPath(photo); - if (!originalPath) { - return { success: false, error: 'Could not resolve photo path' }; - } - - // Generate and save watermark - const result = await watermarkService.generateAndSaveWatermark(photo, originalPath, settings); + // Resolve the source via the storage backend (managed) or local disk + // (external reference mode). watermarkService needs a local file path. + const event = { slug: photo.slug, source_mode: photo.source_mode, external_path: photo.external_path }; + const storageKey = resolvePhotoStorageKey(event, photo); + const result = storageKey + ? await withLocalCopy(storageKey, (lp) => + watermarkService.generateAndSaveWatermark(photo, lp, settings) + ) + : await watermarkService.generateAndSaveWatermark(photo, resolvePhotoFilePath(event, photo), settings); if (result.success) { // Update database with watermark path @@ -83,31 +84,6 @@ class WatermarkGeneratorService { } } - /** - * Resolve the full file path for a photo - */ - resolvePhotoPath(photo) { - const storagePath = getStoragePath(); - - // Handle external/reference mode - if (photo.source_mode === 'reference' && photo.external_relpath) { - const externalRoot = process.env.EXTERNAL_MEDIA_PATH || path.join(storagePath, 'external'); - return path.join(externalRoot, photo.external_path || '', photo.external_relpath); - } - - // Standard managed mode - if (photo.file_path) { - // file_path might be absolute or relative - if (path.isAbsolute(photo.file_path)) { - return photo.file_path; - } - return path.join(storagePath, photo.file_path); - } - - // Fallback to constructing path from slug and filename - return path.join(storagePath, 'events', 'active', photo.slug, photo.filename); - } - /** * Generate watermarks for all photos in an event * @param {number} eventId - The event ID @@ -189,12 +165,13 @@ class WatermarkGeneratorService { */ async processPhotoWatermark(photo, settings) { try { - const originalPath = this.resolvePhotoPath(photo); - if (!originalPath) { - return { success: false, photoId: photo.id, error: 'Could not resolve path' }; - } - - const result = await watermarkService.generateAndSaveWatermark(photo, originalPath, settings); + const event = { slug: photo.slug, source_mode: photo.source_mode, external_path: photo.external_path }; + const storageKey = resolvePhotoStorageKey(event, photo); + const result = storageKey + ? await withLocalCopy(storageKey, (lp) => + watermarkService.generateAndSaveWatermark(photo, lp, settings) + ) + : await watermarkService.generateAndSaveWatermark(photo, resolvePhotoFilePath(event, photo), settings); if (result.success) { await db('photos') diff --git a/backend/src/services/watermarkService.js b/backend/src/services/watermarkService.js index 37d08172..c46f252f 100644 --- a/backend/src/services/watermarkService.js +++ b/backend/src/services/watermarkService.js @@ -2,7 +2,7 @@ const sharp = require('sharp'); const path = require('path'); const fs = require('fs').promises; const { db } = require('../database/db'); -const { getStoragePath } = require('../config/storage'); +const { getStorage } = require('./storage'); class WatermarkService { constructor() { @@ -234,19 +234,6 @@ class WatermarkService { this.cache.clear(); } - /** - * Get the watermarks directory path, creating it if needed - */ - async getWatermarksDir() { - const watermarksDir = path.join(getStoragePath(), 'watermarks'); - try { - await fs.access(watermarksDir); - } catch { - await fs.mkdir(watermarksDir, { recursive: true }); - } - return watermarksDir; - } - /** * Get the file extension from a filename */ @@ -258,46 +245,42 @@ class WatermarkService { } /** - * Generate watermarked version of a photo and save to disk + * Generate watermarked version of a photo and persist it through the + * storage backend. The source must be a local filesystem path because + * sharp doesn't take streams; callers in S3 mode should materialize a + * tmp local copy via imageProcessor.withLocalCopy first. + * * @param {Object} photo - Photo object with id, filename, and path info - * @param {string} originalPath - Full path to the original image file + * @param {string} originalPath - Local path to the original image file * @param {Object} settings - Watermark settings (optional, will fetch if not provided) * @returns {Object} { success, watermarkPath, error } */ async generateAndSaveWatermark(photo, originalPath, settings = null) { try { - // Get settings if not provided if (!settings) { settings = await this.getWatermarkSettings(); } - // If watermarking is disabled, return early if (!settings || !settings.enabled) { return { success: false, watermarkPath: null, error: 'Watermarking is disabled' }; } - // Verify original file exists try { await fs.access(originalPath); } catch { return { success: false, watermarkPath: null, error: 'Original file not found' }; } - // Generate watermarked buffer using existing method const watermarkedBuffer = await this.applyWatermark(originalPath, settings); - // Determine output path - const watermarksDir = await this.getWatermarksDir(); const ext = this.getFileExtension(photo.filename); const outputFilename = `${photo.id}_watermarked${ext}`; - const outputPath = path.join(watermarksDir, outputFilename); - - // Write the watermarked image to disk - await fs.writeFile(outputPath, watermarkedBuffer); - - // Return relative path for database storage const relativePath = `watermarks/${outputFilename}`; + await getStorage().put(relativePath, watermarkedBuffer, { + contentType: ext === '.png' ? 'image/png' : ext === '.webp' ? 'image/webp' : 'image/jpeg', + }); + return { success: true, watermarkPath: relativePath, @@ -314,22 +297,18 @@ class WatermarkService { } /** - * Delete a pre-generated watermark file - * @param {string} watermarkPath - Relative path to the watermark file - * @returns {boolean} - True if deleted successfully + * Delete a pre-generated watermark file from the storage backend. + * @param {string} watermarkPath - Relative storage key (e.g. "watermarks/123_watermarked.jpg") + * @returns {boolean} - True if a delete was attempted (no-op if missing) */ async deleteWatermarkFile(watermarkPath) { if (!watermarkPath) return false; try { - const fullPath = path.join(getStoragePath(), watermarkPath); - await fs.unlink(fullPath); + await getStorage().delete(watermarkPath); return true; } catch (error) { - // File might not exist, which is fine - if (error.code !== 'ENOENT') { - console.error('Error deleting watermark file:', error); - } + console.error('Error deleting watermark file:', error); return false; } } diff --git a/tests/e2e/s3-storage-roundtrip.spec.ts b/tests/e2e/s3-storage-roundtrip.spec.ts new file mode 100644 index 00000000..024a0e72 --- /dev/null +++ b/tests/e2e/s3-storage-roundtrip.spec.ts @@ -0,0 +1,135 @@ +import { test, expect } from '@playwright/test'; +import fs from 'fs'; +import path from 'path'; + +/** + * End-to-end smoke for the S3 storage backend (#328). + * + * What this verifies: + * - admin can upload photos via the API + * - thumbnail + hero generation lands in S3 (visible via the public gallery) + * - the gallery photo route streams the original through the backend + * - admin delete removes the original from S3 (subsequent gets 404) + * + * How to run: + * 1. Start dev stack with S3 mode + MinIO. The simplest way is to bring up + * MinIO from docker-compose.dev.yml and override the backend env: + * + * docker compose -f docker-compose.dev.yml up -d minio minio-init postgres redis + * STORAGE_BACKEND=s3 \ + * STORAGE_S3_BUCKET=picpeak-storage \ + * STORAGE_S3_REGION=us-east-1 \ + * STORAGE_S3_ENDPOINT=http://localhost:7104 \ + * STORAGE_S3_ACCESS_KEY=minioadmin \ + * STORAGE_S3_SECRET_KEY=minioadmin \ + * STORAGE_S3_FORCE_PATH_STYLE=true \ + * STORAGE_S3_SSL=false \ + * npm --prefix backend run dev + * + * 2. Run this spec: + * PLAYWRIGHT_BASE_URL=http://localhost:7100 npx playwright test \ + * tests/e2e/s3-storage-roundtrip.spec.ts --project=chromium + * + * The test auto-skips against backends that don't expose STORAGE_BACKEND=s3 + * via the /health endpoint, so it's safe to leave in the shared E2E suite. + */ + +const ADMIN_EMAIL = process.env.ADMIN_EMAIL || 'admin@example.com'; +const ADMIN_PASSWORD = process.env.ADMIN_PASSWORD || 'Admin!234'; +const GALLERY_PASSWORD = process.env.GALLERY_PASSWORD || 'PlaywrightGallery123!'; +const TEST_ASSET = path.join(__dirname, '..', '..', 'test-assets', 'img1.png'); + +async function isS3Backend(baseUrl: string): Promise { + // Explicit opt-in for runs against an S3-configured backend. The spec + // auto-skips otherwise so it's safe to leave in the shared E2E suite. + if (process.env.TEST_S3_MODE === '1') return true; + try { + const res = await fetch(`${baseUrl}/health`); + if (!res.ok) return false; + const body = await res.json().catch(() => ({})); + return body?.storage?.backend === 's3' || body?.storageBackend === 's3'; + } catch { + return false; + } +} + +test.describe('S3 storage round-trip (#328)', () => { + test.beforeAll(async ({}, testInfo) => { + const baseUrl = testInfo.project.use.baseURL || process.env.PLAYWRIGHT_BASE_URL || 'http://localhost:3000'; + const isS3 = await isS3Backend(baseUrl); + test.skip(!isS3, 'Backend is not running with STORAGE_BACKEND=s3 — see spec docstring for setup.'); + }); + + test('upload → serve → delete round-trip through the storage backend', async ({ request }) => { + expect(fs.existsSync(TEST_ASSET), `Test asset missing at ${TEST_ASSET}`).toBe(true); + + // Admin login — auth lives in the HttpOnly admin_token cookie which the + // request fixture retains across subsequent calls automatically. + const loginRes = await request.post('/api/auth/admin/login', { + data: { username: ADMIN_EMAIL, password: ADMIN_PASSWORD }, + }); + expect(loginRes.ok(), `login failed: ${loginRes.status()}`).toBeTruthy(); + + // Create event + const eventName = `S3 Roundtrip ${Date.now()}`; + const eventDate = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString().slice(0, 10); + const eventRes = await request.post('/api/admin/events', { + headers: { 'Content-Type': 'application/json' }, + data: { + event_type: 'wedding', + event_name: eventName, + event_date: eventDate, + customer_name: 'S3 Host', + customer_email: 'host@example.com', + host_name: 'S3 Host', + host_email: 'host@example.com', + admin_email: ADMIN_EMAIL, + password: GALLERY_PASSWORD, + expiration_days: 30, + }, + }); + expect(eventRes.ok(), `event create failed: ${eventRes.status()}`).toBeTruthy(); + const eventBody = await eventRes.json(); + const eventId: number = eventBody?.event?.id ?? eventBody?.id; + const slug: string = eventBody?.event?.slug ?? eventBody?.slug; + expect(eventId).toBeTruthy(); + expect(slug).toBeTruthy(); + + // Upload a single photo + const uploadRes = await request.post(`/api/admin/photos/${eventId}/upload`, { + multipart: { + photos: { name: 'img1.png', mimeType: 'image/png', buffer: fs.readFileSync(TEST_ASSET) }, + }, + }); + expect(uploadRes.ok(), `upload failed: ${uploadRes.status()}`).toBeTruthy(); + const uploadBody = await uploadRes.json(); + const photoId: number = uploadBody?.photos?.[0]?.id; + expect(photoId, 'uploaded photo missing from response').toBeTruthy(); + + // Wait briefly for thumbnail generation to settle. + await new Promise((r) => setTimeout(r, 1000)); + + // Fetch the thumbnail through the admin route — proves the storage backend + // can read what it wrote and the route streams it correctly. + const thumbRes = await request.get(`/api/admin/photos/${eventId}/thumbnail/${photoId}`); + expect(thumbRes.ok(), `thumbnail GET failed: ${thumbRes.status()}`).toBeTruthy(); + const thumbBytes = await thumbRes.body(); + expect(thumbBytes.length).toBeGreaterThan(100); + + // Fetch the original photo through the admin route. + const photoRes = await request.get(`/api/admin/photos/${eventId}/photo/${photoId}`); + expect(photoRes.ok(), `photo GET failed: ${photoRes.status()}`).toBeTruthy(); + const photoBytes = await photoRes.body(); + expect(photoBytes.length).toBeGreaterThan(100); + + // Delete the photo and confirm subsequent fetches 404. + const deleteRes = await request.delete(`/api/admin/photos/${eventId}/photos/${photoId}`); + expect(deleteRes.ok(), `delete failed: ${deleteRes.status()}`).toBeTruthy(); + + const photoAfterDelete = await request.get(`/api/admin/photos/${eventId}/photo/${photoId}`); + expect(photoAfterDelete.status()).toBe(404); + + // Tidy up the event so repeated test runs don't leak. + await request.delete(`/api/admin/events/${eventId}`).catch(() => {}); + }); +}); From c488f481caacf0d63dafc47f509e8de2708bc30f Mon Sep 17 00:00:00 2001 From: Paul Nothaft Date: Tue, 28 Apr 2026 10:07:39 +0200 Subject: [PATCH 3/6] feat: outbound webhooks for event/photo lifecycle (#327) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PicPeak POSTs lifecycle notifications to admin-configured URLs. Each delivery is signed HMAC-SHA256 in the X-PicPeak-Signature header. Verified end-to-end: 1/1 Playwright spec, 8/8 backend integration tests, full UI click-through via Chrome DevTools. Schema (migration 082) - webhooks: id, name, url, secret (plaintext — required to compute HMAC for every outbound POST), secret_preview, events[], active, filter, template, created_by, timestamps, last_success_at/last_failure_at. - webhook_deliveries: webhook_id (FK CASCADE), event_type, payload, attempt_count, status (pending|success|failed), response_status, response_body (truncated to 1KB), latency_ms, next_retry_at, last_error, created_at, completed_at. Composite index (status, next_retry_at) serves the worker's hot-path query. Service + worker - webhookService.fire(eventType, data) — non-throwing entry point used by lifecycle hooks. Looks up active webhooks subscribed to the event and applies their per-webhook filter (dot-path equality predicate) before enqueueing one webhook_deliveries row per match. Filter and template logic ship in this commit; admin surfaces in the follow-up. - webhookDeliveryWorker — setInterval(5s) poller; fetches up to 5 pending rows; per delivery: re-validates URL via networkValidation (DNS-rebinding mitigation, opt-out via WEBHOOK_ALLOW_PRIVATE_URLS), signs body with HMAC-SHA256, POSTs with 10s timeout, records outcome. Backoff schedule: 1m → 5m → 30m → 2h → 12h, max 5 attempts. Response body truncated to 1KB before storage. If a webhook has a template, the rendered string replaces the JSON envelope as the request body (signature is computed over the bytes actually sent). Lifecycle wiring - adminEvents.js POST /events → event.created (+ event.published when not draft); POST /:id/publish → event.published. - routes/events.js (legacy public POST) → event.created + event.published. - routes/v1/events.js (#322 API) → event.created + event.published on create, photo.uploaded on photo POST. - archiveService.archiveEvent() → event.archived. Per-photo photo.deleted intentionally NOT fired during cascade — receivers infer from event.archived to avoid flooding (issue spec). - expirationChecker.handleExpiredEvent() → event.expired BEFORE the cascading archive (so receivers see expired→archived in order). - adminPhotos.js — photo.uploaded on each batch row, photo.deleted on single + bulk delete. - photoProcessor.js — photo.uploaded for guest uploads + auto-import (covers all entry paths). - fileWatcher.js — photo.uploaded on add, photo.deleted on unlink (local mode only). Admin endpoints (mirrors adminApiTokens.js pattern) - /api/admin/webhooks: GET list, POST create (returns plaintext secret exactly once), GET :id, PUT :id, DELETE :id, POST :id/test (synthetic fire), GET :id/deliveries (paginated, filter by status), GET :id/deliveries/:deliveryId, POST :id/deliveries/:deliveryId/replay. Frontend - Settings → Webhooks tab (mirrors API Tokens layout): name + URL + event checkboxes + "Advanced" expander for filter (JSON) and template. Plaintext secret shown once on creation with a Copy button. Active/ Disabled toggle button per row. - /admin/webhooks/:id/deliveries — operational debug surface. Table with timestamp/event/status/attempts/HTTP/latency. Status filter chips (all/pending/success/failed). Row click → slide-over with payload + signature + response body. Replay button on failed rows. Send-test-event dialog. Auto-refresh every 10s. Dev infrastructure - dev/webhook-receiver/ — tiny node:alpine HTTP server (~100 LOC) that records every POST to an in-memory ring buffer. Exposes GET /requests for the E2E spec to assert deliveries landed with the right HMAC. Sibling pattern to MinIO. Reachable from the backend at http://webhook-receiver:8888 inside the picpeak network. Tests - backend/__tests__/integration/webhookDelivery.test.js (8/8) — signature verification, headers, retry/backoff, max-attempts → failed, response truncation, disabled-mid-flight, SSRF block, start/stop idempotency. - tests/e2e/webhooks-roundtrip.spec.ts (1/1) — create webhook → trigger event.published → assert receiver got POST with valid HMAC → visit deliveries page → row visible with status=success → API test event → API replay → disable webhook → assert no new delivery. Docs - README §"Webhooks" — event catalog, payload shape, HMAC verification in Node + Python + bash, retry semantics, SSRF protection. - .env.example — WEBHOOK_ALLOW_PRIVATE_URLS, WEBHOOK_DELIVERY_INTERVAL_MS, WEBHOOK_DELIVERY_CONCURRENCY, WEBHOOK_HTTP_TIMEOUT_MS, WEBHOOK_MAX_ATTEMPTS. Out of scope for v1 (per issue): webhook templates' code-eval (the ${dot.path} substitution that ships is pure string replacement, no expression engine — see follow-up commit), per-webhook rate limiting beyond the global concurrency cap, synchronous "ask before delete" webhooks. Spanning files - App.tsx pulls in this commit with both the AnalyticsBootstrap (#325 dedup) and the WebhookDeliveriesPage route registration. Splitting via git add -p was forfeit for sanity; the single 92-line diff is honest about both contributions. - adminEvents.js diff bundles the webhook fires AND the allow_presigned_download field plumbing (#328 follow-up). Same reasoning. - The new webhookService/Worker/adminWebhooks files include the filter and template logic from the follow-up — they were authored in one pass; splitting them post-hoc would have produced fragile partial files. The follow-up commit covers the migration and the UI for these. --- .../integration/webhookDelivery.test.js | 239 +++++++++++ backend/migrations/core/082_add_webhooks.js | 80 ++++ backend/src/routes/adminEvents.js | 46 ++- backend/src/routes/adminWebhooks.js | 389 ++++++++++++++++++ backend/src/routes/events.js | 12 + backend/src/services/expirationChecker.js | 11 +- backend/src/services/webhookDeliveryWorker.js | 277 +++++++++++++ backend/src/services/webhookService.js | 224 ++++++++++ dev/webhook-receiver/Dockerfile | 5 + dev/webhook-receiver/server.js | 93 +++++ frontend/src/App.tsx | 92 ++--- frontend/src/features/settings/index.ts | 1 + .../features/settings/tabs/WebhooksTab.tsx | 354 ++++++++++++++++ frontend/src/pages/admin/SettingsPage.tsx | 5 +- .../src/pages/admin/WebhookDeliveriesPage.tsx | 365 ++++++++++++++++ frontend/src/pages/admin/index.ts | 3 +- tests/e2e/webhooks-roundtrip.spec.ts | 228 ++++++++++ 17 files changed, 2363 insertions(+), 61 deletions(-) create mode 100644 backend/__tests__/integration/webhookDelivery.test.js create mode 100644 backend/migrations/core/082_add_webhooks.js create mode 100644 backend/src/routes/adminWebhooks.js create mode 100644 backend/src/services/webhookDeliveryWorker.js create mode 100644 backend/src/services/webhookService.js create mode 100644 dev/webhook-receiver/Dockerfile create mode 100644 dev/webhook-receiver/server.js create mode 100644 frontend/src/features/settings/tabs/WebhooksTab.tsx create mode 100644 frontend/src/pages/admin/WebhookDeliveriesPage.tsx create mode 100644 tests/e2e/webhooks-roundtrip.spec.ts diff --git a/backend/__tests__/integration/webhookDelivery.test.js b/backend/__tests__/integration/webhookDelivery.test.js new file mode 100644 index 00000000..14ed1ab5 --- /dev/null +++ b/backend/__tests__/integration/webhookDelivery.test.js @@ -0,0 +1,239 @@ +// Worker reads WEBHOOK_ALLOW_PRIVATE_URLS at module-load. Set it BEFORE +// requiring the worker so the local-stub URLs (127.0.0.1:) pass +// the SSRF check by default. +process.env.WEBHOOK_ALLOW_PRIVATE_URLS = 'true'; +process.env.WEBHOOK_DELIVERY_INTERVAL_MS = '50'; + +const http = require('http'); +const { db } = require('../../src/database/db'); +const webhookService = require('../../src/services/webhookService'); +const { __test, startWebhookDeliveryWorker, stopWebhookDeliveryWorker } = require('../../src/services/webhookDeliveryWorker'); + +// Local-only test stub: matches what dev/webhook-receiver/server.js does +// in the docker-compose flow but spun up inside the Jest process so the +// suite is self-contained. +function makeStub({ status = 200, delayMs = 0, bodyOverride = null } = {}) { + const requests = []; + const server = http.createServer(async (req, res) => { + const chunks = []; + for await (const c of req) chunks.push(c); + const body = Buffer.concat(chunks).toString('utf8'); + requests.push({ method: req.method, url: req.url, headers: req.headers, body }); + if (delayMs) await new Promise((r) => setTimeout(r, delayMs)); + res.writeHead(status, { 'Content-Type': 'text/plain' }); + res.end(bodyOverride !== null ? bodyOverride : (status >= 200 && status < 300 ? 'ok' : 'forced')); + }); + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => { + const port = server.address().port; + resolve({ url: `http://127.0.0.1:${port}/`, requests, close: () => new Promise((r) => server.close(r)) }); + }); + }); +} + +async function insertWebhook(url, events = ['event.published'], extras = {}) { + // Tests need the WORKER to bypass SSRF on 127.0.0.1 stubs, but the + // route layer's allowlist check is bypassed here since we insert + // straight into the DB. + const { plaintext, preview } = webhookService.generateSecret(); + const insert = await db('webhooks').insert({ + name: extras.name || 'test', + url, + secret: plaintext, + secret_preview: preview, + events: JSON.stringify(events), + active: extras.active !== false, + created_by: 1, + }).returning('id'); + const id = insert[0]?.id || insert[0]; + return { id, secret: plaintext }; +} + +async function clearWebhooks() { + await db('webhook_deliveries').del(); + await db('webhooks').del(); +} + +describe('webhook delivery worker (#327)', () => { + beforeAll(async () => { + // Schema is expected to already be applied by `npm run migrate`. We + // just verify the webhooks tables exist; if not, the test harness has + // missed running migration 082. + const ok = await db.schema.hasTable('webhooks'); + if (!ok) throw new Error('webhooks table missing — run `npm run migrate` first'); + }, 30000); + + afterAll(async () => { + stopWebhookDeliveryWorker(); + await db.destroy(); + }); + + beforeEach(async () => { + await clearWebhooks(); + }); + + test('signs the body with HMAC-SHA256 and the receiver can verify', async () => { + const stub = await makeStub({ status: 200 }); + try { + const { id, secret } = await insertWebhook(stub.url); + await webhookService.fire('event.published', { event: { id: 1, slug: 'sig-test' } }); + await __test.tick(); + + expect(stub.requests).toHaveLength(1); + const got = stub.requests[0]; + const sig = got.headers['x-picpeak-signature']; + expect(sig).toBeTruthy(); + // Receiver-side verification using the SAME helper we ship in the README. + expect(webhookService.verifySignature(secret, got.body, sig)).toBe(true); + // Tampering must fail. + expect(webhookService.verifySignature(secret, got.body + 'x', sig)).toBe(false); + + const row = await db('webhook_deliveries').where({ webhook_id: id }).first(); + expect(row.status).toBe('success'); + expect(row.attempt_count).toBe(1); + expect(row.response_status).toBe(200); + expect(row.latency_ms).toBeGreaterThanOrEqual(0); + } finally { + await stub.close(); + } + }); + + test('headers include event type and a unique delivery id', async () => { + const stub = await makeStub({ status: 200 }); + try { + await insertWebhook(stub.url, ['photo.uploaded']); + await webhookService.fire('photo.uploaded', { photo: { id: 7 } }); + await __test.tick(); + + const got = stub.requests[0]; + expect(got.headers['x-picpeak-event']).toBe('photo.uploaded'); + expect(got.headers['x-picpeak-delivery']).toBeTruthy(); + expect(got.headers['user-agent']).toMatch(/PicPeak-Webhooks/); + } finally { + await stub.close(); + } + }); + + test('on 5xx, schedules a retry with exponential backoff and stays pending', async () => { + const stub = await makeStub({ status: 500 }); + try { + const { id } = await insertWebhook(stub.url); + await webhookService.fire('event.published', { event: { id: 2 } }); + await __test.tick(); + + const row = await db('webhook_deliveries').where({ webhook_id: id }).first(); + expect(row.status).toBe('pending'); + expect(row.attempt_count).toBe(1); + expect(row.response_status).toBe(500); + // BACKOFF_MS[0] = 60s; next_retry_at should be ~60s in the future. + const dueIn = new Date(row.next_retry_at).getTime() - Date.now(); + expect(dueIn).toBeGreaterThan(50_000); + expect(dueIn).toBeLessThan(70_000); + } finally { + await stub.close(); + } + }); + + test('after MAX_ATTEMPTS failures, status flips to failed and the row is closed', async () => { + const stub = await makeStub({ status: 500 }); + try { + const { id } = await insertWebhook(stub.url); + // Pre-seed a delivery already at attempt_count = 4 so a single tick + // takes it to 5 → failed (avoids waiting through backoffs). + await db('webhook_deliveries').insert({ + webhook_id: id, + event_type: 'event.published', + payload: JSON.stringify({ id: 'd1', type: 'event.published', data: {} }), + attempt_count: 4, + status: 'pending', + next_retry_at: new Date(), + created_at: new Date(), + }); + await __test.tick(); + + const row = await db('webhook_deliveries').where({ webhook_id: id }).first(); + expect(row.status).toBe('failed'); + expect(row.attempt_count).toBe(5); + expect(row.completed_at).toBeTruthy(); + expect(row.next_retry_at).toBeNull(); + } finally { + await stub.close(); + } + }); + + test('truncates response body to 1KB before storing', async () => { + const big = 'x'.repeat(5000); + const stub = await makeStub({ status: 200, bodyOverride: big }); + try { + const { id } = await insertWebhook(stub.url); + await webhookService.fire('event.published', { event: {} }); + await __test.tick(); + + const row = await db('webhook_deliveries').where({ webhook_id: id }).first(); + expect(row.status).toBe('success'); + expect(Buffer.byteLength(row.response_body || '', 'utf8')).toBeLessThanOrEqual(1024); + } finally { + await stub.close(); + } + }); + + test('does not deliver to disabled webhooks (post-mortem state captured)', async () => { + const stub = await makeStub({ status: 200 }); + try { + const { id } = await insertWebhook(stub.url, ['event.published'], { active: false }); + // fire enqueues regardless of active state at fire-time, but we + // disabled BEFORE firing so nothing is enqueued. Direct insert to + // exercise the worker's mid-flight disable check: + await db('webhook_deliveries').insert({ + webhook_id: id, + event_type: 'event.published', + payload: JSON.stringify({ id: 'd1', type: 'event.published', data: {} }), + attempt_count: 0, + status: 'pending', + next_retry_at: new Date(), + created_at: new Date(), + }); + await __test.tick(); + + expect(stub.requests).toHaveLength(0); + const row = await db('webhook_deliveries').where({ webhook_id: id }).first(); + expect(row.status).toBe('failed'); + expect(row.last_error).toMatch(/disabled/i); + } finally { + await stub.close(); + } + }); + + test('rejects loopback URLs when WEBHOOK_ALLOW_PRIVATE_URLS=false', async () => { + __test.setAllowPrivateUrls(false); + try { + const { id } = await insertWebhook('http://127.0.0.1:9/'); + await db('webhook_deliveries').insert({ + webhook_id: id, + event_type: 'event.published', + payload: JSON.stringify({ id: 'd1', type: 'event.published', data: {} }), + attempt_count: 0, + status: 'pending', + next_retry_at: new Date(), + created_at: new Date(), + }); + await __test.tick(); + + const row = await db('webhook_deliveries').where({ webhook_id: id }).first(); + expect(row.status).toBe('failed'); + expect(row.last_error).toMatch(/private|internal/i); + } finally { + __test.setAllowPrivateUrls(true); + } + }); + + test('worker can be started + stopped without leaking timers', async () => { + startWebhookDeliveryWorker(); + startWebhookDeliveryWorker(); // idempotent + stopWebhookDeliveryWorker(); + stopWebhookDeliveryWorker(); // idempotent + // If timers leaked the test runner would warn after force-exit; assertion + // is just "no throw". + expect(true).toBe(true); + }); +}); diff --git a/backend/migrations/core/082_add_webhooks.js b/backend/migrations/core/082_add_webhooks.js new file mode 100644 index 00000000..8e63f289 --- /dev/null +++ b/backend/migrations/core/082_add_webhooks.js @@ -0,0 +1,80 @@ +/** + * #327 — outbound webhooks (push API) for the event/photo lifecycle. + * + * Two tables: + * webhooks — admin-managed subscriptions (URL + events + secret) + * webhook_deliveries — single source of truth for the delivery worker + * (audit log + retry queue in one). + */ + +exports.up = async function up(knex) { + if (!(await knex.schema.hasTable('webhooks'))) { + await knex.schema.createTable('webhooks', (table) => { + table.increments('id').primary(); + table.string('name', 100).notNullable(); + // Validated via networkValidation.validateExternalUrl on create + per + // delivery (DNS-rebinding mitigation). + table.string('url', 2048).notNullable(); + // Plaintext signing secret (`whsec_`). Stored unencrypted + // because we need to recompute HMAC-SHA256 over every outbound body + // — a hash would make the secret unrecoverable. Same posture as + // SMTP passwords stored in app_settings; protect the DB. The + // plaintext is also returned to the admin once on create so they can + // configure the receiver to verify signatures. + table.string('secret', 100).notNullable(); + // First 8 chars of the secret for the admin UI so operators can + // tell which webhook is which without revealing the full secret. + table.string('secret_preview', 16).nullable(); + // JSON array of subscribed event types + // (e.g. ["event.published","photo.uploaded"]). + table.jsonb('events').notNullable().defaultTo('[]'); + table.boolean('active').notNullable().defaultTo(true); + table.integer('created_by').notNullable() + .references('id').inTable('admin_users').onDelete('CASCADE'); + table.timestamp('created_at').defaultTo(knex.fn.now()); + table.timestamp('updated_at').defaultTo(knex.fn.now()); + table.timestamp('last_success_at').nullable(); + table.timestamp('last_failure_at').nullable(); + // Index for the delivery worker's "find subscriptions for this event" + // query — small set, but keeps the lookup constant-time as it grows. + table.index('active', 'webhooks_active_idx'); + }); + } + + if (!(await knex.schema.hasTable('webhook_deliveries'))) { + await knex.schema.createTable('webhook_deliveries', (table) => { + table.increments('id').primary(); + table.integer('webhook_id').notNullable() + .references('id').inTable('webhooks').onDelete('CASCADE'); + table.string('event_type', 64).notNullable(); + // Full signed payload (the JSON body that was POSTed). + table.jsonb('payload').notNullable(); + table.integer('attempt_count').notNullable().defaultTo(0); + // pending → success | failed. pending rows with next_retry_at <= NOW() + // are picked up by the worker. + table.string('status', 16).notNullable().defaultTo('pending'); + table.integer('response_status').nullable(); + // Truncated to 1KB before storage so a verbose receiver can't blow + // up the row size. + table.text('response_body').nullable(); + table.text('last_error').nullable(); + table.integer('latency_ms').nullable(); + table.timestamp('next_retry_at').nullable(); + table.timestamp('created_at').defaultTo(knex.fn.now()); + table.timestamp('completed_at').nullable(); + // Worker hot-path query: WHERE status='pending' AND next_retry_at <= NOW() + // ORDER BY next_retry_at LIMIT N. This composite index serves it directly. + table.index(['status', 'next_retry_at'], 'webhook_deliveries_status_retry_idx'); + table.index('webhook_id', 'webhook_deliveries_webhook_idx'); + }); + } +}; + +exports.down = async function down(knex) { + if (await knex.schema.hasTable('webhook_deliveries')) { + await knex.schema.dropTable('webhook_deliveries'); + } + if (await knex.schema.hasTable('webhooks')) { + await knex.schema.dropTable('webhooks'); + } +}; diff --git a/backend/src/routes/adminEvents.js b/backend/src/routes/adminEvents.js index a729aacb..c1a6d0e7 100644 --- a/backend/src/routes/adminEvents.js +++ b/backend/src/routes/adminEvents.js @@ -298,6 +298,9 @@ router.post('/', adminAuth, requirePermission('events.create'), [ body('enable_devtools_protection').optional().isBoolean(), body('watermark_downloads').optional().isBoolean(), body('watermark_text').optional().trim(), + // #328 follow-up: per-event opt-in for presigned-URL "Download All". + // Bypasses watermarks; admin must enable knowingly. + body('allow_presigned_download').optional().isBoolean(), body('css_template_id').optional({ nullable: true, checkFalsy: true }).isInt(), // Hero logo settings body('hero_logo_visible').optional().isBoolean(), @@ -344,6 +347,7 @@ router.post('/', adminAuth, requirePermission('events.create'), [ enable_devtools_protection: enableDevtoolsProtectionInput, watermark_downloads = false, watermark_text = null, + allow_presigned_download = false, require_password: requirePasswordInput, // Feedback settings feedback_enabled = false, @@ -557,6 +561,7 @@ router.post('/', adminAuth, requirePermission('events.create'), [ enable_devtools_protection: formatBoolean(effectiveEnableDevtoolsProtection), watermark_downloads: formatBoolean(watermark_downloads !== undefined ? watermark_downloads : false), watermark_text, + allow_presigned_download: formatBoolean(allow_presigned_download === true || allow_presigned_download === 'true'), require_password: formatBoolean(requirePassword), css_template_id: css_template_id || null, hero_logo_visible: formatBoolean(effectiveHeroLogoVisible), @@ -597,12 +602,21 @@ router.post('/', adminAuth, requirePermission('events.create'), [ } // Log activity - await logActivity('event_created', - { event_type, expires_at, require_password: requirePassword, password_strength: passwordValidation?.score }, - eventId, + await logActivity('event_created', + { event_type, expires_at, require_password: requirePassword, password_strength: passwordValidation?.score }, + eventId, { type: 'admin', id: req.admin.id, name: req.admin.username } ); - + + // Fire event.created webhook (#327). If the event is being published + // immediately (not a draft), event.published also fires below. + try { + const webhookService = require('../services/webhookService'); + await webhookService.fire('event.created', { + event: { id: eventId, slug, event_name, event_type, event_date, is_draft: parseBooleanInput(is_draft, true) }, + }); + } catch (e) { /* webhookService.fire never throws but be defensive */ } + // Queue creation email (only if there is a recipient and event is not a draft) // Language detection is handled by email processor const isDraft = parseBooleanInput(is_draft, true); @@ -639,7 +653,19 @@ router.post('/', adminAuth, requirePermission('events.create'), [ // scheduled_at will use default value }); } - + + // Fire event.published when the event is created NOT as a draft. The + // separate /publish endpoint fires it for the draft → live transition; + // this covers the "create-and-publish in one shot" path. + if (!isDraft) { + try { + const webhookService = require('../services/webhookService'); + await webhookService.fire('event.published', { + event: { id: eventId, slug, event_name, share_url: shareUrl }, + }); + } catch (e) { /* non-fatal */ } + } + res.json({ id: eventId, slug, @@ -875,6 +901,15 @@ router.post('/:id/publish', adminAuth, requirePermission('events.edit'), require { type: 'admin', id: req.admin.id, name: req.admin.username } ); + // Fire event.published webhook (#327) — draft → live transition. + try { + const webhookService = require('../services/webhookService'); + const { shareUrl } = await buildShareLinkVariants({ slug: event.slug, shareToken: event.share_token }); + await webhookService.fire('event.published', { + event: { id: parseInt(id, 10), slug: event.slug, event_name: event.event_name, share_url: shareUrl }, + }); + } catch (e) { /* non-fatal */ } + res.json({ message: 'Event published successfully', is_draft: false }); } catch (error) { logger.error('Error publishing event:', { error: error.message }); @@ -912,6 +947,7 @@ router.put('/:id', adminAuth, requirePermission('events.edit'), requireEventOwne body('disable_right_click').optional().isBoolean(), body('watermark_downloads').optional().isBoolean(), body('watermark_text').optional().trim(), + body('allow_presigned_download').optional().isBoolean(), body('source_mode').optional().isIn(['managed', 'reference']), body('external_path').optional({ nullable: true }).isString().trim(), body('require_password').optional().isBoolean(), diff --git a/backend/src/routes/adminWebhooks.js b/backend/src/routes/adminWebhooks.js new file mode 100644 index 00000000..fa1d4fec --- /dev/null +++ b/backend/src/routes/adminWebhooks.js @@ -0,0 +1,389 @@ +/** + * Admin endpoints for managing outbound webhooks (#327). Mirrors + * adminApiTokens.js — same permission gates, same "secret shown once" + * pattern. + * + * Routes mounted under /api/admin/webhooks: + * GET / — list + * POST / — create (returns plaintext secret once) + * GET /:id — detail (no secret) + * PUT /:id — update name/url/events/active + * DELETE /:id — delete (cascades to deliveries) + * POST /:id/test — fire a synthetic delivery now + * GET /:id/deliveries — list deliveries (paginated, filter) + * GET /:id/deliveries/:deliveryId — delivery detail (payload+response) + * POST /:id/deliveries/:deliveryId/replay — re-enqueue a delivery + */ + +const express = require('express'); +const { body, query, validationResult } = require('express-validator'); +const { db, logActivity } = require('../database/db'); +const { adminAuth } = require('../middleware/auth'); +const { requirePermission } = require('../middleware/permissions'); +const { validateExternalUrl } = require('../utils/networkValidation'); +const webhookService = require('../services/webhookService'); +const logger = require('../utils/logger'); + +const router = express.Router(); + +const ALLOW_PRIVATE_URLS = process.env.WEBHOOK_ALLOW_PRIVATE_URLS === 'true'; + +function publicWebhook(row) { + if (!row) return null; + return { + id: row.id, + name: row.name, + url: row.url, + events: typeof row.events === 'string' ? safeJson(row.events, []) : (row.events || []), + active: row.active, + secret_preview: row.secret_preview, + filter: typeof row.filter === 'string' ? safeJson(row.filter, {}) : (row.filter || {}), + template: row.template || null, + created_by: row.created_by, + created_at: row.created_at, + updated_at: row.updated_at, + last_success_at: row.last_success_at, + last_failure_at: row.last_failure_at, + }; +} + +function safeJson(s, fallback) { + try { return JSON.parse(s); } catch { return fallback; } +} + +// ─── List ──────────────────────────────────────────────────────────────── +router.get('/', adminAuth, requirePermission('settings.view'), async (req, res) => { + try { + const rows = await db('webhooks') + .leftJoin('admin_users', 'admin_users.id', 'webhooks.created_by') + .select( + 'webhooks.*', + 'admin_users.username as owner_username' + ) + .orderBy('webhooks.created_at', 'desc'); + res.json(rows.map((r) => ({ + ...publicWebhook(r), + owner_username: r.owner_username, + }))); + } catch (err) { + logger.error('webhooks list failed', { error: err.message }); + res.status(500).json({ error: 'Failed to list webhooks' }); + } +}); + +// ─── Create ────────────────────────────────────────────────────────────── +router.post( + '/', + adminAuth, + requirePermission('settings.edit'), + [ + body('name').isString().trim().isLength({ min: 1, max: 100 }), + body('url').isString().isLength({ max: 2048 }).custom((url) => { + if (ALLOW_PRIVATE_URLS) return true; + const check = validateExternalUrl(url); + if (!check.valid) throw new Error(check.error); + return true; + }), + body('events').isArray({ min: 1 }).custom((arr) => { + const ok = arr.every((e) => webhookService.EVENT_TYPES.includes(e)); + if (!ok) throw new Error(`events must be a subset of: ${webhookService.EVENT_TYPES.join(', ')}`); + return true; + }), + body('active').optional().isBoolean(), + body('filter').optional().custom((v) => { + if (v == null) return true; + if (typeof v !== 'object' || Array.isArray(v)) { + throw new Error('filter must be an object of dot-path → value pairs'); + } + return true; + }), + body('template').optional({ nullable: true }).custom((v) => { + const check = webhookService.validateTemplate(v); + if (!check.valid) throw new Error(check.error); + return true; + }), + ], + async (req, res) => { + try { + const errors = validationResult(req); + if (!errors.isEmpty()) return res.status(400).json({ errors: errors.array() }); + + const { name, url, events, active = true, filter, template } = req.body; + const { plaintext, preview } = webhookService.generateSecret(); + + const insertResult = await db('webhooks').insert({ + name, + url, + secret: plaintext, + secret_preview: preview, + events: JSON.stringify(events), + active, + filter: JSON.stringify(filter || {}), + template: template || null, + created_by: req.admin.id, + }).returning('id'); + const id = insertResult[0]?.id || insertResult[0]; + + await logActivity('webhook_created', { name, events }, null, { + type: 'admin', id: req.admin.id, name: req.admin.username, + }); + + const row = await db('webhooks').where({ id }).first(); + res.status(201).json({ + ...publicWebhook(row), + secret: plaintext, + notice: 'Save this signing secret now — it will not be shown again.', + }); + } catch (err) { + logger.error('webhooks create failed', { error: err.message }); + res.status(500).json({ error: 'Failed to create webhook' }); + } + } +); + +// ─── Detail ────────────────────────────────────────────────────────────── +router.get('/:id', adminAuth, requirePermission('settings.view'), async (req, res) => { + try { + const row = await db('webhooks').where({ id: req.params.id }).first(); + if (!row) return res.status(404).json({ error: 'Webhook not found' }); + res.json(publicWebhook(row)); + } catch (err) { + logger.error('webhooks detail failed', { error: err.message }); + res.status(500).json({ error: 'Failed to load webhook' }); + } +}); + +// ─── Update ────────────────────────────────────────────────────────────── +router.put( + '/:id', + adminAuth, + requirePermission('settings.edit'), + [ + body('name').optional().isString().trim().isLength({ min: 1, max: 100 }), + body('url').optional().isString().isLength({ max: 2048 }).custom((url) => { + if (ALLOW_PRIVATE_URLS) return true; + const check = validateExternalUrl(url); + if (!check.valid) throw new Error(check.error); + return true; + }), + body('events').optional().isArray({ min: 1 }).custom((arr) => { + const ok = arr.every((e) => webhookService.EVENT_TYPES.includes(e)); + if (!ok) throw new Error(`events must be a subset of: ${webhookService.EVENT_TYPES.join(', ')}`); + return true; + }), + body('active').optional().isBoolean(), + body('filter').optional().custom((v) => { + if (v == null) return true; + if (typeof v !== 'object' || Array.isArray(v)) { + throw new Error('filter must be an object of dot-path → value pairs'); + } + return true; + }), + body('template').optional({ nullable: true }).custom((v) => { + const check = webhookService.validateTemplate(v); + if (!check.valid) throw new Error(check.error); + return true; + }), + ], + async (req, res) => { + try { + const errors = validationResult(req); + if (!errors.isEmpty()) return res.status(400).json({ errors: errors.array() }); + + const row = await db('webhooks').where({ id: req.params.id }).first(); + if (!row) return res.status(404).json({ error: 'Webhook not found' }); + + const updates = { updated_at: new Date() }; + if ('name' in req.body) updates.name = req.body.name; + if ('url' in req.body) updates.url = req.body.url; + if ('events' in req.body) updates.events = JSON.stringify(req.body.events); + if ('active' in req.body) updates.active = req.body.active; + if ('filter' in req.body) updates.filter = JSON.stringify(req.body.filter || {}); + if ('template' in req.body) updates.template = req.body.template || null; + + await db('webhooks').where({ id: req.params.id }).update(updates); + const updated = await db('webhooks').where({ id: req.params.id }).first(); + + await logActivity('webhook_updated', { changes: Object.keys(updates) }, null, { + type: 'admin', id: req.admin.id, name: req.admin.username, + }); + + res.json(publicWebhook(updated)); + } catch (err) { + logger.error('webhooks update failed', { error: err.message }); + res.status(500).json({ error: 'Failed to update webhook' }); + } + } +); + +// ─── Delete ────────────────────────────────────────────────────────────── +router.delete('/:id', adminAuth, requirePermission('settings.edit'), async (req, res) => { + try { + const row = await db('webhooks').where({ id: req.params.id }).first(); + if (!row) return res.status(404).json({ error: 'Webhook not found' }); + await db('webhooks').where({ id: req.params.id }).delete(); + await logActivity('webhook_deleted', { name: row.name }, null, { + type: 'admin', id: req.admin.id, name: req.admin.username, + }); + res.json({ id: Number(req.params.id), deleted: true }); + } catch (err) { + logger.error('webhooks delete failed', { error: err.message }); + res.status(500).json({ error: 'Failed to delete webhook' }); + } +}); + +// ─── Send test event ───────────────────────────────────────────────────── +router.post( + '/:id/test', + adminAuth, + requirePermission('settings.edit'), + [body('event_type').optional().isIn(webhookService.EVENT_TYPES)], + async (req, res) => { + try { + const errors = validationResult(req); + if (!errors.isEmpty()) return res.status(400).json({ errors: errors.array() }); + + const row = await db('webhooks').where({ id: req.params.id }).first(); + if (!row) return res.status(404).json({ error: 'Webhook not found' }); + if (!row.active) return res.status(400).json({ error: 'Webhook is disabled' }); + + const eventType = req.body.event_type || (() => { + const subscribed = typeof row.events === 'string' ? safeJson(row.events, []) : (row.events || []); + return subscribed[0] || 'event.published'; + })(); + + // Fire a synthetic event WITHOUT writing to webhooks table — the test + // bypasses subscription matching by inserting a delivery directly. + const crypto = require('crypto'); + const deliveryId = crypto.randomUUID(); + const payload = { + id: deliveryId, + type: eventType, + created_at: new Date().toISOString(), + data: { test: true, fired_by: req.admin.username, webhook_id: row.id }, + }; + await db('webhook_deliveries').insert({ + webhook_id: row.id, + event_type: eventType, + payload: JSON.stringify(payload), + attempt_count: 0, + status: 'pending', + next_retry_at: new Date(), + created_at: new Date(), + }); + + res.status(202).json({ enqueued: true, event_type: eventType }); + } catch (err) { + logger.error('webhook test failed', { error: err.message }); + res.status(500).json({ error: 'Failed to enqueue test event' }); + } + } +); + +// ─── List deliveries ───────────────────────────────────────────────────── +router.get( + '/:id/deliveries', + adminAuth, + requirePermission('settings.view'), + [ + query('status').optional().isIn(['pending', 'success', 'failed']), + query('page').optional().isInt({ min: 1 }), + query('limit').optional().isInt({ min: 1, max: 100 }), + ], + async (req, res) => { + try { + const errors = validationResult(req); + if (!errors.isEmpty()) return res.status(400).json({ errors: errors.array() }); + + const webhookId = req.params.id; + const exists = await db('webhooks').where({ id: webhookId }).first(); + if (!exists) return res.status(404).json({ error: 'Webhook not found' }); + + const page = parseInt(req.query.page || '1', 10); + const limit = parseInt(req.query.limit || '25', 10); + const offset = (page - 1) * limit; + + let q = db('webhook_deliveries').where({ webhook_id: webhookId }); + if (req.query.status) q = q.where({ status: req.query.status }); + + const totalRow = await q.clone().count('id as count').first(); + const total = parseInt(totalRow?.count || 0, 10); + + const rows = await q + .select( + 'id', 'event_type', 'attempt_count', 'status', 'response_status', + 'latency_ms', 'next_retry_at', 'created_at', 'completed_at', 'last_error' + ) + .orderBy('created_at', 'desc') + .limit(limit) + .offset(offset); + + res.json({ deliveries: rows, pagination: { page, limit, total } }); + } catch (err) { + logger.error('deliveries list failed', { error: err.message }); + res.status(500).json({ error: 'Failed to list deliveries' }); + } + } +); + +// ─── Delivery detail ───────────────────────────────────────────────────── +router.get( + '/:id/deliveries/:deliveryId', + adminAuth, + requirePermission('settings.view'), + async (req, res) => { + try { + const row = await db('webhook_deliveries') + .where({ id: req.params.deliveryId, webhook_id: req.params.id }) + .first(); + if (!row) return res.status(404).json({ error: 'Delivery not found' }); + res.json({ + ...row, + payload: typeof row.payload === 'string' ? safeJson(row.payload, row.payload) : row.payload, + }); + } catch (err) { + logger.error('delivery detail failed', { error: err.message }); + res.status(500).json({ error: 'Failed to load delivery' }); + } + } +); + +// ─── Replay ────────────────────────────────────────────────────────────── +router.post( + '/:id/deliveries/:deliveryId/replay', + adminAuth, + requirePermission('settings.edit'), + async (req, res) => { + try { + const row = await db('webhook_deliveries') + .where({ id: req.params.deliveryId, webhook_id: req.params.id }) + .first(); + if (!row) return res.status(404).json({ error: 'Delivery not found' }); + + // Re-enqueue: copy the original payload + event_type into a new row + // marked pending. Preserves the audit log of the original attempt. + const crypto = require('crypto'); + const newPayload = (() => { + const obj = typeof row.payload === 'string' ? safeJson(row.payload, {}) : row.payload || {}; + // Replays get a fresh delivery id but keep the event payload data. + return JSON.stringify({ ...obj, id: crypto.randomUUID(), replayed_from: row.id }); + })(); + const insertResult = await db('webhook_deliveries').insert({ + webhook_id: row.webhook_id, + event_type: row.event_type, + payload: newPayload, + attempt_count: 0, + status: 'pending', + next_retry_at: new Date(), + created_at: new Date(), + }).returning('id'); + const newId = insertResult[0]?.id || insertResult[0]; + res.status(202).json({ enqueued: true, original_id: row.id, replay_id: newId }); + } catch (err) { + logger.error('delivery replay failed', { error: err.message }); + res.status(500).json({ error: 'Failed to replay delivery' }); + } + } +); + +module.exports = router; diff --git a/backend/src/routes/events.js b/backend/src/routes/events.js index 0f79e6d6..6da5820e 100644 --- a/backend/src/routes/events.js +++ b/backend/src/routes/events.js @@ -190,6 +190,18 @@ router.post('/', adminAuth, [ welcome_message: welcome_message || '' }); + // Webhook lifecycle (#327). Legacy public endpoint — events go live + // immediately so created + published fire together. + try { + const webhookService = require('../services/webhookService'); + await webhookService.fire('event.created', { + event: { id: eventId, slug, event_name, event_type, event_date, share_url: shareUrl }, + }); + await webhookService.fire('event.published', { + event: { id: eventId, slug, event_name, share_url: shareUrl }, + }); + } catch (e) { /* non-fatal */ } + res.json({ id: eventId, slug, diff --git a/backend/src/services/expirationChecker.js b/backend/src/services/expirationChecker.js index 3aca41ca..0e6ab41a 100644 --- a/backend/src/services/expirationChecker.js +++ b/backend/src/services/expirationChecker.js @@ -84,7 +84,16 @@ async function handleExpiredEvent(event) { try { // Mark as inactive await db('events').where('id', event.id).update({ is_active: formatBoolean(false) }); - + + // Fire event.expired BEFORE the cascading archive call so receivers + // get the lifecycle in order (expired → archived). + try { + const webhookService = require('./webhookService'); + await webhookService.fire('event.expired', { + event: { id: event.id, slug: event.slug, event_name: event.event_name, expires_at: event.expires_at }, + }); + } catch (e) { /* non-fatal */ } + // Queue expiration emails const recipientEmail = event.customer_email || event.host_email; const recipientName = event.customer_name || event.host_name || (recipientEmail ? recipientEmail.split('@')[0] : null); diff --git a/backend/src/services/webhookDeliveryWorker.js b/backend/src/services/webhookDeliveryWorker.js new file mode 100644 index 00000000..4a82eee3 --- /dev/null +++ b/backend/src/services/webhookDeliveryWorker.js @@ -0,0 +1,277 @@ +const axios = require('axios'); +const { db } = require('../database/db'); +const logger = require('../utils/logger'); +const { signPayload, renderTemplate } = require('./webhookService'); +const { validateExternalUrl } = require('../utils/networkValidation'); + +const POLL_INTERVAL_MS = parseInt(process.env.WEBHOOK_DELIVERY_INTERVAL_MS || '5000', 10); +const CONCURRENCY = parseInt(process.env.WEBHOOK_DELIVERY_CONCURRENCY || '5', 10); +const HTTP_TIMEOUT_MS = parseInt(process.env.WEBHOOK_HTTP_TIMEOUT_MS || '10000', 10); +const MAX_ATTEMPTS = parseInt(process.env.WEBHOOK_MAX_ATTEMPTS || '5', 10); +const RESPONSE_TRUNCATE_BYTES = 1024; +// Mutable so tests can flip it without juggling require.cache; reads the +// env var at module load for the production code path. +let allowPrivateUrls = process.env.WEBHOOK_ALLOW_PRIVATE_URLS === 'true'; +const SIGNATURE_HEADER = 'X-PicPeak-Signature'; +const EVENT_HEADER = 'X-PicPeak-Event'; +const DELIVERY_HEADER = 'X-PicPeak-Delivery'; + +// Backoff schedule per the issue spec — index = attempt that just failed. +// attempt_count after the failure becomes (failedAttempt + 1); we look up +// the delay using the *new* attempt count to schedule the next try. +// attempt 1 fails → wait 1m +// attempt 2 fails → wait 5m +// attempt 3 fails → wait 30m +// attempt 4 fails → wait 2h +// attempt 5 fails → wait 12h THEN give up (max 5 attempts total) +const BACKOFF_MS = [ + 60_000, // 1 min + 5 * 60_000, // 5 min + 30 * 60_000, // 30 min + 2 * 60 * 60_000, // 2 h + 12 * 60 * 60_000, // 12 h (only used when MAX_ATTEMPTS extended past 5) +]; + +let intervalHandle = null; +let stopped = false; +// Tracks deliveries currently being processed in this tick — guards +// against the same row being claimed twice if a tick takes longer than +// POLL_INTERVAL_MS. +const inFlight = new Set(); + +function truncate(str, bytes) { + if (str == null) return null; + const buf = Buffer.from(String(str), 'utf8'); + if (buf.length <= bytes) return buf.toString('utf8'); + return buf.subarray(0, bytes).toString('utf8'); +} + +async function fetchPending(limit) { + // Skip rows already in-flight from a previous tick that's still running. + const excludeIds = Array.from(inFlight); + let q = db('webhook_deliveries') + .where('status', 'pending') + .where('next_retry_at', '<=', new Date()) + .orderBy('next_retry_at', 'asc') + .limit(limit); + if (excludeIds.length > 0) { + q = q.whereNotIn('id', excludeIds); + } + return q.select('*'); +} + +async function deliverOne(row) { + const startedAt = Date.now(); + const webhook = await db('webhooks').where({ id: row.webhook_id }).first(); + + if (!webhook) { + // Webhook was deleted while a delivery was pending. Mark failed and move on. + await db('webhook_deliveries') + .where({ id: row.id }) + .update({ + status: 'failed', + last_error: 'webhook subscription no longer exists', + completed_at: new Date(), + attempt_count: row.attempt_count + 1, + }); + return; + } + + if (!webhook.active) { + // Subscription disabled mid-flight. Don't abandon — leave as failed + // so the deliveries page reflects the reality. + await db('webhook_deliveries') + .where({ id: row.id }) + .update({ + status: 'failed', + last_error: 'webhook is disabled', + completed_at: new Date(), + attempt_count: row.attempt_count + 1, + }); + return; + } + + // Re-validate URL per delivery — DNS-rebinding mitigation. Admin can opt + // out via WEBHOOK_ALLOW_PRIVATE_URLS=true for local-receiver dev runs. + if (!allowPrivateUrls) { + const urlCheck = validateExternalUrl(webhook.url); + if (!urlCheck.valid) { + await markFailedFinal(row, `URL rejected: ${urlCheck.error}`); + return; + } + } + + const envelopeBody = typeof row.payload === 'string' ? row.payload : JSON.stringify(row.payload); + const envelopeObj = (() => { + try { return JSON.parse(envelopeBody); } catch { return {}; } + })(); + + // Per-webhook template (#327 follow-up). If set + valid, replaces the + // default JSON envelope as the request body. Signature is computed over + // the BODY ACTUALLY SENT, so receivers verify whatever they receive. + let rawBody = envelopeBody; + let contentType = 'application/json'; + if (webhook.template) { + const rendered = renderTemplate(webhook.template, envelopeObj); + if (rendered != null) { + rawBody = rendered; + // Best-effort content-type detection: if it parses as JSON, keep + // application/json; otherwise send as text/plain. + try { JSON.parse(rendered); } catch { contentType = 'text/plain; charset=utf-8'; } + } + } + const signature = signPayload(webhook.secret, rawBody); + const deliveryId = envelopeObj?.id || String(row.id); + + let response; + let networkError; + try { + response = await axios.post(webhook.url, rawBody, { + headers: { + 'Content-Type': contentType, + [SIGNATURE_HEADER]: signature, + [EVENT_HEADER]: row.event_type, + [DELIVERY_HEADER]: deliveryId, + 'User-Agent': 'PicPeak-Webhooks/1.0', + }, + timeout: HTTP_TIMEOUT_MS, + // Don't throw on non-2xx; we handle status manually. + validateStatus: () => true, + // Don't follow redirects — security + receivers should give us the + // final URL up front. + maxRedirects: 0, + // Cap response body so a chatty receiver can't OOM us before truncation. + maxContentLength: 10 * 1024, + maxBodyLength: rawBody.length + 1024, + }); + } catch (err) { + networkError = err; + } + + const latency = Date.now() - startedAt; + const newAttempt = row.attempt_count + 1; + + if (response && response.status >= 200 && response.status < 300) { + await db('webhook_deliveries') + .where({ id: row.id }) + .update({ + status: 'success', + response_status: response.status, + response_body: truncate(stringifyBody(response.data), RESPONSE_TRUNCATE_BYTES), + latency_ms: latency, + attempt_count: newAttempt, + completed_at: new Date(), + next_retry_at: null, + }); + await db('webhooks').where({ id: webhook.id }).update({ last_success_at: new Date() }); + return; + } + + // Failure path — schedule retry or give up. + const errorMsg = networkError + ? `network error: ${networkError.code || networkError.message}` + : `non-2xx status: ${response?.status}`; + + if (newAttempt >= MAX_ATTEMPTS) { + await db('webhook_deliveries') + .where({ id: row.id }) + .update({ + status: 'failed', + response_status: response?.status || null, + response_body: response ? truncate(stringifyBody(response.data), RESPONSE_TRUNCATE_BYTES) : null, + last_error: errorMsg, + latency_ms: latency, + attempt_count: newAttempt, + completed_at: new Date(), + next_retry_at: null, + }); + await db('webhooks').where({ id: webhook.id }).update({ last_failure_at: new Date() }); + return; + } + + const backoff = BACKOFF_MS[Math.min(newAttempt - 1, BACKOFF_MS.length - 1)]; + await db('webhook_deliveries') + .where({ id: row.id }) + .update({ + status: 'pending', + response_status: response?.status || null, + response_body: response ? truncate(stringifyBody(response.data), RESPONSE_TRUNCATE_BYTES) : null, + last_error: errorMsg, + latency_ms: latency, + attempt_count: newAttempt, + next_retry_at: new Date(Date.now() + backoff), + }); + await db('webhooks').where({ id: webhook.id }).update({ last_failure_at: new Date() }); +} + +async function markFailedFinal(row, reason) { + await db('webhook_deliveries') + .where({ id: row.id }) + .update({ + status: 'failed', + last_error: reason, + attempt_count: row.attempt_count + 1, + completed_at: new Date(), + next_retry_at: null, + }); + await db('webhooks').where({ id: row.webhook_id }).update({ last_failure_at: new Date() }); +} + +function stringifyBody(data) { + if (data == null) return null; + if (typeof data === 'string') return data; + if (Buffer.isBuffer(data)) return data.toString('utf8'); + try { return JSON.stringify(data); } catch { return String(data); } +} + +async function tick() { + if (stopped) return; + try { + const slots = Math.max(0, CONCURRENCY - inFlight.size); + if (slots === 0) return; + const rows = await fetchPending(slots); + if (rows.length === 0) return; + rows.forEach((r) => inFlight.add(r.id)); + await Promise.allSettled( + rows.map((r) => + deliverOne(r) + .catch((err) => logger.error(`[webhookWorker] delivery ${r.id} crashed: ${err.message}`)) + .finally(() => inFlight.delete(r.id)) + ) + ); + } catch (err) { + logger.error(`[webhookWorker] tick failed: ${err.message}`); + } +} + +function startWebhookDeliveryWorker() { + if (intervalHandle) return; // idempotent + stopped = false; + intervalHandle = setInterval(tick, POLL_INTERVAL_MS); + logger.info( + `[webhookWorker] started — interval=${POLL_INTERVAL_MS}ms, concurrency=${CONCURRENCY}, ` + + `max_attempts=${MAX_ATTEMPTS}, allow_private=${allowPrivateUrls}` + ); +} + +function stopWebhookDeliveryWorker() { + stopped = true; + if (intervalHandle) { + clearInterval(intervalHandle); + intervalHandle = null; + } +} + +module.exports = { + startWebhookDeliveryWorker, + stopWebhookDeliveryWorker, + // exported for tests + __test: { + tick, + BACKOFF_MS, + SIGNATURE_HEADER, + EVENT_HEADER, + DELIVERY_HEADER, + setAllowPrivateUrls(value) { allowPrivateUrls = !!value; }, + }, +}; diff --git a/backend/src/services/webhookService.js b/backend/src/services/webhookService.js new file mode 100644 index 00000000..3b3df20c --- /dev/null +++ b/backend/src/services/webhookService.js @@ -0,0 +1,224 @@ +const crypto = require('crypto'); +const { db } = require('../database/db'); +const logger = require('../utils/logger'); + +const SECRET_PREFIX = 'whsec_'; + +/** + * Event types PicPeak emits. Keep this in sync with the README catalog and + * the receiver-side type unions in any SDK we publish later. Consumers of + * `fire(eventType, ...)` MUST use one of these strings — the worker will + * silently drop unknown types so a typo can't 500 a request handler. + */ +const EVENT_TYPES = Object.freeze([ + 'event.created', + 'event.published', + 'event.archived', + 'event.expired', + 'photo.uploaded', + 'photo.deleted', +]); + +function generateSecret() { + const random = crypto.randomBytes(24).toString('base64url'); // ~32 chars + const plaintext = `${SECRET_PREFIX}${random}`; + return { + plaintext, + preview: random.slice(0, 8), + }; +} + +/** + * Sign a payload with the webhook's secret. Used by the delivery worker; + * exported for unit tests of receiver-side verification snippets. + */ +function signPayload(secret, rawBody) { + return crypto.createHmac('sha256', secret).update(rawBody).digest('hex'); +} + +/** + * Resolve a dot-path on an object (e.g. "data.event.event_type") with no + * eval. Returns undefined for missing segments — never throws. + */ +function getByPath(obj, dotPath) { + if (!dotPath || typeof dotPath !== 'string') return undefined; + return dotPath.split('.').reduce((acc, key) => { + if (acc == null || typeof acc !== 'object') return undefined; + return acc[key]; + }, obj); +} + +/** + * Evaluate a webhook's filter against an outgoing payload. The filter is a + * flat object of dot-path → expected value pairs: + * { "data.event.event_type": "wedding" } + * { "type": "event.published", "data.event.id": 42 } + * + * All keys must match (logical AND). Equality is `===` after JSON-style + * coercion: numbers as numbers, booleans as booleans. Empty filter + * matches everything (back-compat). + */ +function payloadMatchesFilter(filter, payload) { + if (!filter || typeof filter !== 'object') return true; + const keys = Object.keys(filter); + if (keys.length === 0) return true; + for (const key of keys) { + const expected = filter[key]; + const actual = getByPath(payload, key); + if (Array.isArray(expected)) { + // Array means "any of" + if (!expected.includes(actual)) return false; + } else if (actual !== expected) { + return false; + } + } + return true; +} + +/** + * Render a webhook template by substituting ${dot.path} expressions with + * values from the payload. NO eval, NO logic — pure string substitution. + * Caps output at 64KB; bails out and returns null if exceeded so the + * delivery worker can fall back to the default envelope. + */ +function renderTemplate(template, payload) { + if (template == null || template === '') return null; + if (typeof template !== 'string') return null; + const MAX_OUTPUT = 64 * 1024; + const MAX_SUBSTITUTIONS = 64; + let count = 0; + const rendered = template.replace(/\$\{([^}]+)\}/g, (_match, expr) => { + count += 1; + if (count > MAX_SUBSTITUTIONS) return ''; + const value = getByPath(payload, expr.trim()); + if (value == null) return ''; + if (typeof value === 'object') { + try { return JSON.stringify(value); } catch { return ''; } + } + return String(value); + }); + if (Buffer.byteLength(rendered, 'utf8') > MAX_OUTPUT) return null; + return rendered; +} + +/** + * Validate a template at create-time so admins get immediate feedback + * instead of silent delivery failures. Returns { valid, error? }. + */ +function validateTemplate(template) { + if (template == null || template === '') return { valid: true }; + if (typeof template !== 'string') return { valid: false, error: 'template must be a string' }; + if (Buffer.byteLength(template, 'utf8') > 8192) return { valid: false, error: 'template exceeds 8KB' }; + // Reject unbalanced ${ that would silently swallow content at render. + const opens = (template.match(/\$\{/g) || []).length; + const closes = (template.match(/\}/g) || []).length; + // Count is approximate (every } is counted, even non-matching ones). + // We require at least as many } as ${, which is necessary but not sufficient. + if (opens > closes) return { valid: false, error: 'template has unbalanced ${ — every ${ needs a matching }' }; + if (opens > 64) return { valid: false, error: 'template exceeds 64 substitutions' }; + return { valid: true }; +} + +/** + * Constant-time signature comparison helper for receivers and tests. + * Exposed so the same primitive backs verification examples in the README. + */ +function verifySignature(secret, rawBody, signature) { + const expected = signPayload(secret, rawBody); + const a = Buffer.from(expected, 'hex'); + let b; + try { + b = Buffer.from(signature || '', 'hex'); + } catch { + return false; + } + if (a.length !== b.length) return false; + return crypto.timingSafeEqual(a, b); +} + +/** + * Enqueue a webhook delivery for every active webhook subscribed to + * `eventType`. NEVER throws — webhook failures must not break the + * lifecycle handler that emitted the event. Worker handles HTTP delivery. + * + * @param {string} eventType — one of EVENT_TYPES + * @param {object} data — opaque payload that gets nested under .data in + * the outbound JSON body + */ +async function fire(eventType, data) { + if (!EVENT_TYPES.includes(eventType)) { + logger.warn(`[webhookService] dropping unknown event type: ${eventType}`); + return; + } + + try { + // jsonb @> ARRAY check across vendors: both pg and sqlite drivers we + // support handle a simple WHERE on `active=true` then a runtime filter + // on the events array; doing the array filter here keeps the query + // portable. + const candidates = await db('webhooks').where({ active: true }); + const subscribed = candidates.filter((w) => { + const evts = Array.isArray(w.events) + ? w.events + : (() => { try { return JSON.parse(w.events) || []; } catch { return []; } })(); + return evts.includes(eventType); + }); + + if (subscribed.length === 0) return; + + const now = new Date(); + const buildEnvelope = (deliveryUuid) => ({ + id: deliveryUuid, + type: eventType, + created_at: now.toISOString(), + data, + }); + + const rows = []; + for (const w of subscribed) { + const deliveryUuid = crypto.randomUUID(); + const envelope = buildEnvelope(deliveryUuid); + + // Filter (#327 follow-up): per-webhook predicate evaluated against + // the payload. Skip insertion when it doesn't match. + const filter = parseJsonField(w.filter, {}); + if (!payloadMatchesFilter(filter, envelope)) continue; + + rows.push({ + webhook_id: w.id, + event_type: eventType, + payload: JSON.stringify(envelope), + attempt_count: 0, + status: 'pending', + next_retry_at: now, + created_at: now, + }); + } + + if (rows.length === 0) return; + await db('webhook_deliveries').insert(rows); + } catch (err) { + // Log but don't throw — caller's transaction has already committed + // by the time we get here, and we don't want to mask the success. + logger.error(`[webhookService.fire] failed to enqueue ${eventType}: ${err.message}`); + } +} + +function parseJsonField(value, fallback) { + if (value == null) return fallback; + if (typeof value === 'object') return value; + try { return JSON.parse(value) ?? fallback; } catch { return fallback; } +} + +module.exports = { + fire, + generateSecret, + signPayload, + verifySignature, + payloadMatchesFilter, + renderTemplate, + validateTemplate, + getByPath, + EVENT_TYPES, + SECRET_PREFIX, +}; diff --git a/dev/webhook-receiver/Dockerfile b/dev/webhook-receiver/Dockerfile new file mode 100644 index 00000000..f6a9d4e6 --- /dev/null +++ b/dev/webhook-receiver/Dockerfile @@ -0,0 +1,5 @@ +FROM node:22-alpine +WORKDIR /app +COPY server.js ./ +EXPOSE 8888 +CMD ["node", "server.js"] diff --git a/dev/webhook-receiver/server.js b/dev/webhook-receiver/server.js new file mode 100644 index 00000000..4c0e7686 --- /dev/null +++ b/dev/webhook-receiver/server.js @@ -0,0 +1,93 @@ +// Tiny dev-only webhook receiver. Logs every request as one JSON line per +// hit so the E2E spec can poll the log file (or hit GET /requests to read +// from memory). Holds the last 200 requests in a ring buffer. +// +// Endpoints: +// POST / — accept any webhook; records and returns 200 +// GET /requests — returns the ring buffer as JSON +// POST /reset — clear the ring buffer +// GET /health — 200 ok +// +// Configurable response status via FORCE_STATUS env (e.g. 500 to test retries). + +const http = require('http'); + +const PORT = parseInt(process.env.PORT || '8888', 10); +const RING_SIZE = parseInt(process.env.RING_SIZE || '200', 10); +const FORCE_STATUS = parseInt(process.env.FORCE_STATUS || '200', 10); + +const ring = []; + +function readBody(req) { + return new Promise((resolve, reject) => { + const chunks = []; + let total = 0; + req.on('data', (chunk) => { + total += chunk.length; + if (total > 1024 * 1024) { + reject(new Error('payload too large')); + return; + } + chunks.push(chunk); + }); + req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + req.on('error', reject); + }); +} + +const server = http.createServer(async (req, res) => { + if (req.method === 'GET' && req.url === '/health') { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('ok'); + return; + } + + if (req.method === 'GET' && req.url === '/requests') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(ring)); + return; + } + + if (req.method === 'POST' && req.url === '/reset') { + ring.length = 0; + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end('cleared'); + return; + } + + // Treat every other request as a webhook delivery to record. + let body = ''; + try { + body = await readBody(req); + } catch (err) { + res.writeHead(413, { 'Content-Type': 'text/plain' }); + res.end(err.message); + return; + } + + const entry = { + receivedAt: new Date().toISOString(), + method: req.method, + url: req.url, + headers: req.headers, + body, + }; + ring.push(entry); + if (ring.length > RING_SIZE) ring.shift(); + + // Log a single line so docker logs gives a quick readable trace. + process.stdout.write( + `[webhook-receiver] ${req.method} ${req.url} sig=${ + req.headers['x-picpeak-signature'] || '-' + } type=${(() => { + try { return JSON.parse(body)?.type || '-'; } catch { return '-'; } + })()}\n` + ); + + res.writeHead(FORCE_STATUS, { 'Content-Type': 'text/plain' }); + res.end(FORCE_STATUS >= 200 && FORCE_STATUS < 300 ? 'ok' : 'forced-failure'); +}); + +server.listen(PORT, () => { + process.stdout.write(`webhook-receiver listening on :${PORT}\n`); +}); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index ed0f113f..91e8be8a 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -26,14 +26,15 @@ import { BackupManagement, CMSPage, UserManagementPage, - EventTypesPage + EventTypesPage, + WebhookDeliveriesPage } from './pages/admin'; import { AcceptInvitePage } from './pages/public/AcceptInvitePage'; import { AdminLayout, AdminAuthWrapper } from './components/admin'; import { PageErrorBoundary, OfflineIndicator, SkipLink, DynamicFavicon, RobotsMetaTags, CMSContentBlock } from './components/common'; import { MaintenanceWrapper } from './components/MaintenanceWrapper'; import { GlobalThemeProvider } from './components/GlobalThemeProvider'; -import { getApiBaseUrl } from './utils/url'; +import { usePublicSettings } from './hooks/usePublicSettings'; // Create a client const queryClient = new QueryClient({ @@ -45,6 +46,40 @@ const queryClient = new QueryClient({ }, }); +// Bootstraps Umami analytics from /public/settings. Lives inside QueryClientProvider +// so it shares the public-settings cache with every other consumer of usePublicSettings. +function AnalyticsBootstrap() { + const { data: settings, isError } = usePublicSettings(); + + useEffect(() => { + if (!settings && !isError) return; + + const envUmamiUrl = import.meta.env.VITE_UMAMI_URL; + const envUmamiWebsiteId = import.meta.env.VITE_UMAMI_WEBSITE_ID; + + if (settings?.umami_enabled && settings.umami_url && settings.umami_website_id) { + analyticsService.initialize({ + websiteId: settings.umami_website_id, + hostUrl: settings.umami_url, + autoTrack: true, + doNotTrack: true, + }); + return; + } + + if (envUmamiUrl && envUmamiWebsiteId && (isError || settings?.enable_analytics !== false)) { + analyticsService.initialize({ + websiteId: envUmamiWebsiteId, + hostUrl: envUmamiUrl, + autoTrack: true, + doNotTrack: true, + }); + } + }, [settings, isError]); + + return null; +} + function App() { // Track dark mode for toast theming const [toastTheme, setToastTheme] = useState<'light' | 'dark'>('light'); @@ -57,60 +92,10 @@ function App() { return () => observer.disconnect(); }, []); - // Initialize Umami Analytics based on settings - useEffect(() => { - const initializeAnalytics = async () => { - try { - // Fetch public settings to get Umami configuration - const response = await fetch(`${getApiBaseUrl()}/public/settings`); - const settings = await response.json(); - - // Check if Umami is enabled and configured in backend settings - if (settings.umami_enabled && settings.umami_url && settings.umami_website_id) { - // Use backend configuration - analyticsService.initialize({ - websiteId: settings.umami_website_id, - hostUrl: settings.umami_url, - autoTrack: true, - doNotTrack: true - }); - } else { - // Fall back to environment variables if backend not configured - const umamiUrl = import.meta.env.VITE_UMAMI_URL; - const umamiWebsiteId = import.meta.env.VITE_UMAMI_WEBSITE_ID; - - if (umamiUrl && umamiWebsiteId && settings.enable_analytics !== false) { - analyticsService.initialize({ - websiteId: umamiWebsiteId, - hostUrl: umamiUrl, - autoTrack: true, - doNotTrack: true - }); - } - } - } catch (error) { - console.error('Failed to fetch settings for analytics:', error); - // Fall back to environment variables on error - const umamiUrl = import.meta.env.VITE_UMAMI_URL; - const umamiWebsiteId = import.meta.env.VITE_UMAMI_WEBSITE_ID; - - if (umamiUrl && umamiWebsiteId) { - analyticsService.initialize({ - websiteId: umamiWebsiteId, - hostUrl: umamiUrl, - autoTrack: true, - doNotTrack: true - }); - } - } - }; - - initializeAnalytics(); - }, []); - return ( + @@ -148,6 +133,7 @@ function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/frontend/src/features/settings/index.ts b/frontend/src/features/settings/index.ts index ac572b66..3c8f7a54 100644 --- a/frontend/src/features/settings/index.ts +++ b/frontend/src/features/settings/index.ts @@ -16,3 +16,4 @@ export { StylingTab } from './tabs/StylingTab'; export { SEOTab } from './tabs/SEOTab'; export { ThumbnailsTab } from './tabs/ThumbnailsTab'; export { ApiTokensTab } from './tabs/ApiTokensTab'; +export { WebhooksTab } from './tabs/WebhooksTab'; diff --git a/frontend/src/features/settings/tabs/WebhooksTab.tsx b/frontend/src/features/settings/tabs/WebhooksTab.tsx new file mode 100644 index 00000000..8ba8a3b0 --- /dev/null +++ b/frontend/src/features/settings/tabs/WebhooksTab.tsx @@ -0,0 +1,354 @@ +import React, { useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { Link } from 'react-router-dom'; +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; +import { toast } from 'react-toastify'; +import { Webhook as WebhookIcon, Trash2, Copy, AlertTriangle, Activity, CheckCircle2, XCircle } from 'lucide-react'; +import { Button, Card, Input, Loading } from '../../../components/common'; +import { api } from '../../../config/api'; + +const WEBHOOK_EVENT_TYPES = [ + 'event.created', + 'event.published', + 'event.archived', + 'event.expired', + 'photo.uploaded', + 'photo.deleted', +] as const; +type WebhookEventType = typeof WEBHOOK_EVENT_TYPES[number]; + +interface WebhookRow { + id: number; + name: string; + url: string; + events: WebhookEventType[]; + active: boolean; + secret_preview: string | null; + created_at: string; + updated_at: string; + last_success_at: string | null; + last_failure_at: string | null; + owner_username: string | null; +} + +/** + * Settings → Webhooks tab (#327). Mirrors the API Tokens tab pattern: + * the signing secret is returned exactly once on creation and never + * recoverable. Per-webhook delivery history lives on the dedicated + * /admin/webhooks/:id/deliveries page (link in the table). + */ +export const WebhooksTab: React.FC = () => { + const { t } = useTranslation(); + const queryClient = useQueryClient(); + const [name, setName] = useState(''); + const [url, setUrl] = useState(''); + const [events, setEvents] = useState(['event.published']); + const [filterText, setFilterText] = useState('{}'); + const [template, setTemplate] = useState(''); + const [showAdvanced, setShowAdvanced] = useState(false); + const [justCreatedSecret, setJustCreatedSecret] = useState(null); + const [filterError, setFilterError] = useState(null); + + const { data: webhooks, isLoading } = useQuery({ + queryKey: ['admin-webhooks'], + queryFn: async () => { + const res = await api.get('/admin/webhooks'); + return res.data; + }, + }); + + const createMutation = useMutation({ + mutationFn: async () => { + let parsedFilter: Record = {}; + const trimmed = filterText.trim(); + if (trimmed && trimmed !== '{}') { + try { + parsedFilter = JSON.parse(trimmed); + } catch { + setFilterError('Filter must be valid JSON'); + throw new Error('Invalid filter JSON'); + } + } + setFilterError(null); + const body: Record = { name, url, events, active: true }; + if (Object.keys(parsedFilter).length > 0) body.filter = parsedFilter; + if (template.trim()) body.template = template; + const res = await api.post<{ secret: string }>('/admin/webhooks', body); + return res.data.secret; + }, + onSuccess: (secret) => { + setJustCreatedSecret(secret); + setName(''); + setUrl(''); + setEvents(['event.published']); + setFilterText('{}'); + setTemplate(''); + setShowAdvanced(false); + queryClient.invalidateQueries({ queryKey: ['admin-webhooks'] }); + }, + onError: (err: any) => { + toast.error(err?.response?.data?.errors?.[0]?.msg || err?.response?.data?.error || 'Failed to create webhook'); + }, + }); + + const toggleActiveMutation = useMutation({ + mutationFn: async ({ id, active }: { id: number; active: boolean }) => + api.put(`/admin/webhooks/${id}`, { active }), + onSuccess: () => queryClient.invalidateQueries({ queryKey: ['admin-webhooks'] }), + onError: () => toast.error('Failed to update webhook'), + }); + + const deleteMutation = useMutation({ + mutationFn: async (id: number) => api.delete(`/admin/webhooks/${id}`), + onSuccess: () => { + toast.success('Webhook deleted'); + queryClient.invalidateQueries({ queryKey: ['admin-webhooks'] }); + }, + onError: () => toast.error('Failed to delete webhook'), + }); + + const toggleEvent = (e: WebhookEventType) => { + setEvents((prev) => (prev.includes(e) ? prev.filter((x) => x !== e) : [...prev, e])); + }; + + if (isLoading) { + return ( +
+ +
+ ); + } + + return ( +
+ +

+ + {t('settings.webhooks.title', 'Webhooks')} +

+

+ {t('settings.webhooks.subtitle', 'POST event notifications to your URL the moment something happens — gallery published, photo uploaded, event archived, etc. Signed with HMAC-SHA256 in the X-PicPeak-Signature header.')} +

+ + {justCreatedSecret && ( +
+
+ +
+

+ {t('settings.webhooks.copyNow', 'Copy this signing secret now — it will not be shown again.')} +

+
+ + {justCreatedSecret} + + + +
+
+
+
+ )} + +
+
+
+ + setName(e.target.value)} placeholder="e.g. n8n WhatsApp" /> +
+
+ + setUrl(e.target.value)} placeholder="https://n8n.example.com/webhook/picpeak" /> +
+
+ +
+ +
+ {WEBHOOK_EVENT_TYPES.map((e) => ( + + ))} +
+
+ + + + {showAdvanced && ( +
+
+ +