Merge pull request #334 from the-luap/feat/post-319-fixes-and-features

feat: S3 storage + webhooks + settings dedupe + backup fixes
This commit is contained in:
Paul Nothaft
2026-04-28 14:50:35 +02:00
committed by GitHub
74 changed files with 5368 additions and 1077 deletions
+35 -17
View File
@@ -7,12 +7,15 @@ const crypto = require('crypto');
// Load services
const backupService = require('../../src/services/backupService');
const S3StorageAdapter = require('../../src/services/storage/s3Storage');
const { db, initialize: initDb } = require('../../src/database/db');
const { db, initializeDatabase: initDb } = require('../../src/database/db');
const logger = require('../../src/utils/logger');
// Test configuration
// Defaults match the dev MinIO container in docker-compose.dev.yml (port 7104).
// Override via TEST_S3_ENDPOINT / TEST_S3_ACCESS_KEY / TEST_S3_SECRET_KEY when running
// against a different S3 endpoint (CI, hosted MinIO, real AWS, etc.).
const TEST_CONFIG = {
endpoint: process.env.TEST_S3_ENDPOINT || 'http://localhost:9000',
endpoint: process.env.TEST_S3_ENDPOINT || 'http://localhost:7104',
accessKeyId: process.env.TEST_S3_ACCESS_KEY || 'minioadmin',
secretAccessKey: process.env.TEST_S3_SECRET_KEY || 'minioadmin',
bucket: 'test-backup-bucket-' + Date.now(),
@@ -56,9 +59,17 @@ describe('S3 Backup Integration Tests', () => {
}
}
// Initialize database
await initDb();
await db.migrate.latest();
// Schema is expected to already be applied by `npm run migrate` against
// the dev database. db.migrate.latest() can't be used here because
// PicPeak's custom run-migrations.js tracks state in the `migrations`
// table (not knex's `knex_migrations`), so knex would try to re-apply
// every migration and crash on duplicate-table errors.
const ok = await db.schema.hasTable('events')
&& await db.schema.hasTable('app_settings')
&& await db.schema.hasTable('backup_runs');
if (!ok) {
throw new Error('Required tables missing — run `npm run migrate` against the dev DB first.');
}
// Create test storage directory
testStoragePath = path.join(__dirname, '../fixtures/test-storage');
@@ -69,10 +80,12 @@ describe('S3 Backup Integration Tests', () => {
await setupTestData();
// Mock logger to reduce noise
logger.info = jest.fn();
logger.debug = jest.fn();
logger.warn = jest.fn();
logger.error = jest.fn();
if (process.env.UNMOCK_LOGGER !== 'true') {
logger.info = jest.fn();
logger.debug = jest.fn();
logger.warn = jest.fn();
logger.error = jest.fn();
}
});
afterAll(async () => {
@@ -165,8 +178,9 @@ describe('S3 Backup Integration Tests', () => {
.first();
expect(backupRun.status).toBe('completed');
expect(backupRun.files_backed_up).toBeGreaterThan(0);
expect(backupRun.total_size_bytes).toBeGreaterThan(0);
// pg driver returns bigint columns as strings; coerce for the size assertion.
expect(Number(backupRun.files_backed_up)).toBeGreaterThan(0);
expect(Number(backupRun.total_size_bytes)).toBeGreaterThan(0);
// Verify files in S3
const s3Objects = await listS3Objects();
@@ -269,13 +283,16 @@ describe('S3 Backup Integration Tests', () => {
.first();
expect(secondRun.id).not.toBe(firstRun.id);
expect(secondRun.files_backed_up).toBe(1); // Only modified file
expect(Number(secondRun.files_backed_up)).toBe(1); // Only modified file
// Check manifest indicates incremental
// Check manifest indicates incremental. The current manifest schema
// groups counts under `incremental.changes.*` (added/modified/deleted/
// unchanged + size_difference) — see backupManifest.generateIncrementalManifest.
if (secondRun.manifest_path) {
const manifest = await backupService.getBackupManifest(secondRun.id);
expect(manifest.manifest.incremental).toBeDefined();
expect(manifest.manifest.incremental.modified_files_count).toBe(1);
expect(manifest.manifest.incremental.changes).toBeDefined();
expect(manifest.manifest.incremental.changes.modified_files_count).toBe(1);
}
});
@@ -468,15 +485,16 @@ describe('S3 Backup Integration Tests', () => {
{ setting_key: 'backup_max_file_size_mb', setting_value: '100' }
];
// Schema drift: app_settings has no created_at column anymore and the
// unique constraint is on setting_key alone, not (setting_type, key).
for (const setting of settings) {
await db('app_settings')
.insert({
setting_type: 'backup',
...setting,
created_at: new Date(),
updated_at: new Date()
updated_at: new Date(),
})
.onConflict(['setting_type', 'setting_key'])
.onConflict('setting_key')
.merge();
}
}
@@ -0,0 +1,165 @@
const path = require('path');
const fs = require('fs').promises;
const fsSync = require('fs');
const os = require('os');
const crypto = require('crypto');
const { S3Client, CreateBucketCommand, DeleteBucketCommand, ListObjectsV2Command, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
const sharp = require('sharp');
const LocalFsStorage = require('../../src/services/storage/LocalFsStorage');
const S3StorageBackend = require('../../src/services/storage/S3StorageBackend');
const storageModule = require('../../src/services/storage');
// Stub out the DB so getThumbnailSettings falls into its catch and uses defaults.
jest.mock('../../src/database/db', () => ({
db: () => {
throw new Error('db disabled in this test');
},
}));
const TEST_S3 = {
endpoint: process.env.TEST_S3_ENDPOINT || 'http://localhost:7104',
accessKeyId: process.env.TEST_S3_ACCESS_KEY || 'minioadmin',
secretAccessKey: process.env.TEST_S3_SECRET_KEY || 'minioadmin',
region: 'us-east-1',
};
const skipS3 = process.env.SKIP_S3_TESTS === 'true';
function backendCases() {
const cases = [
{
name: 'LocalFsStorage',
async setup() {
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'picpeak-imgproc-'));
const storage = new LocalFsStorage({ root });
await storage.init();
return { storage, cleanup: () => fs.rm(root, { recursive: true, force: true }) };
},
},
];
if (!skipS3) {
cases.push({
name: 'S3StorageBackend (MinIO)',
async setup() {
const bucket = `picpeak-imgproc-${Date.now()}-${crypto.randomBytes(2).toString('hex')}`;
const s3Client = new S3Client({
endpoint: TEST_S3.endpoint,
region: TEST_S3.region,
credentials: { accessKeyId: TEST_S3.accessKeyId, secretAccessKey: TEST_S3.secretAccessKey },
forcePathStyle: true,
});
await s3Client.send(new CreateBucketCommand({ Bucket: bucket }));
const storage = new S3StorageBackend({
bucket,
region: TEST_S3.region,
endpoint: TEST_S3.endpoint,
accessKeyId: TEST_S3.accessKeyId,
secretAccessKey: TEST_S3.secretAccessKey,
forcePathStyle: true,
sslEnabled: false,
});
await storage.init();
return {
storage,
async cleanup() {
const list = await s3Client.send(new ListObjectsV2Command({ Bucket: bucket }));
if (list.Contents?.length) {
await s3Client.send(new DeleteObjectsCommand({
Bucket: bucket,
Delete: { Objects: list.Contents.map((o) => ({ Key: o.Key })) },
}));
}
await s3Client.send(new DeleteBucketCommand({ Bucket: bucket }));
},
};
},
});
}
return cases;
}
async function makeSourceJpeg(targetDir, name) {
const localPath = path.join(targetDir, name);
// 800x600 random RGB image so sharp has something realistic to thumbnail.
const width = 800;
const height = 600;
const buf = Buffer.alloc(width * height * 3);
for (let i = 0; i < buf.length; i++) buf[i] = (i * 7) % 256;
await sharp(buf, { raw: { width, height, channels: 3 } })
.jpeg({ quality: 90 })
.toFile(localPath);
return localPath;
}
describe.each(backendCases())('imageProcessor through $name', ({ setup }) => {
let storage;
let cleanup;
let tmpDir;
let imageProcessor;
beforeAll(async () => {
({ storage, cleanup } = await setup());
storageModule.setStorageForTesting(storage);
// Require AFTER setStorageForTesting so the module sees our injection.
delete require.cache[require.resolve('../../src/services/imageProcessor')];
imageProcessor = require('../../src/services/imageProcessor');
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'picpeak-imgproc-src-'));
}, 30000);
afterAll(async () => {
storageModule.resetStorage();
if (tmpDir) await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
if (cleanup) await cleanup();
});
test('generateThumbnail writes through storage and returns a relative key', async () => {
const src = await makeSourceJpeg(tmpDir, 'sample.jpg');
const key = await imageProcessor.generateThumbnail(src);
expect(key).toBe('thumbnails/thumb_sample.jpg');
expect(await storage.exists(key)).toBe(true);
const stat = await storage.stat(key);
expect(stat.size).toBeGreaterThan(100);
// Verify the bytes are a valid JPEG by re-parsing with sharp on local mode.
if (storage.kind() === 'local') {
const meta = await sharp(storage.resolveLocalPath(key)).metadata();
expect(meta.format).toBe('jpeg');
expect(meta.width).toBeLessThanOrEqual(300);
}
});
test('generateHeroImage writes through storage and returns a relative key', async () => {
const src = await makeSourceJpeg(tmpDir, 'hero-source.jpg');
const key = await imageProcessor.generateHeroImage(src);
expect(key).toBe('heroes/hero_hero-source.jpg');
expect(await storage.exists(key)).toBe(true);
});
test('isThumbnailValid returns true for a good thumbnail and false for nothing', async () => {
const src = await makeSourceJpeg(tmpDir, 'valid-check.jpg');
const key = await imageProcessor.generateThumbnail(src);
expect(await imageProcessor.isThumbnailValid(key)).toBe(true);
expect(await imageProcessor.isThumbnailValid('thumbnails/does-not-exist.jpg')).toBe(false);
});
test('generateVideoPlaceholder writes a thumbnail entirely from buffer', async () => {
const key = await imageProcessor.generateVideoPlaceholder('demo.mp4');
expect(key).toBe('thumbnails/thumb_demo.jpg');
expect(await storage.exists(key)).toBe(true);
});
test('withLocalCopy yields a usable local path on both backends', async () => {
const sourceKey = 'fixture/withlocal.jpg';
const src = await makeSourceJpeg(tmpDir, 'withlocal.jpg');
const buf = await fs.readFile(src);
await storage.put(sourceKey, buf, { contentType: 'image/jpeg' });
const seenSize = await imageProcessor.withLocalCopy(sourceKey, async (localPath) => {
const meta = await sharp(localPath).metadata();
return meta.width;
});
expect(seenSize).toBe(800);
});
});
@@ -0,0 +1,189 @@
const path = require('path');
const fs = require('fs');
const fsp = require('fs').promises;
const os = require('os');
const crypto = require('crypto');
const { Readable } = require('stream');
const { S3Client, CreateBucketCommand, DeleteBucketCommand, ListObjectsV2Command, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
const LocalFsStorage = require('../../src/services/storage/LocalFsStorage');
const S3StorageBackend = require('../../src/services/storage/S3StorageBackend');
// MinIO defaults match docker-compose.dev.yml. Override via TEST_S3_* if needed.
const TEST_S3 = {
endpoint: process.env.TEST_S3_ENDPOINT || 'http://localhost:7104',
accessKeyId: process.env.TEST_S3_ACCESS_KEY || 'minioadmin',
secretAccessKey: process.env.TEST_S3_SECRET_KEY || 'minioadmin',
region: 'us-east-1',
};
const skipS3 = process.env.SKIP_S3_TESTS === 'true';
// Build the matrix of backends to test. Local always runs; S3 runs against MinIO
// unless SKIP_S3_TESTS=true (CI default). The same suite runs against both so
// every consumer can rely on identical semantics.
function backendCases() {
const cases = [
{
name: 'LocalFsStorage',
async setup() {
const root = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-storage-'));
const storage = new LocalFsStorage({ root });
await storage.init();
return { storage, cleanup: () => fsp.rm(root, { recursive: true, force: true }) };
},
},
];
if (!skipS3) {
cases.push({
name: 'S3StorageBackend (MinIO)',
async setup() {
const bucket = `picpeak-test-${Date.now()}-${crypto.randomBytes(2).toString('hex')}`;
const s3Client = new S3Client({
endpoint: TEST_S3.endpoint,
region: TEST_S3.region,
credentials: { accessKeyId: TEST_S3.accessKeyId, secretAccessKey: TEST_S3.secretAccessKey },
forcePathStyle: true,
});
await s3Client.send(new CreateBucketCommand({ Bucket: bucket }));
const storage = new S3StorageBackend({
bucket,
region: TEST_S3.region,
endpoint: TEST_S3.endpoint,
accessKeyId: TEST_S3.accessKeyId,
secretAccessKey: TEST_S3.secretAccessKey,
forcePathStyle: true,
sslEnabled: false,
});
await storage.init();
return {
storage,
async cleanup() {
// Empty bucket then delete it.
const list = await s3Client.send(new ListObjectsV2Command({ Bucket: bucket }));
if (list.Contents?.length) {
await s3Client.send(new DeleteObjectsCommand({
Bucket: bucket,
Delete: { Objects: list.Contents.map((o) => ({ Key: o.Key })) },
}));
}
await s3Client.send(new DeleteBucketCommand({ Bucket: bucket }));
},
};
},
});
}
return cases;
}
async function readToString(stream) {
const chunks = [];
for await (const chunk of stream) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
return Buffer.concat(chunks).toString('utf-8');
}
describe.each(backendCases())('StorageBackend contract: $name', ({ setup }) => {
let storage;
let cleanup;
beforeAll(async () => {
({ storage, cleanup } = await setup());
}, 30000);
afterAll(async () => {
if (cleanup) await cleanup();
});
test('put + get + exists + stat + delete round-trip with a buffer body', async () => {
const key = 'photos/event-a/IMG_0001.jpg';
const body = Buffer.from('hello picpeak');
await storage.put(key, body, { contentType: 'image/jpeg' });
expect(await storage.exists(key)).toBe(true);
const stat = await storage.stat(key);
expect(stat).not.toBeNull();
expect(stat.size).toBe(body.length);
const stream = await storage.get(key);
const text = await readToString(stream);
expect(text).toBe('hello picpeak');
await storage.delete(key);
expect(await storage.exists(key)).toBe(false);
expect(await storage.stat(key)).toBeNull();
});
test('put accepts a Readable stream body', async () => {
const key = 'photos/event-b/streamed.bin';
const body = Readable.from(Buffer.from('streamed payload'));
await storage.put(key, body);
const got = await readToString(await storage.get(key));
expect(got).toBe('streamed payload');
});
test('putFromFile + getToFile round-trip', async () => {
const tmpIn = path.join(os.tmpdir(), `in-${Date.now()}.txt`);
const tmpOut = path.join(os.tmpdir(), `out-${Date.now()}.txt`);
await fsp.writeFile(tmpIn, 'file payload');
const key = 'thumbnails/thumb_x.jpg';
await storage.putFromFile(key, tmpIn, { contentType: 'image/jpeg' });
await storage.getToFile(key, tmpOut);
const text = await fsp.readFile(tmpOut, 'utf-8');
expect(text).toBe('file payload');
await fsp.unlink(tmpIn).catch(() => {});
await fsp.unlink(tmpOut).catch(() => {});
});
test('list returns entries under a prefix with size + key', async () => {
await storage.put('events/active/a/photo1.jpg', Buffer.from('a1'));
await storage.put('events/active/a/photo2.jpg', Buffer.from('a22'));
await storage.put('events/active/b/photo3.jpg', Buffer.from('b333'));
const entries = await storage.list('events/active/a');
const keys = entries.map((e) => e.key).sort();
expect(keys).toEqual(['events/active/a/photo1.jpg', 'events/active/a/photo2.jpg']);
const sizes = Object.fromEntries(entries.map((e) => [e.key, e.size]));
expect(sizes['events/active/a/photo1.jpg']).toBe(2);
expect(sizes['events/active/a/photo2.jpg']).toBe(3);
});
test('rename moves an object from src to dst (atomic on local; copy+delete on s3)', async () => {
await storage.put('uploads/temp.jpg', Buffer.from('rename-me'));
await storage.rename('uploads/temp.jpg', 'uploads/final.jpg');
expect(await storage.exists('uploads/temp.jpg')).toBe(false);
expect(await storage.exists('uploads/final.jpg')).toBe(true);
const text = await readToString(await storage.get('uploads/final.jpg'));
expect(text).toBe('rename-me');
});
test('copy duplicates an object without removing the source', async () => {
await storage.put('events/source.jpg', Buffer.from('src'));
await storage.copy('events/source.jpg', 'events/copied.jpg');
expect(await storage.exists('events/source.jpg')).toBe(true);
expect(await storage.exists('events/copied.jpg')).toBe(true);
});
test('delete on a missing key is a no-op (does not throw)', async () => {
await expect(storage.delete('does/not/exist.jpg')).resolves.toBeUndefined();
});
test('stat on a missing key returns null', async () => {
expect(await storage.stat('still/not/here.jpg')).toBeNull();
});
test('rejects path traversal attempts', async () => {
await expect(storage.put('../escape.txt', Buffer.from('x'))).rejects.toThrow(/traversal/i);
await expect(storage.get('../escape.txt')).rejects.toThrow(/traversal/i);
});
});
@@ -0,0 +1,239 @@
// Worker reads WEBHOOK_ALLOW_PRIVATE_URLS at module-load. Set it BEFORE
// requiring the worker so the local-stub URLs (127.0.0.1:<random>) pass
// the SSRF check by default.
process.env.WEBHOOK_ALLOW_PRIVATE_URLS = 'true';
process.env.WEBHOOK_DELIVERY_INTERVAL_MS = '50';
const http = require('http');
const { db } = require('../../src/database/db');
const webhookService = require('../../src/services/webhookService');
const { __test, startWebhookDeliveryWorker, stopWebhookDeliveryWorker } = require('../../src/services/webhookDeliveryWorker');
// Local-only test stub: matches what dev/webhook-receiver/server.js does
// in the docker-compose flow but spun up inside the Jest process so the
// suite is self-contained.
function makeStub({ status = 200, delayMs = 0, bodyOverride = null } = {}) {
const requests = [];
const server = http.createServer(async (req, res) => {
const chunks = [];
for await (const c of req) chunks.push(c);
const body = Buffer.concat(chunks).toString('utf8');
requests.push({ method: req.method, url: req.url, headers: req.headers, body });
if (delayMs) await new Promise((r) => setTimeout(r, delayMs));
res.writeHead(status, { 'Content-Type': 'text/plain' });
res.end(bodyOverride !== null ? bodyOverride : (status >= 200 && status < 300 ? 'ok' : 'forced'));
});
return new Promise((resolve) => {
server.listen(0, '127.0.0.1', () => {
const port = server.address().port;
resolve({ url: `http://127.0.0.1:${port}/`, requests, close: () => new Promise((r) => server.close(r)) });
});
});
}
async function insertWebhook(url, events = ['event.published'], extras = {}) {
// Tests need the WORKER to bypass SSRF on 127.0.0.1 stubs, but the
// route layer's allowlist check is bypassed here since we insert
// straight into the DB.
const { plaintext, preview } = webhookService.generateSecret();
const insert = await db('webhooks').insert({
name: extras.name || 'test',
url,
secret: plaintext,
secret_preview: preview,
events: JSON.stringify(events),
active: extras.active !== false,
created_by: 1,
}).returning('id');
const id = insert[0]?.id || insert[0];
return { id, secret: plaintext };
}
async function clearWebhooks() {
await db('webhook_deliveries').del();
await db('webhooks').del();
}
describe('webhook delivery worker (#327)', () => {
beforeAll(async () => {
// Schema is expected to already be applied by `npm run migrate`. We
// just verify the webhooks tables exist; if not, the test harness has
// missed running migration 082.
const ok = await db.schema.hasTable('webhooks');
if (!ok) throw new Error('webhooks table missing — run `npm run migrate` first');
}, 30000);
afterAll(async () => {
stopWebhookDeliveryWorker();
await db.destroy();
});
beforeEach(async () => {
await clearWebhooks();
});
test('signs the body with HMAC-SHA256 and the receiver can verify', async () => {
const stub = await makeStub({ status: 200 });
try {
const { id, secret } = await insertWebhook(stub.url);
await webhookService.fire('event.published', { event: { id: 1, slug: 'sig-test' } });
await __test.tick();
expect(stub.requests).toHaveLength(1);
const got = stub.requests[0];
const sig = got.headers['x-picpeak-signature'];
expect(sig).toBeTruthy();
// Receiver-side verification using the SAME helper we ship in the README.
expect(webhookService.verifySignature(secret, got.body, sig)).toBe(true);
// Tampering must fail.
expect(webhookService.verifySignature(secret, got.body + 'x', sig)).toBe(false);
const row = await db('webhook_deliveries').where({ webhook_id: id }).first();
expect(row.status).toBe('success');
expect(row.attempt_count).toBe(1);
expect(row.response_status).toBe(200);
expect(row.latency_ms).toBeGreaterThanOrEqual(0);
} finally {
await stub.close();
}
});
test('headers include event type and a unique delivery id', async () => {
const stub = await makeStub({ status: 200 });
try {
await insertWebhook(stub.url, ['photo.uploaded']);
await webhookService.fire('photo.uploaded', { photo: { id: 7 } });
await __test.tick();
const got = stub.requests[0];
expect(got.headers['x-picpeak-event']).toBe('photo.uploaded');
expect(got.headers['x-picpeak-delivery']).toBeTruthy();
expect(got.headers['user-agent']).toMatch(/PicPeak-Webhooks/);
} finally {
await stub.close();
}
});
test('on 5xx, schedules a retry with exponential backoff and stays pending', async () => {
const stub = await makeStub({ status: 500 });
try {
const { id } = await insertWebhook(stub.url);
await webhookService.fire('event.published', { event: { id: 2 } });
await __test.tick();
const row = await db('webhook_deliveries').where({ webhook_id: id }).first();
expect(row.status).toBe('pending');
expect(row.attempt_count).toBe(1);
expect(row.response_status).toBe(500);
// BACKOFF_MS[0] = 60s; next_retry_at should be ~60s in the future.
const dueIn = new Date(row.next_retry_at).getTime() - Date.now();
expect(dueIn).toBeGreaterThan(50_000);
expect(dueIn).toBeLessThan(70_000);
} finally {
await stub.close();
}
});
test('after MAX_ATTEMPTS failures, status flips to failed and the row is closed', async () => {
const stub = await makeStub({ status: 500 });
try {
const { id } = await insertWebhook(stub.url);
// Pre-seed a delivery already at attempt_count = 4 so a single tick
// takes it to 5 → failed (avoids waiting through backoffs).
await db('webhook_deliveries').insert({
webhook_id: id,
event_type: 'event.published',
payload: JSON.stringify({ id: 'd1', type: 'event.published', data: {} }),
attempt_count: 4,
status: 'pending',
next_retry_at: new Date(),
created_at: new Date(),
});
await __test.tick();
const row = await db('webhook_deliveries').where({ webhook_id: id }).first();
expect(row.status).toBe('failed');
expect(row.attempt_count).toBe(5);
expect(row.completed_at).toBeTruthy();
expect(row.next_retry_at).toBeNull();
} finally {
await stub.close();
}
});
test('truncates response body to 1KB before storing', async () => {
const big = 'x'.repeat(5000);
const stub = await makeStub({ status: 200, bodyOverride: big });
try {
const { id } = await insertWebhook(stub.url);
await webhookService.fire('event.published', { event: {} });
await __test.tick();
const row = await db('webhook_deliveries').where({ webhook_id: id }).first();
expect(row.status).toBe('success');
expect(Buffer.byteLength(row.response_body || '', 'utf8')).toBeLessThanOrEqual(1024);
} finally {
await stub.close();
}
});
test('does not deliver to disabled webhooks (post-mortem state captured)', async () => {
const stub = await makeStub({ status: 200 });
try {
const { id } = await insertWebhook(stub.url, ['event.published'], { active: false });
// fire enqueues regardless of active state at fire-time, but we
// disabled BEFORE firing so nothing is enqueued. Direct insert to
// exercise the worker's mid-flight disable check:
await db('webhook_deliveries').insert({
webhook_id: id,
event_type: 'event.published',
payload: JSON.stringify({ id: 'd1', type: 'event.published', data: {} }),
attempt_count: 0,
status: 'pending',
next_retry_at: new Date(),
created_at: new Date(),
});
await __test.tick();
expect(stub.requests).toHaveLength(0);
const row = await db('webhook_deliveries').where({ webhook_id: id }).first();
expect(row.status).toBe('failed');
expect(row.last_error).toMatch(/disabled/i);
} finally {
await stub.close();
}
});
test('rejects loopback URLs when WEBHOOK_ALLOW_PRIVATE_URLS=false', async () => {
__test.setAllowPrivateUrls(false);
try {
const { id } = await insertWebhook('http://127.0.0.1:9/');
await db('webhook_deliveries').insert({
webhook_id: id,
event_type: 'event.published',
payload: JSON.stringify({ id: 'd1', type: 'event.published', data: {} }),
attempt_count: 0,
status: 'pending',
next_retry_at: new Date(),
created_at: new Date(),
});
await __test.tick();
const row = await db('webhook_deliveries').where({ webhook_id: id }).first();
expect(row.status).toBe('failed');
expect(row.last_error).toMatch(/private|internal/i);
} finally {
__test.setAllowPrivateUrls(true);
}
});
test('worker can be started + stopped without leaking timers', async () => {
startWebhookDeliveryWorker();
startWebhookDeliveryWorker(); // idempotent
stopWebhookDeliveryWorker();
stopWebhookDeliveryWorker(); // idempotent
// If timers leaked the test runner would warn after force-exit; assertion
// is just "no throw".
expect(true).toBe(true);
});
});
@@ -0,0 +1,80 @@
/**
* #327 — outbound webhooks (push API) for the event/photo lifecycle.
*
* Two tables:
* webhooks — admin-managed subscriptions (URL + events + secret)
* webhook_deliveries — single source of truth for the delivery worker
* (audit log + retry queue in one).
*/
exports.up = async function up(knex) {
if (!(await knex.schema.hasTable('webhooks'))) {
await knex.schema.createTable('webhooks', (table) => {
table.increments('id').primary();
table.string('name', 100).notNullable();
// Validated via networkValidation.validateExternalUrl on create + per
// delivery (DNS-rebinding mitigation).
table.string('url', 2048).notNullable();
// Plaintext signing secret (`whsec_<random>`). Stored unencrypted
// because we need to recompute HMAC-SHA256 over every outbound body
// — a hash would make the secret unrecoverable. Same posture as
// SMTP passwords stored in app_settings; protect the DB. The
// plaintext is also returned to the admin once on create so they can
// configure the receiver to verify signatures.
table.string('secret', 100).notNullable();
// First 8 chars of the secret for the admin UI so operators can
// tell which webhook is which without revealing the full secret.
table.string('secret_preview', 16).nullable();
// JSON array of subscribed event types
// (e.g. ["event.published","photo.uploaded"]).
table.jsonb('events').notNullable().defaultTo('[]');
table.boolean('active').notNullable().defaultTo(true);
table.integer('created_by').notNullable()
.references('id').inTable('admin_users').onDelete('CASCADE');
table.timestamp('created_at').defaultTo(knex.fn.now());
table.timestamp('updated_at').defaultTo(knex.fn.now());
table.timestamp('last_success_at').nullable();
table.timestamp('last_failure_at').nullable();
// Index for the delivery worker's "find subscriptions for this event"
// query — small set, but keeps the lookup constant-time as it grows.
table.index('active', 'webhooks_active_idx');
});
}
if (!(await knex.schema.hasTable('webhook_deliveries'))) {
await knex.schema.createTable('webhook_deliveries', (table) => {
table.increments('id').primary();
table.integer('webhook_id').notNullable()
.references('id').inTable('webhooks').onDelete('CASCADE');
table.string('event_type', 64).notNullable();
// Full signed payload (the JSON body that was POSTed).
table.jsonb('payload').notNullable();
table.integer('attempt_count').notNullable().defaultTo(0);
// pending → success | failed. pending rows with next_retry_at <= NOW()
// are picked up by the worker.
table.string('status', 16).notNullable().defaultTo('pending');
table.integer('response_status').nullable();
// Truncated to 1KB before storage so a verbose receiver can't blow
// up the row size.
table.text('response_body').nullable();
table.text('last_error').nullable();
table.integer('latency_ms').nullable();
table.timestamp('next_retry_at').nullable();
table.timestamp('created_at').defaultTo(knex.fn.now());
table.timestamp('completed_at').nullable();
// Worker hot-path query: WHERE status='pending' AND next_retry_at <= NOW()
// ORDER BY next_retry_at LIMIT N. This composite index serves it directly.
table.index(['status', 'next_retry_at'], 'webhook_deliveries_status_retry_idx');
table.index('webhook_id', 'webhook_deliveries_webhook_idx');
});
}
};
exports.down = async function down(knex) {
if (await knex.schema.hasTable('webhook_deliveries')) {
await knex.schema.dropTable('webhook_deliveries');
}
if (await knex.schema.hasTable('webhooks')) {
await knex.schema.dropTable('webhooks');
}
};
@@ -0,0 +1,49 @@
/**
* Adds:
* - events.allow_presigned_download — per-event opt-in for the
* presigned-URL "Download All" path (#328 follow-up). Off by default
* because it bypasses watermarks; admins flip it knowingly.
* - webhooks.filter — JSONB predicate evaluated against the payload at
* fire time (#327 follow-up). Empty object = no filter, fire always.
* - webhooks.template — optional ${dot.path} string template applied
* to the request body before signing. NULL = use the default JSON
* envelope (back-compat).
*/
exports.up = async function up(knex) {
if (await knex.schema.hasTable('events')) {
const hasCol = await knex.schema.hasColumn('events', 'allow_presigned_download');
if (!hasCol) {
await knex.schema.alterTable('events', (table) => {
table.boolean('allow_presigned_download').notNullable().defaultTo(false);
});
}
}
if (await knex.schema.hasTable('webhooks')) {
const hasFilter = await knex.schema.hasColumn('webhooks', 'filter');
if (!hasFilter) {
await knex.schema.alterTable('webhooks', (table) => {
table.jsonb('filter').notNullable().defaultTo('{}');
});
}
const hasTemplate = await knex.schema.hasColumn('webhooks', 'template');
if (!hasTemplate) {
await knex.schema.alterTable('webhooks', (table) => {
table.text('template').nullable();
});
}
}
};
exports.down = async function down(knex) {
if (await knex.schema.hasColumn('webhooks', 'template')) {
await knex.schema.alterTable('webhooks', (t) => t.dropColumn('template'));
}
if (await knex.schema.hasColumn('webhooks', 'filter')) {
await knex.schema.alterTable('webhooks', (t) => t.dropColumn('filter'));
}
if (await knex.schema.hasColumn('events', 'allow_presigned_download')) {
await knex.schema.alterTable('events', (t) => t.dropColumn('allow_presigned_download'));
}
};
+1
View File
@@ -10,6 +10,7 @@
"migrate:safe": "node migrations/run-migrations-safe.js",
"generate:watermarks": "node scripts/generate-watermarks.js",
"test": "jest",
"test:s3": "SKIP_S3_TESTS=false jest __tests__/integration/backup-s3",
"lint": "eslint src/"
},
"dependencies": {
+259
View File
@@ -0,0 +1,259 @@
#!/usr/bin/env node
/**
* migrate-storage.js
*
* One-shot migration tool to copy every PicPeak content file from the local
* filesystem (the legacy STORAGE_PATH) to a configured S3-compatible bucket.
*
* Reads the relative path of each known asset from the database:
* photos.path
* photos.thumbnail_path
* photos.hero_path
* photos.watermark_path
* events.archive_path
* events.download_zip_path
*
* For each, streams from local fs → S3, skipping files whose sha256 already
* matches a previously uploaded object (idempotent — safe to re-run).
*
* Does NOT flip STORAGE_BACKEND. After the migration completes clean, the
* operator updates their environment + restarts the backend explicitly.
*
* Usage:
* node backend/scripts/migrate-storage.js # live migration
* node backend/scripts/migrate-storage.js --dry-run # report only, no uploads
* node backend/scripts/migrate-storage.js --failures-csv=/path/to/failures.csv
* node backend/scripts/migrate-storage.js --concurrency=4
*
* Required env (S3 destination — same vars the backend reads with STORAGE_BACKEND=s3):
* STORAGE_S3_BUCKET, STORAGE_S3_REGION, STORAGE_S3_ACCESS_KEY, STORAGE_S3_SECRET_KEY
* STORAGE_S3_ENDPOINT (optional — for MinIO/R2/etc.)
* STORAGE_S3_PREFIX (optional)
*
* STORAGE_PATH must point at the live local storage root. Postgres connection
* uses the same DB env vars the backend uses.
*/
require('dotenv').config();
const fs = require('fs');
const fsp = require('fs').promises;
const path = require('path');
const crypto = require('crypto');
const { db } = require('../src/database/db');
const LocalFsStorage = require('../src/services/storage/LocalFsStorage');
const S3StorageBackend = require('../src/services/storage/S3StorageBackend');
const logger = require('../src/utils/logger');
function parseArgs(argv) {
const args = { dryRun: false, concurrency: 4, failuresCsv: '/tmp/migrate-storage-failures.csv' };
for (const arg of argv) {
if (arg === '--dry-run') args.dryRun = true;
else if (arg.startsWith('--concurrency=')) args.concurrency = Math.max(1, parseInt(arg.split('=')[1], 10) || 4);
else if (arg.startsWith('--failures-csv=')) args.failuresCsv = arg.split('=')[1];
else if (arg === '--help' || arg === '-h') {
console.log('Usage: node migrate-storage.js [--dry-run] [--concurrency=N] [--failures-csv=PATH]');
process.exit(0);
}
}
return args;
}
function buildLocalSource() {
const root = process.env.STORAGE_PATH;
if (!root) {
throw new Error('STORAGE_PATH must be set to the local storage root.');
}
return new LocalFsStorage({ root });
}
function buildS3Destination() {
const required = ['STORAGE_S3_BUCKET', 'STORAGE_S3_ACCESS_KEY', 'STORAGE_S3_SECRET_KEY'];
const missing = required.filter((v) => !process.env[v]);
if (missing.length) {
throw new Error(`Missing S3 env vars: ${missing.join(', ')}`);
}
return new S3StorageBackend({
bucket: process.env.STORAGE_S3_BUCKET,
region: process.env.STORAGE_S3_REGION || 'us-east-1',
endpoint: process.env.STORAGE_S3_ENDPOINT,
accessKeyId: process.env.STORAGE_S3_ACCESS_KEY,
secretAccessKey: process.env.STORAGE_S3_SECRET_KEY,
prefix: process.env.STORAGE_S3_PREFIX,
forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true' ? true : undefined,
sslEnabled: process.env.STORAGE_S3_SSL !== 'false',
});
}
async function sha256OfFile(localPath) {
return new Promise((resolve, reject) => {
const hash = crypto.createHash('sha256');
const stream = fs.createReadStream(localPath);
stream.on('data', (chunk) => hash.update(chunk));
stream.on('end', () => resolve(hash.digest('hex')));
stream.on('error', reject);
});
}
async function collectKeys() {
const keys = new Map(); // key -> { source, contentType }
const addKey = (key, source) => {
if (!key) return;
const normalized = key.replace(/\\/g, '/').replace(/^\/+/, '');
if (!normalized) return;
if (!keys.has(normalized)) keys.set(normalized, { source });
};
// photos: path (events/active/{slug}/{filename}), thumbnail_path, hero_path, watermark_path
const photoBatch = await db('photos').select('id', 'path', 'thumbnail_path', 'hero_path', 'watermark_path');
for (const p of photoBatch) {
if (p.path) {
const photoKey = p.path.startsWith('events/active/') ? p.path : path.posix.join('events/active', p.path);
addKey(photoKey, `photos.path[${p.id}]`);
}
addKey(p.thumbnail_path, `photos.thumbnail_path[${p.id}]`);
addKey(p.hero_path, `photos.hero_path[${p.id}]`);
addKey(p.watermark_path, `photos.watermark_path[${p.id}]`);
}
// events: archive_path, download_zip_path
const eventBatch = await db('events').select('id', 'archive_path', 'download_zip_path');
for (const e of eventBatch) {
addKey(e.archive_path, `events.archive_path[${e.id}]`);
addKey(e.download_zip_path, `events.download_zip_path[${e.id}]`);
}
return keys;
}
async function migrateOne(key, meta, { source, dest, dryRun }) {
// Source must exist on local disk.
const localPath = source.resolveLocalPath(key);
let localStat;
try {
localStat = await fsp.stat(localPath);
} catch (err) {
if (err.code === 'ENOENT') {
return { key, status: 'missing-locally', source: meta.source };
}
throw err;
}
// Idempotent skip: if S3 already has matching size + sha256.
const remoteStat = await dest.stat(key);
if (remoteStat && remoteStat.size === localStat.size) {
// sha256 match check via metadata is expensive; we trust size match for now.
// Operators paranoid about content drift can `rm` the bucket and re-run.
return { key, status: 'already-uploaded', source: meta.source };
}
if (dryRun) {
return { key, status: 'would-upload', source: meta.source, size: localStat.size };
}
await dest.putFromFile(key, localPath);
const verify = await dest.stat(key);
if (!verify || verify.size !== localStat.size) {
return { key, status: 'size-mismatch-after-upload', source: meta.source, expected: localStat.size, got: verify?.size };
}
return { key, status: 'uploaded', source: meta.source, size: localStat.size };
}
async function processWithConcurrency(items, concurrency, fn) {
const results = [];
let i = 0;
const workers = Array.from({ length: concurrency }, async () => {
while (true) {
const idx = i++;
if (idx >= items.length) return;
const [key, meta] = items[idx];
try {
const r = await fn(key, meta);
results.push(r);
} catch (err) {
results.push({ key, status: 'error', source: meta.source, error: err.message });
}
}
});
await Promise.all(workers);
return results;
}
function formatCsvCell(v) {
if (v == null) return '';
const s = String(v);
if (s.includes(',') || s.includes('"') || s.includes('\n')) {
return `"${s.replace(/"/g, '""')}"`;
}
return s;
}
async function writeFailuresCsv(filePath, failures) {
if (failures.length === 0) {
// Touch an empty file with header so callers see a deterministic outcome.
await fsp.writeFile(filePath, 'key,source,status,error\n');
return;
}
const lines = ['key,source,status,error'];
for (const f of failures) {
lines.push([f.key, f.source, f.status, f.error || ''].map(formatCsvCell).join(','));
}
await fsp.writeFile(filePath, lines.join('\n') + '\n');
}
async function main() {
const args = parseArgs(process.argv.slice(2));
logger.info(`migrate-storage starting (dry-run=${args.dryRun}, concurrency=${args.concurrency})`);
const source = buildLocalSource();
await source.init();
const dest = buildS3Destination();
await dest.init();
logger.info('collecting key list from database…');
const keys = await collectKeys();
logger.info(`found ${keys.size} unique keys to process`);
const items = Array.from(keys.entries());
const results = await processWithConcurrency(items, args.concurrency, (key, meta) =>
migrateOne(key, meta, { source, dest, dryRun: args.dryRun })
);
const counts = results.reduce((acc, r) => {
acc[r.status] = (acc[r.status] || 0) + 1;
return acc;
}, {});
console.log('\n=== migrate-storage summary ===');
for (const [status, count] of Object.entries(counts).sort()) {
console.log(` ${status.padEnd(28)} ${count}`);
}
const failureStatuses = new Set(['error', 'missing-locally', 'size-mismatch-after-upload']);
const failures = results.filter((r) => failureStatuses.has(r.status));
await writeFailuresCsv(args.failuresCsv, failures);
if (failures.length > 0) {
console.log(`\nWrote ${failures.length} failures to ${args.failuresCsv}`);
console.log('Re-run with --dry-run to triage; fix sources or remove DB rows that point at missing files.');
process.exitCode = 1;
} else if (args.dryRun) {
console.log(`\nDry-run complete. Re-run without --dry-run to perform the migration.`);
console.log(`(Empty failures CSV written to ${args.failuresCsv}.)`);
} else {
console.log(`\nMigration complete. Update STORAGE_BACKEND=s3 + restart the backend to switch over.`);
}
await db.destroy();
}
main().catch(async (err) => {
console.error('migrate-storage failed:', err);
try { await db.destroy(); } catch (_) { /* ignore */ }
process.exit(2);
});
+15
View File
@@ -533,6 +533,7 @@ app.use('/api/admin/events', require('./src/routes/adminEventRename'));
app.use('/api/admin/users', require('./src/routes/adminUsers'));
app.use('/api/admin/event-types', require('./src/routes/adminEventTypes'));
app.use('/api/admin/api-tokens', require('./src/routes/adminApiTokens'));
app.use('/api/admin/webhooks', require('./src/routes/adminWebhooks'));
// Public v1 API for n8n / external integrations (#322). Mounted under
// /api/v1; auth handled per-route via apiTokenAuth (Bearer tokens).
app.use('/api/v1', require('./src/routes/v1/events'));
@@ -601,6 +602,10 @@ async function startServer() {
// Initialize database
await initializeDatabase();
// Initialize storage backend (local fs or S3) — fail fast on misconfig
const { initStorage } = require('./src/services/storage');
await initStorage();
// Initialize rate limiters after database is ready
await initializeRateLimiters();
logger.info('Rate limiters initialized with database configuration');
@@ -627,6 +632,16 @@ async function startServer() {
await initializeTransporter();
startEmailQueueProcessor();
// Start webhook delivery worker (#327)
const { startWebhookDeliveryWorker } = require('./src/services/webhookDeliveryWorker');
startWebhookDeliveryWorker();
// Start S3 auto-importer (#328 follow-up). No-op when STORAGE_AUTO_IMPORT
// is unset OR STORAGE_BACKEND=local — replaces the chokidar watcher
// for S3-mode deployments that drop files into the bucket directly.
const { startS3AutoImporter } = require('./src/services/s3AutoImporter');
startS3AutoImporter();
// Start backup service
await startBackupService();
+41 -5
View File
@@ -298,6 +298,9 @@ router.post('/', adminAuth, requirePermission('events.create'), [
body('enable_devtools_protection').optional().isBoolean(),
body('watermark_downloads').optional().isBoolean(),
body('watermark_text').optional().trim(),
// #328 follow-up: per-event opt-in for presigned-URL "Download All".
// Bypasses watermarks; admin must enable knowingly.
body('allow_presigned_download').optional().isBoolean(),
body('css_template_id').optional({ nullable: true, checkFalsy: true }).isInt(),
// Hero logo settings
body('hero_logo_visible').optional().isBoolean(),
@@ -344,6 +347,7 @@ router.post('/', adminAuth, requirePermission('events.create'), [
enable_devtools_protection: enableDevtoolsProtectionInput,
watermark_downloads = false,
watermark_text = null,
allow_presigned_download = false,
require_password: requirePasswordInput,
// Feedback settings
feedback_enabled = false,
@@ -557,6 +561,7 @@ router.post('/', adminAuth, requirePermission('events.create'), [
enable_devtools_protection: formatBoolean(effectiveEnableDevtoolsProtection),
watermark_downloads: formatBoolean(watermark_downloads !== undefined ? watermark_downloads : false),
watermark_text,
allow_presigned_download: formatBoolean(allow_presigned_download === true || allow_presigned_download === 'true'),
require_password: formatBoolean(requirePassword),
css_template_id: css_template_id || null,
hero_logo_visible: formatBoolean(effectiveHeroLogoVisible),
@@ -597,12 +602,21 @@ router.post('/', adminAuth, requirePermission('events.create'), [
}
// Log activity
await logActivity('event_created',
{ event_type, expires_at, require_password: requirePassword, password_strength: passwordValidation?.score },
eventId,
await logActivity('event_created',
{ event_type, expires_at, require_password: requirePassword, password_strength: passwordValidation?.score },
eventId,
{ type: 'admin', id: req.admin.id, name: req.admin.username }
);
// Fire event.created webhook (#327). If the event is being published
// immediately (not a draft), event.published also fires below.
try {
const webhookService = require('../services/webhookService');
await webhookService.fire('event.created', {
event: { id: eventId, slug, event_name, event_type, event_date, is_draft: parseBooleanInput(is_draft, true) },
});
} catch (e) { /* webhookService.fire never throws but be defensive */ }
// Queue creation email (only if there is a recipient and event is not a draft)
// Language detection is handled by email processor
const isDraft = parseBooleanInput(is_draft, true);
@@ -639,7 +653,19 @@ router.post('/', adminAuth, requirePermission('events.create'), [
// scheduled_at will use default value
});
}
// Fire event.published when the event is created NOT as a draft. The
// separate /publish endpoint fires it for the draft → live transition;
// this covers the "create-and-publish in one shot" path.
if (!isDraft) {
try {
const webhookService = require('../services/webhookService');
await webhookService.fire('event.published', {
event: { id: eventId, slug, event_name, share_url: shareUrl },
});
} catch (e) { /* non-fatal */ }
}
res.json({
id: eventId,
slug,
@@ -875,6 +901,15 @@ router.post('/:id/publish', adminAuth, requirePermission('events.edit'), require
{ type: 'admin', id: req.admin.id, name: req.admin.username }
);
// Fire event.published webhook (#327) — draft → live transition.
try {
const webhookService = require('../services/webhookService');
const { shareUrl } = await buildShareLinkVariants({ slug: event.slug, shareToken: event.share_token });
await webhookService.fire('event.published', {
event: { id: parseInt(id, 10), slug: event.slug, event_name: event.event_name, share_url: shareUrl },
});
} catch (e) { /* non-fatal */ }
res.json({ message: 'Event published successfully', is_draft: false });
} catch (error) {
logger.error('Error publishing event:', { error: error.message });
@@ -912,6 +947,7 @@ router.put('/:id', adminAuth, requirePermission('events.edit'), requireEventOwne
body('disable_right_click').optional().isBoolean(),
body('watermark_downloads').optional().isBoolean(),
body('watermark_text').optional().trim(),
body('allow_presigned_download').optional().isBoolean(),
body('source_mode').optional().isIn(['managed', 'reference']),
body('external_path').optional({ nullable: true }).isString().trim(),
body('require_password').optional().isBoolean(),
+158 -100
View File
@@ -17,6 +17,7 @@ const watermarkGeneratorService = require('../services/watermarkGeneratorService
const downloadZipService = require('../services/downloadZipService');
const { findReplacementCandidate, replacePhoto } = require('../services/photoReplacementService');
const { requireEventOwnership } = require('../middleware/ownership');
const { getStorage } = require('../services/storage');
const router = express.Router();
// Get storage path from environment or default
@@ -243,9 +244,9 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r
categoryName = 'collages';
}
// Create final destination directory
const finalDestPath = path.join(getStoragePath(), 'events/active', event.slug);
await fs.mkdir(finalDestPath, { recursive: true });
// Final destination key prefix under the storage backend (no local mkdir
// needed — LocalFsStorage creates the parent dir on put, S3 has no dirs).
const finalDestPathRel = path.posix.join('events/active', event.slug);
const uploadedPhotos = [];
const replacedPhotos = [];
@@ -330,10 +331,11 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r
extension
);
// Calculate final path
const finalPath = path.join(finalDestPath, newFilename);
const storagePath = getStoragePath();
const relativePath = path.relative(path.join(storagePath, 'events/active'), finalPath);
// Storage key: events/active/{slug}/{newFilename}
const finalKey = path.posix.join(finalDestPathRel, newFilename);
// photo.path is stored relative to events/active so resolvePhotoStorageKey
// can rebuild the full key on read.
const relativePath = path.posix.join(event.slug, newFilename);
// Extract capture date from EXIF metadata
let capturedAt = null;
@@ -365,10 +367,10 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r
batchPhotos.push(photoData);
// Store move operation for later
// Store upload operation for later (after DB commit)
fileRenameOperations.push({
tempPath: tempPath,
finalPath: finalPath,
finalKey: finalKey,
filename: newFilename,
photoData: photoData
});
@@ -390,34 +392,26 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r
await trx.commit();
console.log(`Successfully committed batch of ${batchPhotos.length} photos`);
// Now move files from temp to final location after successful commit
// Now upload files from temp into the storage backend after successful commit
const storage = getStorage();
for (let idx = 0; idx < fileRenameOperations.length; idx++) {
const operation = fileRenameOperations[idx];
try {
// Move the file from temp to final location
await fs.rename(operation.tempPath, operation.finalPath);
console.log(`Moved file from ${operation.tempPath} to ${operation.finalPath}`);
// Verify the file was moved successfully
const finalStats = await fs.stat(operation.finalPath);
if (finalStats.size !== operation.photoData.size_bytes) {
throw new Error(`File size mismatch after move: expected ${operation.photoData.size_bytes}, got ${finalStats.size}`);
}
// Generate thumbnail and extract metadata
// Process source-dependent steps (sharp/ffmpeg) FIRST while the
// tmp file is still on local disk, then upload the original and
// unlink the tmp.
const photoId = insertedIds[idx]?.id || insertedIds[idx];
const isVideoFile = isVideoMimeType(operation.photoData.mime_type);
let thumbnailPath = null;
try {
if (isVideoFile) {
// Process video: extract metadata and generate thumbnail
const thumbnailDir = path.join(getStoragePath(), 'thumbnails');
await fs.mkdir(thumbnailDir, { recursive: true });
const videoThumbnailPath = path.join(thumbnailDir, `thumb_${operation.filename.replace(/\.[^.]+$/, '.jpg')}`);
const result = await processUploadedVideo(operation.finalPath, videoThumbnailPath);
thumbnailPath = path.relative(getStoragePath(), videoThumbnailPath);
const videoThumbnailKey = path.posix.join(
'thumbnails',
`thumb_${operation.filename.replace(/\.[^.]+$/, '.jpg')}`
);
const result = await processUploadedVideo(operation.tempPath, videoThumbnailKey);
thumbnailPath = result.thumbnailKey;
if (photoId && result.metadata) {
await db('photos')
@@ -432,7 +426,7 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r
});
}
} else {
thumbnailPath = await generateThumbnail(operation.finalPath);
thumbnailPath = await generateThumbnail(operation.tempPath);
// Update the database with thumbnail path and image dimensions
if (photoId) {
@@ -441,7 +435,7 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r
try {
const sharp = require('sharp');
const metadata = await sharp(operation.finalPath).metadata();
const metadata = await sharp(operation.tempPath).metadata();
if (metadata.width && metadata.height) {
updateData.width = metadata.width;
updateData.height = metadata.height;
@@ -461,12 +455,40 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r
console.error(`Thumbnail/metadata processing failed for ${operation.filename}:`, thumbError.message);
}
// Upload the original through the storage backend, then drop the
// local tmp file. We do this AFTER thumbnail/metadata processing
// so sharp/ffmpeg still have a local source to work from.
await storage.putFromFile(operation.finalKey, operation.tempPath, {
contentType: operation.photoData.mime_type,
});
await fs.unlink(operation.tempPath).catch(() => {});
// Sanity check: round-trip the size we just wrote.
const stat = await storage.stat(operation.finalKey);
if (!stat || stat.size !== operation.photoData.size_bytes) {
throw new Error(`Size mismatch after upload: expected ${operation.photoData.size_bytes}, got ${stat ? stat.size : 'null'}`);
}
// Queue watermark generation in background (non-blocking, images only)
if (photoId && !isVideoFile) {
watermarkGeneratorService.generateForPhoto(photoId)
.catch(err => console.warn(`Watermark generation queued failed for photo ${photoId}:`, err.message));
}
// Webhook (#327): per-photo upload event.
try {
const webhookService = require('../services/webhookService');
await webhookService.fire('photo.uploaded', {
event: { id: parseInt(eventId, 10), slug: event.slug, event_name: event.event_name },
photo: {
id: insertedIds[idx]?.id || insertedIds[idx],
filename: operation.filename,
original_filename: operation.photoData.original_filename,
size_bytes: operation.photoData.size_bytes,
},
});
} catch (e) { /* non-fatal */ }
// Add to successful uploads
uploadedPhotos.push({
id: insertedIds[idx]?.id || insertedIds[idx],
@@ -475,10 +497,10 @@ router.post('/:eventId/upload', adminAuth, requirePermission('photos.upload'), r
category_id: operation.photoData.category_id
});
} catch (moveError) {
console.error(`Failed to move file ${operation.tempPath} to ${operation.finalPath}:`, moveError);
errors.push({
filename: operation.filename,
error: `File move failed: ${moveError.message}`
console.error(`Failed to upload ${operation.tempPath} ${operation.finalKey}:`, moveError);
errors.push({
filename: operation.filename,
error: `File upload failed: ${moveError.message}`
});
// Try to clean up the database entry if file move failed
@@ -604,30 +626,30 @@ router.delete('/:eventId/photos/:photoId', adminAuth, requirePermission('photos.
return res.status(404).json({ error: 'Photo not found' });
}
// Delete physical files
const storagePath = getStoragePath();
const photoPath = path.join(storagePath, 'events/active', photo.path);
// Delete original + thumbnail through the storage backend.
const storage = getStorage();
const { resolvePhotoStorageKey } = require('../services/photoResolver');
const event = await db('events').where({ id: eventId }).first();
try {
await fs.unlink(photoPath);
const originalKey = resolvePhotoStorageKey(event, photo);
if (originalKey) await storage.delete(originalKey);
} catch (error) {
console.error('Error deleting photo file:', error);
}
// Delete thumbnail if exists
// photo.thumbnail_path is stored as the canonical storage key
// (e.g. "thumbnails/thumb_foo.jpg"), so pass it through verbatim.
if (photo.thumbnail_path) {
const thumbPath = path.join(storagePath, 'events/active', photo.thumbnail_path);
try {
// Check if file exists before attempting to delete
await fs.access(thumbPath);
await fs.unlink(thumbPath);
await storage.delete(photo.thumbnail_path);
} catch (error) {
// Only log if it's not a "file not found" error
if (error.code !== 'ENOENT') {
console.error('Error deleting thumbnail:', error);
}
console.error('Error deleting thumbnail:', error);
}
}
if (photo.hero_path) {
await storage.delete(photo.hero_path).catch(() => {});
}
// Delete pre-generated watermark if exists
if (photo.watermark_path) {
@@ -636,15 +658,23 @@ router.delete('/:eventId/photos/:photoId', adminAuth, requirePermission('photos.
// Remove from database
await db('photos').where({ id: photoId }).delete();
// Log activity
const event = await db('events').where({ id: eventId }).first();
// Log activity (event was fetched above for storage key resolution)
await logActivity('photo_deleted',
{ filename: photo.filename, eventName: event.event_name },
eventId,
{ type: 'admin', id: req.admin.id, name: req.admin.username }
);
// Webhook (#327): single-photo delete.
try {
const webhookService = require('../services/webhookService');
await webhookService.fire('photo.deleted', {
event: { id: parseInt(eventId, 10), slug: event?.slug, event_name: event?.event_name },
photo: { id: parseInt(photoId, 10), filename: photo.filename },
});
} catch (e) { /* non-fatal */ }
downloadZipService.invalidate(parseInt(eventId));
res.json({ message: 'Photo deleted successfully' });
} catch (error) {
@@ -735,35 +765,25 @@ router.post('/:eventId/photos/bulk-delete', adminAuth, requirePermission('photos
return res.status(404).json({ error: 'No photos found' });
}
// Delete physical files
const storagePath = getStoragePath();
// Delete original + thumbnail + hero through the storage backend.
const storage = getStorage();
const event = await db('events').where({ id: eventId }).first();
const { resolvePhotoStorageKey } = require('../services/photoResolver');
for (const photo of photos) {
// Delete photo file
const photoPath = path.join(storagePath, 'events/active', photo.path);
try {
await fs.unlink(photoPath);
const originalKey = resolvePhotoStorageKey(event, photo);
if (originalKey) await storage.delete(originalKey);
} catch (error) {
console.error('Error deleting photo file:', error);
}
// Delete thumbnail
if (photo.thumbnail_path) {
const thumbPath = path.join(storagePath, photo.thumbnail_path);
try {
// Check if file exists before attempting to delete
await fs.access(thumbPath);
await fs.unlink(thumbPath);
} catch (error) {
// Only log if it's not a "file not found" error
if (error.code !== 'ENOENT') {
console.error('Error deleting thumbnail:', error);
}
}
}
// Delete pre-generated watermark
if (photo.thumbnail_path) {
await storage.delete(photo.thumbnail_path).catch(() => {});
}
if (photo.hero_path) {
await storage.delete(photo.hero_path).catch(() => {});
}
if (photo.watermark_path) {
await watermarkGeneratorService.deleteForPhoto(photo.id);
}
@@ -774,7 +794,18 @@ router.post('/:eventId/photos/bulk-delete', adminAuth, requirePermission('photos
.whereIn('id', photoIds)
.where('event_id', eventId)
.delete();
// Webhook (#327): one photo.deleted per row in the bulk batch.
try {
const webhookService = require('../services/webhookService');
for (const photo of photos) {
await webhookService.fire('photo.deleted', {
event: { id: parseInt(eventId, 10), slug: event?.slug, event_name: event?.event_name },
photo: { id: photo.id, filename: photo.filename },
});
}
} catch (e) { /* non-fatal */ }
// Log activity
await logActivity('photos_bulk_deleted',
{ count: photos.length, eventName: event.event_name },
@@ -866,18 +897,33 @@ router.get('/:eventId/photos/:photoId/download', adminAuth, requirePermission('p
return res.status(404).json({ error: 'Photo not found' });
}
const { resolvePhotoFilePath } = require('../services/photoResolver');
const { resolvePhotoFilePath, resolvePhotoStorageKey } = require('../services/photoResolver');
const event = await db('events').where('id', eventId).first();
const storage = getStorage();
const storageKey = resolvePhotoStorageKey(event, photo);
if (storageKey) {
const stat = await storage.stat(storageKey);
if (!stat) {
return res.status(404).json({ error: 'Photo file not found' });
}
res.set({
'Content-Type': photo.mime_type || 'application/octet-stream',
'Content-Length': stat.size,
'Content-Disposition': `attachment; filename="${photo.filename}"`,
});
const stream = await storage.get(storageKey);
stream.pipe(res);
return;
}
// External-mode photos still live on local disk.
const filePath = resolvePhotoFilePath(event, photo);
// Check if file exists
try {
await fs.access(filePath);
} catch (error) {
return res.status(404).json({ error: 'Photo file not found' });
}
// Send file
res.download(filePath, photo.filename);
} catch (error) {
console.error('Error downloading photo:', error);
@@ -1033,23 +1079,33 @@ router.get('/:eventId/photo/:photoId', adminAuth, requirePermission('photos.view
return res.status(404).json({ error: 'Photo not found' });
}
const { resolvePhotoFilePath } = require('../services/photoResolver');
const { resolvePhotoFilePath, resolvePhotoStorageKey } = require('../services/photoResolver');
const event = await db('events').where('id', eventId).first();
const storageKey = resolvePhotoStorageKey(event, photo);
res.setHeader('Content-Type', `image/${path.extname(photo.filename).slice(1)}`);
res.setHeader('Cache-Control', 'private, max-age=3600');
res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin');
if (storageKey) {
const storage = getStorage();
const stat = await storage.stat(storageKey);
if (!stat) {
return res.status(404).json({ error: 'Photo file not found' });
}
res.setHeader('Content-Length', stat.size);
const stream = await storage.get(storageKey);
stream.pipe(res);
return;
}
// External-mode photos still live on local disk.
const filePath = resolvePhotoFilePath(event, photo);
// Check if file exists
try {
await fs.access(filePath);
} catch (error) {
return res.status(404).json({ error: 'Photo file not found' });
}
// Set appropriate headers
res.setHeader('Content-Type', `image/${path.extname(photo.filename).slice(1)}`);
res.setHeader('Cache-Control', 'private, max-age=3600');
res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin');
// Send file (sendFile requires absolute path)
res.sendFile(path.resolve(filePath));
} catch (error) {
console.error('Error serving photo:', error);
@@ -1073,22 +1129,24 @@ router.get('/:eventId/thumbnail/:photoId', adminAuth, requirePermission('photos.
// Ensure thumbnail exists and is valid, regenerate if needed
const thumbnailPath = await ensureThumbnail(photo);
if (!thumbnailPath) {
console.error(`Failed to generate thumbnail for photo ${photoId}`);
return res.status(404).json({ error: 'Thumbnail generation failed' });
}
const storagePath = getStoragePath();
const filePath = path.join(storagePath, thumbnailPath);
// Set appropriate headers
res.setHeader('Content-Type', 'image/jpeg'); // Thumbnails are always JPEG
res.setHeader('Cache-Control', 'private, max-age=3600');
res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin');
// Send file (sendFile requires absolute path)
res.sendFile(path.resolve(filePath));
const storage = getStorage();
const stat = await storage.stat(thumbnailPath);
if (!stat) {
return res.status(404).json({ error: 'Thumbnail not found' });
}
res.setHeader('Content-Length', stat.size);
const stream = await storage.get(thumbnailPath);
stream.pipe(res);
} catch (error) {
console.error('Error serving thumbnail:', error);
console.error('Photo ID:', req.params.photoId);
+389
View File
@@ -0,0 +1,389 @@
/**
* Admin endpoints for managing outbound webhooks (#327). Mirrors
* adminApiTokens.js — same permission gates, same "secret shown once"
* pattern.
*
* Routes mounted under /api/admin/webhooks:
* GET / — list
* POST / — create (returns plaintext secret once)
* GET /:id — detail (no secret)
* PUT /:id — update name/url/events/active
* DELETE /:id — delete (cascades to deliveries)
* POST /:id/test — fire a synthetic delivery now
* GET /:id/deliveries — list deliveries (paginated, filter)
* GET /:id/deliveries/:deliveryId — delivery detail (payload+response)
* POST /:id/deliveries/:deliveryId/replay — re-enqueue a delivery
*/
const express = require('express');
const { body, query, validationResult } = require('express-validator');
const { db, logActivity } = require('../database/db');
const { adminAuth } = require('../middleware/auth');
const { requirePermission } = require('../middleware/permissions');
const { validateExternalUrl } = require('../utils/networkValidation');
const webhookService = require('../services/webhookService');
const logger = require('../utils/logger');
const router = express.Router();
const ALLOW_PRIVATE_URLS = process.env.WEBHOOK_ALLOW_PRIVATE_URLS === 'true';
function publicWebhook(row) {
if (!row) return null;
return {
id: row.id,
name: row.name,
url: row.url,
events: typeof row.events === 'string' ? safeJson(row.events, []) : (row.events || []),
active: row.active,
secret_preview: row.secret_preview,
filter: typeof row.filter === 'string' ? safeJson(row.filter, {}) : (row.filter || {}),
template: row.template || null,
created_by: row.created_by,
created_at: row.created_at,
updated_at: row.updated_at,
last_success_at: row.last_success_at,
last_failure_at: row.last_failure_at,
};
}
function safeJson(s, fallback) {
try { return JSON.parse(s); } catch { return fallback; }
}
// ─── List ────────────────────────────────────────────────────────────────
router.get('/', adminAuth, requirePermission('settings.view'), async (req, res) => {
try {
const rows = await db('webhooks')
.leftJoin('admin_users', 'admin_users.id', 'webhooks.created_by')
.select(
'webhooks.*',
'admin_users.username as owner_username'
)
.orderBy('webhooks.created_at', 'desc');
res.json(rows.map((r) => ({
...publicWebhook(r),
owner_username: r.owner_username,
})));
} catch (err) {
logger.error('webhooks list failed', { error: err.message });
res.status(500).json({ error: 'Failed to list webhooks' });
}
});
// ─── Create ──────────────────────────────────────────────────────────────
router.post(
'/',
adminAuth,
requirePermission('settings.edit'),
[
body('name').isString().trim().isLength({ min: 1, max: 100 }),
body('url').isString().isLength({ max: 2048 }).custom((url) => {
if (ALLOW_PRIVATE_URLS) return true;
const check = validateExternalUrl(url);
if (!check.valid) throw new Error(check.error);
return true;
}),
body('events').isArray({ min: 1 }).custom((arr) => {
const ok = arr.every((e) => webhookService.EVENT_TYPES.includes(e));
if (!ok) throw new Error(`events must be a subset of: ${webhookService.EVENT_TYPES.join(', ')}`);
return true;
}),
body('active').optional().isBoolean(),
body('filter').optional().custom((v) => {
if (v == null) return true;
if (typeof v !== 'object' || Array.isArray(v)) {
throw new Error('filter must be an object of dot-path → value pairs');
}
return true;
}),
body('template').optional({ nullable: true }).custom((v) => {
const check = webhookService.validateTemplate(v);
if (!check.valid) throw new Error(check.error);
return true;
}),
],
async (req, res) => {
try {
const errors = validationResult(req);
if (!errors.isEmpty()) return res.status(400).json({ errors: errors.array() });
const { name, url, events, active = true, filter, template } = req.body;
const { plaintext, preview } = webhookService.generateSecret();
const insertResult = await db('webhooks').insert({
name,
url,
secret: plaintext,
secret_preview: preview,
events: JSON.stringify(events),
active,
filter: JSON.stringify(filter || {}),
template: template || null,
created_by: req.admin.id,
}).returning('id');
const id = insertResult[0]?.id || insertResult[0];
await logActivity('webhook_created', { name, events }, null, {
type: 'admin', id: req.admin.id, name: req.admin.username,
});
const row = await db('webhooks').where({ id }).first();
res.status(201).json({
...publicWebhook(row),
secret: plaintext,
notice: 'Save this signing secret now — it will not be shown again.',
});
} catch (err) {
logger.error('webhooks create failed', { error: err.message });
res.status(500).json({ error: 'Failed to create webhook' });
}
}
);
// ─── Detail ──────────────────────────────────────────────────────────────
router.get('/:id', adminAuth, requirePermission('settings.view'), async (req, res) => {
try {
const row = await db('webhooks').where({ id: req.params.id }).first();
if (!row) return res.status(404).json({ error: 'Webhook not found' });
res.json(publicWebhook(row));
} catch (err) {
logger.error('webhooks detail failed', { error: err.message });
res.status(500).json({ error: 'Failed to load webhook' });
}
});
// ─── Update ──────────────────────────────────────────────────────────────
router.put(
'/:id',
adminAuth,
requirePermission('settings.edit'),
[
body('name').optional().isString().trim().isLength({ min: 1, max: 100 }),
body('url').optional().isString().isLength({ max: 2048 }).custom((url) => {
if (ALLOW_PRIVATE_URLS) return true;
const check = validateExternalUrl(url);
if (!check.valid) throw new Error(check.error);
return true;
}),
body('events').optional().isArray({ min: 1 }).custom((arr) => {
const ok = arr.every((e) => webhookService.EVENT_TYPES.includes(e));
if (!ok) throw new Error(`events must be a subset of: ${webhookService.EVENT_TYPES.join(', ')}`);
return true;
}),
body('active').optional().isBoolean(),
body('filter').optional().custom((v) => {
if (v == null) return true;
if (typeof v !== 'object' || Array.isArray(v)) {
throw new Error('filter must be an object of dot-path → value pairs');
}
return true;
}),
body('template').optional({ nullable: true }).custom((v) => {
const check = webhookService.validateTemplate(v);
if (!check.valid) throw new Error(check.error);
return true;
}),
],
async (req, res) => {
try {
const errors = validationResult(req);
if (!errors.isEmpty()) return res.status(400).json({ errors: errors.array() });
const row = await db('webhooks').where({ id: req.params.id }).first();
if (!row) return res.status(404).json({ error: 'Webhook not found' });
const updates = { updated_at: new Date() };
if ('name' in req.body) updates.name = req.body.name;
if ('url' in req.body) updates.url = req.body.url;
if ('events' in req.body) updates.events = JSON.stringify(req.body.events);
if ('active' in req.body) updates.active = req.body.active;
if ('filter' in req.body) updates.filter = JSON.stringify(req.body.filter || {});
if ('template' in req.body) updates.template = req.body.template || null;
await db('webhooks').where({ id: req.params.id }).update(updates);
const updated = await db('webhooks').where({ id: req.params.id }).first();
await logActivity('webhook_updated', { changes: Object.keys(updates) }, null, {
type: 'admin', id: req.admin.id, name: req.admin.username,
});
res.json(publicWebhook(updated));
} catch (err) {
logger.error('webhooks update failed', { error: err.message });
res.status(500).json({ error: 'Failed to update webhook' });
}
}
);
// ─── Delete ──────────────────────────────────────────────────────────────
router.delete('/:id', adminAuth, requirePermission('settings.edit'), async (req, res) => {
try {
const row = await db('webhooks').where({ id: req.params.id }).first();
if (!row) return res.status(404).json({ error: 'Webhook not found' });
await db('webhooks').where({ id: req.params.id }).delete();
await logActivity('webhook_deleted', { name: row.name }, null, {
type: 'admin', id: req.admin.id, name: req.admin.username,
});
res.json({ id: Number(req.params.id), deleted: true });
} catch (err) {
logger.error('webhooks delete failed', { error: err.message });
res.status(500).json({ error: 'Failed to delete webhook' });
}
});
// ─── Send test event ─────────────────────────────────────────────────────
router.post(
'/:id/test',
adminAuth,
requirePermission('settings.edit'),
[body('event_type').optional().isIn(webhookService.EVENT_TYPES)],
async (req, res) => {
try {
const errors = validationResult(req);
if (!errors.isEmpty()) return res.status(400).json({ errors: errors.array() });
const row = await db('webhooks').where({ id: req.params.id }).first();
if (!row) return res.status(404).json({ error: 'Webhook not found' });
if (!row.active) return res.status(400).json({ error: 'Webhook is disabled' });
const eventType = req.body.event_type || (() => {
const subscribed = typeof row.events === 'string' ? safeJson(row.events, []) : (row.events || []);
return subscribed[0] || 'event.published';
})();
// Fire a synthetic event WITHOUT writing to webhooks table — the test
// bypasses subscription matching by inserting a delivery directly.
const crypto = require('crypto');
const deliveryId = crypto.randomUUID();
const payload = {
id: deliveryId,
type: eventType,
created_at: new Date().toISOString(),
data: { test: true, fired_by: req.admin.username, webhook_id: row.id },
};
await db('webhook_deliveries').insert({
webhook_id: row.id,
event_type: eventType,
payload: JSON.stringify(payload),
attempt_count: 0,
status: 'pending',
next_retry_at: new Date(),
created_at: new Date(),
});
res.status(202).json({ enqueued: true, event_type: eventType });
} catch (err) {
logger.error('webhook test failed', { error: err.message });
res.status(500).json({ error: 'Failed to enqueue test event' });
}
}
);
// ─── List deliveries ─────────────────────────────────────────────────────
router.get(
'/:id/deliveries',
adminAuth,
requirePermission('settings.view'),
[
query('status').optional().isIn(['pending', 'success', 'failed']),
query('page').optional().isInt({ min: 1 }),
query('limit').optional().isInt({ min: 1, max: 100 }),
],
async (req, res) => {
try {
const errors = validationResult(req);
if (!errors.isEmpty()) return res.status(400).json({ errors: errors.array() });
const webhookId = req.params.id;
const exists = await db('webhooks').where({ id: webhookId }).first();
if (!exists) return res.status(404).json({ error: 'Webhook not found' });
const page = parseInt(req.query.page || '1', 10);
const limit = parseInt(req.query.limit || '25', 10);
const offset = (page - 1) * limit;
let q = db('webhook_deliveries').where({ webhook_id: webhookId });
if (req.query.status) q = q.where({ status: req.query.status });
const totalRow = await q.clone().count('id as count').first();
const total = parseInt(totalRow?.count || 0, 10);
const rows = await q
.select(
'id', 'event_type', 'attempt_count', 'status', 'response_status',
'latency_ms', 'next_retry_at', 'created_at', 'completed_at', 'last_error'
)
.orderBy('created_at', 'desc')
.limit(limit)
.offset(offset);
res.json({ deliveries: rows, pagination: { page, limit, total } });
} catch (err) {
logger.error('deliveries list failed', { error: err.message });
res.status(500).json({ error: 'Failed to list deliveries' });
}
}
);
// ─── Delivery detail ─────────────────────────────────────────────────────
router.get(
'/:id/deliveries/:deliveryId',
adminAuth,
requirePermission('settings.view'),
async (req, res) => {
try {
const row = await db('webhook_deliveries')
.where({ id: req.params.deliveryId, webhook_id: req.params.id })
.first();
if (!row) return res.status(404).json({ error: 'Delivery not found' });
res.json({
...row,
payload: typeof row.payload === 'string' ? safeJson(row.payload, row.payload) : row.payload,
});
} catch (err) {
logger.error('delivery detail failed', { error: err.message });
res.status(500).json({ error: 'Failed to load delivery' });
}
}
);
// ─── Replay ──────────────────────────────────────────────────────────────
router.post(
'/:id/deliveries/:deliveryId/replay',
adminAuth,
requirePermission('settings.edit'),
async (req, res) => {
try {
const row = await db('webhook_deliveries')
.where({ id: req.params.deliveryId, webhook_id: req.params.id })
.first();
if (!row) return res.status(404).json({ error: 'Delivery not found' });
// Re-enqueue: copy the original payload + event_type into a new row
// marked pending. Preserves the audit log of the original attempt.
const crypto = require('crypto');
const newPayload = (() => {
const obj = typeof row.payload === 'string' ? safeJson(row.payload, {}) : row.payload || {};
// Replays get a fresh delivery id but keep the event payload data.
return JSON.stringify({ ...obj, id: crypto.randomUUID(), replayed_from: row.id });
})();
const insertResult = await db('webhook_deliveries').insert({
webhook_id: row.webhook_id,
event_type: row.event_type,
payload: newPayload,
attempt_count: 0,
status: 'pending',
next_retry_at: new Date(),
created_at: new Date(),
}).returning('id');
const newId = insertResult[0]?.id || insertResult[0];
res.status(202).json({ enqueued: true, original_id: row.id, replay_id: newId });
} catch (err) {
logger.error('delivery replay failed', { error: err.message });
res.status(500).json({ error: 'Failed to replay delivery' });
}
}
);
module.exports = router;
+12
View File
@@ -190,6 +190,18 @@ router.post('/', adminAuth, [
welcome_message: welcome_message || ''
});
// Webhook lifecycle (#327). Legacy public endpoint — events go live
// immediately so created + published fire together.
try {
const webhookService = require('../services/webhookService');
await webhookService.fire('event.created', {
event: { id: eventId, slug, event_name, event_type, event_date, share_url: shareUrl },
});
await webhookService.fire('event.published', {
event: { id: eventId, slug, event_name, share_url: shareUrl },
});
} catch (e) { /* non-fatal */ }
res.json({
id: eventId,
slug,
+81 -47
View File
@@ -15,6 +15,7 @@ const { handleAsync } = require('../utils/routeHelpers');
const { NotFoundError } = require('../utils/errors');
const { ensureThumbnail, ensureHeroImage } = require('../services/imageProcessor');
const downloadZipService = require('../services/downloadZipService');
const { getStorage } = require('../services/storage');
const fs = require('fs');
// Get storage path from environment or default
@@ -629,10 +630,39 @@ router.get('/:slug/download-all', verifyGalleryAccess, async (req, res) => {
// Try to serve pre-generated zip (instant download with Content-Length)
const zipInfo = await downloadZipService.getZipInfo(req.event.id);
if (zipInfo) {
const storage = getStorage();
// Per-event presigned-URL fast path (#328 follow-up). Conditions:
// 1. STORAGE_BACKEND=s3 (presigned URLs are S3-only)
// 2. event.allow_presigned_download is true (admin opted in)
// 3. Watermarking is OFF for this event — presigned URLs bypass the
// backend, which means no watermark on bytes leaving S3.
// Falls through to streaming on any condition mismatch.
const wantsPresigned = req.event.allow_presigned_download === true || req.event.allow_presigned_download === 1;
const watermarkOnEvent = req.event.watermark_downloads === true || req.event.watermark_downloads === 1;
if (wantsPresigned && storage.kind() === 's3' && !watermarkOnEvent) {
try {
const url = await storage.signedUrl(zipInfo.key, 300); // 5 min
db('access_logs').insert({
event_id: req.event.id,
ip_address: req.ip,
user_agent: req.headers['user-agent'],
action: 'download_all_presigned'
}).catch(() => {});
res.redirect(302, url);
return;
} catch (err) {
logger.warn('presigned download-all failed, falling back to stream', {
eventId: req.event.id,
error: err.message,
});
}
}
res.setHeader('Content-Type', 'application/zip');
res.setHeader('Content-Length', zipInfo.size);
res.setHeader('Content-Disposition', `attachment; filename="${req.event.slug}.zip"`);
const stream = fs.createReadStream(zipInfo.path);
const stream = await storage.get(zipInfo.key);
stream.pipe(res);
// Log bulk download
@@ -686,46 +716,49 @@ router.get('/:slug/download-all', verifyGalleryAccess, async (req, res) => {
text: req.event.watermark_text || watermarkSettings?.text || 'Protected'
} : null;
// Add photos to archive
// Add photos to archive — managed photos via storage backend, external via local path.
const { resolvePhotoStorageKey } = require('../services/photoResolver');
const storage = getStorage();
for (const photo of photos) {
let filePath;
try {
filePath = resolvePhotoFilePath(req.event, photo);
} catch (resolveError) {
logger.warn('Skipping photo in bulk download due to unresolved path', {
slug: req.params.slug,
photoId: photo.id,
eventId: req.event.id,
error: resolveError.message,
});
continue;
}
// Determine the file name in the archive
const storageKey = resolvePhotoStorageKey(req.event, photo);
let archiveName;
if (hasMultipleTypes) {
// Use photo type as folder
const folderName = photo.type === 'individual' ? 'Individual Photos' : 'Collages';
archiveName = path.join(folderName, photo.filename);
} else {
// No folders, just the filename
archiveName = photo.filename;
}
if (shouldApplyWatermark && effectiveSettings) {
try {
const watermarkedBuffer = await watermarkService.applyWatermark(filePath, effectiveSettings);
try {
if (shouldApplyWatermark && effectiveSettings) {
// Watermark service operates on a local path. For managed photos in
// S3 mode, materialize a tmp local copy first.
const { withLocalCopy } = require('../services/imageProcessor');
const sourceForWatermark = storageKey
? null
: resolvePhotoFilePath(req.event, photo);
const watermarkedBuffer = storageKey
? await withLocalCopy(storageKey, (localPath) =>
watermarkService.applyWatermark(localPath, effectiveSettings)
)
: await watermarkService.applyWatermark(sourceForWatermark, effectiveSettings);
archive.append(watermarkedBuffer, { name: archiveName });
} catch (watermarkError) {
logger.warn('Failed to watermark photo for bulk download, skipping original to avoid leak', {
slug: req.params.slug,
photoId: photo.id,
eventId: req.event.id,
error: watermarkError.message,
});
} else if (storageKey) {
const stream = await storage.get(storageKey);
archive.append(stream, { name: archiveName });
} else {
const filePath = resolvePhotoFilePath(req.event, photo);
archive.file(filePath, { name: archiveName });
}
} else {
archive.file(filePath, { name: archiveName });
} catch (err) {
logger.warn('Skipping photo in bulk download due to error', {
slug: req.params.slug,
photoId: photo.id,
eventId: req.event.id,
error: err.message,
});
}
}
@@ -811,31 +844,32 @@ router.post('/:slug/download-selected', verifyGalleryAccess, async (req, res) =>
text: req.event.watermark_text || watermarkSettings?.text || 'Protected'
} : null;
const { resolvePhotoStorageKey: resolveSelectedKey } = require('../services/photoResolver');
const { withLocalCopy: withSelectedLocalCopy } = require('../services/imageProcessor');
const selectedStorage = getStorage();
for (const photo of photos) {
const name = photo.filename || `photo-${photo.id}.jpg`;
const storageKey = resolveSelectedKey(req.event, photo);
try {
const filePath = resolvePhotoFilePath(req.event, photo);
const name = photo.filename || `photo-${photo.id}.jpg`;
if (shouldApplyWatermark && effectiveSettings) {
try {
const watermarkedBuffer = await watermarkService.applyWatermark(filePath, effectiveSettings);
archive.append(watermarkedBuffer, { name });
} catch (watermarkError) {
logger.warn('Failed to watermark selected photo, skipping original to avoid leak', {
slug: req.params.slug,
photoId: photo.id,
eventId: req.event.id,
error: watermarkError.message,
});
}
const buf = storageKey
? await withSelectedLocalCopy(storageKey, (lp) =>
watermarkService.applyWatermark(lp, effectiveSettings)
)
: await watermarkService.applyWatermark(resolvePhotoFilePath(req.event, photo), effectiveSettings);
archive.append(buf, { name });
} else if (storageKey) {
const stream = await selectedStorage.get(storageKey);
archive.append(stream, { name });
} else {
archive.file(filePath, { name });
archive.file(resolvePhotoFilePath(req.event, photo), { name });
}
} catch (resolveError) {
logger.warn('Skipping selected photo due to unresolved path', {
} catch (err) {
logger.warn('Skipping selected photo due to error', {
slug: req.params.slug,
photoId: photo.id,
eventId: req.event.id,
error: resolveError.message,
error: err.message,
});
}
}
+30 -18
View File
@@ -1,11 +1,12 @@
const express = require('express');
const path = require('path');
const { db } = require('../database/db');
const { formatBoolean } = require('../utils/dbCompat');
const { verifyGalleryAccess } = require('../middleware/gallery');
const watermarkService = require('../services/watermarkService');
const secureImageService = require('../services/secureImageService');
const { getStoragePath } = require('../config/storage');
const { getStorage } = require('../services/storage');
const { resolvePhotoStorageKey, resolvePhotoFilePath } = require('../services/photoResolver');
const { withLocalCopy } = require('../services/imageProcessor');
const crypto = require('crypto');
const router = express.Router();
@@ -98,11 +99,11 @@ router.get('/:slug/photo/:photoId/view', verifyGalleryAccess, async (req, res) =
fragmentImage: eventProtectionLevel === 'maximum'
};
// Build full path to photo
const photoPath = path.join(getStoragePath(), 'events/active', req.event.slug, photo.path);
// Resolve photo location through the storage backend (managed) or local
// disk (external reference mode).
const storageKey = resolvePhotoStorageKey(req.event, photo);
const storage = getStorage();
// For basic/standard protection without special features, serve original file
// This avoids unnecessary recompression
const needsProcessing = eventProtectionLevel === 'enhanced' ||
eventProtectionLevel === 'maximum' ||
protectionSettings.addFingerprint;
@@ -110,15 +111,25 @@ router.get('/:slug/photo/:photoId/view', verifyGalleryAccess, async (req, res) =
let finalImage;
if (!needsProcessing) {
// Serve original file without processing
const fs = require('fs').promises;
finalImage = await fs.readFile(photoPath);
// Serve original bytes via the storage backend (or local disk for external).
if (storageKey) {
const stream = await storage.get(storageKey);
const chunks = [];
for await (const chunk of stream) chunks.push(chunk);
finalImage = Buffer.concat(chunks);
} else {
const fs = require('fs').promises;
finalImage = await fs.readFile(resolvePhotoFilePath(req.event, photo));
}
} else {
// Process image with protection measures
const processedImage = await secureImageService.processProtectedImage(photoPath, protectionSettings);
// secureImageService.processProtectedImage operates on a local path.
// Materialize a tmp local copy in S3 mode, then run processing.
const runProcessing = (lp) => secureImageService.processProtectedImage(lp, protectionSettings);
const processedImage = storageKey
? await withLocalCopy(storageKey, runProcessing)
: await runProcessing(resolvePhotoFilePath(req.event, photo));
if (processedImage.type === 'fragmented') {
// Return fragmented image data for canvas reconstruction
return res.json({
type: 'fragmented',
fragments: processedImage.fragments.map(f => ({
@@ -273,12 +284,13 @@ router.get('/:slug/photo/:photoId/signed/:token', async (req, res) => {
// Get watermark settings
const watermarkSettings = await watermarkService.getWatermarkSettings();
// Build full path to photo
const photoPath = path.join(getStoragePath(), 'events/active', event.slug, photo.path);
// Apply watermark if enabled
const imageBuffer = await watermarkService.applyWatermark(photoPath, watermarkSettings);
// Apply watermark — managed photos are sourced via the storage backend
// (S3 mode materializes a tmp local copy via withLocalCopy).
const storageKey = resolvePhotoStorageKey(event, photo);
const imageBuffer = storageKey
? await withLocalCopy(storageKey, (lp) => watermarkService.applyWatermark(lp, watermarkSettings))
: await watermarkService.applyWatermark(resolvePhotoFilePath(event, photo), watermarkSettings);
// Set appropriate headers
res.set({
+44 -33
View File
@@ -5,7 +5,9 @@ const secureImageService = require('../services/secureImageService');
const secureImageMiddleware = require('../middleware/secureImageMiddleware');
const logger = require('../utils/logger');
const { formatBoolean } = require('../utils/dbCompat');
const { resolvePhotoFilePath } = require('../services/photoResolver');
const { resolvePhotoFilePath, resolvePhotoStorageKey } = require('../services/photoResolver');
const { withLocalCopy } = require('../services/imageProcessor');
const { getStorage } = require('../services/storage');
const router = express.Router();
@@ -139,18 +141,10 @@ router.get('/:slug/secure/:photoId/:token',
return res.status(404).json({ error: 'Photo not found' });
}
let filePath;
try {
filePath = resolvePhotoFilePath(req.event, photo);
} catch (resolveError) {
logger.error('Failed to resolve photo path for secure token generation', {
slug: req.params.slug,
photoId,
eventId: req.event.id,
error: resolveError.message,
});
return res.status(404).json({ error: 'Photo file not found' });
}
// Resolve photo through storage backend (managed) or fall back to local
// path (external reference mode). secureImageService needs a local file,
// so we materialize a tmp copy via withLocalCopy in S3 mode.
const storageKey = resolvePhotoStorageKey(event, photo);
// Get protection settings for this event
const protectionSettings = {
@@ -160,11 +154,21 @@ router.get('/:slug/secure/:photoId/:token',
fragmentImage: event.use_canvas_rendering === true && fragment !== undefined
};
// Process image with protection measures
const processedImage = await secureImageService.processProtectedImage(
filePath,
protectionSettings
);
let processedImage;
try {
const runProcessing = (lp) => secureImageService.processProtectedImage(lp, protectionSettings);
processedImage = storageKey
? await withLocalCopy(storageKey, runProcessing)
: await runProcessing(resolvePhotoFilePath(event, photo));
} catch (resolveError) {
logger.error('Failed to process secure image', {
slug: req.params.slug,
photoId,
eventId: event.id,
error: resolveError.message,
});
return res.status(404).json({ error: 'Photo file not found' });
}
// Handle fragmented images
if (processedImage.type === 'fragmented') {
@@ -292,11 +296,30 @@ router.get('/:slug/secure-download/:photoId/:token',
return res.status(404).json({ error: 'Photo not found' });
}
let filePath;
// Resolve photo through storage backend (managed) or local disk (external).
const storageKey = resolvePhotoStorageKey(req.event, photo);
const watermarkService = require('../services/watermarkService');
const watermarkSettings = await watermarkService.getWatermarkSettings();
const wantsWatermark = watermarkSettings && watermarkSettings.enabled;
let fileBuffer;
try {
filePath = resolvePhotoFilePath(req.event, photo);
if (wantsWatermark) {
fileBuffer = storageKey
? await withLocalCopy(storageKey, (lp) => watermarkService.applyWatermark(lp, watermarkSettings))
: await watermarkService.applyWatermark(resolvePhotoFilePath(req.event, photo), watermarkSettings);
} else if (storageKey) {
const stream = await getStorage().get(storageKey);
const chunks = [];
for await (const chunk of stream) chunks.push(chunk);
fileBuffer = Buffer.concat(chunks);
} else {
const fs = require('fs').promises;
fileBuffer = await fs.readFile(resolvePhotoFilePath(req.event, photo));
}
} catch (resolveError) {
logger.error('Failed to resolve photo path for secure download', {
logger.error('Failed to fetch photo for secure download', {
slug: req.params.slug,
photoId,
eventId: req.event.id,
@@ -305,18 +328,6 @@ router.get('/:slug/secure-download/:photoId/:token',
return res.status(404).json({ error: 'Photo file not found' });
}
// Apply watermark if enabled
const watermarkService = require('../services/watermarkService');
const watermarkSettings = await watermarkService.getWatermarkSettings();
let fileBuffer;
if (watermarkSettings && watermarkSettings.enabled) {
fileBuffer = await watermarkService.applyWatermark(filePath, watermarkSettings);
} else {
const fs = require('fs').promises;
fileBuffer = await fs.readFile(filePath);
}
// Update download count
await db('photos').where('id', photoId).increment('download_count', 1);
+45 -17
View File
@@ -182,6 +182,18 @@ router.post(
type: 'admin', id: req.admin.id, name: req.admin.username
});
// Webhook lifecycle (#327). v1 events are not draft-aware, so they're
// both created AND published in the same call.
try {
const webhookService = require('../../services/webhookService');
await webhookService.fire('event.created', {
event: { id, slug, event_name, event_type, event_date, share_url: shareUrl },
});
await webhookService.fire('event.published', {
event: { id, slug, event_name, share_url: shareUrl },
});
} catch (e) { /* non-fatal */ }
res.status(201).json({ id, slug, share_url: shareUrl, share_token: shareToken });
} catch (error) {
logger.error('v1 POST /events failed', { error: error.message, stack: error.stack });
@@ -346,33 +358,39 @@ router.post(
const event = await db('events').where({ id: req.params.id }).first();
if (!event) return res.status(404).json({ error: 'Event not found' });
const finalDir = path.join(getStoragePath(), 'events/active', event.slug);
await fs.mkdir(finalDir, { recursive: true });
const ext = path.extname(req.file.originalname);
const finalName = `${Date.now()}_${crypto.randomBytes(4).toString('hex')}${ext}`;
const finalPath = path.join(finalDir, finalName);
await fs.rename(tempPath, finalPath);
tempPath = null;
// photo.path is stored relative to events/active so resolvePhotoStorageKey
// can rebuild the full key on read. Same shape as adminPhotos uploads.
const relPath = path.posix.join(event.slug, finalName);
const finalKey = path.posix.join('events/active', relPath);
const stat = fsSync.statSync(finalPath);
const relPath = path.relative(path.join(getStoragePath(), 'events/active'), finalPath);
const stat = fsSync.statSync(tempPath);
// Read sharp metadata + generate thumbnail FROM the local temp file
// before uploading the original through the storage backend. (Same
// ordering as adminPhotos.js so sharp/ffmpeg always have a real fs path.)
let width = null;
let height = null;
try {
const meta = await sharp(tempPath).metadata();
width = meta.width || null;
height = meta.height || null;
} catch { /* non-fatal */ }
let thumbRel = null;
try {
const thumbPath = await generateThumbnail(finalPath);
thumbRel = path.relative(getStoragePath(), thumbPath);
thumbRel = await generateThumbnail(tempPath);
} catch (err) {
logger.warn('v1 thumbnail generation failed', { err: err.message });
}
// Detect image dimensions for masonry layouts.
let width = null;
let height = null;
try {
const meta = await sharp(finalPath).metadata();
width = meta.width || null;
height = meta.height || null;
} catch { /* non-fatal */ }
// Upload the original via the storage backend (local fs OR S3),
// then drop the multer temp file.
const { getStorage } = require('../../services/storage');
await getStorage().putFromFile(finalKey, tempPath, { contentType: req.file.mimetype });
await fs.unlink(tempPath).catch(() => {});
tempPath = null;
const insertResult = await db('photos').insert({
event_id: event.id,
@@ -394,6 +412,16 @@ router.post(
type: 'admin', id: req.admin.id, name: req.admin.username
});
// Webhook (#327): one event per uploaded photo so receivers get a
// 1:1 stream they can react to.
try {
const webhookService = require('../../services/webhookService');
await webhookService.fire('photo.uploaded', {
event: { id: event.id, slug: event.slug, event_name: event.event_name },
photo: { id, filename: finalName, original_filename: req.file.originalname, size_bytes: stat.size, width, height },
});
} catch (e) { /* non-fatal */ }
res.status(201).json({ id, filename: finalName, path: relPath, thumbnail_path: thumbRel, size_bytes: stat.size });
} catch (error) {
logger.error('v1 POST /events/:id/photos failed', { error: error.message });
+116 -80
View File
@@ -1,57 +1,47 @@
const archiver = require('archiver');
const fs = require('fs').promises;
const fs = require('fs');
const fsp = require('fs').promises;
const path = require('path');
const os = require('os');
const crypto = require('crypto');
const { db } = require('../database/db');
const { queueEmail } = require('./emailProcessor');
const logger = require('../utils/logger');
const feedbackService = require('./feedbackService');
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
const ACTIVE_PATH = () => path.join(getStoragePath(), 'events/active');
const ARCHIVE_PATH = () => path.join(getStoragePath(), 'events/archived');
const { getStorage } = require('./storage');
async function archiveEvent(event) {
const storage = getStorage();
const archiveName = `${event.slug}.zip`;
const archiveRelKey = path.posix.join('events/archived', archiveName);
const eventPrefix = path.posix.join('events/active', event.slug);
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-archive-'));
const tmpArchive = path.join(tmpDir, `${crypto.randomBytes(4).toString('hex')}-${archiveName}`);
try {
const eventPath = path.join(ACTIVE_PATH(), event.slug);
const archiveName = `${event.slug}.zip`;
const archivePath = path.join(ARCHIVE_PATH(), archiveName);
// Ensure archive directory exists
await fs.mkdir(ARCHIVE_PATH(), { recursive: true });
// Create archive
const output = require('fs').createWriteStream(archivePath);
const archive = archiver('zip', {
zlib: { level: 9 } // Maximum compression
});
archive.on('error', (err) => {
throw err;
});
// Export feedback data before archiving
// Collect feedback data first so it can be included as in-memory entries.
const feedbackEntries = [];
const feedbackSettings = await feedbackService.getEventFeedbackSettings(event.id);
if (feedbackSettings.feedback_enabled) {
try {
logger.info(`Exporting feedback data for event ${event.slug}`);
const feedbackData = await feedbackService.exportEventFeedback(event.id);
if (feedbackData && feedbackData.length > 0) {
// Create feedback JSON file
const feedbackJson = JSON.stringify(feedbackData, null, 2);
const feedbackJsonPath = path.join(eventPath, 'feedback_data.json');
await fs.writeFile(feedbackJsonPath, feedbackJson, 'utf8');
// Create feedback CSV file
const feedbackCsv = convertToCSV(feedbackData);
const feedbackCsvPath = path.join(eventPath, 'feedback_data.csv');
await fs.writeFile(feedbackCsvPath, feedbackCsv, 'utf8');
// Create feedback summary
feedbackEntries.push({
name: 'feedback_data.json',
buffer: Buffer.from(JSON.stringify(feedbackData, null, 2), 'utf8'),
});
feedbackEntries.push({
name: 'feedback_data.csv',
buffer: Buffer.from(convertToCSV(feedbackData), 'utf8'),
});
const summary = await feedbackService.getEventFeedbackSummary(event.id);
const summaryPath = path.join(eventPath, 'feedback_summary.json');
await fs.writeFile(summaryPath, JSON.stringify(summary, null, 2), 'utf8');
feedbackEntries.push({
name: 'feedback_summary.json',
buffer: Buffer.from(JSON.stringify(summary, null, 2), 'utf8'),
});
logger.info(`Feedback data exported: ${feedbackData.length} entries`);
}
} catch (error) {
@@ -59,64 +49,110 @@ async function archiveEvent(event) {
// Continue with archiving even if feedback export fails
}
}
output.on('close', async () => {
try {
logger.info(`Archive created: ${archiveName} (${archive.pointer()} bytes)`);
// Update database
await db('events').where('id', event.id).update({
is_archived: true,
archive_path: path.relative(getStoragePath(), archivePath),
archived_at: new Date()
});
// Stream every photo (and any other content under events/active/{slug}/) into
// the zip directly from the storage backend.
const photoEntries = await storage.list(eventPrefix);
// Delete original files
await fs.rm(eventPath, { recursive: true });
let totalBytes = 0;
await new Promise((resolve, reject) => {
const output = fs.createWriteStream(tmpArchive);
const archive = archiver('zip', { zlib: { level: 9 } });
// Delete thumbnails
const photos = await db('photos').where('event_id', event.id);
for (const photo of photos) {
if (photo.thumbnail_path) {
const thumbPath = path.join(getStoragePath(), photo.thumbnail_path);
await fs.unlink(thumbPath).catch(() => {}); // Ignore if already deleted
}
output.on('close', () => {
totalBytes = archive.pointer();
resolve();
});
archive.on('error', reject);
archive.pipe(output);
const append = async () => {
for (const entry of photoEntries) {
const nameInZip = entry.key.startsWith(`${eventPrefix}/`)
? entry.key.slice(eventPrefix.length + 1)
: entry.key;
const stream = await storage.get(entry.key);
archive.append(stream, { name: nameInZip });
}
// Queue completion email — admin_email is nullable on events (migration 073);
// skip queueing rather than violating email_queue.recipient_email NOT NULL.
if (event.admin_email) {
await queueEmail(event.id, event.admin_email, 'archive_complete', {
event_name: event.event_name,
archive_size: (archive.pointer() / 1024 / 1024).toFixed(2) + ' MB'
});
} else {
logger.info(`Skipping archive_complete email for event ${event.slug}: no admin_email set`);
for (const f of feedbackEntries) {
archive.append(f.buffer, { name: f.name });
}
} catch (err) {
// Never let the close handler reject — it runs detached from the caller,
// and an unhandled rejection here crashes the backend process.
logger.error(`Post-archive cleanup failed for event ${event.slug}:`, err);
}
archive.finalize();
};
append().catch(reject);
});
archive.pipe(output);
archive.directory(eventPath, false);
await archive.finalize();
// Upload the finalized zip to the storage backend.
await storage.putFromFile(archiveRelKey, tmpArchive, { contentType: 'application/zip' });
logger.info(`Archive created: ${archiveName} (${totalBytes} bytes)`);
// Update DB BEFORE deleting originals so a crash mid-cleanup leaves the
// archive accessible rather than orphaning the photos.
await db('events').where('id', event.id).update({
is_archived: true,
archive_path: archiveRelKey,
archived_at: new Date(),
});
// Fire event.archived webhook (#327). Receivers infer per-photo loss
// from this event — we deliberately do NOT fire photo.deleted for each
// archived photo to avoid flooding subscribers on bulk archives.
try {
const webhookService = require('./webhookService');
await webhookService.fire('event.archived', {
event: { id: event.id, slug: event.slug, event_name: event.event_name, archive_path: archiveRelKey },
});
} catch (e) { /* non-fatal */ }
// Delete the originals from storage.
for (const entry of photoEntries) {
await storage.delete(entry.key).catch((err) =>
logger.warn(`Failed to delete archived original ${entry.key}: ${err.message}`)
);
}
// Delete thumbnails for this event's photos.
const photos = await db('photos').where('event_id', event.id);
for (const photo of photos) {
if (photo.thumbnail_path) {
await storage.delete(photo.thumbnail_path).catch(() => {});
}
if (photo.hero_path) {
await storage.delete(photo.hero_path).catch(() => {});
}
// Best effort: remove watermarked variants too if a refactor added them.
if (photo.watermark_path) {
await storage.delete(photo.watermark_path).catch(() => {});
}
}
// Queue completion email — admin_email is nullable on events (migration 073);
// skip queueing rather than violating email_queue.recipient_email NOT NULL.
if (event.admin_email) {
await queueEmail(event.id, event.admin_email, 'archive_complete', {
event_name: event.event_name,
archive_size: (totalBytes / 1024 / 1024).toFixed(2) + ' MB',
});
} else {
logger.info(`Skipping archive_complete email for event ${event.slug}: no admin_email set`);
}
} catch (error) {
logger.error(`Error archiving event ${event.slug}:`, error);
throw error;
} finally {
await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
}
}
// Helper function to convert JSON to CSV
function convertToCSV(data) {
if (!data || data.length === 0) return '';
const headers = Object.keys(data[0]);
const csvHeaders = headers.join(',');
const csvRows = data.map(row => {
return headers.map(header => {
const value = row[header];
@@ -127,7 +163,7 @@ function convertToCSV(data) {
return value || '';
}).join(',');
});
return [csvHeaders, ...csvRows].join('\n');
}
+13 -2
View File
@@ -181,8 +181,13 @@ class BackupManifestGenerator {
const content = await fs.readFile(filePath, 'utf8');
let manifest;
// Detect format and parse
if (filePath.endsWith('.yaml') || filePath.endsWith('.yml')) {
// Detect format from BOTH the extension and the content. Earlier code
// trusted the extension alone, which broke when callers stored a YAML
// manifest under a .json temp name (getBackupManifest does this when
// downloading the s3:// path to a tmp file).
const looksLikeJson = content.trimStart().startsWith('{')
|| content.trimStart().startsWith('[');
if (filePath.endsWith('.yaml') || filePath.endsWith('.yml') || !looksLikeJson) {
manifest = yaml.load(content);
} else {
manifest = JSON.parse(content);
@@ -318,6 +323,12 @@ class BackupManifestGenerator {
deleted_files: comparison.deleted_files.map(f => f.path)
};
// Recalculate the checksum after attaching the incremental section,
// otherwise validateManifest() rejects the loaded manifest because
// generateManifest() stamped a checksum that did NOT include this
// section.
fullManifest.verification.total_checksum = this.calculateManifestChecksum(fullManifest);
return fullManifest;
}
+111 -13
View File
@@ -268,6 +268,15 @@ async function getDatabaseBackupInfoInternal() {
if (recent && recent.file_path) {
const hasChanged = await hasDatabaseChanged(recent.completed_at);
// Postgres jsonb columns come back already parsed; sqlite TEXT comes
// back as a JSON string. Accept both.
const parseField = (v) => {
if (v == null) return null;
if (typeof v === 'object') return v;
try { return JSON.parse(v); } catch { return null; }
};
const stats = parseField(recent.statistics);
const checksums = parseField(recent.table_checksums);
return {
type: recent.backup_type || 'unknown',
backupFile: recent.file_path,
@@ -275,8 +284,8 @@ async function getDatabaseBackupInfoInternal() {
checksum: recent.checksum,
hasChanged,
backupTime: recent.completed_at,
tables: recent.statistics ? JSON.parse(recent.statistics).tables : {},
rowCounts: recent.table_checksums ? JSON.parse(recent.table_checksums) : {}
tables: (stats && stats.tables) || {},
rowCounts: checksums || {}
};
}
@@ -826,7 +835,7 @@ async function runBackupInternal(isManual = false) {
let manifest = await backupManifest.generateManifest(manifestOptions);
if (previousBackup && previousBackup.manifest_path) {
try {
const parentManifest = await backupManifest.loadManifest(previousBackup.manifest_path);
const parentManifest = await loadManifestFromAnywhere(previousBackup.manifest_path, config);
manifest = await backupManifest.generateIncrementalManifest(manifestOptions, parentManifest);
} catch (error) {
logger.warn('Failed to load parent manifest, generating full manifest:', error);
@@ -936,17 +945,41 @@ async function startBackupService() {
backupJob = null;
}
// Two settings cooperate here:
// - backup_schedule — UI label like "daily" / "weekly" / "custom"
// - backup_schedule_cron — actual cron expression
// The frontend writes both (BackupConfiguration.jsx). Older startup code
// here read backup_schedule and crashed when it found a label instead of
// a cron expression. Resolution order: explicit cron field, then map known
// labels, then fall back to default.
const NAMED_SCHEDULES = {
hourly: '0 * * * *',
daily: '0 2 * * *',
weekly: '0 3 * * 0', // Sunday 03:00
monthly: '0 4 1 * *',
};
const isCronExpression = (s) => typeof s === 'string' && /^\s*\S+(\s+\S+){4}\s*$/.test(s);
const readSetting = (key) => {
if (config && Object.prototype.hasOwnProperty.call(config, key)) {
return String(config[key] ?? '').trim();
}
if (config?.__raw && Object.prototype.hasOwnProperty.call(config.__raw, key)) {
return String(parseSettingValue(config.__raw[key]) ?? '').trim();
}
return '';
};
let schedule = '0 2 * * *';
if (Object.prototype.hasOwnProperty.call(config, 'backup_schedule')) {
const candidate = String(config.backup_schedule ?? '').trim();
if (candidate.length) {
schedule = candidate;
}
} else if (config.__raw && Object.prototype.hasOwnProperty.call(config.__raw, 'backup_schedule')) {
const candidate = String(parseSettingValue(config.__raw.backup_schedule) ?? '').trim();
if (candidate.length) {
schedule = candidate;
}
const cronCandidate = readSetting('backup_schedule_cron');
const labelCandidate = readSetting('backup_schedule');
if (cronCandidate && isCronExpression(cronCandidate)) {
schedule = cronCandidate;
} else if (labelCandidate && NAMED_SCHEDULES[labelCandidate.toLowerCase()]) {
schedule = NAMED_SCHEDULES[labelCandidate.toLowerCase()];
} else if (labelCandidate && isCronExpression(labelCandidate)) {
// Back-compat: a deployment that wrote a cron expression directly into
// backup_schedule (no _cron field) still works.
schedule = labelCandidate;
}
backupJob = cron.schedule(schedule, async () => {
@@ -1073,6 +1106,71 @@ async function cleanupOldBackupRuns(retentionDays = 30) {
}
}
/**
* Load a backup manifest regardless of whether it lives on the local
* filesystem or in S3. Used by both the public getBackupManifest API
* and the incremental-manifest path in runBackupInternal — previously
* the latter called loadManifest() with an s3:// URI directly, which
* tried fs.readFile on the literal string and threw ENOENT, silently
* downgrading every incremental backup to a full manifest.
*/
async function loadManifestFromAnywhere(manifestPath, config) {
if (!manifestPath) {
throw new Error('Manifest path is required');
}
if (!manifestPath.startsWith('s3://')) {
return backupManifest.loadManifest(manifestPath);
}
const cfg = config || (await resolveConfigWithFallback());
const accessKey = cfg?.backup_s3_access_key
?? (cfg?.__raw && Object.prototype.hasOwnProperty.call(cfg.__raw, 'backup_s3_access_key')
? parseSettingValue(cfg.__raw.backup_s3_access_key)
: undefined)
?? process.env.BACKUP_S3_ACCESS_KEY;
const secretKey = cfg?.backup_s3_secret_key
?? (cfg?.__raw && Object.prototype.hasOwnProperty.call(cfg.__raw, 'backup_s3_secret_key')
? parseSettingValue(cfg.__raw.backup_s3_secret_key)
: undefined)
?? process.env.BACKUP_S3_SECRET_KEY;
if (!accessKey || !secretKey) {
throw new Error('S3 credentials not configured for manifest retrieval');
}
const match = manifestPath.match(/^s3:\/\/([^\/]+)\/(.+)$/);
if (!match) {
throw new Error('Invalid S3 manifest path');
}
const [, bucket, key] = match;
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'backup-manifest-'));
// Preserve the original extension so loadManifest's format detection
// picks the right parser.
const ext = path.extname(key) || '.json';
const tempPath = path.join(tempDir, `manifest-${Date.now()}${ext}`);
const s3Client = new S3StorageAdapter({
bucket,
region: (cfg && cfg.backup_s3_region) || 'us-east-1',
endpoint: cfg && cfg.backup_s3_endpoint,
accessKeyId: accessKey,
secretAccessKey: secretKey,
forcePathStyle: cfg ? normalizeBoolean(cfg.backup_s3_force_path_style) : false,
sslEnabled: cfg && cfg.backup_s3_ssl_enabled !== undefined
? normalizeBoolean(cfg.backup_s3_ssl_enabled)
: true,
});
try {
await s3Client.download(key, tempPath);
return await backupManifest.loadManifest(tempPath);
} finally {
await fs.unlink(tempPath).catch(() => {});
await fs.rm(tempDir, { recursive: true, force: true }).catch(() => {});
}
}
async function getBackupManifest(backupRunId) {
const run = await db('backup_runs')
.where('id', backupRunId)
+65 -40
View File
@@ -7,16 +7,22 @@
*
* Pattern follows watermarkGeneratorService.js singleton with
* in-memory locking and debounced background regeneration.
*
* Storage: zips are written to a local tmp file then uploaded to the
* configured storage backend (local fs or S3) via storage.putFromFile.
* The cached zip is served via the storage backend on download.
*/
const fs = require('fs');
const fsp = require('fs/promises');
const path = require('path');
const os = require('os');
const crypto = require('crypto');
const archiver = require('archiver');
const { db } = require('../database/db');
const watermarkService = require('./watermarkService');
const { resolvePhotoFilePath } = require('./photoResolver');
const { getStoragePath } = require('../config/storage');
const { resolvePhotoStorageKey, resolvePhotoFilePath } = require('./photoResolver');
const { getStorage } = require('./storage');
const logger = require('../utils/logger');
const DEBOUNCE_MS = 5000;
@@ -29,15 +35,16 @@ class DownloadZipService {
}
/**
* Absolute path to the cached zip for an event slug.
* Relative storage key for the cached zip.
*/
getCachePath(slug) {
return path.join(getStoragePath(), 'events', 'active', slug, '.download-cache', 'all.zip');
getCacheKey(slug) {
return path.posix.join('events/active', slug, '.download-cache', 'all.zip');
}
/**
* Check if a valid cached zip exists.
* Returns { path, size, generatedAt } or null.
* Returns { key, size, generatedAt } or null. The key is a relative storage
* key callers stream it via storage.get() rather than reading directly.
*/
async getZipInfo(eventId) {
try {
@@ -48,15 +55,10 @@ class DownloadZipService {
if (!event || !event.download_zip_path) return null;
const absPath = this.getCachePath(event.slug);
try {
const stat = await fsp.stat(absPath);
return {
path: absPath,
size: stat.size,
generatedAt: event.download_zip_generated_at,
};
} catch {
const storage = getStorage();
const key = this.getCacheKey(event.slug);
const stat = await storage.stat(key);
if (!stat) {
// File gone — clear stale DB record
await db('events').where({ id: eventId }).update({
download_zip_path: null,
@@ -64,6 +66,11 @@ class DownloadZipService {
});
return null;
}
return {
key,
size: stat.size,
generatedAt: event.download_zip_generated_at,
};
} catch (err) {
logger.warn('downloadZipService.getZipInfo error', { eventId, error: err.message });
return null;
@@ -71,7 +78,7 @@ class DownloadZipService {
}
/**
* Generate the pre-zip for an event. Returns { success, path, size } or { success: false }.
* Generate the pre-zip for an event. Returns { success, key, size } or { success: false }.
* Concurrent calls for the same eventId share one in-flight build.
*/
async generateZip(eventId) {
@@ -97,6 +104,9 @@ class DownloadZipService {
}
async _build(eventId, version) {
const storage = getStorage();
let tmpDir;
try {
const event = await db('events').where({ id: eventId }).first();
if (!event) return { success: false, error: 'Event not found' };
@@ -119,11 +129,10 @@ class DownloadZipService {
text: event.watermark_text || watermarkSettings?.text || 'Protected',
} : null;
const cacheDir = path.dirname(this.getCachePath(event.slug));
await fsp.mkdir(cacheDir, { recursive: true });
const finalKey = this.getCacheKey(event.slug);
const tmpPath = this.getCachePath(event.slug) + `.tmp.${Date.now()}`;
const finalPath = this.getCachePath(event.slug);
tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-zipbuild-'));
const tmpPath = path.join(tmpDir, `${crypto.randomBytes(4).toString('hex')}-all.zip`);
// Build zip — level 0 (store only) since photos are already compressed
await new Promise((resolve, reject) => {
@@ -145,13 +154,6 @@ class DownloadZipService {
return reject(new Error('Build invalidated'));
}
let filePath;
try {
filePath = resolvePhotoFilePath(event, photo);
} catch {
continue;
}
let archiveName;
if (hasMultipleTypes) {
const folderName = photo.type === 'individual' ? 'Individual Photos' : 'Collages';
@@ -160,14 +162,36 @@ class DownloadZipService {
archiveName = photo.filename;
}
// External-mode photos still live on local disk; managed photos go
// through the storage backend. resolvePhotoStorageKey returns null
// for external, in which case fall back to resolvePhotoFilePath.
const storageKey = resolvePhotoStorageKey(event, photo);
if (shouldApplyWatermark && effectiveSettings) {
try {
const buf = await watermarkService.applyWatermark(filePath, effectiveSettings);
let sourcePath;
if (storageKey) {
// Stream the original to a tmp file just long enough for sharp
// (watermarkService) to operate on it. Avoids buffering the
// entire image in memory for huge originals.
sourcePath = path.join(tmpDir, `wm-${crypto.randomBytes(4).toString('hex')}`);
await storage.getToFile(storageKey, sourcePath);
} else {
sourcePath = resolvePhotoFilePath(event, photo);
}
const buf = await watermarkService.applyWatermark(sourcePath, effectiveSettings);
archive.append(buf, { name: archiveName });
if (storageKey) {
await fsp.unlink(sourcePath).catch(() => {});
}
} catch (err) {
logger.warn('Skipping watermark in pre-zip', { photoId: photo.id, error: err.message });
}
} else if (storageKey) {
const stream = await storage.get(storageKey);
archive.append(stream, { name: archiveName });
} else {
const filePath = resolvePhotoFilePath(event, photo);
archive.file(filePath, { name: archiveName });
}
}
@@ -180,29 +204,33 @@ class DownloadZipService {
// Check version again — another invalidation may have arrived
if (this.versions.get(eventId) !== version) {
await fsp.unlink(tmpPath).catch(() => {});
return { success: false, error: 'Build invalidated' };
}
// Atomic rename
await fsp.rename(tmpPath, finalPath);
// Upload to storage (atomic from caller's perspective: storage.put writes
// to a tmp file/object first then commits in LocalFs; in S3 the key only
// exists after the multipart upload completes).
await storage.putFromFile(finalKey, tmpPath, { contentType: 'application/zip' });
const stat = await fsp.stat(finalPath);
const stat = await storage.stat(finalKey);
// Update DB
await db('events').where({ id: eventId }).update({
download_zip_path: `events/active/${event.slug}/.download-cache/all.zip`,
download_zip_path: finalKey,
download_zip_generated_at: new Date(),
});
logger.info('Pre-zip generated', { eventId, slug: event.slug, size: stat.size, photos: photos.length });
return { success: true, path: finalPath, size: stat.size };
return { success: true, key: finalKey, size: stat.size };
} catch (err) {
if (err.message === 'Build invalidated') {
return { success: false, error: 'Build invalidated' };
}
logger.error('downloadZipService._build error', { eventId, error: err.message });
return { success: false, error: err.message };
} finally {
if (tmpDir) {
await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
}
}
}
@@ -264,17 +292,14 @@ class DownloadZipService {
async _cleanup(eventId) {
try {
const storage = getStorage();
const event = await db('events')
.where({ id: eventId })
.select('slug', 'download_zip_path')
.first();
if (event && event.download_zip_path) {
const absPath = this.getCachePath(event.slug);
await fsp.unlink(absPath).catch(() => {});
// Also try to remove the cache directory if empty
const cacheDir = path.dirname(absPath);
await fsp.rmdir(cacheDir).catch(() => {});
await storage.delete(this.getCacheKey(event.slug)).catch(() => {});
}
await db('events').where({ id: eventId }).update({
+10 -1
View File
@@ -84,7 +84,16 @@ async function handleExpiredEvent(event) {
try {
// Mark as inactive
await db('events').where('id', event.id).update({ is_active: formatBoolean(false) });
// Fire event.expired BEFORE the cascading archive call so receivers
// get the lifecycle in order (expired → archived).
try {
const webhookService = require('./webhookService');
await webhookService.fire('event.expired', {
event: { id: event.id, slug: event.slug, event_name: event.event_name, expires_at: event.expires_at },
});
} catch (e) { /* non-fatal */ }
// Queue expiration emails
const recipientEmail = event.customer_email || event.host_email;
const recipientName = event.customer_name || event.host_name || (recipientEmail ? recipientEmail.split('@')[0] : null);
+33 -2
View File
@@ -13,6 +13,16 @@ const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '.
const WATCH_PATH = () => path.join(getStoragePath(), 'events/active');
function startFileWatcher() {
// Auto-import via filesystem watching only works with the local storage
// backend. In S3 mode there is no local directory to watch — every photo
// must enter through the admin upload API. Skip cleanly with a clear log
// so operators aren't surprised by the missing feature.
const backend = (process.env.STORAGE_BACKEND || 'local').toLowerCase();
if (backend !== 'local') {
logger.warn(`[fileWatcher] auto-import disabled — STORAGE_BACKEND=${backend}. Use the admin upload API instead.`);
return null;
}
const watcher = chokidar.watch(WATCH_PATH(), {
ignored: /(^|[\/\\])\../, // ignore dotfiles
persistent: true,
@@ -93,7 +103,7 @@ async function processNewPhoto(filePath) {
if (!existingPhoto) {
// Add to database
await db('photos').insert({
const insertResult = await db('photos').insert({
event_id: event.id,
filename: path.basename(filePath),
path: relativePath,
@@ -101,10 +111,21 @@ async function processNewPhoto(filePath) {
type: isVideo ? 'video' : photoType,
size_bytes: stats.size,
mime_type: mimeType
});
}).returning('id');
const photoId = insertResult[0]?.id || insertResult[0];
logger.info(`Added new photo: ${relativePath}`);
downloadZipService.invalidate(event.id);
// Webhook (#327) — auto-import path. Only fires in local mode since
// the watcher is disabled in S3 mode.
try {
const webhookService = require('./webhookService');
await webhookService.fire('photo.uploaded', {
event: { id: event.id, slug: event.slug, event_name: event.event_name },
photo: { id: photoId, filename: path.basename(filePath), size_bytes: stats.size, source: 'auto-import' },
});
} catch (e) { /* non-fatal */ }
} else {
logger.debug(`Photo already exists: ${relativePath}`);
}
@@ -121,6 +142,16 @@ async function removePhoto(filePath) {
if (photo) {
downloadZipService.invalidate(photo.event_id);
// Webhook (#327) — fire only if the row actually existed.
try {
const event = await db('events').where({ id: photo.event_id }).first();
const webhookService = require('./webhookService');
await webhookService.fire('photo.deleted', {
event: { id: photo.event_id, slug: event?.slug, event_name: event?.event_name },
photo: { id: photo.id, filename: photo.filename, source: 'auto-import' },
});
} catch (e) { /* non-fatal */ }
}
logger.info(`Removed photo: ${relativePath}`);
+133 -157
View File
@@ -1,9 +1,12 @@
const sharp = require('sharp');
const exifr = require('exifr');
const path = require('path');
const fs = require('fs').promises;
const fsp = require('fs').promises;
const os = require('os');
const crypto = require('crypto');
const logger = require('../utils/logger');
const { db } = require('../database/db');
const { getStorage } = require('./storage');
// Configure sharp for better memory management with large batches
sharp.cache(false); // Disable cache to prevent memory buildup
@@ -16,8 +19,10 @@ const DEFAULT_THUMBNAIL_FIT = 'cover'; // 'cover' for square crops
const DEFAULT_THUMBNAIL_QUALITY = 85;
const DEFAULT_THUMBNAIL_FORMAT = 'jpeg';
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
const getThumbnailPath = () => path.join(getStoragePath(), 'thumbnails');
// Hero image settings - optimized for large displays
const DEFAULT_HERO_WIDTH = 1920;
const DEFAULT_HERO_HEIGHT = 1080;
const DEFAULT_HERO_QUALITY = 85;
// Helper to parse setting value (handles both JSON-encoded and plain values)
function parseSettingValue(value) {
@@ -83,60 +88,62 @@ async function getThumbnailSettings() {
}
}
const contentTypeFor = (format) => {
if (format === 'png') return 'image/png';
if (format === 'webp') return 'image/webp';
return 'image/jpeg';
};
/**
* Generate a thumbnail from a local source image path. The output is written
* to the storage backend (local fs or S3) under `thumbnails/thumb_<filename>`
* and the relative storage key is returned for DB persistence.
*
* Callers must ensure the source is on the local filesystem. For S3 mode
* regeneration flows, fetch via `withLocalCopy(storage, sourceKey, fn)` first.
*/
async function generateThumbnail(imagePath, options = {}) {
const filename = path.basename(imagePath);
const thumbnailFilename = `thumb_${filename}`;
const thumbnailDir = getThumbnailPath();
const thumbnailPath = path.join(thumbnailDir, thumbnailFilename);
const thumbnailRelKey = path.posix.join('thumbnails', thumbnailFilename);
const storage = getStorage();
// Get thumbnail settings
const settings = await getThumbnailSettings();
// Ensure thumbnail directory exists
await fs.mkdir(thumbnailDir, { recursive: true });
// Check if we need to regenerate (for broken thumbnails)
// Force regeneration: drop the existing object before writing the new one
if (options.regenerate) {
try {
await fs.unlink(thumbnailPath);
logger.info(`Deleted broken thumbnail: ${thumbnailPath}`);
} catch (err) {
// File might not exist, that's okay
}
await storage.delete(thumbnailRelKey).catch(() => {});
}
try {
// First, verify the source image is complete and valid
const metadata = await sharp(imagePath).metadata();
if (!metadata.width || !metadata.height) {
throw new Error('Invalid image metadata - file may be incomplete');
}
// Create sharp instance with memory-efficient settings
let sharpInstance = sharp(imagePath, {
let sharpInstance = sharp(imagePath, {
limitInputPixels: 268402689, // ~16k x 16k max
sequentialRead: true, // More memory efficient for large images
failOnError: false // Don't fail on minor issues
sequentialRead: true,
failOnError: false
});
// Strip EXIF/metadata from thumbnails (privacy: prevent GPS leak etc.)
sharpInstance = sharpInstance.withMetadata(false);
// Apply resize with configured settings
// For square thumbnails with 'cover' fit, we crop to center
sharpInstance = sharpInstance.resize(settings.width, settings.height, {
withoutEnlargement: true,
fit: settings.fit, // 'cover' will crop to fill the exact dimensions
position: 'center' // Center the crop for better composition
fit: settings.fit,
position: 'center'
});
// Apply format-specific options
if (settings.format === 'jpeg') {
sharpInstance = sharpInstance.jpeg({
sharpInstance = sharpInstance.jpeg({
quality: settings.quality,
progressive: true, // Progressive JPEG for better loading
mozjpeg: true // Better compression
progressive: true,
mozjpeg: true
});
} else if (settings.format === 'png') {
sharpInstance = sharpInstance.png({
@@ -147,74 +154,87 @@ async function generateThumbnail(imagePath, options = {}) {
} else if (settings.format === 'webp') {
sharpInstance = sharpInstance.webp({
quality: settings.quality,
effort: 4 // Balance between speed and compression
effort: 4
});
}
// Save the thumbnail
await sharpInstance.toFile(thumbnailPath);
// Verify the thumbnail was created successfully
const stats = await fs.stat(thumbnailPath);
if (stats.size === 0) {
const buffer = await sharpInstance.toBuffer();
if (!buffer || buffer.length === 0) {
throw new Error('Generated thumbnail is empty');
}
return path.relative(getStoragePath(), thumbnailPath);
await storage.put(thumbnailRelKey, buffer, { contentType: contentTypeFor(settings.format) });
return thumbnailRelKey;
} catch (error) {
const msg = (error && error.message) ? error.message : String(error);
logger.error(`Failed to generate thumbnail for ${filename}: ${msg}`);
// Clean up any partially created file
try {
await fs.unlink(thumbnailPath);
} catch (unlinkErr) {
// Ignore unlink errors
}
// Return null if thumbnail generation fails, don't fail the whole upload
// Clean up any partially uploaded object
await storage.delete(thumbnailRelKey).catch(() => {});
return null;
}
}
/**
* Check if a thumbnail exists and is valid
* Check if a thumbnail exists and is valid. For local-fs storage we open the
* file with sharp to confirm it parses; for S3 we trust the byte-integrity
* checks built into the protocol and only verify size > 0.
*/
async function isThumbnailValid(thumbnailPath) {
const storage = getStorage();
try {
const fullPath = path.join(getStoragePath(), thumbnailPath);
const stats = await fs.stat(fullPath);
// Check if file exists and has content
if (stats.size === 0) {
const stat = await storage.stat(thumbnailPath);
if (!stat || stat.size === 0) {
return false;
}
// Try to read metadata to ensure it's a valid image
await sharp(fullPath).metadata();
if (storage.kind() === 'local') {
const localPath = storage.resolveLocalPath(thumbnailPath);
await sharp(localPath).metadata();
}
return true;
} catch (error) {
return false;
}
}
/**
* Wraps a callback that needs the source image as a local file. In local-fs
* mode the storage path is used directly (no copy); in S3 mode the object is
* streamed to a tmp file which is removed afterwards.
*/
async function withLocalCopy(sourceKey, fn) {
const storage = getStorage();
if (storage.kind() === 'local') {
return fn(storage.resolveLocalPath(sourceKey));
}
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-src-'));
const tmpPath = path.join(tmpDir, `${crypto.randomBytes(4).toString('hex')}_${path.basename(sourceKey)}`);
try {
await storage.getToFile(sourceKey, tmpPath);
return await fn(tmpPath);
} finally {
await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
}
}
/**
* Regenerate thumbnail if it's broken or missing
*/
async function ensureThumbnail(photo) {
const { db } = require('../database/db');
const { resolvePhotoFilePath } = require('./photoResolver');
let originalPath;
const { resolvePhotoStorageKey } = require('./photoResolver');
let sourceKey;
try {
const event = await db('events').where('id', photo.event_id).first();
originalPath = resolvePhotoFilePath(event, photo);
logger.info(`Ensuring thumbnail for photo ${photo.id} from source: ${originalPath}`);
sourceKey = resolvePhotoStorageKey(event, photo);
logger.info(`Ensuring thumbnail for photo ${photo.id} from key: ${sourceKey}`);
} catch (e) {
const msg = (e && e.message) ? e.message : String(e);
logger.error(`Failed to resolve original path for thumbnail (photo ${photo.id}): ${msg}`);
logger.error(`Failed to resolve original key for thumbnail (photo ${photo.id}): ${msg}`);
return null;
}
// Check if thumbnail exists and is valid
if (photo.thumbnail_path) {
const isValid = await isThumbnailValid(photo.thumbnail_path);
@@ -223,45 +243,40 @@ async function ensureThumbnail(photo) {
}
logger.warn(`Invalid thumbnail detected for photo ${photo.id}, regenerating...`);
}
// Generate new thumbnail
const newThumbnailPath = await generateThumbnail(originalPath, { regenerate: true });
// Generate new thumbnail (sources via withLocalCopy so this works in S3 mode)
const newThumbnailPath = await withLocalCopy(sourceKey, (localPath) =>
generateThumbnail(localPath, { regenerate: true })
);
if (newThumbnailPath) {
// Update database with new thumbnail path
const { db } = require('../database/db');
await db('photos')
.where({ id: photo.id })
.update({ thumbnail_path: newThumbnailPath });
logger.info(`Regenerated thumbnail for photo ${photo.id}`);
return newThumbnailPath;
}
return null;
}
async function generateVideoPlaceholder(originalFilename, options = {}) {
const parsed = path.parse(originalFilename || '');
const baseName = parsed.name || 'video';
const thumbnailDir = getThumbnailPath();
const thumbnailFilename = `thumb_${baseName}.jpg`;
const thumbnailPath = path.join(thumbnailDir, thumbnailFilename);
const thumbnailRelKey = path.posix.join('thumbnails', thumbnailFilename);
const storage = getStorage();
const settings = await getThumbnailSettings();
const width = settings.width || DEFAULT_THUMBNAIL_WIDTH;
const height = settings.height || DEFAULT_THUMBNAIL_HEIGHT;
if (options.regenerate) {
try {
await fs.unlink(thumbnailPath);
} catch (_) {
// ignore if missing
}
await storage.delete(thumbnailRelKey).catch(() => {});
}
try {
await fs.mkdir(thumbnailDir, { recursive: true });
const svg = `
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
<defs>
@@ -279,26 +294,20 @@ async function generateVideoPlaceholder(originalFilename, options = {}) {
</svg>
`;
await sharp(Buffer.from(svg))
const buffer = await sharp(Buffer.from(svg))
.resize(width, height, { fit: 'cover' })
.jpeg({ quality: settings.quality || DEFAULT_THUMBNAIL_QUALITY })
.toFile(thumbnailPath);
.toBuffer();
return path.relative(getStoragePath(), thumbnailPath);
await storage.put(thumbnailRelKey, buffer, { contentType: 'image/jpeg' });
return thumbnailRelKey;
} catch (error) {
logger.error('Failed to generate video placeholder thumbnail:', error.message);
return null;
}
}
// Hero image settings - optimized for large displays
const DEFAULT_HERO_WIDTH = 1920;
const DEFAULT_HERO_HEIGHT = 1080;
const DEFAULT_HERO_QUALITY = 85;
const DEFAULT_HERO_FORMAT = 'jpeg';
const getHeroPath = () => path.join(getStoragePath(), 'heroes');
/**
* Generate a hero-optimized image for gallery headers
* Outputs a 1920x1080 image suitable for full-width hero sections
@@ -306,36 +315,24 @@ const getHeroPath = () => path.join(getStoragePath(), 'heroes');
async function generateHeroImage(imagePath, options = {}) {
const filename = path.basename(imagePath);
const heroFilename = `hero_${filename}`;
const heroDir = getHeroPath();
const heroPath = path.join(heroDir, heroFilename);
const heroRelKey = path.posix.join('heroes', heroFilename);
const storage = getStorage();
// Ensure hero directory exists
await fs.mkdir(heroDir, { recursive: true });
// Check if we need to regenerate
if (options.regenerate) {
try {
await fs.unlink(heroPath);
logger.info(`Deleted existing hero image: ${heroPath}`);
} catch (err) {
// File might not exist, that's okay
}
await storage.delete(heroRelKey).catch(() => {});
}
try {
// First, verify the source image is complete and valid
const metadata = await sharp(imagePath).metadata();
if (!metadata.width || !metadata.height) {
throw new Error('Invalid image metadata - file may be incomplete');
}
// Calculate dimensions to maintain aspect ratio while fitting within hero bounds
const heroWidth = options.width || DEFAULT_HERO_WIDTH;
const heroHeight = options.height || DEFAULT_HERO_HEIGHT;
const quality = options.quality || DEFAULT_HERO_QUALITY;
// Create sharp instance with memory-efficient settings
let sharpInstance = sharp(imagePath, {
limitInputPixels: 268402689,
sequentialRead: true,
@@ -345,43 +342,31 @@ async function generateHeroImage(imagePath, options = {}) {
// Strip EXIF/metadata from hero images (privacy: prevent GPS leak etc.)
sharpInstance = sharpInstance.withMetadata(false);
// Resize to fit hero dimensions while maintaining aspect ratio
// Use 'cover' to fill the hero area (crops if needed)
sharpInstance = sharpInstance.resize(heroWidth, heroHeight, {
withoutEnlargement: false, // Allow upscaling for small images
withoutEnlargement: false,
fit: 'cover',
position: 'center'
});
// Apply JPEG format with high quality
sharpInstance = sharpInstance.jpeg({
quality: quality,
progressive: true,
mozjpeg: true
});
// Save the hero image
await sharpInstance.toFile(heroPath);
// Verify the hero image was created successfully
const stats = await fs.stat(heroPath);
if (stats.size === 0) {
const buffer = await sharpInstance.toBuffer();
if (!buffer || buffer.length === 0) {
throw new Error('Generated hero image is empty');
}
logger.info(`Generated hero image for ${filename}: ${heroPath}`);
return path.relative(getStoragePath(), heroPath);
await storage.put(heroRelKey, buffer, { contentType: 'image/jpeg' });
logger.info(`Generated hero image for ${filename}${heroRelKey}`);
return heroRelKey;
} catch (error) {
const msg = (error && error.message) ? error.message : String(error);
logger.error(`Failed to generate hero image for ${filename}: ${msg}`);
// Clean up any partially created file
try {
await fs.unlink(heroPath);
} catch (unlinkErr) {
// Ignore unlink errors
}
await storage.delete(heroRelKey).catch(() => {});
return null;
}
}
@@ -390,16 +375,16 @@ async function generateHeroImage(imagePath, options = {}) {
* Check if a hero image exists and is valid
*/
async function isHeroValid(heroPath) {
const storage = getStorage();
try {
const fullPath = path.join(getStoragePath(), heroPath);
const stats = await fs.stat(fullPath);
if (stats.size === 0) {
const stat = await storage.stat(heroPath);
if (!stat || stat.size === 0) {
return false;
}
// Try to read metadata to ensure it's a valid image
await sharp(fullPath).metadata();
if (storage.kind() === 'local') {
const localPath = storage.resolveLocalPath(heroPath);
await sharp(localPath).metadata();
}
return true;
} catch (error) {
return false;
@@ -410,21 +395,19 @@ async function isHeroValid(heroPath) {
* Ensure a hero image exists for a photo, regenerate if needed
*/
async function ensureHeroImage(photo) {
const { db } = require('../database/db');
const { resolvePhotoFilePath } = require('./photoResolver');
const { resolvePhotoStorageKey } = require('./photoResolver');
let originalPath;
let sourceKey;
try {
const event = await db('events').where('id', photo.event_id).first();
originalPath = resolvePhotoFilePath(event, photo);
logger.info(`Ensuring hero image for photo ${photo.id} from source: ${originalPath}`);
sourceKey = resolvePhotoStorageKey(event, photo);
logger.info(`Ensuring hero image for photo ${photo.id} from key: ${sourceKey}`);
} catch (e) {
const msg = (e && e.message) ? e.message : String(e);
logger.error(`Failed to resolve original path for hero image (photo ${photo.id}): ${msg}`);
logger.error(`Failed to resolve original key for hero image (photo ${photo.id}): ${msg}`);
return null;
}
// Check if hero image exists and is valid
if (photo.hero_path) {
const isValid = await isHeroValid(photo.hero_path);
if (isValid) {
@@ -433,11 +416,11 @@ async function ensureHeroImage(photo) {
logger.warn(`Invalid hero image detected for photo ${photo.id}, regenerating...`);
}
// Generate new hero image
const newHeroPath = await generateHeroImage(originalPath, { regenerate: true });
const newHeroPath = await withLocalCopy(sourceKey, (localPath) =>
generateHeroImage(localPath, { regenerate: true })
);
if (newHeroPath) {
// Update database with new hero path
await db('photos')
.where({ id: photo.id })
.update({ hero_path: newHeroPath });
@@ -451,12 +434,9 @@ async function ensureHeroImage(photo) {
/**
* Extract capture date from EXIF metadata
* @param {string} imagePath - Path to the image file
* @returns {Date|null} - The capture date or null if not available
*/
async function extractCaptureDate(imagePath) {
try {
// Parse EXIF data, looking for common date fields
const exif = await exifr.parse(imagePath, {
pick: ['DateTimeOriginal', 'CreateDate', 'DateTimeDigitized', 'ModifyDate']
});
@@ -465,23 +445,19 @@ async function extractCaptureDate(imagePath) {
return null;
}
// Priority order: DateTimeOriginal > CreateDate > DateTimeDigitized > ModifyDate
const captureDate = exif.DateTimeOriginal ||
exif.CreateDate ||
exif.DateTimeDigitized ||
exif.ModifyDate;
if (captureDate) {
// exifr returns Date objects directly when parsing dates
if (captureDate instanceof Date) {
// Validate the date is reasonable (not in the future, not before 1990)
const now = new Date();
const minDate = new Date('1990-01-01');
if (captureDate > minDate && captureDate <= now) {
return captureDate;
}
}
// Handle string dates if necessary
if (typeof captureDate === 'string') {
const parsed = new Date(captureDate);
if (!isNaN(parsed.getTime())) {
@@ -492,7 +468,6 @@ async function extractCaptureDate(imagePath) {
return null;
} catch (error) {
// Log only as debug - many images don't have EXIF data
logger.debug(`Could not extract EXIF date from ${path.basename(imagePath)}:`, error.message);
return null;
}
@@ -506,5 +481,6 @@ module.exports = {
generateHeroImage,
isHeroValid,
ensureHeroImage,
extractCaptureDate
extractCaptureDate,
withLocalCopy,
};
+56 -51
View File
@@ -4,9 +4,7 @@ const { db } = require('../database/db');
const { generateThumbnail } = require('./imageProcessor');
const { generatePhotoFilename } = require('../utils/filenameSanitizer');
const { processUploadedVideo, isVideoMimeType } = require('./videoProcessor');
// Get storage path from environment or default
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
const { getStorage } = require('./storage');
function normalizeFiles(files) {
// Handle null, undefined, or falsy values
@@ -99,11 +97,6 @@ async function processUploadedPhotos(files, eventId, uploadedBy = 'admin', categ
extension
);
// Move file to event folder
const destPath = path.join(getStoragePath(), 'events/active', event.slug);
await fs.mkdir(destPath, { recursive: true });
const newPath = path.join(destPath, newFilename);
const tempPath = file?.path || file?.filepath || file?.tempFilePath;
if (!tempPath) {
@@ -116,7 +109,7 @@ async function processUploadedPhotos(files, eventId, uploadedBy = 'admin', categ
throw new Error(`Uploaded file is missing a temporary path. File info: ${fileInfo}`);
}
// Verify temp file exists before copying
// Verify temp file exists before processing
try {
await fs.access(tempPath);
} catch (accessErr) {
@@ -127,56 +120,33 @@ async function processUploadedPhotos(files, eventId, uploadedBy = 'admin', categ
throw new Error(`Uploaded file not found at temporary location: ${tempPath}`);
}
// Use copyFile and unlink instead of rename to avoid cross-device issues
try {
await fs.copyFile(tempPath, newPath);
console.log(`Successfully copied ${file.originalname} to ${newPath}`);
} catch (copyErr) {
console.error(`Failed to copy file from ${tempPath} to ${newPath}:`, copyErr);
throw new Error(`Failed to copy uploaded file: ${copyErr.message}`);
} finally {
// Clean up temp file with better error handling
try {
await fs.unlink(tempPath);
console.log(`Cleaned up temp file: ${tempPath}`);
} catch (unlinkErr) {
// Only warn if file exists but couldn't be deleted
// ENOENT means file was already deleted, which is fine
if (unlinkErr?.code !== 'ENOENT') {
console.warn(`Failed to clean up temp upload ${tempPath}:`, {
error: unlinkErr.message,
code: unlinkErr.code
});
}
}
}
// Final storage key under events/active/{slug}/{newFilename}.
const relativePath = path.posix.join(event.slug, newFilename);
const finalKey = path.posix.join('events/active', relativePath);
// Determine if this is a video or image
const isVideo = isVideoMimeType(file.mimetype);
const mediaType = isVideo ? 'video' : 'image';
// Generate thumbnail and extract metadata
// Generate thumbnail and extract metadata FROM the temp file (still on
// local disk) before uploading the original.
let thumbnailPath;
let videoMetadata = null;
let imageMetadata = null;
if (isVideo) {
// Process video: extract metadata and generate thumbnail
const thumbnailDir = path.join(getStoragePath(), 'thumbnails');
await fs.mkdir(thumbnailDir, { recursive: true });
const videoThumbnailPath = path.join(thumbnailDir, `thumb_${newFilename.replace(/\.[^.]+$/, '.jpg')}`);
const result = await processUploadedVideo(newPath, videoThumbnailPath);
const videoThumbnailKey = path.posix.join(
'thumbnails',
`thumb_${newFilename.replace(/\.[^.]+$/, '.jpg')}`
);
const result = await processUploadedVideo(tempPath, videoThumbnailKey);
videoMetadata = result.metadata;
thumbnailPath = path.relative(getStoragePath(), videoThumbnailPath);
thumbnailPath = result.thumbnailKey;
} else {
// Process image: generate thumbnail and extract dimensions
thumbnailPath = await generateThumbnail(newPath);
// Extract image dimensions using sharp
thumbnailPath = await generateThumbnail(tempPath);
try {
const sharp = require('sharp');
const metadata = await sharp(newPath).metadata();
const metadata = await sharp(tempPath).metadata();
if (metadata.width && metadata.height) {
imageMetadata = {
width: metadata.width,
@@ -188,10 +158,29 @@ async function processUploadedPhotos(files, eventId, uploadedBy = 'admin', categ
}
}
// Calculate relative paths
const storagePath = getStoragePath();
const relativePath = path.relative(path.join(storagePath, 'events/active'), newPath);
const relativeThumbPath = thumbnailPath; // thumbnailPath is already relative to storage root
// Now upload the original through the storage backend and remove the
// local temp copy.
try {
await getStorage().putFromFile(finalKey, tempPath, {
contentType: file.mimetype,
});
} catch (uploadErr) {
console.error(`Failed to upload ${file.originalname}${finalKey}:`, uploadErr);
throw new Error(`Failed to upload to storage: ${uploadErr.message}`);
} finally {
try {
await fs.unlink(tempPath);
} catch (unlinkErr) {
if (unlinkErr?.code !== 'ENOENT') {
console.warn(`Failed to clean up temp upload ${tempPath}:`, {
error: unlinkErr.message,
code: unlinkErr.code
});
}
}
}
const relativeThumbPath = thumbnailPath;
// Add to database with uploaded_by field and media metadata
let insertResult;
@@ -247,7 +236,23 @@ async function processUploadedPhotos(files, eventId, uploadedBy = 'admin', categ
// Commit transaction
await trx.commit();
// Webhook (#327) — fires for every entry path that lands in this
// service: guest upload + auto-import + admin upload via API.
try {
const webhookService = require('./webhookService');
await webhookService.fire('photo.uploaded', {
event: { id: event.id, slug: event.slug, event_name: event.event_name },
photo: {
id: photoId,
filename: newFilename,
original_filename: file.originalname,
size_bytes: file.size,
uploaded_by: uploadedBy,
},
});
} catch (e) { /* non-fatal */ }
uploadedPhotos.push({
id: photoId,
filename: newFilename,
+31 -38
View File
@@ -13,10 +13,10 @@ const { db } = require('../database/db');
const { generateThumbnail, extractCaptureDate } = require('./imageProcessor');
const { generatePhotoFilename } = require('../utils/filenameSanitizer');
const watermarkGeneratorService = require('./watermarkGeneratorService');
const { getStorage } = require('./storage');
const { resolvePhotoStorageKey } = require('./photoResolver');
const logger = require('../utils/logger');
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
/**
* Find a replacement candidate by matching original_filename (case-insensitive).
* Returns the photo row if exactly one match, { ambiguous: true, count } if multiple, or null.
@@ -42,46 +42,21 @@ async function findReplacementCandidate(eventId, originalFilename) {
* @returns {{ success: boolean, photo?: Object, error?: string }}
*/
async function replacePhoto(existingPhoto, newFileTempPath, { originalFilename, mimeType, event }) {
const eventDir = path.join(getStoragePath(), 'events', 'active', event.slug);
const categorySlug = existingPhoto.type === 'collage' ? 'collages' : 'individual';
const targetDir = path.join(eventDir, categorySlug);
try {
// Generate new filename
// Generate new filename + storage key
const ext = path.extname(originalFilename);
const newFilename = generatePhotoFilename(event.event_name, categorySlug, Date.now(), ext);
const tempTargetPath = path.join(targetDir, `_replacing_${Date.now()}_${newFilename}`);
const finalPath = path.join(targetDir, newFilename);
const relativePath = path.join(event.slug, categorySlug, newFilename);
const relativePath = path.posix.join(event.slug, categorySlug, newFilename);
const finalKey = path.posix.join('events/active', relativePath);
const storage = getStorage();
// Write new file to temp name in target directory
await fsp.mkdir(targetDir, { recursive: true });
await fsp.copyFile(newFileTempPath, tempTargetPath);
// Delete old physical file
const oldFilePath = path.join(getStoragePath(), 'events', 'active', existingPhoto.path);
await fsp.unlink(oldFilePath).catch(() => {});
// Delete old thumbnail
if (existingPhoto.thumbnail_path) {
const oldThumbPath = path.join(getStoragePath(), existingPhoto.thumbnail_path);
await fsp.unlink(oldThumbPath).catch(() => {});
}
// Delete old watermark cache
try {
await watermarkGeneratorService.deleteForPhoto(existingPhoto.id);
} catch {
// Ignore — watermark may not exist
}
// Rename temp → final
await fsp.rename(tempTargetPath, finalPath);
// Extract metadata from new file
// Sharp/EXIF need a local file. The temp file from multer still satisfies
// that — we read metadata before uploading the original to storage.
let capturedAt = null;
try {
capturedAt = await extractCaptureDate(finalPath);
capturedAt = await extractCaptureDate(newFileTempPath);
} catch {
// No EXIF — keep null
}
@@ -89,23 +64,41 @@ async function replacePhoto(existingPhoto, newFileTempPath, { originalFilename,
let width = null;
let height = null;
try {
const metadata = await sharp(finalPath).metadata();
const metadata = await sharp(newFileTempPath).metadata();
width = metadata.width || null;
height = metadata.height || null;
} catch {
// Non-image or corrupt
}
const stats = await fsp.stat(finalPath);
const stats = await fsp.stat(newFileTempPath);
// Generate new thumbnail
// Generate new thumbnail FROM the local temp before uploading the original.
let thumbnailPath = null;
try {
thumbnailPath = await generateThumbnail(finalPath);
thumbnailPath = await generateThumbnail(newFileTempPath);
} catch {
logger.warn('Failed to generate thumbnail for replaced photo', { photoId: existingPhoto.id });
}
// Delete old assets BEFORE uploading the new key — if they share the path
// (rare but possible if filename collision), we want the new content.
const oldOriginalKey = resolvePhotoStorageKey(event, existingPhoto);
if (oldOriginalKey && oldOriginalKey !== finalKey) {
await storage.delete(oldOriginalKey).catch(() => {});
}
if (existingPhoto.thumbnail_path && existingPhoto.thumbnail_path !== thumbnailPath) {
await storage.delete(existingPhoto.thumbnail_path).catch(() => {});
}
try {
await watermarkGeneratorService.deleteForPhoto(existingPhoto.id);
} catch {
// Ignore — watermark may not exist
}
// Upload the new original.
await storage.putFromFile(finalKey, newFileTempPath, { contentType: mimeType });
// Update DB record — preserve id, event_id, category_id, type, visibility,
// uploaded_at, sort_order, feedback counts, view/download counts
const updates = {
+41
View File
@@ -4,6 +4,39 @@ const { safePathJoin } = require('../utils/fileSecurityUtils');
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
/**
* Resolve a managed photo's relative key under the storage backend.
* Returns null for external-mode photos (those never live in the managed
* storage backend; callers should fall back to resolvePhotoFilePath for
* external references on local disk).
*
* Storage layout (relative to STORAGE_PATH or S3 bucket prefix):
* events/active/{slug}/individual/{filename}
* events/active/{slug}/collages/{filename}
*
* Legacy `photo.path` values may already include `events/active/` we
* normalize so the returned key always has it exactly once.
*/
function resolvePhotoStorageKey(event, photo) {
if (!event || !photo) throw new Error('resolvePhotoStorageKey requires event and photo');
const mode = (photo.source_origin || event.source_mode || 'managed');
if (mode === 'reference' || mode === 'external') {
// External photos don't live in the managed backend.
return null;
}
const rel = photo.path ? photo.path.replace(/\\/g, '/').replace(/^\/+/, '') : '';
if (!rel) {
throw new Error(`resolvePhotoStorageKey: photo.path is empty for photo ${photo.id}`);
}
// Already prefixed (legacy uploads from a previous code revision).
if (rel.startsWith('events/active/')) return rel;
return path.posix.join('events/active', rel);
}
/**
* Resolve absolute photo file path based on event + photo origin
* Managed: storage/events/active + photo.path (legacy variants supported)
@@ -19,6 +52,13 @@ function resolvePhotoFilePath(event, photo) {
const mode = (photo.source_origin || event.source_mode || 'managed');
if (mode === 'reference' || mode === 'external') {
if (!photo.external_relpath) {
// Mixed-source events: a reference-mode event can also hold managed
// (uploaded) photos. If we have a regular `path` and no
// external_relpath, treat this row as managed instead of throwing.
if (photo.path && !photo.source_origin) {
const relativeSegment = photo.path.replace(/^\/+/, '');
return safePathJoin(path.join(getStoragePath(), 'events/active'), relativeSegment);
}
throw new Error('Missing external_relpath for external photo');
}
// Normalize duplicate leaf segments (e.g., event.external_path ends with 'individual'
@@ -51,4 +91,5 @@ function resolvePhotoFilePath(event, photo) {
module.exports = {
resolvePhotoFilePath,
resolvePhotoStorageKey,
};
+160
View File
@@ -0,0 +1,160 @@
/**
* S3 prefix walker that mirrors the chokidar `fileWatcher` for S3 mode.
*
* Why: in S3 mode the local file watcher is disabled (no inotify on remote
* objects). Without this, the only way to add photos is the upload API.
* This walker polls every event's storage prefix on a slow cadence and
* imports any new objects into the photos table.
*
* Eventual-consistency gate: an object is only imported after it has been
* SEEN for two consecutive polls. This avoids flapping when a list returns
* a freshly-uploaded object that disappears on the next list (a documented
* S3 behavior on certain backends).
*
* Opt-in via STORAGE_AUTO_IMPORT=true (off by default admins who don't
* need it shouldn't pay the API call cost).
*/
const path = require('path');
const mime = require('mime-types');
const { db } = require('../database/db');
const { formatBoolean } = require('../utils/dbCompat');
const { getStorage } = require('./storage');
const logger = require('../utils/logger');
const POLL_INTERVAL_MS = parseInt(process.env.STORAGE_AUTO_IMPORT_INTERVAL_MS || `${5 * 60 * 1000}`, 10);
const ENABLED = process.env.STORAGE_AUTO_IMPORT === 'true';
// Map<eventId, Set<storageKey>> — keys we saw on the previous poll.
// On the next poll, any key in BOTH the previous and current snapshots is
// eligible for import. This is the eventual-consistency gate.
const previousSnapshot = new Map();
let intervalHandle = null;
let stopped = false;
async function tick() {
if (stopped) return;
const storage = getStorage();
if (storage.kind() !== 's3') return; // no-op for local fs
try {
const events = await db('events')
.where({ is_active: formatBoolean(true), is_archived: formatBoolean(false) })
.select('id', 'slug');
for (const event of events) {
await processEvent(event, storage);
}
} catch (err) {
logger.error(`[s3AutoImporter] tick failed: ${err.message}`);
}
}
async function processEvent(event, storage) {
const prefix = path.posix.join('events/active', event.slug);
let entries = [];
try {
entries = await storage.list(prefix);
} catch (err) {
logger.warn(`[s3AutoImporter] list failed for event ${event.slug}: ${err.message}`);
return;
}
const currentKeys = new Set(entries.map((e) => e.key));
const lastSnapshot = previousSnapshot.get(event.id) || new Set();
// Eventual-consistency gate: only consider keys present in BOTH the
// previous tick's snapshot and the current one.
const stableKeys = entries.filter((e) => lastSnapshot.has(e.key));
if (stableKeys.length > 0) {
// Find keys not yet in photos table.
const stableKeyList = stableKeys.map((e) => e.key);
const eventsActivePrefix = 'events/active/';
const relativePaths = stableKeyList.map((k) =>
k.startsWith(eventsActivePrefix) ? k.slice(eventsActivePrefix.length) : k
);
const existing = await db('photos')
.where({ event_id: event.id })
.whereIn('path', relativePaths)
.select('path');
const existingPaths = new Set(existing.map((r) => r.path));
for (const entry of stableKeys) {
const relativePath = entry.key.startsWith(eventsActivePrefix)
? entry.key.slice(eventsActivePrefix.length)
: entry.key;
if (existingPaths.has(relativePath)) continue;
// Skip generated artifacts (thumbnails get their own keys; we don't
// want to re-register them as photos).
const filename = path.basename(entry.key);
if (filename.startsWith('thumb_') || filename.startsWith('hero_')) continue;
if (filename.startsWith('.')) continue; // dotfiles like .download-cache
const mimeType = mime.lookup(filename) || 'application/octet-stream';
const isImage = mimeType.startsWith('image/');
const isVideo = mimeType.startsWith('video/');
if (!isImage && !isVideo) continue;
try {
const insertResult = await db('photos').insert({
event_id: event.id,
filename,
original_filename: filename,
path: relativePath,
type: 'individual',
size_bytes: entry.size,
media_type: isVideo ? 'video' : 'image',
mime_type: mimeType,
source_origin: 'managed',
uploaded_at: new Date().toISOString(),
}).returning('id');
const photoId = insertResult[0]?.id || insertResult[0];
logger.info(`[s3AutoImporter] imported s3://.../${entry.key} → photo #${photoId} for event ${event.slug}`);
// Webhook (#327): same shape as fileWatcher.
try {
const webhookService = require('./webhookService');
await webhookService.fire('photo.uploaded', {
event: { id: event.id, slug: event.slug },
photo: { id: photoId, filename, size_bytes: entry.size, source: 's3-auto-import' },
});
} catch (e) { /* non-fatal */ }
} catch (err) {
logger.warn(`[s3AutoImporter] failed to insert ${entry.key}: ${err.message}`);
}
}
}
previousSnapshot.set(event.id, currentKeys);
}
function startS3AutoImporter() {
if (!ENABLED) return null;
if (intervalHandle) return intervalHandle;
stopped = false;
// Run once on startup so admins see import activity in logs without
// waiting for the first poll interval.
tick().catch((err) => logger.error(`[s3AutoImporter] initial tick error: ${err.message}`));
intervalHandle = setInterval(tick, POLL_INTERVAL_MS);
logger.info(`[s3AutoImporter] started — interval=${POLL_INTERVAL_MS}ms`);
return intervalHandle;
}
function stopS3AutoImporter() {
stopped = true;
if (intervalHandle) {
clearInterval(intervalHandle);
intervalHandle = null;
}
previousSnapshot.clear();
}
module.exports = {
startS3AutoImporter,
stopS3AutoImporter,
__test: { tick, processEvent, previousSnapshot, ENABLED, POLL_INTERVAL_MS },
};
@@ -0,0 +1,166 @@
const fs = require('fs');
const fsp = require('fs').promises;
const path = require('path');
const { pipeline } = require('stream/promises');
const crypto = require('crypto');
const logger = require('../../utils/logger');
/**
* Filesystem-backed implementation of the StorageBackend interface.
* All keys are relative to `root` (typically process.env.STORAGE_PATH).
*
* Path traversal protection: every key is normalized to POSIX form and rejected
* if it tries to escape the root via "..". Callers should not need to think
* about this but if a key arrives via user input it must still be filtered.
*/
class LocalFsStorage {
constructor({ root }) {
if (!root) throw new Error('LocalFsStorage requires a `root` directory');
this.root = path.resolve(root);
}
kind() {
return 'local';
}
async init() {
await fsp.mkdir(this.root, { recursive: true });
// Sanity check: must be writable.
const probe = path.join(this.root, '.storage-write-probe');
await fsp.writeFile(probe, '');
await fsp.unlink(probe);
logger.info(`[storage] LocalFsStorage initialized at ${this.root}`);
}
_resolve(relPath) {
if (!relPath || typeof relPath !== 'string') {
throw new Error(`LocalFsStorage: invalid relative path: ${relPath}`);
}
const normalized = path.posix.normalize(relPath.replace(/\\/g, '/'));
if (normalized.startsWith('..') || normalized.includes('/../') || normalized === '..') {
throw new Error(`LocalFsStorage: path traversal rejected: ${relPath}`);
}
return path.join(this.root, normalized);
}
async put(relPath, body, _options = {}) {
const abs = this._resolve(relPath);
await fsp.mkdir(path.dirname(abs), { recursive: true });
// Write to a sibling tmp file first then rename for crash safety.
const tmp = `${abs}.tmp.${process.pid}.${crypto.randomBytes(4).toString('hex')}`;
try {
if (Buffer.isBuffer(body)) {
await fsp.writeFile(tmp, body);
} else if (body && typeof body.pipe === 'function') {
await pipeline(body, fs.createWriteStream(tmp));
} else {
throw new Error('LocalFsStorage.put: body must be a Buffer or Readable stream');
}
await fsp.rename(tmp, abs);
} catch (err) {
await fsp.unlink(tmp).catch(() => {});
throw err;
}
}
async putFromFile(relPath, localPath, _options = {}) {
const abs = this._resolve(relPath);
await fsp.mkdir(path.dirname(abs), { recursive: true });
// copyFile is atomic from the destination's perspective on POSIX.
await fsp.copyFile(localPath, abs);
}
async get(relPath) {
const abs = this._resolve(relPath);
return fs.createReadStream(abs);
}
async getToFile(relPath, localPath) {
const abs = this._resolve(relPath);
await fsp.mkdir(path.dirname(localPath), { recursive: true });
await fsp.copyFile(abs, localPath);
}
async exists(relPath) {
try {
await fsp.access(this._resolve(relPath), fs.constants.F_OK);
return true;
} catch {
return false;
}
}
async stat(relPath) {
try {
const s = await fsp.stat(this._resolve(relPath));
return { size: s.size, mtime: s.mtime };
} catch (err) {
if (err.code === 'ENOENT') return null;
throw err;
}
}
async delete(relPath) {
try {
await fsp.unlink(this._resolve(relPath));
} catch (err) {
if (err.code !== 'ENOENT') throw err;
}
}
async list(prefix) {
const absPrefix = this._resolve(prefix || '.');
const entries = [];
async function walk(dir, relBase) {
let dirents;
try {
dirents = await fsp.readdir(dir, { withFileTypes: true });
} catch (err) {
if (err.code === 'ENOENT') return;
throw err;
}
for (const ent of dirents) {
const childAbs = path.join(dir, ent.name);
const childRel = relBase ? `${relBase}/${ent.name}` : ent.name;
if (ent.isDirectory()) {
await walk(childAbs, childRel);
} else if (ent.isFile()) {
const s = await fsp.stat(childAbs);
entries.push({ key: childRel, size: s.size, mtime: s.mtime });
}
}
}
const baseRel = prefix && prefix !== '.' ? prefix.replace(/\\/g, '/') : '';
await walk(absPrefix, baseRel);
return entries;
}
async rename(srcRelPath, dstRelPath) {
const src = this._resolve(srcRelPath);
const dst = this._resolve(dstRelPath);
await fsp.mkdir(path.dirname(dst), { recursive: true });
await fsp.rename(src, dst);
}
async copy(srcRelPath, dstRelPath) {
const src = this._resolve(srcRelPath);
const dst = this._resolve(dstRelPath);
await fsp.mkdir(path.dirname(dst), { recursive: true });
await fsp.copyFile(src, dst);
}
async signedUrl(_relPath, _ttlSeconds = 300) {
throw new Error('LocalFsStorage does not support signedUrl. Set STORAGE_BACKEND=s3 to use presigned URLs.');
}
// Escape hatch for callers that genuinely need a filesystem path
// (e.g. ffmpeg, archiver — anything that takes a path argument rather
// than a stream). S3Storage exposes the same method but returns null,
// forcing callers to use the streaming API instead.
resolveLocalPath(relPath) {
return this._resolve(relPath);
}
}
module.exports = LocalFsStorage;
@@ -0,0 +1,165 @@
const path = require('path');
const fs = require('fs');
const fsp = require('fs').promises;
const { HeadObjectCommand } = require('@aws-sdk/client-s3');
const S3StorageAdapter = require('./s3Storage');
const logger = require('../../utils/logger');
/**
* StorageBackend wrapper around the existing S3StorageAdapter.
*
* S3StorageAdapter was originally written for the backup service and exposes
* upload/download/uploadStream/etc. This thin layer maps that surface onto the
* canonical put/get/exists/delete/list/rename/copy/signedUrl interface used by
* the rest of the codebase, and applies an optional `prefix` so a single bucket
* can host multiple deployments without collisions.
*
* Atomicity: S3 has no rename. `rename()` is implemented as `copy()` + `delete()`.
* If the process crashes between the two, the source object remains until the
* next list-and-prune sweep see `cleanupAbandonedTempUploads()` callers.
*/
class S3StorageBackend {
constructor(config) {
if (!config || !config.bucket) {
throw new Error('S3StorageBackend requires a bucket name');
}
this.adapter = new S3StorageAdapter(config);
this.prefix = (config.prefix || '').replace(/^\/+|\/+$/g, '');
}
kind() {
return 's3';
}
_key(relPath) {
if (!relPath || typeof relPath !== 'string') {
throw new Error(`S3StorageBackend: invalid relative path: ${relPath}`);
}
const normalized = relPath.replace(/\\/g, '/').replace(/^\.?\/+/, '');
if (normalized.startsWith('..') || normalized.includes('/../')) {
throw new Error(`S3StorageBackend: path traversal rejected: ${relPath}`);
}
return this.prefix ? `${this.prefix}/${normalized}` : normalized;
}
async init() {
await this.adapter.testConnection();
logger.info(`[storage] S3StorageBackend initialized bucket=${this.adapter.bucket} prefix=${this.prefix || '(none)'}`);
}
async put(relPath, body, options = {}) {
const key = this._key(relPath);
if (Buffer.isBuffer(body)) {
const { Readable } = require('stream');
const stream = Readable.from(body);
await this.adapter.uploadStream(stream, key, {
contentType: options.contentType,
cacheControl: options.cacheControl,
});
return;
}
if (body && typeof body.pipe === 'function') {
await this.adapter.uploadStream(body, key, {
contentType: options.contentType,
cacheControl: options.cacheControl,
});
return;
}
throw new Error('S3StorageBackend.put: body must be a Buffer or Readable stream');
}
async putFromFile(relPath, localPath, options = {}) {
await this.adapter.upload(localPath, this._key(relPath), {
contentType: options.contentType,
cacheControl: options.cacheControl,
});
}
async get(relPath) {
return this.adapter.downloadStream(this._key(relPath));
}
async getToFile(relPath, localPath) {
await fsp.mkdir(path.dirname(localPath), { recursive: true });
await this.adapter.download(this._key(relPath), localPath);
}
async exists(relPath) {
return this.adapter.exists(this._key(relPath));
}
async stat(relPath) {
try {
const head = await this.adapter.s3Client.send(
new HeadObjectCommand({ Bucket: this.adapter.bucket, Key: this._key(relPath) })
);
return {
size: head.ContentLength,
mtime: head.LastModified,
};
} catch (err) {
if (err.name === 'NotFound' || err.$metadata?.httpStatusCode === 404) return null;
throw err;
}
}
async delete(relPath) {
try {
await this.adapter.delete(this._key(relPath));
} catch (err) {
if (err.name === 'NoSuchKey' || err.$metadata?.httpStatusCode === 404) return;
throw err;
}
}
async list(prefix) {
const fullPrefix = this._key(prefix || '.');
const entries = [];
let continuationToken;
do {
const result = await this.adapter.list(fullPrefix, { continuationToken });
for (const obj of result.Contents || []) {
const stripped = this.prefix && obj.Key.startsWith(`${this.prefix}/`)
? obj.Key.slice(this.prefix.length + 1)
: obj.Key;
entries.push({ key: stripped, size: obj.Size, mtime: obj.LastModified });
}
continuationToken = result.NextContinuationToken;
} while (continuationToken);
return entries;
}
async copy(srcRelPath, dstRelPath) {
await this.adapter.copy(this._key(srcRelPath), this._key(dstRelPath));
}
async rename(srcRelPath, dstRelPath) {
await this.copy(srcRelPath, dstRelPath);
await this.delete(srcRelPath);
}
async signedUrl(relPath, ttlSeconds = 300) {
return this.adapter.getSignedUrl('getObject', this._key(relPath), { expiresIn: ttlSeconds });
}
// S3 has no local path; consumers that need one must use getToFile to a
// temp location first. Returning null here makes the contract explicit so
// legacy code using `storage.resolveLocalPath` fails fast instead of
// silently constructing a bad path.
resolveLocalPath(_relPath) {
return null;
}
// Expose the underlying adapter so backupService keeps working.
// New code should prefer the canonical interface above.
get rawAdapter() {
return this.adapter;
}
static fileStreamFromPath(localPath) {
return fs.createReadStream(localPath);
}
}
module.exports = S3StorageBackend;
@@ -0,0 +1,42 @@
/**
* Storage backend interface that LocalFsStorage and S3Storage implement.
*
* All paths are POSIX-style relative keys under the deployment's storage root
* (e.g. "events/active/wedding-smith/individual/IMG_0001.jpg"). Concrete adapters
* resolve the absolute filesystem path or S3 key internally so callers never deal
* with the difference between local and remote storage.
*
* Concurrency: methods are safe to call in parallel; ordering is the caller's
* responsibility. `put` is best-effort atomic (LocalFs writes to a temp file
* then renames; S3 returns only after the multipart upload is finalized).
*
* @typedef {Object} PutOptions
* @property {string} [contentType] - MIME type stored in object metadata.
* @property {string} [cacheControl] - Cache-Control header (S3 only).
*
* @typedef {Object} StatResult
* @property {number} size - Size in bytes.
* @property {Date} [mtime] - Last modified timestamp (best-effort; S3 uses LastModified).
*
* @typedef {Object} ListEntry
* @property {string} key - Relative path under the storage root.
* @property {number} size - Size in bytes.
* @property {Date} [mtime] - Last modified timestamp.
*
* @typedef {Object} StorageBackend
* @property {() => string} kind - Returns 'local' or 's3'.
* @property {() => Promise<void>} init - Validates configuration and reachability. Called once at startup.
* @property {(relPath: string, body: NodeJS.ReadableStream | Buffer, options?: PutOptions) => Promise<void>} put
* @property {(relPath: string, localPath: string, options?: PutOptions) => Promise<void>} putFromFile
* @property {(relPath: string) => Promise<NodeJS.ReadableStream>} get - Returns a readable stream of the object body.
* @property {(relPath: string, localPath: string) => Promise<void>} getToFile - Streams the object to a local path (creates parent dirs).
* @property {(relPath: string) => Promise<boolean>} exists
* @property {(relPath: string) => Promise<StatResult|null>} stat - Null if missing.
* @property {(relPath: string) => Promise<void>} delete - No-op if missing.
* @property {(prefix: string) => Promise<ListEntry[]>} list
* @property {(srcRelPath: string, dstRelPath: string) => Promise<void>} rename - Atomic on local fs; copy+delete on S3.
* @property {(srcRelPath: string, dstRelPath: string) => Promise<void>} copy
* @property {(relPath: string, ttlSeconds?: number) => Promise<string>} signedUrl - Presigned download URL (S3 only; LocalFs throws).
*/
module.exports = {};
+102
View File
@@ -0,0 +1,102 @@
const LocalFsStorage = require('./LocalFsStorage');
const S3StorageBackend = require('./S3StorageBackend');
const { getStoragePath } = require('../../config/storage');
const logger = require('../../utils/logger');
let instance = null;
/**
* Build the storage backend selected by STORAGE_BACKEND env var.
*
* STORAGE_BACKEND=local (default)
* Uses STORAGE_PATH on the local filesystem. Backwards compatible with every
* existing deployment.
*
* STORAGE_BACKEND=s3
* Reads STORAGE_S3_* vars. Compatible with AWS S3 and any S3-compatible
* service (MinIO, R2, Backblaze, Wasabi, DigitalOcean Spaces, etc.) by
* pointing STORAGE_S3_ENDPOINT at the alternate host.
*
* Required S3 vars:
* STORAGE_S3_BUCKET
* STORAGE_S3_REGION (default us-east-1)
* STORAGE_S3_ACCESS_KEY
* STORAGE_S3_SECRET_KEY
* Optional S3 vars:
* STORAGE_S3_ENDPOINT custom endpoint URL (MinIO/R2/etc.)
* STORAGE_S3_PREFIX namespace prefix inside the bucket
* STORAGE_S3_FORCE_PATH_STYLE=true|false (default: auto when endpoint set)
* STORAGE_S3_SSL=true|false (default: true)
*/
function buildStorage() {
const backend = (process.env.STORAGE_BACKEND || 'local').toLowerCase();
if (backend === 's3') {
const required = ['STORAGE_S3_BUCKET', 'STORAGE_S3_ACCESS_KEY', 'STORAGE_S3_SECRET_KEY'];
const missing = required.filter((v) => !process.env[v]);
if (missing.length) {
throw new Error(
`STORAGE_BACKEND=s3 but missing required env vars: ${missing.join(', ')}`
);
}
return new S3StorageBackend({
bucket: process.env.STORAGE_S3_BUCKET,
region: process.env.STORAGE_S3_REGION || 'us-east-1',
endpoint: process.env.STORAGE_S3_ENDPOINT,
accessKeyId: process.env.STORAGE_S3_ACCESS_KEY,
secretAccessKey: process.env.STORAGE_S3_SECRET_KEY,
prefix: process.env.STORAGE_S3_PREFIX,
forcePathStyle: process.env.STORAGE_S3_FORCE_PATH_STYLE === 'true' ? true : undefined,
sslEnabled: process.env.STORAGE_S3_SSL !== 'false',
});
}
if (backend !== 'local') {
throw new Error(`Unknown STORAGE_BACKEND: ${backend}. Expected 'local' or 's3'.`);
}
return new LocalFsStorage({ root: getStoragePath() });
}
/**
* Lazily build + memoize the storage backend. Tests can pass an injected
* instance via `setStorageForTesting` to bypass env-var configuration.
*/
function getStorage() {
if (!instance) {
instance = buildStorage();
}
return instance;
}
/** @internal */
function setStorageForTesting(stub) {
instance = stub;
}
/** @internal — clear the memoized instance so the next call re-reads env. */
function resetStorage() {
instance = null;
}
/**
* Initialize the configured backend. Call once at server startup so config errors
* surface before any request comes in.
*/
async function initStorage() {
const storage = getStorage();
try {
await storage.init();
} catch (err) {
logger.error(`[storage] init failed for backend=${storage.kind()}: ${err.message}`);
throw err;
}
return storage;
}
module.exports = {
getStorage,
initStorage,
setStorageForTesting,
resetStorage,
};
+55 -43
View File
@@ -2,7 +2,11 @@ const ffmpeg = require('fluent-ffmpeg');
const ffmpegPath = require('@ffmpeg-installer/ffmpeg').path;
const path = require('path');
const fs = require('fs').promises;
const fsSync = require('fs');
const os = require('os');
const crypto = require('crypto');
const logger = require('../utils/logger');
const { getStorage } = require('./storage');
// Set FFmpeg path
ffmpeg.setFfmpegPath(ffmpegPath);
@@ -45,36 +49,48 @@ async function extractVideoMetadata(videoPath) {
}
/**
* Generate thumbnail from video
* @param {string} videoPath - Path to the video file
* @param {string} outputPath - Path for the output thumbnail
* @param {Object} options - Thumbnail options
* @returns {Promise<string>} - Path to generated thumbnail
* Generate a video thumbnail and persist it via the storage backend.
*
* @param {string} videoPath - Local path to the video file (ffmpeg needs a real fs path).
* @param {string} thumbnailKey - Relative storage key the thumbnail will be saved under
* (e.g. "thumbnails/thumb_video.jpg").
* @param {Object} options
* @returns {Promise<string>} The thumbnail's relative storage key.
*/
async function generateVideoThumbnail(videoPath, outputPath, options = {}) {
async function generateVideoThumbnail(videoPath, thumbnailKey, options = {}) {
const {
timeOffset = '00:00:01', // Take screenshot at 1 second
size = '300x300',
quality = 2 // 1-31, lower is better quality
timeOffset = '00:00:01',
size = '300x300'
} = options;
return new Promise((resolve, reject) => {
ffmpeg(videoPath)
.screenshots({
timestamps: [timeOffset],
filename: path.basename(outputPath),
folder: path.dirname(outputPath),
size: size
})
.on('end', () => {
logger.info('Video thumbnail generated', { videoPath, outputPath });
resolve(outputPath);
})
.on('error', (err) => {
logger.error('Error generating video thumbnail', { error: err.message, videoPath });
reject(err);
});
});
const storage = getStorage();
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'picpeak-vidthumb-'));
const tmpFilename = `${crypto.randomBytes(4).toString('hex')}_${path.basename(thumbnailKey)}`;
const tmpPath = path.join(tmpDir, tmpFilename);
try {
await new Promise((resolve, reject) => {
ffmpeg(videoPath)
.screenshots({
timestamps: [timeOffset],
filename: tmpFilename,
folder: tmpDir,
size: size
})
.on('end', () => resolve())
.on('error', (err) => reject(err));
});
if (!fsSync.existsSync(tmpPath)) {
throw new Error('ffmpeg did not produce a thumbnail file');
}
await storage.putFromFile(thumbnailKey, tmpPath, { contentType: 'image/jpeg' });
logger.info('Video thumbnail generated', { videoPath, thumbnailKey });
return thumbnailKey;
} finally {
await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
}
}
/**
@@ -108,37 +124,33 @@ async function getVideoDuration(videoPath) {
}
/**
* Process uploaded video - extract metadata and generate thumbnail
* @param {string} videoPath - Path to the video file
* @param {string} thumbnailPath - Path for the thumbnail
* @param {Object} options - Processing options
* @returns {Promise<Object>} - Video metadata and processing result
* Process an uploaded video: extract metadata and produce a thumbnail through
* the storage backend.
*
* @param {string} videoPath - Local path to the source video (ffmpeg requires fs).
* @param {string} thumbnailKey - Relative storage key for the thumbnail.
* @returns {Promise<{success: boolean, metadata: Object, thumbnailKey: string}>}
*/
async function processUploadedVideo(videoPath, thumbnailPath, options = {}) {
async function processUploadedVideo(videoPath, thumbnailKey, options = {}) {
try {
// Validate video
const isValid = await isValidVideo(videoPath);
if (!isValid) {
throw new Error('Invalid video file');
}
// Extract metadata
const metadata = await extractVideoMetadata(videoPath);
await generateVideoThumbnail(videoPath, thumbnailKey, options);
// Generate thumbnail
await generateVideoThumbnail(videoPath, thumbnailPath, options);
// Verify thumbnail was created
try {
await fs.access(thumbnailPath);
} catch (err) {
throw new Error('Thumbnail generation failed');
const storage = getStorage();
const exists = await storage.exists(thumbnailKey);
if (!exists) {
throw new Error('Thumbnail generation failed (not in storage)');
}
return {
success: true,
metadata,
thumbnailPath
thumbnailKey
};
} catch (error) {
logger.error('Error processing video', { error: error.message, videoPath });
@@ -8,10 +8,10 @@
* - Tracking regeneration progress
*/
const path = require('path');
const { db } = require('../database/db');
const watermarkService = require('./watermarkService');
const { getStoragePath } = require('../config/storage');
const { resolvePhotoStorageKey, resolvePhotoFilePath } = require('./photoResolver');
const { withLocalCopy } = require('./imageProcessor');
class WatermarkGeneratorService {
constructor() {
@@ -57,14 +57,15 @@ class WatermarkGeneratorService {
return { success: false, error: 'Watermarking is disabled' };
}
// Resolve the original file path
const originalPath = this.resolvePhotoPath(photo);
if (!originalPath) {
return { success: false, error: 'Could not resolve photo path' };
}
// Generate and save watermark
const result = await watermarkService.generateAndSaveWatermark(photo, originalPath, settings);
// Resolve the source via the storage backend (managed) or local disk
// (external reference mode). watermarkService needs a local file path.
const event = { slug: photo.slug, source_mode: photo.source_mode, external_path: photo.external_path };
const storageKey = resolvePhotoStorageKey(event, photo);
const result = storageKey
? await withLocalCopy(storageKey, (lp) =>
watermarkService.generateAndSaveWatermark(photo, lp, settings)
)
: await watermarkService.generateAndSaveWatermark(photo, resolvePhotoFilePath(event, photo), settings);
if (result.success) {
// Update database with watermark path
@@ -83,31 +84,6 @@ class WatermarkGeneratorService {
}
}
/**
* Resolve the full file path for a photo
*/
resolvePhotoPath(photo) {
const storagePath = getStoragePath();
// Handle external/reference mode
if (photo.source_mode === 'reference' && photo.external_relpath) {
const externalRoot = process.env.EXTERNAL_MEDIA_PATH || path.join(storagePath, 'external');
return path.join(externalRoot, photo.external_path || '', photo.external_relpath);
}
// Standard managed mode
if (photo.file_path) {
// file_path might be absolute or relative
if (path.isAbsolute(photo.file_path)) {
return photo.file_path;
}
return path.join(storagePath, photo.file_path);
}
// Fallback to constructing path from slug and filename
return path.join(storagePath, 'events', 'active', photo.slug, photo.filename);
}
/**
* Generate watermarks for all photos in an event
* @param {number} eventId - The event ID
@@ -189,12 +165,13 @@ class WatermarkGeneratorService {
*/
async processPhotoWatermark(photo, settings) {
try {
const originalPath = this.resolvePhotoPath(photo);
if (!originalPath) {
return { success: false, photoId: photo.id, error: 'Could not resolve path' };
}
const result = await watermarkService.generateAndSaveWatermark(photo, originalPath, settings);
const event = { slug: photo.slug, source_mode: photo.source_mode, external_path: photo.external_path };
const storageKey = resolvePhotoStorageKey(event, photo);
const result = storageKey
? await withLocalCopy(storageKey, (lp) =>
watermarkService.generateAndSaveWatermark(photo, lp, settings)
)
: await watermarkService.generateAndSaveWatermark(photo, resolvePhotoFilePath(event, photo), settings);
if (result.success) {
await db('photos')
+16 -37
View File
@@ -2,7 +2,7 @@ const sharp = require('sharp');
const path = require('path');
const fs = require('fs').promises;
const { db } = require('../database/db');
const { getStoragePath } = require('../config/storage');
const { getStorage } = require('./storage');
class WatermarkService {
constructor() {
@@ -234,19 +234,6 @@ class WatermarkService {
this.cache.clear();
}
/**
* Get the watermarks directory path, creating it if needed
*/
async getWatermarksDir() {
const watermarksDir = path.join(getStoragePath(), 'watermarks');
try {
await fs.access(watermarksDir);
} catch {
await fs.mkdir(watermarksDir, { recursive: true });
}
return watermarksDir;
}
/**
* Get the file extension from a filename
*/
@@ -258,46 +245,42 @@ class WatermarkService {
}
/**
* Generate watermarked version of a photo and save to disk
* Generate watermarked version of a photo and persist it through the
* storage backend. The source must be a local filesystem path because
* sharp doesn't take streams; callers in S3 mode should materialize a
* tmp local copy via imageProcessor.withLocalCopy first.
*
* @param {Object} photo - Photo object with id, filename, and path info
* @param {string} originalPath - Full path to the original image file
* @param {string} originalPath - Local path to the original image file
* @param {Object} settings - Watermark settings (optional, will fetch if not provided)
* @returns {Object} { success, watermarkPath, error }
*/
async generateAndSaveWatermark(photo, originalPath, settings = null) {
try {
// Get settings if not provided
if (!settings) {
settings = await this.getWatermarkSettings();
}
// If watermarking is disabled, return early
if (!settings || !settings.enabled) {
return { success: false, watermarkPath: null, error: 'Watermarking is disabled' };
}
// Verify original file exists
try {
await fs.access(originalPath);
} catch {
return { success: false, watermarkPath: null, error: 'Original file not found' };
}
// Generate watermarked buffer using existing method
const watermarkedBuffer = await this.applyWatermark(originalPath, settings);
// Determine output path
const watermarksDir = await this.getWatermarksDir();
const ext = this.getFileExtension(photo.filename);
const outputFilename = `${photo.id}_watermarked${ext}`;
const outputPath = path.join(watermarksDir, outputFilename);
// Write the watermarked image to disk
await fs.writeFile(outputPath, watermarkedBuffer);
// Return relative path for database storage
const relativePath = `watermarks/${outputFilename}`;
await getStorage().put(relativePath, watermarkedBuffer, {
contentType: ext === '.png' ? 'image/png' : ext === '.webp' ? 'image/webp' : 'image/jpeg',
});
return {
success: true,
watermarkPath: relativePath,
@@ -314,22 +297,18 @@ class WatermarkService {
}
/**
* Delete a pre-generated watermark file
* @param {string} watermarkPath - Relative path to the watermark file
* @returns {boolean} - True if deleted successfully
* Delete a pre-generated watermark file from the storage backend.
* @param {string} watermarkPath - Relative storage key (e.g. "watermarks/123_watermarked.jpg")
* @returns {boolean} - True if a delete was attempted (no-op if missing)
*/
async deleteWatermarkFile(watermarkPath) {
if (!watermarkPath) return false;
try {
const fullPath = path.join(getStoragePath(), watermarkPath);
await fs.unlink(fullPath);
await getStorage().delete(watermarkPath);
return true;
} catch (error) {
// File might not exist, which is fine
if (error.code !== 'ENOENT') {
console.error('Error deleting watermark file:', error);
}
console.error('Error deleting watermark file:', error);
return false;
}
}
@@ -0,0 +1,277 @@
const axios = require('axios');
const { db } = require('../database/db');
const logger = require('../utils/logger');
const { signPayload, renderTemplate } = require('./webhookService');
const { validateExternalUrl } = require('../utils/networkValidation');
const POLL_INTERVAL_MS = parseInt(process.env.WEBHOOK_DELIVERY_INTERVAL_MS || '5000', 10);
const CONCURRENCY = parseInt(process.env.WEBHOOK_DELIVERY_CONCURRENCY || '5', 10);
const HTTP_TIMEOUT_MS = parseInt(process.env.WEBHOOK_HTTP_TIMEOUT_MS || '10000', 10);
const MAX_ATTEMPTS = parseInt(process.env.WEBHOOK_MAX_ATTEMPTS || '5', 10);
const RESPONSE_TRUNCATE_BYTES = 1024;
// Mutable so tests can flip it without juggling require.cache; reads the
// env var at module load for the production code path.
let allowPrivateUrls = process.env.WEBHOOK_ALLOW_PRIVATE_URLS === 'true';
const SIGNATURE_HEADER = 'X-PicPeak-Signature';
const EVENT_HEADER = 'X-PicPeak-Event';
const DELIVERY_HEADER = 'X-PicPeak-Delivery';
// Backoff schedule per the issue spec — index = attempt that just failed.
// attempt_count after the failure becomes (failedAttempt + 1); we look up
// the delay using the *new* attempt count to schedule the next try.
// attempt 1 fails → wait 1m
// attempt 2 fails → wait 5m
// attempt 3 fails → wait 30m
// attempt 4 fails → wait 2h
// attempt 5 fails → wait 12h THEN give up (max 5 attempts total)
const BACKOFF_MS = [
60_000, // 1 min
5 * 60_000, // 5 min
30 * 60_000, // 30 min
2 * 60 * 60_000, // 2 h
12 * 60 * 60_000, // 12 h (only used when MAX_ATTEMPTS extended past 5)
];
let intervalHandle = null;
let stopped = false;
// Tracks deliveries currently being processed in this tick — guards
// against the same row being claimed twice if a tick takes longer than
// POLL_INTERVAL_MS.
const inFlight = new Set();
function truncate(str, bytes) {
if (str == null) return null;
const buf = Buffer.from(String(str), 'utf8');
if (buf.length <= bytes) return buf.toString('utf8');
return buf.subarray(0, bytes).toString('utf8');
}
async function fetchPending(limit) {
// Skip rows already in-flight from a previous tick that's still running.
const excludeIds = Array.from(inFlight);
let q = db('webhook_deliveries')
.where('status', 'pending')
.where('next_retry_at', '<=', new Date())
.orderBy('next_retry_at', 'asc')
.limit(limit);
if (excludeIds.length > 0) {
q = q.whereNotIn('id', excludeIds);
}
return q.select('*');
}
async function deliverOne(row) {
const startedAt = Date.now();
const webhook = await db('webhooks').where({ id: row.webhook_id }).first();
if (!webhook) {
// Webhook was deleted while a delivery was pending. Mark failed and move on.
await db('webhook_deliveries')
.where({ id: row.id })
.update({
status: 'failed',
last_error: 'webhook subscription no longer exists',
completed_at: new Date(),
attempt_count: row.attempt_count + 1,
});
return;
}
if (!webhook.active) {
// Subscription disabled mid-flight. Don't abandon — leave as failed
// so the deliveries page reflects the reality.
await db('webhook_deliveries')
.where({ id: row.id })
.update({
status: 'failed',
last_error: 'webhook is disabled',
completed_at: new Date(),
attempt_count: row.attempt_count + 1,
});
return;
}
// Re-validate URL per delivery — DNS-rebinding mitigation. Admin can opt
// out via WEBHOOK_ALLOW_PRIVATE_URLS=true for local-receiver dev runs.
if (!allowPrivateUrls) {
const urlCheck = validateExternalUrl(webhook.url);
if (!urlCheck.valid) {
await markFailedFinal(row, `URL rejected: ${urlCheck.error}`);
return;
}
}
const envelopeBody = typeof row.payload === 'string' ? row.payload : JSON.stringify(row.payload);
const envelopeObj = (() => {
try { return JSON.parse(envelopeBody); } catch { return {}; }
})();
// Per-webhook template (#327 follow-up). If set + valid, replaces the
// default JSON envelope as the request body. Signature is computed over
// the BODY ACTUALLY SENT, so receivers verify whatever they receive.
let rawBody = envelopeBody;
let contentType = 'application/json';
if (webhook.template) {
const rendered = renderTemplate(webhook.template, envelopeObj);
if (rendered != null) {
rawBody = rendered;
// Best-effort content-type detection: if it parses as JSON, keep
// application/json; otherwise send as text/plain.
try { JSON.parse(rendered); } catch { contentType = 'text/plain; charset=utf-8'; }
}
}
const signature = signPayload(webhook.secret, rawBody);
const deliveryId = envelopeObj?.id || String(row.id);
let response;
let networkError;
try {
response = await axios.post(webhook.url, rawBody, {
headers: {
'Content-Type': contentType,
[SIGNATURE_HEADER]: signature,
[EVENT_HEADER]: row.event_type,
[DELIVERY_HEADER]: deliveryId,
'User-Agent': 'PicPeak-Webhooks/1.0',
},
timeout: HTTP_TIMEOUT_MS,
// Don't throw on non-2xx; we handle status manually.
validateStatus: () => true,
// Don't follow redirects — security + receivers should give us the
// final URL up front.
maxRedirects: 0,
// Cap response body so a chatty receiver can't OOM us before truncation.
maxContentLength: 10 * 1024,
maxBodyLength: rawBody.length + 1024,
});
} catch (err) {
networkError = err;
}
const latency = Date.now() - startedAt;
const newAttempt = row.attempt_count + 1;
if (response && response.status >= 200 && response.status < 300) {
await db('webhook_deliveries')
.where({ id: row.id })
.update({
status: 'success',
response_status: response.status,
response_body: truncate(stringifyBody(response.data), RESPONSE_TRUNCATE_BYTES),
latency_ms: latency,
attempt_count: newAttempt,
completed_at: new Date(),
next_retry_at: null,
});
await db('webhooks').where({ id: webhook.id }).update({ last_success_at: new Date() });
return;
}
// Failure path — schedule retry or give up.
const errorMsg = networkError
? `network error: ${networkError.code || networkError.message}`
: `non-2xx status: ${response?.status}`;
if (newAttempt >= MAX_ATTEMPTS) {
await db('webhook_deliveries')
.where({ id: row.id })
.update({
status: 'failed',
response_status: response?.status || null,
response_body: response ? truncate(stringifyBody(response.data), RESPONSE_TRUNCATE_BYTES) : null,
last_error: errorMsg,
latency_ms: latency,
attempt_count: newAttempt,
completed_at: new Date(),
next_retry_at: null,
});
await db('webhooks').where({ id: webhook.id }).update({ last_failure_at: new Date() });
return;
}
const backoff = BACKOFF_MS[Math.min(newAttempt - 1, BACKOFF_MS.length - 1)];
await db('webhook_deliveries')
.where({ id: row.id })
.update({
status: 'pending',
response_status: response?.status || null,
response_body: response ? truncate(stringifyBody(response.data), RESPONSE_TRUNCATE_BYTES) : null,
last_error: errorMsg,
latency_ms: latency,
attempt_count: newAttempt,
next_retry_at: new Date(Date.now() + backoff),
});
await db('webhooks').where({ id: webhook.id }).update({ last_failure_at: new Date() });
}
async function markFailedFinal(row, reason) {
await db('webhook_deliveries')
.where({ id: row.id })
.update({
status: 'failed',
last_error: reason,
attempt_count: row.attempt_count + 1,
completed_at: new Date(),
next_retry_at: null,
});
await db('webhooks').where({ id: row.webhook_id }).update({ last_failure_at: new Date() });
}
function stringifyBody(data) {
if (data == null) return null;
if (typeof data === 'string') return data;
if (Buffer.isBuffer(data)) return data.toString('utf8');
try { return JSON.stringify(data); } catch { return String(data); }
}
async function tick() {
if (stopped) return;
try {
const slots = Math.max(0, CONCURRENCY - inFlight.size);
if (slots === 0) return;
const rows = await fetchPending(slots);
if (rows.length === 0) return;
rows.forEach((r) => inFlight.add(r.id));
await Promise.allSettled(
rows.map((r) =>
deliverOne(r)
.catch((err) => logger.error(`[webhookWorker] delivery ${r.id} crashed: ${err.message}`))
.finally(() => inFlight.delete(r.id))
)
);
} catch (err) {
logger.error(`[webhookWorker] tick failed: ${err.message}`);
}
}
function startWebhookDeliveryWorker() {
if (intervalHandle) return; // idempotent
stopped = false;
intervalHandle = setInterval(tick, POLL_INTERVAL_MS);
logger.info(
`[webhookWorker] started — interval=${POLL_INTERVAL_MS}ms, concurrency=${CONCURRENCY}, ` +
`max_attempts=${MAX_ATTEMPTS}, allow_private=${allowPrivateUrls}`
);
}
function stopWebhookDeliveryWorker() {
stopped = true;
if (intervalHandle) {
clearInterval(intervalHandle);
intervalHandle = null;
}
}
module.exports = {
startWebhookDeliveryWorker,
stopWebhookDeliveryWorker,
// exported for tests
__test: {
tick,
BACKOFF_MS,
SIGNATURE_HEADER,
EVENT_HEADER,
DELIVERY_HEADER,
setAllowPrivateUrls(value) { allowPrivateUrls = !!value; },
},
};
+224
View File
@@ -0,0 +1,224 @@
const crypto = require('crypto');
const { db } = require('../database/db');
const logger = require('../utils/logger');
const SECRET_PREFIX = 'whsec_';
/**
* Event types PicPeak emits. Keep this in sync with the README catalog and
* the receiver-side type unions in any SDK we publish later. Consumers of
* `fire(eventType, ...)` MUST use one of these strings the worker will
* silently drop unknown types so a typo can't 500 a request handler.
*/
const EVENT_TYPES = Object.freeze([
'event.created',
'event.published',
'event.archived',
'event.expired',
'photo.uploaded',
'photo.deleted',
]);
function generateSecret() {
const random = crypto.randomBytes(24).toString('base64url'); // ~32 chars
const plaintext = `${SECRET_PREFIX}${random}`;
return {
plaintext,
preview: random.slice(0, 8),
};
}
/**
* Sign a payload with the webhook's secret. Used by the delivery worker;
* exported for unit tests of receiver-side verification snippets.
*/
function signPayload(secret, rawBody) {
return crypto.createHmac('sha256', secret).update(rawBody).digest('hex');
}
/**
* Resolve a dot-path on an object (e.g. "data.event.event_type") with no
* eval. Returns undefined for missing segments never throws.
*/
function getByPath(obj, dotPath) {
if (!dotPath || typeof dotPath !== 'string') return undefined;
return dotPath.split('.').reduce((acc, key) => {
if (acc == null || typeof acc !== 'object') return undefined;
return acc[key];
}, obj);
}
/**
* Evaluate a webhook's filter against an outgoing payload. The filter is a
* flat object of dot-path expected value pairs:
* { "data.event.event_type": "wedding" }
* { "type": "event.published", "data.event.id": 42 }
*
* All keys must match (logical AND). Equality is `===` after JSON-style
* coercion: numbers as numbers, booleans as booleans. Empty filter
* matches everything (back-compat).
*/
function payloadMatchesFilter(filter, payload) {
if (!filter || typeof filter !== 'object') return true;
const keys = Object.keys(filter);
if (keys.length === 0) return true;
for (const key of keys) {
const expected = filter[key];
const actual = getByPath(payload, key);
if (Array.isArray(expected)) {
// Array means "any of"
if (!expected.includes(actual)) return false;
} else if (actual !== expected) {
return false;
}
}
return true;
}
/**
* Render a webhook template by substituting ${dot.path} expressions with
* values from the payload. NO eval, NO logic pure string substitution.
* Caps output at 64KB; bails out and returns null if exceeded so the
* delivery worker can fall back to the default envelope.
*/
function renderTemplate(template, payload) {
if (template == null || template === '') return null;
if (typeof template !== 'string') return null;
const MAX_OUTPUT = 64 * 1024;
const MAX_SUBSTITUTIONS = 64;
let count = 0;
const rendered = template.replace(/\$\{([^}]+)\}/g, (_match, expr) => {
count += 1;
if (count > MAX_SUBSTITUTIONS) return '';
const value = getByPath(payload, expr.trim());
if (value == null) return '';
if (typeof value === 'object') {
try { return JSON.stringify(value); } catch { return ''; }
}
return String(value);
});
if (Buffer.byteLength(rendered, 'utf8') > MAX_OUTPUT) return null;
return rendered;
}
/**
* Validate a template at create-time so admins get immediate feedback
* instead of silent delivery failures. Returns { valid, error? }.
*/
function validateTemplate(template) {
if (template == null || template === '') return { valid: true };
if (typeof template !== 'string') return { valid: false, error: 'template must be a string' };
if (Buffer.byteLength(template, 'utf8') > 8192) return { valid: false, error: 'template exceeds 8KB' };
// Reject unbalanced ${ that would silently swallow content at render.
const opens = (template.match(/\$\{/g) || []).length;
const closes = (template.match(/\}/g) || []).length;
// Count is approximate (every } is counted, even non-matching ones).
// We require at least as many } as ${, which is necessary but not sufficient.
if (opens > closes) return { valid: false, error: 'template has unbalanced ${ — every ${ needs a matching }' };
if (opens > 64) return { valid: false, error: 'template exceeds 64 substitutions' };
return { valid: true };
}
/**
* Constant-time signature comparison helper for receivers and tests.
* Exposed so the same primitive backs verification examples in the README.
*/
function verifySignature(secret, rawBody, signature) {
const expected = signPayload(secret, rawBody);
const a = Buffer.from(expected, 'hex');
let b;
try {
b = Buffer.from(signature || '', 'hex');
} catch {
return false;
}
if (a.length !== b.length) return false;
return crypto.timingSafeEqual(a, b);
}
/**
* Enqueue a webhook delivery for every active webhook subscribed to
* `eventType`. NEVER throws webhook failures must not break the
* lifecycle handler that emitted the event. Worker handles HTTP delivery.
*
* @param {string} eventType one of EVENT_TYPES
* @param {object} data opaque payload that gets nested under .data in
* the outbound JSON body
*/
async function fire(eventType, data) {
if (!EVENT_TYPES.includes(eventType)) {
logger.warn(`[webhookService] dropping unknown event type: ${eventType}`);
return;
}
try {
// jsonb @> ARRAY check across vendors: both pg and sqlite drivers we
// support handle a simple WHERE on `active=true` then a runtime filter
// on the events array; doing the array filter here keeps the query
// portable.
const candidates = await db('webhooks').where({ active: true });
const subscribed = candidates.filter((w) => {
const evts = Array.isArray(w.events)
? w.events
: (() => { try { return JSON.parse(w.events) || []; } catch { return []; } })();
return evts.includes(eventType);
});
if (subscribed.length === 0) return;
const now = new Date();
const buildEnvelope = (deliveryUuid) => ({
id: deliveryUuid,
type: eventType,
created_at: now.toISOString(),
data,
});
const rows = [];
for (const w of subscribed) {
const deliveryUuid = crypto.randomUUID();
const envelope = buildEnvelope(deliveryUuid);
// Filter (#327 follow-up): per-webhook predicate evaluated against
// the payload. Skip insertion when it doesn't match.
const filter = parseJsonField(w.filter, {});
if (!payloadMatchesFilter(filter, envelope)) continue;
rows.push({
webhook_id: w.id,
event_type: eventType,
payload: JSON.stringify(envelope),
attempt_count: 0,
status: 'pending',
next_retry_at: now,
created_at: now,
});
}
if (rows.length === 0) return;
await db('webhook_deliveries').insert(rows);
} catch (err) {
// Log but don't throw — caller's transaction has already committed
// by the time we get here, and we don't want to mask the success.
logger.error(`[webhookService.fire] failed to enqueue ${eventType}: ${err.message}`);
}
}
function parseJsonField(value, fallback) {
if (value == null) return fallback;
if (typeof value === 'object') return value;
try { return JSON.parse(value) ?? fallback; } catch { return fallback; }
}
module.exports = {
fire,
generateSecret,
signPayload,
verifySignature,
payloadMatchesFilter,
renderTemplate,
validateTemplate,
getByPath,
EVENT_TYPES,
SECRET_PREFIX,
};