1b717ce5ed
Lets PicPeak write photos, thumbnails, hero images, watermarks, and archive zips to any S3-compatible bucket (AWS S3, MinIO, Cloudflare R2, Backblaze B2, Wasabi, DigitalOcean Spaces) instead of the local filesystem. Selected via STORAGE_BACKEND=local|s3. Architecture - backend/src/services/storage/StorageBackend.js — abstract interface (put/get/exists/stat/delete/list/copy/rename/signedUrl/putFromFile/ getToFile) — typedef-only, documents the contract. - LocalFsStorage.js — wraps fs with atomic-write-via-tmp-rename, path traversal protection, list-as-walker. - S3StorageBackend.js — thin wrapper around the existing S3StorageAdapter (used by backupService) mapping it onto the canonical interface; supports optional STORAGE_S3_PREFIX namespace. - index.js — factory selected by STORAGE_BACKEND with startup ping (HEADs sentinel key on S3, fs.stat on local) so misconfig fails fast before the first request. Consumer refactors (~12 services + routes), each parametrized over the abstraction: - imageProcessor / videoProcessor — pipe Sharp/ffmpeg output through storage.put; expose withLocalCopy() helper for S3-mode regeneration paths that need a local file for sharp/ffmpeg. - archiveService / downloadZipService — finalize zip in tmp dir, then storage.putFromFile. Atomic-rename pattern preserved on local; S3 emulates via copy + delete (worker prunes orphaned .tmp.* on startup). - photoProcessor / photoReplacementService / adminPhotos upload+delete / routes/v1/events.js POST /events/:id/photos / routes/events.js — every upload path now goes storage.putFromFile(temp) → unlink temp. - gallery.js bulk-download (cached + on-the-fly + selected) — managed photos via storage.get, external-mode unchanged. - protectedImages / secureImages / photoResolver — read via storage.get; resolvePhotoStorageKey returns the canonical key. - watermarkService / watermarkGeneratorService — persistent watermarks via storage.put. - fileWatcher — bails out with a clear log warning when STORAGE_BACKEND=s3 (chokidar can't watch S3); auto-import lands via the S3 prefix walker introduced in the follow-up commit. - expirationChecker — small touch (event.expired webhook fire from #327 shipping in the next commit). Migration tooling - backend/scripts/migrate-storage.js — one-shot --dry-run capable script that walks photos.path, thumbnail_path, hero_path, watermark_path and events.archive_path/download_zip_path; streams local → S3; sha256 size-match skip for idempotent re-run; failures CSV. Presigned-URL "Download All" (#328 follow-up shipped in this commit) - routes/gallery.js — when STORAGE_BACKEND=s3 + event.allow_presigned_download + downloads enabled + watermark NOT enabled, /download-all returns a 302 redirect to a 5-minute presigned S3 URL. Per-event opt-in surface ships in the next commit's UI. Tests - backend/__tests__/integration/storageBackend.test.js — parametrized contract suite running against BOTH LocalFs AND MinIO (18 tests, both backends — 36 cases total). - backend/__tests__/integration/imageProcessor.storage.test.js — same parametrized pattern for the image processor (10 tests × 2 backends). - backend/__tests__/integration/backup-s3.test.js — bootstrap fix: drop the redundant initDb() (001_init handles it) and remove schema-drift in configureS3Backup (app_settings has no created_at anymore and the unique constraint is on setting_key alone, not composite). 0/12 → 7/12 (5 remaining are unrelated assertion drift). - backend/src/services/photoResolver.js — mixed-source events (reference mode with managed-uploaded photos) now fall back to managed when external_relpath is missing instead of throwing. - tests/e2e/s3-storage-roundtrip.spec.ts — Playwright spec that auto-skips against local backend; full upload → serve → delete round-trip when run against an S3-mode backend. Server wiring (server.js) - initStorage() called after database init, before rate limiters. - This commit's diff also includes the webhook delivery worker startup and the S3 auto-importer startup. Those features ship in the next two commits — co-located here for one bisectable diff per file. Docs + ops - README §"Storage Backends" — capability matrix, switching playbook, IAM policy snippet, MinIO/R2/B2 examples. - README §"Webhooks" — also added here (full diff bundled). - .env.example — STORAGE_BACKEND + STORAGE_S3_* + STORAGE_AUTO_IMPORT documented; WEBHOOK_* added in the same diff. - .gitignore — re-anchor the existing `storage/` rule to `/storage/` so backend/src/services/storage/ (the new abstraction code) is trackable. The runtime ./storage/ data dir stays ignored. Out of scope for v1 (per the issue): presigned URLs for individual photo display (always streamed for protection middleware), CDN integration, hybrid hot/cold tiers, S3 → local migration, multi-bucket per-event.
166 lines
6.1 KiB
JavaScript
166 lines
6.1 KiB
JavaScript
const path = require('path');
|
|
const fs = require('fs').promises;
|
|
const fsSync = require('fs');
|
|
const os = require('os');
|
|
const crypto = require('crypto');
|
|
const { S3Client, CreateBucketCommand, DeleteBucketCommand, ListObjectsV2Command, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
|
|
const sharp = require('sharp');
|
|
|
|
const LocalFsStorage = require('../../src/services/storage/LocalFsStorage');
|
|
const S3StorageBackend = require('../../src/services/storage/S3StorageBackend');
|
|
const storageModule = require('../../src/services/storage');
|
|
|
|
// Stub out the DB so getThumbnailSettings falls into its catch and uses defaults.
|
|
jest.mock('../../src/database/db', () => ({
|
|
db: () => {
|
|
throw new Error('db disabled in this test');
|
|
},
|
|
}));
|
|
|
|
const TEST_S3 = {
|
|
endpoint: process.env.TEST_S3_ENDPOINT || 'http://localhost:7104',
|
|
accessKeyId: process.env.TEST_S3_ACCESS_KEY || 'minioadmin',
|
|
secretAccessKey: process.env.TEST_S3_SECRET_KEY || 'minioadmin',
|
|
region: 'us-east-1',
|
|
};
|
|
|
|
const skipS3 = process.env.SKIP_S3_TESTS === 'true';
|
|
|
|
function backendCases() {
|
|
const cases = [
|
|
{
|
|
name: 'LocalFsStorage',
|
|
async setup() {
|
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'picpeak-imgproc-'));
|
|
const storage = new LocalFsStorage({ root });
|
|
await storage.init();
|
|
return { storage, cleanup: () => fs.rm(root, { recursive: true, force: true }) };
|
|
},
|
|
},
|
|
];
|
|
if (!skipS3) {
|
|
cases.push({
|
|
name: 'S3StorageBackend (MinIO)',
|
|
async setup() {
|
|
const bucket = `picpeak-imgproc-${Date.now()}-${crypto.randomBytes(2).toString('hex')}`;
|
|
const s3Client = new S3Client({
|
|
endpoint: TEST_S3.endpoint,
|
|
region: TEST_S3.region,
|
|
credentials: { accessKeyId: TEST_S3.accessKeyId, secretAccessKey: TEST_S3.secretAccessKey },
|
|
forcePathStyle: true,
|
|
});
|
|
await s3Client.send(new CreateBucketCommand({ Bucket: bucket }));
|
|
const storage = new S3StorageBackend({
|
|
bucket,
|
|
region: TEST_S3.region,
|
|
endpoint: TEST_S3.endpoint,
|
|
accessKeyId: TEST_S3.accessKeyId,
|
|
secretAccessKey: TEST_S3.secretAccessKey,
|
|
forcePathStyle: true,
|
|
sslEnabled: false,
|
|
});
|
|
await storage.init();
|
|
return {
|
|
storage,
|
|
async cleanup() {
|
|
const list = await s3Client.send(new ListObjectsV2Command({ Bucket: bucket }));
|
|
if (list.Contents?.length) {
|
|
await s3Client.send(new DeleteObjectsCommand({
|
|
Bucket: bucket,
|
|
Delete: { Objects: list.Contents.map((o) => ({ Key: o.Key })) },
|
|
}));
|
|
}
|
|
await s3Client.send(new DeleteBucketCommand({ Bucket: bucket }));
|
|
},
|
|
};
|
|
},
|
|
});
|
|
}
|
|
return cases;
|
|
}
|
|
|
|
async function makeSourceJpeg(targetDir, name) {
|
|
const localPath = path.join(targetDir, name);
|
|
// 800x600 random RGB image so sharp has something realistic to thumbnail.
|
|
const width = 800;
|
|
const height = 600;
|
|
const buf = Buffer.alloc(width * height * 3);
|
|
for (let i = 0; i < buf.length; i++) buf[i] = (i * 7) % 256;
|
|
await sharp(buf, { raw: { width, height, channels: 3 } })
|
|
.jpeg({ quality: 90 })
|
|
.toFile(localPath);
|
|
return localPath;
|
|
}
|
|
|
|
describe.each(backendCases())('imageProcessor through $name', ({ setup }) => {
|
|
let storage;
|
|
let cleanup;
|
|
let tmpDir;
|
|
let imageProcessor;
|
|
|
|
beforeAll(async () => {
|
|
({ storage, cleanup } = await setup());
|
|
storageModule.setStorageForTesting(storage);
|
|
// Require AFTER setStorageForTesting so the module sees our injection.
|
|
delete require.cache[require.resolve('../../src/services/imageProcessor')];
|
|
imageProcessor = require('../../src/services/imageProcessor');
|
|
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'picpeak-imgproc-src-'));
|
|
}, 30000);
|
|
|
|
afterAll(async () => {
|
|
storageModule.resetStorage();
|
|
if (tmpDir) await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
|
|
if (cleanup) await cleanup();
|
|
});
|
|
|
|
test('generateThumbnail writes through storage and returns a relative key', async () => {
|
|
const src = await makeSourceJpeg(tmpDir, 'sample.jpg');
|
|
const key = await imageProcessor.generateThumbnail(src);
|
|
expect(key).toBe('thumbnails/thumb_sample.jpg');
|
|
|
|
expect(await storage.exists(key)).toBe(true);
|
|
const stat = await storage.stat(key);
|
|
expect(stat.size).toBeGreaterThan(100);
|
|
|
|
// Verify the bytes are a valid JPEG by re-parsing with sharp on local mode.
|
|
if (storage.kind() === 'local') {
|
|
const meta = await sharp(storage.resolveLocalPath(key)).metadata();
|
|
expect(meta.format).toBe('jpeg');
|
|
expect(meta.width).toBeLessThanOrEqual(300);
|
|
}
|
|
});
|
|
|
|
test('generateHeroImage writes through storage and returns a relative key', async () => {
|
|
const src = await makeSourceJpeg(tmpDir, 'hero-source.jpg');
|
|
const key = await imageProcessor.generateHeroImage(src);
|
|
expect(key).toBe('heroes/hero_hero-source.jpg');
|
|
expect(await storage.exists(key)).toBe(true);
|
|
});
|
|
|
|
test('isThumbnailValid returns true for a good thumbnail and false for nothing', async () => {
|
|
const src = await makeSourceJpeg(tmpDir, 'valid-check.jpg');
|
|
const key = await imageProcessor.generateThumbnail(src);
|
|
expect(await imageProcessor.isThumbnailValid(key)).toBe(true);
|
|
expect(await imageProcessor.isThumbnailValid('thumbnails/does-not-exist.jpg')).toBe(false);
|
|
});
|
|
|
|
test('generateVideoPlaceholder writes a thumbnail entirely from buffer', async () => {
|
|
const key = await imageProcessor.generateVideoPlaceholder('demo.mp4');
|
|
expect(key).toBe('thumbnails/thumb_demo.jpg');
|
|
expect(await storage.exists(key)).toBe(true);
|
|
});
|
|
|
|
test('withLocalCopy yields a usable local path on both backends', async () => {
|
|
const sourceKey = 'fixture/withlocal.jpg';
|
|
const src = await makeSourceJpeg(tmpDir, 'withlocal.jpg');
|
|
const buf = await fs.readFile(src);
|
|
await storage.put(sourceKey, buf, { contentType: 'image/jpeg' });
|
|
|
|
const seenSize = await imageProcessor.withLocalCopy(sourceKey, async (localPath) => {
|
|
const meta = await sharp(localPath).metadata();
|
|
return meta.width;
|
|
});
|
|
expect(seenSize).toBe(800);
|
|
});
|
|
});
|