1b717ce5ed
Lets PicPeak write photos, thumbnails, hero images, watermarks, and archive zips to any S3-compatible bucket (AWS S3, MinIO, Cloudflare R2, Backblaze B2, Wasabi, DigitalOcean Spaces) instead of the local filesystem. Selected via STORAGE_BACKEND=local|s3. Architecture - backend/src/services/storage/StorageBackend.js — abstract interface (put/get/exists/stat/delete/list/copy/rename/signedUrl/putFromFile/ getToFile) — typedef-only, documents the contract. - LocalFsStorage.js — wraps fs with atomic-write-via-tmp-rename, path traversal protection, list-as-walker. - S3StorageBackend.js — thin wrapper around the existing S3StorageAdapter (used by backupService) mapping it onto the canonical interface; supports optional STORAGE_S3_PREFIX namespace. - index.js — factory selected by STORAGE_BACKEND with startup ping (HEADs sentinel key on S3, fs.stat on local) so misconfig fails fast before the first request. Consumer refactors (~12 services + routes), each parametrized over the abstraction: - imageProcessor / videoProcessor — pipe Sharp/ffmpeg output through storage.put; expose withLocalCopy() helper for S3-mode regeneration paths that need a local file for sharp/ffmpeg. - archiveService / downloadZipService — finalize zip in tmp dir, then storage.putFromFile. Atomic-rename pattern preserved on local; S3 emulates via copy + delete (worker prunes orphaned .tmp.* on startup). - photoProcessor / photoReplacementService / adminPhotos upload+delete / routes/v1/events.js POST /events/:id/photos / routes/events.js — every upload path now goes storage.putFromFile(temp) → unlink temp. - gallery.js bulk-download (cached + on-the-fly + selected) — managed photos via storage.get, external-mode unchanged. - protectedImages / secureImages / photoResolver — read via storage.get; resolvePhotoStorageKey returns the canonical key. - watermarkService / watermarkGeneratorService — persistent watermarks via storage.put. - fileWatcher — bails out with a clear log warning when STORAGE_BACKEND=s3 (chokidar can't watch S3); auto-import lands via the S3 prefix walker introduced in the follow-up commit. - expirationChecker — small touch (event.expired webhook fire from #327 shipping in the next commit). Migration tooling - backend/scripts/migrate-storage.js — one-shot --dry-run capable script that walks photos.path, thumbnail_path, hero_path, watermark_path and events.archive_path/download_zip_path; streams local → S3; sha256 size-match skip for idempotent re-run; failures CSV. Presigned-URL "Download All" (#328 follow-up shipped in this commit) - routes/gallery.js — when STORAGE_BACKEND=s3 + event.allow_presigned_download + downloads enabled + watermark NOT enabled, /download-all returns a 302 redirect to a 5-minute presigned S3 URL. Per-event opt-in surface ships in the next commit's UI. Tests - backend/__tests__/integration/storageBackend.test.js — parametrized contract suite running against BOTH LocalFs AND MinIO (18 tests, both backends — 36 cases total). - backend/__tests__/integration/imageProcessor.storage.test.js — same parametrized pattern for the image processor (10 tests × 2 backends). - backend/__tests__/integration/backup-s3.test.js — bootstrap fix: drop the redundant initDb() (001_init handles it) and remove schema-drift in configureS3Backup (app_settings has no created_at anymore and the unique constraint is on setting_key alone, not composite). 0/12 → 7/12 (5 remaining are unrelated assertion drift). - backend/src/services/photoResolver.js — mixed-source events (reference mode with managed-uploaded photos) now fall back to managed when external_relpath is missing instead of throwing. - tests/e2e/s3-storage-roundtrip.spec.ts — Playwright spec that auto-skips against local backend; full upload → serve → delete round-trip when run against an S3-mode backend. Server wiring (server.js) - initStorage() called after database init, before rate limiters. - This commit's diff also includes the webhook delivery worker startup and the S3 auto-importer startup. Those features ship in the next two commits — co-located here for one bisectable diff per file. Docs + ops - README §"Storage Backends" — capability matrix, switching playbook, IAM policy snippet, MinIO/R2/B2 examples. - README §"Webhooks" — also added here (full diff bundled). - .env.example — STORAGE_BACKEND + STORAGE_S3_* + STORAGE_AUTO_IMPORT documented; WEBHOOK_* added in the same diff. - .gitignore — re-anchor the existing `storage/` rule to `/storage/` so backend/src/services/storage/ (the new abstraction code) is trackable. The runtime ./storage/ data dir stays ignored. Out of scope for v1 (per the issue): presigned URLs for individual photo display (always streamed for protection middleware), CDN integration, hybrid hot/cold tiers, S3 → local migration, multi-bucket per-event.
167 lines
5.2 KiB
JavaScript
167 lines
5.2 KiB
JavaScript
const fs = require('fs');
|
|
const fsp = require('fs').promises;
|
|
const path = require('path');
|
|
const { pipeline } = require('stream/promises');
|
|
const crypto = require('crypto');
|
|
|
|
const logger = require('../../utils/logger');
|
|
|
|
/**
|
|
* Filesystem-backed implementation of the StorageBackend interface.
|
|
* All keys are relative to `root` (typically process.env.STORAGE_PATH).
|
|
*
|
|
* Path traversal protection: every key is normalized to POSIX form and rejected
|
|
* if it tries to escape the root via "..". Callers should not need to think
|
|
* about this — but if a key arrives via user input it must still be filtered.
|
|
*/
|
|
class LocalFsStorage {
|
|
constructor({ root }) {
|
|
if (!root) throw new Error('LocalFsStorage requires a `root` directory');
|
|
this.root = path.resolve(root);
|
|
}
|
|
|
|
kind() {
|
|
return 'local';
|
|
}
|
|
|
|
async init() {
|
|
await fsp.mkdir(this.root, { recursive: true });
|
|
// Sanity check: must be writable.
|
|
const probe = path.join(this.root, '.storage-write-probe');
|
|
await fsp.writeFile(probe, '');
|
|
await fsp.unlink(probe);
|
|
logger.info(`[storage] LocalFsStorage initialized at ${this.root}`);
|
|
}
|
|
|
|
_resolve(relPath) {
|
|
if (!relPath || typeof relPath !== 'string') {
|
|
throw new Error(`LocalFsStorage: invalid relative path: ${relPath}`);
|
|
}
|
|
const normalized = path.posix.normalize(relPath.replace(/\\/g, '/'));
|
|
if (normalized.startsWith('..') || normalized.includes('/../') || normalized === '..') {
|
|
throw new Error(`LocalFsStorage: path traversal rejected: ${relPath}`);
|
|
}
|
|
return path.join(this.root, normalized);
|
|
}
|
|
|
|
async put(relPath, body, _options = {}) {
|
|
const abs = this._resolve(relPath);
|
|
await fsp.mkdir(path.dirname(abs), { recursive: true });
|
|
// Write to a sibling tmp file first then rename for crash safety.
|
|
const tmp = `${abs}.tmp.${process.pid}.${crypto.randomBytes(4).toString('hex')}`;
|
|
try {
|
|
if (Buffer.isBuffer(body)) {
|
|
await fsp.writeFile(tmp, body);
|
|
} else if (body && typeof body.pipe === 'function') {
|
|
await pipeline(body, fs.createWriteStream(tmp));
|
|
} else {
|
|
throw new Error('LocalFsStorage.put: body must be a Buffer or Readable stream');
|
|
}
|
|
await fsp.rename(tmp, abs);
|
|
} catch (err) {
|
|
await fsp.unlink(tmp).catch(() => {});
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
async putFromFile(relPath, localPath, _options = {}) {
|
|
const abs = this._resolve(relPath);
|
|
await fsp.mkdir(path.dirname(abs), { recursive: true });
|
|
// copyFile is atomic from the destination's perspective on POSIX.
|
|
await fsp.copyFile(localPath, abs);
|
|
}
|
|
|
|
async get(relPath) {
|
|
const abs = this._resolve(relPath);
|
|
return fs.createReadStream(abs);
|
|
}
|
|
|
|
async getToFile(relPath, localPath) {
|
|
const abs = this._resolve(relPath);
|
|
await fsp.mkdir(path.dirname(localPath), { recursive: true });
|
|
await fsp.copyFile(abs, localPath);
|
|
}
|
|
|
|
async exists(relPath) {
|
|
try {
|
|
await fsp.access(this._resolve(relPath), fs.constants.F_OK);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async stat(relPath) {
|
|
try {
|
|
const s = await fsp.stat(this._resolve(relPath));
|
|
return { size: s.size, mtime: s.mtime };
|
|
} catch (err) {
|
|
if (err.code === 'ENOENT') return null;
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
async delete(relPath) {
|
|
try {
|
|
await fsp.unlink(this._resolve(relPath));
|
|
} catch (err) {
|
|
if (err.code !== 'ENOENT') throw err;
|
|
}
|
|
}
|
|
|
|
async list(prefix) {
|
|
const absPrefix = this._resolve(prefix || '.');
|
|
const entries = [];
|
|
async function walk(dir, relBase) {
|
|
let dirents;
|
|
try {
|
|
dirents = await fsp.readdir(dir, { withFileTypes: true });
|
|
} catch (err) {
|
|
if (err.code === 'ENOENT') return;
|
|
throw err;
|
|
}
|
|
for (const ent of dirents) {
|
|
const childAbs = path.join(dir, ent.name);
|
|
const childRel = relBase ? `${relBase}/${ent.name}` : ent.name;
|
|
if (ent.isDirectory()) {
|
|
await walk(childAbs, childRel);
|
|
} else if (ent.isFile()) {
|
|
const s = await fsp.stat(childAbs);
|
|
entries.push({ key: childRel, size: s.size, mtime: s.mtime });
|
|
}
|
|
}
|
|
}
|
|
const baseRel = prefix && prefix !== '.' ? prefix.replace(/\\/g, '/') : '';
|
|
await walk(absPrefix, baseRel);
|
|
return entries;
|
|
}
|
|
|
|
async rename(srcRelPath, dstRelPath) {
|
|
const src = this._resolve(srcRelPath);
|
|
const dst = this._resolve(dstRelPath);
|
|
await fsp.mkdir(path.dirname(dst), { recursive: true });
|
|
await fsp.rename(src, dst);
|
|
}
|
|
|
|
async copy(srcRelPath, dstRelPath) {
|
|
const src = this._resolve(srcRelPath);
|
|
const dst = this._resolve(dstRelPath);
|
|
await fsp.mkdir(path.dirname(dst), { recursive: true });
|
|
await fsp.copyFile(src, dst);
|
|
}
|
|
|
|
async signedUrl(_relPath, _ttlSeconds = 300) {
|
|
throw new Error('LocalFsStorage does not support signedUrl. Set STORAGE_BACKEND=s3 to use presigned URLs.');
|
|
}
|
|
|
|
// Escape hatch for callers that genuinely need a filesystem path
|
|
// (e.g. ffmpeg, archiver — anything that takes a path argument rather
|
|
// than a stream). S3Storage exposes the same method but returns null,
|
|
// forcing callers to use the streaming API instead.
|
|
resolveLocalPath(relPath) {
|
|
return this._resolve(relPath);
|
|
}
|
|
}
|
|
|
|
module.exports = LocalFsStorage;
|