Clients who need smaller files no longer make the photographer re-export. Two capabilities, both off by default. STANDARD RESOLUTION — the size a gallery hands out for every ordinary download (single, selected, download-all). Global default in Settings, overridable per gallery with the NULL=inherit tri-state. The pre-built download-all zip is built AT the standard resolution, so changing it invalidates those archives, including a fan-out to inheriting galleries. RESOLUTION PICKER — opt-in modal letting guests choose a different size. Custom archives are built as a DB-backed job the client polls, never cached. The picker never offers a size above the standard, and Original reappears only when the admin explicitly allows it. Resize is fit:'inside' + withoutEnlargement — aspect preserved, never upscaled — applied before the watermark, since the mark is sized relative to its input. Three rounds of external review hardened this: job archives are bound to the requester's visibility scope and re-validated at delivery, the streamed download-all path applies the cap, queue admission is bounded, and rejected resolutions no longer inflate download stats. Closes #858.
This commit is contained in:
Paul Nothaft
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* downloadJobCleanupService — TTL sweep for custom-resolution download jobs
|
||||
* (#858).
|
||||
*
|
||||
* Job archives are one-off renditions of a gallery at a size nothing else
|
||||
* caches against, so they are pure disposable bytes: once the TTL passes the
|
||||
* row and its zip go away. Without this sweep, every guest who ever picked a
|
||||
* non-standard resolution would leave a full gallery-sized archive behind in
|
||||
* `.download-cache` forever.
|
||||
*
|
||||
* Runs every 20 minutes, offset from the hourly jobs so the three cleanup
|
||||
* schedulers don't all wake at once.
|
||||
*/
|
||||
|
||||
const cron = require('node-cron');
|
||||
const logger = require('../utils/logger');
|
||||
const downloadJobService = require('./downloadJobService');
|
||||
|
||||
function startDownloadJobCleanup() {
|
||||
// A restart leaves any in-flight build with no worker. Fail those rows once
|
||||
// at startup so their owners get a clear error instead of polling forever.
|
||||
downloadJobService.recoverOrphanedJobs().catch((err) =>
|
||||
logger.error('Download job recovery failed', { error: err.message }));
|
||||
|
||||
cron.schedule('7,27,47 * * * *', async () => {
|
||||
await runDownloadJobCleanup();
|
||||
});
|
||||
logger.info('Download job cleanup scheduler started');
|
||||
}
|
||||
|
||||
async function runDownloadJobCleanup() {
|
||||
try {
|
||||
await downloadJobService.sweepExpired();
|
||||
} catch (err) {
|
||||
logger.error('Download job cleanup failed', { error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
startDownloadJobCleanup,
|
||||
// exported for tests / manual invocation
|
||||
runDownloadJobCleanup,
|
||||
};
|
||||
@@ -0,0 +1,418 @@
|
||||
/**
|
||||
* Custom-resolution download jobs (#858).
|
||||
*
|
||||
* The plain download-all is served from the pre-built cache, which is built at
|
||||
* the gallery's STANDARD resolution. When a guest picks a different size there
|
||||
* is nothing to cache against — so instead of holding an HTTP connection open
|
||||
* while sharp chews through a whole gallery (a reverse proxy would time it out
|
||||
* long before it finished), the archive is built as a job:
|
||||
*
|
||||
* POST .../download-jobs → { token, status: 'pending' }
|
||||
* GET .../download-jobs/:token → poll { status, progress }
|
||||
* GET .../download-jobs/:token/file → the finished zip
|
||||
*
|
||||
* State lives in the `download_jobs` table rather than in memory: an in-memory
|
||||
* map loses every "ready" job on restart and is simply wrong the moment the
|
||||
* backend runs more than one replica.
|
||||
*
|
||||
* Artifacts land in the same `.download-cache` directory as the pre-built zip.
|
||||
* That directory is a dotfile, and s3AutoImporter skips dotfiles, so job zips
|
||||
* can never be mistaken for gallery photos and re-imported.
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const fs = require('fs');
|
||||
const fsp = require('fs').promises;
|
||||
const crypto = require('crypto');
|
||||
const archiver = require('archiver');
|
||||
const { db } = require('../database/db');
|
||||
const { getStorage } = require('./storage');
|
||||
const { getUseOriginalFilenames, getZipEntryNames } = require('./downloadFilenameService');
|
||||
const { renderPhotoForDownload, resolveWatermarkSettings } = require('./downloadRendition');
|
||||
const { resolvePhotoStorageKey, resolvePhotoFilePath } = require('./photoResolver');
|
||||
const { parseResolution } = require('../utils/downloadResolutions');
|
||||
const { applyPhotoVisibilityFilter, canSeeHiddenPhotos } = require('../utils/photoVisibility');
|
||||
const logger = require('../utils/logger');
|
||||
|
||||
// How long a finished archive stays downloadable before the sweep deletes it.
|
||||
const JOB_TTL_MS = 60 * 60 * 1000; // 1 hour
|
||||
// Guards against a handful of guests each kicking off a whole-gallery resize.
|
||||
const MAX_CONCURRENT_BUILDS = 2;
|
||||
// Hard ceiling on queued+running builds. Gallery routes are not behind the
|
||||
// general rate limiter, so without this a token holder could vary the photo
|
||||
// subset to enqueue unbounded 500-photo resizes — each one parking a promise
|
||||
// and eventually a full gallery's worth of CPU and disk.
|
||||
const MAX_QUEUED_BUILDS = 8;
|
||||
// A build heartbeats while it works. A row whose heartbeat is older than this
|
||||
// has no live worker (crashed or restarted) and may be failed by recovery.
|
||||
const LEASE_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Collapse an access level to the visibility scope that decides WHICH photos a
|
||||
* requester may receive. Anything that can see hidden photos is one scope;
|
||||
* ordinary guests are another. Part of the job dedup identity so archives are
|
||||
* never shared across the boundary.
|
||||
*/
|
||||
function visibilityScope(accessLevel) {
|
||||
return canSeeHiddenPhotos(accessLevel) ? 'hidden' : 'public';
|
||||
}
|
||||
|
||||
class DownloadJobService {
|
||||
constructor() {
|
||||
this.running = 0;
|
||||
// jobId -> in-flight build promise. Only jobs present here are safe to
|
||||
// rejoin; rows left 'pending'/'building' by a previous process are not.
|
||||
this.liveBuilds = new Map();
|
||||
// Slots claimed between the admission check and the liveBuilds entry.
|
||||
// Without it, concurrent requests all pass the check before any registers.
|
||||
this.reserved = 0;
|
||||
}
|
||||
|
||||
cacheDir(slug) {
|
||||
return path.posix.join('events/active', slug, '.download-cache');
|
||||
}
|
||||
|
||||
jobKey(slug, token) {
|
||||
return path.posix.join(this.cacheDir(slug), `job-${token}.zip`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable identity for "the same archive".
|
||||
*
|
||||
* SECURITY: `visibilityScope` and the RESOLVED photo id list are part of the
|
||||
* identity, not just the requested one. A PIN client sees hidden photos that
|
||||
* an ordinary guest must not; without the scope in the key, a guest asking
|
||||
* for the same size would be handed the client's job token and could
|
||||
* download hidden photos, because the delivery route only checks the event
|
||||
* id. Hashing the resolved id set also stops a stale archive being reused
|
||||
* after photos are added, removed or hidden.
|
||||
*/
|
||||
dedupKey(eventId, resolution, resolvedPhotoIds, watermark, visibilityScope) {
|
||||
const ids = [...resolvedPhotoIds].map(Number).sort((a, b) => a - b).join(',');
|
||||
// The full watermark SETTINGS, not just the on/off flag: an admin who
|
||||
// edits the watermark text or logo while it stays enabled would otherwise
|
||||
// have the old mark served from a ready job for the rest of its TTL.
|
||||
const wm = watermark
|
||||
? crypto.createHash('sha256').update(JSON.stringify(watermark)).digest('hex').slice(0, 16)
|
||||
: 'raw';
|
||||
return crypto.createHash('sha256')
|
||||
.update(`${eventId}|${resolution}|${visibilityScope}|${ids}|${wm}`)
|
||||
.digest('hex')
|
||||
.slice(0, 64);
|
||||
}
|
||||
|
||||
/**
|
||||
* The photos a given requester would actually receive. Used both to build
|
||||
* the dedup identity and to build the archive, so the two can never drift.
|
||||
*/
|
||||
photoQuery(eventId, photoIds, accessLevel) {
|
||||
let query = db('photos')
|
||||
.leftJoin('photo_categories', 'photos.category_id', 'photo_categories.id')
|
||||
.where('photos.event_id', eventId)
|
||||
.where(function () {
|
||||
this.whereNull('photos.category_id')
|
||||
.orWhere('photo_categories.allow_downloads', true)
|
||||
.orWhereNull('photo_categories.allow_downloads');
|
||||
});
|
||||
if (photoIds && photoIds.length) {
|
||||
query = query.whereIn('photos.id', photoIds);
|
||||
}
|
||||
return applyPhotoVisibilityFilter(query, accessLevel);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a job that already satisfies this request — either still building or
|
||||
* finished and not yet expired. Failed jobs are ignored so a transient error
|
||||
* doesn't poison every later attempt.
|
||||
*
|
||||
* `pending`/`building` rows are only reusable while THIS process is actually
|
||||
* building them: after a restart those rows have no live worker, so rejoining
|
||||
* one would leave the client polling until the TTL expires.
|
||||
*/
|
||||
async findReusable(eventId, dedupKey) {
|
||||
const rows = await db('download_jobs')
|
||||
.where({ event_id: eventId, dedup_key: dedupKey })
|
||||
.whereIn('status', ['pending', 'building', 'ready'])
|
||||
.where('expires_at', '>', new Date().toISOString())
|
||||
.orderBy('id', 'desc');
|
||||
return rows.find((r) => r.status === 'ready' || this.liveBuilds.has(r.id)) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create (or join) a job. Returns the job row. Building continues in the
|
||||
* background — callers poll getStatus().
|
||||
*/
|
||||
async createJob({ event, resolution, photoIds, accessLevel }) {
|
||||
const watermark = await resolveWatermarkSettings(event);
|
||||
|
||||
// Resolve the photo set up front, under THIS requester's visibility, so
|
||||
// the dedup identity reflects what they may actually receive.
|
||||
const resolved = await this.photoQuery(event.id, photoIds, accessLevel).select('photos.id');
|
||||
const resolvedIds = resolved.map((r) => r.id);
|
||||
if (resolvedIds.length === 0) {
|
||||
const err = new Error('No photos available for this selection');
|
||||
err.code = 'NO_PHOTOS';
|
||||
throw err;
|
||||
}
|
||||
|
||||
const scope = visibilityScope(accessLevel);
|
||||
const dedupKey = this.dedupKey(event.id, resolution, resolvedIds, watermark, scope);
|
||||
|
||||
const existing = await this.findReusable(event.id, dedupKey);
|
||||
if (existing) return existing;
|
||||
|
||||
// Refuse rather than queue without bound. The client shows this as a
|
||||
// retryable error, which is far better than accepting work the box can't
|
||||
// absorb and timing the user out anyway.
|
||||
//
|
||||
// The slot is reserved SYNCHRONOUSLY here — checking liveBuilds.size and
|
||||
// only populating it after the awaited insert let a burst of concurrent
|
||||
// requests all pass the check before any of them registered.
|
||||
if (this.reserved + this.liveBuilds.size >= MAX_QUEUED_BUILDS) {
|
||||
const err = new Error('Too many downloads are being prepared right now');
|
||||
err.code = 'BUSY';
|
||||
throw err;
|
||||
}
|
||||
this.reserved += 1;
|
||||
|
||||
try {
|
||||
const token = crypto.randomBytes(32).toString('hex');
|
||||
const expiresAt = new Date(Date.now() + JOB_TTL_MS).toISOString();
|
||||
|
||||
const inserted = await db('download_jobs').insert({
|
||||
token,
|
||||
event_id: event.id,
|
||||
resolution,
|
||||
photo_ids: JSON.stringify(resolvedIds),
|
||||
dedup_key: dedupKey,
|
||||
visibility_scope: scope,
|
||||
status: 'pending',
|
||||
heartbeat_at: new Date().toISOString(),
|
||||
created_at: new Date().toISOString(),
|
||||
expires_at: expiresAt,
|
||||
}).returning('id');
|
||||
const id = inserted[0]?.id ?? inserted[0];
|
||||
|
||||
// Fire and forget — the row is the source of truth for progress. The
|
||||
// liveBuilds entry is what makes a pending/building row reusable; a row
|
||||
// without one is an orphan from a previous process.
|
||||
const promise = this._build(id, event, resolution, resolvedIds, watermark, accessLevel)
|
||||
.catch((err) => logger.error('Download job build failed', { jobId: id, error: err.message }))
|
||||
.finally(() => this.liveBuilds.delete(id));
|
||||
this.liveBuilds.set(id, promise);
|
||||
|
||||
return await db('download_jobs').where({ id }).first();
|
||||
} finally {
|
||||
// The slot is now accounted for by liveBuilds (or the insert failed).
|
||||
this.reserved -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
async getStatus(token) {
|
||||
return db('download_jobs').where({ token }).first();
|
||||
}
|
||||
|
||||
/** Exposed so the delivery route can re-check the requester's scope. */
|
||||
visibilityScopeFor(accessLevel) {
|
||||
return visibilityScope(accessLevel);
|
||||
}
|
||||
|
||||
/**
|
||||
* Is every photo in this finished archive STILL visible to the requester?
|
||||
*
|
||||
* The scope check alone isn't enough: a photo hidden after a public job went
|
||||
* ready stays inside that zip, and both job and requester are still
|
||||
* 'public', so the archive would keep serving it for the rest of its TTL.
|
||||
* Re-running the visibility query at delivery closes that window.
|
||||
*/
|
||||
async isStillDeliverable(job, event, accessLevel) {
|
||||
let ids;
|
||||
try {
|
||||
ids = JSON.parse(job.photo_ids || '[]');
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
if (!Array.isArray(ids) || ids.length === 0) return false;
|
||||
|
||||
// Every packaged photo must still be visible to this requester.
|
||||
const visible = await this.photoQuery(job.event_id, ids, accessLevel).select('photos.id');
|
||||
if (visible.length !== ids.length) return false;
|
||||
|
||||
// …and the archive must still match the CURRENT rendition policy. Turning
|
||||
// a watermark on, editing it, or revoking a resolution after the job went
|
||||
// ready would otherwise keep serving the old bytes for the rest of the
|
||||
// TTL. Recomputing the identity is the cheapest way to notice: any input
|
||||
// that changes the archive changes the key.
|
||||
const watermark = await resolveWatermarkSettings(event);
|
||||
const expected = this.dedupKey(
|
||||
job.event_id, job.resolution, ids, watermark, visibilityScope(accessLevel)
|
||||
);
|
||||
return expected === job.dedup_key;
|
||||
}
|
||||
|
||||
async _fail(id, message) {
|
||||
await db('download_jobs').where({ id }).update({
|
||||
status: 'failed',
|
||||
error: String(message).slice(0, 500),
|
||||
completed_at: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
async _build(id, event, resolution, photoIds, watermarkSettings, accessLevel) {
|
||||
// Back-pressure: a queued job stays 'pending' (which the UI shows as
|
||||
// "preparing") rather than piling more sharp pipelines onto a box that is
|
||||
// already saturated.
|
||||
while (this.running >= MAX_CONCURRENT_BUILDS) {
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
this.running += 1;
|
||||
|
||||
let tmpDir;
|
||||
try {
|
||||
await db('download_jobs').where({ id }).update({ status: 'building', heartbeat_at: new Date().toISOString() });
|
||||
|
||||
// Same query that produced the dedup identity, so the archive can never
|
||||
// contain photos the requester wasn't entitled to at creation time.
|
||||
const photos = await this.photoQuery(event.id, photoIds, accessLevel)
|
||||
.select('photos.*')
|
||||
.orderBy('photos.uploaded_at', 'desc');
|
||||
|
||||
if (photos.length === 0) {
|
||||
await this._fail(id, 'No photos available for this selection');
|
||||
return;
|
||||
}
|
||||
|
||||
const box = parseResolution(resolution);
|
||||
const storage = getStorage();
|
||||
const useOriginal = await getUseOriginalFilenames();
|
||||
const entryNames = getZipEntryNames(photos, useOriginal);
|
||||
|
||||
tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-dljob-'));
|
||||
const tmpPath = path.join(tmpDir, `${crypto.randomBytes(4).toString('hex')}.zip`);
|
||||
|
||||
let appended = 0;
|
||||
const appendedIds = [];
|
||||
await new Promise((resolve, reject) => {
|
||||
const output = fs.createWriteStream(tmpPath);
|
||||
// level 0 — photos are already compressed, so deflate only burns CPU.
|
||||
const archive = archiver('zip', { zlib: { level: 0 } });
|
||||
output.on('close', resolve);
|
||||
archive.on('error', reject);
|
||||
archive.pipe(output);
|
||||
|
||||
(async () => {
|
||||
for (let i = 0; i < photos.length; i += 1) {
|
||||
const photo = photos[i];
|
||||
const name = entryNames[i] || `photo-${photo.id}.jpg`;
|
||||
try {
|
||||
const rendered = await renderPhotoForDownload(event, photo, box, watermarkSettings);
|
||||
if (rendered) {
|
||||
archive.append(rendered, { name });
|
||||
} else {
|
||||
const key = resolvePhotoStorageKey(event, photo);
|
||||
if (key) {
|
||||
archive.append(await storage.get(key), { name });
|
||||
} else {
|
||||
archive.file(resolvePhotoFilePath(event, photo), { name });
|
||||
}
|
||||
}
|
||||
appended += 1;
|
||||
appendedIds.push(photo.id);
|
||||
// Progress is coarse (photo count, not bytes) but it is what the
|
||||
// modal needs to show movement on a long build.
|
||||
if (appended % 10 === 0 || appended === photos.length) {
|
||||
// Doubles as the lease heartbeat — see LEASE_TIMEOUT_MS.
|
||||
await db('download_jobs').where({ id })
|
||||
.update({ photo_count: appended, heartbeat_at: new Date().toISOString() });
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('Skipping photo in download job', { jobId: id, photoId: photo.id, error: err.message });
|
||||
}
|
||||
}
|
||||
archive.finalize();
|
||||
})().catch(reject);
|
||||
});
|
||||
|
||||
if (appended === 0) {
|
||||
await this._fail(id, 'No photos could be packaged');
|
||||
return;
|
||||
}
|
||||
|
||||
const stat = await fsp.stat(tmpPath);
|
||||
const key = this.jobKey(event.slug, (await db('download_jobs').where({ id }).first()).token);
|
||||
await storage.putFromFile(key, tmpPath);
|
||||
|
||||
await db('download_jobs').where({ id }).update({
|
||||
status: 'ready',
|
||||
zip_path: key,
|
||||
size_bytes: stat.size,
|
||||
photo_count: appended,
|
||||
// Only what actually landed in the zip: a missing/corrupt source is
|
||||
// skipped, and counting it as downloaded would inflate that photo's
|
||||
// stats for a file the guest never received. Kept SEPARATE from
|
||||
// photo_ids, which is the requested set the dedup fingerprint was
|
||||
// computed from — overwriting it would make every job with a skipped
|
||||
// photo fail the delivery fingerprint check.
|
||||
delivered_photo_ids: JSON.stringify(appendedIds),
|
||||
completed_at: new Date().toISOString(),
|
||||
});
|
||||
logger.info('Download job ready', { jobId: id, photos: appended, bytes: stat.size });
|
||||
} catch (err) {
|
||||
logger.error('Download job error', { jobId: id, error: err.message });
|
||||
await this._fail(id, err.message).catch(() => {});
|
||||
} finally {
|
||||
this.running -= 1;
|
||||
if (tmpDir) await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fail rows left mid-build by a previous process. Without this they linger
|
||||
* as 'pending'/'building' until the TTL, and although findReusable now skips
|
||||
* them, the client that owns such a token would poll a job that can never
|
||||
* finish. Called once at startup.
|
||||
*/
|
||||
async recoverOrphanedJobs() {
|
||||
// Only rows whose LEASE has expired. A live worker heartbeats while it
|
||||
// builds, so in a multi-replica deployment a rolling restart can't have
|
||||
// one replica fail jobs another is still working on — which a blanket
|
||||
// "fail everything pending" would do.
|
||||
const staleBefore = new Date(Date.now() - LEASE_TIMEOUT_MS).toISOString();
|
||||
const orphaned = await db('download_jobs')
|
||||
.whereIn('status', ['pending', 'building'])
|
||||
.where(function () {
|
||||
this.whereNull('heartbeat_at').orWhere('heartbeat_at', '<', staleBefore);
|
||||
})
|
||||
.update({
|
||||
status: 'failed',
|
||||
error: 'Interrupted by a server restart — please request the download again',
|
||||
completed_at: new Date().toISOString(),
|
||||
});
|
||||
if (orphaned > 0) {
|
||||
logger.info(`Failed ${orphaned} download job(s) orphaned by a restart`);
|
||||
}
|
||||
return orphaned;
|
||||
}
|
||||
|
||||
/** Delete expired jobs and their artifacts. Returns how many were removed. */
|
||||
async sweepExpired() {
|
||||
const now = new Date().toISOString();
|
||||
const expired = await db('download_jobs').where('expires_at', '<=', now);
|
||||
if (expired.length === 0) return 0;
|
||||
|
||||
const storage = getStorage();
|
||||
for (const job of expired) {
|
||||
if (job.zip_path) {
|
||||
await storage.delete(job.zip_path).catch((e) =>
|
||||
logger.warn('Failed deleting download job artifact', { jobId: job.id, error: e.message }));
|
||||
}
|
||||
}
|
||||
await db('download_jobs').whereIn('id', expired.map((j) => j.id)).del();
|
||||
logger.info(`Download job sweep removed ${expired.length} expired job(s)`);
|
||||
return expired.length;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = new DownloadJobService();
|
||||
module.exports.JOB_TTL_MS = JOB_TTL_MS;
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* Download renditions (#858).
|
||||
*
|
||||
* One place that answers "what bytes does this photo ship as for this
|
||||
* download?" — used by the single-photo route, the selected-photos archive,
|
||||
* the cached download-all build, and the custom-resolution job builder.
|
||||
*
|
||||
* The ordering matters and is the whole reason this is centralised: the
|
||||
* watermark is sized relative to its input's width, so it MUST be applied
|
||||
* after the resize. Watermarking the original and then downscaling would
|
||||
* resample the mark and burn CPU on pixels that get thrown away.
|
||||
*
|
||||
* Returns null when the photo needs no transformation at all, which lets
|
||||
* callers stream the original straight from storage instead of buffering it.
|
||||
*/
|
||||
|
||||
const { resolvePhotoStorageKey, resolvePhotoFilePath } = require('./photoResolver');
|
||||
const { withLocalCopy, resizeToBox } = require('./imageProcessor');
|
||||
const watermarkService = require('./watermarkService');
|
||||
const { getStorage } = require('./storage');
|
||||
const fs = require('fs');
|
||||
|
||||
/** Videos have no resize path — they always ship as stored. */
|
||||
function isVideo(photo) {
|
||||
return photo.media_type === 'video'
|
||||
|| (photo.mime_type && String(photo.mime_type).startsWith('video/'));
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {object} event
|
||||
* @param {object} photo
|
||||
* @param {object?} box {width,height} or null for original size
|
||||
* @param {object?} watermarkSettings effective settings, or null to skip
|
||||
* @returns {Promise<Buffer|null>} null = serve the stored bytes unchanged
|
||||
*/
|
||||
async function renderPhotoForDownload(event, photo, box, watermarkSettings) {
|
||||
const wantsResize = !!box && !isVideo(photo);
|
||||
const wantsWatermark = !!(watermarkSettings && watermarkSettings.enabled);
|
||||
if (!wantsResize && !wantsWatermark) return null;
|
||||
|
||||
const storageKey = resolvePhotoStorageKey(event, photo);
|
||||
|
||||
const transform = async (localPath) => {
|
||||
// No resize → hand applyWatermark the PATH. Buffer inputs intentionally
|
||||
// bypass its cache, so buffering here would re-run sharp over the
|
||||
// full-size original for every download of an unresized gallery.
|
||||
if (!wantsResize) {
|
||||
return watermarkService.applyWatermark(localPath, watermarkSettings);
|
||||
}
|
||||
const buffer = await resizeToBox(await fs.promises.readFile(localPath), box);
|
||||
return wantsWatermark
|
||||
? watermarkService.applyWatermark(buffer, watermarkSettings)
|
||||
: buffer;
|
||||
};
|
||||
|
||||
// Managed photos live behind the storage abstraction (possibly S3); external
|
||||
// / reference photos are already on a local mount.
|
||||
return storageKey
|
||||
? withLocalCopy(storageKey, transform)
|
||||
: transform(resolvePhotoFilePath(event, photo));
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the effective watermark settings for an event, or null when no
|
||||
* watermark applies. Same global-OR-event rule the download routes already
|
||||
* used, lifted here so the job builder can't drift from it.
|
||||
*/
|
||||
async function resolveWatermarkSettings(event) {
|
||||
const settings = await watermarkService.getWatermarkSettings();
|
||||
const eventEnabled = event.watermark_downloads === true || event.watermark_downloads === 1;
|
||||
const shouldApply = (settings && settings.enabled) || eventEnabled;
|
||||
if (!shouldApply) return null;
|
||||
return {
|
||||
...settings,
|
||||
enabled: true,
|
||||
text: event.watermark_text || settings?.text || 'Protected',
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
renderPhotoForDownload,
|
||||
resolveWatermarkSettings,
|
||||
isVideo,
|
||||
getStorage,
|
||||
};
|
||||
@@ -24,6 +24,8 @@ const watermarkService = require('./watermarkService');
|
||||
const { resolvePhotoStorageKey, resolvePhotoFilePath } = require('./photoResolver');
|
||||
const { getStorage } = require('./storage');
|
||||
const { getUseOriginalFilenames, getZipEntryNames } = require('./downloadFilenameService');
|
||||
const { renderPhotoForDownload } = require('./downloadRendition');
|
||||
const { resolveEventDownloadPolicy } = require('../utils/downloadResolutions');
|
||||
const logger = require('../utils/logger');
|
||||
|
||||
const DEBOUNCE_MS = 5000;
|
||||
@@ -137,6 +139,12 @@ class DownloadZipService {
|
||||
text: event.watermark_text || watermarkSettings?.text || 'Protected',
|
||||
} : null;
|
||||
|
||||
// The cached archive is built AT the gallery's standard resolution
|
||||
// (#858) — 'original' keeps the historical behaviour. Any change to the
|
||||
// standard invalidates this zip via the settings write paths, so a
|
||||
// cached archive always matches the currently configured size.
|
||||
const { standardBox } = await resolveEventDownloadPolicy(event);
|
||||
|
||||
const finalKey = this.getCacheKey(event.slug);
|
||||
|
||||
tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-zipbuild-'));
|
||||
@@ -182,26 +190,20 @@ class DownloadZipService {
|
||||
// for external, in which case fall back to resolvePhotoFilePath.
|
||||
const storageKey = resolvePhotoStorageKey(event, photo);
|
||||
|
||||
if (shouldApplyWatermark && effectiveSettings) {
|
||||
try {
|
||||
let sourcePath;
|
||||
if (storageKey) {
|
||||
// Stream the original to a tmp file just long enough for sharp
|
||||
// (watermarkService) to operate on it. Avoids buffering the
|
||||
// entire image in memory for huge originals.
|
||||
sourcePath = path.join(tmpDir, `wm-${crypto.randomBytes(4).toString('hex')}`);
|
||||
await storage.getToFile(storageKey, sourcePath);
|
||||
} else {
|
||||
sourcePath = resolvePhotoFilePath(event, photo);
|
||||
}
|
||||
const buf = await watermarkService.applyWatermark(sourcePath, effectiveSettings);
|
||||
archive.append(buf, { name: archiveName });
|
||||
if (storageKey) {
|
||||
await fsp.unlink(sourcePath).catch(() => {});
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('Skipping watermark in pre-zip', { photoId: photo.id, error: err.message });
|
||||
}
|
||||
// Resize to the gallery's standard resolution (#858) and/or
|
||||
// watermark. Returns null when neither applies, so an
|
||||
// original-size unwatermarked gallery still streams straight
|
||||
// from storage with nothing buffered.
|
||||
let rendered = null;
|
||||
try {
|
||||
rendered = await renderPhotoForDownload(event, photo, standardBox, effectiveSettings);
|
||||
} catch (err) {
|
||||
logger.warn('Skipping photo in pre-zip', { photoId: photo.id, error: err.message });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (rendered) {
|
||||
archive.append(rendered, { name: archiveName });
|
||||
} else if (storageKey) {
|
||||
const stream = await storage.get(storageKey);
|
||||
archive.append(stream, { name: archiveName });
|
||||
|
||||
@@ -736,7 +736,75 @@ async function extractCaptureDate(imagePath) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Downscale to fit inside a box, for the download-resolution feature (#858).
|
||||
*
|
||||
* `fit: 'inside'` + `withoutEnlargement` is exactly the "up to" semantic the
|
||||
* requester asked for on #858: the box is a maximum, aspect ratio is kept
|
||||
* (so a 3:2 box leaves a 4:3 photo slightly smaller than the box on one
|
||||
* edge), and an image already smaller than the box is returned untouched
|
||||
* rather than upscaled into mush.
|
||||
*
|
||||
* Takes and returns a Buffer so callers can chain resize → watermark without
|
||||
* a tmp file. Returns the input unchanged when `box` is null ('original').
|
||||
* Never throws: on a corrupt/undecodable source it logs and returns the input,
|
||||
* because failing a download outright is worse than serving the full size.
|
||||
*/
|
||||
async function resizeToBox(inputBuffer, box, options = {}) {
|
||||
if (!box || !box.width || !box.height) return inputBuffer;
|
||||
try {
|
||||
const probe = sharp(inputBuffer, { limitInputPixels: 268402689, failOn: 'none' });
|
||||
const metadata = await probe.metadata();
|
||||
// Already inside the box — hand back the original bytes rather than
|
||||
// re-encoding, which would only cost quality and CPU.
|
||||
if (metadata.width && metadata.height
|
||||
&& metadata.width <= box.width && metadata.height <= box.height) {
|
||||
return inputBuffer;
|
||||
}
|
||||
|
||||
const format = (metadata.format || '').toLowerCase();
|
||||
// Animated sources must be re-opened with `animated: true`, otherwise
|
||||
// sharp keeps only the first frame and the download silently loses its
|
||||
// animation. `.rotate()` would flatten an animated source, so it is
|
||||
// applied only to stills (where EXIF orientation actually exists).
|
||||
const animated = (metadata.pages || 1) > 1;
|
||||
const image = animated
|
||||
? sharp(inputBuffer, { limitInputPixels: 268402689, failOn: 'none', animated: true })
|
||||
: probe.rotate();
|
||||
|
||||
let pipeline = image
|
||||
.resize(box.width, box.height, { fit: 'inside', withoutEnlargement: true });
|
||||
|
||||
// Re-encode in the SOURCE format. The download routes keep the original
|
||||
// filename and mime type, so emitting JPEG for a .gif would ship
|
||||
// mislabelled bytes. GIF is an accepted upload format (multerConfig.photos).
|
||||
//
|
||||
// HEIC/HEIF is the exception: sharp builds generally cannot ENCODE it, and
|
||||
// the browser can't display the original anyway (see originalNeedsPreview
|
||||
// in gallery.js). Rather than emit JPEG bytes under a .heic name, leave
|
||||
// those downloads at original size — correct-but-larger beats
|
||||
// mislabelled-and-broken.
|
||||
if (format === 'heif' || format === 'heic') {
|
||||
return inputBuffer;
|
||||
}
|
||||
if (format === 'png') {
|
||||
pipeline = pipeline.png({ compressionLevel: 6 });
|
||||
} else if (format === 'webp') {
|
||||
pipeline = pipeline.webp({ quality: options.quality || 90 });
|
||||
} else if (format === 'gif') {
|
||||
pipeline = pipeline.gif();
|
||||
} else {
|
||||
pipeline = pipeline.jpeg({ quality: options.quality || 90, mozjpeg: true });
|
||||
}
|
||||
return await pipeline.toBuffer();
|
||||
} catch (e) {
|
||||
logger.warn(`resizeToBox failed (${box.width}x${box.height}), serving original: ${e.message}`);
|
||||
return inputBuffer;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
resizeToBox,
|
||||
generateThumbnail,
|
||||
isThumbnailValid,
|
||||
ensureThumbnail,
|
||||
|
||||
@@ -90,18 +90,29 @@ class WatermarkService {
|
||||
/**
|
||||
* Apply watermark to an image
|
||||
*/
|
||||
/**
|
||||
* `imagePath` may be a path OR an in-memory Buffer (#858). Buffers let the
|
||||
* download paths resize first and watermark second without a second tmp
|
||||
* file — which matters because the mark is sized relative to the input's
|
||||
* own width, so it has to be applied at the OUTPUT size to come out right.
|
||||
*/
|
||||
async applyWatermark(imagePath, settings) {
|
||||
const isBuffer = Buffer.isBuffer(imagePath);
|
||||
try {
|
||||
if (!settings || !settings.enabled) {
|
||||
// Return original image if watermarking is disabled
|
||||
return await fs.readFile(imagePath);
|
||||
return isBuffer ? imagePath : await fs.readFile(imagePath);
|
||||
}
|
||||
|
||||
// Check cache first
|
||||
const cacheKey = `${imagePath}_${JSON.stringify(settings)}`;
|
||||
const cached = this.cache.get(cacheKey);
|
||||
if (cached && Date.now() - cached.timestamp < this.cacheMaxAge) {
|
||||
return cached.buffer;
|
||||
// Check cache first. Buffer inputs are already-resized intermediates:
|
||||
// they have no stable key (hashing megabytes per photo would cost more
|
||||
// than the watermark) and no reuse across requests, so skip the cache.
|
||||
const cacheKey = isBuffer ? null : `${imagePath}_${JSON.stringify(settings)}`;
|
||||
if (cacheKey) {
|
||||
const cached = this.cache.get(cacheKey);
|
||||
if (cached && Date.now() - cached.timestamp < this.cacheMaxAge) {
|
||||
return cached.buffer;
|
||||
}
|
||||
}
|
||||
|
||||
// Load the main image
|
||||
@@ -199,20 +210,24 @@ class WatermarkService {
|
||||
watermarkedBuffer = await watermarkedImage.jpeg({ quality: 100, mozjpeg: true }).toBuffer();
|
||||
}
|
||||
|
||||
// Cache the result
|
||||
this.cache.set(cacheKey, {
|
||||
buffer: watermarkedBuffer,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
// Cache the result (path inputs only — see cacheKey above)
|
||||
if (cacheKey) {
|
||||
this.cache.set(cacheKey, {
|
||||
buffer: watermarkedBuffer,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
|
||||
// Clean old cache entries
|
||||
this.cleanCache();
|
||||
// Clean old cache entries
|
||||
this.cleanCache();
|
||||
}
|
||||
|
||||
return watermarkedBuffer;
|
||||
} catch (error) {
|
||||
logger.error('Error applying watermark:', error);
|
||||
// Return original image on error
|
||||
return await fs.readFile(imagePath);
|
||||
// Return the un-watermarked input on error. Buffer inputs are already
|
||||
// in memory — readFile() would treat the Buffer as a path and throw,
|
||||
// turning a cosmetic watermark failure into a failed download.
|
||||
return isBuffer ? imagePath : await fs.readFile(imagePath);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user