paul/picpeak
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(external-media): pre-generate thumbnails so reference-mode galleries load fast (#423)

Browse Source 
External (source_origin='external') photos always had thumbnail_path=NULL,
so the gallery returned thumbnail_url=null and every layout fell back to
streaming the full original from the NAS via the secure-image route.
With ~100 NAS-mounted photos that meant minutes of wall-clock load time,
sequential per tile.

Two halves:

1. import-external route generates the thumbnail right after each
   successful insert and writes thumbnail_path on the row. Best-effort:
   a single failure logs a warning and leaves thumbnail_path=NULL —
   ensureThumbnail will retry lazily on first view. Synchronous in the
   loop adds ~100-300ms per image; for the worst-case 1000-photo import
   that's still under the typical request timeout.

2. ensureThumbnail() in imageProcessor handles external photos too —
   resolves the local NAS mount path via resolvePhotoFilePath instead of
   the storage-backend key. This covers existing externals already in
   the database that were imported before this fix: first gallery view
   per photo regenerates the thumbnail, subsequent views are fast.

Filename-collision protection: external thumbnails use
`thumb_ext<photoId>_<basename>` so two events both referencing
e.g. `IMG_0001.jpg` on different NAS subtrees can't clobber each other's
thumbnail. generateThumbnail accepts a new options.outputBasename to
support this without changing the managed-photo behaviour.

Verified locally with a 3-photo external dir and a real NAS-style import:
  POST /api/admin/external-media/events/N/import-external
  → {imported:3, thumbnailsGenerated:3, thumbnailsFailed:0}
  /api/gallery/<slug>/photos returns thumbnail_url for every photo
  Lazy-regen path: clearing thumbnail_path + deleting the file, then
  hitting /thumbnail/N regenerates and repopulates the row in 42ms.

Closes #423.
This commit is contained in:
Paul Nothaft
2026-05-08 19:17:15 +02:00
parent f57429faf2
commit f3d0f161c9
2 changed files with 94 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/routes/adminExternalMedia.js
+38 -3
View File
@@ -7,6 +7,7 @@ const { list, resolveExternalPath, getExternalMediaRoot } = require('../services
const { db, logActivity } = require('../database/db');
const sharp = require('sharp');
const logger = require('../utils/logger');
const { generateThumbnail } = require('../services/imageProcessor');
const router = express.Router();
@@ -93,6 +94,8 @@ router.post('/events/:id/import-external', adminAuth, requirePermission('photos.
}
let imported = 0;
let thumbnailsGenerated = 0;
let thumbnailsFailed = 0;
// Insert photos
for (const f of dedupeMap.values()) {
@@ -137,6 +140,34 @@ router.post('/events/:id/import-external', adminAuth, requirePermission('photos.
})
.returning('id');
const photoId = Array.isArray(inserted) && inserted.length
? (typeof inserted[0] === 'object' ? inserted[0].id : inserted[0])
: null;
// Generate the thumbnail right away so the gallery grid can use the
// managed thumbnail endpoint instead of falling back to the full
// NAS-streamed original (#423). Best-effort: a single failure logs
// a warning and leaves thumbnail_path=null — the gallery's
// ensureThumbnail will retry lazily on first view. The cost of
// doing this synchronously is ~100-300ms per image; for the
// worst-case 1000-photo import that's still under the 5-minute
// request timeout typical of the import flow.
if (photoId != null) {
try {
const outputBasename = `ext${photoId}_${path.basename(f.rel)}`;
const thumbnailPath = await generateThumbnail(f.full, { outputBasename });
if (thumbnailPath) {
await db('photos').where({ id: photoId }).update({ thumbnail_path: thumbnailPath });
thumbnailsGenerated++;
} else {
thumbnailsFailed++;
}
} catch (thumbErr) {
thumbnailsFailed++;
logger.warn(`Thumbnail generation failed for external photo ${photoId} (${f.rel}): ${thumbErr.message}`);
}
}
imported += (inserted?.length ? 1 : 0);
} catch (e) {
skipped++;
@@ -146,10 +177,14 @@ router.post('/events/:id/import-external', adminAuth, requirePermission('photos.
// Update event fields
await db('events').where('id', eventId).update({ source_mode: 'reference', external_path });
// Queue thumbnail generation lazily by reading thumbnails via ensure endpoint as needed
await logActivity('external_import_completed', { event_id: eventId, imported, skipped, external_path }, eventId, { type: 'admin' });
await logActivity(
'external_import_completed',
{ event_id: eventId, imported, skipped, thumbnailsGenerated, thumbnailsFailed, external_path },
eventId,
{ type: 'admin' }
);
res.json({ imported, skipped, thumbnailsQueued: 0 });
res.json({ imported, skipped, thumbnailsGenerated, thumbnailsFailed });
} catch (error) {
logger.error('External media import failed', {
eventId: req.params.id,
backend/src/services/imageProcessor.js
+56 -18
View File
@@ -101,10 +101,17 @@ const contentTypeFor = (format) => {
*
* Callers must ensure the source is on the local filesystem. For S3 mode
* regeneration flows, fetch via `withLocalCopy(storage, sourceKey, fn)` first.
*
* options.outputBasename — override the basename portion of the thumbnail
* filename (default: basename of imagePath). Used for external/reference
* photos where the source basename can collide across events (#423) — the
* import path passes a per-photo unique basename so two events both
* referencing `IMG_0001.jpg` don't clobber each other's thumbnail.
*/
async function generateThumbnail(imagePath, options = {}) {
const filename = path.basename(imagePath);
const thumbnailFilename = `thumb_${filename}`;
const sourceBasename = path.basename(imagePath);
const outputBasename = options.outputBasename || sourceBasename;
const thumbnailFilename = `thumb_${outputBasename}`;
const thumbnailRelKey = path.posix.join('thumbnails', thumbnailFilename);
const storage = getStorage();
@@ -168,7 +175,7 @@ async function generateThumbnail(imagePath, options = {}) {
return thumbnailRelKey;
} catch (error) {
const msg = (error && error.message) ? error.message : String(error);
logger.error(`Failed to generate thumbnail for ${filename}: ${msg}`);
logger.error(`Failed to generate thumbnail for ${sourceBasename}: ${msg}`);
// Clean up any partially uploaded object
await storage.delete(thumbnailRelKey).catch(() => {});
@@ -220,22 +227,25 @@ async function withLocalCopy(sourceKey, fn) {
}
/**
* Regenerate thumbnail if it's broken or missing
* Regenerate thumbnail if it's broken or missing.
*
* Works for both managed photos (stored via the storage backend, possibly
* S3) and external/reference photos (#423 — sourced from a local mount
* outside the managed storage tree, e.g. NAS over SMB/NFS). External
* photos historically had thumbnail_path=null, which forced the gallery
* to fall back to streaming the full original on every tile — minutes of
* load time for a 100-photo NAS-mounted gallery.
*/
async function ensureThumbnail(photo) {
const { resolvePhotoStorageKey } = require('./photoResolver');
let sourceKey;
try {
const event = await db('events').where('id', photo.event_id).first();
sourceKey = resolvePhotoStorageKey(event, photo);
logger.info(`Ensuring thumbnail for photo ${photo.id} from key: ${sourceKey}`);
} catch (e) {
const msg = (e && e.message) ? e.message : String(e);
logger.error(`Failed to resolve original key for thumbnail (photo ${photo.id}): ${msg}`);
const { resolvePhotoStorageKey, resolvePhotoFilePath } = require('./photoResolver');
const event = await db('events').where('id', photo.event_id).first();
if (!event) {
logger.error(`ensureThumbnail: event ${photo.event_id} not found for photo ${photo.id}`);
return null;
}
// Check if thumbnail exists and is valid
// Check if thumbnail exists and is valid (works for any source).
if (photo.thumbnail_path) {
const isValid = await isThumbnailValid(photo.thumbnail_path);
if (isValid) {
@@ -244,10 +254,38 @@ async function ensureThumbnail(photo) {
logger.warn(`Invalid thumbnail detected for photo ${photo.id}, regenerating...`);
}
// Generate new thumbnail (sources via withLocalCopy so this works in S3 mode)
const newThumbnailPath = await withLocalCopy(sourceKey, (localPath) =>
generateThumbnail(localPath, { regenerate: true })
);
const isExternal = photo.source_origin === 'external' || photo.source_origin === 'reference';
let newThumbnailPath;
if (isExternal) {
// External: source is on a local mount path. No withLocalCopy needed
// (storage-backend abstraction doesn't apply — this is a direct fs
// read). Use a per-photo unique outputBasename so two events both
// referencing the same NAS basename can't clobber each other's thumb.
let localPath;
try {
localPath = resolvePhotoFilePath(event, photo);
} catch (e) {
logger.error(`Failed to resolve external file for thumbnail (photo ${photo.id}): ${e.message}`);
return null;
}
const sourceBasename = path.basename(photo.external_relpath || photo.filename || `photo-${photo.id}`);
const outputBasename = `ext${photo.id}_${sourceBasename}`;
logger.info(`Ensuring thumbnail for external photo ${photo.id} from ${localPath}`);
newThumbnailPath = await generateThumbnail(localPath, { regenerate: true, outputBasename });
} else {
let sourceKey;
try {
sourceKey = resolvePhotoStorageKey(event, photo);
} catch (e) {
logger.error(`Failed to resolve original key for thumbnail (photo ${photo.id}): ${e.message}`);
return null;
}
logger.info(`Ensuring thumbnail for photo ${photo.id} from key: ${sourceKey}`);
newThumbnailPath = await withLocalCopy(sourceKey, (localPath) =>
generateThumbnail(localPath, { regenerate: true })
);
}
if (newThumbnailPath) {
await db('photos')