paul/picpeak
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(admin): move the maintenance sweeps' run state into the database (#1181) (#1188)

Browse Source 
Stable twin of #1184. Both photo sweeps tracked whether they were running in a
module-level variable, which is invisible to every other replica: a status poll
routed to an idle replica reports isRunning false while another is mid-run, and
the next POST starts a second pass over the whole library.

Migration 179 adds one row per job, claimed with a conditional UPDATE whose
affected-row count is the answer. The lease is fenced on a per-claim token so a
runner superseded by a stale takeover cannot renew a claim it has lost or
release one it no longer owns; renewal runs on a timer spanning the claim
through release, since one hung NAS read can outlast the stale window inside a
single iteration. maintenance_jobs is excluded from .picpeak archives.

Gated on settings.edit / settings.view rather than main's system.manage, which
does not exist on this branch — they are what settings.edit was later split
into, so both branches let the same people through.

Co-authored-by: Paul Nothaft <[email protected]>
This commit is contained in:
Paul Nothaft
2026-08-26 09:08:14 +02:00
committed by GitHub
co-authored by Paul Nothaft
parent 7f0ed23ea4
commit 58ccecc304
6 changed files with 865 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/routes/adminPhotoDimensions.js
+268 -155
View File
@@ -7,43 +7,106 @@ const fs = require('fs').promises;
const logger = require('../utils/logger');
const { resolvePhotoFilePath } = require('../services/photoResolver');
const maintenanceJobs = require('../services/maintenanceJobState');
// Module-level progress state
let repairProgress = {
isRunning: false,
lastResult: null
};
// Run state for both sweeps lives in the database, not in this process
// (#1181). It used to be a module-level object per job, which is correct on a
// single replica and wrong behind a load balancer: the status poll answers
// from whichever process it reaches, so an idle replica reports isRunning
// false while another is mid-run, the UI re-enables the button, and the next
// POST starts a duplicate pass over the entire library.
//
// Two separate rows, for the same reason the two objects were separate: the
// jobs walk the same photos but read different things out of them, and one
// running must not block or report for the other.
const { JOB_DIMENSION_REPAIR, JOB_CAPTURE_DATE_BACKFILL } = maintenanceJobs;
// Same shape, separate state: the two jobs walk the same photos but read
// different things out of them, and one running must not block or report for
// the other.
let captureDateProgress = {
isRunning: false,
lastResult: null
};
const { HEARTBEAT_INTERVAL_MS } = maintenanceJobs;
/**
* Renew the lease on a timer for as long as the run holds it.
*
* On a timer, not between photos: a single hung read on a stalled NAS mount or
* a slow S3 object can outlast the whole stale window inside one iteration, and
* a renewal that only fires between photos never gets to run. The lease would
* expire while the job was demonstrably alive, another replica would take it
* over, and the two would walk the same rows — precisely the case the lease
* exists to prevent. The timer also covers the candidate query, which on a
* large library is itself slow.
*
* `lost()` reports whether the claim has since been taken over. The loops check
* it between photos and stop: mid-photo interruption is not possible, so the
* worst case is one extra row written by the old runner, and its release is
* fenced on the token anyway.
*/
function startLeaseKeeper(jobName, token) {
let lost = false;
const timer = setInterval(async () => {
try {
if (!(await maintenanceJobs.heartbeat(jobName, token))) {
lost = true;
clearInterval(timer);
}
} catch (err) {
// heartbeat() already swallows query errors and reports the claim as
// held; this is belt-and-braces so an unexpected throw cannot kill the
// timer callback and silently stop all renewals.
logger.warn(`Lease renewal error for ${jobName}: ${err.message}`);
}
}, HEARTBEAT_INTERVAL_MS);
// Do not hold the event loop open on account of a maintenance sweep.
if (typeof timer.unref === 'function') timer.unref();
return {
lost: () => lost,
stop: () => clearInterval(timer),
};
}
// Repair photo dimensions (background job)
router.post('/repair-dimensions', adminAuth, requirePermission('photos.edit'), async (req, res) => {
try {
if (repairProgress.isRunning) {
// Claimed before the candidate query, not after: that query is an await,
// and two requests arriving inside it would both read "not running" and
// both start a pass. The claim is a conditional UPDATE, so it settles the
// race across replicas as well as within one.
const token = await maintenanceJobs.claim(JOB_DIMENSION_REPAIR);
if (!token) {
return res.status(409).json({ error: 'Repair is already running' });
}
// Started at the claim, not at the loop: on a large or loaded install the
// candidate SELECT below (plus the setImmediate hop) can itself outlast the
// stale window, and an unrenewed claim would be taken over before the sweep
// had read its first photo. Handed to the background section, which stops
// it; every early exit here stops it too.
const lease = startLeaseKeeper(JOB_DIMENSION_REPAIR, token);
const photos = await db('photos')
.join('events', 'photos.event_id', 'events.id')
.where(function () {
this.whereNull('photos.width').orWhereNull('photos.height');
})
.where(function () {
this.where('photos.media_type', '!=', 'video').orWhereNull('photos.media_type');
})
.select(
'photos.id', 'photos.path', 'photos.filename',
'photos.source_origin', 'photos.external_relpath', 'photos.event_id',
'events.source_mode', 'events.external_path', 'events.slug'
);
let photos;
try {
photos = await db('photos')
.join('events', 'photos.event_id', 'events.id')
.where(function () {
this.whereNull('photos.width').orWhereNull('photos.height');
})
.where(function () {
this.where('photos.media_type', '!=', 'video').orWhereNull('photos.media_type');
})
.select(
'photos.id', 'photos.path', 'photos.filename',
'photos.source_origin', 'photos.external_relpath', 'photos.event_id',
'events.source_mode', 'events.external_path', 'events.slug'
);
} catch (err) {
lease.stop();
await maintenanceJobs.release(JOB_DIMENSION_REPAIR, token);
throw err;
}
if (photos.length === 0) {
// Released with no result: nothing ran, so the numbers from the last
// real run stay on screen rather than being blanked by a no-op.
lease.stop();
await maintenanceJobs.release(JOB_DIMENSION_REPAIR, token);
return res.json({ message: 'No photos need dimension repair', count: 0 });
}
@@ -54,70 +117,92 @@ router.post('/repair-dimensions', adminAuth, requirePermission('photos.edit'), a
});
// Process in background
repairProgress.isRunning = true;
repairProgress.lastResult = null;
setImmediate(async () => {
let sharp;
try {
sharp = require('sharp');
} catch (err) {
logger.error('Sharp not available for dimension repair:', err.message);
repairProgress.isRunning = false;
repairProgress.lastResult = { success: 0, failed: 0, error: 'Sharp not available' };
lease.stop();
await maintenanceJobs.release(JOB_DIMENSION_REPAIR, token, { success: 0, failed: 0, error: 'Sharp not available' });
return;
}
let successCount = 0;
let errorCount = 0;
let lostClaim = false;
for (const photo of photos) {
try {
const event = { source_mode: photo.source_mode, external_path: photo.external_path, slug: photo.slug };
let fullPath;
try {
fullPath = resolvePhotoFilePath(event, photo);
} catch (err) {
logger.warn(`Photo ${photo.id} has no resolvable path, skipping dimension repair: ${err.message}`);
errorCount++;
continue;
}
// Everything below runs detached from the request, so an unexpected
// throw has nobody to report to. Without this the claim would sit held
// until it aged out of the staleness window, disabling the button for
// that whole time on every replica.
try {
for (const photo of photos) {
// The timer does the renewing; this only notices that it has
// already failed, so the loop stops instead of running on beside the
// replica that took the claim over.
if (lease.lost()) { lostClaim = true; break; }
try {
await fs.access(fullPath);
} catch (err) {
logger.warn(`File not found for photo ${photo.id}: ${fullPath}`);
errorCount++;
continue;
}
const metadata = await sharp(fullPath).metadata();
if (metadata.width && metadata.height) {
await db('photos')
.where({ id: photo.id })
.update({
width: metadata.width,
height: metadata.height
});
successCount++;
if (successCount % 50 === 0) {
logger.info(`Dimension repair progress: ${successCount} updated...`);
const event = { source_mode: photo.source_mode, external_path: photo.external_path, slug: photo.slug };
let fullPath;
try {
fullPath = resolvePhotoFilePath(event, photo);
} catch (err) {
logger.warn(`Photo ${photo.id} has no resolvable path, skipping dimension repair: ${err.message}`);
errorCount++;
continue;
}
} else {
logger.warn(`Could not extract dimensions for photo ${photo.id}`);
try {
await fs.access(fullPath);
} catch (err) {
logger.warn(`File not found for photo ${photo.id}: ${fullPath}`);
errorCount++;
continue;
}
const metadata = await sharp(fullPath).metadata();
if (metadata.width && metadata.height) {
await db('photos')
.where({ id: photo.id })
.update({
width: metadata.width,
height: metadata.height
});
successCount++;
if (successCount % 50 === 0) {
logger.info(`Dimension repair progress: ${successCount} updated...`);
}
} else {
logger.warn(`Could not extract dimensions for photo ${photo.id}`);
errorCount++;
}
} catch (error) {
logger.error(`Error repairing dimensions for photo ${photo.id}:`, error);
errorCount++;
}
} catch (error) {
logger.error(`Error repairing dimensions for photo ${photo.id}:`, error);
errorCount++;
}
}
repairProgress.isRunning = false;
repairProgress.lastResult = { success: successCount, failed: errorCount };
logger.info(`Dimension repair complete: ${successCount} success, ${errorCount} errors`);
if (lostClaim) {
// Another replica declared this run stale and took it over. It owns
// the row now, so releasing would clear ITS flag — release() refuses
// on the token, but there is nothing to report either way.
logger.warn(`Dimension repair stopped: claim taken over after ${successCount} updated, ${errorCount} errors`);
return;
}
await maintenanceJobs.release(JOB_DIMENSION_REPAIR, token, { success: successCount, failed: errorCount });
logger.info(`Dimension repair complete: ${successCount} success, ${errorCount} errors`);
} catch (err) {
logger.error('Dimension repair aborted:', err);
await maintenanceJobs
.release(JOB_DIMENSION_REPAIR, token, { success: successCount, failed: errorCount, error: err.message })
.catch(() => {});
} finally {
lease.stop();
}
});
} catch (error) {
logger.error('Error starting dimension repair:', error);
@@ -146,13 +231,15 @@ router.get('/repair-dimensions/status', adminAuth, requirePermission('photos.vie
const total = Number(totalPhotos.count);
const withDims = Number(withDimensions.count);
// Read from the shared row, so this answers the same on every replica.
const state = await maintenanceJobs.read(JOB_DIMENSION_REPAIR);
res.json({
total,
withDimensions: withDims,
withoutDimensions: total - withDims,
isRunning: repairProgress.isRunning,
lastResult: repairProgress.lastResult
isRunning: state.isRunning,
lastResult: state.lastResult
});
} catch (error) {
logger.error('Error fetching dimension repair status:', error);
@@ -192,15 +279,18 @@ router.get('/repair-dimensions/status', adminAuth, requirePermission('photos.vie
// the same people through.
router.post('/repair-capture-dates', adminAuth, requirePermission('settings.edit'), async (req, res) => {
try {
if (captureDateProgress.isRunning) {
// Claimed here, not after the candidate query: that query is an await, and
// two POSTs arriving inside it would both read "not running" and both start
// a pass over the same rows. Being a conditional UPDATE, the claim settles
// that between replicas too. Every early exit below has to release it
// again, hence the try/catch around the query.
const token = await maintenanceJobs.claim(JOB_CAPTURE_DATE_BACKFILL);
if (!token) {
return res.status(409).json({ error: 'Capture date backfill is already running' });
}
// Claimed here, not after the candidate query: that query is an await, and
// two POSTs arriving inside it would both read isRunning === false and both
// start a pass over the same rows. Every early exit below has to release it
// again, hence the try/catch around the query.
captureDateProgress.isRunning = true;
captureDateProgress.lastResult = null;
// Started at the claim so the candidate SELECT below is covered too — see
// the dimension repair above.
const lease = startLeaseKeeper(JOB_CAPTURE_DATE_BACKFILL, token);
let photos;
try {
@@ -235,12 +325,15 @@ router.post('/repair-capture-dates', adminAuth, requirePermission('settings.edit
'events.source_mode', 'events.external_path', 'events.slug'
);
} catch (err) {
captureDateProgress.isRunning = false;
lease.stop();
await maintenanceJobs.release(JOB_CAPTURE_DATE_BACKFILL, token);
throw err;
}
if (photos.length === 0) {
captureDateProgress.isRunning = false;
// No result passed: nothing ran, so the last real run's numbers survive.
lease.stop();
await maintenanceJobs.release(JOB_CAPTURE_DATE_BACKFILL, token);
return res.json({ message: 'No photos need a capture date', count: 0 });
}
@@ -255,91 +348,109 @@ router.post('/repair-capture-dates', adminAuth, requirePermission('settings.edit
let successCount = 0;
let missingCount = 0;
let errorCount = 0;
let lostClaim = false;
for (const photo of photos) {
try {
const event = { source_mode: photo.source_mode, external_path: photo.external_path, slug: photo.slug };
const isExternal = photo.source_origin === 'external' || photo.source_origin === 'reference';
// Same reasoning as the dimension repair: detached from the request, so
// an unexpected throw must not leave the claim held.
try {
for (const photo of photos) {
// The timer renews; this only notices it has already failed.
if (lease.lost()) { lostClaim = true; break; }
// Two source shapes, same split the thumbnail regenerator uses
// (imageProcessor.js:391-420). External rows live on a local mount
// and are read directly; managed rows live behind the storage
// backend, which on an S3 install is not a filesystem at all — going
// through resolvePhotoFilePath there would build a STORAGE_PATH that
// holds nothing and fail every managed photo.
let captured;
if (isExternal) {
let fullPath;
try {
fullPath = resolvePhotoFilePath(event, photo);
} catch (err) {
logger.warn(`Photo ${photo.id} has no resolvable path, skipping capture date: ${err.message}`);
errorCount++;
continue;
try {
const event = { source_mode: photo.source_mode, external_path: photo.external_path, slug: photo.slug };
const isExternal = photo.source_origin === 'external' || photo.source_origin === 'reference';
// Two source shapes, same split the thumbnail regenerator uses
// (imageProcessor.js:391-420). External rows live on a local mount
// and are read directly; managed rows live behind the storage
// backend, which on an S3 install is not a filesystem at all — going
// through resolvePhotoFilePath there would build a STORAGE_PATH that
// holds nothing and fail every managed photo.
let captured;
if (isExternal) {
let fullPath;
try {
fullPath = resolvePhotoFilePath(event, photo);
} catch (err) {
logger.warn(`Photo ${photo.id} has no resolvable path, skipping capture date: ${err.message}`);
errorCount++;
continue;
}
try {
await fs.access(fullPath);
} catch (err) {
logger.warn(`File not found for photo ${photo.id}: ${fullPath}`);
errorCount++;
continue;
}
captured = await extractCaptureDate(fullPath);
} else {
let sourceKey;
try {
sourceKey = resolvePhotoStorageKey(event, photo);
} catch (err) {
logger.warn(`Photo ${photo.id} has no resolvable storage key, skipping capture date: ${err.message}`);
errorCount++;
continue;
}
// In local-fs mode withLocalCopy hands back the resolved path
// without checking it exists, so the access probe stays. In S3
// mode a missing object throws out of getToFile and lands in the
// outer catch — both end up counted as failures, which is what a
// missing original is.
captured = await withLocalCopy(sourceKey, async (localPath) => {
await fs.access(localPath);
return extractCaptureDate(localPath);
});
}
try {
await fs.access(fullPath);
} catch (err) {
logger.warn(`File not found for photo ${photo.id}: ${fullPath}`);
errorCount++;
continue;
}
captured = await extractCaptureDate(fullPath);
} else {
let sourceKey;
try {
sourceKey = resolvePhotoStorageKey(event, photo);
} catch (err) {
logger.warn(`Photo ${photo.id} has no resolvable storage key, skipping capture date: ${err.message}`);
errorCount++;
continue;
}
// In local-fs mode withLocalCopy hands back the resolved path
// without checking it exists, so the access probe stays. In S3
// mode a missing object throws out of getToFile and lands in the
// outer catch — both end up counted as failures, which is what a
// missing original is.
captured = await withLocalCopy(sourceKey, async (localPath) => {
await fs.access(localPath);
return extractCaptureDate(localPath);
});
}
if (!captured) {
if (!captured) {
// No date recovered. Usually genuine — plenty of sources carry no
// EXIF — but extractCaptureDate also returns null when the file is
// unreadable as an image, so this bucket is "nothing to write",
// not "definitely has no EXIF". The failure counter above is the
// one that means the storage is broken.
missingCount++;
continue;
}
missingCount++;
continue;
}
// whereNull, not a blanket set: the job can run for a long time on a
// large library, and an import or a replacement finishing meanwhile
// has already written a date this pass would otherwise overwrite
// with the same-or-worse value.
const updated = await db('photos')
.where({ id: photo.id })
.whereNull('captured_at')
.update({ captured_at: captured.toISOString() });
if (updated) successCount++;
// whereNull, not a blanket set: the job can run for a long time on a
// large library, and an import or a replacement finishing meanwhile
// has already written a date this pass would otherwise overwrite
// with the same-or-worse value.
const updated = await db('photos')
.where({ id: photo.id })
.whereNull('captured_at')
.update({ captured_at: captured.toISOString() });
if (updated) successCount++;
if (successCount % 50 === 0 && successCount > 0) {
logger.info(`Capture date backfill progress: ${successCount} updated...`);
if (successCount % 50 === 0 && successCount > 0) {
logger.info(`Capture date backfill progress: ${successCount} updated...`);
}
} catch (error) {
logger.error(`Error backfilling capture date for photo ${photo.id}:`, error);
errorCount++;
}
} catch (error) {
logger.error(`Error backfilling capture date for photo ${photo.id}:`, error);
errorCount++;
}
}
captureDateProgress.isRunning = false;
captureDateProgress.lastResult = { success: successCount, noExif: missingCount, failed: errorCount };
logger.info(`Capture date backfill complete: ${successCount} updated, ${missingCount} without EXIF, ${errorCount} errors`);
if (lostClaim) {
logger.warn(`Capture date backfill stopped: claim taken over after ${successCount} updated, ${errorCount} errors`);
return;
}
await maintenanceJobs.release(JOB_CAPTURE_DATE_BACKFILL, token, { success: successCount, noExif: missingCount, failed: errorCount });
logger.info(`Capture date backfill complete: ${successCount} updated, ${missingCount} without EXIF, ${errorCount} errors`);
} catch (err) {
logger.error('Capture date backfill aborted:', err);
await maintenanceJobs
.release(JOB_CAPTURE_DATE_BACKFILL, token, { success: successCount, noExif: missingCount, failed: errorCount, error: err.message })
.catch(() => {});
} finally {
lease.stop();
}
});
} catch (error) {
logger.error('Error starting capture date backfill:', error);
@@ -383,13 +494,15 @@ router.get('/repair-capture-dates/status', adminAuth, requirePermission('setting
const total = Number(counts.total);
const withCaptureDate = Number(counts.dated);
// Read from the shared row, so this answers the same on every replica.
const state = await maintenanceJobs.read(JOB_CAPTURE_DATE_BACKFILL);
res.json({
total,
withCaptureDate,
withoutCaptureDate: total - withCaptureDate,
isRunning: captureDateProgress.isRunning,
lastResult: captureDateProgress.lastResult
isRunning: state.isRunning,
lastResult: state.lastResult
});
} catch (error) {
logger.error('Error fetching capture date backfill status:', error);