fix: interrupt idle worker waits during shutdown

This commit is contained in:
Paul Nothaft
2026-09-08 15:54:01 +02:00
parent 20a291eb34
commit a31a2e25e2
4 changed files with 232 additions and 14 deletions
+19 -7
View File
@@ -29,6 +29,7 @@
const os = require('os');
const { db } = require('../database/db');
const logger = require('../utils/logger');
const { createInterruptibleSleep } = require('../utils/interruptibleSleep');
const { processPhoto } = require('./photoProcessor');
const POLL_INTERVAL_MS = parseInt(process.env.UPLOAD_PROCESSOR_POLL_MS || '1000', 10);
@@ -69,7 +70,9 @@ let running = false;
let workerHandles = [];
let janitorHandle = null;
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
let waits = null;
let stopping = null;
const sleep = ms => waits.sleep(ms);
function isPostgres() {
const c = db.client.config.client;
@@ -175,12 +178,13 @@ async function janitorLoop() {
}
function start() {
if (running) return;
if (running || stopping) return;
if (process.env.UPLOAD_PROCESSOR_DISABLED === 'true') {
logger.info('backgroundProcessor: disabled via UPLOAD_PROCESSOR_DISABLED');
return;
}
waits = createInterruptibleSleep();
running = true;
workerHandles = [];
for (let i = 0; i < CONCURRENCY; i++) {
@@ -199,12 +203,20 @@ function start() {
);
}
async function stop() {
if (!running) return;
function stop() {
if (stopping) return stopping;
if (!running) return Promise.resolve();
running = false;
await Promise.all([...workerHandles, janitorHandle].filter(Boolean));
workerHandles = [];
janitorHandle = null;
// Interrupt idle/backoff waits only. Claims, processing and janitor work
// already in flight still drain before the database can be closed.
waits.cancel();
stopping = Promise.all([...workerHandles, janitorHandle].filter(Boolean)).finally(() => {
workerHandles = [];
janitorHandle = null;
waits = null;
stopping = null;
});
return stopping;
}
module.exports = { start, stop, claimNextPhoto };
+21 -7
View File
@@ -28,6 +28,7 @@
const { db } = require('../database/db');
const logger = require('../utils/logger');
const { createInterruptibleSleep } = require('../utils/interruptibleSleep');
const { processPhotoFaces } = require('./faceProcessor');
const { SidecarUnavailableError } = require('./faceClient');
const { TransientSourceError } = require('./faceProcessor');
@@ -220,7 +221,9 @@ let running = false;
let workerHandles = [];
let janitorHandle = null;
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
let waits = null;
let stopping = null;
const sleep = ms => waits.sleep(ms);
function isPostgres() {
const c = db.client.config.client;
@@ -289,6 +292,8 @@ async function workerLoop(workerIdx) {
continue;
}
if (!running) break;
let claimed;
try {
claimed = await claimNextPhoto(currentlyDeferredEventIds());
@@ -407,12 +412,13 @@ async function janitorLoop() {
}
function start() {
if (running) return;
if (running || stopping) return;
if (process.env.FACE_PROCESSOR_DISABLED === 'true') {
logger.info('faceQueue: disabled via FACE_PROCESSOR_DISABLED');
return;
}
waits = createInterruptibleSleep();
running = true;
workerHandles = [];
for (let i = 0; i < CONCURRENCY; i++) {
@@ -432,12 +438,20 @@ function start() {
);
}
async function stop() {
if (!running) return;
function stop() {
if (stopping) return stopping;
if (!running) return Promise.resolve();
running = false;
await Promise.all([...workerHandles, janitorHandle].filter(Boolean));
workerHandles = [];
janitorHandle = null;
// Interrupt idle/backoff waits only. Claims, processing and janitor work
// already in flight still drain before the database can be closed.
waits.cancel();
stopping = Promise.all([...workerHandles, janitorHandle].filter(Boolean)).finally(() => {
workerHandles = [];
janitorHandle = null;
waits = null;
stopping = null;
});
return stopping;
}
// drainConsolidation, touchedEvents and consolidationRetryAt are exported for
+25
View File
@@ -0,0 +1,25 @@
/** A run owns its waits; cancelling wakes every waiter and clears its timer. */
function createInterruptibleSleep() {
let cancelled = false;
const waiters = new Set();
return {
sleep(ms) {
if (cancelled) return Promise.resolve();
return new Promise(resolve => {
const finish = () => {
clearTimeout(timer);
waiters.delete(finish);
resolve();
};
const timer = setTimeout(finish, ms);
waiters.add(finish);
});
},
cancel() {
cancelled = true;
for (const finish of waiters) finish();
},
};
}
module.exports = { createInterruptibleSleep };