fix(file-watcher): bound concurrent photo processing (#846)
* fix(file-watcher): bound concurrent photo processing chokidar fires 'add' once per file — with no ignoreInitial option the boot scan fires it for every existing file, and a bulk drop into the watch folder fires it for every new one at once. Each handler runs DB lookups plus (for new files) a full sharp pipeline; sharp.concurrency(2) only caps libvips threads WITHIN one operation, not the number of parallel pipelines, so unbounded handlers can OOM small hosts. Gate both 'add' and 'unlink' through a shared p-limit (FILE_WATCHER_CONCURRENCY, default 2, floor 1) — mass deletes otherwise burst DB work and ZIP-cache invalidation the same way. p-limit is pinned to ^3.1.0, the last CommonJS release. Adapted from the filpgame fork (426ca491) — thanks @filpgame; extended to cover 'unlink', documented in .env.example, plus a lock-in test for the existing Sharp cache/concurrency caps this bound relies on. * chore(compose): pass FILE_WATCHER_CONCURRENCY into the backend container (codex review of #846) The backend service uses an explicit environment list (no env_file), so the documented override never reached the container in the default compose deployments. Added to both compose files + root .env.example.
This commit is contained in:
@@ -2,6 +2,7 @@ const chokidar = require('chokidar');
|
||||
const path = require('path');
|
||||
const fs = require('fs').promises;
|
||||
const sharp = require('sharp');
|
||||
const pLimit = require('p-limit');
|
||||
const { db } = require('../database/db');
|
||||
const { formatBoolean } = require('../utils/dbCompat');
|
||||
const { generateThumbnail, generateVideoPlaceholder } = require('./imageProcessor');
|
||||
@@ -13,6 +14,20 @@ const downloadZipService = require('./downloadZipService');
|
||||
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
||||
const WATCH_PATH = () => path.join(getStoragePath(), 'events/active');
|
||||
|
||||
// Bound concurrent watcher work. chokidar fires 'add' once per file — with no
|
||||
// ignoreInitial option the boot scan fires it for EVERY existing file, and a
|
||||
// bulk drop into the watch folder fires it for every new one at once. Each
|
||||
// handler runs DB lookups and (for new files) a full sharp pipeline;
|
||||
// sharp.concurrency(2) only caps libvips threads WITHIN one operation, not the
|
||||
// number of parallel pipelines, so unbounded handlers can OOM small hosts.
|
||||
// 'unlink' shares the limiter: mass deletes otherwise burst DB work and
|
||||
// ZIP-cache invalidation the same way.
|
||||
const configuredConcurrency = Number.parseInt(process.env.FILE_WATCHER_CONCURRENCY || '2', 10);
|
||||
const watcherConcurrency = Number.isFinite(configuredConcurrency)
|
||||
? Math.max(1, configuredConcurrency)
|
||||
: 2;
|
||||
const processLimit = pLimit(watcherConcurrency);
|
||||
|
||||
function startFileWatcher() {
|
||||
// Auto-import via filesystem watching only works with the local storage
|
||||
// backend. In S3 mode there is no local directory to watch — every photo
|
||||
@@ -34,19 +49,15 @@ function startFileWatcher() {
|
||||
});
|
||||
|
||||
watcher
|
||||
.on('add', async (filePath) => {
|
||||
try {
|
||||
await processNewPhoto(filePath);
|
||||
} catch (error) {
|
||||
.on('add', (filePath) => {
|
||||
processLimit(() => processNewPhoto(filePath)).catch((error) => {
|
||||
logger.error('Error processing new photo:', error);
|
||||
}
|
||||
});
|
||||
})
|
||||
.on('unlink', async (filePath) => {
|
||||
try {
|
||||
await removePhoto(filePath);
|
||||
} catch (error) {
|
||||
.on('unlink', (filePath) => {
|
||||
processLimit(() => removePhoto(filePath)).catch((error) => {
|
||||
logger.error('Error removing photo:', error);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
logger.info('File watcher started');
|
||||
|
||||
Reference in New Issue
Block a user