diff --git a/.env.example b/.env.example index 87fab8e9..0a55d867 100644 --- a/.env.example +++ b/.env.example @@ -178,6 +178,17 @@ VITE_API_URL=/api # lower to 1 on very small hosts. Default: 2 # FILE_WATCHER_CONCURRENCY=2 +# External-media folder watcher (issue 1187). Reference-mode events can opt in +# per event (Event → Source Mode → "Watch folder for new files"); new images in +# the folder are then imported without pressing Import. Deleted files are +# never removed from the gallery. +# EXTERNAL_MEDIA_WATCH=true # global kill switch +# EXTERNAL_MEDIA_WATCH_POLLING=false # true = stat-polling instead of inotify (NFS/SMB mounts) +# EXTERNAL_MEDIA_WATCH_POLL_INTERVAL_MS=5000 +# EXTERNAL_MEDIA_WATCH_SWEEP_INTERVAL_MS=900000 # timer-driven pass over every watched event; 0 disables +# EXTERNAL_MEDIA_WATCH_DEBOUNCE_MS=10000 # quiet period after the last change before the import runs +# EXTERNAL_MEDIA_WATCH_STABILITY_MS=5000 # how long a file must stop growing before it counts as written + # Release Channel # Options: 'stable' (default), 'beta', or specific version like 'v2.3.0' # 'stable' uses the :stable tag (same as :latest on main) diff --git a/backend/.env.example b/backend/.env.example index 3ed79921..242e8319 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -112,6 +112,17 @@ ARCHIVE_PATH=/app/storage/events/archived # generation from exhausting memory on small hosts. Default: 2 # FILE_WATCHER_CONCURRENCY=2 +# External-media folder watcher (issue 1187). Reference-mode events can opt in +# per event (Event → Source Mode → "Watch folder for new files"); new images in +# the folder are then imported without pressing Import. Deleted files are +# never removed from the gallery. +# EXTERNAL_MEDIA_WATCH=true # global kill switch +# EXTERNAL_MEDIA_WATCH_POLLING=false # true = stat-polling instead of inotify (NFS/SMB mounts) +# EXTERNAL_MEDIA_WATCH_POLL_INTERVAL_MS=5000 +# EXTERNAL_MEDIA_WATCH_SWEEP_INTERVAL_MS=900000 # timer-driven pass over every watched event; 0 disables +# EXTERNAL_MEDIA_WATCH_DEBOUNCE_MS=10000 # quiet period after the last change before the import runs +# EXTERNAL_MEDIA_WATCH_STABILITY_MS=5000 # how long a file must stop growing before it counts as written + # Analytics Backend Configuration (OPTIONAL) # Used for server-side tracking only # Primary configuration should be done through Admin UI > Settings > Analytics diff --git a/backend/__tests__/services/externalMediaWatcher.test.js b/backend/__tests__/services/externalMediaWatcher.test.js new file mode 100644 index 00000000..95787261 --- /dev/null +++ b/backend/__tests__/services/externalMediaWatcher.test.js @@ -0,0 +1,361 @@ +/** + * External-media folder watcher (issue 1187). + * + * Drives the real service against a real temp folder and the real import + * pass (sharp and thumbnail generation mocked, as in the other external + * import suites). Covers the contracts the feature rests on: + * + * - only events that opted in, are in reference mode, live and not + * archived get a watcher, and reconcile() follows the row both ways; + * - the timer sweep imports what appeared since the last pass, through the + * same code path as the Import button, and never deletes; + * - a change in the folder triggers a debounced import on its own; + * - a claim held elsewhere makes the watcher stand down rather than walk + * the tree a second time; + * - quiet system passes stay out of the activity log, real imports go in. + */ + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); + +// Fixture files are written "in the past": an automatic pass leaves a file +// modified inside the settle window for the next pass (that is the point of +// the check), so anything a test expects to be imported straight away must +// not look like a copy still in flight. +const writeOld = async (file, content) => { + await fs.promises.writeFile(file, content); + const old = new Date(Date.now() - 60000); + await fs.promises.utimes(file, old, old); +}; + +const waitFor = async (predicate, { timeoutMs = 8000, stepMs = 50 } = {}) => { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await predicate()) return true; + await new Promise((r) => setTimeout(r, stepMs)); + } + return false; +}; + +describe('externalMediaWatcher (issue 1187)', () => { + let tmpDir; let mediaRoot; let db; let watcher; let jobState; + + beforeAll(async () => { + tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'picpeak-extwatch-')); + mediaRoot = path.join(tmpDir, 'media'); + await fs.promises.mkdir(path.join(mediaRoot, 'nas', 'individual'), { recursive: true }); + await fs.promises.mkdir(path.join(mediaRoot, 'other'), { recursive: true }); + await writeOld(path.join(mediaRoot, 'nas', 'individual', 'a.jpg'), 'not-a-real-jpeg'); + + process.env.NODE_ENV = 'test'; + process.env.EXTERNAL_MEDIA_ROOT = mediaRoot; + process.env.JWT_SECRET = process.env.JWT_SECRET || 'extwatch-secret'; + // Short timers, and stat-polling so the change test does not depend on + // the host's inotify/FSEvents behaviour for a temp directory. + process.env.EXTERNAL_MEDIA_WATCH_DEBOUNCE_MS = '100'; + process.env.EXTERNAL_MEDIA_WATCH_STABILITY_MS = '150'; + process.env.EXTERNAL_MEDIA_WATCH_POLLING = 'true'; + process.env.EXTERNAL_MEDIA_WATCH_POLL_INTERVAL_MS = '100'; + process.env.EXTERNAL_MEDIA_WATCH_SWEEP_INTERVAL_MS = '0'; + process.env.EXTERNAL_MEDIA_WATCH_RECONCILE_INTERVAL_MS = '3600000'; + + jest.resetModules(); + jest.doMock('sharp', () => () => ({ metadata: async () => ({ width: 100, height: 200 }) })); + jest.doMock('../../src/services/imageProcessor', () => ({ + generateThumbnail: jest.fn(async () => 'thumbnails/mock.jpg'), + extractCaptureDate: jest.fn(async () => null), + orientedDimensions: (m) => ({ width: m.width, height: m.height }), + ensureThumbnail: jest.fn(), + })); + jest.doMock('../../src/utils/logger', () => ({ + debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn(), + })); + + ({ db } = await require('../integration/helpers/crmDb').bootCrmDb()); + watcher = require('../../src/services/externalMediaWatcher'); + jobState = require('../../src/services/maintenanceJobState'); + }, 180000); + + afterAll(async () => { + await watcher.stopExternalMediaWatcher(); + if (db) await db.destroy?.(); + await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + }); + + beforeEach(async () => { + await watcher.stopExternalMediaWatcher(); + await db('activity_logs').del(); + await db('photos').del(); + await db('external_import_exclusions').del(); + await db('events').del(); + }); + + async function seedEvent(overrides = {}) { + const [e] = await db('events').insert({ + slug: `extwatch-${Math.random().toString(36).slice(2, 8)}`, + event_type: 'wedding', + event_name: 'extwatch', + event_date: '2026-01-01', + host_email: 'h@example.com', + admin_email: 'a@example.com', + password_hash: 'x', + share_link: `extwatch-${Math.random()}`, + expires_at: new Date().toISOString(), + source_mode: 'reference', + external_path: 'nas', + external_watch: 1, + is_active: 1, + is_archived: 0, + ...overrides, + }).returning('id'); + return typeof e === 'object' ? e.id : e; + } + + const relpaths = async (eventId) => (await db('photos').where({ event_id: eventId }).select('external_relpath')) + .map((r) => r.external_relpath).sort(); + + it('lists only reference events that opted in, are active and not archived', async () => { + const watched = await seedEvent(); + await seedEvent({ external_watch: 0 }); + await seedEvent({ source_mode: 'managed', external_path: null }); + await seedEvent({ is_archived: 1 }); + await seedEvent({ is_active: 0 }); + + const rows = await watcher.listWatchedEvents(); + expect(rows.map((r) => r.id)).toEqual([watched]); + }); + + it('reconcile starts a watcher for an opted-in event, imports once, and stops it when the row changes', async () => { + const eventId = await seedEvent(); + + await watcher.reconcile(); + expect(watcher.watchedEventIds()).toEqual([eventId]); + // Ticking the box is enough: what is already in the folder comes in now, + // not at the next sweep. + expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); + + // A folder that does not exist is not watched — and is retried, not failed. + await db('events').where('id', eventId).update({ external_path: 'does-not-exist' }); + await watcher.reconcile(); + expect(watcher.watchedEventIds()).toEqual([]); + + await db('events').where('id', eventId).update({ external_path: 'nas' }); + await watcher.reconcile(); + expect(watcher.watchedEventIds()).toEqual([eventId]); + + await db('events').where('id', eventId).update({ external_watch: 0 }); + await watcher.reconcile(); + expect(watcher.watchedEventIds()).toEqual([]); + }); + + it('sweep imports new files through the shared import pass and never deletes', async () => { + const eventId = await seedEvent(); + await watcher.reconcile(); + + await watcher.sweep(); + expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); + + // An import that changed something is logged, with the system actor ... + expect(await db('activity_logs').where({ activity_type: 'external_import_completed' }).count('* as n').first()) + .toMatchObject({ n: 1 }); + + await writeOld(path.join(mediaRoot, 'nas', 'individual', 'b.jpg'), 'also-not-a-jpeg'); + await fs.promises.unlink(path.join(mediaRoot, 'nas', 'individual', 'a.jpg')); + await watcher.sweep(); + + // ... b.jpg is in, a.jpg's row is kept although the file is gone. + expect(await relpaths(eventId)).toEqual([ + path.join('nas', 'individual', 'a.jpg'), + path.join('nas', 'individual', 'b.jpg'), + ]); + + const logs = await db('activity_logs').where({ activity_type: 'external_import_completed' }); + expect(logs).toHaveLength(2); + expect(logs.every((l) => l.actor_type === 'system')).toBe(true); + + // ... a quiet pass is not: no third entry. + await watcher.sweep(); + expect(await db('activity_logs').where({ activity_type: 'external_import_completed' }).count('* as n').first()) + .toMatchObject({ n: 2 }); + + // Leave the folder as the next test expects it. + await fs.promises.unlink(path.join(mediaRoot, 'nas', 'individual', 'b.jpg')); + await writeOld(path.join(mediaRoot, 'nas', 'individual', 'a.jpg'), 'not-a-real-jpeg'); + }); + + it('a file appearing in the folder triggers a debounced import on its own', async () => { + const eventId = await seedEvent(); + await watcher.reconcile(); + await watcher.sweep(); + expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); + + await fs.promises.writeFile(path.join(mediaRoot, 'nas', 'individual', 'c.jpg'), 'new-arrival'); + + const arrived = await waitFor(async () => (await relpaths(eventId)).includes(path.join('nas', 'individual', 'c.jpg'))); + expect(arrived).toBe(true); + + await fs.promises.unlink(path.join(mediaRoot, 'nas', 'individual', 'c.jpg')); + }, 20000); + + it('stands down while another runner holds the claim for the event', async () => { + const eventId = await seedEvent(); + + const { jobNameFor } = require('../../src/services/externalImportService'); + await jobState.ensure(jobNameFor(eventId)); + const token = await jobState.claim(jobNameFor(eventId)); + expect(token).toBeTruthy(); + + expect(await watcher.runImport(eventId, 'sweep')).toBeNull(); + expect(await relpaths(eventId)).toEqual([]); + + await jobState.release(jobNameFor(eventId), token); + const result = await watcher.runImport(eventId, 'sweep'); + expect(result).toMatchObject({ imported: 1 }); + }); + + it('does not bring back a photo an admin deleted, until the manual Import asks for it', async () => { + const eventId = await seedEvent(); + const { importExternalFolder, recordExclusions } = require('../../src/services/externalImportService'); + + await watcher.reconcile(); + expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); + + // The delete routes record the exclusion before removing the row. + const [row] = await db('photos').where({ event_id: eventId }); + await recordExclusions(eventId, [row]); + await db('photos').where({ id: row.id }).del(); + + const swept = await watcher.runImport(eventId, 'sweep'); + expect(swept).toMatchObject({ imported: 0, excluded: 1 }); + expect(await relpaths(eventId)).toEqual([]); + + // Pressing Import is the explicit intent: the file comes back and the + // exclusion is cleared, so later automatic passes keep it. + const manual = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'admin' } }); + expect(manual).toMatchObject({ imported: 1 }); + expect(await db('external_import_exclusions').where({ event_id: eventId })).toHaveLength(0); + }); + + it('leaves a file that is still changing for the next pass', async () => { + const eventId = await seedEvent(); + const { importExternalFolder } = require('../../src/services/externalImportService'); + const growing = path.join(mediaRoot, 'nas', 'individual', 'growing.jpg'); + await fs.promises.writeFile(growing, 'part-one'); + + // mtime is "now", inside the settle window: deferred, not inserted. + const first = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true, settleMs: 300 }); + expect(first).toMatchObject({ imported: 1, deferred: 1 }); + expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); + + // Old mtime but the size moves during the wait: still deferred. + const old = new Date(Date.now() - 60000); + await fs.promises.utimes(growing, old, old); + const grow = setTimeout(() => fs.promises.appendFile(growing, '-part-two').then(() => fs.promises.utimes(growing, old, old)), 100); + const second = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true, settleMs: 300 }); + clearTimeout(grow); + expect(second).toMatchObject({ imported: 0, deferred: 1 }); + + // Quiet now: imported. + await fs.promises.utimes(growing, old, old); + const third = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true, settleMs: 300 }); + expect(third).toMatchObject({ imported: 1, deferred: 0 }); + + await fs.promises.unlink(growing); + }); + + it('a photo deleted while the pass is settling stays deleted', async () => { + const eventId = await seedEvent(); + const { importExternalFolder, recordExclusions } = require('../../src/services/externalImportService'); + await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'admin' } }); + const [row] = await db('photos').where({ event_id: eventId }); + + // A new file makes the pass wait for the settle window; inside that + // window the admin deletes a.jpg. The snapshot taken before the wait + // saw a.jpg as present, so only a per-file check can keep it out. + await writeOld(path.join(mediaRoot, 'nas', 'individual', 'd.jpg'), 'new-file'); + setTimeout(async () => { + await recordExclusions(eventId, [row]); + await db('photos').where({ id: row.id }).del(); + }, 100); + const result = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true, settleMs: 400 }); + + expect(result).toMatchObject({ imported: 1, excluded: 1 }); + expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'd.jpg')]); + await fs.promises.unlink(path.join(mediaRoot, 'nas', 'individual', 'd.jpg')); + }); + + it('records an exclusion for a replaced external photo too', async () => { + const eventId = await seedEvent(); + const { importExternalFolder, recordExclusions } = require('../../src/services/externalImportService'); + await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'admin' } }); + const [row] = await db('photos').where({ event_id: eventId }); + + // photoReplacementService flips the row to managed but keeps the relpath. + await db('photos').where({ id: row.id }).update({ source_origin: 'managed' }); + const replaced = await db('photos').where({ id: row.id }).first(); + await recordExclusions(eventId, [replaced]); + await db('photos').where({ id: row.id }).del(); + + expect(await watcher.runImport(eventId, 'sweep')).toMatchObject({ imported: 0, excluded: 1 }); + expect(await relpaths(eventId)).toEqual([]); + }); + + it('an automatic pass stops when the event stopped qualifying since it was scheduled', async () => { + const eventId = await seedEvent(); + const { importExternalFolder } = require('../../src/services/externalImportService'); + await db('events').where('id', eventId).update({ external_watch: 0 }); + expect(await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true })) + .toMatchObject({ imported: 0 }); + expect(await relpaths(eventId)).toEqual([]); + }); + + it('an automatic pass never rewrites the event folder, and stops if it moved', async () => { + const eventId = await seedEvent(); + const { importExternalFolder } = require('../../src/services/externalImportService'); + + // The pass was started for 'nas' but the admin has since pointed the + // event at 'other': nothing imported, row untouched. + await db('events').where('id', eventId).update({ external_path: 'other' }); + const result = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true }); + expect(result).toMatchObject({ imported: 0 }); + expect(await relpaths(eventId)).toEqual([]); + expect((await db('events').where('id', eventId).first()).external_path).toBe('other'); + + // The manual Import is what writes the folder onto the event. + await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'admin' } }); + expect((await db('events').where('id', eventId).first()).external_path).toBe('nas'); + }); + + it('re-arms itself for a file that was deferred, so a disabled sweep is not needed', async () => { + const eventId = await seedEvent(); + // Written just now: the pass that starts with the watcher defers it. + await fs.promises.writeFile(path.join(mediaRoot, 'nas', 'individual', 'e.jpg'), 'fresh'); + await watcher.reconcile(); + expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); + + const arrived = await waitFor(async () => (await relpaths(eventId)).includes(path.join('nas', 'individual', 'e.jpg'))); + expect(arrived).toBe(true); + await fs.promises.unlink(path.join(mediaRoot, 'nas', 'individual', 'e.jpg')); + }, 20000); + + it('skips an event that was archived or deactivated after it was scheduled', async () => { + const archived = await seedEvent(); + await db('events').where('id', archived).update({ is_archived: 1 }); + expect(await watcher.runImport(archived, 'change')).toBeNull(); + + const inactive = await seedEvent(); + await db('events').where('id', inactive).update({ is_active: 0 }); + expect(await watcher.runImport(inactive, 'sweep')).toBeNull(); + + expect(await db('photos').count('* as n').first()).toMatchObject({ n: 0 }); + }); + + it('follows the row at run time: an event that opted out since scheduling is skipped', async () => { + const eventId = await seedEvent(); + await db('events').where('id', eventId).update({ external_watch: 0 }); + + expect(await watcher.runImport(eventId, 'change')).toBeNull(); + expect(await relpaths(eventId)).toEqual([]); + }); +}); diff --git a/backend/migrations/core/208_events_external_watch.js b/backend/migrations/core/208_events_external_watch.js new file mode 100644 index 00000000..227854b8 --- /dev/null +++ b/backend/migrations/core/208_events_external_watch.js @@ -0,0 +1,36 @@ +/** + * `events.external_watch` — per-event opt-in for the external-media folder + * watcher (issue 1187). + * + * Managed uploads are picked up by fileWatcher.js as soon as they land in + * storage/events/active. A reference-mode event has no equivalent: new files + * copied into its NAS folder sit there until an admin opens the event and + * presses Import. services/externalMediaWatcher.js closes that gap for events + * that ask for it. + * + * Opt-in per event rather than a global switch: every watched folder is a set + * of inotify handles (or, on a mount that does not deliver events, a polling + * stat of the whole tree), and a large install with hundreds of reference + * events should not pay that for the ones nobody is still adding files to. + * + * Boolean with a false default so an existing install changes nothing on + * upgrade — the column is read through formatBoolean() so SQLite's 0/1 and + * Postgres' true/false both work. + */ + +exports.up = async function (knex) { + if (!(await knex.schema.hasColumn('events', 'external_watch'))) { + await knex.schema.alterTable('events', (table) => { + table.boolean('external_watch').notNullable().defaultTo(false); + }); + console.log('208: added events.external_watch'); + } +}; + +exports.down = async function (knex) { + if (await knex.schema.hasColumn('events', 'external_watch')) { + await knex.schema.alterTable('events', (table) => { + table.dropColumn('external_watch'); + }); + } +}; diff --git a/backend/migrations/core/209_external_import_exclusions.js b/backend/migrations/core/209_external_import_exclusions.js new file mode 100644 index 00000000..93e6fd32 --- /dev/null +++ b/backend/migrations/core/209_external_import_exclusions.js @@ -0,0 +1,35 @@ +/** + * `external_import_exclusions` — files an admin deleted from a reference + * event, so the folder watcher (issue 1187) does not bring them back. + * + * Deleting an external photo removes its row but leaves the NAS original + * alone (resolvePhotoStorageKey returns null for external rows, on purpose). + * The manual Import only ran when an admin pressed it, so the deleted file + * came back only if they asked. The watcher runs on its own, and a full pass + * that skips only rows the event still has would re-import every deleted + * photo on the next sweep — republishing what an admin removed, without any + * new file arriving. + * + * One row per (event, root-relative path). Automatic passes skip these; the + * manual Import button ignores the list and clears the rows for whatever it + * imports, since pressing it is the explicit intent the exclusion exists to + * protect. + */ + +exports.up = async function (knex) { + if (!(await knex.schema.hasTable('external_import_exclusions'))) { + await knex.schema.createTable('external_import_exclusions', (t) => { + t.increments('id').primary(); + t.integer('event_id').notNullable().references('id').inTable('events').onDelete('CASCADE'); + // Same shape as photos.external_relpath: relative to EXTERNAL_MEDIA_ROOT. + t.text('external_relpath').notNullable(); + t.timestamp('created_at').defaultTo(knex.fn.now()); + t.unique(['event_id', 'external_relpath'], { indexName: 'external_import_exclusions_event_relpath_unique' }); + }); + console.log('209: created external_import_exclusions'); + } +}; + +exports.down = async function (knex) { + await knex.schema.dropTableIfExists('external_import_exclusions'); +}; diff --git a/backend/server.js b/backend/server.js index 921e9b33..d2e9d321 100644 --- a/backend/server.js +++ b/backend/server.js @@ -1132,6 +1132,15 @@ async function startServer() { // Start file watcher startFileWatcher(); + // External-media folder watcher (issue 1187): imports new files into + // reference events that opted in. Not gated on STORAGE_BACKEND like the + // managed watcher — EXTERNAL_MEDIA_ROOT is always a local path. + try { + const { startExternalMediaWatcher } = require('./src/services/externalMediaWatcher'); + startExternalMediaWatcher(); + } catch (err) { + logger.warn('External-media watcher failed to start:', err.message); + } // Start expiration checker startExpirationChecker(); diff --git a/backend/src/routes/adminEvents/crud.js b/backend/src/routes/adminEvents/crud.js index a9288448..58bb4b21 100644 --- a/backend/src/routes/adminEvents/crud.js +++ b/backend/src/routes/adminEvents/crud.js @@ -7,7 +7,7 @@ const { db, logActivity } = require('../../database/db'); const { formatBoolean } = require('../../utils/dbCompat'); const { slugify } = require('../../utils/slug'); const { adminAuth } = require('../../middleware/auth'); -const { requirePermission } = require('../../middleware/permissions'); +const { requirePermission, userHasAllPermissions } = require('../../middleware/permissions'); const { IDENTITY_PRESERVING_NORMALIZE_EMAIL } = require('../../utils/emailNormalization'); const bcrypt = require('bcrypt'); const crypto = require('crypto'); @@ -1601,6 +1601,7 @@ module.exports = (router) => { body('allow_presigned_download').optional().isBoolean(), body('source_mode').optional().isIn(['managed', 'reference']), body('external_path').optional({ nullable: true }).isString().trim(), + body('external_watch').optional().isBoolean(), body('require_password').optional().isBoolean(), // Download protection settings body('protection_level').optional().isIn(['basic', 'standard', 'enhanced', 'maximum']), @@ -1811,8 +1812,47 @@ module.exports = (router) => { updates.external_path = trimmedPath || null; } + // The permission guard below inspects `external_watch` / `external_path` + // by exact name, but SQLite resolves column names case-insensitively, so + // `External_Watch` would sail past it and still land on the column. + // Anything that is one of these two keys in any spelling other than the + // canonical one is dropped here, before the guard. + for (const key of Object.keys(updates)) { + const lower = key.toLowerCase(); + if ((lower === 'external_watch' || lower === 'external_path') && key !== lower) delete updates[key]; + } + + // Folder watcher opt-in (issue 1187). Written through formatBoolean + // like the other event flags so SQLite gets 0/1; a managed event has + // no folder to watch, so the switch is cleared with the path. + if (Object.prototype.hasOwnProperty.call(updates, 'external_watch')) { + updates.external_watch = formatBoolean(updates.external_watch === true || updates.external_watch === 'true'); + } + + // Enabling the watcher, or pointing an enabled one at another folder, + // makes the server import on this admin's behalf — which the manual + // Import endpoint requires photos.upload for. events.edit alone must + // not be a way around that. Only transitions are checked: a save that + // leaves an already-watched event as it is stays an events.edit + // operation, so a role without photos.upload can still edit the rest. + if (Object.prototype.hasOwnProperty.call(updates, 'external_watch') || Object.prototype.hasOwnProperty.call(updates, 'external_path')) { + const current = await db('events').where('id', id).select('external_watch', 'external_path').first(); + const wasWatched = Boolean(current?.external_watch); + const willWatch = Object.prototype.hasOwnProperty.call(updates, 'external_watch') + ? Boolean(updates.external_watch) + : wasWatched; + const pathChanges = Object.prototype.hasOwnProperty.call(updates, 'external_path') + && (updates.external_path || null) !== (current?.external_path || null); + if (willWatch && ((!wasWatched) || pathChanges)) { + if (!(await userHasAllPermissions(req.admin.id, ['photos.upload']))) { + return res.status(403).json({ error: 'The photos.upload permission is required to enable automatic imports for this folder' }); + } + } + } + if (updates.source_mode === 'managed') { updates.external_path = null; + updates.external_watch = formatBoolean(false); } if (updates.source_mode === 'reference' && (updates.external_path === null || updates.external_path === undefined)) { diff --git a/backend/src/routes/adminExternalMedia.js b/backend/src/routes/adminExternalMedia.js index 4bce828c..8221f5a3 100644 --- a/backend/src/routes/adminExternalMedia.js +++ b/backend/src/routes/adminExternalMedia.js @@ -1,32 +1,17 @@ const express = require('express'); -const path = require('path'); -const fs = require('fs').promises; const { adminAuth } = require('../middleware/auth'); const { requirePermission } = require('../middleware/permissions'); const { requireEventOwnership } = require('../middleware/ownership'); -const { list, resolveExternalPath } = require('../services/externalMediaService'); -const { db, logActivity } = require('../database/db'); -const sharp = require('sharp'); +const { list } = require('../services/externalMediaService'); const logger = require('../utils/logger'); -const { generateThumbnail, extractCaptureDate, orientedDimensions } = require('../services/imageProcessor'); -const { isUniqueViolation } = require('../utils/dbErrors'); +const { + importExternalFolder, + ImportInProgressError, + EventNotFoundError, +} = require('../services/externalImportService'); const router = express.Router(); -// Events with an import running in THIS process (#1162). -// -// The second line of defence, not the first: migration 186 puts a unique index -// on (event_id, external_relpath), and that is what actually makes a duplicate -// impossible — it holds across replicas, across restarts, and against anything -// that inserts external rows without going through this route. -// -// This set exists for the reason the duplicates got filed in the first place: -// a large tree takes long enough that the run LOOKS hung, so admins click -// again. Letting that second run walk the whole tree only to have every insert -// bounce off the index wastes minutes of CPU and reports a nonsense -// `skipped: 6012` back. Failing it immediately with 409 says what happened. -const importsInFlight = new Set(); - // GET /api/admin/external-media/list?path=relative/dir router.get('/list', adminAuth, requirePermission('photos.view'), async (req, res) => { try { @@ -42,324 +27,42 @@ router.get('/list', adminAuth, requirePermission('photos.view'), async (req, res } }); -// Helper to recursively collect files under a directory, filtered by image extensions -async function walkDir(dir, baseDir) { - const results = []; - const entries = await fs.readdir(dir, { withFileTypes: true }); - for (const e of entries) { - if (e.name.startsWith('.')) continue; - const full = path.join(dir, e.name); - if (e.isDirectory()) { - results.push(...await walkDir(full, baseDir)); - } else if (e.isFile()) { - const ext = path.extname(e.name).toLowerCase(); - if (['.jpg', '.jpeg', '.png', '.webp'].includes(ext)) { - const rel = path.relative(baseDir, full); - results.push({ full, rel, name: e.name }); - } - } - } - return results; -} - // POST /api/admin/events/:id/import-external // Body: { external_path: string, recursive?: boolean, map?: { individual?: string, collages?: string } } +// +// The import itself lives in services/externalImportService.js so the folder +// watcher (issue 1187) runs the identical pass without an HTTP request. This +// handler only validates, maps the service's errors to status codes, and +// records who asked. router.post('/events/:id/import-external', adminAuth, requirePermission('photos.upload'), requireEventOwnership, async (req, res) => { const eventId = parseInt(req.params.id); - if (importsInFlight.has(eventId)) { - return res.status(409).json({ - error: 'An import is already running for this event. Wait for it to finish before starting another.' - }); - } - importsInFlight.add(eventId); + const { external_path, recursive = true, map = { individual: 'individual', collages: 'collages' } } = req.body || {}; + if (!external_path) return res.status(400).json({ error: 'external_path is required' }); + try { - const { external_path, recursive = true, map = { individual: 'individual', collages: 'collages' } } = req.body || {}; - if (!external_path) return res.status(400).json({ error: 'external_path is required' }); - - // Load event - const event = await db('events').where('id', eventId).first(); - if (!event) return res.status(404).json({ error: 'Event not found' }); - - const baseAbs = resolveExternalPath({ external_path }, ''); - - // What gets STORED on the row (#1163). `f.rel` stays relative to the - // imported folder because the type inference below reads its first segment - // ('individual' / 'collages'); external_relpath is written relative to - // EXTERNAL_MEDIA_ROOT so the row does not depend on a column this very - // handler is about to overwrite. - const basePrefix = String(external_path).replace(/^\/+|\/+$/g, ''); - const toRootRelative = (rel) => (basePrefix ? path.join(basePrefix, rel) : rel); - - // Collect files - const files = recursive ? await walkDir(baseAbs, baseAbs) : (await fs.readdir(baseAbs, { withFileTypes: true })) - .filter(e => e.isFile()) - .map(e => ({ full: path.join(baseAbs, e.name), rel: e.name, name: e.name })) - .filter(f => ['.jpg', '.jpeg', '.png', '.webp'].includes(path.extname(f.name).toLowerCase())); - - // Prepare file metadata and deduplicate by filename within type (keep largest) - let skipped = 0; - const preparedFiles = []; - for (const f of files) { - try { - const stats = await fs.stat(f.full); - const segs = f.rel.split(path.sep); - let type = 'individual'; - if (segs[0] === map.collages) type = 'collage'; - if (segs[0] === map.individual) type = 'individual'; - preparedFiles.push({ ...f, type, size: stats.size }); - } catch (err) { - skipped++; - } - } - - const dedupeMap = new Map(); - for (const file of preparedFiles) { - const dedupeKey = `${file.type}:${path.basename(file.rel).toLowerCase()}`; - const existing = dedupeMap.get(dedupeKey); - if (!existing || file.size > existing.size) { - if (existing) skipped++; - dedupeMap.set(dedupeKey, file); - } else { - skipped++; - } - } - - // Point the event at the new directory BEFORE inserting anything. - // - // Two reasons, both about what a half-finished import leaves behind. The - // update used to run after the loop, so an import that died at photo 500 - // of 1000 left those 500 rows carrying external_relpath into the NEW tree - // while the event still resolved against the OLD one — every one of them - // unreadable. And with face detection on, enqueueEvent accepts - // processing_status NULL (faceProcessor.js:243-246), which these inserts - // leave unset, so an admin hitting the toggle or Re-scan mid-import could - // queue those same rows against the stale path and burn them to 'failed'. - // - // Safe to do first for existing MANAGED photos: photo.source_origin takes - // precedence over event.source_mode in both resolvers (photoResolver.js:23, - // :52) and is NOT NULL defaulting to 'managed', so flipping source_mode - // does not touch them. - // - // Safe for existing EXTERNAL rows too, as of #1163. It was not: relpaths - // were stored relative to external_path, so overwriting the column here - // rebased every row already in the event onto the new folder — quietly, - // because their thumbnails were already on local disk and the grid carried - // on rendering. Rows now carry a root-relative path and this write cannot - // reach them. - await db('events').where('id', eventId).update({ source_mode: 'reference', external_path }); - - let imported = 0; - let thumbnailsGenerated = 0; - let thumbnailsFailed = 0; - - // Face detection (#1090). Managed uploads are enqueued by photoProcessor, - // which sets face_status 'pending' once a photo is processed - // (photoProcessor.js:573) — but external media never goes through it, it - // is inserted directly here. Before #1090 that was invisible, because - // faceProcessor skipped externals anyway; now that they are scannable, an - // import into an already-enabled event would still sit unscanned until - // someone pressed Re-scan. - // - // Ids are collected unconditionally and the setting is read at the END, - // not here: this loop can run for many minutes on a large library, and an - // admin who enables detection during it would otherwise leave every photo - // imported after that moment stuck at NULL forever — the toggle endpoint - // only queues rows that already existed when it fired. - // - // The event path is already committed (above), so the enqueue below is - // free of the ordering hazard it used to carry. It stays at the end anyway - // so the setting can be read after the loop, and it only touches rows that - // are still untouched — see the whereNull there. No video guard needed: - // walkDir collects only jpg/jpeg/png/webp. - const importedPhotoIds = []; - - // Insert photos - for (const f of dedupeMap.values()) { - // Infer type by subfolder names - const segs = f.rel.split(path.sep); - let type = 'individual'; - if (segs[0] === map.collages) type = 'collage'; - if (segs[0] === map.individual) type = 'individual'; - - const relFromRoot = toRootRelative(f.rel); - - try { - // Fast path only. This SELECT settles the common case — a re-import of - // a folder already in the event — without paying for a stat and a - // Sharp metadata read per file. It is NOT the guard: those two calls - // sit between here and the INSERT below, which is exactly the window - // two overlapping imports both walked through (#1162). The unique - // index from migration 186 is the guard, and the catch below is how - // this loop converges when it fires. - const exists = await db('photos') - .where({ event_id: eventId, external_relpath: relFromRoot }) - .first(); - if (exists) { skipped++; continue; } - const stats = await fs.stat(f.full); - - // Extract dimensions via Sharp - let width = null; - let height = null; - try { - const metadata = await sharp(f.full).metadata(); - // Oriented, not raw: a portrait shot from a body that tags rather - // than rotates reports landscape dimensions, and the grid would size - // its tile from those (#1185). - ({ width, height } = orientedDimensions(metadata)); - } catch (dimErr) { - logger.warn(`Could not extract dimensions for ${f.rel}: ${dimErr.message}`); - } - - // Capture date from EXIF (#1172). Managed uploads get this from - // photoProcessor, which external media never goes through — so - // captured_at stayed NULL for every externally imported photo, and the - // gallery's "Date Taken" sort silently degraded into import order via - // its COALESCE fallback. On a library imported in two batches that put - // the first days of a trip after the last ones. - // - // Read here because the file is already open a few lines above for the - // dimensions, so this costs one more read of the same source rather - // than a second pass over the mount. - // - // Best-effort, exactly like the dimensions: a source without EXIF, or - // one Sharp/exifr cannot parse, imports with captured_at NULL and - // falls back to uploaded_at as before. - let capturedAt = null; - try { - capturedAt = await extractCaptureDate(f.full); - } catch (dateErr) { - logger.warn(`Could not extract capture date for ${f.rel}: ${dateErr.message}`); - } - - let inserted; - try { - inserted = await db('photos') - .insert({ - event_id: eventId, - filename: f.name, - // The camera-original name (#745). External ingest never sets - // original_filename, and NAS-mounted galleries are among the - // most likely to be driven from Lightroom — without this the - // round-trip has nothing to match a RAW against. - source_filename: f.name, - // Keep path as a hint for legacy code but not used for resolution in external mode - path: path.join(event.slug, f.name), - thumbnail_path: null, - type, - size_bytes: stats.size, - width, - height, - source_origin: 'external', - external_relpath: relFromRoot, - // .toISOString() rather than the Date: inside jest, Dates handed - // to the sqlite3 binding land as the literal string - // "[object Object]" (see CLAUDE.md). Strings round-trip on both - // engines. - captured_at: capturedAt ? capturedAt.toISOString() : null - }) - .returning('id'); - } catch (insertErr) { - // Another writer inserted this exact path while we were reading - // metadata. That is the outcome the index exists to produce, and it - // is a skip rather than a failure — the row is there, it just isn't - // ours. Counting it as `skipped` keeps the reported totals honest; - // before the index this landed in the outer catch as a nameless - // failure, or (more often) never fired at all and duplicated the row. - if (isUniqueViolation(insertErr)) { skipped++; continue; } - throw insertErr; - } - - const photoId = Array.isArray(inserted) && inserted.length - ? (typeof inserted[0] === 'object' ? inserted[0].id : inserted[0]) - : null; - - // Generate the thumbnail right away so the gallery grid can use the - // managed thumbnail endpoint instead of falling back to the full - // NAS-streamed original (#423). Best-effort: a single failure logs - // a warning and leaves thumbnail_path=null — the gallery's - // ensureThumbnail will retry lazily on first view. The cost of - // doing this synchronously is ~100-300ms per image; for the - // worst-case 1000-photo import that's still under the 5-minute - // request timeout typical of the import flow. - if (photoId != null) { - try { - const outputBasename = `ext${photoId}_${path.basename(f.rel)}`; - const thumbnailPath = await generateThumbnail(f.full, { outputBasename }); - if (thumbnailPath) { - await db('photos').where({ id: photoId }).update({ thumbnail_path: thumbnailPath }); - thumbnailsGenerated++; - } else { - thumbnailsFailed++; - } - } catch (thumbErr) { - thumbnailsFailed++; - logger.warn(`Thumbnail generation failed for external photo ${photoId} (${f.rel}): ${thumbErr.message}`); - } - } - - if (photoId != null) importedPhotoIds.push(photoId); - imported += (inserted?.length ? 1 : 0); - } catch (e) { - skipped++; - } - } - - // The event already resolves to the new directory (set before the loop), - // so the queue is safe to open. Still done here rather than on insert so - // the setting below is read after the loop. Guarded the same way - // photoProcessor guards it - // (both the global flag and the per-event toggle), so installs without the - // feature still never write a face_status. Re-read here rather than before - // the loop so a toggle flipped mid-import is honoured. - let queueFaces = false; - try { - const { isEnabledForEvent } = require('../services/faceSettings'); - const freshEvent = await db('events').where('id', eventId).first(); - queueFaces = await isEnabledForEvent(freshEvent); - } catch (err) { - // Never let the face feature break an import — the photos are the point. - logger.warn(`Could not resolve face settings for event ${eventId}: ${err.message}`); - } - - // Chunked because SQLite caps a statement at 999 bound parameters and an - // import can be far larger than that. - if (queueFaces && importedPhotoIds.length) { - let queued = 0; - for (let i = 0; i < importedPhotoIds.length; i += 500) { - // whereNull, not a blanket set. Committing the event path before the - // loop means a toggle or Re-scan firing mid-import can now genuinely - // queue and even finish some of these rows — so an unconditional - // update would drag 'done' rows back to 'pending' for a duplicate - // scan, and knock 'processing' rows out from under the worker - // mid-flight. Only rows nothing has touched are ours to queue. - queued += await db('photos') - .whereIn('id', importedPhotoIds.slice(i, i + 500)) - .whereNull('face_status') - .update({ face_status: 'pending' }); - } - logger.info(`Queued ${queued} of ${importedPhotoIds.length} imported external photo(s) for face scanning (event ${eventId})`); - } - - await logActivity( - 'external_import_completed', - { event_id: eventId, imported, skipped, thumbnailsGenerated, thumbnailsFailed, external_path }, + const result = await importExternalFolder({ eventId, - { type: 'admin' } - ); - - res.json({ imported, skipped, thumbnailsGenerated, thumbnailsFailed }); + externalPath: external_path, + recursive, + map, + actor: { type: 'admin', id: req.admin?.id, name: req.admin?.username }, + }); + res.json(result); } catch (error) { + if (error instanceof EventNotFoundError) { + return res.status(404).json({ error: 'Event not found' }); + } + if (error instanceof ImportInProgressError) { + // Another run holds this event — a double-click, or the watcher on any + // replica mid-pass. Say so rather than walking the tree a second time. + return res.status(409).json({ error: error.message }); + } logger.error('External media import failed', { eventId: req.params.id, externalPath: req.body?.external_path, error: error.message }); res.status(500).json({ error: 'Failed to import external media' }); - } finally { - // In `finally` and not at the end of `try`: an import that throws must - // still release the event, or a single failure locks out every retry - // until the process restarts. - importsInFlight.delete(eventId); } }); diff --git a/backend/src/routes/adminPhotos.js b/backend/src/routes/adminPhotos.js index dc7c59f6..bab076d2 100644 --- a/backend/src/routes/adminPhotos.js +++ b/backend/src/routes/adminPhotos.js @@ -796,6 +796,9 @@ router.delete('/:eventId/photos/:photoId', adminAuth, requirePermission('photos. logger.warn(`deletePhoto: face purge failed for photo ${photoId}`, { error: err.message }); } + // An external photo's file stays on the NAS; make sure the folder watcher + // (issue 1187) does not re-import it on its next pass. + await require('../services/externalImportService').recordExclusions(Number(eventId), [photo]); await db('photos').where({ id: photoId }).delete(); // Log activity (event was fetched above for storage key resolution) @@ -1037,6 +1040,8 @@ router.post('/:eventId/photos/bulk-delete', adminAuth, requirePermission('photos } // Delete from database + // Same as the single delete: keep the watcher from bringing these back. + await require('../services/externalImportService').recordExclusions(Number(eventId), photos); await db('photos') .whereIn('id', photoIds) .where('event_id', eventId) diff --git a/backend/src/services/externalImportService.js b/backend/src/services/externalImportService.js new file mode 100644 index 00000000..52624224 --- /dev/null +++ b/backend/src/services/externalImportService.js @@ -0,0 +1,585 @@ +/** + * Import the files under an external-media folder into an event. + * + * This used to live inline in POST /events/:id/import-external. It moved here + * because the folder watcher (issue 1187) needs to run the exact same pass + * the Import button runs — same walk, same dedupe, same insert, same + * thumbnail and face handling — without going through HTTP. + * + * Mutual exclusion is the database claim from maintenanceJobState, keyed per + * event. It replaces the in-process Set the route used to keep (#1162): that + * Set stopped a double-click in ONE process, but a watcher on a second + * replica, or an admin clicking Import while the watcher is mid-run on + * another, would still walk the same tree twice. The unique index from + * migration 186 keeps that from duplicating rows; the claim keeps it from + * wasting the walk. The run heartbeats as it goes so a claim from a process + * that died is taken over rather than held forever. + */ + +const path = require('path'); +const fs = require('fs').promises; +const sharp = require('sharp'); +const { db, logActivity } = require('../database/db'); +const logger = require('../utils/logger'); +const { resolveExternalPath } = require('./externalMediaService'); +const { generateThumbnail, extractCaptureDate, orientedDimensions } = require('./imageProcessor'); +const { isUniqueViolation } = require('../utils/dbErrors'); +const jobState = require('./maintenanceJobState'); + +const IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.webp']; + +class ImportInProgressError extends Error { + constructor(eventId) { + super('An import is already running for this event. Wait for it to finish before starting another.'); + this.name = 'ImportInProgressError'; + this.eventId = eventId; + } +} + +class EventNotFoundError extends Error { + constructor(eventId) { + super('Event not found'); + this.name = 'EventNotFoundError'; + this.eventId = eventId; + } +} + +const jobNameFor = (eventId) => `external_import:${eventId}`; + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +/** + * Remember that an admin deleted these photos, so an automatic pass does not + * bring them back (migration 209). Called from the photo delete routes. + * + * Keyed on external_relpath alone, not on source_origin: a replaced external + * photo (photoReplacementService) becomes `managed` but deliberately keeps + * its relpath, and deleting that replacement must not republish the NAS + * original either. Rows without a relpath were never imported from the + * folder and are skipped. Insert failures are swallowed on purpose: a + * duplicate means the row is already there, and anything else must not turn + * a successful delete into a 500. + */ +async function recordExclusions(eventId, photos) { + for (const photo of photos) { + if (!photo.external_relpath) continue; + try { + await db('external_import_exclusions').insert({ event_id: eventId, external_relpath: photo.external_relpath }); + } catch (err) { + if (!isUniqueViolation(err)) { + logger.warn(`Could not record import exclusion for photo ${photo.id}: ${err.message}`); + } + } + } +} + +/** + * Root-relative paths of the rows an event already has, for a set of + * candidates. Chunked for SQLite's bound-parameter cap. + */ +async function existingRelpaths(eventId, relpaths) { + const found = new Set(); + for (let i = 0; i < relpaths.length; i += 500) { + const rows = await db('photos') + .where({ event_id: eventId }) + .whereIn('external_relpath', relpaths.slice(i, i + 500)) + .select('external_relpath'); + for (const r of rows) found.add(r.external_relpath); + } + return found; +} + +// Helper to recursively collect files under a directory, filtered by image extensions +async function walkDir(dir, baseDir) { + const results = []; + const entries = await fs.readdir(dir, { withFileTypes: true }); + for (const e of entries) { + if (e.name.startsWith('.')) continue; + const full = path.join(dir, e.name); + if (e.isDirectory()) { + results.push(...await walkDir(full, baseDir)); + } else if (e.isFile()) { + const ext = path.extname(e.name).toLowerCase(); + if (IMAGE_EXTENSIONS.includes(ext)) { + const rel = path.relative(baseDir, full); + results.push({ full, rel, name: e.name }); + } + } + } + return results; +} + +/** + * Run one import pass. + * + * @param {object} opts + * @param {number} opts.eventId + * @param {string} opts.externalPath folder relative to EXTERNAL_MEDIA_ROOT + * @param {boolean} [opts.recursive=true] + * @param {{individual?: string, collages?: string}} [opts.map] + * @param {object|null} [opts.actor] passed through to logActivity + * @param {boolean} [opts.automatic=false] the watcher's pass, as opposed to + * the Import button. An automatic pass never rewrites the event's source + * configuration (it follows the row, and stops if the row moved under it), + * skips files an admin deleted from this event (migration 209), and defers + * files that are still changing. The manual Import writes the folder it was + * given onto the event, ignores the exclusion list and clears it for what it + * imports. + * @param {number} [opts.settleMs=0] automatic passes: a new file modified + * within the last settleMs, or whose size moves during a wait of settleMs, + * is left for the next pass instead of being imported half-copied + * @returns {Promise<{imported:number, skipped:number, deferred:number, excluded:number, thumbnailsGenerated:number, thumbnailsFailed:number}>} + * @throws {EventNotFoundError} when the event does not exist + * @throws {ImportInProgressError} when another run holds the claim for this event + */ +async function importExternalFolder({ + eventId, + externalPath, + recursive = true, + map = { individual: 'individual', collages: 'collages' }, + actor = null, + automatic = false, + settleMs = 0, +}) { + const external_path = externalPath; + + // Load event + const event = await db('events').where('id', eventId).first(); + if (!event) throw new EventNotFoundError(eventId); + + // The claim comes AFTER the event lookup and BEFORE anything touches the + // filesystem: a large tree takes long enough that a run looks hung and + // admins click again, and letting that second run walk the whole tree only + // to bounce every insert off the unique index wastes minutes of CPU and + // reports a nonsense `skipped: 6012` back. Failing fast says what happened. + const jobName = jobNameFor(eventId); + await jobState.ensure(jobName); + const token = await jobState.claim(jobName); + if (!token) throw new ImportInProgressError(eventId); + + // Renew the lease on a clock for the WHOLE run, walk included. A large tree + // on a slow mount can take longer than the stale window just to list, and + // a runner declared stale during that phase would hand the folder to a + // second runner while this one is about to start inserting. `lost` is + // checked before the event is touched and on every loop iteration. + let lost = false; + + // Automatic passes follow the row. The watcher decided to run this pass + // from a snapshot of the event that may be a minute old by the time the + // walk and the settle wait are done — and a long pass over a slow mount + // can outlive several admin saves. The pass stops as soon as the event no + // longer qualifies: watcher off, archived, deactivated, switched to + // managed, or pointed at another folder. + const stillEligible = async () => { + const now = await db('events') + .where('id', eventId) + .select('source_mode', 'external_path', 'external_watch', 'is_active', 'is_archived') + .first(); + return Boolean(now) + && now.source_mode === 'reference' + && (now.external_path || '') === String(external_path) + && Boolean(now.external_watch) + && Boolean(now.is_active) + && !now.is_archived; + }; + + const heartbeatTimer = setInterval(() => { + jobState.heartbeat(jobName, token).then((ok) => { if (!ok) lost = true; }); + if (automatic) { + stillEligible().then((ok) => { if (!ok) lost = true; }).catch(() => {}); + } + }, jobState.HEARTBEAT_INTERVAL_MS); + heartbeatTimer.unref?.(); + + try { + const baseAbs = resolveExternalPath({ external_path }, ''); + + // What gets STORED on the row (#1163). `f.rel` stays relative to the + // imported folder because the type inference below reads its first segment + // ('individual' / 'collages'); external_relpath is written relative to + // EXTERNAL_MEDIA_ROOT so the row does not depend on a column this very + // function is about to overwrite. + const basePrefix = String(external_path).replace(/^\/+|\/+$/g, ''); + const toRootRelative = (rel) => (basePrefix ? path.join(basePrefix, rel) : rel); + + // Collect files + const files = recursive ? await walkDir(baseAbs, baseAbs) : (await fs.readdir(baseAbs, { withFileTypes: true })) + .filter(e => e.isFile()) + .map(e => ({ full: path.join(baseAbs, e.name), rel: e.name, name: e.name })) + .filter(f => IMAGE_EXTENSIONS.includes(path.extname(f.name).toLowerCase())); + + // Prepare file metadata and deduplicate by filename within type (keep largest) + let skipped = 0; + const preparedFiles = []; + for (const f of files) { + try { + const stats = await fs.stat(f.full); + const segs = f.rel.split(path.sep); + let type = 'individual'; + if (segs[0] === map.collages) type = 'collage'; + if (segs[0] === map.individual) type = 'individual'; + preparedFiles.push({ ...f, type, size: stats.size }); + } catch (err) { + skipped++; + } + } + + const dedupeMap = new Map(); + for (const file of preparedFiles) { + const dedupeKey = `${file.type}:${path.basename(file.rel).toLowerCase()}`; + const existing = dedupeMap.get(dedupeKey); + if (!existing || file.size > existing.size) { + if (existing) skipped++; + dedupeMap.set(dedupeKey, file); + } else { + skipped++; + } + } + + let deferred = 0; + let excluded = 0; + + // Automatic passes: a file still being copied onto the mount is left for + // the next pass. chokidar's awaitWriteFinish settles only the file that + // fired the event, not its siblings, and the sweep sees no events at all + // — so the check is done here, on the candidates that would actually be + // inserted: anything modified within settleMs, or whose size moves across + // a wait of settleMs, is deferred. One wait per pass, not per file. The + // manual Import does not need it: an admin presses it when the copy is + // done. + if (automatic && settleMs > 0) { + const candidates = [...dedupeMap.values()]; + const known = await existingRelpaths(eventId, candidates.map((f) => toRootRelative(f.rel))); + const fresh = candidates.filter((f) => !known.has(toRootRelative(f.rel))); + + { + const unsettled = []; + const before = new Map(); + for (const f of fresh) { + if (!dedupeMap.has(`${f.type}:${path.basename(f.rel).toLowerCase()}`)) continue; + try { + const st = await fs.stat(f.full); + if (Date.now() - st.mtimeMs < settleMs) unsettled.push(f); + else before.set(f.full, st.size); + } catch (_) { + unsettled.push(f); + } + } + if (before.size) { + await sleep(settleMs); + for (const f of fresh) { + if (!before.has(f.full)) continue; + try { + const st = await fs.stat(f.full); + if (st.size !== before.get(f.full) || Date.now() - st.mtimeMs < settleMs) unsettled.push(f); + } catch (_) { + unsettled.push(f); + } + } + } + for (const f of unsettled) { + deferred++; + dedupeMap.delete(`${f.type}:${path.basename(f.rel).toLowerCase()}`); + } + if (deferred) logger.info(`External import for event ${eventId}: ${deferred} file(s) still changing, left for the next pass`); + } + } + + if (lost) throw new ImportInProgressError(eventId); + + if (automatic) { + // Follow the row, never write it. Writing source_mode/external_path + // here would silently undo a save made while the tree was being walked + // or the settle wait was running; and importing into an event that no + // longer qualifies is wrong too. Re-read and stop if the row moved. + // The heartbeat tick keeps re-checking during the loop. + if (!(await stillEligible())) { + logger.info(`External import for event ${eventId}: event no longer eligible, nothing imported`); + const empty = { imported: 0, skipped, deferred, excluded, thumbnailsGenerated: 0, thumbnailsFailed: 0 }; + await jobState.release(jobName, token, null); + return empty; + } + } else { + // Point the event at the new directory BEFORE inserting anything. + // + // Two reasons, both about what a half-finished import leaves behind. The + // update used to run after the loop, so an import that died at photo 500 + // of 1000 left those 500 rows carrying external_relpath into the NEW tree + // while the event still resolved against the OLD one — every one of them + // unreadable. And with face detection on, enqueueEvent accepts + // processing_status NULL (faceProcessor.js:243-246), which these inserts + // leave unset, so an admin hitting the toggle or Re-scan mid-import could + // queue those same rows against the stale path and burn them to 'failed'. + // + // Safe to do first for existing MANAGED photos: photo.source_origin takes + // precedence over event.source_mode in both resolvers (photoResolver.js:23, + // :52) and is NOT NULL defaulting to 'managed', so flipping source_mode + // does not touch them. + // + // Safe for existing EXTERNAL rows too, as of #1163. It was not: relpaths + // were stored relative to external_path, so overwriting the column here + // rebased every row already in the event onto the new folder — quietly, + // because their thumbnails were already on local disk and the grid carried + // on rendering. Rows now carry a root-relative path and this write cannot + // reach them. + await db('events').where('id', eventId).update({ source_mode: 'reference', external_path }); + } + + let imported = 0; + let thumbnailsGenerated = 0; + let thumbnailsFailed = 0; + + // Face detection (#1090). Managed uploads are enqueued by photoProcessor, + // which sets face_status 'pending' once a photo is processed + // (photoProcessor.js:573) — but external media never goes through it, it + // is inserted directly here. Before #1090 that was invisible, because + // faceProcessor skipped externals anyway; now that they are scannable, an + // import into an already-enabled event would still sit unscanned until + // someone pressed Re-scan. + // + // Ids are collected unconditionally and the setting is read at the END, + // not here: this loop can run for many minutes on a large library, and an + // admin who enables detection during it would otherwise leave every photo + // imported after that moment stuck at NULL forever — the toggle endpoint + // only queues rows that already existed when it fired. + // + // The event path is already committed (above), so the enqueue below is + // free of the ordering hazard it used to carry. It stays at the end anyway + // so the setting can be read after the loop, and it only touches rows that + // are still untouched — see the whereNull there. No video guard needed: + // walkDir collects only jpg/jpeg/png/webp. + const importedPhotoIds = []; + + let superseded = false; + + // Insert photos + for (const f of dedupeMap.values()) { + if (lost) { + // Either another process took the claim over — it is walking this + // same folder now, and the unique index makes anything we insert from + // here a wasted stat + decode — or (automatic passes) the event + // stopped qualifying. Stop and let whoever owns it now finish. + superseded = true; + logger.warn(`External import for event ${eventId} stopped mid-run (claim lost or event no longer eligible) after ${imported} imported`); + break; + } + + // Infer type by subfolder names + const segs = f.rel.split(path.sep); + let type = 'individual'; + if (segs[0] === map.collages) type = 'collage'; + if (segs[0] === map.individual) type = 'individual'; + + const relFromRoot = toRootRelative(f.rel); + + try { + // Fast path only. This SELECT settles the common case — a re-import of + // a folder already in the event — without paying for a stat and a + // Sharp metadata read per file. It is NOT the guard: those two calls + // sit between here and the INSERT below, which is exactly the window + // two overlapping imports both walked through (#1162). The unique + // index from migration 186 is the guard, and the catch below is how + // this loop converges when it fires. + const exists = await db('photos') + .where({ event_id: eventId, external_relpath: relFromRoot }) + .first(); + if (exists) { skipped++; continue; } + + // Automatic passes: checked HERE, per file, not against a snapshot + // taken before the loop — an admin can delete a photo (which records + // the exclusion) while this pass is walking or settling, and a + // snapshot would let the loop re-insert it moments later. + if (automatic) { + const excludedRow = await db('external_import_exclusions') + .where({ event_id: eventId, external_relpath: relFromRoot }) + .first(); + if (excludedRow) { excluded++; continue; } + } + const stats = await fs.stat(f.full); + + // Extract dimensions via Sharp + let width = null; + let height = null; + try { + const metadata = await sharp(f.full).metadata(); + // Oriented, not raw: a portrait shot from a body that tags rather + // than rotates reports landscape dimensions, and the grid would size + // its tile from those (#1185). + ({ width, height } = orientedDimensions(metadata)); + } catch (dimErr) { + logger.warn(`Could not extract dimensions for ${f.rel}: ${dimErr.message}`); + } + + // Capture date from EXIF (#1172). Managed uploads get this from + // photoProcessor, which external media never goes through — so + // captured_at stayed NULL for every externally imported photo, and the + // gallery's "Date Taken" sort silently degraded into import order via + // its COALESCE fallback. On a library imported in two batches that put + // the first days of a trip after the last ones. + // + // Read here because the file is already open a few lines above for the + // dimensions, so this costs one more read of the same source rather + // than a second pass over the mount. + // + // Best-effort, exactly like the dimensions: a source without EXIF, or + // one Sharp/exifr cannot parse, imports with captured_at NULL and + // falls back to uploaded_at as before. + let capturedAt = null; + try { + capturedAt = await extractCaptureDate(f.full); + } catch (dateErr) { + logger.warn(`Could not extract capture date for ${f.rel}: ${dateErr.message}`); + } + + let inserted; + try { + inserted = await db('photos') + .insert({ + event_id: eventId, + filename: f.name, + // The camera-original name (#745). External ingest never sets + // original_filename, and NAS-mounted galleries are among the + // most likely to be driven from Lightroom — without this the + // round-trip has nothing to match a RAW against. + source_filename: f.name, + // Keep path as a hint for legacy code but not used for resolution in external mode + path: path.join(event.slug, f.name), + thumbnail_path: null, + type, + size_bytes: stats.size, + width, + height, + source_origin: 'external', + external_relpath: relFromRoot, + // .toISOString() rather than the Date: inside jest, Dates handed + // to the sqlite3 binding land as the literal string + // "[object Object]" (see CLAUDE.md). Strings round-trip on both + // engines. + captured_at: capturedAt ? capturedAt.toISOString() : null + }) + .returning('id'); + } catch (insertErr) { + // Another writer inserted this exact path while we were reading + // metadata. That is the outcome the index exists to produce, and it + // is a skip rather than a failure — the row is there, it just isn't + // ours. Counting it as `skipped` keeps the reported totals honest; + // before the index this landed in the outer catch as a nameless + // failure, or (more often) never fired at all and duplicated the row. + if (isUniqueViolation(insertErr)) { skipped++; continue; } + throw insertErr; + } + + const photoId = Array.isArray(inserted) && inserted.length + ? (typeof inserted[0] === 'object' ? inserted[0].id : inserted[0]) + : null; + + // Generate the thumbnail right away so the gallery grid can use the + // managed thumbnail endpoint instead of falling back to the full + // NAS-streamed original (#423). Best-effort: a single failure logs + // a warning and leaves thumbnail_path=null — the gallery's + // ensureThumbnail will retry lazily on first view. The cost of + // doing this synchronously is ~100-300ms per image; for the + // worst-case 1000-photo import that's still under the 5-minute + // request timeout typical of the import flow. + if (photoId != null) { + try { + const outputBasename = `ext${photoId}_${path.basename(f.rel)}`; + const thumbnailPath = await generateThumbnail(f.full, { outputBasename }); + if (thumbnailPath) { + await db('photos').where({ id: photoId }).update({ thumbnail_path: thumbnailPath }); + thumbnailsGenerated++; + } else { + thumbnailsFailed++; + } + } catch (thumbErr) { + thumbnailsFailed++; + logger.warn(`Thumbnail generation failed for external photo ${photoId} (${f.rel}): ${thumbErr.message}`); + } + } + + if (photoId != null) importedPhotoIds.push(photoId); + imported += (inserted?.length ? 1 : 0); + + // The manual Import is the explicit intent the exclusion list exists + // to protect: what it brings back is no longer excluded. + if (!automatic && photoId != null) { + await db('external_import_exclusions').where({ event_id: eventId, external_relpath: relFromRoot }).delete(); + } + } catch (e) { + skipped++; + } + } + + // The event already resolves to the new directory (set before the loop), + // so the queue is safe to open. Still done here rather than on insert so + // the setting below is read after the loop. Guarded the same way + // photoProcessor guards it + // (both the global flag and the per-event toggle), so installs without the + // feature still never write a face_status. Re-read here rather than before + // the loop so a toggle flipped mid-import is honoured. + let queueFaces = false; + try { + const { isEnabledForEvent } = require('./faceSettings'); + const freshEvent = await db('events').where('id', eventId).first(); + queueFaces = await isEnabledForEvent(freshEvent); + } catch (err) { + // Never let the face feature break an import — the photos are the point. + logger.warn(`Could not resolve face settings for event ${eventId}: ${err.message}`); + } + + // Chunked because SQLite caps a statement at 999 bound parameters and an + // import can be far larger than that. + if (queueFaces && importedPhotoIds.length) { + let queued = 0; + for (let i = 0; i < importedPhotoIds.length; i += 500) { + // whereNull, not a blanket set. Committing the event path before the + // loop means a toggle or Re-scan firing mid-import can now genuinely + // queue and even finish some of these rows — so an unconditional + // update would drag 'done' rows back to 'pending' for a duplicate + // scan, and knock 'processing' rows out from under the worker + // mid-flight. Only rows nothing has touched are ours to queue. + queued += await db('photos') + .whereIn('id', importedPhotoIds.slice(i, i + 500)) + .whereNull('face_status') + .update({ face_status: 'pending' }); + } + logger.info(`Queued ${queued} of ${importedPhotoIds.length} imported external photo(s) for face scanning (event ${eventId})`); + } + + const result = { imported, skipped, deferred, excluded, thumbnailsGenerated, thumbnailsFailed }; + + // Only a run that changed something goes into the activity log. The + // watcher re-runs this pass on a timer for every watched event, and a + // "0 imported, 6012 skipped" row every fifteen minutes per event would + // bury the entries an admin is actually looking for. + if (imported > 0 || actor?.type !== 'system') { + await logActivity( + 'external_import_completed', + { event_id: eventId, ...result, external_path }, + eventId, + actor || { type: 'admin' } + ); + } + + if (!superseded) await jobState.release(jobName, token, result); + return result; + } catch (error) { + // Release without a result so the last real outcome is kept, and in the + // catch rather than a finally so a run that lost its claim does not clear + // the new owner's flag — release() is token-scoped and refuses that anyway, + // but there is no reason to make the call. + await jobState.release(jobName, token, null); + throw error; + } finally { + clearInterval(heartbeatTimer); + } +} + +module.exports = { + importExternalFolder, + recordExclusions, + ImportInProgressError, + EventNotFoundError, + jobNameFor, + IMAGE_EXTENSIONS, +}; diff --git a/backend/src/services/externalMediaWatcher.js b/backend/src/services/externalMediaWatcher.js new file mode 100644 index 00000000..e780db06 --- /dev/null +++ b/backend/src/services/externalMediaWatcher.js @@ -0,0 +1,371 @@ +/** + * Folder watcher for reference-mode events (issue 1187). + * + * Managed uploads dropped into storage/events/active are imported by + * fileWatcher.js without anyone touching the admin UI. External media had no + * equivalent: a reference event's NAS folder keeps growing, and every new + * batch waits for someone to open the event and press Import. This service + * runs that same import pass — literally the same function the button calls — + * whenever a watched folder changes, and again on a timer. + * + * Shape, and why: + * + * - Per-event opt-in (`events.external_watch`, migration 208), not a global + * switch. Each watched tree is a set of inotify handles or, on a mount + * that does not deliver events, a polling stat of every file in it. An + * install with hundreds of reference events should only pay that for the + * ones still receiving files. + * + * - Change events trigger a debounced full pass over the folder, not a + * per-file insert. The import already skips rows it has, so a full pass + * costs one directory walk plus work for the new files — and it keeps + * exactly one code path for external ingest, with the dedupe, the type + * inference, the thumbnail and the face enqueue all in one place. + * + * - A periodic sweep is the fallback, not an optimisation. NFS and SMB + * mounts routinely deliver no inotify events at all for writes made from + * another host, which for a NAS is the normal case. Without the sweep a + * watcher on such a mount would look configured and silently do nothing. + * EXTERNAL_MEDIA_WATCH_POLLING=true switches chokidar to stat-polling for + * installs that want change-driven imports on those mounts anyway. + * + * - Deletions are ignored, deliberately. The manual import only ever adds. + * A file vanishing from the NAS is at least as likely to be a folder + * being reorganised, a mount dropping out, or a copy in progress as it is + * an intentional removal — and acting on it would delete a guest-visible + * photo. Rows whose file is gone stay, exactly as they do today. + * + * - Every replica watches. Which one actually imports is settled by the + * per-event claim inside importExternalFolder; the others get + * ImportInProgressError and stand down. That is what makes the same code + * correct for the AIO container, a two-container install, and a + * multi-replica deploy behind a load balancer. + * + * - Not gated on STORAGE_BACKEND. Unlike the managed watcher, + * EXTERNAL_MEDIA_ROOT is always a local filesystem path — reference + * galleries are not migrated to S3 — so an S3 install can watch too. + * + * Reconciliation runs on a timer against the events table rather than being + * signalled by the toggle endpoint: the endpoint may execute on a different + * replica than the one holding the watcher, and a minute of latency on a + * checkbox is a better trade than cross-process signalling. + */ + +const path = require('path'); +const fs = require('fs').promises; +const chokidar = require('chokidar'); +const { db } = require('../database/db'); +const { formatBoolean } = require('../utils/dbCompat'); +const logger = require('../utils/logger'); +const { resolveExternalPath } = require('./externalMediaService'); +const { + importExternalFolder, + ImportInProgressError, + IMAGE_EXTENSIONS, +} = require('./externalImportService'); + +const envInt = (name, fallback) => { + const parsed = Number.parseInt(process.env[name] || '', 10); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +}; + +// Kill switch. Default on: the feature is already opt-in per event, so an +// install with no watched events runs nothing but the reconcile query. +const ENABLED = (process.env.EXTERNAL_MEDIA_WATCH || 'true').toLowerCase() !== 'false'; +// Stat-polling instead of inotify, for mounts that do not deliver events. +const USE_POLLING = (process.env.EXTERNAL_MEDIA_WATCH_POLLING || 'false').toLowerCase() === 'true'; +const POLL_INTERVAL_MS = envInt('EXTERNAL_MEDIA_WATCH_POLL_INTERVAL_MS', 5000); +// How long a file must stop growing before it counts as written. Generous: +// a NAS copy of a 40 MB raw export can stall for seconds mid-file. +const STABILITY_MS = envInt('EXTERNAL_MEDIA_WATCH_STABILITY_MS', 5000); +// Quiet period after the last change before the pass runs, so a batch copy +// of 300 files becomes one import rather than 300. +const DEBOUNCE_MS = envInt('EXTERNAL_MEDIA_WATCH_DEBOUNCE_MS', 10000); +// Timer-driven pass over every watched event. 0 disables the sweep. +const SWEEP_INTERVAL_MS = envInt('EXTERNAL_MEDIA_WATCH_SWEEP_INTERVAL_MS', 15 * 60 * 1000); +// How often the set of watched events is re-read from the database. +const RECONCILE_INTERVAL_MS = envInt('EXTERNAL_MEDIA_WATCH_RECONCILE_INTERVAL_MS', 60 * 1000); + +const ACTOR = { type: 'system', name: 'external-media-watcher' }; + +// eventId -> { externalPath, watcher, timer } +const watched = new Map(); +// Folders reported missing, so the reconcile loop logs each once rather than +// once a minute until the mount comes back. +const missingLogged = new Set(); +let reconcileTimer = null; +let sweepTimer = null; +let started = false; + +/** + * Events that asked to be watched and can be: reference mode, a folder set, + * live, not archived. Exported for the test. + */ +async function listWatchedEvents() { + return db('events') + .where({ source_mode: 'reference', external_watch: formatBoolean(true) }) + .whereNotNull('external_path') + .where('is_active', formatBoolean(true)) + .where(function () { + this.where('is_archived', formatBoolean(false)).orWhereNull('is_archived'); + }) + .select('id', 'slug', 'external_path'); +} + +const isImage = (filePath) => IMAGE_EXTENSIONS.includes(path.extname(filePath).toLowerCase()); + +/** + * One import pass for an event. The event is re-read first: the folder may + * have moved or the toggle may have been cleared since the change that + * scheduled this, and the pass must follow the row, not the scheduler's memory. + * + * Returns the import result, or null when nothing ran. + */ +async function runImport(eventId, reason) { + const event = await db('events').where('id', eventId).first(); + // The same eligibility listWatchedEvents() applies, re-checked at run time: + // reconcile only looks once a minute, and an event archived or deactivated + // inside that window must not gain photos from a pass scheduled before. + if ( + !event || !event.external_watch || event.source_mode !== 'reference' || !event.external_path + || !event.is_active || event.is_archived + ) { + return null; + } + try { + const result = await importExternalFolder({ + eventId, + externalPath: event.external_path, + recursive: true, + actor: ACTOR, + // Automatic pass: follow the row rather than writing it, keep out what + // an admin deleted, leave files still being copied for the next pass + // (see externalImportService). + automatic: true, + settleMs: STABILITY_MS, + }); + if (result.imported > 0 || result.deferred > 0) { + logger.info(`[externalMediaWatcher] event ${eventId} (${event.slug}): imported ${result.imported}, skipped ${result.skipped}, deferred ${result.deferred}, excluded ${result.excluded} (${reason})`); + } else { + logger.debug(`[externalMediaWatcher] event ${eventId}: nothing new (${reason})`); + } + // A deferred file gets no second 'add' event (ignoreInitial, and the copy + // that made it unsettled already fired its one), and the sweep may be + // disabled — so the pass re-arms itself until the folder is quiet. + if (result.deferred > 0) scheduleImport(eventId); + return result; + } catch (err) { + if (err instanceof ImportInProgressError) { + // Someone else — the Import button, or this watcher on another replica + // — is already walking this folder. A change-triggered pass re-arms so + // files that landed during that run are not left for the sweep; the + // sweep itself just tries again next tick. + logger.debug(`[externalMediaWatcher] event ${eventId}: import already running, ${reason === 'change' ? 're-arming' : 'skipping'}`); + if (reason === 'change') scheduleImport(eventId); + return null; + } + logger.error(`[externalMediaWatcher] import failed for event ${eventId}: ${err.message}`); + return null; + } +} + +/** + * Debounced trigger. Each new change pushes the run back, so a batch copy + * settles into a single pass once the folder has been quiet for DEBOUNCE_MS. + */ +function scheduleImport(eventId) { + const entry = watched.get(eventId); + if (!entry) return; + if (entry.timer) clearTimeout(entry.timer); + entry.timer = setTimeout(() => { + entry.timer = null; + runImport(eventId, 'change').catch((err) => { + logger.error(`[externalMediaWatcher] scheduled import failed for event ${eventId}: ${err.message}`); + }); + }, DEBOUNCE_MS); + entry.timer.unref?.(); +} + +/** + * Start a watcher for one event. Returns true when it is now watched, false + * when it could not be (bad path, folder not there yet). + */ +async function startWatching(event) { + let absPath; + try { + absPath = resolveExternalPath({ external_path: event.external_path }, ''); + } catch (err) { + // A path outside EXTERNAL_MEDIA_ROOT cannot be watched, and should not + // have been saved. Log and leave it; nothing to clean up. + logger.warn(`[externalMediaWatcher] event ${event.id}: refusing to watch '${event.external_path}': ${err.message}`); + return false; + } + + try { + const stat = await fs.stat(absPath); + if (!stat.isDirectory()) throw new Error('not a directory'); + } catch (err) { + // The mount is not there (yet). Reconcile retries every minute, and the + // sweep will not run for an event that is not in `watched` either — a + // folder that cannot be listed has nothing to import. + if (!missingLogged.has(event.id)) { + missingLogged.add(event.id); + logger.warn(`[externalMediaWatcher] event ${event.id} (${event.slug}): folder '${event.external_path}' is not readable (${err.message}); will retry`); + } + return false; + } + missingLogged.delete(event.id); + + const watcher = chokidar.watch(absPath, { + // The sweep and the first reconcile cover what is already there; firing + // 'add' for every existing file on boot would schedule an import of a + // folder that was just imported. + ignoreInitial: true, + persistent: true, + ignored: (p) => path.basename(p).startsWith('.'), + usePolling: USE_POLLING, + interval: POLL_INTERVAL_MS, + binaryInterval: POLL_INTERVAL_MS, + awaitWriteFinish: { + stabilityThreshold: STABILITY_MS, + pollInterval: Math.min(1000, STABILITY_MS), + }, + }); + + const entry = { externalPath: event.external_path, watcher, timer: null }; + watched.set(event.id, entry); + + watcher + .on('add', (filePath) => { + if (isImage(filePath)) scheduleImport(event.id); + }) + .on('unlink', (filePath) => { + // Deliberately not acted on — see the header comment. + if (isImage(filePath)) logger.debug(`[externalMediaWatcher] event ${event.id}: file removed, row kept: ${path.relative(absPath, filePath)}`); + }) + .on('error', (err) => { + logger.warn(`[externalMediaWatcher] event ${event.id}: watcher error: ${err.message}`); + }); + + logger.info(`[externalMediaWatcher] watching event ${event.id} (${event.slug}) at '${event.external_path}'${USE_POLLING ? ' (polling)' : ''}`); + return true; +} + +async function stopWatching(eventId) { + const entry = watched.get(eventId); + if (!entry) return; + watched.delete(eventId); + if (entry.timer) clearTimeout(entry.timer); + try { + await entry.watcher.close(); + } catch (err) { + logger.debug(`[externalMediaWatcher] close failed for event ${eventId}: ${err.message}`); + } + logger.info(`[externalMediaWatcher] stopped watching event ${eventId}`); +} + +/** + * Bring the set of live watchers in line with the events table: start for + * events that opted in since the last pass, restart the ones whose folder + * moved, stop the ones that opted out, archived, or went inactive. + * + * A watcher that just started gets one pass straight away. chokidar is told + * to ignore what is already in the folder, so without this an admin ticking + * the box — or a mount coming back after an outage, or a fresh replica + * booting — would see nothing happen until the next sweep. The pass skips + * rows the event already has, so on a folder that was imported by hand it + * costs one directory walk. + */ +async function reconcile() { + let events; + try { + events = await listWatchedEvents(); + } catch (err) { + logger.warn(`[externalMediaWatcher] could not list watched events: ${err.message}`); + return; + } + + const wanted = new Map(events.map((e) => [e.id, e])); + + for (const eventId of [...watched.keys()]) { + const next = wanted.get(eventId); + if (!next || next.external_path !== watched.get(eventId).externalPath) { + await stopWatching(eventId); + } + } + + const fresh = []; + for (const event of wanted.values()) { + if (!watched.has(event.id) && await startWatching(event)) { + fresh.push(event.id); + } + } + + for (const eventId of fresh) { + await runImport(eventId, 'start'); + } +} + +/** + * Timer-driven pass over every watched event, one at a time. Sequential on + * purpose: each pass reads and decodes new files off the mount, and running + * them in parallel would only make a slow NAS slower. + */ +async function sweep() { + for (const eventId of [...watched.keys()]) { + await runImport(eventId, 'sweep'); + } +} + +function startExternalMediaWatcher() { + if (!ENABLED) { + logger.info('[externalMediaWatcher] disabled via EXTERNAL_MEDIA_WATCH=false'); + return null; + } + if (started) return null; + started = true; + + reconcile().catch((err) => logger.warn(`[externalMediaWatcher] initial reconcile failed: ${err.message}`)); + + reconcileTimer = setInterval(() => { + reconcile().catch((err) => logger.warn(`[externalMediaWatcher] reconcile failed: ${err.message}`)); + }, RECONCILE_INTERVAL_MS); + reconcileTimer.unref?.(); + + if (SWEEP_INTERVAL_MS > 0) { + sweepTimer = setInterval(() => { + sweep().catch((err) => logger.warn(`[externalMediaWatcher] sweep failed: ${err.message}`)); + }, SWEEP_INTERVAL_MS); + sweepTimer.unref?.(); + } + + logger.info(`[externalMediaWatcher] started (sweep every ${SWEEP_INTERVAL_MS > 0 ? `${Math.round(SWEEP_INTERVAL_MS / 60000)} min` : 'never'}, debounce ${DEBOUNCE_MS} ms${USE_POLLING ? ', polling' : ''})`); + return { reconcile, sweep }; +} + +/** + * Tear everything down. For tests and for a clean shutdown; the process exits + * fine without it because every timer is unref'd. + */ +async function stopExternalMediaWatcher() { + if (reconcileTimer) clearInterval(reconcileTimer); + if (sweepTimer) clearInterval(sweepTimer); + reconcileTimer = null; + sweepTimer = null; + for (const eventId of [...watched.keys()]) { + await stopWatching(eventId); + } + missingLogged.clear(); + started = false; +} + +module.exports = { + startExternalMediaWatcher, + stopExternalMediaWatcher, + // Exposed for the test suite. + reconcile, + sweep, + runImport, + listWatchedEvents, + watchedEventIds: () => [...watched.keys()], +}; diff --git a/backend/src/services/maintenanceJobState.js b/backend/src/services/maintenanceJobState.js index 1d69c962..4ced166a 100644 --- a/backend/src/services/maintenanceJobState.js +++ b/backend/src/services/maintenanceJobState.js @@ -40,6 +40,7 @@ const os = require('os'); const crypto = require('crypto'); const { db } = require('../database/db'); const logger = require('../utils/logger'); +const { isUniqueViolation } = require('../utils/dbErrors'); const JOB_DIMENSION_REPAIR = 'photo_dimension_repair'; const JOB_CAPTURE_DATE_BACKFILL = 'photo_capture_date_backfill'; @@ -63,6 +64,26 @@ const OWNER = `${os.hostname()}:${process.pid}`; const nowIso = () => new Date().toISOString(); const cutoffIso = (staleAfterMs) => new Date(Date.now() - staleAfterMs).toISOString(); +/** + * Make sure a row exists for `jobName`, so claim() has something to UPDATE. + * + * The maintenance sweeps are seeded by migration 189 and never need this. It + * exists for job names that are only known at runtime — an external-media + * import is claimed per event (`external_import:`), and events are created + * long after any migration ran. Two replicas racing to seed the same name is + * settled by the primary key: the loser's insert bounces and the row is there + * either way. + */ +async function ensure(jobName) { + const row = await db('maintenance_jobs').where({ job_name: jobName }).first(); + if (row) return; + try { + await db('maintenance_jobs').insert({ job_name: jobName, is_running: false }); + } catch (err) { + if (!isUniqueViolation(err)) throw err; + } +} + /** * Try to become the one runner of `jobName`. * @@ -168,6 +189,7 @@ async function read(jobName, { staleAfterMs = DEFAULT_STALE_MS } = {}) { } module.exports = { + ensure, claim, heartbeat, release, diff --git a/docker-compose.production.yml b/docker-compose.production.yml index 420c2510..bd6e8bdc 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -102,6 +102,13 @@ services: - PICPEAK_RELEASE_CHANNEL=${PICPEAK_CHANNEL:-stable} # Watch-folder auto-import: max photos processed in parallel (default 2). - FILE_WATCHER_CONCURRENCY=${FILE_WATCHER_CONCURRENCY:-2} + # External-media folder watcher (issue 1187); all optional, see .env.example. + - EXTERNAL_MEDIA_WATCH=${EXTERNAL_MEDIA_WATCH:-true} + - EXTERNAL_MEDIA_WATCH_POLLING=${EXTERNAL_MEDIA_WATCH_POLLING:-false} + - EXTERNAL_MEDIA_WATCH_POLL_INTERVAL_MS=${EXTERNAL_MEDIA_WATCH_POLL_INTERVAL_MS:-5000} + - EXTERNAL_MEDIA_WATCH_SWEEP_INTERVAL_MS=${EXTERNAL_MEDIA_WATCH_SWEEP_INTERVAL_MS:-900000} + - EXTERNAL_MEDIA_WATCH_DEBOUNCE_MS=${EXTERNAL_MEDIA_WATCH_DEBOUNCE_MS:-10000} + - EXTERNAL_MEDIA_WATCH_STABILITY_MS=${EXTERNAL_MEDIA_WATCH_STABILITY_MS:-5000} # Face recognition (#1074). Defaults to the sidecar's compose service # name; nothing touches it until the `faces` feature flag is enabled in # admin settings, so installs without the picpeak-ml container are diff --git a/docker-compose.yml b/docker-compose.yml index 2ff5b814..55766657 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -90,6 +90,13 @@ services: - STORAGE_PATH=/app/storage # Watch-folder auto-import: max photos processed in parallel (default 2). - FILE_WATCHER_CONCURRENCY=${FILE_WATCHER_CONCURRENCY:-2} + # External-media folder watcher (issue 1187); all optional, see .env.example. + - EXTERNAL_MEDIA_WATCH=${EXTERNAL_MEDIA_WATCH:-true} + - EXTERNAL_MEDIA_WATCH_POLLING=${EXTERNAL_MEDIA_WATCH_POLLING:-false} + - EXTERNAL_MEDIA_WATCH_POLL_INTERVAL_MS=${EXTERNAL_MEDIA_WATCH_POLL_INTERVAL_MS:-5000} + - EXTERNAL_MEDIA_WATCH_SWEEP_INTERVAL_MS=${EXTERNAL_MEDIA_WATCH_SWEEP_INTERVAL_MS:-900000} + - EXTERNAL_MEDIA_WATCH_DEBOUNCE_MS=${EXTERNAL_MEDIA_WATCH_DEBOUNCE_MS:-10000} + - EXTERNAL_MEDIA_WATCH_STABILITY_MS=${EXTERNAL_MEDIA_WATCH_STABILITY_MS:-5000} # Face recognition (#1074). The URL defaults to the sidecar's compose # service name, so the common case needs no configuration. None of this # is touched until the `faces` feature flag is enabled in admin diff --git a/frontend/src/i18n/locales/de.json b/frontend/src/i18n/locales/de.json index 05b4abb8..cff4c6c8 100644 --- a/frontend/src/i18n/locales/de.json +++ b/frontend/src/i18n/locales/de.json @@ -1876,6 +1876,10 @@ "sourceModeHelp": "Nutzen Sie den verwalteten Modus für direkte Uploads oder verweisen Sie auf einen gemounteten /external-media Ordner.", "externalFolder": "Externer Ordner", "externalFolderHint": "Diese Ordner stammen aus dem /external-media Mount innerhalb des Containers oder Hosts.", + "externalWatch": "Ordner auf neue Dateien überwachen", + "externalWatchHint": "Neue Bilder, die in diesen Ordner kopiert werden, werden automatisch importiert – genau wie über den Import-Button. Aus dem Ordner entfernte Dateien werden nie aus der Galerie gelöscht.", + "externalWatchActive": "Ordner wird überwacht – neue Dateien werden automatisch importiert.", + "externalWatchNoPermission": "Erfordert die Berechtigung, Fotos hochzuladen.", "externalFolderRequired": "Bitte wählen Sie vor dem Speichern einen externen Ordner aus.", "externalFolderEmpty": "Keine Unterordner", "clearSelection": "Löschen", diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index 81320a9b..aaa4ddc4 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -1373,6 +1373,10 @@ "sourceModeHelp": "Use managed mode for direct uploads or point to a mounted /external-media folder when using local storage.", "externalFolder": "External Folder", "externalFolderHint": "These folders are read from the /external-media mount inside your container or host.", + "externalWatch": "Watch folder for new files", + "externalWatchHint": "New images copied into this folder are imported automatically, the same way the Import button does it. Files removed from the folder are never deleted from the gallery.", + "externalWatchActive": "Folder is watched — new files are imported automatically.", + "externalWatchNoPermission": "Requires the permission to upload photos.", "externalFolderRequired": "Please select an external folder before saving.", "externalFolderEmpty": "No subfolders", "clearSelection": "Clear", diff --git a/frontend/src/pages/admin/EventDetailsPage.tsx b/frontend/src/pages/admin/EventDetailsPage.tsx index 383dc2d7..93ce7a68 100644 --- a/frontend/src/pages/admin/EventDetailsPage.tsx +++ b/frontend/src/pages/admin/EventDetailsPage.tsx @@ -424,6 +424,7 @@ export const EventDetailsPage: React.FC = () => { customer_phone: event.customer_phone || '', source_mode: event.source_mode === 'reference' ? 'reference' : 'managed', external_path: event.external_path || '', + external_watch: Boolean(event.external_watch), require_password: normalizeRequirePassword(event.require_password), new_password: '', confirm_new_password: '', @@ -626,6 +627,9 @@ export const EventDetailsPage: React.FC = () => { updateData.external_path = editForm.source_mode === 'reference' ? externalPathToSave : null; + // Always sent, like og_image_share_enabled: the backend writes through + // formatBoolean, so a save can switch the watcher off again. + updateData.external_watch = editForm.source_mode === 'reference' && editForm.external_watch; if (editForm.customer_name !== undefined && editForm.customer_name !== null) { updateData.customer_name = editForm.customer_name; } diff --git a/frontend/src/pages/admin/event-details/EventInformationCard.tsx b/frontend/src/pages/admin/event-details/EventInformationCard.tsx index 6311b2be..56a47650 100644 --- a/frontend/src/pages/admin/event-details/EventInformationCard.tsx +++ b/frontend/src/pages/admin/event-details/EventInformationCard.tsx @@ -27,6 +27,7 @@ import type { AdminPhoto } from '../../../services/photos.service'; import type { FeedbackSettings as FeedbackSettingsType } from '../../../services/feedback.service'; import { ExternalFolderPicker } from './ExternalFolderPicker'; import { safeParseDate } from './utils'; +import { usePermission } from '../../../hooks/usePermission'; import type { EditFormState } from './types'; interface EventInformationCardProps { @@ -64,6 +65,11 @@ export const EventInformationCard: React.FC = ({ onRevealNow }) => { const { t } = useTranslation(); + // Enabling the watcher makes the server import on the admin's behalf, which + // the backend gates on photos.upload like the Import button. Mirror that + // here rather than letting the save bounce with a 403. + const canEnableWatch = usePermission('photos.upload'); + const { format } = useLocalizedDate(); const queryClient = useQueryClient(); const [logoUploading, setLogoUploading] = useState(false); @@ -356,6 +362,28 @@ export const EventInformationCard: React.FC = ({

{t('events.externalFolderHint', 'These folders come from the /external-media mount inside the container. Ensure it is accessible to the backend process.')}

+ )} @@ -850,6 +878,11 @@ export const EventInformationCard: React.FC = ({ {event.source_mode === 'reference' && event.external_path ? ( /external-media/{event.external_path} ) : null} + {event.source_mode === 'reference' && event.external_watch ? ( + + {t('events.externalWatchActive', 'Folder is watched — new files are imported automatically.')} + + ) : null}
diff --git a/frontend/src/pages/admin/event-details/types.ts b/frontend/src/pages/admin/event-details/types.ts index 2ff3a7a3..e57c1a8a 100644 --- a/frontend/src/pages/admin/event-details/types.ts +++ b/frontend/src/pages/admin/event-details/types.ts @@ -16,6 +16,7 @@ export type EditFormState = { customer_phone: string; source_mode: 'managed' | 'reference'; external_path: string; + external_watch: boolean; require_password: boolean; new_password: string; confirm_new_password: string; @@ -71,6 +72,7 @@ export const INITIAL_EDIT_FORM: EditFormState = { customer_phone: '', source_mode: 'managed', external_path: '', + external_watch: false, require_password: true, new_password: '', confirm_new_password: '', diff --git a/frontend/src/services/events.service.ts b/frontend/src/services/events.service.ts index ead45c25..f07076b2 100644 --- a/frontend/src/services/events.service.ts +++ b/frontend/src/services/events.service.ts @@ -79,6 +79,7 @@ interface UpdateEventData { hero_photo_id?: number | null; source_mode?: 'managed' | 'reference'; external_path?: string | null; + external_watch?: boolean; photo_cap?: number | null; default_photo_sort?: string; // Per-event opt-in for hero photo as social-share preview (#474). diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 8a39245a..b5168766 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -39,6 +39,8 @@ export interface Event { unique_visitors?: number; source_mode?: 'managed' | 'reference' | string; external_path?: string | null; + // Folder watcher opt-in (issue 1187). SQLite hands back 0/1, Postgres a boolean. + external_watch?: boolean | number | null; // Download protection fields allow_downloads?: boolean; protection_level?: 'basic' | 'standard' | 'enhanced' | 'maximum';