/** * External-media folder watcher (issue 1187). * * Drives the real service against a real temp folder and the real import * pass (sharp and thumbnail generation mocked, as in the other external * import suites). Covers the contracts the feature rests on: * * - only events that opted in, are in reference mode, live and not * archived get a watcher, and reconcile() follows the row both ways; * - the timer sweep imports what appeared since the last pass, through the * same code path as the Import button, and never deletes; * - a change in the folder triggers a debounced import on its own; * - a claim held elsewhere makes the watcher stand down rather than walk * the tree a second time; * - quiet system passes stay out of the activity log, real imports go in. */ const fs = require('fs'); const path = require('path'); const os = require('os'); // Fixture files are written "in the past": an automatic pass leaves a file // modified inside the settle window for the next pass (that is the point of // the check), so anything a test expects to be imported straight away must // not look like a copy still in flight. const writeOld = async (file, content) => { await fs.promises.writeFile(file, content); const old = new Date(Date.now() - 60000); await fs.promises.utimes(file, old, old); }; const waitFor = async (predicate, { timeoutMs = 8000, stepMs = 50 } = {}) => { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { if (await predicate()) return true; await new Promise((r) => setTimeout(r, stepMs)); } return false; }; describe('externalMediaWatcher (issue 1187)', () => { let tmpDir; let mediaRoot; let db; let watcher; let jobState; beforeAll(async () => { tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'picpeak-extwatch-')); mediaRoot = path.join(tmpDir, 'media'); await fs.promises.mkdir(path.join(mediaRoot, 'nas', 'individual'), { recursive: true }); await fs.promises.mkdir(path.join(mediaRoot, 'other'), { recursive: true }); await writeOld(path.join(mediaRoot, 'nas', 'individual', 'a.jpg'), 'not-a-real-jpeg'); process.env.NODE_ENV = 'test'; process.env.EXTERNAL_MEDIA_ROOT = mediaRoot; process.env.JWT_SECRET = process.env.JWT_SECRET || 'extwatch-secret'; // Short timers, and stat-polling so the change test does not depend on // the host's inotify/FSEvents behaviour for a temp directory. process.env.EXTERNAL_MEDIA_WATCH_DEBOUNCE_MS = '100'; process.env.EXTERNAL_MEDIA_WATCH_STABILITY_MS = '150'; process.env.EXTERNAL_MEDIA_WATCH_POLLING = 'true'; process.env.EXTERNAL_MEDIA_WATCH_POLL_INTERVAL_MS = '100'; process.env.EXTERNAL_MEDIA_WATCH_SWEEP_INTERVAL_MS = '0'; process.env.EXTERNAL_MEDIA_WATCH_RECONCILE_INTERVAL_MS = '3600000'; jest.resetModules(); jest.doMock('sharp', () => () => ({ metadata: async () => ({ width: 100, height: 200 }) })); jest.doMock('../../src/services/imageProcessor', () => ({ generateThumbnail: jest.fn(async () => 'thumbnails/mock.jpg'), extractCaptureDate: jest.fn(async () => null), orientedDimensions: (m) => ({ width: m.width, height: m.height }), ensureThumbnail: jest.fn(), })); jest.doMock('../../src/utils/logger', () => ({ debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn(), })); ({ db } = await require('../integration/helpers/crmDb').bootCrmDb()); watcher = require('../../src/services/externalMediaWatcher'); jobState = require('../../src/services/maintenanceJobState'); }, 180000); afterAll(async () => { await watcher.stopExternalMediaWatcher(); if (db) await db.destroy?.(); await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); }); beforeEach(async () => { await watcher.stopExternalMediaWatcher(); await db('activity_logs').del(); await db('photos').del(); await db('external_import_exclusions').del(); await db('events').del(); }); async function seedEvent(overrides = {}) { const [e] = await db('events').insert({ slug: `extwatch-${Math.random().toString(36).slice(2, 8)}`, event_type: 'wedding', event_name: 'extwatch', event_date: '2026-01-01', host_email: 'h@example.com', admin_email: 'a@example.com', password_hash: 'x', share_link: `extwatch-${Math.random()}`, expires_at: new Date().toISOString(), source_mode: 'reference', external_path: 'nas', external_watch: 1, is_active: 1, is_archived: 0, ...overrides, }).returning('id'); return typeof e === 'object' ? e.id : e; } const relpaths = async (eventId) => (await db('photos').where({ event_id: eventId }).select('external_relpath')) .map((r) => r.external_relpath).sort(); it('lists only reference events that opted in, are active and not archived', async () => { const watched = await seedEvent(); await seedEvent({ external_watch: 0 }); await seedEvent({ source_mode: 'managed', external_path: null }); await seedEvent({ is_archived: 1 }); await seedEvent({ is_active: 0 }); const rows = await watcher.listWatchedEvents(); expect(rows.map((r) => r.id)).toEqual([watched]); }); it('reconcile starts a watcher for an opted-in event, imports once, and stops it when the row changes', async () => { const eventId = await seedEvent(); await watcher.reconcile(); expect(watcher.watchedEventIds()).toEqual([eventId]); // Ticking the box is enough: what is already in the folder comes in now, // not at the next sweep. expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); // A folder that does not exist is not watched — and is retried, not failed. await db('events').where('id', eventId).update({ external_path: 'does-not-exist' }); await watcher.reconcile(); expect(watcher.watchedEventIds()).toEqual([]); await db('events').where('id', eventId).update({ external_path: 'nas' }); await watcher.reconcile(); expect(watcher.watchedEventIds()).toEqual([eventId]); await db('events').where('id', eventId).update({ external_watch: 0 }); await watcher.reconcile(); expect(watcher.watchedEventIds()).toEqual([]); }); it('sweep imports new files through the shared import pass and never deletes', async () => { const eventId = await seedEvent(); await watcher.reconcile(); await watcher.sweep(); expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); // An import that changed something is logged, with the system actor ... expect(await db('activity_logs').where({ activity_type: 'external_import_completed' }).count('* as n').first()) .toMatchObject({ n: 1 }); await writeOld(path.join(mediaRoot, 'nas', 'individual', 'b.jpg'), 'also-not-a-jpeg'); await fs.promises.unlink(path.join(mediaRoot, 'nas', 'individual', 'a.jpg')); await watcher.sweep(); // ... b.jpg is in, a.jpg's row is kept although the file is gone. expect(await relpaths(eventId)).toEqual([ path.join('nas', 'individual', 'a.jpg'), path.join('nas', 'individual', 'b.jpg'), ]); const logs = await db('activity_logs').where({ activity_type: 'external_import_completed' }); expect(logs).toHaveLength(2); expect(logs.every((l) => l.actor_type === 'system')).toBe(true); // ... a quiet pass is not: no third entry. await watcher.sweep(); expect(await db('activity_logs').where({ activity_type: 'external_import_completed' }).count('* as n').first()) .toMatchObject({ n: 2 }); // Leave the folder as the next test expects it. await fs.promises.unlink(path.join(mediaRoot, 'nas', 'individual', 'b.jpg')); await writeOld(path.join(mediaRoot, 'nas', 'individual', 'a.jpg'), 'not-a-real-jpeg'); }); it('a file appearing in the folder triggers a debounced import on its own', async () => { const eventId = await seedEvent(); await watcher.reconcile(); await watcher.sweep(); expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); await fs.promises.writeFile(path.join(mediaRoot, 'nas', 'individual', 'c.jpg'), 'new-arrival'); const arrived = await waitFor(async () => (await relpaths(eventId)).includes(path.join('nas', 'individual', 'c.jpg'))); expect(arrived).toBe(true); await fs.promises.unlink(path.join(mediaRoot, 'nas', 'individual', 'c.jpg')); }, 20000); it('stands down while another runner holds the claim for the event', async () => { const eventId = await seedEvent(); const { jobNameFor } = require('../../src/services/externalImportService'); await jobState.ensure(jobNameFor(eventId)); const token = await jobState.claim(jobNameFor(eventId)); expect(token).toBeTruthy(); expect(await watcher.runImport(eventId, 'sweep')).toBeNull(); expect(await relpaths(eventId)).toEqual([]); await jobState.release(jobNameFor(eventId), token); const result = await watcher.runImport(eventId, 'sweep'); expect(result).toMatchObject({ imported: 1 }); }); it('does not bring back a photo an admin deleted, until the manual Import asks for it', async () => { const eventId = await seedEvent(); const { importExternalFolder, recordExclusions } = require('../../src/services/externalImportService'); await watcher.reconcile(); expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); // The delete routes record the exclusion before removing the row. const [row] = await db('photos').where({ event_id: eventId }); await recordExclusions(eventId, [row]); await db('photos').where({ id: row.id }).del(); const swept = await watcher.runImport(eventId, 'sweep'); expect(swept).toMatchObject({ imported: 0, excluded: 1 }); expect(await relpaths(eventId)).toEqual([]); // Pressing Import is the explicit intent: the file comes back and the // exclusion is cleared, so later automatic passes keep it. const manual = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'admin' } }); expect(manual).toMatchObject({ imported: 1 }); expect(await db('external_import_exclusions').where({ event_id: eventId })).toHaveLength(0); }); it('leaves a file that is still changing for the next pass', async () => { const eventId = await seedEvent(); const { importExternalFolder } = require('../../src/services/externalImportService'); const growing = path.join(mediaRoot, 'nas', 'individual', 'growing.jpg'); await fs.promises.writeFile(growing, 'part-one'); // mtime is "now", inside the settle window: deferred, not inserted. const first = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true, settleMs: 300 }); expect(first).toMatchObject({ imported: 1, deferred: 1 }); expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); // Old mtime but the size moves during the wait: still deferred. const old = new Date(Date.now() - 60000); await fs.promises.utimes(growing, old, old); const grow = setTimeout(() => fs.promises.appendFile(growing, '-part-two').then(() => fs.promises.utimes(growing, old, old)), 100); const second = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true, settleMs: 300 }); clearTimeout(grow); expect(second).toMatchObject({ imported: 0, deferred: 1 }); // Quiet now: imported. await fs.promises.utimes(growing, old, old); const third = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true, settleMs: 300 }); expect(third).toMatchObject({ imported: 1, deferred: 0 }); await fs.promises.unlink(growing); }); it('a photo deleted while the pass is settling stays deleted', async () => { const eventId = await seedEvent(); const { importExternalFolder, recordExclusions } = require('../../src/services/externalImportService'); await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'admin' } }); const [row] = await db('photos').where({ event_id: eventId }); // A new file makes the pass wait for the settle window; inside that // window the admin deletes a.jpg. The snapshot taken before the wait // saw a.jpg as present, so only a per-file check can keep it out. await writeOld(path.join(mediaRoot, 'nas', 'individual', 'd.jpg'), 'new-file'); setTimeout(async () => { await recordExclusions(eventId, [row]); await db('photos').where({ id: row.id }).del(); }, 100); const result = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true, settleMs: 400 }); expect(result).toMatchObject({ imported: 1, excluded: 1 }); expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'd.jpg')]); await fs.promises.unlink(path.join(mediaRoot, 'nas', 'individual', 'd.jpg')); }); it('records an exclusion for a replaced external photo too', async () => { const eventId = await seedEvent(); const { importExternalFolder, recordExclusions } = require('../../src/services/externalImportService'); await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'admin' } }); const [row] = await db('photos').where({ event_id: eventId }); // photoReplacementService flips the row to managed but keeps the relpath. await db('photos').where({ id: row.id }).update({ source_origin: 'managed' }); const replaced = await db('photos').where({ id: row.id }).first(); await recordExclusions(eventId, [replaced]); await db('photos').where({ id: row.id }).del(); expect(await watcher.runImport(eventId, 'sweep')).toMatchObject({ imported: 0, excluded: 1 }); expect(await relpaths(eventId)).toEqual([]); }); it('an automatic pass stops when the event stopped qualifying since it was scheduled', async () => { const eventId = await seedEvent(); const { importExternalFolder } = require('../../src/services/externalImportService'); await db('events').where('id', eventId).update({ external_watch: 0 }); expect(await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true })) .toMatchObject({ imported: 0 }); expect(await relpaths(eventId)).toEqual([]); }); it('an automatic pass never rewrites the event folder, and stops if it moved', async () => { const eventId = await seedEvent(); const { importExternalFolder } = require('../../src/services/externalImportService'); // The pass was started for 'nas' but the admin has since pointed the // event at 'other': nothing imported, row untouched. await db('events').where('id', eventId).update({ external_path: 'other' }); const result = await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'system' }, automatic: true }); expect(result).toMatchObject({ imported: 0 }); expect(await relpaths(eventId)).toEqual([]); expect((await db('events').where('id', eventId).first()).external_path).toBe('other'); // The manual Import is what writes the folder onto the event. await importExternalFolder({ eventId, externalPath: 'nas', actor: { type: 'admin' } }); expect((await db('events').where('id', eventId).first()).external_path).toBe('nas'); }); it('re-arms itself for a file that was deferred, so a disabled sweep is not needed', async () => { const eventId = await seedEvent(); // Written just now: the pass that starts with the watcher defers it. await fs.promises.writeFile(path.join(mediaRoot, 'nas', 'individual', 'e.jpg'), 'fresh'); await watcher.reconcile(); expect(await relpaths(eventId)).toEqual([path.join('nas', 'individual', 'a.jpg')]); const arrived = await waitFor(async () => (await relpaths(eventId)).includes(path.join('nas', 'individual', 'e.jpg'))); expect(arrived).toBe(true); await fs.promises.unlink(path.join(mediaRoot, 'nas', 'individual', 'e.jpg')); }, 20000); it('skips an event that was archived or deactivated after it was scheduled', async () => { const archived = await seedEvent(); await db('events').where('id', archived).update({ is_archived: 1 }); expect(await watcher.runImport(archived, 'change')).toBeNull(); const inactive = await seedEvent(); await db('events').where('id', inactive).update({ is_active: 0 }); expect(await watcher.runImport(inactive, 'sweep')).toBeNull(); expect(await db('photos').count('* as n').first()).toMatchObject({ n: 0 }); }); it('follows the row at run time: an event that opted out since scheduling is skipped', async () => { const eventId = await seedEvent(); await db('events').where('id', eventId).update({ external_watch: 0 }); expect(await watcher.runImport(eventId, 'change')).toBeNull(); expect(await relpaths(eventId)).toEqual([]); }); });