picpeak/backend/__tests__/integration/faceExternalImportEnqueue.test.js

/**
 * External imports are queued for face scanning, in the right order (#1090).
 *
 * Managed uploads are enqueued by photoProcessor, which writes face_status
 * 'pending' once a photo is processed (photoProcessor.js:573 — "the only
 * correct place to enqueue"). External media never goes through photoProcessor:
 * adminExternalMedia inserts rows directly, so they stayed NULL and were only
 * ever picked up by a manual Re-scan.
 *
 * The ordering matters as much as the enqueue. events.external_path is written
 * only AFTER the whole import loop, so marking rows 'pending' as they are
 * inserted publishes claimable work while the event still points at the old
 * directory — or none at all, on a first import. The face worker polls
 * continuously, would resolve those photos against the wrong path, and mark
 * them permanently 'failed', a state only an explicit Re-scan clears.
 *
 * This drives the real route rather than re-implementing it, so removing the
 * enqueue fails the first test and moving it back onto the insert fails the
 * second.
 */

const fs = require('fs');
const path = require('path');
const os = require('os');
const express = require('express');
const request = require('supertest');

describe('external import queues faces (#1090)', () => {
  let tmpDir; let db; let app; let mediaRoot;
  // Recorded from inside the per-photo thumbnail call, i.e. mid-loop.
  let pendingSeenDuringLoop = 0;
  let externalPathDuringLoop;
  // When set to an event id, the mocked thumbnail call turns detection on
  // mid-loop, standing in for an admin flipping the toggle during an import.
  let flipFacesOnDuringLoop = null;
  // Stands in for a concurrent Re-scan completing a row mid-import.
  let markDoneDuringLoop = false;

  beforeAll(async () => {
    tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'picpeak-extenq-'));
    mediaRoot = path.join(tmpDir, 'media');
    await fs.promises.mkdir(path.join(mediaRoot, 'nas', 'individual'), { recursive: true });
    for (const name of ['a.jpg', 'b.jpg', 'c.jpg']) {
      await fs.promises.writeFile(path.join(mediaRoot, 'nas', 'individual', name), 'not-a-real-jpeg');
    }

    process.env.NODE_ENV = 'test';
    process.env.TEST_DATABASE_PATH = path.join(tmpDir, 'data', 'db.sqlite');
    await fs.promises.mkdir(path.dirname(process.env.TEST_DATABASE_PATH), { recursive: true });
    process.env.STORAGE_PATH = path.join(tmpDir, 'storage');
    process.env.EXTERNAL_MEDIA_ROOT = mediaRoot;
    process.env.JWT_SECRET = process.env.JWT_SECRET || 'extenq-secret';

    jest.resetModules();

    jest.doMock('../../src/middleware/auth', () => ({
      adminAuth: (req, _res, next) => { req.admin = { id: 1, username: 'tester', roleName: 'admin' }; next(); },
    }));
    jest.doMock('../../src/middleware/permissions', () => ({
      requirePermission: () => (_req, _res, next) => next(),
    }));
    jest.doMock('../../src/middleware/ownership', () => ({
      requireEventOwnership: (_req, _res, next) => next(),
    }));

    // Runs once per photo, inside the import loop — the only hook that can
    // observe the intermediate state the ordering bug would expose.
    jest.doMock('../../src/services/imageProcessor', () => ({
      generateThumbnail: jest.fn(async () => {
        const { db: liveDb } = require('../../src/database/db');
        const rows = await liveDb('photos').where({ face_status: 'pending' });
        pendingSeenDuringLoop += rows.length;
        const ev = await liveDb('events').first();
        externalPathDuringLoop = ev ? ev.external_path : undefined;
        if (markDoneDuringLoop) {
          const rows = await liveDb('photos').orderBy('id', 'asc').limit(1);
          if (rows.length) {
            await liveDb('photos').where({ id: rows[0].id }).update({ face_status: 'done' });
          }
        }
        if (flipFacesOnDuringLoop) {
          await liveDb('events').where({ id: flipFacesOnDuringLoop })
            .update({ face_recognition_enabled: true });
        }
        return 'thumbnails/mock.jpg';
      }),
      ensureThumbnail: jest.fn(),
    }));

    jest.doMock('../../src/utils/logger', () => ({
      debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn(),
    }));

    // bootCrmDb runs every migrations/core/*.up() directly — knex's Migrator
    // deadlocks on 001_init's nested initializeDatabase() call.
    ({ db } = await require('./helpers/crmDb').bootCrmDb());

    app = express();
    app.use(express.json());
    app.use('/api/admin/external-media', require('../../src/routes/adminExternalMedia'));
  }, 180000);

  afterAll(async () => {
    if (db) await db.destroy?.();
    await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
  });

  async function seedEvent({ facesEnabled, flagOn }) {
    await db('feature_flags').insert({ key: 'faces', value: flagOn })
      .onConflict('key').merge()
      .catch(async () => { await db('feature_flags').where({ key: 'faces' }).update({ value: flagOn }); });
    // The flag read is TTL-cached (requireFeatureFlag.js:26-34); production
    // invalidates after every write, and so must this.
    require('../../src/middleware/requireFeatureFlag').invalidateFeatureFlagCache();

    await db('photos').del();
    await db('events').del();
    const [e] = await db('events').insert({
      slug: `extenq-${Math.random().toString(36).slice(2, 8)}`,
      event_type: 'wedding',
      event_name: 'extenq',
      event_date: '2026-01-01',
      host_email: 'h@example.com',
      admin_email: 'a@example.com',
      password_hash: 'x',
      share_link: `extenq-${Math.random()}`,
      expires_at: new Date().toISOString(),
      face_recognition_enabled: facesEnabled,
      source_mode: 'reference',
    }).returning('id');

    pendingSeenDuringLoop = 0;
    externalPathDuringLoop = undefined;
    markDoneDuringLoop = false;
    return typeof e === 'object' ? e.id : e;
  }

  async function runImport(eventId) {
    return request(app)
      .post(`/api/admin/external-media/events/${eventId}/import-external`)
      .send({ external_path: 'nas', recursive: true });
  }

  it('queues imported photos when detection is on', async () => {
    const eventId = await seedEvent({ facesEnabled: true, flagOn: true });

    const res = await runImport(eventId);
    expect(res.status).toBe(200);

    const photos = await db('photos').where({ event_id: eventId });
    expect(photos.length).toBeGreaterThan(0);
    // The regression: these stayed NULL and waited for a manual Re-scan.
    expect(photos.every((p) => p.face_status === 'pending')).toBe(true);
  });

  it('does not publish claimable rows before events.external_path is written', async () => {
    const eventId = await seedEvent({ facesEnabled: true, flagOn: true });

    await runImport(eventId);

    // Observed from inside the loop: nothing is claimable yet. The event path
    // is already committed (see the test above), so this is no longer load
    // bearing for correctness — but keeping the enqueue at the end is what lets
    // the feature setting be read after the loop, so the invariant stays.
    expect(pendingSeenDuringLoop).toBe(0);

    // ...and afterwards both are in place.
    const ev = await db('events').where({ id: eventId }).first();
    expect(ev.external_path).toBe('nas');
    expect((await db('photos').where({ event_id: eventId, face_status: 'pending' })).length)
      .toBe((await db('photos').where({ event_id: eventId })).length);
  });

  it('honours a toggle flipped DURING the import', async () => {
    // The setting is read after the loop, not before: on a large library the
    // loop runs for minutes, and the toggle endpoint only queues rows that
    // already existed when it fired. Reading it up front would strand every
    // photo imported after that moment at NULL forever.
    const eventId = await seedEvent({ facesEnabled: false, flagOn: true });
    flipFacesOnDuringLoop = eventId;

    await runImport(eventId);
    flipFacesOnDuringLoop = null;

    const photos = await db('photos').where({ event_id: eventId });
    expect(photos.length).toBeGreaterThan(0);
    expect(photos.every((p) => p.face_status === 'pending')).toBe(true);
  });

  it('commits events.external_path before the first row is inserted', async () => {
    // enqueueEvent accepts processing_status NULL (faceProcessor.js:243-246),
    // which these inserts leave unset — so a toggle or Re-scan firing mid-import
    // can queue partial rows. If the event still pointed at the old directory
    // they would resolve against it and burn to 'failed'. Setting the path
    // first also means a half-finished import leaves rows that still resolve,
    // instead of rows stranded against the previous path.
    const eventId = await seedEvent({ facesEnabled: true, flagOn: true });

    await runImport(eventId);

    // Sampled from inside the per-photo thumbnail call, i.e. while rows are
    // still being inserted.
    expect(externalPathDuringLoop).toBe('nas');
  });

  it('does not re-queue rows a concurrent scan already handled', async () => {
    // Committing the event path before the loop means a toggle or Re-scan
    // firing mid-import can now genuinely queue and even finish some of these
    // rows. A blanket update at the end would drag 'done' rows back to
    // 'pending' for a duplicate sidecar scan and knock 'processing' rows out
    // from under the worker.
    const eventId = await seedEvent({ facesEnabled: true, flagOn: true });
    markDoneDuringLoop = true;

    await runImport(eventId);
    markDoneDuringLoop = false;

    const done = await db('photos').where({ event_id: eventId, face_status: 'done' });
    expect(done.length).toBeGreaterThan(0); // the concurrent scan's work survived
  });

  it('leaves face_status untouched when the per-event toggle is off', async () => {
    const eventId = await seedEvent({ facesEnabled: false, flagOn: true });
    await runImport(eventId);
    const photos = await db('photos').where({ event_id: eventId });
    expect(photos.length).toBeGreaterThan(0);
    expect(photos.every((p) => p.face_status === null)).toBe(true);
  });

  it('leaves face_status untouched when the global flag is off', async () => {
    // Installs without the feature must never accumulate face_status rows —
    // the same invariant photoProcessor's guard protects.
    const eventId = await seedEvent({ facesEnabled: true, flagOn: false });
    await runImport(eventId);
    const photos = await db('photos').where({ event_id: eventId });
    expect(photos.length).toBeGreaterThan(0);
    expect(photos.every((p) => p.face_status === null)).toBe(true);
  });
});