fix(gallery): bound the cached-zip builder's reads and cap rebuild concurrency (stable) (#1421)

Backport of the two halves that landed on main, which together are what the
issue asked for: a per-build read cap plus a separate build-concurrency cap.

Per build: the builder opened one storage read per photo and handed each to
archiver, which drains them one at a time — so every read past the one being
written parked an S3 socket holding unread bytes. Nothing reclaimed them:
archiver's abort() does not touch source streams, and the SDK arms its socket
timeout on a 3s delay then clears it as soon as response headers land, so a
fast response never gets one. Reads are now capped at two and destroyed on
every exit, including the invalidated one, which previously cleaned up nothing
at all. Invalidation also cancels an in-flight build directly rather than
leaving a note for the loop, which matters once the loop can be parked waiting
for a slot a stalled archive will never free.

Across builds: invalidateAll() invalidates every event holding a cached zip and
each invalidate() arms its own debounce timer in the same tick, so they all
fired together and every one started building at once. Background rebuilds now
run two at a time. The cap is on that path only — a foreground generateZip,
where a guest is waiting on the download, is never queued behind a burst.

Two deliberate differences from the main twins. The read cap uses the shared
archiveStreamGuard helper this branch already has rather than main's inline
copy — same contract, less duplicated code. And main's stop() drain has no
counterpart here because this branch has no stop(), so that machinery is left
out rather than carried as dead code.

Relates to issue 1399

Co-authored-by: Paul Nothaft <[email protected]>

backend/__tests__/services/downloadZipRegenConcurrency.test.js

@@ -0,0 +1,127 @@
+/**
+ * Background zip rebuilds are capped (#1399).
+ *
+ * invalidateAll() invalidates every event holding a cached zip, and each
+ * invalidate() arms its own debounce timer in the same tick — so they all fire
+ * together. Every build opens its own storage reads, so a settings change
+ * across 25 events was enough to exhaust the S3 agent pool and stall uploads,
+ * thumbnails and gallery reads until the burst drained.
+ *
+ * The cap is on the BACKGROUND path only: a guest waiting on a download must
+ * not be queued behind a settings-change burst.
+ */
+jest.mock('../../src/database/db', () => ({ db: jest.fn() }));
+jest.mock('../../src/utils/logger', () => ({
+  info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn(),
+}));
+
+const { db } = require('../../src/database/db');
+const service = require('../../src/services/downloadZipService');
+
+const flush = () => new Promise((r) => setImmediate(r));
+
+describe('downloadZipService background regen concurrency (#1399)', () => {
+  let peak;
+  let inFlight;
+  let release;
+
+  beforeEach(() => {
+    // setImmediate must stay real: the flush() helper below rides on it, and
+    // jest's modern fake timers mock it too.
+    jest.useFakeTimers({ doNotFake: ['setImmediate'] });
+    peak = 0;
+    inFlight = 0;
+    release = [];
+    service.regenActive = 0;
+    service.regenWaiters = [];
+    service.debounceTimers.clear();
+    service.activeBuilds.clear();
+
+    jest.spyOn(service, 'generateZip').mockImplementation(() => {
+      inFlight += 1;
+      peak = Math.max(peak, inFlight);
+      return new Promise((resolve) => {
+        release.push(() => { inFlight -= 1; resolve(); });
+      });
+    });
+    jest.spyOn(service, '_cleanup').mockResolvedValue(undefined);
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+    jest.restoreAllMocks();
+  });
+
+  it('never runs more than two rebuilds at once, however many fire together', async () => {
+    const rows = Array.from({ length: 12 }, (_, i) => ({ id: i + 1 }));
+    db.mockReturnValue({
+      whereNotNull: () => ({ select: () => Promise.resolve(rows) }),
+    });
+
+    await service.invalidateAll();
+    // Every debounce timer was armed in the same tick — fire them all.
+    jest.runAllTimers();
+    await flush();
+
+    expect(peak).toBe(2);
+    expect(service.generateZip).toHaveBeenCalledTimes(2);
+  });
+
+  it('starts the next rebuild as each one finishes', async () => {
+    const rows = Array.from({ length: 5 }, (_, i) => ({ id: i + 1 }));
+    db.mockReturnValue({
+      whereNotNull: () => ({ select: () => Promise.resolve(rows) }),
+    });
+
+    await service.invalidateAll();
+    jest.runAllTimers();
+    await flush();
+    expect(service.generateZip).toHaveBeenCalledTimes(2);
+
+    release.shift()();
+    await flush();
+    expect(service.generateZip).toHaveBeenCalledTimes(3);
+    expect(peak).toBe(2);
+
+    while (release.length) { release.shift()(); await flush(); }
+    expect(service.generateZip).toHaveBeenCalledTimes(5);
+    expect(peak).toBe(2);
+  });
+
+  it('does not queue a foreground download behind the burst', async () => {
+    const rows = Array.from({ length: 6 }, (_, i) => ({ id: i + 1 }));
+    db.mockReturnValue({
+      whereNotNull: () => ({ select: () => Promise.resolve(rows) }),
+    });
+
+    await service.invalidateAll();
+    jest.runAllTimers();
+    await flush();
+    expect(service.generateZip).toHaveBeenCalledTimes(2);
+
+    // A guest asking for a zip right now calls generateZip directly. It must
+    // not park behind the two rebuilds already holding the slots.
+    service.generateZip(999);
+    await flush();
+    expect(service.generateZip).toHaveBeenCalledWith(999);
+    expect(inFlight).toBe(3);
+  });
+
+  it('leaves the queue empty once every rebuild has run', async () => {
+    const rows = Array.from({ length: 5 }, (_, i) => ({ id: i + 1 }));
+    db.mockReturnValue({
+      whereNotNull: () => ({ select: () => Promise.resolve(rows) }),
+    });
+
+    await service.invalidateAll();
+    jest.runAllTimers();
+    await flush();
+    expect(service.regenWaiters.length).toBeGreaterThan(0);
+
+    while (release.length) { release.shift()(); await flush(); }
+    // Nothing parked, nothing counted as running — no slot leaked on the way
+    // through, which is what would quietly wedge the next burst.
+    expect(service.regenWaiters).toHaveLength(0);
+    expect(service.regenActive).toBe(0);
+  });
+});