fix(archives): stream-extract restore for >2 GiB + preserve original_filename via manifest (#640)

Two related backup-integrity fixes from 8digit's fork (issue #640 items
#3 + #4), bundled because they touch the same two files and ship better
together than apart.

### Stream-extract restore for >2 GiB archives

`adminArchives.js:170` was using `adm-zip`, which loads the entire ZIP
into a Node Buffer before extracting. Node has a hard 2 GiB Buffer cap,
so any restore over that limit fails with `ERR_FS_FILE_TOO_LARGE` — and
since the frontend `onError` toast is the generic "Something went wrong",
the cause stays invisible. Real-world wedding archives routinely cross
2 GiB; affected restores have likely been silent failures.

Swapped `adm-zip` for `node-stream-zip` which streams each entry to disk
as it's processed — no full-file Buffer, no 2 GiB ceiling. API shape:

```js
const zip = new StreamZip.async({ file: archivePath });
const entries = Object.values(await zip.entries());
await zip.extract(null, eventDir);
await zip.close();
```

Re-import logic (photos, categories, sizes) unchanged; only field rename
`entry.entryName` → `entry.name`. Credit: 8digit/picpeak@69033c6.

### Preserve `original_filename` via photos manifest

Archive → restore round-trip currently loses `original_filename` (the
post-#508 column tracking the camera-side name) because the gallery
filenames are renamed on upload and can't be derived from the extracted
files. This matters now that the Lightroom export (#623) depends on
`original_filename` — a restored event lost that signal.

- **`archiveService.js`**: writes `photos_manifest.json` into the archive
  containing per-photo `{filename, original_filename, type, uploaded_at,
  category_name}`. Non-fatal: a manifest write failure falls through to
  legacy behaviour (filename used as original_filename, same as before).
- **`adminArchives.js`**: reads the manifest on restore, builds a
  `Map<filename → manifest>`, and assigns
  `original_filename = manifest?.original_filename || filename`.
  Archives produced before this lands have no manifest — restore logs a
  one-shot notice and falls back to filename, preserving backward compat.

Credit: 8digit/picpeak@eb018aa.

### Deps

- Removed `adm-zip ^0.5.16`
- Added `node-stream-zip ^1.15.0`

### What's NOT in this PR

8digit's commit also fixed the production compose healthcheck (`curl`
isn't in our Alpine image); that's already been addressed upstream in
the meantime. The frontend `onError` swallow on the restore toast is a
separate small follow-up.

### Test plan

- [x] `node -c` on both files clean
- [x] `node-stream-zip` async API verified at load time
- [ ] Manual: archive a multi-GB event → restore → confirm photos
      re-import with original_filename preserved
- [ ] Manual: restore an archive produced before this lands → confirm
      fallback to filename works (no manifest path crashes)
- [ ] Manual: confirm the new photos_manifest.json is inside the
      generated archive (`unzip -l <archive>.zip | grep manifest`)
This commit is contained in:
Paul Nothaft
2026-06-18 22:18:54 +02:00
parent 8212a647c7
commit e4e79a0b3a
4 changed files with 110 additions and 30 deletions
+51 -17
View File
@@ -7,7 +7,7 @@ const { slugify } = require('../utils/slug');
const { adminAuth } = require('../middleware/auth');
const { requirePermission } = require('../middleware/permissions');
const archiver = require('archiver');
const AdmZip = require('adm-zip');
const StreamZip = require('node-stream-zip');
const { requireEventOwnership } = require('../middleware/ownership');
const router = express.Router();
@@ -167,32 +167,62 @@ router.post('/:id/restore', adminAuth, requirePermission('archives.restore'), re
// Extract the archive
try {
const zip = new AdmZip(fullArchivePath);
// node-stream-zip streams each entry to disk on extract — adm-zip used
// to load the whole archive into a Node Buffer up front, which capped
// restore at 2 GiB (ERR_FS_FILE_TOO_LARGE). Real-world wedding archives
// routinely cross that line. Credit: 8digit/picpeak@69033c6.
const zip = new StreamZip.async({ file: fullArchivePath });
const eventsDir = path.join(storagePath, 'events/active');
const eventDir = path.join(eventsDir, archive.slug);
// Create event directory if it doesn't exist
await fs.mkdir(eventDir, { recursive: true });
// Log ZIP contents for debugging
console.log(`Extracting archive to: ${eventDir}`);
const entries = zip.getEntries();
const entries = Object.values(await zip.entries());
console.log(`Archive contains ${entries.length} entries`);
// Extract files to the event directory
zip.extractAllTo(eventDir, true);
// Stream-extract everything to disk
await zip.extract(null, eventDir);
await zip.close();
// Load photos manifest if present. The gallery filenames are renamed on
// upload, so `original_filename` (and category linkage) can't be derived
// from the extracted files alone — they're only recoverable from the
// manifest the archive process writes. Older archives have no manifest;
// we fall back to filename for those.
const manifestByFilename = new Map();
try {
const manifestRaw = await fs.readFile(
path.join(eventDir, 'photos_manifest.json'), 'utf8',
);
const parsed = JSON.parse(manifestRaw);
if (Array.isArray(parsed)) {
for (const m of parsed) {
if (m && m.filename) manifestByFilename.set(m.filename, m);
}
}
console.log(`Loaded photos manifest: ${manifestByFilename.size} entries`);
} catch (e) {
if (e.code !== 'ENOENT') {
console.warn('Photos manifest present but unreadable; falling back to filenames', e.message);
} else {
console.log('No photos manifest in archive (older archive); original_filename falls back to filename');
}
}
// Get list of extracted files to update database
const extractedPhotos = [];
// First, collect all category information from the ZIP structure
const categoriesMap = new Map();
for (const entry of entries) {
if (!entry.isDirectory && entry.entryName.match(/\.(jpg|jpeg|png|gif|webp)$/i)) {
const filename = path.basename(entry.entryName);
const dirPath = path.dirname(entry.entryName);
const actualFilePath = path.join(eventDir, entry.entryName);
if (!entry.isDirectory && entry.name.match(/\.(jpg|jpeg|png|gif|webp)$/i)) {
const filename = path.basename(entry.name);
const dirPath = path.dirname(entry.name);
const actualFilePath = path.join(eventDir, entry.name);
try {
// Check if file was extracted successfully
@@ -239,10 +269,14 @@ router.post('/:id/restore', adminAuth, requirePermission('archives.restore'), re
if (!existingPhoto) {
// Store relative path from storage root
const relativePath = path.relative(storagePath, actualFilePath);
const manifestEntry = manifestByFilename.get(filename);
extractedPhotos.push({
event_id: archive.id,
filename: filename,
original_filename: filename,
// Recover original_filename from the manifest if present;
// legacy archives without a manifest lose nothing (filename
// is what they had before).
original_filename: manifestEntry?.original_filename || filename,
path: relativePath,
thumbnail_path: null, // Will be regenerated by thumbnail service
type: path.extname(filename).substring(1).toLowerCase(),
@@ -253,7 +287,7 @@ router.post('/:id/restore', adminAuth, requirePermission('archives.restore'), re
}
} catch (statError) {
console.error(`Failed to stat file: ${actualFilePath}`);
console.error(`Entry name was: ${entry.entryName}`);
console.error(`Entry name was: ${entry.name}`);
console.error('Error:', statError.message);
// Skip this file if we can't stat it
continue;
+33
View File
@@ -26,6 +26,36 @@ async function archiveEvent(event) {
const tmpArchive = path.join(tmpDir, `${crypto.randomBytes(4).toString('hex')}-${archiveName}`);
try {
// Photos manifest — the gallery filenames are renamed on upload, so
// `original_filename` (and category linkage) can't be derived from the
// extracted files alone. Persisting a manifest inside the archive lets a
// future restore round-trip recover those fields. Falls back to bare
// filename for archives produced before this lands (see restore path).
let photosManifestEntry = null;
try {
const manifestRows = await db('photos')
.leftJoin('photo_categories', 'photos.category_id', 'photo_categories.id')
.where('photos.event_id', event.id)
.select(
'photos.filename',
'photos.original_filename',
'photos.type',
'photos.uploaded_at',
'photo_categories.name as category_name',
);
if (manifestRows.length > 0) {
photosManifestEntry = {
name: 'photos_manifest.json',
buffer: Buffer.from(JSON.stringify(manifestRows, null, 2), 'utf8'),
};
logger.info(`Photos manifest prepared: ${manifestRows.length} entries`);
}
} catch (error) {
logger.error(`Error building photos manifest for event ${event.slug}:`, error);
// Non-fatal — restore will fall back to filename as original_filename
// for events archived without a manifest, same as the legacy behaviour.
}
// Collect feedback data first so it can be included as in-memory entries.
const feedbackEntries = [];
const feedbackSettings = await feedbackService.getEventFeedbackSettings(event.id);
@@ -116,6 +146,9 @@ async function archiveEvent(event) {
for (const f of feedbackEntries) {
archive.append(f.buffer, { name: f.name });
}
if (photosManifestEntry) {
archive.append(photosManifestEntry.buffer, { name: photosManifestEntry.name });
}
archive.finalize();
};