fix(archives): stream-extract restore for >2 GiB + preserve original_filename via manifest (#640)
Two related backup-integrity fixes from 8digit's fork (issue #640 items #3 + #4), bundled because they touch the same two files and ship better together than apart. ### Stream-extract restore for >2 GiB archives `adminArchives.js:170` was using `adm-zip`, which loads the entire ZIP into a Node Buffer before extracting. Node has a hard 2 GiB Buffer cap, so any restore over that limit fails with `ERR_FS_FILE_TOO_LARGE` — and since the frontend `onError` toast is the generic "Something went wrong", the cause stays invisible. Real-world wedding archives routinely cross 2 GiB; affected restores have likely been silent failures. Swapped `adm-zip` for `node-stream-zip` which streams each entry to disk as it's processed — no full-file Buffer, no 2 GiB ceiling. API shape: ```js const zip = new StreamZip.async({ file: archivePath }); const entries = Object.values(await zip.entries()); await zip.extract(null, eventDir); await zip.close(); ``` Re-import logic (photos, categories, sizes) unchanged; only field rename `entry.entryName` → `entry.name`. Credit: 8digit/picpeak@69033c6. ### Preserve `original_filename` via photos manifest Archive → restore round-trip currently loses `original_filename` (the post-#508 column tracking the camera-side name) because the gallery filenames are renamed on upload and can't be derived from the extracted files. This matters now that the Lightroom export (#623) depends on `original_filename` — a restored event lost that signal. - **`archiveService.js`**: writes `photos_manifest.json` into the archive containing per-photo `{filename, original_filename, type, uploaded_at, category_name}`. Non-fatal: a manifest write failure falls through to legacy behaviour (filename used as original_filename, same as before). - **`adminArchives.js`**: reads the manifest on restore, builds a `Map<filename → manifest>`, and assigns `original_filename = manifest?.original_filename || filename`. Archives produced before this lands have no manifest — restore logs a one-shot notice and falls back to filename, preserving backward compat. Credit: 8digit/picpeak@eb018aa. ### Deps - Removed `adm-zip ^0.5.16` - Added `node-stream-zip ^1.15.0` ### What's NOT in this PR 8digit's commit also fixed the production compose healthcheck (`curl` isn't in our Alpine image); that's already been addressed upstream in the meantime. The frontend `onError` swallow on the restore toast is a separate small follow-up. ### Test plan - [x] `node -c` on both files clean - [x] `node-stream-zip` async API verified at load time - [ ] Manual: archive a multi-GB event → restore → confirm photos re-import with original_filename preserved - [ ] Manual: restore an archive produced before this lands → confirm fallback to filename works (no manifest path crashes) - [ ] Manual: confirm the new photos_manifest.json is inside the generated archive (`unzip -l <archive>.zip | grep manifest`)
This commit is contained in:
Paul Nothaft
Generated
+25
-12
@@ -1,17 +1,16 @@
|
||||
{
|
||||
"name": "picpeak-backend",
|
||||
"version": "3.60.6-beta.0",
|
||||
"version": "3.65.1-beta.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "picpeak-backend",
|
||||
"version": "3.60.6-beta.0",
|
||||
"version": "3.65.1-beta.0",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.850.0",
|
||||
"@aws-sdk/lib-storage": "^3.850.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.850.0",
|
||||
"adm-zip": "^0.5.16",
|
||||
"archiver": "^5.3.1",
|
||||
"axios": "1.15.2",
|
||||
"bcrypt": "6.0.0",
|
||||
@@ -39,6 +38,7 @@
|
||||
"mime-types": "^3.0.1",
|
||||
"multer": "^2.0.2",
|
||||
"node-cron": "^3.0.2",
|
||||
"node-stream-zip": "^1.15.0",
|
||||
"nodemailer": "^8.0.5",
|
||||
"pdf-lib": "^1.17.1",
|
||||
"pdfkit": "^0.17.2",
|
||||
@@ -314,6 +314,7 @@
|
||||
"resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1000.0.tgz",
|
||||
"integrity": "sha512-7kPy33qNGq3NfwHC0412T6LDK1bp4+eiPzetX0sVd9cpTSXuQDKpoOFnB0Njj6uZjJDcLS3n2OeyarwwgkQ0Ow==",
|
||||
"license": "Apache-2.0",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@aws-crypto/sha1-browser": "5.2.0",
|
||||
"@aws-crypto/sha256-browser": "5.2.0",
|
||||
@@ -1042,6 +1043,7 @@
|
||||
"integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/code-frame": "^7.27.1",
|
||||
"@babel/generator": "^7.28.5",
|
||||
@@ -3804,6 +3806,7 @@
|
||||
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"acorn": "bin/acorn"
|
||||
},
|
||||
@@ -3821,15 +3824,6 @@
|
||||
"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/adm-zip": {
|
||||
"version": "0.5.16",
|
||||
"resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz",
|
||||
"integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.0"
|
||||
}
|
||||
},
|
||||
"node_modules/agent-base": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
|
||||
@@ -4373,6 +4367,7 @@
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"baseline-browser-mapping": "^2.9.0",
|
||||
"caniuse-lite": "^1.0.30001759",
|
||||
@@ -5518,6 +5513,7 @@
|
||||
"deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@eslint-community/eslint-utils": "^4.2.0",
|
||||
"@eslint-community/regexpp": "^4.6.1",
|
||||
@@ -5765,6 +5761,7 @@
|
||||
"resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz",
|
||||
"integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"accepts": "~1.3.8",
|
||||
"array-flatten": "1.1.1",
|
||||
@@ -6791,6 +6788,7 @@
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"@babel/runtime": "^7.27.6"
|
||||
},
|
||||
@@ -9058,6 +9056,19 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/node-stream-zip": {
|
||||
"version": "1.15.0",
|
||||
"resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.15.0.tgz",
|
||||
"integrity": "sha512-LN4fydt9TqhZhThkZIVQnF9cwjU3qmUH9h78Mx/K7d3VvfRqqwthLwJEUOEL0QPZ0XQmNN7be5Ggit5+4dq3Bw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=0.12.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/antelle"
|
||||
}
|
||||
},
|
||||
"node_modules/nodemailer": {
|
||||
"version": "8.0.10",
|
||||
"resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-8.0.10.tgz",
|
||||
@@ -9555,6 +9566,7 @@
|
||||
"resolved": "https://registry.npmjs.org/pdfkit/-/pdfkit-0.17.2.tgz",
|
||||
"integrity": "sha512-UnwF5fXy08f0dnp4jchFYAROKMNTaPqb/xgR8GtCzIcqoTnbOqtp3bwKvO4688oHI6vzEEs8Q6vqqEnC5IUELw==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"crypto-js": "^4.2.0",
|
||||
"fontkit": "^2.0.4",
|
||||
@@ -10621,6 +10633,7 @@
|
||||
"resolved": "https://registry.npmjs.org/selderee/-/selderee-0.12.0.tgz",
|
||||
"integrity": "sha512-b1YMh3+DHZp59DLna3qVwQ5iOla/nrI6mLBNW02XxU77M3046Df6VLkoaJyFz20VsGIG5kkp+FK0kg4K4HnUFw==",
|
||||
"license": "MIT",
|
||||
"peer": true,
|
||||
"dependencies": {
|
||||
"parseley": "~0.13.1"
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user