Merge pull request #731 from Luca-Timo/feat/setup-wizard-and-backup-roundtrip
feat: first-run setup wizard (feature selection + config) and portable .picpeak backup roundtrip
This commit is contained in:
Paul Nothaft
@@ -0,0 +1,94 @@
|
||||
'use strict';
|
||||
|
||||
// Validates the engine-neutral .picpeak export: it must produce a real zip with
|
||||
// a manifest + per-table NDJSON, exclude knex bookkeeping, and honour the photo
|
||||
// toggle. Uses the shared CRM DB harness (temp SQLite) — no docker needed.
|
||||
process.env.JWT_SECRET = process.env.JWT_SECRET || 'test-secret-at-least-32-characters-long!!';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const StreamZip = require('node-stream-zip');
|
||||
const { bootCrmDb } = require('./helpers/crmDb');
|
||||
|
||||
let db;
|
||||
let cleanup;
|
||||
let tmpDir;
|
||||
let createPicpeak;
|
||||
|
||||
// bootCrmDb MUST run before requiring the service (which transitively requires
|
||||
// db.js) so the export reads this test's DB, not the default path.
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup, tmpDir } = await bootCrmDb());
|
||||
process.env.STORAGE_PATH = tmpDir; // isolate file collection to the temp dir
|
||||
({ createPicpeak } = require('../../src/services/picpeakExportService'));
|
||||
}, 60000);
|
||||
|
||||
afterAll(async () => {
|
||||
await cleanup();
|
||||
});
|
||||
|
||||
async function readZip(filePath) {
|
||||
const zip = new StreamZip.async({ file: filePath });
|
||||
const entries = Object.keys(await zip.entries());
|
||||
const manifest = JSON.parse((await zip.entryData('manifest.json')).toString('utf8'));
|
||||
await zip.close();
|
||||
return { entries, manifest };
|
||||
}
|
||||
|
||||
describe('picpeak export (.picpeak logical export)', () => {
|
||||
it('produces a .picpeak with a manifest and per-table NDJSON', async () => {
|
||||
const { filePath, manifest } = await createPicpeak({ includePhotos: false });
|
||||
try {
|
||||
expect(filePath.endsWith('.picpeak')).toBe(true);
|
||||
expect(fs.existsSync(filePath)).toBe(true);
|
||||
|
||||
expect(manifest.format).toBe(1);
|
||||
expect(manifest.kind).toBe('picpeak-backup');
|
||||
expect(manifest.database.engine).toBe('sqlite');
|
||||
expect(manifest.options.includePhotos).toBe(false);
|
||||
expect(manifest.contains_secrets).toBe(true);
|
||||
// Migrations seed real tables (e.g. app_settings) — expect several.
|
||||
expect(Object.keys(manifest.tables).length).toBeGreaterThan(0);
|
||||
expect(Object.keys(manifest.tables)).toContain('app_settings');
|
||||
|
||||
const { entries, manifest: zipped } = await readZip(filePath);
|
||||
expect(entries).toContain('manifest.json');
|
||||
expect(entries.some((n) => n.startsWith('data/') && n.endsWith('.ndjson'))).toBe(true);
|
||||
expect(entries).toContain('data/app_settings.ndjson');
|
||||
// Manifest inside the zip matches the returned one.
|
||||
expect(zipped.tables).toEqual(manifest.tables);
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('never exports knex bookkeeping tables', async () => {
|
||||
const { filePath, manifest } = await createPicpeak({ includePhotos: false });
|
||||
try {
|
||||
const names = Object.keys(manifest.tables);
|
||||
expect(names).not.toContain('knex_migrations');
|
||||
expect(names).not.toContain('knex_migrations_lock');
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('row counts in the manifest match the NDJSON line counts', async () => {
|
||||
// Insert a couple of settings so at least one table is non-empty.
|
||||
await db('app_settings')
|
||||
.insert({ setting_key: 'picpeak_export_test_a', setting_value: JSON.stringify('1'), setting_type: 'string' })
|
||||
.onConflict('setting_key').merge();
|
||||
|
||||
const { filePath, manifest } = await createPicpeak({ includePhotos: false });
|
||||
try {
|
||||
const zip = new StreamZip.async({ file: filePath });
|
||||
const buf = await zip.entryData('data/app_settings.ndjson');
|
||||
await zip.close();
|
||||
const lines = buf.toString('utf8').split('\n').filter((l) => l.trim().length > 0);
|
||||
expect(lines.length).toBe(manifest.tables.app_settings.rowCount);
|
||||
expect(manifest.tables.app_settings.rowCount).toBeGreaterThan(0);
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
'use strict';
|
||||
|
||||
// Full .picpeak roundtrip on a temp SQLite DB:
|
||||
// 1. seed a "backup" instance (admin A + a marker setting)
|
||||
// 2. export → .picpeak
|
||||
// 3. simulate a reinstall: wipe, create a DIFFERENT current admin B, mutate data
|
||||
// 4. import the backup with currentAdminId = B
|
||||
// 5. assert the backup data is restored AND the current account (B) survives,
|
||||
// while the backup's admin (A) is also present (different email → added).
|
||||
process.env.JWT_SECRET = process.env.JWT_SECRET || 'test-secret-at-least-32-characters-long!!';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { bootCrmDb } = require('./helpers/crmDb');
|
||||
|
||||
let db;
|
||||
let cleanup;
|
||||
let tmpDir;
|
||||
let createPicpeak;
|
||||
let importFromPicpeak;
|
||||
let validateManifest;
|
||||
let superAdminRoleId;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup, tmpDir } = await bootCrmDb());
|
||||
process.env.STORAGE_PATH = tmpDir;
|
||||
({ createPicpeak } = require('../../src/services/picpeakExportService'));
|
||||
({ importFromPicpeak, validateManifest } = require('../../src/services/picpeakImportService'));
|
||||
const role = await db('roles').where({ name: 'super_admin' }).first();
|
||||
superAdminRoleId = role.id;
|
||||
}, 60000);
|
||||
|
||||
afterAll(async () => {
|
||||
await cleanup();
|
||||
});
|
||||
|
||||
const adminRow = (email, hash) => ({
|
||||
username: email,
|
||||
email,
|
||||
password_hash: hash,
|
||||
role_id: superAdminRoleId,
|
||||
is_active: true,
|
||||
must_change_password: false,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
});
|
||||
|
||||
async function setMarker(value) {
|
||||
await db('app_settings')
|
||||
.insert({ setting_key: 'roundtrip_marker', setting_value: JSON.stringify(value), setting_type: 'string' })
|
||||
.onConflict('setting_key').merge();
|
||||
}
|
||||
async function getMarker() {
|
||||
const row = await db('app_settings').where({ setting_key: 'roundtrip_marker' }).first();
|
||||
return row ? JSON.parse(row.setting_value) : null;
|
||||
}
|
||||
|
||||
describe('.picpeak roundtrip (export → import)', () => {
|
||||
it('restores backup data and preserves the current account', async () => {
|
||||
// 1. Seed the "source" instance.
|
||||
await db('admin_users').del();
|
||||
await db('admin_users').insert(adminRow('[email protected]', 'HASH_A'));
|
||||
await setMarker('from_backup');
|
||||
|
||||
// 2. Export.
|
||||
const { filePath } = await createPicpeak({ includePhotos: false });
|
||||
|
||||
try {
|
||||
// 3. Simulate a reinstall: fresh current admin B, mutated data.
|
||||
await db('admin_users').del();
|
||||
const [bId] = await db('admin_users').insert(adminRow('[email protected]', 'HASH_B')).returning('id');
|
||||
const currentAdminId = typeof bId === 'object' ? bId.id : bId;
|
||||
await setMarker('mutated_after_backup');
|
||||
|
||||
// 4. Import, preserving the current admin.
|
||||
const result = await importFromPicpeak({ filePath: undefined, picpeakPath: filePath, currentAdminId });
|
||||
expect(result.restored).toBe(true);
|
||||
expect(result.tables).toBeGreaterThan(0);
|
||||
|
||||
// 5a. Backup data restored (marker reverted to the backup value).
|
||||
expect(await getMarker()).toBe('from_backup');
|
||||
|
||||
// 5b. The backup's admin is present (different email → added).
|
||||
const a = await db('admin_users').whereRaw('lower(email) = lower(?)', ['[email protected]']).first();
|
||||
expect(a).toBeTruthy();
|
||||
expect(a.password_hash).toBe('HASH_A');
|
||||
|
||||
// 5c. The current account SURVIVES the override, with its own credentials.
|
||||
const b = await db('admin_users').whereRaw('lower(email) = lower(?)', ['[email protected]']).first();
|
||||
expect(b).toBeTruthy();
|
||||
expect(b.password_hash).toBe('HASH_B');
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('overwrites a backup admin that collides with the current account email', async () => {
|
||||
// Source has an admin at the SAME email the current operator will use.
|
||||
await db('admin_users').del();
|
||||
await db('admin_users').insert(adminRow('[email protected]', 'OLD_HASH'));
|
||||
await setMarker('collision_case');
|
||||
const { filePath } = await createPicpeak({ includePhotos: false });
|
||||
|
||||
try {
|
||||
// Reinstall: current admin uses the same email but a NEW password.
|
||||
await db('admin_users').del();
|
||||
const [id] = await db('admin_users').insert(adminRow('[email protected]', 'NEW_HASH')).returning('id');
|
||||
const currentAdminId = typeof id === 'object' ? id.id : id;
|
||||
|
||||
await importFromPicpeak({ picpeakPath: filePath, currentAdminId });
|
||||
|
||||
// Exactly one admin at that email, and it keeps the CURRENT password.
|
||||
const rows = await db('admin_users').whereRaw('lower(email) = lower(?)', ['[email protected]']);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].password_hash).toBe('NEW_HASH');
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('restores files/ and reports filesRestored', async () => {
|
||||
// A business-doc that lives in storage → travels in the backup.
|
||||
const docDir = path.join(tmpDir, 'business-docs');
|
||||
const marker = path.join(docDir, 'roundtrip-doc.txt');
|
||||
fs.mkdirSync(docDir, { recursive: true });
|
||||
fs.writeFileSync(marker, 'hello');
|
||||
await db('admin_users').del();
|
||||
const [id] = await db('admin_users').insert(adminRow('[email protected]', 'H')).returning('id');
|
||||
const currentAdminId = typeof id === 'object' ? id.id : id;
|
||||
|
||||
const { filePath } = await createPicpeak({ includePhotos: false });
|
||||
try {
|
||||
fs.rmSync(marker); // delete on disk so the restore must bring it back
|
||||
const result = await importFromPicpeak({ picpeakPath: filePath, currentAdminId });
|
||||
expect(result.filesRestored).toBeGreaterThanOrEqual(1);
|
||||
expect(fs.existsSync(marker)).toBe(true);
|
||||
expect(fs.readFileSync(marker, 'utf8')).toBe('hello');
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
|
||||
fs.rmSync(docDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('.picpeak manifest validation', () => {
|
||||
it('rejects a database-engine mismatch', async () => {
|
||||
// Harness runs on SQLite, so a pg manifest must be refused.
|
||||
const blockers = await validateManifest({
|
||||
kind: 'picpeak-backup', format: 1, database: { engine: 'pg' }, tables: {},
|
||||
});
|
||||
expect(blockers.some((b) => /engine/i.test(b))).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects a backup from a newer schema (forward-only)', async () => {
|
||||
// validateManifest reads knex_migrations for the target's latest migration;
|
||||
// the harness has none, so create it with an older migration than the backup.
|
||||
await db.schema.createTable('knex_migrations', (t) => {
|
||||
t.increments('id');
|
||||
t.string('name');
|
||||
t.integer('batch');
|
||||
t.timestamp('migration_time');
|
||||
});
|
||||
try {
|
||||
await db('knex_migrations').insert({ name: '100_baseline', batch: 1 });
|
||||
const blockers = await validateManifest({
|
||||
kind: 'picpeak-backup', format: 1,
|
||||
database: { engine: 'sqlite', latest_migration: '999_from_the_future' },
|
||||
tables: {},
|
||||
});
|
||||
expect(blockers.some((b) => /newer/i.test(b))).toBe(true);
|
||||
} finally {
|
||||
await db.schema.dropTableIfExists('knex_migrations');
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a file that is not a PicPeak backup', async () => {
|
||||
const blockers = await validateManifest({ some: 'random-json' });
|
||||
expect(blockers.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -131,6 +131,71 @@ router.post('/run', adminAuth, requirePermission('backup.create'), async (req, r
|
||||
}
|
||||
});
|
||||
|
||||
// Generate + download a portable ".picpeak" export — an engine-neutral logical
|
||||
// snapshot (DB rows as NDJSON + PDFs/business-docs) that can be re-uploaded to
|
||||
// another instance via the web UI. `?includePhotos=true` also bundles original
|
||||
// gallery photos (larger); otherwise the admin re-uploads them per gallery.
|
||||
//
|
||||
// SECURITY: the file contains plaintext secrets (SMTP password, admin password
|
||||
// hashes, API keys). The download UI must warn before offering it. We surface
|
||||
// the flag as a response header too so the client can double-confirm.
|
||||
router.get('/picpeak/export', adminAuth, requirePermission('backup.create'), async (req, res) => {
|
||||
const fsSync = require('fs');
|
||||
try {
|
||||
const includePhotos = req.query.includePhotos === 'true' || req.query.includePhotos === '1';
|
||||
const { createPicpeak } = require('../services/picpeakExportService');
|
||||
const { filePath } = await createPicpeak({ includePhotos });
|
||||
const filename = path.basename(filePath);
|
||||
res.setHeader('X-Picpeak-Contains-Secrets', 'true');
|
||||
res.download(filePath, filename, (err) => {
|
||||
// Best-effort cleanup of the temp .picpeak (and its temp dir) after send.
|
||||
fsSync.rm(path.dirname(filePath), { recursive: true, force: true }, () => {});
|
||||
if (err) logger.error('[picpeak-export] download failed', { error: err.message });
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('[picpeak-export] failed to create export', { error: error.message });
|
||||
if (!res.headersSent) res.status(500).json({ error: 'Failed to create .picpeak export' });
|
||||
}
|
||||
});
|
||||
|
||||
// Multipart upload for .picpeak restore — streamed to a temp file. Runs AFTER
|
||||
// auth so an unauthenticated request can't push a large file to disk.
|
||||
const os = require('os');
|
||||
const multer = require('multer');
|
||||
const picpeakUpload = multer({
|
||||
storage: multer.diskStorage({
|
||||
destination: (req, file, cb) => cb(null, os.tmpdir()),
|
||||
filename: (req, file, cb) => cb(null, `picpeak-upload-${Date.now()}-${crypto.randomBytes(6).toString('hex')}.picpeak`),
|
||||
}),
|
||||
limits: { fileSize: 5 * 1024 * 1024 * 1024 }, // 5 GB — .picpeak with photos can be large
|
||||
});
|
||||
|
||||
// Upload + restore a .picpeak onto THIS instance. DESTRUCTIVE: full override of
|
||||
// all data except the current logged-in account (the client shows an explicit
|
||||
// confirmation before calling this). Returns `usesExternalMedia` so the UI can
|
||||
// prompt the admin to reconfigure the external-media mount afterwards.
|
||||
router.post('/picpeak/import', adminAuth, requirePermission('backup.restore'), picpeakUpload.single('backup'), async (req, res) => {
|
||||
const fsSync = require('fs');
|
||||
if (!req.file) return res.status(400).json({ error: 'No backup file uploaded' });
|
||||
const picpeakPath = req.file.path;
|
||||
try {
|
||||
const { importFromPicpeak } = require('../services/picpeakImportService');
|
||||
const result = await importFromPicpeak({ picpeakPath, currentAdminId: req.user && req.user.id });
|
||||
res.json({
|
||||
success: true,
|
||||
tables: result.tables,
|
||||
filesRestored: result.filesRestored,
|
||||
usesExternalMedia: result.usesExternalMedia,
|
||||
});
|
||||
} catch (error) {
|
||||
const status = error.statusCode || 500;
|
||||
logger.error('[picpeak-import] restore failed', { error: error.message });
|
||||
res.status(status).json({ error: error.message || 'Restore failed', validation: error.validation });
|
||||
} finally {
|
||||
fsSync.unlink(picpeakPath, () => {});
|
||||
}
|
||||
});
|
||||
|
||||
// Get backup run details
|
||||
router.get('/runs/:id', adminAuth, requirePermission('backup.view'), async (req, res) => {
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
'use strict';
|
||||
|
||||
// Portable ".picpeak" export — a single, self-describing archive that can be
|
||||
// downloaded from one instance and re-uploaded to another via the web UI only
|
||||
// (see picpeakImportService for the receiving half).
|
||||
//
|
||||
// Deliberately ENGINE-NEUTRAL: instead of a native pg_dump / sqlite .backup
|
||||
// (which can only ever restore into the same engine and version), each table is
|
||||
// written as NDJSON. The target rebuilds its own schema by running migrations,
|
||||
// then loads these rows into it — so an older backup restores cleanly onto a
|
||||
// newer target (forward-only), and pg↔pg / sqlite↔sqlite both work.
|
||||
//
|
||||
// This module is purely additive: it introduces a new artifact and touches no
|
||||
// existing backup/restore path.
|
||||
|
||||
const fs = require('fs');
|
||||
const fsp = require('fs').promises;
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const crypto = require('crypto');
|
||||
const archiver = require('archiver');
|
||||
const { db } = require('../database/db');
|
||||
const knexConfig = require('../../knexfile');
|
||||
const { getStoragePath } = require('../config/storage');
|
||||
const logger = require('../utils/logger');
|
||||
const packageJson = require('../../package.json');
|
||||
|
||||
// Bump only on a breaking change to the on-disk layout below.
|
||||
const PICPEAK_FORMAT_VERSION = 1;
|
||||
|
||||
// Never exported as data — the target owns these (its own migrations set them).
|
||||
const EXCLUDED_TABLES = new Set(['knex_migrations', 'knex_migrations_lock']);
|
||||
|
||||
// Storage subdirs holding non-recalculable blobs — always included.
|
||||
const DOC_DIRS = ['business-docs', 'uploads'];
|
||||
// Original gallery photos — only when includePhotos is true (large; otherwise
|
||||
// the admin re-uploads originals per gallery and previews are re-rendered).
|
||||
const PHOTO_DIRS = ['events/active', 'events/archived'];
|
||||
|
||||
const isPostgres = () => knexConfig.client === 'pg';
|
||||
|
||||
// db.raw returns `{ rows: [...] }` on Postgres and a bare array on SQLite.
|
||||
const rawRows = (result) => (isPostgres() ? result.rows : result);
|
||||
|
||||
// All user tables, minus knex bookkeeping. Introspected at runtime so the
|
||||
// export never rots as tables are added (no hardcoded list to maintain).
|
||||
async function listDataTables() {
|
||||
let names;
|
||||
if (isPostgres()) {
|
||||
const result = await db.raw(`
|
||||
SELECT table_name AS name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
|
||||
ORDER BY table_name
|
||||
`);
|
||||
names = rawRows(result).map((r) => r.name);
|
||||
} else {
|
||||
const result = await db.raw(`
|
||||
SELECT name FROM sqlite_master
|
||||
WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
|
||||
ORDER BY name
|
||||
`);
|
||||
names = rawRows(result).map((r) => r.name);
|
||||
}
|
||||
return names.filter((n) => !EXCLUDED_TABLES.has(n));
|
||||
}
|
||||
|
||||
// The latest applied migration — recorded in the manifest so the importer can
|
||||
// refuse a backup that is NEWER than the target (forward-only guarantee).
|
||||
async function getLatestMigration() {
|
||||
try {
|
||||
const rows = await db('knex_migrations').orderBy('id', 'desc').limit(1);
|
||||
return rows[0]?.name || null;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Write one table to <dataDir>/<table>.ndjson (one JSON object per line).
|
||||
// Returns { rowCount, checksum } for the manifest. JSON.stringify serialises
|
||||
// Dates to ISO strings, which re-import cleanly on both engines.
|
||||
//
|
||||
// Uses a plain select rather than knex `.stream()`: streaming on Postgres pulls
|
||||
// in the optional `pg-query-stream` dependency (not bundled), so it throws on
|
||||
// pg. A select works on both engines with no extra dependency. Rows are DB
|
||||
// metadata (blobs live on disk under files/), so holding a table in memory is
|
||||
// fine for the instance sizes PicPeak targets.
|
||||
async function writeTableNdjson(table, dataDir) {
|
||||
const outPath = path.join(dataDir, `${table}.ndjson`);
|
||||
const hash = crypto.createHash('sha256');
|
||||
const rows = await db(table).select('*');
|
||||
const lines = rows.map((row) => {
|
||||
const line = JSON.stringify(row);
|
||||
hash.update(`${line}\n`);
|
||||
return line;
|
||||
});
|
||||
await fsp.writeFile(outPath, lines.length ? `${lines.join('\n')}\n` : '', 'utf8');
|
||||
return { rowCount: rows.length, checksum: hash.digest('hex') };
|
||||
}
|
||||
|
||||
// Recursively collect files under a storage subdir as { abs, rel } where rel is
|
||||
// relative to the storage root (so the importer restores the same layout).
|
||||
async function collectDir(subdir, storageRoot, acc) {
|
||||
const abs = path.join(storageRoot, subdir);
|
||||
let entries;
|
||||
try {
|
||||
entries = await fsp.readdir(abs, { withFileTypes: true });
|
||||
} catch (_) {
|
||||
return; // subdir may not exist on this install — skip silently
|
||||
}
|
||||
for (const entry of entries) {
|
||||
const childRel = path.join(subdir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await collectDir(childRel, storageRoot, acc);
|
||||
} else if (entry.isFile()) {
|
||||
acc.push({ abs: path.join(storageRoot, childRel), rel: childRel });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function collectFiles(includePhotos) {
|
||||
const storageRoot = getStoragePath();
|
||||
const dirs = includePhotos ? [...DOC_DIRS, ...PHOTO_DIRS] : [...DOC_DIRS];
|
||||
const acc = [];
|
||||
for (const d of dirs) {
|
||||
await collectDir(d, storageRoot, acc);
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a .picpeak archive.
|
||||
* @param {Object} opts
|
||||
* @param {boolean} [opts.includePhotos=false] include original gallery photos
|
||||
* @param {string} [opts.outDir] where to write the file (defaults to a temp dir)
|
||||
* @returns {Promise<{ filePath: string, manifest: object }>}
|
||||
*/
|
||||
async function createPicpeak({ includePhotos = false, outDir } = {}) {
|
||||
const staging = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-export-'));
|
||||
const dataDir = path.join(staging, 'data');
|
||||
await fsp.mkdir(dataDir, { recursive: true });
|
||||
|
||||
try {
|
||||
// 1. Dump every table to NDJSON, tracking counts + checksums.
|
||||
const tables = await listDataTables();
|
||||
const tableMeta = {};
|
||||
for (const table of tables) {
|
||||
tableMeta[table] = await writeTableNdjson(table, dataDir);
|
||||
}
|
||||
|
||||
// 2. Gather the non-recalculable blobs (PDFs, business-docs, uploads, and
|
||||
// optionally original photos).
|
||||
const files = await collectFiles(includePhotos);
|
||||
|
||||
// 3. Manifest — everything the importer needs to validate + reconstruct.
|
||||
const manifest = {
|
||||
format: PICPEAK_FORMAT_VERSION,
|
||||
kind: 'picpeak-backup',
|
||||
created_at: new Date().toISOString(),
|
||||
app_version: packageJson.version || null,
|
||||
database: {
|
||||
engine: isPostgres() ? 'pg' : 'sqlite',
|
||||
latest_migration: await getLatestMigration(),
|
||||
},
|
||||
options: { includePhotos: !!includePhotos },
|
||||
tables: tableMeta,
|
||||
file_count: files.length,
|
||||
// NOTE: contains secrets (SMTP password, admin hashes, API keys) in plain
|
||||
// text — the download surface must warn about this.
|
||||
contains_secrets: true,
|
||||
};
|
||||
await fsp.writeFile(
|
||||
path.join(staging, 'manifest.json'),
|
||||
JSON.stringify(manifest, null, 2),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
// 4. Zip staging (manifest + data/) plus the blobs under files/. The final
|
||||
// .picpeak lands in outDir (caller-managed) or a fresh temp dir; either
|
||||
// way the NDJSON scratch (which holds plaintext secrets) is always
|
||||
// removed in `finally` below.
|
||||
const targetDir = outDir || (await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-out-')));
|
||||
await fsp.mkdir(targetDir, { recursive: true });
|
||||
const stamp = manifest.created_at.replace(/[:.]/g, '-');
|
||||
const filePath = path.join(targetDir, `picpeak-backup-${stamp}.picpeak`);
|
||||
|
||||
try {
|
||||
await new Promise((resolve, reject) => {
|
||||
const output = fs.createWriteStream(filePath);
|
||||
const archive = archiver('zip', { zlib: { level: 9 } });
|
||||
output.on('close', resolve);
|
||||
output.on('error', reject);
|
||||
archive.on('error', reject);
|
||||
// Surface archiver warnings (e.g. a file vanished mid-run) instead of
|
||||
// silently shipping an incomplete archive.
|
||||
archive.on('warning', (err) => reject(err));
|
||||
archive.pipe(output);
|
||||
archive.file(path.join(staging, 'manifest.json'), { name: 'manifest.json' });
|
||||
archive.directory(dataDir, 'data');
|
||||
for (const f of files) {
|
||||
archive.file(f.abs, { name: path.posix.join('files', f.rel.split(path.sep).join('/')) });
|
||||
}
|
||||
archive.finalize();
|
||||
});
|
||||
} catch (err) {
|
||||
// Archiver failed → the partial .picpeak holds plaintext secrets and is
|
||||
// useless; remove our own temp out dir so it isn't orphaned. A
|
||||
// caller-supplied outDir is left untouched.
|
||||
if (!outDir) await fsp.rm(targetDir, { recursive: true, force: true }).catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`[picpeak-export] wrote ${filePath} (${tables.length} tables, ${files.length} files, includePhotos=${!!includePhotos})`
|
||||
);
|
||||
return { filePath, manifest };
|
||||
} finally {
|
||||
// Always remove the NDJSON scratch dir — it contains a plaintext dump of
|
||||
// every table (secrets included). The final .picpeak is elsewhere.
|
||||
await fsp.rm(staging, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
PICPEAK_FORMAT_VERSION,
|
||||
EXCLUDED_TABLES,
|
||||
createPicpeak,
|
||||
// exported for reuse/testing
|
||||
listDataTables,
|
||||
collectFiles,
|
||||
};
|
||||
@@ -0,0 +1,271 @@
|
||||
'use strict';
|
||||
|
||||
// Receiving half of the GUI-only backup roundtrip: takes a ".picpeak" produced
|
||||
// by picpeakExportService and restores it onto THIS instance.
|
||||
//
|
||||
// Restore semantics (agreed design): FULL OVERRIDE — every table is wiped and
|
||||
// replaced by the backup's rows — EXCEPT the current logged-in admin account,
|
||||
// which is preserved so the operator is never locked out. A backup admin whose
|
||||
// email collides with the current account is overwritten with the current
|
||||
// account's credentials (so the operator's known password keeps working).
|
||||
//
|
||||
// Same-engine only (pg↔pg / sqlite↔sqlite) and forward-only (an older backup
|
||||
// restores onto a newer instance; a newer backup is refused). The target's own
|
||||
// schema is used as-is — we never replay the backup's DDL.
|
||||
|
||||
const fs = require('fs');
|
||||
const fsp = require('fs').promises;
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const StreamZip = require('node-stream-zip');
|
||||
const { db } = require('../database/db');
|
||||
const knexConfig = require('../../knexfile');
|
||||
const { getStoragePath } = require('../config/storage');
|
||||
const { hasColumnCached } = require('../utils/schemaCache');
|
||||
const logger = require('../utils/logger');
|
||||
const { PICPEAK_FORMAT_VERSION, EXCLUDED_TABLES, listDataTables } = require('./picpeakExportService');
|
||||
|
||||
const isPostgres = () => knexConfig.client === 'pg';
|
||||
|
||||
// Compare migrations by their numeric filename prefix (001_, 107_, 129_ …).
|
||||
function migrationOrder(name) {
|
||||
const m = String(name || '').match(/^(\d+)/);
|
||||
return m ? parseInt(m[1], 10) : -1;
|
||||
}
|
||||
|
||||
async function readManifestFromZip(picpeakPath) {
|
||||
const zip = new StreamZip.async({ file: picpeakPath });
|
||||
try {
|
||||
return JSON.parse((await zip.entryData('manifest.json')).toString('utf8'));
|
||||
} finally {
|
||||
await zip.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Returns an array of human-readable blockers ([] = OK to restore).
|
||||
async function validateManifest(manifest) {
|
||||
const errors = [];
|
||||
if (!manifest || manifest.kind !== 'picpeak-backup') {
|
||||
return ['This file is not a PicPeak backup (.picpeak).'];
|
||||
}
|
||||
if (Number(manifest.format) > PICPEAK_FORMAT_VERSION) {
|
||||
errors.push('This backup was created by a newer version of PicPeak. Update this instance first.');
|
||||
}
|
||||
const engine = isPostgres() ? 'pg' : 'sqlite';
|
||||
if (manifest.database && manifest.database.engine && manifest.database.engine !== engine) {
|
||||
errors.push(`Database engine mismatch: the backup is "${manifest.database.engine}" but this instance is "${engine}". Restore is only supported between matching engines.`);
|
||||
}
|
||||
// Forward-only: the target schema must be at least as new as the backup's.
|
||||
let targetLatest = null;
|
||||
try {
|
||||
const applied = await db('knex_migrations').orderBy('id', 'desc').limit(1);
|
||||
targetLatest = applied[0] ? applied[0].name : null;
|
||||
} catch (_) {
|
||||
// No knex_migrations table (e.g. some test harnesses) — skip the check.
|
||||
}
|
||||
const backupLatest = manifest.database ? manifest.database.latest_migration : null;
|
||||
if (backupLatest && targetLatest && migrationOrder(backupLatest) > migrationOrder(targetLatest)) {
|
||||
errors.push('This backup is from a newer database schema than this instance. Update this instance to at least the backup version before restoring.');
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function parseNdjson(filePath) {
|
||||
if (!fs.existsSync(filePath)) return [];
|
||||
return fs
|
||||
.readFileSync(filePath, 'utf8')
|
||||
.split('\n')
|
||||
.filter((l) => l.trim().length > 0)
|
||||
.map((l) => JSON.parse(l));
|
||||
}
|
||||
|
||||
// Re-insert the operator's account inside the restore transaction so they keep
|
||||
// working credentials. If the backup already loaded an admin with the same
|
||||
// email, overwrite that row's credentials with the current account's (current
|
||||
// creds win); otherwise insert the snapshot with a fresh id.
|
||||
async function reinjectCurrentAdmin(trx, currentAdmin) {
|
||||
if (!currentAdmin) return;
|
||||
const existing = await trx('admin_users').whereRaw('lower(email) = lower(?)', [currentAdmin.email]).first();
|
||||
if (existing) {
|
||||
await trx('admin_users').where({ id: existing.id }).update({
|
||||
password_hash: currentAdmin.password_hash,
|
||||
is_active: currentAdmin.is_active,
|
||||
must_change_password: currentAdmin.must_change_password,
|
||||
});
|
||||
} else {
|
||||
const row = { ...currentAdmin };
|
||||
delete row.id; // let the engine assign a fresh id to avoid collision
|
||||
await trx('admin_users').insert(row);
|
||||
}
|
||||
}
|
||||
|
||||
// The json/jsonb columns of a table (Postgres only). The pg driver returns
|
||||
// jsonb as parsed JS values, so on re-insert they must be serialised back to
|
||||
// valid JSON text — otherwise a scalar like the string "PicPeak" is sent
|
||||
// unquoted and pg rejects it ("invalid input syntax for type json").
|
||||
async function jsonColumnsFor(trx, table) {
|
||||
if (!isPostgres()) return new Set();
|
||||
const res = await trx.raw(
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = ? AND data_type IN ('json', 'jsonb')",
|
||||
[table]
|
||||
);
|
||||
return new Set(res.rows.map((r) => r.column_name));
|
||||
}
|
||||
|
||||
function serialiseJsonColumns(rows, jsonCols) {
|
||||
if (!jsonCols.size) return rows;
|
||||
return rows.map((row) => {
|
||||
const out = { ...row };
|
||||
for (const col of jsonCols) {
|
||||
if (out[col] !== undefined && out[col] !== null) out[col] = JSON.stringify(out[col]);
|
||||
}
|
||||
return out;
|
||||
});
|
||||
}
|
||||
|
||||
// Whole-DB replace in one transaction with FK enforcement suspended (pg:
|
||||
// session_replication_role=replica on the trx connection, reset before commit;
|
||||
// sqlite: defer_foreign_keys so checks run at commit). knex_migrations is never
|
||||
// in the data set, so the target's schema/migration state is left intact.
|
||||
async function replaceAllTables(tables, dataDir, currentAdmin) {
|
||||
await db.transaction(async (trx) => {
|
||||
if (isPostgres()) {
|
||||
try {
|
||||
await trx.raw("SET session_replication_role = 'replica'");
|
||||
} catch (_) {
|
||||
// session_replication_role requires a Postgres SUPERUSER. The bundled
|
||||
// postgres image's role is one; managed Postgres (RDS / Cloud SQL / …)
|
||||
// app users usually are not. Fail fast with a clear message BEFORE any
|
||||
// rows are deleted — the transaction rolls back, so nothing is wiped.
|
||||
const err = new Error(
|
||||
'Restore needs a PostgreSQL superuser to suspend foreign-key checks during the full replace, but this instance’s database user is not a superuser (common on managed Postgres such as RDS or Cloud SQL). Restore onto the bundled Postgres, or grant the role superuser for the restore.'
|
||||
);
|
||||
err.statusCode = 400;
|
||||
throw err;
|
||||
}
|
||||
} else {
|
||||
await trx.raw('PRAGMA defer_foreign_keys = ON');
|
||||
}
|
||||
|
||||
for (const table of tables) {
|
||||
await trx(table).del();
|
||||
}
|
||||
for (const table of tables) {
|
||||
const rows = parseNdjson(path.join(dataDir, `${table}.ndjson`));
|
||||
if (!rows.length) continue;
|
||||
const jsonCols = await jsonColumnsFor(trx, table);
|
||||
await trx.batchInsert(table, serialiseJsonColumns(rows, jsonCols), 100);
|
||||
}
|
||||
|
||||
await reinjectCurrentAdmin(trx, currentAdmin);
|
||||
|
||||
// Reset the pg session flag BEFORE the connection returns to the pool.
|
||||
if (isPostgres()) await trx.raw("SET session_replication_role = 'origin'");
|
||||
});
|
||||
}
|
||||
|
||||
// Copy the archive's files/ tree into storage, overwriting existing files.
|
||||
async function restoreFiles(stagingDir) {
|
||||
const src = path.join(stagingDir, 'files');
|
||||
if (!fs.existsSync(src)) return 0;
|
||||
const storageRoot = getStoragePath();
|
||||
let count = 0;
|
||||
async function walk(rel) {
|
||||
const abs = path.join(src, rel);
|
||||
for (const entry of await fsp.readdir(abs, { withFileTypes: true })) {
|
||||
const childRel = path.join(rel, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await walk(childRel);
|
||||
} else if (entry.isFile()) {
|
||||
const dest = path.join(storageRoot, childRel);
|
||||
await fsp.mkdir(path.dirname(dest), { recursive: true });
|
||||
await fsp.copyFile(path.join(src, childRel), dest);
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
await walk('');
|
||||
return count;
|
||||
}
|
||||
|
||||
// Does the restored data reference an external-media library? If so the caller
|
||||
// shows a banner telling the admin to (re)configure the external-media mount on
|
||||
// this instance — those files are NOT in the backup by design.
|
||||
async function detectExternalMedia() {
|
||||
try {
|
||||
if (await hasColumnCached('events', 'external_path')) {
|
||||
const row = await db('events').whereNotNull('external_path').first();
|
||||
if (row) return true;
|
||||
}
|
||||
if (await hasColumnCached('photos', 'external_relpath')) {
|
||||
const row = await db('photos').whereNotNull('external_relpath').first();
|
||||
if (row) return true;
|
||||
}
|
||||
} catch (_) {
|
||||
// Best-effort — a detection miss is not worth failing the restore.
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore a .picpeak onto this instance.
|
||||
* @param {Object} opts
|
||||
* @param {string} opts.picpeakPath path to the uploaded/staged .picpeak
|
||||
* @param {number} [opts.currentAdminId] admin to preserve across the wipe
|
||||
* @returns {Promise<{restored:boolean, tables:number, filesRestored:number, usesExternalMedia:boolean, manifest:object}>}
|
||||
*/
|
||||
async function importFromPicpeak({ picpeakPath, currentAdminId }) {
|
||||
const manifest = await readManifestFromZip(picpeakPath);
|
||||
const blockers = await validateManifest(manifest);
|
||||
if (blockers.length) {
|
||||
const err = new Error(blockers[0]);
|
||||
err.statusCode = 400;
|
||||
err.validation = blockers;
|
||||
throw err;
|
||||
}
|
||||
|
||||
const currentAdmin = currentAdminId
|
||||
? await db('admin_users').where({ id: currentAdminId }).first()
|
||||
: null;
|
||||
|
||||
const staging = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-import-'));
|
||||
try {
|
||||
const zip = new StreamZip.async({ file: picpeakPath });
|
||||
try {
|
||||
await zip.extract(null, staging);
|
||||
} finally {
|
||||
await zip.close();
|
||||
}
|
||||
|
||||
const dataDir = path.join(staging, 'data');
|
||||
// Only touch tables that (a) the uploaded manifest lists AND (b) actually
|
||||
// exist as real tables in THIS database. listDataTables() already excludes
|
||||
// knex_migrations/_lock (EXCLUDED_TABLES), so a crafted or corrupted
|
||||
// .picpeak can never make the restore delete the migration bookkeeping — or
|
||||
// any table that isn't a genuine data table here.
|
||||
const dbTables = new Set(await listDataTables());
|
||||
const manifestTables = Object.keys(manifest.tables || {});
|
||||
const tables = manifestTables.filter((tbl) => dbTables.has(tbl) && !EXCLUDED_TABLES.has(tbl));
|
||||
const skipped = manifestTables.filter((tbl) => !tables.includes(tbl));
|
||||
if (skipped.length) {
|
||||
logger.warn(`[picpeak-import] ignoring ${skipped.length} backup table(s) not present in this DB (or protected): ${skipped.join(', ')}`);
|
||||
}
|
||||
|
||||
await replaceAllTables(tables, dataDir, currentAdmin);
|
||||
const filesRestored = await restoreFiles(staging);
|
||||
const usesExternalMedia = await detectExternalMedia();
|
||||
|
||||
logger.info(
|
||||
`[picpeak-import] restored ${tables.length} tables, ${filesRestored} files (externalMedia=${usesExternalMedia})`
|
||||
);
|
||||
return { restored: true, tables: tables.length, filesRestored, usesExternalMedia, manifest };
|
||||
} finally {
|
||||
await fsp.rm(staging, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
importFromPicpeak,
|
||||
readManifestFromZip,
|
||||
validateManifest,
|
||||
};
|
||||
Reference in New Issue
Block a user