feat(backup): .picpeak import/restore (full override, keeps current account)
Receiving half of the roundtrip. picpeakImportService.importFromPicpeak(): - Validates the manifest: rejects non-picpeak files, a newer format, an engine mismatch (pg↔pg / sqlite↔sqlite only), and a backup from a NEWER schema than this instance (forward-only). knex_migrations absence is tolerated (test harnesses). - Snapshots the current logged-in admin, then wipes + reloads every table from the backup NDJSON in one transaction with FK enforcement suspended (pg: session_replication_role=replica reset before commit; sqlite: defer_foreign_keys). knex_migrations is never touched, so the target's schema/migration state is preserved. - Re-injects the current account so the operator is never locked out; a backup admin colliding on email is overwritten with the current creds. - Restores files/ into storage and detects external-media references so the caller can prompt to reconfigure the mount. Roundtrip integration test proves: backup data restored, current account survives a full override (different email → added), and the email-collision case keeps the operator's password.
This commit is contained in:
Luca
@@ -0,0 +1,119 @@
|
||||
'use strict';
|
||||
|
||||
// Full .picpeak roundtrip on a temp SQLite DB:
|
||||
// 1. seed a "backup" instance (admin A + a marker setting)
|
||||
// 2. export → .picpeak
|
||||
// 3. simulate a reinstall: wipe, create a DIFFERENT current admin B, mutate data
|
||||
// 4. import the backup with currentAdminId = B
|
||||
// 5. assert the backup data is restored AND the current account (B) survives,
|
||||
// while the backup's admin (A) is also present (different email → added).
|
||||
process.env.JWT_SECRET = process.env.JWT_SECRET || 'test-secret-at-least-32-characters-long!!';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { bootCrmDb } = require('./helpers/crmDb');
|
||||
|
||||
let db;
|
||||
let cleanup;
|
||||
let tmpDir;
|
||||
let createPicpeak;
|
||||
let importFromPicpeak;
|
||||
let superAdminRoleId;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup, tmpDir } = await bootCrmDb());
|
||||
process.env.STORAGE_PATH = tmpDir;
|
||||
({ createPicpeak } = require('../../src/services/picpeakExportService'));
|
||||
({ importFromPicpeak } = require('../../src/services/picpeakImportService'));
|
||||
const role = await db('roles').where({ name: 'super_admin' }).first();
|
||||
superAdminRoleId = role.id;
|
||||
}, 60000);
|
||||
|
||||
afterAll(async () => {
|
||||
await cleanup();
|
||||
});
|
||||
|
||||
const adminRow = (email, hash) => ({
|
||||
username: email,
|
||||
email,
|
||||
password_hash: hash,
|
||||
role_id: superAdminRoleId,
|
||||
is_active: true,
|
||||
must_change_password: false,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
});
|
||||
|
||||
async function setMarker(value) {
|
||||
await db('app_settings')
|
||||
.insert({ setting_key: 'roundtrip_marker', setting_value: JSON.stringify(value), setting_type: 'string' })
|
||||
.onConflict('setting_key').merge();
|
||||
}
|
||||
async function getMarker() {
|
||||
const row = await db('app_settings').where({ setting_key: 'roundtrip_marker' }).first();
|
||||
return row ? JSON.parse(row.setting_value) : null;
|
||||
}
|
||||
|
||||
describe('.picpeak roundtrip (export → import)', () => {
|
||||
it('restores backup data and preserves the current account', async () => {
|
||||
// 1. Seed the "source" instance.
|
||||
await db('admin_users').del();
|
||||
await db('admin_users').insert(adminRow('[email protected]', 'HASH_A'));
|
||||
await setMarker('from_backup');
|
||||
|
||||
// 2. Export.
|
||||
const { filePath } = await createPicpeak({ includePhotos: false });
|
||||
|
||||
try {
|
||||
// 3. Simulate a reinstall: fresh current admin B, mutated data.
|
||||
await db('admin_users').del();
|
||||
const [bId] = await db('admin_users').insert(adminRow('[email protected]', 'HASH_B')).returning('id');
|
||||
const currentAdminId = typeof bId === 'object' ? bId.id : bId;
|
||||
await setMarker('mutated_after_backup');
|
||||
|
||||
// 4. Import, preserving the current admin.
|
||||
const result = await importFromPicpeak({ filePath: undefined, picpeakPath: filePath, currentAdminId });
|
||||
expect(result.restored).toBe(true);
|
||||
expect(result.tables).toBeGreaterThan(0);
|
||||
|
||||
// 5a. Backup data restored (marker reverted to the backup value).
|
||||
expect(await getMarker()).toBe('from_backup');
|
||||
|
||||
// 5b. The backup's admin is present (different email → added).
|
||||
const a = await db('admin_users').whereRaw('lower(email) = lower(?)', ['[email protected]']).first();
|
||||
expect(a).toBeTruthy();
|
||||
expect(a.password_hash).toBe('HASH_A');
|
||||
|
||||
// 5c. The current account SURVIVES the override, with its own credentials.
|
||||
const b = await db('admin_users').whereRaw('lower(email) = lower(?)', ['[email protected]']).first();
|
||||
expect(b).toBeTruthy();
|
||||
expect(b.password_hash).toBe('HASH_B');
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('overwrites a backup admin that collides with the current account email', async () => {
|
||||
// Source has an admin at the SAME email the current operator will use.
|
||||
await db('admin_users').del();
|
||||
await db('admin_users').insert(adminRow('[email protected]', 'OLD_HASH'));
|
||||
await setMarker('collision_case');
|
||||
const { filePath } = await createPicpeak({ includePhotos: false });
|
||||
|
||||
try {
|
||||
// Reinstall: current admin uses the same email but a NEW password.
|
||||
await db('admin_users').del();
|
||||
const [id] = await db('admin_users').insert(adminRow('[email protected]', 'NEW_HASH')).returning('id');
|
||||
const currentAdminId = typeof id === 'object' ? id.id : id;
|
||||
|
||||
await importFromPicpeak({ picpeakPath: filePath, currentAdminId });
|
||||
|
||||
// Exactly one admin at that email, and it keeps the CURRENT password.
|
||||
const rows = await db('admin_users').whereRaw('lower(email) = lower(?)', ['[email protected]']);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].password_hash).toBe('NEW_HASH');
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,219 @@
|
||||
'use strict';
|
||||
|
||||
// Receiving half of the GUI-only backup roundtrip: takes a ".picpeak" produced
|
||||
// by picpeakExportService and restores it onto THIS instance.
|
||||
//
|
||||
// Restore semantics (agreed design): FULL OVERRIDE — every table is wiped and
|
||||
// replaced by the backup's rows — EXCEPT the current logged-in admin account,
|
||||
// which is preserved so the operator is never locked out. A backup admin whose
|
||||
// email collides with the current account is overwritten with the current
|
||||
// account's credentials (so the operator's known password keeps working).
|
||||
//
|
||||
// Same-engine only (pg↔pg / sqlite↔sqlite) and forward-only (an older backup
|
||||
// restores onto a newer instance; a newer backup is refused). The target's own
|
||||
// schema is used as-is — we never replay the backup's DDL.
|
||||
|
||||
const fs = require('fs');
|
||||
const fsp = require('fs').promises;
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const StreamZip = require('node-stream-zip');
|
||||
const { db } = require('../database/db');
|
||||
const knexConfig = require('../../knexfile');
|
||||
const { getStoragePath } = require('../config/storage');
|
||||
const { hasColumnCached } = require('../utils/schemaCache');
|
||||
const logger = require('../utils/logger');
|
||||
const { PICPEAK_FORMAT_VERSION } = require('./picpeakExportService');
|
||||
|
||||
const isPostgres = () => knexConfig.client === 'pg';
|
||||
|
||||
// Compare migrations by their numeric filename prefix (001_, 107_, 129_ …).
|
||||
function migrationOrder(name) {
|
||||
const m = String(name || '').match(/^(\d+)/);
|
||||
return m ? parseInt(m[1], 10) : -1;
|
||||
}
|
||||
|
||||
async function readManifestFromZip(picpeakPath) {
|
||||
const zip = new StreamZip.async({ file: picpeakPath });
|
||||
try {
|
||||
return JSON.parse((await zip.entryData('manifest.json')).toString('utf8'));
|
||||
} finally {
|
||||
await zip.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Returns an array of human-readable blockers ([] = OK to restore).
|
||||
async function validateManifest(manifest) {
|
||||
const errors = [];
|
||||
if (!manifest || manifest.kind !== 'picpeak-backup') {
|
||||
return ['This file is not a PicPeak backup (.picpeak).'];
|
||||
}
|
||||
if (Number(manifest.format) > PICPEAK_FORMAT_VERSION) {
|
||||
errors.push('This backup was created by a newer version of PicPeak. Update this instance first.');
|
||||
}
|
||||
const engine = isPostgres() ? 'pg' : 'sqlite';
|
||||
if (manifest.database && manifest.database.engine && manifest.database.engine !== engine) {
|
||||
errors.push(`Database engine mismatch: the backup is "${manifest.database.engine}" but this instance is "${engine}". Restore is only supported between matching engines.`);
|
||||
}
|
||||
// Forward-only: the target schema must be at least as new as the backup's.
|
||||
let targetLatest = null;
|
||||
try {
|
||||
const applied = await db('knex_migrations').orderBy('id', 'desc').limit(1);
|
||||
targetLatest = applied[0] ? applied[0].name : null;
|
||||
} catch (_) {
|
||||
// No knex_migrations table (e.g. some test harnesses) — skip the check.
|
||||
}
|
||||
const backupLatest = manifest.database ? manifest.database.latest_migration : null;
|
||||
if (backupLatest && targetLatest && migrationOrder(backupLatest) > migrationOrder(targetLatest)) {
|
||||
errors.push('This backup is from a newer database schema than this instance. Update this instance to at least the backup version before restoring.');
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
function parseNdjson(filePath) {
|
||||
if (!fs.existsSync(filePath)) return [];
|
||||
return fs
|
||||
.readFileSync(filePath, 'utf8')
|
||||
.split('\n')
|
||||
.filter((l) => l.trim().length > 0)
|
||||
.map((l) => JSON.parse(l));
|
||||
}
|
||||
|
||||
// Re-insert the operator's account inside the restore transaction so they keep
|
||||
// working credentials. If the backup already loaded an admin with the same
|
||||
// email, overwrite that row's credentials with the current account's (current
|
||||
// creds win); otherwise insert the snapshot with a fresh id.
|
||||
async function reinjectCurrentAdmin(trx, currentAdmin) {
|
||||
if (!currentAdmin) return;
|
||||
const existing = await trx('admin_users').whereRaw('lower(email) = lower(?)', [currentAdmin.email]).first();
|
||||
if (existing) {
|
||||
await trx('admin_users').where({ id: existing.id }).update({
|
||||
password_hash: currentAdmin.password_hash,
|
||||
is_active: currentAdmin.is_active,
|
||||
must_change_password: currentAdmin.must_change_password,
|
||||
});
|
||||
} else {
|
||||
const row = { ...currentAdmin };
|
||||
delete row.id; // let the engine assign a fresh id to avoid collision
|
||||
await trx('admin_users').insert(row);
|
||||
}
|
||||
}
|
||||
|
||||
// Whole-DB replace in one transaction with FK enforcement suspended (pg:
|
||||
// session_replication_role=replica on the trx connection, reset before commit;
|
||||
// sqlite: defer_foreign_keys so checks run at commit). knex_migrations is never
|
||||
// in the data set, so the target's schema/migration state is left intact.
|
||||
async function replaceAllTables(tables, dataDir, currentAdmin) {
|
||||
await db.transaction(async (trx) => {
|
||||
if (isPostgres()) await trx.raw("SET session_replication_role = 'replica'");
|
||||
else await trx.raw('PRAGMA defer_foreign_keys = ON');
|
||||
|
||||
for (const table of tables) {
|
||||
await trx(table).del();
|
||||
}
|
||||
for (const table of tables) {
|
||||
const rows = parseNdjson(path.join(dataDir, `${table}.ndjson`));
|
||||
if (rows.length) await trx.batchInsert(table, rows, 100);
|
||||
}
|
||||
|
||||
await reinjectCurrentAdmin(trx, currentAdmin);
|
||||
|
||||
// Reset the pg session flag BEFORE the connection returns to the pool.
|
||||
if (isPostgres()) await trx.raw("SET session_replication_role = 'origin'");
|
||||
});
|
||||
}
|
||||
|
||||
// Copy the archive's files/ tree into storage, overwriting existing files.
|
||||
async function restoreFiles(stagingDir) {
|
||||
const src = path.join(stagingDir, 'files');
|
||||
if (!fs.existsSync(src)) return 0;
|
||||
const storageRoot = getStoragePath();
|
||||
let count = 0;
|
||||
async function walk(rel) {
|
||||
const abs = path.join(src, rel);
|
||||
for (const entry of await fsp.readdir(abs, { withFileTypes: true })) {
|
||||
const childRel = path.join(rel, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await walk(childRel);
|
||||
} else if (entry.isFile()) {
|
||||
const dest = path.join(storageRoot, childRel);
|
||||
await fsp.mkdir(path.dirname(dest), { recursive: true });
|
||||
await fsp.copyFile(path.join(src, childRel), dest);
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
await walk('');
|
||||
return count;
|
||||
}
|
||||
|
||||
// Does the restored data reference an external-media library? If so the caller
|
||||
// shows a banner telling the admin to (re)configure the external-media mount on
|
||||
// this instance — those files are NOT in the backup by design.
|
||||
async function detectExternalMedia() {
|
||||
try {
|
||||
if (await hasColumnCached('events', 'external_path')) {
|
||||
const row = await db('events').whereNotNull('external_path').first();
|
||||
if (row) return true;
|
||||
}
|
||||
if (await hasColumnCached('photos', 'external_relpath')) {
|
||||
const row = await db('photos').whereNotNull('external_relpath').first();
|
||||
if (row) return true;
|
||||
}
|
||||
} catch (_) {
|
||||
// Best-effort — a detection miss is not worth failing the restore.
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore a .picpeak onto this instance.
|
||||
* @param {Object} opts
|
||||
* @param {string} opts.picpeakPath path to the uploaded/staged .picpeak
|
||||
* @param {number} [opts.currentAdminId] admin to preserve across the wipe
|
||||
* @returns {Promise<{restored:boolean, tables:number, filesRestored:number, usesExternalMedia:boolean, manifest:object}>}
|
||||
*/
|
||||
async function importFromPicpeak({ picpeakPath, currentAdminId }) {
|
||||
const manifest = await readManifestFromZip(picpeakPath);
|
||||
const blockers = await validateManifest(manifest);
|
||||
if (blockers.length) {
|
||||
const err = new Error(blockers[0]);
|
||||
err.statusCode = 400;
|
||||
err.validation = blockers;
|
||||
throw err;
|
||||
}
|
||||
|
||||
const currentAdmin = currentAdminId
|
||||
? await db('admin_users').where({ id: currentAdminId }).first()
|
||||
: null;
|
||||
|
||||
const staging = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-import-'));
|
||||
try {
|
||||
const zip = new StreamZip.async({ file: picpeakPath });
|
||||
try {
|
||||
await zip.extract(null, staging);
|
||||
} finally {
|
||||
await zip.close();
|
||||
}
|
||||
|
||||
const dataDir = path.join(staging, 'data');
|
||||
const tables = Object.keys(manifest.tables || {});
|
||||
|
||||
await replaceAllTables(tables, dataDir, currentAdmin);
|
||||
const filesRestored = await restoreFiles(staging);
|
||||
const usesExternalMedia = await detectExternalMedia();
|
||||
|
||||
logger.info(
|
||||
`[picpeak-import] restored ${tables.length} tables, ${filesRestored} files (externalMedia=${usesExternalMedia})`
|
||||
);
|
||||
return { restored: true, tables: tables.length, filesRestored, usesExternalMedia, manifest };
|
||||
} finally {
|
||||
await fsp.rm(staging, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
importFromPicpeak,
|
||||
readManifestFromZip,
|
||||
validateManifest,
|
||||
};
|
||||
Reference in New Issue
Block a user