feat(backup): .picpeak portable export (engine-neutral logical snapshot)
First half of the GUI-only backup roundtrip. Adds a self-describing ".picpeak" archive that can be downloaded from one instance and (later) re-uploaded to another via the web UI only. - picpeakExportService.createPicpeak(): dumps every table as NDJSON (tables introspected at runtime — no hardcoded list, won't rot), plus a manifest (format version, app version, DB engine, latest migration, per-table row counts + checksums, includePhotos, contains_secrets), plus files/ (business-docs + uploads always; original gallery photos only when includePhotos). NDJSON is engine-neutral so the target rebuilds schema via migrations then loads rows — enabling pg↔pg / sqlite↔sqlite and forward-only auto-migrate. - GET /admin/backup/picpeak/export?includePhotos= streams the file and sets X-Picpeak-Contains-Secrets (the file holds plaintext SMTP pass, admin hashes, API keys — the UI must warn). - Purely additive: no existing backup/restore path is touched. Integration test proves the archive shape, knex-table exclusion, and row-count/NDJSON consistency (85 tables on the seed schema).
This commit is contained in:
Luca
@@ -0,0 +1,94 @@
|
||||
'use strict';
|
||||
|
||||
// Validates the engine-neutral .picpeak export: it must produce a real zip with
|
||||
// a manifest + per-table NDJSON, exclude knex bookkeeping, and honour the photo
|
||||
// toggle. Uses the shared CRM DB harness (temp SQLite) — no docker needed.
|
||||
process.env.JWT_SECRET = process.env.JWT_SECRET || 'test-secret-at-least-32-characters-long!!';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const StreamZip = require('node-stream-zip');
|
||||
const { bootCrmDb } = require('./helpers/crmDb');
|
||||
|
||||
let db;
|
||||
let cleanup;
|
||||
let tmpDir;
|
||||
let createPicpeak;
|
||||
|
||||
// bootCrmDb MUST run before requiring the service (which transitively requires
|
||||
// db.js) so the export reads this test's DB, not the default path.
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup, tmpDir } = await bootCrmDb());
|
||||
process.env.STORAGE_PATH = tmpDir; // isolate file collection to the temp dir
|
||||
({ createPicpeak } = require('../../src/services/picpeakExportService'));
|
||||
}, 60000);
|
||||
|
||||
afterAll(async () => {
|
||||
await cleanup();
|
||||
});
|
||||
|
||||
async function readZip(filePath) {
|
||||
const zip = new StreamZip.async({ file: filePath });
|
||||
const entries = Object.keys(await zip.entries());
|
||||
const manifest = JSON.parse((await zip.entryData('manifest.json')).toString('utf8'));
|
||||
await zip.close();
|
||||
return { entries, manifest };
|
||||
}
|
||||
|
||||
describe('picpeak export (.picpeak logical export)', () => {
|
||||
it('produces a .picpeak with a manifest and per-table NDJSON', async () => {
|
||||
const { filePath, manifest } = await createPicpeak({ includePhotos: false });
|
||||
try {
|
||||
expect(filePath.endsWith('.picpeak')).toBe(true);
|
||||
expect(fs.existsSync(filePath)).toBe(true);
|
||||
|
||||
expect(manifest.format).toBe(1);
|
||||
expect(manifest.kind).toBe('picpeak-backup');
|
||||
expect(manifest.database.engine).toBe('sqlite');
|
||||
expect(manifest.options.includePhotos).toBe(false);
|
||||
expect(manifest.contains_secrets).toBe(true);
|
||||
// Migrations seed real tables (e.g. app_settings) — expect several.
|
||||
expect(Object.keys(manifest.tables).length).toBeGreaterThan(0);
|
||||
expect(Object.keys(manifest.tables)).toContain('app_settings');
|
||||
|
||||
const { entries, manifest: zipped } = await readZip(filePath);
|
||||
expect(entries).toContain('manifest.json');
|
||||
expect(entries.some((n) => n.startsWith('data/') && n.endsWith('.ndjson'))).toBe(true);
|
||||
expect(entries).toContain('data/app_settings.ndjson');
|
||||
// Manifest inside the zip matches the returned one.
|
||||
expect(zipped.tables).toEqual(manifest.tables);
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('never exports knex bookkeeping tables', async () => {
|
||||
const { filePath, manifest } = await createPicpeak({ includePhotos: false });
|
||||
try {
|
||||
const names = Object.keys(manifest.tables);
|
||||
expect(names).not.toContain('knex_migrations');
|
||||
expect(names).not.toContain('knex_migrations_lock');
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('row counts in the manifest match the NDJSON line counts', async () => {
|
||||
// Insert a couple of settings so at least one table is non-empty.
|
||||
await db('app_settings')
|
||||
.insert({ setting_key: 'picpeak_export_test_a', setting_value: JSON.stringify('1'), setting_type: 'string' })
|
||||
.onConflict('setting_key').merge();
|
||||
|
||||
const { filePath, manifest } = await createPicpeak({ includePhotos: false });
|
||||
try {
|
||||
const zip = new StreamZip.async({ file: filePath });
|
||||
const buf = await zip.entryData('data/app_settings.ndjson');
|
||||
await zip.close();
|
||||
const lines = buf.toString('utf8').split('\n').filter((l) => l.trim().length > 0);
|
||||
expect(lines.length).toBe(manifest.tables.app_settings.rowCount);
|
||||
expect(manifest.tables.app_settings.rowCount).toBeGreaterThan(0);
|
||||
} finally {
|
||||
fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -131,6 +131,33 @@ router.post('/run', adminAuth, requirePermission('backup.create'), async (req, r
|
||||
}
|
||||
});
|
||||
|
||||
// Generate + download a portable ".picpeak" export — an engine-neutral logical
|
||||
// snapshot (DB rows as NDJSON + PDFs/business-docs) that can be re-uploaded to
|
||||
// another instance via the web UI. `?includePhotos=true` also bundles original
|
||||
// gallery photos (larger); otherwise the admin re-uploads them per gallery.
|
||||
//
|
||||
// SECURITY: the file contains plaintext secrets (SMTP password, admin password
|
||||
// hashes, API keys). The download UI must warn before offering it. We surface
|
||||
// the flag as a response header too so the client can double-confirm.
|
||||
router.get('/picpeak/export', adminAuth, requirePermission('backup.create'), async (req, res) => {
|
||||
const fsSync = require('fs');
|
||||
try {
|
||||
const includePhotos = req.query.includePhotos === 'true' || req.query.includePhotos === '1';
|
||||
const { createPicpeak } = require('../services/picpeakExportService');
|
||||
const { filePath } = await createPicpeak({ includePhotos });
|
||||
const filename = path.basename(filePath);
|
||||
res.setHeader('X-Picpeak-Contains-Secrets', 'true');
|
||||
res.download(filePath, filename, (err) => {
|
||||
// Best-effort cleanup of the temp .picpeak (and its temp dir) after send.
|
||||
fsSync.rm(path.dirname(filePath), { recursive: true, force: true }, () => {});
|
||||
if (err) logger.error('[picpeak-export] download failed', { error: err.message });
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('[picpeak-export] failed to create export', { error: error.message });
|
||||
if (!res.headersSent) res.status(500).json({ error: 'Failed to create .picpeak export' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get backup run details
|
||||
router.get('/runs/:id', adminAuth, requirePermission('backup.view'), async (req, res) => {
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
'use strict';
|
||||
|
||||
// Portable ".picpeak" export — a single, self-describing archive that can be
|
||||
// downloaded from one instance and re-uploaded to another via the web UI only
|
||||
// (see picpeakImportService for the receiving half).
|
||||
//
|
||||
// Deliberately ENGINE-NEUTRAL: instead of a native pg_dump / sqlite .backup
|
||||
// (which can only ever restore into the same engine and version), each table is
|
||||
// written as NDJSON. The target rebuilds its own schema by running migrations,
|
||||
// then loads these rows into it — so an older backup restores cleanly onto a
|
||||
// newer target (forward-only), and pg↔pg / sqlite↔sqlite both work.
|
||||
//
|
||||
// This module is purely additive: it introduces a new artifact and touches no
|
||||
// existing backup/restore path.
|
||||
|
||||
const fs = require('fs');
|
||||
const fsp = require('fs').promises;
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const crypto = require('crypto');
|
||||
const archiver = require('archiver');
|
||||
const { db } = require('../database/db');
|
||||
const knexConfig = require('../../knexfile');
|
||||
const { getStoragePath } = require('../config/storage');
|
||||
const logger = require('../utils/logger');
|
||||
const packageJson = require('../../package.json');
|
||||
|
||||
// Bump only on a breaking change to the on-disk layout below.
|
||||
const PICPEAK_FORMAT_VERSION = 1;
|
||||
|
||||
// Never exported as data — the target owns these (its own migrations set them).
|
||||
const EXCLUDED_TABLES = new Set(['knex_migrations', 'knex_migrations_lock']);
|
||||
|
||||
// Storage subdirs holding non-recalculable blobs — always included.
|
||||
const DOC_DIRS = ['business-docs', 'uploads'];
|
||||
// Original gallery photos — only when includePhotos is true (large; otherwise
|
||||
// the admin re-uploads originals per gallery and previews are re-rendered).
|
||||
const PHOTO_DIRS = ['events/active', 'events/archived'];
|
||||
|
||||
const isPostgres = () => knexConfig.client === 'pg';
|
||||
|
||||
// db.raw returns `{ rows: [...] }` on Postgres and a bare array on SQLite.
|
||||
const rawRows = (result) => (isPostgres() ? result.rows : result);
|
||||
|
||||
// All user tables, minus knex bookkeeping. Introspected at runtime so the
|
||||
// export never rots as tables are added (no hardcoded list to maintain).
|
||||
async function listDataTables() {
|
||||
let names;
|
||||
if (isPostgres()) {
|
||||
const result = await db.raw(`
|
||||
SELECT table_name AS name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
|
||||
ORDER BY table_name
|
||||
`);
|
||||
names = rawRows(result).map((r) => r.name);
|
||||
} else {
|
||||
const result = await db.raw(`
|
||||
SELECT name FROM sqlite_master
|
||||
WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
|
||||
ORDER BY name
|
||||
`);
|
||||
names = rawRows(result).map((r) => r.name);
|
||||
}
|
||||
return names.filter((n) => !EXCLUDED_TABLES.has(n));
|
||||
}
|
||||
|
||||
// The latest applied migration — recorded in the manifest so the importer can
|
||||
// refuse a backup that is NEWER than the target (forward-only guarantee).
|
||||
async function getLatestMigration() {
|
||||
try {
|
||||
const rows = await db('knex_migrations').orderBy('id', 'desc').limit(1);
|
||||
return rows[0]?.name || null;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Stream one table to <dataDir>/<table>.ndjson (one JSON object per line).
|
||||
// Returns { rowCount, checksum } for the manifest. JSON.stringify serialises
|
||||
// Dates to ISO strings, which re-import cleanly on both engines.
|
||||
async function writeTableNdjson(table, dataDir) {
|
||||
const outPath = path.join(dataDir, `${table}.ndjson`);
|
||||
const out = fs.createWriteStream(outPath, { encoding: 'utf8' });
|
||||
const hash = crypto.createHash('sha256');
|
||||
let rowCount = 0;
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
out.on('error', reject);
|
||||
const stream = db(table).stream();
|
||||
stream.on('error', reject);
|
||||
stream.on('data', (row) => {
|
||||
const line = `${JSON.stringify(row)}\n`;
|
||||
hash.update(line);
|
||||
rowCount += 1;
|
||||
if (!out.write(line)) {
|
||||
stream.pause();
|
||||
out.once('drain', () => stream.resume());
|
||||
}
|
||||
});
|
||||
stream.on('end', () => out.end(resolve));
|
||||
});
|
||||
|
||||
return { rowCount, checksum: hash.digest('hex') };
|
||||
}
|
||||
|
||||
// Recursively collect files under a storage subdir as { abs, rel } where rel is
|
||||
// relative to the storage root (so the importer restores the same layout).
|
||||
async function collectDir(subdir, storageRoot, acc) {
|
||||
const abs = path.join(storageRoot, subdir);
|
||||
let entries;
|
||||
try {
|
||||
entries = await fsp.readdir(abs, { withFileTypes: true });
|
||||
} catch (_) {
|
||||
return; // subdir may not exist on this install — skip silently
|
||||
}
|
||||
for (const entry of entries) {
|
||||
const childRel = path.join(subdir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await collectDir(childRel, storageRoot, acc);
|
||||
} else if (entry.isFile()) {
|
||||
acc.push({ abs: path.join(storageRoot, childRel), rel: childRel });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function collectFiles(includePhotos) {
|
||||
const storageRoot = getStoragePath();
|
||||
const dirs = includePhotos ? [...DOC_DIRS, ...PHOTO_DIRS] : [...DOC_DIRS];
|
||||
const acc = [];
|
||||
for (const d of dirs) {
|
||||
await collectDir(d, storageRoot, acc);
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a .picpeak archive.
|
||||
* @param {Object} opts
|
||||
* @param {boolean} [opts.includePhotos=false] include original gallery photos
|
||||
* @param {string} [opts.outDir] where to write the file (defaults to a temp dir)
|
||||
* @returns {Promise<{ filePath: string, manifest: object }>}
|
||||
*/
|
||||
async function createPicpeak({ includePhotos = false, outDir } = {}) {
|
||||
const staging = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-export-'));
|
||||
const dataDir = path.join(staging, 'data');
|
||||
await fsp.mkdir(dataDir, { recursive: true });
|
||||
|
||||
try {
|
||||
// 1. Dump every table to NDJSON, tracking counts + checksums.
|
||||
const tables = await listDataTables();
|
||||
const tableMeta = {};
|
||||
for (const table of tables) {
|
||||
tableMeta[table] = await writeTableNdjson(table, dataDir);
|
||||
}
|
||||
|
||||
// 2. Gather the non-recalculable blobs (PDFs, business-docs, uploads, and
|
||||
// optionally original photos).
|
||||
const files = await collectFiles(includePhotos);
|
||||
|
||||
// 3. Manifest — everything the importer needs to validate + reconstruct.
|
||||
const manifest = {
|
||||
format: PICPEAK_FORMAT_VERSION,
|
||||
kind: 'picpeak-backup',
|
||||
created_at: new Date().toISOString(),
|
||||
app_version: packageJson.version || null,
|
||||
database: {
|
||||
engine: isPostgres() ? 'pg' : 'sqlite',
|
||||
latest_migration: await getLatestMigration(),
|
||||
},
|
||||
options: { includePhotos: !!includePhotos },
|
||||
tables: tableMeta,
|
||||
file_count: files.length,
|
||||
// NOTE: contains secrets (SMTP password, admin hashes, API keys) in plain
|
||||
// text — the download surface must warn about this.
|
||||
contains_secrets: true,
|
||||
};
|
||||
await fsp.writeFile(
|
||||
path.join(staging, 'manifest.json'),
|
||||
JSON.stringify(manifest, null, 2),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
// 4. Zip staging (manifest + data/) plus the blobs under files/. The final
|
||||
// .picpeak lands in outDir (caller-managed) or a fresh temp dir; either
|
||||
// way the NDJSON scratch (which holds plaintext secrets) is always
|
||||
// removed in `finally` below.
|
||||
const targetDir = outDir || (await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-out-')));
|
||||
await fsp.mkdir(targetDir, { recursive: true });
|
||||
const stamp = manifest.created_at.replace(/[:.]/g, '-');
|
||||
const filePath = path.join(targetDir, `picpeak-backup-${stamp}.picpeak`);
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
const output = fs.createWriteStream(filePath);
|
||||
const archive = archiver('zip', { zlib: { level: 9 } });
|
||||
output.on('close', resolve);
|
||||
output.on('error', reject);
|
||||
archive.on('error', reject);
|
||||
// Surface archiver warnings (e.g. a file vanished mid-run) instead of
|
||||
// silently shipping an incomplete archive.
|
||||
archive.on('warning', (err) => reject(err));
|
||||
archive.pipe(output);
|
||||
archive.file(path.join(staging, 'manifest.json'), { name: 'manifest.json' });
|
||||
archive.directory(dataDir, 'data');
|
||||
for (const f of files) {
|
||||
archive.file(f.abs, { name: path.posix.join('files', f.rel.split(path.sep).join('/')) });
|
||||
}
|
||||
archive.finalize();
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`[picpeak-export] wrote ${filePath} (${tables.length} tables, ${files.length} files, includePhotos=${!!includePhotos})`
|
||||
);
|
||||
return { filePath, manifest };
|
||||
} finally {
|
||||
// Always remove the NDJSON scratch dir — it contains a plaintext dump of
|
||||
// every table (secrets included). The final .picpeak is elsewhere.
|
||||
await fsp.rm(staging, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
PICPEAK_FORMAT_VERSION,
|
||||
createPicpeak,
|
||||
// exported for reuse/testing
|
||||
listDataTables,
|
||||
collectFiles,
|
||||
};
|
||||
Reference in New Issue
Block a user