diff --git a/backend/__tests__/integration/picpeakExport.test.js b/backend/__tests__/integration/picpeakExport.test.js new file mode 100644 index 00000000..bb443d6c --- /dev/null +++ b/backend/__tests__/integration/picpeakExport.test.js @@ -0,0 +1,94 @@ +'use strict'; + +// Validates the engine-neutral .picpeak export: it must produce a real zip with +// a manifest + per-table NDJSON, exclude knex bookkeeping, and honour the photo +// toggle. Uses the shared CRM DB harness (temp SQLite) — no docker needed. +process.env.JWT_SECRET = process.env.JWT_SECRET || 'test-secret-at-least-32-characters-long!!'; + +const fs = require('fs'); +const path = require('path'); +const StreamZip = require('node-stream-zip'); +const { bootCrmDb } = require('./helpers/crmDb'); + +let db; +let cleanup; +let tmpDir; +let createPicpeak; + +// bootCrmDb MUST run before requiring the service (which transitively requires +// db.js) so the export reads this test's DB, not the default path. +beforeAll(async () => { + ({ db, cleanup, tmpDir } = await bootCrmDb()); + process.env.STORAGE_PATH = tmpDir; // isolate file collection to the temp dir + ({ createPicpeak } = require('../../src/services/picpeakExportService')); +}, 60000); + +afterAll(async () => { + await cleanup(); +}); + +async function readZip(filePath) { + const zip = new StreamZip.async({ file: filePath }); + const entries = Object.keys(await zip.entries()); + const manifest = JSON.parse((await zip.entryData('manifest.json')).toString('utf8')); + await zip.close(); + return { entries, manifest }; +} + +describe('picpeak export (.picpeak logical export)', () => { + it('produces a .picpeak with a manifest and per-table NDJSON', async () => { + const { filePath, manifest } = await createPicpeak({ includePhotos: false }); + try { + expect(filePath.endsWith('.picpeak')).toBe(true); + expect(fs.existsSync(filePath)).toBe(true); + + expect(manifest.format).toBe(1); + expect(manifest.kind).toBe('picpeak-backup'); + expect(manifest.database.engine).toBe('sqlite'); + expect(manifest.options.includePhotos).toBe(false); + expect(manifest.contains_secrets).toBe(true); + // Migrations seed real tables (e.g. app_settings) — expect several. + expect(Object.keys(manifest.tables).length).toBeGreaterThan(0); + expect(Object.keys(manifest.tables)).toContain('app_settings'); + + const { entries, manifest: zipped } = await readZip(filePath); + expect(entries).toContain('manifest.json'); + expect(entries.some((n) => n.startsWith('data/') && n.endsWith('.ndjson'))).toBe(true); + expect(entries).toContain('data/app_settings.ndjson'); + // Manifest inside the zip matches the returned one. + expect(zipped.tables).toEqual(manifest.tables); + } finally { + fs.rmSync(path.dirname(filePath), { recursive: true, force: true }); + } + }); + + it('never exports knex bookkeeping tables', async () => { + const { filePath, manifest } = await createPicpeak({ includePhotos: false }); + try { + const names = Object.keys(manifest.tables); + expect(names).not.toContain('knex_migrations'); + expect(names).not.toContain('knex_migrations_lock'); + } finally { + fs.rmSync(path.dirname(filePath), { recursive: true, force: true }); + } + }); + + it('row counts in the manifest match the NDJSON line counts', async () => { + // Insert a couple of settings so at least one table is non-empty. + await db('app_settings') + .insert({ setting_key: 'picpeak_export_test_a', setting_value: JSON.stringify('1'), setting_type: 'string' }) + .onConflict('setting_key').merge(); + + const { filePath, manifest } = await createPicpeak({ includePhotos: false }); + try { + const zip = new StreamZip.async({ file: filePath }); + const buf = await zip.entryData('data/app_settings.ndjson'); + await zip.close(); + const lines = buf.toString('utf8').split('\n').filter((l) => l.trim().length > 0); + expect(lines.length).toBe(manifest.tables.app_settings.rowCount); + expect(manifest.tables.app_settings.rowCount).toBeGreaterThan(0); + } finally { + fs.rmSync(path.dirname(filePath), { recursive: true, force: true }); + } + }); +}); diff --git a/backend/src/routes/adminBackup.js b/backend/src/routes/adminBackup.js index 7a030edc..e7904a2a 100644 --- a/backend/src/routes/adminBackup.js +++ b/backend/src/routes/adminBackup.js @@ -131,6 +131,33 @@ router.post('/run', adminAuth, requirePermission('backup.create'), async (req, r } }); +// Generate + download a portable ".picpeak" export — an engine-neutral logical +// snapshot (DB rows as NDJSON + PDFs/business-docs) that can be re-uploaded to +// another instance via the web UI. `?includePhotos=true` also bundles original +// gallery photos (larger); otherwise the admin re-uploads them per gallery. +// +// SECURITY: the file contains plaintext secrets (SMTP password, admin password +// hashes, API keys). The download UI must warn before offering it. We surface +// the flag as a response header too so the client can double-confirm. +router.get('/picpeak/export', adminAuth, requirePermission('backup.create'), async (req, res) => { + const fsSync = require('fs'); + try { + const includePhotos = req.query.includePhotos === 'true' || req.query.includePhotos === '1'; + const { createPicpeak } = require('../services/picpeakExportService'); + const { filePath } = await createPicpeak({ includePhotos }); + const filename = path.basename(filePath); + res.setHeader('X-Picpeak-Contains-Secrets', 'true'); + res.download(filePath, filename, (err) => { + // Best-effort cleanup of the temp .picpeak (and its temp dir) after send. + fsSync.rm(path.dirname(filePath), { recursive: true, force: true }, () => {}); + if (err) logger.error('[picpeak-export] download failed', { error: err.message }); + }); + } catch (error) { + logger.error('[picpeak-export] failed to create export', { error: error.message }); + if (!res.headersSent) res.status(500).json({ error: 'Failed to create .picpeak export' }); + } +}); + // Get backup run details router.get('/runs/:id', adminAuth, requirePermission('backup.view'), async (req, res) => { try { diff --git a/backend/src/services/picpeakExportService.js b/backend/src/services/picpeakExportService.js new file mode 100644 index 00000000..663ca252 --- /dev/null +++ b/backend/src/services/picpeakExportService.js @@ -0,0 +1,228 @@ +'use strict'; + +// Portable ".picpeak" export — a single, self-describing archive that can be +// downloaded from one instance and re-uploaded to another via the web UI only +// (see picpeakImportService for the receiving half). +// +// Deliberately ENGINE-NEUTRAL: instead of a native pg_dump / sqlite .backup +// (which can only ever restore into the same engine and version), each table is +// written as NDJSON. The target rebuilds its own schema by running migrations, +// then loads these rows into it — so an older backup restores cleanly onto a +// newer target (forward-only), and pg↔pg / sqlite↔sqlite both work. +// +// This module is purely additive: it introduces a new artifact and touches no +// existing backup/restore path. + +const fs = require('fs'); +const fsp = require('fs').promises; +const path = require('path'); +const os = require('os'); +const crypto = require('crypto'); +const archiver = require('archiver'); +const { db } = require('../database/db'); +const knexConfig = require('../../knexfile'); +const { getStoragePath } = require('../config/storage'); +const logger = require('../utils/logger'); +const packageJson = require('../../package.json'); + +// Bump only on a breaking change to the on-disk layout below. +const PICPEAK_FORMAT_VERSION = 1; + +// Never exported as data — the target owns these (its own migrations set them). +const EXCLUDED_TABLES = new Set(['knex_migrations', 'knex_migrations_lock']); + +// Storage subdirs holding non-recalculable blobs — always included. +const DOC_DIRS = ['business-docs', 'uploads']; +// Original gallery photos — only when includePhotos is true (large; otherwise +// the admin re-uploads originals per gallery and previews are re-rendered). +const PHOTO_DIRS = ['events/active', 'events/archived']; + +const isPostgres = () => knexConfig.client === 'pg'; + +// db.raw returns `{ rows: [...] }` on Postgres and a bare array on SQLite. +const rawRows = (result) => (isPostgres() ? result.rows : result); + +// All user tables, minus knex bookkeeping. Introspected at runtime so the +// export never rots as tables are added (no hardcoded list to maintain). +async function listDataTables() { + let names; + if (isPostgres()) { + const result = await db.raw(` + SELECT table_name AS name + FROM information_schema.tables + WHERE table_schema = 'public' AND table_type = 'BASE TABLE' + ORDER BY table_name + `); + names = rawRows(result).map((r) => r.name); + } else { + const result = await db.raw(` + SELECT name FROM sqlite_master + WHERE type = 'table' AND name NOT LIKE 'sqlite_%' + ORDER BY name + `); + names = rawRows(result).map((r) => r.name); + } + return names.filter((n) => !EXCLUDED_TABLES.has(n)); +} + +// The latest applied migration — recorded in the manifest so the importer can +// refuse a backup that is NEWER than the target (forward-only guarantee). +async function getLatestMigration() { + try { + const rows = await db('knex_migrations').orderBy('id', 'desc').limit(1); + return rows[0]?.name || null; + } catch (_) { + return null; + } +} + +// Stream one table to /.ndjson (one JSON object per line). +// Returns { rowCount, checksum } for the manifest. JSON.stringify serialises +// Dates to ISO strings, which re-import cleanly on both engines. +async function writeTableNdjson(table, dataDir) { + const outPath = path.join(dataDir, `${table}.ndjson`); + const out = fs.createWriteStream(outPath, { encoding: 'utf8' }); + const hash = crypto.createHash('sha256'); + let rowCount = 0; + + await new Promise((resolve, reject) => { + out.on('error', reject); + const stream = db(table).stream(); + stream.on('error', reject); + stream.on('data', (row) => { + const line = `${JSON.stringify(row)}\n`; + hash.update(line); + rowCount += 1; + if (!out.write(line)) { + stream.pause(); + out.once('drain', () => stream.resume()); + } + }); + stream.on('end', () => out.end(resolve)); + }); + + return { rowCount, checksum: hash.digest('hex') }; +} + +// Recursively collect files under a storage subdir as { abs, rel } where rel is +// relative to the storage root (so the importer restores the same layout). +async function collectDir(subdir, storageRoot, acc) { + const abs = path.join(storageRoot, subdir); + let entries; + try { + entries = await fsp.readdir(abs, { withFileTypes: true }); + } catch (_) { + return; // subdir may not exist on this install — skip silently + } + for (const entry of entries) { + const childRel = path.join(subdir, entry.name); + if (entry.isDirectory()) { + await collectDir(childRel, storageRoot, acc); + } else if (entry.isFile()) { + acc.push({ abs: path.join(storageRoot, childRel), rel: childRel }); + } + } +} + +async function collectFiles(includePhotos) { + const storageRoot = getStoragePath(); + const dirs = includePhotos ? [...DOC_DIRS, ...PHOTO_DIRS] : [...DOC_DIRS]; + const acc = []; + for (const d of dirs) { + await collectDir(d, storageRoot, acc); + } + return acc; +} + +/** + * Build a .picpeak archive. + * @param {Object} opts + * @param {boolean} [opts.includePhotos=false] include original gallery photos + * @param {string} [opts.outDir] where to write the file (defaults to a temp dir) + * @returns {Promise<{ filePath: string, manifest: object }>} + */ +async function createPicpeak({ includePhotos = false, outDir } = {}) { + const staging = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-export-')); + const dataDir = path.join(staging, 'data'); + await fsp.mkdir(dataDir, { recursive: true }); + + try { + // 1. Dump every table to NDJSON, tracking counts + checksums. + const tables = await listDataTables(); + const tableMeta = {}; + for (const table of tables) { + tableMeta[table] = await writeTableNdjson(table, dataDir); + } + + // 2. Gather the non-recalculable blobs (PDFs, business-docs, uploads, and + // optionally original photos). + const files = await collectFiles(includePhotos); + + // 3. Manifest — everything the importer needs to validate + reconstruct. + const manifest = { + format: PICPEAK_FORMAT_VERSION, + kind: 'picpeak-backup', + created_at: new Date().toISOString(), + app_version: packageJson.version || null, + database: { + engine: isPostgres() ? 'pg' : 'sqlite', + latest_migration: await getLatestMigration(), + }, + options: { includePhotos: !!includePhotos }, + tables: tableMeta, + file_count: files.length, + // NOTE: contains secrets (SMTP password, admin hashes, API keys) in plain + // text — the download surface must warn about this. + contains_secrets: true, + }; + await fsp.writeFile( + path.join(staging, 'manifest.json'), + JSON.stringify(manifest, null, 2), + 'utf8' + ); + + // 4. Zip staging (manifest + data/) plus the blobs under files/. The final + // .picpeak lands in outDir (caller-managed) or a fresh temp dir; either + // way the NDJSON scratch (which holds plaintext secrets) is always + // removed in `finally` below. + const targetDir = outDir || (await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-out-'))); + await fsp.mkdir(targetDir, { recursive: true }); + const stamp = manifest.created_at.replace(/[:.]/g, '-'); + const filePath = path.join(targetDir, `picpeak-backup-${stamp}.picpeak`); + + await new Promise((resolve, reject) => { + const output = fs.createWriteStream(filePath); + const archive = archiver('zip', { zlib: { level: 9 } }); + output.on('close', resolve); + output.on('error', reject); + archive.on('error', reject); + // Surface archiver warnings (e.g. a file vanished mid-run) instead of + // silently shipping an incomplete archive. + archive.on('warning', (err) => reject(err)); + archive.pipe(output); + archive.file(path.join(staging, 'manifest.json'), { name: 'manifest.json' }); + archive.directory(dataDir, 'data'); + for (const f of files) { + archive.file(f.abs, { name: path.posix.join('files', f.rel.split(path.sep).join('/')) }); + } + archive.finalize(); + }); + + logger.info( + `[picpeak-export] wrote ${filePath} (${tables.length} tables, ${files.length} files, includePhotos=${!!includePhotos})` + ); + return { filePath, manifest }; + } finally { + // Always remove the NDJSON scratch dir — it contains a plaintext dump of + // every table (secrets included). The final .picpeak is elsewhere. + await fsp.rm(staging, { recursive: true, force: true }).catch(() => {}); + } +} + +module.exports = { + PICPEAK_FORMAT_VERSION, + createPicpeak, + // exported for reuse/testing + listDataTables, + collectFiles, +};