feat(backup): .picpeak portable export (engine-neutral logical snapshot)

First half of the GUI-only backup roundtrip. Adds a self-describing
".picpeak" archive that can be downloaded from one instance and (later)
re-uploaded to another via the web UI only.

- picpeakExportService.createPicpeak(): dumps every table as NDJSON
  (tables introspected at runtime — no hardcoded list, won't rot), plus
  a manifest (format version, app version, DB engine, latest migration,
  per-table row counts + checksums, includePhotos, contains_secrets),
  plus files/ (business-docs + uploads always; original gallery photos
  only when includePhotos). NDJSON is engine-neutral so the target
  rebuilds schema via migrations then loads rows — enabling pg↔pg /
  sqlite↔sqlite and forward-only auto-migrate.
- GET /admin/backup/picpeak/export?includePhotos= streams the file and
  sets X-Picpeak-Contains-Secrets (the file holds plaintext SMTP pass,
  admin hashes, API keys — the UI must warn).
- Purely additive: no existing backup/restore path is touched.

Integration test proves the archive shape, knex-table exclusion, and
row-count/NDJSON consistency (85 tables on the seed schema).

backend/__tests__/integration/picpeakExport.test.js

@@ -0,0 +1,94 @@
+'use strict';
+
+// Validates the engine-neutral .picpeak export: it must produce a real zip with
+// a manifest + per-table NDJSON, exclude knex bookkeeping, and honour the photo
+// toggle. Uses the shared CRM DB harness (temp SQLite) — no docker needed.
+process.env.JWT_SECRET = process.env.JWT_SECRET || 'test-secret-at-least-32-characters-long!!';
+
+const fs = require('fs');
+const path = require('path');
+const StreamZip = require('node-stream-zip');
+const { bootCrmDb } = require('./helpers/crmDb');
+
+let db;
+let cleanup;
+let tmpDir;
+let createPicpeak;
+
+// bootCrmDb MUST run before requiring the service (which transitively requires
+// db.js) so the export reads this test's DB, not the default path.
+beforeAll(async () => {
+  ({ db, cleanup, tmpDir } = await bootCrmDb());
+  process.env.STORAGE_PATH = tmpDir; // isolate file collection to the temp dir
+  ({ createPicpeak } = require('../../src/services/picpeakExportService'));
+}, 60000);
+
+afterAll(async () => {
+  await cleanup();
+});
+
+async function readZip(filePath) {
+  const zip = new StreamZip.async({ file: filePath });
+  const entries = Object.keys(await zip.entries());
+  const manifest = JSON.parse((await zip.entryData('manifest.json')).toString('utf8'));
+  await zip.close();
+  return { entries, manifest };
+}
+
+describe('picpeak export (.picpeak logical export)', () => {
+  it('produces a .picpeak with a manifest and per-table NDJSON', async () => {
+    const { filePath, manifest } = await createPicpeak({ includePhotos: false });
+    try {
+      expect(filePath.endsWith('.picpeak')).toBe(true);
+      expect(fs.existsSync(filePath)).toBe(true);
+
+      expect(manifest.format).toBe(1);
+      expect(manifest.kind).toBe('picpeak-backup');
+      expect(manifest.database.engine).toBe('sqlite');
+      expect(manifest.options.includePhotos).toBe(false);
+      expect(manifest.contains_secrets).toBe(true);
+      // Migrations seed real tables (e.g. app_settings) — expect several.
+      expect(Object.keys(manifest.tables).length).toBeGreaterThan(0);
+      expect(Object.keys(manifest.tables)).toContain('app_settings');
+
+      const { entries, manifest: zipped } = await readZip(filePath);
+      expect(entries).toContain('manifest.json');
+      expect(entries.some((n) => n.startsWith('data/') && n.endsWith('.ndjson'))).toBe(true);
+      expect(entries).toContain('data/app_settings.ndjson');
+      // Manifest inside the zip matches the returned one.
+      expect(zipped.tables).toEqual(manifest.tables);
+    } finally {
+      fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
+    }
+  });
+
+  it('never exports knex bookkeeping tables', async () => {
+    const { filePath, manifest } = await createPicpeak({ includePhotos: false });
+    try {
+      const names = Object.keys(manifest.tables);
+      expect(names).not.toContain('knex_migrations');
+      expect(names).not.toContain('knex_migrations_lock');
+    } finally {
+      fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
+    }
+  });
+
+  it('row counts in the manifest match the NDJSON line counts', async () => {
+    // Insert a couple of settings so at least one table is non-empty.
+    await db('app_settings')
+      .insert({ setting_key: 'picpeak_export_test_a', setting_value: JSON.stringify('1'), setting_type: 'string' })
+      .onConflict('setting_key').merge();
+
+    const { filePath, manifest } = await createPicpeak({ includePhotos: false });
+    try {
+      const zip = new StreamZip.async({ file: filePath });
+      const buf = await zip.entryData('data/app_settings.ndjson');
+      await zip.close();
+      const lines = buf.toString('utf8').split('\n').filter((l) => l.trim().length > 0);
+      expect(lines.length).toBe(manifest.tables.app_settings.rowCount);
+      expect(manifest.tables.app_settings.rowCount).toBeGreaterThan(0);
+    } finally {
+      fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
+    }
+  });
+});

backend/src/routes/adminBackup.js

@@ -131,6 +131,33 @@ router.post('/run', adminAuth, requirePermission('backup.create'), async (req, r
   }
 });
 
+// Generate + download a portable ".picpeak" export — an engine-neutral logical
+// snapshot (DB rows as NDJSON + PDFs/business-docs) that can be re-uploaded to
+// another instance via the web UI. `?includePhotos=true` also bundles original
+// gallery photos (larger); otherwise the admin re-uploads them per gallery.
+//
+// SECURITY: the file contains plaintext secrets (SMTP password, admin password
+// hashes, API keys). The download UI must warn before offering it. We surface
+// the flag as a response header too so the client can double-confirm.
+router.get('/picpeak/export', adminAuth, requirePermission('backup.create'), async (req, res) => {
+  const fsSync = require('fs');
+  try {
+    const includePhotos = req.query.includePhotos === 'true' || req.query.includePhotos === '1';
+    const { createPicpeak } = require('../services/picpeakExportService');
+    const { filePath } = await createPicpeak({ includePhotos });
+    const filename = path.basename(filePath);
+    res.setHeader('X-Picpeak-Contains-Secrets', 'true');
+    res.download(filePath, filename, (err) => {
+      // Best-effort cleanup of the temp .picpeak (and its temp dir) after send.
+      fsSync.rm(path.dirname(filePath), { recursive: true, force: true }, () => {});
+      if (err) logger.error('[picpeak-export] download failed', { error: err.message });
+    });
+  } catch (error) {
+    logger.error('[picpeak-export] failed to create export', { error: error.message });
+    if (!res.headersSent) res.status(500).json({ error: 'Failed to create .picpeak export' });
+  }
+});
+
 // Get backup run details
 router.get('/runs/:id', adminAuth, requirePermission('backup.view'), async (req, res) => {
   try {

backend/src/services/picpeakExportService.js

@@ -0,0 +1,228 @@
+'use strict';
+
+// Portable ".picpeak" export — a single, self-describing archive that can be
+// downloaded from one instance and re-uploaded to another via the web UI only
+// (see picpeakImportService for the receiving half).
+//
+// Deliberately ENGINE-NEUTRAL: instead of a native pg_dump / sqlite .backup
+// (which can only ever restore into the same engine and version), each table is
+// written as NDJSON. The target rebuilds its own schema by running migrations,
+// then loads these rows into it — so an older backup restores cleanly onto a
+// newer target (forward-only), and pg↔pg / sqlite↔sqlite both work.
+//
+// This module is purely additive: it introduces a new artifact and touches no
+// existing backup/restore path.
+
+const fs = require('fs');
+const fsp = require('fs').promises;
+const path = require('path');
+const os = require('os');
+const crypto = require('crypto');
+const archiver = require('archiver');
+const { db } = require('../database/db');
+const knexConfig = require('../../knexfile');
+const { getStoragePath } = require('../config/storage');
+const logger = require('../utils/logger');
+const packageJson = require('../../package.json');
+
+// Bump only on a breaking change to the on-disk layout below.
+const PICPEAK_FORMAT_VERSION = 1;
+
+// Never exported as data — the target owns these (its own migrations set them).
+const EXCLUDED_TABLES = new Set(['knex_migrations', 'knex_migrations_lock']);
+
+// Storage subdirs holding non-recalculable blobs — always included.
+const DOC_DIRS = ['business-docs', 'uploads'];
+// Original gallery photos — only when includePhotos is true (large; otherwise
+// the admin re-uploads originals per gallery and previews are re-rendered).
+const PHOTO_DIRS = ['events/active', 'events/archived'];
+
+const isPostgres = () => knexConfig.client === 'pg';
+
+// db.raw returns `{ rows: [...] }` on Postgres and a bare array on SQLite.
+const rawRows = (result) => (isPostgres() ? result.rows : result);
+
+// All user tables, minus knex bookkeeping. Introspected at runtime so the
+// export never rots as tables are added (no hardcoded list to maintain).
+async function listDataTables() {
+  let names;
+  if (isPostgres()) {
+    const result = await db.raw(`
+      SELECT table_name AS name
+      FROM information_schema.tables
+      WHERE table_schema = 'public' AND table_type = 'BASE TABLE'
+      ORDER BY table_name
+    `);
+    names = rawRows(result).map((r) => r.name);
+  } else {
+    const result = await db.raw(`
+      SELECT name FROM sqlite_master
+      WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
+      ORDER BY name
+    `);
+    names = rawRows(result).map((r) => r.name);
+  }
+  return names.filter((n) => !EXCLUDED_TABLES.has(n));
+}
+
+// The latest applied migration — recorded in the manifest so the importer can
+// refuse a backup that is NEWER than the target (forward-only guarantee).
+async function getLatestMigration() {
+  try {
+    const rows = await db('knex_migrations').orderBy('id', 'desc').limit(1);
+    return rows[0]?.name || null;
+  } catch (_) {
+    return null;
+  }
+}
+
+// Stream one table to <dataDir>/<table>.ndjson (one JSON object per line).
+// Returns { rowCount, checksum } for the manifest. JSON.stringify serialises
+// Dates to ISO strings, which re-import cleanly on both engines.
+async function writeTableNdjson(table, dataDir) {
+  const outPath = path.join(dataDir, `${table}.ndjson`);
+  const out = fs.createWriteStream(outPath, { encoding: 'utf8' });
+  const hash = crypto.createHash('sha256');
+  let rowCount = 0;
+
+  await new Promise((resolve, reject) => {
+    out.on('error', reject);
+    const stream = db(table).stream();
+    stream.on('error', reject);
+    stream.on('data', (row) => {
+      const line = `${JSON.stringify(row)}\n`;
+      hash.update(line);
+      rowCount += 1;
+      if (!out.write(line)) {
+        stream.pause();
+        out.once('drain', () => stream.resume());
+      }
+    });
+    stream.on('end', () => out.end(resolve));
+  });
+
+  return { rowCount, checksum: hash.digest('hex') };
+}
+
+// Recursively collect files under a storage subdir as { abs, rel } where rel is
+// relative to the storage root (so the importer restores the same layout).
+async function collectDir(subdir, storageRoot, acc) {
+  const abs = path.join(storageRoot, subdir);
+  let entries;
+  try {
+    entries = await fsp.readdir(abs, { withFileTypes: true });
+  } catch (_) {
+    return; // subdir may not exist on this install — skip silently
+  }
+  for (const entry of entries) {
+    const childRel = path.join(subdir, entry.name);
+    if (entry.isDirectory()) {
+      await collectDir(childRel, storageRoot, acc);
+    } else if (entry.isFile()) {
+      acc.push({ abs: path.join(storageRoot, childRel), rel: childRel });
+    }
+  }
+}
+
+async function collectFiles(includePhotos) {
+  const storageRoot = getStoragePath();
+  const dirs = includePhotos ? [...DOC_DIRS, ...PHOTO_DIRS] : [...DOC_DIRS];
+  const acc = [];
+  for (const d of dirs) {
+    await collectDir(d, storageRoot, acc);
+  }
+  return acc;
+}
+
+/**
+ * Build a .picpeak archive.
+ * @param {Object} opts
+ * @param {boolean} [opts.includePhotos=false] include original gallery photos
+ * @param {string}  [opts.outDir] where to write the file (defaults to a temp dir)
+ * @returns {Promise<{ filePath: string, manifest: object }>}
+ */
+async function createPicpeak({ includePhotos = false, outDir } = {}) {
+  const staging = await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-export-'));
+  const dataDir = path.join(staging, 'data');
+  await fsp.mkdir(dataDir, { recursive: true });
+
+  try {
+    // 1. Dump every table to NDJSON, tracking counts + checksums.
+    const tables = await listDataTables();
+    const tableMeta = {};
+    for (const table of tables) {
+      tableMeta[table] = await writeTableNdjson(table, dataDir);
+    }
+
+    // 2. Gather the non-recalculable blobs (PDFs, business-docs, uploads, and
+    //    optionally original photos).
+    const files = await collectFiles(includePhotos);
+
+    // 3. Manifest — everything the importer needs to validate + reconstruct.
+    const manifest = {
+      format: PICPEAK_FORMAT_VERSION,
+      kind: 'picpeak-backup',
+      created_at: new Date().toISOString(),
+      app_version: packageJson.version || null,
+      database: {
+        engine: isPostgres() ? 'pg' : 'sqlite',
+        latest_migration: await getLatestMigration(),
+      },
+      options: { includePhotos: !!includePhotos },
+      tables: tableMeta,
+      file_count: files.length,
+      // NOTE: contains secrets (SMTP password, admin hashes, API keys) in plain
+      // text — the download surface must warn about this.
+      contains_secrets: true,
+    };
+    await fsp.writeFile(
+      path.join(staging, 'manifest.json'),
+      JSON.stringify(manifest, null, 2),
+      'utf8'
+    );
+
+    // 4. Zip staging (manifest + data/) plus the blobs under files/. The final
+    //    .picpeak lands in outDir (caller-managed) or a fresh temp dir; either
+    //    way the NDJSON scratch (which holds plaintext secrets) is always
+    //    removed in `finally` below.
+    const targetDir = outDir || (await fsp.mkdtemp(path.join(os.tmpdir(), 'picpeak-out-')));
+    await fsp.mkdir(targetDir, { recursive: true });
+    const stamp = manifest.created_at.replace(/[:.]/g, '-');
+    const filePath = path.join(targetDir, `picpeak-backup-${stamp}.picpeak`);
+
+    await new Promise((resolve, reject) => {
+      const output = fs.createWriteStream(filePath);
+      const archive = archiver('zip', { zlib: { level: 9 } });
+      output.on('close', resolve);
+      output.on('error', reject);
+      archive.on('error', reject);
+      // Surface archiver warnings (e.g. a file vanished mid-run) instead of
+      // silently shipping an incomplete archive.
+      archive.on('warning', (err) => reject(err));
+      archive.pipe(output);
+      archive.file(path.join(staging, 'manifest.json'), { name: 'manifest.json' });
+      archive.directory(dataDir, 'data');
+      for (const f of files) {
+        archive.file(f.abs, { name: path.posix.join('files', f.rel.split(path.sep).join('/')) });
+      }
+      archive.finalize();
+    });
+
+    logger.info(
+      `[picpeak-export] wrote ${filePath} (${tables.length} tables, ${files.length} files, includePhotos=${!!includePhotos})`
+    );
+    return { filePath, manifest };
+  } finally {
+    // Always remove the NDJSON scratch dir — it contains a plaintext dump of
+    // every table (secrets included). The final .picpeak is elsewhere.
+    await fsp.rm(staging, { recursive: true, force: true }).catch(() => {});
+  }
+}
+
+module.exports = {
+  PICPEAK_FORMAT_VERSION,
+  createPicpeak,
+  // exported for reuse/testing
+  listDataTables,
+  collectFiles,
+};