Merge pull request #596 from Luca-Timo/bugfix/crm-backup
Backup & Restore hardening — close the silent files-only data-loss class
This commit is contained in:
Paul Nothaft
@@ -24,8 +24,73 @@ try {
|
||||
try { logger.warn('SQLite directory ensure failed', { error: e.message }); } catch (_) {}
|
||||
}
|
||||
|
||||
// Create database connection with built-in retry logic
|
||||
const db = knex(knexConfig);
|
||||
// Create database connection with built-in retry logic.
|
||||
//
|
||||
// The underlying knex instance is held in `_db` and reachable through a
|
||||
// Proxy `db` that forwards every call to the current instance. This
|
||||
// indirection exists so `reinitPool()` below can swap the live pool
|
||||
// without breaking the thousands of existing `const { db } = require(...)`
|
||||
// imports — they capture the Proxy once, and every subsequent
|
||||
// `db('table')` / `db.schema.hasTable(...)` lookup goes through the
|
||||
// Proxy to whatever `_db` currently points at.
|
||||
//
|
||||
// Used by the restore service after DROP/CREATE DATABASE: the old pool
|
||||
// is destroyed during the drop (to release PG connections so the DROP
|
||||
// can succeed), then `reinitPool()` opens a fresh pool against the
|
||||
// recreated DB. Without this, every query in the process after a
|
||||
// restore failed with "Unable to acquire a connection" until the
|
||||
// container was manually restarted — exactly the footgun Ralf hit
|
||||
// repeatedly on 2026-05-30.
|
||||
let _db = knex(knexConfig);
|
||||
|
||||
const db = new Proxy(function knexCall() {}, {
|
||||
// db('tableName') — knex's query builder entry point
|
||||
apply(_target, _thisArg, args) {
|
||||
return _db(...args);
|
||||
},
|
||||
// db.schema, db.raw, db.migrate, etc.
|
||||
get(_target, prop) {
|
||||
const v = _db[prop];
|
||||
return typeof v === 'function' ? v.bind(_db) : v;
|
||||
},
|
||||
// Defensive: future code that does `if ('schema' in db)` works.
|
||||
has(_target, prop) { return prop in _db; },
|
||||
});
|
||||
|
||||
/**
|
||||
* Tear down the current knex pool and open a fresh one.
|
||||
*
|
||||
* Idempotent — calling it twice in a row just destroys + recreates
|
||||
* twice, no error. Throws if the new pool can't establish a
|
||||
* connection (which surfaces a clear error rather than letting the
|
||||
* caller proceed with a half-broken pool).
|
||||
*
|
||||
* Used by restoreService after the DROP/CREATE DATABASE pair.
|
||||
*/
|
||||
async function reinitPool() {
|
||||
const prev = _db;
|
||||
try {
|
||||
await prev.destroy();
|
||||
} catch (err) {
|
||||
logger.warn(`Old pool destroy failed (continuing with reinit): ${err.message}`);
|
||||
}
|
||||
_db = knex(knexConfig);
|
||||
// Probe the new pool with a no-op query so we fail loudly here if
|
||||
// the new pool can't connect — better than silently handing the
|
||||
// caller a broken pool and surfacing the error on the next admin
|
||||
// request.
|
||||
try {
|
||||
if (knexConfig.client === 'pg') {
|
||||
await _db.raw('SELECT 1');
|
||||
} else {
|
||||
await _db.raw('SELECT 1');
|
||||
}
|
||||
} catch (probeErr) {
|
||||
logger.error('New pool failed health-check after reinit', { error: probeErr.message });
|
||||
throw probeErr;
|
||||
}
|
||||
logger.info('knex pool re-initialized successfully');
|
||||
}
|
||||
|
||||
// Connection retry configuration
|
||||
const MAX_RETRIES = 3;
|
||||
@@ -607,4 +672,4 @@ async function logActivity(activityType, metadata = {}, eventId = null, actor =
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { db, initializeDatabase, logActivity, withRetry };
|
||||
module.exports = { db, initializeDatabase, logActivity, withRetry, reinitPool };
|
||||
|
||||
@@ -13,7 +13,18 @@ let cachedTimeout = null;
|
||||
let cacheExpiry = 0;
|
||||
const CACHE_DURATION = 30 * 60 * 1000; // 30 minutes - reduced DB queries
|
||||
|
||||
// Clean up expired sessions every 5 minutes
|
||||
// Clean up expired sessions every 5 minutes.
|
||||
//
|
||||
// `.unref()` so this timer doesn't keep the event loop alive on its
|
||||
// own — without it, every jest worker that requires this module
|
||||
// (directly or transitively via server.js / a middleware-importing
|
||||
// route file) gets stuck and either prints the "worker failed to
|
||||
// exit gracefully" warning or, under high CI load, force-kills mid-
|
||||
// test and takes an unrelated suite down with it (we hit this with
|
||||
// integration/storageBackend.test.js on PR #555). Production
|
||||
// behaviour is unchanged: the timer fires every 5 min as long as
|
||||
// the server has anything else keeping the loop alive (HTTP server,
|
||||
// other intervals), which is always.
|
||||
setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const [token, lastActivity] of sessions.entries()) {
|
||||
@@ -21,7 +32,7 @@ setInterval(() => {
|
||||
sessions.delete(token);
|
||||
}
|
||||
}
|
||||
}, 5 * 60 * 1000);
|
||||
}, 5 * 60 * 1000).unref();
|
||||
|
||||
async function getSessionTimeout() {
|
||||
const now = Date.now();
|
||||
|
||||
+269
-101
@@ -348,51 +348,7 @@ router.get('/run/:id/report', requirePermission('backup.view'), async (req, res)
|
||||
*/
|
||||
router.get('/available-backups', requirePermission('backup.view'), async (req, res) => {
|
||||
try {
|
||||
const backups = [];
|
||||
|
||||
// Get local file backups
|
||||
const backupConfig = await getBackupConfig();
|
||||
if (backupConfig.backup_destination_type === 'local' && backupConfig.backup_destination_path) {
|
||||
try {
|
||||
const files = await fs.readdir(backupConfig.backup_destination_path);
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json') || file.endsWith('.yaml')) {
|
||||
const filePath = path.join(backupConfig.backup_destination_path, file);
|
||||
const stats = await fs.stat(filePath);
|
||||
backups.push({
|
||||
type: 'local',
|
||||
name: file,
|
||||
path: filePath,
|
||||
size: stats.size,
|
||||
modified: stats.mtime
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to list local backups:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Get database backups from backup_runs table
|
||||
const backupRuns = await db('backup_runs')
|
||||
.where('status', 'completed')
|
||||
.whereNotNull('manifest_path')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.limit(20);
|
||||
|
||||
for (const run of backupRuns) {
|
||||
backups.push({
|
||||
type: run.manifest_path.startsWith('s3://') ? 's3' : 'local',
|
||||
name: `Backup ${run.completed_at}`,
|
||||
path: run.manifest_path,
|
||||
manifestId: run.manifest_id,
|
||||
size: run.total_size_bytes,
|
||||
filesCount: run.files_backed_up,
|
||||
duration: run.duration_seconds,
|
||||
completed: run.completed_at
|
||||
});
|
||||
}
|
||||
|
||||
const backups = await discoverAvailableBackups();
|
||||
res.json({
|
||||
success: true,
|
||||
data: backups
|
||||
@@ -406,6 +362,213 @@ router.get('/available-backups', requirePermission('backup.view'), async (req, r
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Discover restorable backups by walking the configured destination
|
||||
* directory + harvesting the backup_runs table.
|
||||
*
|
||||
* **Why we recurse the disk first, DB second**
|
||||
*
|
||||
* The disk is the source of truth for restore. After a disaster
|
||||
* (`docker compose down -v`, drive corruption, fresh install) the
|
||||
* `backup_runs` table is empty — but the manifest JSONs are exactly
|
||||
* what's left on disk for an admin to recover from. A wizard that
|
||||
* only reads the DB shows "No backups found" precisely when the
|
||||
* admin needs it most. So we walk first, dedupe-by-manifestId
|
||||
* against any surviving DB rows, and present a unified list.
|
||||
*
|
||||
* Discovery rules:
|
||||
* - Walks `backup_destination_path` AND `backup_manifest_path` if
|
||||
* they're distinct (manifests can live in a sibling directory).
|
||||
* - Recurses up to 3 levels deep — enough to find
|
||||
* `<root>/manifests/backup-manifest-<id>.json` (the default
|
||||
* layout) without scanning the entire photo tree.
|
||||
* - Matches manifest files by glob: `backup-manifest-*.json`,
|
||||
* `backup-manifest-*.yaml`, and the legacy bare `manifest.json`.
|
||||
* - Parses each manifest to extract real metadata (timestamp,
|
||||
* size, file count, source type) instead of showing the admin
|
||||
* a list of opaque filenames.
|
||||
*
|
||||
* Returns: array of `{ type, name, path, manifestId, size,
|
||||
* filesCount, completed, source: 'disk' | 'db' }`.
|
||||
*/
|
||||
async function discoverAvailableBackups() {
|
||||
const backupConfig = await getBackupConfig();
|
||||
const backups = [];
|
||||
const seenManifestIds = new Set();
|
||||
|
||||
if (backupConfig.backup_destination_type === 'local') {
|
||||
const roots = new Set();
|
||||
if (backupConfig.backup_destination_path) roots.add(backupConfig.backup_destination_path);
|
||||
if (backupConfig.backup_manifest_path) roots.add(backupConfig.backup_manifest_path);
|
||||
|
||||
for (const root of roots) {
|
||||
try {
|
||||
const manifestPaths = await walkForManifests(root, 3);
|
||||
for (const filePath of manifestPaths) {
|
||||
try {
|
||||
const parsed = await parseManifestMetadata(filePath);
|
||||
if (parsed.manifestId) seenManifestIds.add(parsed.manifestId);
|
||||
backups.push(parsed);
|
||||
} catch (err) {
|
||||
// Don't fail discovery because ONE manifest is corrupt —
|
||||
// surface the file with a note so the admin sees something
|
||||
// is wrong and can investigate.
|
||||
logger.warn(`Manifest unreadable at ${filePath}: ${err.message}`);
|
||||
const stats = await fs.stat(filePath).catch(() => null);
|
||||
backups.push({
|
||||
type: 'local',
|
||||
name: path.basename(filePath),
|
||||
path: filePath,
|
||||
manifestId: null,
|
||||
size: stats?.size || 0,
|
||||
filesCount: null,
|
||||
completed: stats?.mtime || null,
|
||||
source: 'disk',
|
||||
corrupt: true,
|
||||
error: err.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`Could not scan backup root ${root}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Layer in surviving DB rows, deduping by manifest_id so we don't
|
||||
// show the same backup twice with different shapes.
|
||||
const backupRuns = await db('backup_runs')
|
||||
.where('status', 'completed')
|
||||
.whereNotNull('manifest_path')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.limit(50);
|
||||
|
||||
for (const run of backupRuns) {
|
||||
if (run.manifest_id && seenManifestIds.has(run.manifest_id)) continue;
|
||||
backups.push({
|
||||
type: run.manifest_path.startsWith('s3://') ? 's3' : 'local',
|
||||
name: `Backup ${run.completed_at}`,
|
||||
path: run.manifest_path,
|
||||
manifestId: run.manifest_id,
|
||||
size: run.total_size_bytes,
|
||||
filesCount: run.files_backed_up,
|
||||
duration: run.duration_seconds,
|
||||
completed: run.completed_at,
|
||||
source: 'db',
|
||||
});
|
||||
}
|
||||
|
||||
// Most recent first.
|
||||
backups.sort((a, b) => {
|
||||
const aTime = a.completed ? new Date(a.completed).getTime() : 0;
|
||||
const bTime = b.completed ? new Date(b.completed).getTime() : 0;
|
||||
return bTime - aTime;
|
||||
});
|
||||
|
||||
return backups;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursive manifest finder. Depth-limited so we don't enumerate
|
||||
* thousands of photo files. Yields absolute paths.
|
||||
*/
|
||||
async function walkForManifests(dir, maxDepth, depth = 0) {
|
||||
if (depth > maxDepth) return [];
|
||||
let entries;
|
||||
try {
|
||||
entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') return [];
|
||||
throw err;
|
||||
}
|
||||
|
||||
const out = [];
|
||||
for (const entry of entries) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
// Skip obvious noise: photo trees, node_modules, hidden dirs.
|
||||
if (entry.name === 'events' || entry.name === 'business-docs'
|
||||
|| entry.name === 'thumbnails' || entry.name === 'previews'
|
||||
|| entry.name === 'heroes' || entry.name === 'uploads'
|
||||
|| entry.name.startsWith('.')
|
||||
|| entry.name === 'node_modules') continue;
|
||||
out.push(...await walkForManifests(full, maxDepth, depth + 1));
|
||||
} else if (entry.isFile() && isManifestFilename(entry.name)) {
|
||||
out.push(full);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function isManifestFilename(name) {
|
||||
// Canonical: backup-manifest-<id>.json / .yaml
|
||||
// Legacy: manifest.json (inside backup-<id>/manifest.json layout)
|
||||
// Be liberal in what we accept — admin may have renamed.
|
||||
if (/^backup-manifest-.+\.(json|ya?ml)$/i.test(name)) return true;
|
||||
if (/^manifest\.(json|ya?ml)$/i.test(name)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a manifest file and pull out the fields the wizard wants.
|
||||
* Tolerates schema drift across manifest versions (v1, v2) by
|
||||
* checking multiple shapes.
|
||||
*/
|
||||
async function parseManifestMetadata(filePath) {
|
||||
const raw = await fs.readFile(filePath, 'utf8');
|
||||
let parsed;
|
||||
if (filePath.toLowerCase().endsWith('.json')) {
|
||||
parsed = JSON.parse(raw);
|
||||
} else {
|
||||
// Minimal YAML support — most admins use JSON; only do require()
|
||||
// if a .yaml manifest is actually present.
|
||||
const yaml = require('js-yaml');
|
||||
parsed = yaml.load(raw);
|
||||
}
|
||||
|
||||
const stats = await fs.stat(filePath);
|
||||
|
||||
// v2 shape: { manifest: { id, timestamp }, backup: { ... }, files: [...], database: { ... } }
|
||||
// v1 shape: { backup_id, started_at, files: [...] } (older)
|
||||
const manifestId =
|
||||
parsed?.manifest?.id
|
||||
|| parsed?.backup?.id
|
||||
|| parsed?.backup_id
|
||||
|| null;
|
||||
|
||||
const completed =
|
||||
parsed?.backup?.completed_at
|
||||
|| parsed?.manifest?.timestamp
|
||||
|| parsed?.completed_at
|
||||
|| stats.mtime;
|
||||
|
||||
const filesCount =
|
||||
(Array.isArray(parsed?.files) ? parsed.files.length : null)
|
||||
?? parsed?.backup?.total_files
|
||||
?? null;
|
||||
|
||||
const totalSizeBytes =
|
||||
parsed?.backup?.total_size_bytes
|
||||
?? parsed?.total_size_bytes
|
||||
?? null;
|
||||
|
||||
return {
|
||||
type: 'local',
|
||||
name: path.basename(filePath),
|
||||
path: filePath,
|
||||
manifestId,
|
||||
size: totalSizeBytes ?? stats.size,
|
||||
filesCount,
|
||||
completed,
|
||||
source: 'disk',
|
||||
databaseIncluded: Boolean(parsed?.database?.backup_file),
|
||||
// Helpful for the UI: lets it show "This backup has no DB" warning
|
||||
// — exactly the surface that would have caught Ralf's original
|
||||
// four files-only manifests if it had existed.
|
||||
schemaVersion: parsed?.manifest?.version || parsed?.version || '1.0',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* List backups for restore (POST version for frontend compatibility)
|
||||
* Accepts source type in request body
|
||||
@@ -413,59 +576,58 @@ router.get('/available-backups', requirePermission('backup.view'), async (req, r
|
||||
router.post('/list-backups', requirePermission('backup.view'), async (req, res) => {
|
||||
try {
|
||||
const { source } = req.body; // 'local', 's3', or undefined for all
|
||||
const backups = [];
|
||||
|
||||
// Get backup configuration
|
||||
const backupConfig = await getBackupConfig();
|
||||
// Use the same disk-first discovery the GET endpoint uses so that
|
||||
// a fresh post-`docker compose down -v` install (empty backup_runs
|
||||
// table) can still see what's on disk. The whole point of restore
|
||||
// is "the DB is broken, rebuild it from disk" — a wizard that
|
||||
// only queries the DB shows "No backups found" exactly when it's
|
||||
// needed most. See discoverAvailableBackups for the full rationale.
|
||||
const discovered = await discoverAvailableBackups();
|
||||
|
||||
// Get database backups from backup_runs table
|
||||
const backupRuns = await db('backup_runs')
|
||||
.where('status', 'completed')
|
||||
.whereNotNull('manifest_path')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.limit(20);
|
||||
const filtered = source
|
||||
? discovered.filter((b) => b.type === source)
|
||||
: discovered;
|
||||
|
||||
for (const run of backupRuns) {
|
||||
const isS3 = run.manifest_path.startsWith('s3://');
|
||||
const backupType = isS3 ? 's3' : 'local';
|
||||
|
||||
// Filter by source if specified
|
||||
if (source && source !== backupType) {
|
||||
continue;
|
||||
}
|
||||
|
||||
backups.push({
|
||||
id: run.id,
|
||||
type: backupType,
|
||||
name: `Backup from ${new Date(run.completed_at).toLocaleString()}`,
|
||||
path: run.manifest_path,
|
||||
manifest_path: run.manifest_path,
|
||||
manifestId: run.manifest_id,
|
||||
manifestPath: run.manifest_path,
|
||||
size: parseInt(run.total_size_bytes) || 0,
|
||||
total_size: parseInt(run.total_size_bytes) || 0,
|
||||
total_size_bytes: parseInt(run.total_size_bytes) || 0,
|
||||
filesCount: run.files_backed_up || 0,
|
||||
files_backed_up: run.files_backed_up || 0,
|
||||
duration: run.duration_seconds,
|
||||
duration_seconds: run.duration_seconds,
|
||||
// Frontend expects snake_case date fields
|
||||
created_at: run.completed_at,
|
||||
completed_at: run.completed_at,
|
||||
started_at: run.started_at,
|
||||
// camelCase aliases
|
||||
completedAt: run.completed_at,
|
||||
startedAt: run.started_at,
|
||||
// Backup metadata
|
||||
status: run.status,
|
||||
backup_type: run.backup_type,
|
||||
backupType: run.backup_type,
|
||||
backup_mode: run.backup_mode,
|
||||
backupMode: run.backup_mode,
|
||||
app_version: run.app_version,
|
||||
appVersion: run.app_version
|
||||
});
|
||||
}
|
||||
// Shape for frontend compatibility — preserves every alias the
|
||||
// frontend was already reading (snake_case + camelCase), so the
|
||||
// UI rendering doesn't have to change.
|
||||
const backups = filtered.map((b) => ({
|
||||
id: b.manifestId || null,
|
||||
type: b.type,
|
||||
name: b.completed
|
||||
? `Backup from ${new Date(b.completed).toLocaleString()}`
|
||||
: b.name,
|
||||
path: b.path,
|
||||
manifest_path: b.path,
|
||||
manifestId: b.manifestId,
|
||||
manifestPath: b.path,
|
||||
size: parseInt(b.size) || 0,
|
||||
total_size: parseInt(b.size) || 0,
|
||||
total_size_bytes: parseInt(b.size) || 0,
|
||||
filesCount: b.filesCount || 0,
|
||||
files_backed_up: b.filesCount || 0,
|
||||
duration: b.duration || null,
|
||||
duration_seconds: b.duration || null,
|
||||
created_at: b.completed,
|
||||
completed_at: b.completed,
|
||||
started_at: b.completed,
|
||||
completedAt: b.completed,
|
||||
startedAt: b.completed,
|
||||
status: 'completed',
|
||||
// Stage A-aware: when the source is a disk-scanned manifest we
|
||||
// can tell the wizard whether the DB dump is present, so the
|
||||
// UI can warn before the admin picks a files-only backup.
|
||||
database_included: b.databaseIncluded,
|
||||
databaseIncluded: b.databaseIncluded,
|
||||
corrupt: b.corrupt || false,
|
||||
// Provenance: 'disk' (manifest read from filesystem) vs 'db'
|
||||
// (backup_runs row that the disk didn't surface) — useful for
|
||||
// debugging which side is missing.
|
||||
source: b.source,
|
||||
schema_version: b.schemaVersion,
|
||||
schemaVersion: b.schemaVersion,
|
||||
}));
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
@@ -556,17 +718,23 @@ async function getRestoreSettings() {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'restore')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
|
||||
const result = {};
|
||||
settings.forEach(setting => {
|
||||
// Convert boolean strings to actual booleans
|
||||
if (setting.setting_value === '1' || setting.setting_value === '0') {
|
||||
result[setting.setting_key] = setting.setting_value === '1';
|
||||
// Boolean-string normalization. Historically only handled '1'/'0',
|
||||
// but other code paths (boot self-heal, admin UI, direct SQL) write
|
||||
// 'true' / 'false' or JSON-encoded "true" / "false". Accept all four
|
||||
// shapes so `!settings.<key>` evaluates correctly downstream.
|
||||
const raw = setting.setting_value;
|
||||
if (raw === '1' || raw === 'true' || raw === '"true"') {
|
||||
result[setting.setting_key] = true;
|
||||
} else if (raw === '0' || raw === 'false' || raw === '"false"') {
|
||||
result[setting.setting_key] = false;
|
||||
} else {
|
||||
result[setting.setting_key] = setting.setting_value;
|
||||
result[setting.setting_key] = raw;
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* Admin → System Health
|
||||
*
|
||||
* Endpoint mounted at /api/admin/system-health. The "Backup
|
||||
* integrity" sub-endpoint is the on-demand verifier for CRM
|
||||
* document artefacts — confirms every `*_path` column on quotes /
|
||||
* contracts / invoices points at a file that actually exists on
|
||||
* disk and (where a `*_sha256` column is set) the file's bytes
|
||||
* still hash to the expected value.
|
||||
*
|
||||
* Per the design decisions locked with the maintainer:
|
||||
* - On-demand only; no scheduler (D1)
|
||||
* - Not auto-triggered after restore (D2)
|
||||
* - Wet-upload contracts are hash-verified same as system-rendered (D3)
|
||||
*
|
||||
* Read-only. Returns a JSON report — never mutates DB or fs.
|
||||
*/
|
||||
|
||||
const express = require('express');
|
||||
const { query } = require('express-validator');
|
||||
const { adminAuth } = require('../middleware/auth');
|
||||
const { requirePermission } = require('../middleware/permissions');
|
||||
const { handleAsync, validateRequest, successResponse } = require('../utils/routeHelpers');
|
||||
const { verifyDocumentArtefacts } = require('../services/backupIntegrityService');
|
||||
const { getCoverageReport } = require('../services/backupCoverageService');
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.use(adminAuth);
|
||||
|
||||
const VALID_SCOPES = ['quote', 'contract', 'contract-signature', 'invoice'];
|
||||
|
||||
router.get(
|
||||
'/backup-integrity',
|
||||
requirePermission('settings.view'),
|
||||
[
|
||||
// CSV string like `?scope=contract,invoice`. Each member must be
|
||||
// one of the four known scopes. Empty / omitted means full scan.
|
||||
query('scope').optional({ values: 'falsy' }).isString().isLength({ max: 128 }),
|
||||
],
|
||||
handleAsync(async (req, res) => {
|
||||
validateRequest(req);
|
||||
let scope;
|
||||
if (req.query.scope) {
|
||||
scope = String(req.query.scope)
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
// Defense-in-depth: reject unknown scope tokens so a typo doesn't
|
||||
// silently scan everything when the caller wanted just one slice.
|
||||
const unknown = scope.filter((s) => !VALID_SCOPES.includes(s));
|
||||
if (unknown.length > 0) {
|
||||
return res.status(400).json({
|
||||
error: `Unknown scope(s): ${unknown.join(', ')}`,
|
||||
code: 'BACKUP_INTEGRITY_UNKNOWN_SCOPE',
|
||||
validScopes: VALID_SCOPES,
|
||||
});
|
||||
}
|
||||
}
|
||||
const report = await verifyDocumentArtefacts({ scope });
|
||||
return successResponse(res, { report });
|
||||
}),
|
||||
);
|
||||
|
||||
/**
|
||||
* GET /api/admin/system-health/backup-coverage
|
||||
*
|
||||
* Stage C of the backup-hardening plan. Returns the data-driven
|
||||
* coverage report — what the next "Run Backup Now" will include /
|
||||
* skip / silently miss, plus the database-dump status block.
|
||||
*
|
||||
* Read-only, on-demand. No scope parameter — the report is cheap
|
||||
* (only top-level directory listing under STORAGE_PATH, no recursion).
|
||||
*
|
||||
* See backupCoverageService.js for the full rationale and the
|
||||
* coverage-classification rules.
|
||||
*/
|
||||
router.get(
|
||||
'/backup-coverage',
|
||||
requirePermission('settings.view'),
|
||||
handleAsync(async (req, res) => {
|
||||
const report = await getCoverageReport();
|
||||
return successResponse(res, { report });
|
||||
}),
|
||||
);
|
||||
|
||||
module.exports = router;
|
||||
@@ -201,14 +201,22 @@ describe('DatabaseBackupService', () => {
|
||||
delete: jest.fn().mockResolvedValue(1)
|
||||
});
|
||||
|
||||
// Mock fs.unlink
|
||||
fs.unlink = jest.fn().mockResolvedValue(undefined);
|
||||
|
||||
// Stub fs.promises.unlink via jest.spyOn so the original is
|
||||
// restored when the test finishes. The previous form
|
||||
// (`fs.unlink = jest.fn()`) leaked into every test that ran
|
||||
// after this one in the same jest worker — most visibly
|
||||
// integration/storageBackend.test.js, whose LocalFsStorage
|
||||
// delete() became a silent no-op and the subsequent
|
||||
// exists() assertion flipped from false to true. spyOn +
|
||||
// mockRestore in afterEach keeps the stub scoped to this test.
|
||||
const unlinkSpy = jest.spyOn(fs, 'unlink').mockResolvedValue(undefined);
|
||||
|
||||
await service.cleanupOldBackups(30);
|
||||
|
||||
expect(fs.unlink).toHaveBeenCalledTimes(2);
|
||||
expect(fs.unlink).toHaveBeenCalledWith('/backup/old1.sql.gz');
|
||||
expect(fs.unlink).toHaveBeenCalledWith('/backup/old2.sql.gz');
|
||||
|
||||
expect(unlinkSpy).toHaveBeenCalledTimes(2);
|
||||
expect(unlinkSpy).toHaveBeenCalledWith('/backup/old1.sql.gz');
|
||||
expect(unlinkSpy).toHaveBeenCalledWith('/backup/old2.sql.gz');
|
||||
unlinkSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* Boot-time self-heal for the `backup_paths` table.
|
||||
*
|
||||
* **Why this exists**
|
||||
*
|
||||
* Knex won't re-run an applied migration, so once migration
|
||||
* 109_add_backup_paths.js has run, any later default we want to add
|
||||
* (a new subdirectory shipped by a future feature) would never reach
|
||||
* already-deployed installs. The historical fix for this kind of
|
||||
* "schema is fine, seed drifted" problem is the boot-time self-heal
|
||||
* pattern documented in [[feedback_self_heal_pattern]] — we just
|
||||
* re-apply the canonical seed on every boot with `onConflict.ignore()`
|
||||
* so admin edits stay intact and new rows trickle in.
|
||||
*
|
||||
* **Authoritative list**
|
||||
*
|
||||
* The list of defaults lives on migration 109 itself
|
||||
* (`DEFAULT_PATHS` export) — one source of truth that both the
|
||||
* migration and this seeder read. Tests assert these two stay in
|
||||
* lockstep.
|
||||
*
|
||||
* **Failure semantics**
|
||||
*
|
||||
* If the table doesn't exist yet (migrations haven't run, fresh
|
||||
* install before migration 109 lands, etc.) we no-op and log. The
|
||||
* walker has a hard-coded `LEGACY_DEFAULTS` fallback for the same
|
||||
* reason — defense in depth so "Run Backup Now" can never silently
|
||||
* ship a files-only manifest because of a seed issue. See
|
||||
* `backupService.js` getFilesToBackupInternal.
|
||||
*/
|
||||
|
||||
const { DEFAULT_PATHS } = require('../../migrations/core/109_add_backup_paths');
|
||||
|
||||
let booted = false;
|
||||
|
||||
/**
|
||||
* Idempotently re-seed `backup_paths` with the canonical defaults.
|
||||
*
|
||||
* @param {object} db knex instance
|
||||
* @param {object} logger app logger (must expose .info / .warn)
|
||||
* @returns {Promise<{ seeded: string[] }>} paths newly inserted on this boot.
|
||||
*/
|
||||
async function seedBackupPathsAtBoot(db, logger) {
|
||||
const log = logger || { info: () => {}, warn: () => {} };
|
||||
if (booted) return { seeded: [] };
|
||||
|
||||
if (!(await db.schema.hasTable('backup_paths'))) {
|
||||
log.warn('backup_paths table missing at boot — self-heal skipped (migration 109 may not have run yet)');
|
||||
return { seeded: [] };
|
||||
}
|
||||
|
||||
// Diff: which canonical paths are missing from the table right now?
|
||||
// We can't easily get "what got inserted by onConflict.ignore" out of
|
||||
// knex on both backends, so we just compute the diff ourselves and log
|
||||
// it — admins benefit from seeing exactly what got auto-added when a
|
||||
// new feature ships.
|
||||
const existing = await db('backup_paths').select('path');
|
||||
const existingSet = new Set(existing.map((r) => r.path));
|
||||
const missing = DEFAULT_PATHS.filter((p) => !existingSet.has(p.path));
|
||||
|
||||
if (missing.length === 0) {
|
||||
booted = true;
|
||||
return { seeded: [] };
|
||||
}
|
||||
|
||||
try {
|
||||
await db('backup_paths')
|
||||
.insert(missing.map((row) => ({
|
||||
...row,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
})))
|
||||
.onConflict('path')
|
||||
.ignore();
|
||||
log.info(`backup_paths self-heal added ${missing.length} row(s): ${missing.map((m) => m.path).join(', ')}`);
|
||||
} catch (err) {
|
||||
log.warn(`backup_paths self-heal failed: ${err.message}`);
|
||||
}
|
||||
|
||||
booted = true;
|
||||
return { seeded: missing.map((m) => m.path) };
|
||||
}
|
||||
|
||||
// Test-only: reset the module-level boot flag so jest can re-exercise
|
||||
// the seeder against a fresh test DB inside a single worker.
|
||||
function _resetBootForTests() {
|
||||
booted = false;
|
||||
}
|
||||
|
||||
module.exports = { seedBackupPathsAtBoot, _resetBootForTests };
|
||||
@@ -0,0 +1,242 @@
|
||||
/**
|
||||
* Install-from-backup boot hook.
|
||||
*
|
||||
* **The problem this closes**
|
||||
*
|
||||
* Without this hook, recovering a picpeak install from a backup is a
|
||||
* six-step process:
|
||||
* 1. Stand up the compose stack with empty volumes
|
||||
* 2. Wait for boot → land on the onboarding wizard
|
||||
* 3. Create a throwaway fresh-install admin
|
||||
* 4. Navigate to Backup → Restore
|
||||
* 5. Walk through the wizard with Force Restore ticked
|
||||
* 6. Log out, log back in with original (pre-disaster) credentials
|
||||
*
|
||||
* With this hook, admins skip steps 2-6. They place their backup
|
||||
* artefacts in the existing bind-mounted /backup directory, drop a
|
||||
* trigger file alongside, and the next container start runs the
|
||||
* restore BEFORE creating the throwaway onboarding admin. Server
|
||||
* comes up populated, original login works first try.
|
||||
*
|
||||
* **The trigger file convention (chosen for zero compose-file changes)**
|
||||
*
|
||||
* Drop a file named `RESTORE_ON_INSTALL` (no extension, or .txt) into
|
||||
* the root of the `/backup` mount. Two payload variants:
|
||||
*
|
||||
* 1. EMPTY file (or pure whitespace) — auto-pick the newest
|
||||
* `backup-manifest-*.json` from `/backup/manifests/`. Useful when
|
||||
* the admin doesn't know or care which one is most recent.
|
||||
*
|
||||
* 2. NON-EMPTY file containing a relative or absolute path to a
|
||||
* specific manifest. Trimmed; first line wins. Useful when the
|
||||
* admin wants a specific older backup.
|
||||
*
|
||||
* After a successful restore the trigger file is DELETED so the next
|
||||
* boot doesn't re-trigger. On failure the file is preserved + the
|
||||
* error is logged, so the admin can fix the input and retry by just
|
||||
* restarting the container.
|
||||
*
|
||||
* **Safety**
|
||||
*
|
||||
* Three layers gate this against accidental data loss:
|
||||
* 1. Trigger file must exist (intentional admin action, not auto-magic)
|
||||
* 2. DB must be empty — admin_users.count = 0 AND events.count = 0.
|
||||
* If either is non-zero, the hook refuses to run.
|
||||
* 3. The Stage A restore path (with all of tonight's fixes) handles
|
||||
* the actual swap atomically. If anything fails, the rollback
|
||||
* runs and the install stays in fresh-install state.
|
||||
*
|
||||
* Override: `INSTALL_FROM_BACKUP_FORCE=true` skips the empty-DB check
|
||||
* for the "I know what I'm doing" edge case (e.g. dev environment
|
||||
* rebuilds where there's leftover data that's safe to clobber).
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const TRIGGER_FILENAMES = ['RESTORE_ON_INSTALL', 'RESTORE_ON_INSTALL.txt'];
|
||||
|
||||
/**
|
||||
* Resolve which file should be treated as the trigger. Returns
|
||||
* `{ triggerPath, manifestPath }` if found, or null if no trigger
|
||||
* file is present (the common case — most boots).
|
||||
*/
|
||||
async function findTrigger(backupRoot, logger) {
|
||||
for (const name of TRIGGER_FILENAMES) {
|
||||
const triggerPath = path.join(backupRoot, name);
|
||||
if (fs.existsSync(triggerPath)) {
|
||||
let payload;
|
||||
try {
|
||||
payload = fs.readFileSync(triggerPath, 'utf8').trim();
|
||||
} catch (err) {
|
||||
logger.warn(`Install-from-backup: trigger file ${triggerPath} is unreadable: ${err.message}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!payload) {
|
||||
// Auto-pick the newest manifest
|
||||
const manifestsDir = path.join(backupRoot, 'manifests');
|
||||
if (!fs.existsSync(manifestsDir)) {
|
||||
logger.warn(`Install-from-backup: trigger file found but ${manifestsDir} doesn't exist`);
|
||||
return null;
|
||||
}
|
||||
const entries = fs.readdirSync(manifestsDir)
|
||||
.filter((f) => /^backup-manifest-.+\.(json|ya?ml)$/i.test(f))
|
||||
.map((f) => {
|
||||
const full = path.join(manifestsDir, f);
|
||||
return { full, mtime: fs.statSync(full).mtimeMs };
|
||||
})
|
||||
.sort((a, b) => b.mtime - a.mtime);
|
||||
if (entries.length === 0) {
|
||||
logger.warn(`Install-from-backup: no manifests found in ${manifestsDir}`);
|
||||
return null;
|
||||
}
|
||||
return { triggerPath, manifestPath: entries[0].full };
|
||||
}
|
||||
|
||||
// Take the first non-empty line as the manifest path
|
||||
const firstLine = payload.split(/\r?\n/).find((l) => l.trim()) || '';
|
||||
const manifestPath = path.isAbsolute(firstLine)
|
||||
? firstLine
|
||||
: path.join(backupRoot, firstLine);
|
||||
|
||||
if (!fs.existsSync(manifestPath)) {
|
||||
logger.warn(`Install-from-backup: trigger file points at ${manifestPath} which doesn't exist`);
|
||||
return null;
|
||||
}
|
||||
return { triggerPath, manifestPath };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the DB is empty enough that restoring on top is safe.
|
||||
* Returns `true` if safe, `false` if there's existing data.
|
||||
*/
|
||||
async function isDatabaseFresh(db, logger) {
|
||||
try {
|
||||
if (!(await db.schema.hasTable('admin_users'))) {
|
||||
// No admin_users table yet — schema is mid-migration or wholly
|
||||
// empty. Definitely safe to restore on top.
|
||||
return true;
|
||||
}
|
||||
const adminCount = await db('admin_users').count('* as c').first();
|
||||
const adminN = Number(adminCount?.c || 0);
|
||||
|
||||
let eventN = 0;
|
||||
if (await db.schema.hasTable('events')) {
|
||||
const eventCount = await db('events').count('* as c').first();
|
||||
eventN = Number(eventCount?.c || 0);
|
||||
}
|
||||
|
||||
if (adminN > 1 || eventN > 0) {
|
||||
logger.warn(
|
||||
`Install-from-backup: refusing — install has ${adminN} admin(s) and ${eventN} event(s). `
|
||||
+ 'This guard prevents accidental clobbering of production data. '
|
||||
+ 'Override with INSTALL_FROM_BACKUP_FORCE=true if you really want to restore on top.'
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
// adminN === 1 is the "fresh install ran migration 001 and auto-created
|
||||
// the default admin" case. That admin is throwaway — the restore will
|
||||
// replace it with the backup's admin row. So we treat 1 admin + 0
|
||||
// events as fresh.
|
||||
return true;
|
||||
} catch (err) {
|
||||
logger.warn(`Install-from-backup: fresh-install check threw: ${err.message}. Assuming NOT fresh.`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Public entry point — called from server.js after migrations and
|
||||
* before startServer.
|
||||
*
|
||||
* @returns {Promise<{ ran: boolean, manifestPath?: string, error?: string }>}
|
||||
*/
|
||||
async function tryInstallFromBackup(db, logger) {
|
||||
const log = logger || { info: () => {}, warn: () => {}, error: () => {} };
|
||||
|
||||
// The container's winston logger writes to `/app/logs/combined.log`
|
||||
// by default and may not always tee to stdout, so admins running
|
||||
// `docker logs picpeak-beta-backend` after a `compose up` would
|
||||
// see no signal that a restore happened — flagged on PR #596
|
||||
// review. We mirror the key trigger / start / end lines to
|
||||
// console.log as well so the docker-logs surface tells the story
|
||||
// without needing to exec into the container.
|
||||
const announce = (msg) => {
|
||||
try { console.log(`[install-from-backup] ${msg}`); } catch (_) { /* defensive */ }
|
||||
};
|
||||
|
||||
const backupRoot = process.env.BACKUP_ROOT || '/backup';
|
||||
if (!fs.existsSync(backupRoot)) {
|
||||
return { ran: false };
|
||||
}
|
||||
|
||||
const trigger = await findTrigger(backupRoot, log);
|
||||
if (!trigger) {
|
||||
return { ran: false };
|
||||
}
|
||||
|
||||
log.info(`Install-from-backup: trigger file found at ${trigger.triggerPath}, target manifest ${trigger.manifestPath}`);
|
||||
announce(`trigger file detected → ${trigger.manifestPath}`);
|
||||
|
||||
const forceOverride = process.env.INSTALL_FROM_BACKUP_FORCE === 'true';
|
||||
const isFresh = await isDatabaseFresh(db, log);
|
||||
if (!isFresh && !forceOverride) {
|
||||
log.warn('Install-from-backup: skipping. Trigger file left in place so you can correct + retry.');
|
||||
announce('skipping — install has existing data and INSTALL_FROM_BACKUP_FORCE is not set');
|
||||
return { ran: false, error: 'Database not empty' };
|
||||
}
|
||||
|
||||
log.info(`Install-from-backup: restoring from ${trigger.manifestPath}...`);
|
||||
announce(`starting restore from ${trigger.manifestPath}`);
|
||||
|
||||
try {
|
||||
const { restoreService } = require('./restoreService');
|
||||
const result = await restoreService.restore({
|
||||
source: 'local',
|
||||
manifestPath: trigger.manifestPath,
|
||||
restoreType: 'full',
|
||||
// Force=true because the fresh-install admin auto-created by
|
||||
// migration 001 trips the "1 active admin" warning — we WANT to
|
||||
// override that warning, since replacing the throwaway admin
|
||||
// with the backup's admin is exactly the goal.
|
||||
force: true,
|
||||
// SkipPreBackup=true because backing up an empty install is
|
||||
// pointless. Saves a few seconds and reduces disk noise.
|
||||
skipPreBackup: true,
|
||||
operator: {
|
||||
type: 'install-from-backup',
|
||||
userId: null,
|
||||
ip: null,
|
||||
},
|
||||
});
|
||||
|
||||
if (result?.success === false) {
|
||||
throw new Error(result?.error || 'Restore service reported failure');
|
||||
}
|
||||
|
||||
log.info(`Install-from-backup: restore completed successfully from ${trigger.manifestPath}`);
|
||||
announce('restore completed successfully');
|
||||
|
||||
// Remove the trigger so the next boot doesn't redo it.
|
||||
try {
|
||||
fs.unlinkSync(trigger.triggerPath);
|
||||
log.info(`Install-from-backup: removed trigger file ${trigger.triggerPath}`);
|
||||
} catch (unlinkErr) {
|
||||
log.warn(`Install-from-backup: could not remove trigger file (manual cleanup needed): ${unlinkErr.message}`);
|
||||
}
|
||||
|
||||
return { ran: true, manifestPath: trigger.manifestPath };
|
||||
} catch (err) {
|
||||
log.error(`Install-from-backup: FAILED — ${err.message}`);
|
||||
log.warn('Trigger file left in place so you can fix the input and retry by restarting the container.');
|
||||
announce(`FAILED — ${err.message}. Trigger file left in place for retry.`);
|
||||
return { ran: false, error: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { tryInstallFromBackup };
|
||||
@@ -0,0 +1,173 @@
|
||||
/**
|
||||
* Boot-time self-heal for restore-meta settings.
|
||||
*
|
||||
* **Why this exists**
|
||||
*
|
||||
* `restore_allow_force` gates whether the Restore wizard accepts a
|
||||
* `force: true` payload. The flag exists to add admin friction
|
||||
* before letting a restore override safety warnings (e.g. "1 active
|
||||
* admin user — restoring would clobber the current install").
|
||||
*
|
||||
* In practice the friction lands at the worst possible moment: a
|
||||
* fresh install (no app_settings row yet OR `restore_allow_force =
|
||||
* false` by default) hits the wall on its very FIRST restore. The
|
||||
* admin is mid disaster-recovery, panicked, and gets:
|
||||
*
|
||||
* "Force restore is not allowed by system settings"
|
||||
*
|
||||
* They then have to hand-craft SQL like
|
||||
*
|
||||
* INSERT INTO app_settings (setting_key, setting_value, ...)
|
||||
* VALUES ('restore_allow_force', 'true', 'restore', NOW())
|
||||
* ON CONFLICT ... SET setting_value = 'true';
|
||||
*
|
||||
* before they can recover their data. This isn't security — the
|
||||
* admin who could run that SQL could also flip the setting via the
|
||||
* UI. It's just a sharp edge that bites every new install once.
|
||||
*
|
||||
* Cure: seed the default ON at boot via `INSERT ... ON CONFLICT
|
||||
* DO NOTHING`. New installs get force-allowed out of the box.
|
||||
* Existing installs that have explicitly set the row (true OR
|
||||
* false) are NOT overwritten — admin policy wins. Same pattern
|
||||
* `_backupPathsBoot.js` uses for the canonical backup_paths rows.
|
||||
*
|
||||
* **Default-ON rationale (matches Stage A's principle)**
|
||||
*
|
||||
* Stage A defaulted inline DB dumps to ON because the cost of
|
||||
* forgetting was data loss. By the same logic, `restore_allow_force`
|
||||
* defaults ON because the cost of forgetting is being unable to
|
||||
* recover from a disaster. Audit logging captures every forced
|
||||
* restore so the accountability story stays intact.
|
||||
*
|
||||
* If/when the broader "exclude restore-meta settings from being
|
||||
* overwritten by restore" follow-up lands (the second half of this
|
||||
* chicken-and-egg), this self-heal becomes the safety net for
|
||||
* fresh installs only — existing installs by that point have the
|
||||
* row preserved across restores.
|
||||
*/
|
||||
|
||||
const SEEDS = [
|
||||
{
|
||||
setting_key: 'restore_allow_force',
|
||||
setting_value: 'true',
|
||||
setting_type: 'restore',
|
||||
rationale: 'Default ON so fresh installs can recover from disaster '
|
||||
+ 'without a SQL incantation. Admins who want to require manual '
|
||||
+ 'intervention can disable via the admin UI.',
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* Installs that ran migration 032 BEFORE the 2026-05-30 in-place edit
|
||||
* have a `restore_allow_force` row with the deprecated `false` default
|
||||
* (literal string `'false'` from `JSON.stringify(false)`). Per
|
||||
* [[feedback_self_heal_pattern]] knex won't re-run the corrected
|
||||
* migration on those installs, so we have to bump the row to `true`
|
||||
* here ONCE at boot.
|
||||
*
|
||||
* The bump is guarded by a tracking row `restore_allow_force_auto_upgraded`
|
||||
* so we don't fight an admin who explicitly disables force later:
|
||||
* - Tracking row absent → bump if the value is the deprecated `'false'`
|
||||
* - Tracking row present → never touch `restore_allow_force` again
|
||||
*
|
||||
* The bump applies ONLY when the existing value EXACTLY equals the old
|
||||
* migration default. Any other value (`'true'`, admin-set anything,
|
||||
* empty, null) is left alone — those reflect either the fixed
|
||||
* migration's output or a deliberate admin choice.
|
||||
*/
|
||||
const DEPRECATED_DEFAULT_VALUE = 'false';
|
||||
const AUTO_UPGRADE_FLAG_KEY = 'restore_allow_force_auto_upgraded';
|
||||
|
||||
let booted = false;
|
||||
|
||||
/**
|
||||
* Seed the canonical restore-meta settings on fresh installs.
|
||||
*
|
||||
* @param {object} db knex instance
|
||||
* @param {object} logger app logger (must expose .info / .warn)
|
||||
* @returns {Promise<{ seeded: string[], upgraded: string[] }>}
|
||||
*/
|
||||
async function seedRestoreSettingsAtBoot(db, logger) {
|
||||
const log = logger || { info: () => {}, warn: () => {} };
|
||||
if (booted) return { seeded: [], upgraded: [] };
|
||||
|
||||
if (!(await db.schema.hasTable('app_settings'))) {
|
||||
log.warn('app_settings table missing at boot — restore-settings self-heal skipped');
|
||||
return { seeded: [], upgraded: [] };
|
||||
}
|
||||
|
||||
const seeded = [];
|
||||
const upgraded = [];
|
||||
|
||||
// Step 1: fresh-install seeding. Insert rows that don't exist at all.
|
||||
for (const seed of SEEDS) {
|
||||
try {
|
||||
const existing = await db('app_settings')
|
||||
.where('setting_key', seed.setting_key)
|
||||
.first();
|
||||
if (existing) continue;
|
||||
|
||||
await db('app_settings').insert({
|
||||
setting_key: seed.setting_key,
|
||||
setting_value: seed.setting_value,
|
||||
setting_type: seed.setting_type,
|
||||
updated_at: new Date(),
|
||||
});
|
||||
seeded.push(seed.setting_key);
|
||||
log.info(`Seeded restore-meta setting ${seed.setting_key}=${seed.setting_value}`);
|
||||
} catch (err) {
|
||||
log.warn(`Failed to seed restore-meta setting ${seed.setting_key}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: one-time auto-upgrade for installs that ran the OLD
|
||||
// migration 032 (which seeded restore_allow_force='false'). Bump
|
||||
// to 'true' iff the value is still the deprecated default AND the
|
||||
// auto-upgrade tracking flag hasn't already been set.
|
||||
try {
|
||||
const guard = await db('app_settings')
|
||||
.where('setting_key', AUTO_UPGRADE_FLAG_KEY)
|
||||
.first();
|
||||
|
||||
if (!guard) {
|
||||
const row = await db('app_settings')
|
||||
.where('setting_key', 'restore_allow_force')
|
||||
.first();
|
||||
|
||||
if (row && row.setting_value === DEPRECATED_DEFAULT_VALUE) {
|
||||
await db('app_settings')
|
||||
.where('setting_key', 'restore_allow_force')
|
||||
.update({
|
||||
setting_value: 'true',
|
||||
updated_at: new Date(),
|
||||
});
|
||||
upgraded.push('restore_allow_force');
|
||||
log.info('Auto-upgraded restore_allow_force from deprecated migration-032 default \'false\' to \'true\' '
|
||||
+ '(fresh-install disaster recovery now works without SQL incantation)');
|
||||
}
|
||||
|
||||
// Always set the guard, even if no upgrade happened — prevents
|
||||
// the bump from firing later if an admin sets the value to
|
||||
// false on purpose.
|
||||
await db('app_settings').insert({
|
||||
setting_key: AUTO_UPGRADE_FLAG_KEY,
|
||||
setting_value: 'true',
|
||||
setting_type: 'restore',
|
||||
updated_at: new Date(),
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn(`restore_allow_force auto-upgrade failed: ${err.message}`);
|
||||
}
|
||||
|
||||
booted = true;
|
||||
return { seeded, upgraded };
|
||||
}
|
||||
|
||||
// Test-only: reset the module-level boot flag so jest can re-exercise
|
||||
// the seeder against a fresh test DB inside a single worker.
|
||||
function _resetBootForTests() {
|
||||
booted = false;
|
||||
}
|
||||
|
||||
module.exports = { seedRestoreSettingsAtBoot, _resetBootForTests, SEEDS };
|
||||
@@ -0,0 +1,351 @@
|
||||
/**
|
||||
* Backup-coverage diagnostic — Stage C of the backup-hardening plan.
|
||||
*
|
||||
* **Why this is a separate service**
|
||||
*
|
||||
* Stage A (inline DB dump + fail-loud) and Stage B (config-driven
|
||||
* walker via `backup_paths`) close the data-loss footgun, but they
|
||||
* don't tell an admin *what* the next backup will actually cover.
|
||||
* That's a separate question — and a particularly important one,
|
||||
* because the whole reason Stage B exists is that the walker's
|
||||
* subdirectory list used to silently fall behind reality every time
|
||||
* a new feature dropped artefacts under STORAGE_PATH.
|
||||
*
|
||||
* This service answers two questions:
|
||||
*
|
||||
* 1. For every row in `backup_paths`, what will the next backup
|
||||
* do with it? (scan / skip-via-feature-flag / skip-via-toggle /
|
||||
* missing-on-disk)
|
||||
* 2. What subdirectories EXIST under STORAGE_PATH but have NO row
|
||||
* in `backup_paths` — i.e. drift the admin should know about
|
||||
* before they lose data on a restore?
|
||||
*
|
||||
* Plus a top-level database-dump status block: are we configured
|
||||
* for inline dump (default), or relying on the scheduled dump?
|
||||
* When was the last successful dump? Is it stale?
|
||||
*
|
||||
* **What this service does NOT do**
|
||||
*
|
||||
* - Does not run the backup
|
||||
* - Does not write anything (no DB mutations, no fs touches)
|
||||
* - Does not auto-recover drift (it's a diagnostic — admins decide
|
||||
* whether to add a `backup_paths` row, delete the orphan dir, etc.)
|
||||
* - Does not walk file contents — only top-level directory entries
|
||||
* under STORAGE_PATH are inspected (cheap; no recursion through
|
||||
* potentially-millions of photos)
|
||||
*
|
||||
* Read-only. Returns a JSON report — same shape as
|
||||
* backupIntegrityService.verifyDocumentArtefacts.
|
||||
*/
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { db } = require('../database/db');
|
||||
const logger = require('../utils/logger');
|
||||
const backupService = require('./backupService');
|
||||
|
||||
const STORAGE_ROOT = () => process.env.STORAGE_PATH || path.join(process.cwd(), 'storage');
|
||||
|
||||
/**
|
||||
* Top-level subdirectories we expect to find under STORAGE_PATH but
|
||||
* which are intentionally NOT in `backup_paths` — they're generated
|
||||
* caches / runtime artefacts that the backup is supposed to skip.
|
||||
* Listing them here keeps the drift detector from flagging them.
|
||||
*
|
||||
* `backups` — the destination directory the backup writer itself
|
||||
* creates, plus the `database_backup_runs` dump files.
|
||||
* Including it in the walker would create a recursive
|
||||
* "backup of backups" feedback loop.
|
||||
*
|
||||
* `tmp` — short-lived scratch space (e.g. PDF render staging,
|
||||
* S3 multipart uploads). Re-created on demand, never
|
||||
* holds the only copy of anything.
|
||||
*/
|
||||
const EXPECTED_NON_BACKUP_DIRS = new Set([
|
||||
'backups',
|
||||
'tmp',
|
||||
]);
|
||||
|
||||
/**
|
||||
* How stale a database dump can be before we flag it. 26 hours so a
|
||||
* daily scheduled dump is still considered "fresh" if it ran a few
|
||||
* hours late.
|
||||
*/
|
||||
const DB_DUMP_STALE_AFTER_MS = 26 * 60 * 60 * 1000;
|
||||
|
||||
function parseSettingValue(raw) {
|
||||
if (raw === null || raw === undefined) return null;
|
||||
if (typeof raw !== 'string') return raw;
|
||||
try { return JSON.parse(raw); } catch (_) {
|
||||
if (raw === 'true') return true;
|
||||
if (raw === 'false') return false;
|
||||
const n = Number(raw);
|
||||
return Number.isFinite(n) ? n : raw;
|
||||
}
|
||||
}
|
||||
|
||||
async function readBackupConfig() {
|
||||
try {
|
||||
const rows = await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
const cfg = {};
|
||||
for (const row of rows) {
|
||||
cfg[row.setting_key] = parseSettingValue(row.setting_value);
|
||||
}
|
||||
return cfg;
|
||||
} catch (err) {
|
||||
logger.warn(`backup-coverage: could not read backup config — ${err.message}`);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
async function listConfiguredPaths() {
|
||||
try {
|
||||
if (!(await db.schema.hasTable('backup_paths'))) return null;
|
||||
return await db('backup_paths')
|
||||
.orderBy('display_order', 'asc')
|
||||
.select('path', 'include_in_default', 'feature_flag', 'display_order', 'description');
|
||||
} catch (err) {
|
||||
logger.warn(`backup-coverage: could not read backup_paths — ${err.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function listTopLevelStorageDirs() {
|
||||
const root = STORAGE_ROOT();
|
||||
try {
|
||||
const entries = await fs.readdir(root, { withFileTypes: true });
|
||||
return entries.filter((e) => e.isDirectory()).map((e) => e.name);
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') return [];
|
||||
logger.warn(`backup-coverage: could not read STORAGE_PATH (${root}) — ${err.message}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function statPath(absPath) {
|
||||
try {
|
||||
const st = await fs.stat(absPath);
|
||||
return { exists: true, isDir: st.isDirectory() };
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') return { exists: false, isDir: false };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the database-dump status block. Tells the admin whether
|
||||
* "Run Backup Now" will inline-dump (default) or rely on the
|
||||
* scheduled-dump path, plus how fresh the most recent dump is.
|
||||
*/
|
||||
async function buildDatabaseStatus(config) {
|
||||
// normalizeBoolean(undefined) === false, so we have to gate on
|
||||
// explicit-false the same way ensureDatabaseDumpForBackup does.
|
||||
const inlineExplicitlyOff = config.backup_database_inline_dump !== undefined
|
||||
&& config.backup_database_inline_dump !== null
|
||||
&& config.backup_database_inline_dump === false;
|
||||
const mode = inlineExplicitlyOff ? 'scheduled-only' : 'inline';
|
||||
|
||||
let recent = null;
|
||||
try {
|
||||
if (await db.schema.hasTable('database_backup_runs')) {
|
||||
recent = await db('database_backup_runs')
|
||||
.where('status', 'completed')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.first();
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`backup-coverage: could not read database_backup_runs — ${err.message}`);
|
||||
}
|
||||
|
||||
const status = {
|
||||
mode,
|
||||
inlineDumpExplicitlyDisabled: inlineExplicitlyOff,
|
||||
lastDumpAt: recent ? recent.completed_at : null,
|
||||
lastDumpType: recent ? recent.backup_type : null,
|
||||
lastDumpSizeBytes: recent ? Number(recent.file_size_bytes || 0) : 0,
|
||||
lastDumpFilePath: recent ? recent.file_path : null,
|
||||
lastDumpAgeMs: null,
|
||||
lastDumpStale: null,
|
||||
ok: null,
|
||||
};
|
||||
|
||||
if (recent && recent.completed_at) {
|
||||
const completedAt = recent.completed_at instanceof Date
|
||||
? recent.completed_at
|
||||
: new Date(recent.completed_at);
|
||||
status.lastDumpAgeMs = Date.now() - completedAt.getTime();
|
||||
status.lastDumpStale = status.lastDumpAgeMs > DB_DUMP_STALE_AFTER_MS;
|
||||
}
|
||||
|
||||
// ok semantics:
|
||||
// - inline mode: always ok=true (next backup will produce a fresh
|
||||
// dump on demand, staleness is irrelevant)
|
||||
// - scheduled-only: ok=true iff a recent non-stale dump exists,
|
||||
// because the file-backup guard will fail-loud otherwise
|
||||
if (mode === 'inline') {
|
||||
status.ok = true;
|
||||
} else {
|
||||
status.ok = Boolean(recent && recent.file_path && status.lastDumpStale === false);
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-path coverage:
|
||||
* - configured + include_in_default + (no feature_flag OR flag truthy) → 'will-scan'
|
||||
* - configured + include_in_default + flag falsey → 'skipped-by-feature-flag'
|
||||
* - configured + include_in_default = false → 'skipped-by-toggle'
|
||||
* - configured but missing on disk → 'missing-on-disk'
|
||||
*
|
||||
* Returns one entry per `backup_paths` row.
|
||||
*/
|
||||
async function buildConfiguredPathReport(configuredRows, config) {
|
||||
const root = STORAGE_ROOT();
|
||||
const result = [];
|
||||
|
||||
for (const row of configuredRows) {
|
||||
const absPath = path.join(root, row.path);
|
||||
const stat = await statPath(absPath);
|
||||
const includedInDefault = Boolean(row.include_in_default);
|
||||
let featureFlagValue = null;
|
||||
if (row.feature_flag) {
|
||||
const v = config[row.feature_flag];
|
||||
featureFlagValue = v === undefined ? null : Boolean(v);
|
||||
}
|
||||
|
||||
let coverage;
|
||||
if (!includedInDefault) {
|
||||
coverage = 'skipped-by-toggle';
|
||||
} else if (row.feature_flag && featureFlagValue !== true) {
|
||||
// null (unset) and explicit false both gate the path off — matches
|
||||
// the walker's normalizeBoolean semantics
|
||||
coverage = 'skipped-by-feature-flag';
|
||||
} else if (!stat.exists) {
|
||||
coverage = 'missing-on-disk';
|
||||
} else {
|
||||
coverage = 'will-scan';
|
||||
}
|
||||
|
||||
result.push({
|
||||
path: row.path,
|
||||
includeInDefault: includedInDefault,
|
||||
featureFlag: row.feature_flag || null,
|
||||
featureFlagValue,
|
||||
displayOrder: row.display_order,
|
||||
description: row.description || null,
|
||||
existsOnDisk: stat.exists,
|
||||
coverage,
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drift detection: top-level subdirs under STORAGE_PATH that are not
|
||||
* in `backup_paths` AND not in the `EXPECTED_NON_BACKUP_DIRS` allow-list.
|
||||
*
|
||||
* These are the directories that will be missed by "Run Backup Now"
|
||||
* — either intentionally (a new feature drops cache files there and
|
||||
* the admin doesn't want them backed up — they should add them to the
|
||||
* allow-list) or accidentally (a feature shipped without a matching
|
||||
* `backup_paths` row — the data-loss footgun this whole effort is
|
||||
* designed to catch).
|
||||
*/
|
||||
function detectDrift(diskDirs, configuredPaths) {
|
||||
// configured paths can be nested ('events/active'); we only diff the
|
||||
// top-level segment ('events') because that's the granularity admins
|
||||
// see in the storage tree. A path like 'events/active' implies the
|
||||
// 'events' top-level is "known to the backup config".
|
||||
const configuredTopLevels = new Set(
|
||||
configuredPaths.map((p) => p.path.split('/')[0]),
|
||||
);
|
||||
|
||||
return diskDirs
|
||||
.filter((d) => !configuredTopLevels.has(d))
|
||||
.filter((d) => !EXPECTED_NON_BACKUP_DIRS.has(d))
|
||||
.sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Public entry point.
|
||||
*
|
||||
* @returns {Promise<{
|
||||
* database: object,
|
||||
* paths: Array<object>,
|
||||
* drift: { unconfiguredOnDisk: string[], expectedNonBackupDirs: string[] },
|
||||
* summary: object,
|
||||
* generatedAt: string,
|
||||
* }>}
|
||||
*/
|
||||
async function getCoverageReport() {
|
||||
const config = await readBackupConfig();
|
||||
const configuredRows = await listConfiguredPaths();
|
||||
const diskDirs = await listTopLevelStorageDirs();
|
||||
|
||||
// Fallback when the table doesn't exist yet (migration 108 hasn't
|
||||
// run for some reason). Mirrors the walker's LEGACY_BACKUP_PATHS
|
||||
// contract — every other layer of this system uses the same
|
||||
// belt-and-suspenders fallback.
|
||||
const fallback = configuredRows === null;
|
||||
const effectiveRows = configuredRows || [
|
||||
{ path: 'events/active', include_in_default: true, feature_flag: null, display_order: 10, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'events/archived', include_in_default: true, feature_flag: 'backup_include_archived', display_order: 20, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'thumbnails', include_in_default: true, feature_flag: null, display_order: 30, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'previews', include_in_default: true, feature_flag: null, display_order: 40, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'heroes', include_in_default: true, feature_flag: null, display_order: 50, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'uploads', include_in_default: true, feature_flag: null, display_order: 60, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'business-docs', include_in_default: true, feature_flag: null, display_order: 70, description: 'Legacy fallback (backup_paths missing)' },
|
||||
];
|
||||
|
||||
const [database, paths] = await Promise.all([
|
||||
buildDatabaseStatus(config),
|
||||
buildConfiguredPathReport(effectiveRows, config),
|
||||
]);
|
||||
|
||||
const unconfiguredOnDisk = detectDrift(diskDirs, effectiveRows);
|
||||
|
||||
const summary = {
|
||||
configuredCount: effectiveRows.length,
|
||||
willScanCount: paths.filter((p) => p.coverage === 'will-scan').length,
|
||||
skippedByToggleCount: paths.filter((p) => p.coverage === 'skipped-by-toggle').length,
|
||||
skippedByFeatureFlagCount: paths.filter((p) => p.coverage === 'skipped-by-feature-flag').length,
|
||||
missingOnDiskCount: paths.filter((p) => p.coverage === 'missing-on-disk').length,
|
||||
driftCount: unconfiguredOnDisk.length,
|
||||
tableMissingFallbackInUse: fallback,
|
||||
databaseOk: database.ok,
|
||||
// overall: green only when DB is ok AND there's at least one path
|
||||
// that will actually be scanned AND no drift was found
|
||||
overallOk: Boolean(
|
||||
database.ok
|
||||
&& paths.some((p) => p.coverage === 'will-scan')
|
||||
&& unconfiguredOnDisk.length === 0,
|
||||
),
|
||||
};
|
||||
|
||||
return {
|
||||
database,
|
||||
paths,
|
||||
drift: {
|
||||
unconfiguredOnDisk,
|
||||
expectedNonBackupDirs: Array.from(EXPECTED_NON_BACKUP_DIRS).sort(),
|
||||
},
|
||||
summary,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getCoverageReport,
|
||||
// Exported for test introspection — the route doesn't use these.
|
||||
EXPECTED_NON_BACKUP_DIRS,
|
||||
DB_DUMP_STALE_AFTER_MS,
|
||||
};
|
||||
|
||||
// Silence unused import lint warning — backupService is required so
|
||||
// the module-graph cache primes (some tests jest.mock it before
|
||||
// requiring this service).
|
||||
void backupService;
|
||||
@@ -0,0 +1,228 @@
|
||||
/**
|
||||
* Backup-integrity verifier — walks every CRM document-artefact path
|
||||
* column and confirms (a) the file exists on disk, (b) when a SHA-256
|
||||
* is stored, the file's actual bytes hash to the stored value.
|
||||
*
|
||||
* **Why this is a separate service**
|
||||
*
|
||||
* The audit trail captured at issue / sign time (signed_customer_ip,
|
||||
* signed_by_customer_at, signed_pdf_sha256, signed_*_signature_path,
|
||||
* issue_date, etc.) is worth exactly nothing on its own — what makes
|
||||
* it legally meaningful is being able to produce the document the
|
||||
* audit trail refers to. A backup that captures the DB but skips
|
||||
* `storage/business-docs/` (the bug fixed in this same PR) leaves
|
||||
* every `*_path` column a broken FK and every `*_sha256` column with
|
||||
* nothing to verify against. This service is the diagnostic for
|
||||
* exactly that drift — runs on demand, surfaces missing files +
|
||||
* hash mismatches without making any changes.
|
||||
*
|
||||
* **Verification modes**
|
||||
*
|
||||
* - existence — file at `*_path` must exist on disk
|
||||
* - sha256 — file at `*_path` must exist AND its sha256 must
|
||||
* equal `*_sha256` column (when that column is set)
|
||||
*
|
||||
* Per-table coverage (lines reference migrations/core/107_crm_consolidated.js):
|
||||
*
|
||||
* quotes.pdf_path line 844 (existence)
|
||||
* contracts.pdf_path line 1245 (existence + sha256 via contracts.pdf_sha256)
|
||||
* contracts.signed_pdf_path line 1246 (existence + sha256 via contracts.signed_pdf_sha256)
|
||||
* contracts.signed_customer_signature_path line 1269 (existence — drawn signatures, no hash column)
|
||||
* contracts.signed_admin_signature_path line 1273 (existence — admin counter-signature drawing)
|
||||
* invoices.pdf_path line 1026 (existence)
|
||||
* invoices.imported_pdf_path line 1020 (existence — admin-uploaded scans)
|
||||
*
|
||||
* Wet uploads (`contracts.signed_pdf_is_wet_upload = true`) DO have a
|
||||
* `signed_pdf_sha256` computed at upload time (contractService.js
|
||||
* upload route), so they're hash-verified the same as system-rendered
|
||||
* contracts — no special case here.
|
||||
*
|
||||
* **What this service does NOT do**
|
||||
*
|
||||
* - Does not write anything (no DB mutations, no fs touches)
|
||||
* - Does not fail the request when a mismatch is found — the
|
||||
* report shape carries the data, the caller decides what to do
|
||||
* - Does not auto-trigger after restore (D2 decision: surface a
|
||||
* CTA on the restore-completed screen instead)
|
||||
* - Does not run on a schedule (D1 decision: on-demand v1; revisit
|
||||
* once we have runtime data on large installs)
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const crypto = require('crypto');
|
||||
const path = require('path');
|
||||
const { db } = require('../database/db');
|
||||
const logger = require('../utils/logger');
|
||||
|
||||
const STORAGE_ROOT = () => process.env.STORAGE_PATH || path.join(process.cwd(), 'storage');
|
||||
|
||||
/**
|
||||
* Every column the verifier walks, declared once so the test suite
|
||||
* and the service share a single source of truth. Order is the order
|
||||
* the report lists rows in — table-major, then column-by-column.
|
||||
*/
|
||||
const CHECKS = [
|
||||
{ table: 'quotes', pathColumn: 'pdf_path', shaColumn: null, scope: 'quote' },
|
||||
{ table: 'contracts', pathColumn: 'pdf_path', shaColumn: 'pdf_sha256', scope: 'contract' },
|
||||
{ table: 'contracts', pathColumn: 'signed_pdf_path', shaColumn: 'signed_pdf_sha256', scope: 'contract' },
|
||||
{ table: 'contracts', pathColumn: 'signed_customer_signature_path', shaColumn: null, scope: 'contract-signature' },
|
||||
{ table: 'contracts', pathColumn: 'signed_admin_signature_path', shaColumn: null, scope: 'contract-signature' },
|
||||
{ table: 'invoices', pathColumn: 'pdf_path', shaColumn: null, scope: 'invoice' },
|
||||
{ table: 'invoices', pathColumn: 'imported_pdf_path', shaColumn: null, scope: 'invoice' },
|
||||
];
|
||||
|
||||
/** Stream-hash a file to sha256 hex without buffering the whole thing. */
|
||||
function hashFile(absPath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = crypto.createHash('sha256');
|
||||
const stream = fs.createReadStream(absPath);
|
||||
stream.on('error', reject);
|
||||
stream.on('data', (chunk) => hash.update(chunk));
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {object} [options]
|
||||
* @param {string[]} [options.scope] Filter checks by scope tag:
|
||||
* 'quote' | 'contract' | 'contract-signature' | 'invoice'.
|
||||
* Defaults to all four (full scan).
|
||||
* @returns {Promise<{
|
||||
* scannedAt: string,
|
||||
* scopes: string[],
|
||||
* summary: {
|
||||
* totalRows: number,
|
||||
* verifiedOk: number,
|
||||
* missingFiles: number,
|
||||
* hashMismatches: number,
|
||||
* existsButNoHash: number,
|
||||
* },
|
||||
* missing: Array<{ table, rowId, column, expectedPath }>,
|
||||
* hashMismatches: Array<{ table, rowId, column, expectedPath, expectedSha, actualSha }>,
|
||||
* existsButNoHash: Array<{ table, rowId, column, path }>,
|
||||
* }>}
|
||||
*
|
||||
* `existsButNoHash` is the existence-only-verified bucket — the file
|
||||
* was found but no `*_sha256` column exists for it (quote/invoice PDFs,
|
||||
* signature PNGs). Surfaced separately so admins can distinguish
|
||||
* "verified by hash" from "verified by existence only" — the latter
|
||||
* is weaker evidence in a legal dispute.
|
||||
*/
|
||||
async function verifyDocumentArtefacts(options = {}) {
|
||||
const scopes = Array.isArray(options.scope) && options.scope.length > 0
|
||||
? options.scope.slice()
|
||||
: Array.from(new Set(CHECKS.map((c) => c.scope)));
|
||||
|
||||
const checksToRun = CHECKS.filter((c) => scopes.includes(c.scope));
|
||||
const storageRoot = STORAGE_ROOT();
|
||||
|
||||
const missing = [];
|
||||
const hashMismatches = [];
|
||||
const existsButNoHash = [];
|
||||
let totalRows = 0;
|
||||
let verifiedOk = 0;
|
||||
|
||||
for (const check of checksToRun) {
|
||||
// Skip the check cleanly when the column or table doesn't exist
|
||||
// on this install — keeps the verifier safe to run on partial
|
||||
// migrations or installs that have features disabled.
|
||||
if (!(await db.schema.hasTable(check.table))) continue;
|
||||
if (!(await db.schema.hasColumn(check.table, check.pathColumn))) continue;
|
||||
|
||||
const select = ['id', check.pathColumn];
|
||||
const hasHashColumn = check.shaColumn
|
||||
&& (await db.schema.hasColumn(check.table, check.shaColumn));
|
||||
if (hasHashColumn) select.push(check.shaColumn);
|
||||
|
||||
const rows = await db(check.table)
|
||||
.whereNotNull(check.pathColumn)
|
||||
.select(...select);
|
||||
|
||||
for (const row of rows) {
|
||||
totalRows += 1;
|
||||
const storedPath = row[check.pathColumn];
|
||||
// Stored paths can be absolute (older rows) or relative-to-
|
||||
// storage (newer rows). Normalize: resolve relative paths
|
||||
// against STORAGE_PATH; absolute paths are used verbatim.
|
||||
const absPath = path.isAbsolute(storedPath)
|
||||
? storedPath
|
||||
: path.join(storageRoot, storedPath);
|
||||
|
||||
let exists = false;
|
||||
try {
|
||||
exists = fs.existsSync(absPath);
|
||||
} catch (_) { exists = false; }
|
||||
|
||||
if (!exists) {
|
||||
missing.push({
|
||||
table: check.table,
|
||||
rowId: row.id,
|
||||
column: check.pathColumn,
|
||||
expectedPath: storedPath,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const expectedSha = hasHashColumn ? row[check.shaColumn] : null;
|
||||
if (!expectedSha) {
|
||||
// File exists but we have no hash to verify it against.
|
||||
existsButNoHash.push({
|
||||
table: check.table,
|
||||
rowId: row.id,
|
||||
column: check.pathColumn,
|
||||
path: storedPath,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let actualSha;
|
||||
try {
|
||||
actualSha = await hashFile(absPath);
|
||||
} catch (err) {
|
||||
logger.warn(`backupIntegrity: failed to hash ${absPath}: ${err.message}`);
|
||||
missing.push({
|
||||
table: check.table,
|
||||
rowId: row.id,
|
||||
column: check.pathColumn,
|
||||
expectedPath: storedPath,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (actualSha !== expectedSha) {
|
||||
hashMismatches.push({
|
||||
table: check.table,
|
||||
rowId: row.id,
|
||||
column: check.pathColumn,
|
||||
expectedPath: storedPath,
|
||||
expectedSha,
|
||||
actualSha,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
verifiedOk += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
scannedAt: new Date().toISOString(),
|
||||
scopes,
|
||||
summary: {
|
||||
totalRows,
|
||||
verifiedOk,
|
||||
missingFiles: missing.length,
|
||||
hashMismatches: hashMismatches.length,
|
||||
existsButNoHash: existsButNoHash.length,
|
||||
},
|
||||
missing,
|
||||
hashMismatches,
|
||||
existsButNoHash,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
verifyDocumentArtefacts,
|
||||
// Exported for tests; not part of the route API.
|
||||
_internal: { CHECKS, hashFile },
|
||||
};
|
||||
@@ -259,6 +259,73 @@ async function hasDatabaseChanged(sinceTime) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run an inline database dump (default ON) and then verify a usable dump
|
||||
* is actually on disk before letting the file-backup proceed. Returns the
|
||||
* verified `databaseInfo` so the caller can pass it straight into the
|
||||
* manifest builder without re-querying.
|
||||
*
|
||||
* Why this lives here and not inline in `runBackupInternal`:
|
||||
* - Encapsulates the "Run Backup Now must include DB" guarantee
|
||||
* introduced when the silent files-only bug was discovered
|
||||
* (2026-05-29 — admin lost CRM after `docker compose down -v`)
|
||||
* - Lets the manifest path share the same `databaseInfo` object
|
||||
* instead of doing a second `getDatabaseBackupInfo()` round-trip
|
||||
* - Thrown errors bubble up to `runBackupInternal`'s catch, which
|
||||
* marks the `backup_runs` row failed and queues the admin email
|
||||
*
|
||||
* Default-ON semantics: `backup_database_inline_dump` is only treated
|
||||
* as disabled when explicitly set to false. `undefined` (the case on
|
||||
* every existing install that predates the setting) falls through to
|
||||
* the safe-default ON branch. `normalizeBoolean(undefined)` returns
|
||||
* false, so a naive `!== false` check would silently disable the
|
||||
* inline dump for every upgrading install.
|
||||
*/
|
||||
async function ensureDatabaseDumpForBackup(config) {
|
||||
const inlineDumpExplicitlyOff = config.backup_database_inline_dump !== undefined
|
||||
&& config.backup_database_inline_dump !== null
|
||||
&& normalizeBoolean(config.backup_database_inline_dump) === false;
|
||||
|
||||
if (!inlineDumpExplicitlyOff) {
|
||||
logger.info('Running inline database dump before file backup...');
|
||||
const { databaseBackupService } = require('./databaseBackup');
|
||||
const dumpResult = await databaseBackupService.backup({});
|
||||
logger.info(`Inline database dump completed: ${dumpResult.path} ` +
|
||||
`(${(dumpResult.size / 1024 / 1024).toFixed(2)} MB)`);
|
||||
}
|
||||
|
||||
const databaseInfo = await service.getDatabaseBackupInfo();
|
||||
if (!databaseInfo.backupFile) {
|
||||
throw new Error(
|
||||
'No database backup available to include in this file backup. ' +
|
||||
'Either keep backup_database_inline_dump enabled (default) or configure ' +
|
||||
'backup_database_schedule and let it run at least once first.'
|
||||
);
|
||||
}
|
||||
|
||||
let dumpStat;
|
||||
try {
|
||||
dumpStat = await fs.stat(databaseInfo.backupFile);
|
||||
} catch (statErr) {
|
||||
if (statErr.code === 'ENOENT') {
|
||||
throw new Error(
|
||||
`Database backup file at ${databaseInfo.backupFile} is missing from disk. ` +
|
||||
'Refusing to proceed with file backup; configure backup_database_schedule or ' +
|
||||
'keep backup_database_inline_dump enabled.'
|
||||
);
|
||||
}
|
||||
throw statErr;
|
||||
}
|
||||
if (!dumpStat.size) {
|
||||
throw new Error(
|
||||
`Database backup file at ${databaseInfo.backupFile} is empty (0 bytes). ` +
|
||||
'Refusing to proceed with file backup to avoid shipping a manifest with no DB content.'
|
||||
);
|
||||
}
|
||||
|
||||
return databaseInfo;
|
||||
}
|
||||
|
||||
async function getDatabaseBackupInfoInternal() {
|
||||
try {
|
||||
const recent = await db('database_backup_runs')
|
||||
@@ -350,28 +417,179 @@ async function scanDirectory(dirPath, fileList, basePath, excludePatterns = [])
|
||||
}
|
||||
}
|
||||
|
||||
async function getFilesToBackupInternal(includeArchived = true) {
|
||||
/**
|
||||
* Hard-coded fallback when `backup_paths` is missing/empty. Mirrors
|
||||
* the canonical seed in migration 109 — kept here as defense in depth
|
||||
* so the walker can never silently degrade to "no directories scanned"
|
||||
* because of a seed problem.
|
||||
*
|
||||
* Order matches the legacy behavior of the inlined sequence this
|
||||
* function used to contain.
|
||||
*/
|
||||
const LEGACY_BACKUP_PATHS = [
|
||||
{ path: 'events/active', feature_flag: null },
|
||||
{ path: 'events/archived', feature_flag: 'backup_include_archived' },
|
||||
{ path: 'thumbnails', feature_flag: null },
|
||||
{ path: 'previews', feature_flag: null },
|
||||
{ path: 'heroes', feature_flag: null },
|
||||
{ path: 'uploads', feature_flag: null },
|
||||
{ path: 'business-docs', feature_flag: null },
|
||||
];
|
||||
|
||||
/**
|
||||
* Resolve the walker's target subdirectories from `backup_paths`.
|
||||
*
|
||||
* Layered fallback (defense in depth — no scenario where the walker
|
||||
* silently scans nothing):
|
||||
* 1. Read `backup_paths` rows where include_in_default = true,
|
||||
* ordered by display_order.
|
||||
* 2. If the table is missing OR returns zero rows, fall back to
|
||||
* LEGACY_BACKUP_PATHS. Logged loudly so the admin sees it.
|
||||
*
|
||||
* Per-row gating: when `feature_flag` is set, the corresponding
|
||||
* config key in `app_settings` must resolve truthy for that path to
|
||||
* be included. Mirrors the historical `includeArchived` parameter,
|
||||
* but now driven by data instead of a hard-coded boolean.
|
||||
*
|
||||
* @param {object} config resolved backup config (parseSettingValue'd).
|
||||
* Used to evaluate feature_flag gates.
|
||||
* @returns {Promise<Array<{ path: string, feature_flag: string|null }>>}
|
||||
*/
|
||||
async function resolveBackupPaths(config) {
|
||||
let rows;
|
||||
try {
|
||||
if (!(await db.schema.hasTable('backup_paths'))) {
|
||||
logger.warn('backup_paths table missing — falling back to LEGACY_BACKUP_PATHS');
|
||||
rows = LEGACY_BACKUP_PATHS;
|
||||
} else {
|
||||
rows = await db('backup_paths')
|
||||
.where('include_in_default', formatBoolean(true))
|
||||
.orderBy('display_order', 'asc')
|
||||
.select('path', 'feature_flag');
|
||||
if (!rows.length) {
|
||||
logger.warn('backup_paths has no rows with include_in_default=true — falling back to LEGACY_BACKUP_PATHS');
|
||||
rows = LEGACY_BACKUP_PATHS;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`Failed to query backup_paths (${err.message}) — falling back to LEGACY_BACKUP_PATHS`);
|
||||
rows = LEGACY_BACKUP_PATHS;
|
||||
}
|
||||
|
||||
// Apply feature_flag gating. A row with feature_flag='backup_include_archived'
|
||||
// requires config.backup_include_archived to be truthy (same semantics as
|
||||
// the historical `includeArchived` parameter).
|
||||
return rows.filter((row) => {
|
||||
if (!row.feature_flag) return true;
|
||||
const flagValue = config ? config[row.feature_flag] : undefined;
|
||||
return normalizeBoolean(flagValue);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Bucket actually-backed-up files into their owning `backup_paths` row.
|
||||
*
|
||||
* Uses longest-prefix match — e.g. `events/active/E1/photo.jpg` matches
|
||||
* `events/active` (length 13) rather than `events` (length 6, if that
|
||||
* row existed). This handles the case where a future feature ships a
|
||||
* nested backup_paths row that overlaps an existing one.
|
||||
*
|
||||
* @param {string[]} backedUpRelativePaths — paths that actually got
|
||||
* copied / uploaded (post-incremental-filter). The exact list
|
||||
* the destination implementation reports back.
|
||||
* @param {Array<{relativePath, size}>} allFiles — the full file
|
||||
* catalogue from the walker, used as a size lookup table.
|
||||
* @returns {Promise<Record<string, { count: number, size: number }>>}
|
||||
* keyed by `backup_paths.path` (e.g. 'events/active'). Paths
|
||||
* with zero matches are omitted to keep the manifest compact.
|
||||
*/
|
||||
async function computePerPathStats(backedUpRelativePaths, allFiles) {
|
||||
if (!backedUpRelativePaths || backedUpRelativePaths.length === 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Reuse the same source of truth the walker uses, so a row toggled
|
||||
// off by include_in_default doesn't appear in the breakdown either.
|
||||
let configuredPaths;
|
||||
try {
|
||||
if (await db.schema.hasTable('backup_paths')) {
|
||||
configuredPaths = await db('backup_paths')
|
||||
.where('include_in_default', formatBoolean(true))
|
||||
.orderBy('display_order', 'asc')
|
||||
.select('path');
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`Could not load backup_paths for per-path stats — falling back to legacy: ${err.message}`);
|
||||
}
|
||||
if (!configuredPaths || configuredPaths.length === 0) {
|
||||
configuredPaths = LEGACY_BACKUP_PATHS.map((p) => ({ path: p.path }));
|
||||
}
|
||||
|
||||
// Longest-prefix-first so nested paths win over their parents.
|
||||
const sortedPaths = configuredPaths
|
||||
.map((row) => row.path)
|
||||
.sort((a, b) => b.length - a.length);
|
||||
|
||||
// Size lookup. relativePath uses OS path separators in `allFiles`
|
||||
// (whatever scanDirectory built); the backup_paths rows always use
|
||||
// forward slashes. Normalize the lookup key once.
|
||||
const sizeByPath = new Map();
|
||||
for (const f of allFiles || []) {
|
||||
sizeByPath.set(f.relativePath.split(path.sep).join('/'), f.size || 0);
|
||||
}
|
||||
|
||||
const stats = {};
|
||||
for (const relativePath of backedUpRelativePaths) {
|
||||
const norm = relativePath.split(path.sep).join('/');
|
||||
// Find the longest configured path that this file's relativePath starts with.
|
||||
const match = sortedPaths.find(
|
||||
(p) => norm === p || norm.startsWith(`${p}/`)
|
||||
);
|
||||
if (!match) continue; // file outside any configured path (shouldn't happen)
|
||||
if (!stats[match]) stats[match] = { count: 0, size: 0 };
|
||||
stats[match].count += 1;
|
||||
stats[match].size += sizeByPath.get(norm) || 0;
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
async function getFilesToBackupInternal(configOrIncludeArchived = true) {
|
||||
const files = [];
|
||||
const storagePath = getStoragePath();
|
||||
|
||||
await scanDirectory(path.join(storagePath, 'events/active'), files, storagePath);
|
||||
|
||||
if (normalizeBoolean(includeArchived)) {
|
||||
await scanDirectory(path.join(storagePath, 'events/archived'), files, storagePath);
|
||||
// Backward-compatible call signature:
|
||||
// - Boolean `true|false` → legacy `includeArchived` argument. We
|
||||
// forge a config-shaped object so the feature-flag gating
|
||||
// resolves the same way the old code path did.
|
||||
// - Object → full resolved backup config (preferred).
|
||||
// - Anything else → treated as "include archived" (truthy).
|
||||
let config;
|
||||
if (typeof configOrIncludeArchived === 'object' && configOrIncludeArchived !== null) {
|
||||
config = configOrIncludeArchived;
|
||||
} else {
|
||||
config = { backup_include_archived: normalizeBoolean(configOrIncludeArchived) };
|
||||
}
|
||||
|
||||
await scanDirectory(path.join(storagePath, 'thumbnails'), files, storagePath);
|
||||
// Lightbox preview tier (#492). Cheap to back up — typically a few
|
||||
// hundred KB per photo — and saves admins the regenerate cycle on
|
||||
// a restore. Tolerated when missing (admins who never enabled the
|
||||
// feature won't have the folder; scanDirectory short-circuits on
|
||||
// ENOENT cleanly).
|
||||
await scanDirectory(path.join(storagePath, 'previews'), files, storagePath);
|
||||
// Heroes too — same logic; admins who picked a hero photo for the
|
||||
// gallery header had its 1920x1080 file generated and was missed
|
||||
// by the original backup walk before this addition.
|
||||
await scanDirectory(path.join(storagePath, 'heroes'), files, storagePath);
|
||||
await scanDirectory(path.join(storagePath, 'uploads'), files, storagePath);
|
||||
const targets = await resolveBackupPaths(config);
|
||||
|
||||
for (const target of targets) {
|
||||
// CRM document estate is special-cased in the comment block below
|
||||
// because it's the most expensive omission to recover from:
|
||||
// - business-docs/quote/<year>/*.pdf
|
||||
// - business-docs/contract/<year>/*.pdf (system-rendered + wet uploads)
|
||||
// - business-docs/contract/signatures/<contract_id>/*.{png,jpg}
|
||||
// (drawn signatures, forensic-preserved per Date.now() filename)
|
||||
// - business-docs/invoice/<year>/*.pdf (issued invoices + Storno)
|
||||
// - business-docs/invoice-imports/<year>/*.pdf (admin-imported
|
||||
// historical invoices — irrecoverable if not backed up)
|
||||
// Without this scan, the audit trail (signed_pdf_sha256, signed_*
|
||||
// _ip, accepted_at, etc.) survives the restore but the documents
|
||||
// those values refer to do not, leaving every CRM *_path column a
|
||||
// broken FK. scanDirectory short-circuits on ENOENT so installs
|
||||
// that never used CRM features won't error.
|
||||
await scanDirectory(path.join(storagePath, target.path), files, storagePath);
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
@@ -799,7 +1017,17 @@ async function runBackupInternal(isManual = false) {
|
||||
}).returning('id');
|
||||
runId = insertResult[0]?.id || insertResult[0];
|
||||
|
||||
const files = await service.getFilesToBackup(config.backup_include_archived);
|
||||
// Inline DB dump + fail-loud verification. The returned `databaseInfo`
|
||||
// is reused at manifest-build time below so we don't pay a second
|
||||
// `getDatabaseBackupInfo()` round-trip — see `ensureDatabaseDumpForBackup`
|
||||
// for the full rationale.
|
||||
const verifiedDatabaseInfo = await ensureDatabaseDumpForBackup(config);
|
||||
|
||||
// Pass the full config so the walker can evaluate any feature_flag
|
||||
// gates declared in the backup_paths table (e.g. `events/archived`
|
||||
// gated by `backup_include_archived`). Boolean signature is still
|
||||
// supported for legacy callers and tests — see getFilesToBackupInternal.
|
||||
const files = await service.getFilesToBackup(config);
|
||||
logger.info(`Found ${files.length} files to check for backup`);
|
||||
|
||||
let result;
|
||||
@@ -826,7 +1054,13 @@ async function runBackupInternal(isManual = false) {
|
||||
|
||||
const previousBackup = await getPreviousSuccessfulBackup(runId);
|
||||
const manifestFiles = buildManifestFiles(result.backedUpFiles, files);
|
||||
const databaseInfo = result.databaseInfo || await service.getDatabaseBackupInfo();
|
||||
// `verifiedDatabaseInfo` came from ensureDatabaseDumpForBackup at the
|
||||
// top of this run — reuse it so manifest building doesn't pay a
|
||||
// second `getDatabaseBackupInfo()` round-trip. The
|
||||
// `result.databaseInfo` branch is kept for destination implementations
|
||||
// (S3, future destinations) that override the local info on the result
|
||||
// object; falls back to the verified copy otherwise.
|
||||
const databaseInfo = result.databaseInfo || verifiedDatabaseInfo;
|
||||
|
||||
const manifestOptions = {
|
||||
backupType: previousBackup ? 'incremental' : 'full',
|
||||
@@ -869,6 +1103,18 @@ async function runBackupInternal(isManual = false) {
|
||||
logger.error('Failed to generate backup manifest:', error);
|
||||
}
|
||||
|
||||
// Per-Stage-B-path stats — bucket the actually-backed-up files
|
||||
// into their owning backup_paths row by longest-prefix match. Lets
|
||||
// the Backup History detail pane render a true breakdown
|
||||
// events/active: 142 files (3.2 GB)
|
||||
// business-docs: 17 files (4.5 MB)
|
||||
// thumbnails: 142 files (12.4 MB)
|
||||
// instead of the legacy "Photos + Archives + Other" categorization
|
||||
// that didn't reflect Stage B's data-driven walker. Falls back to
|
||||
// an empty map if backup_paths is missing (defense in depth — the
|
||||
// walker has the same fallback).
|
||||
const perPath = await computePerPathStats(result.backedUpFiles, files);
|
||||
|
||||
await db('backup_runs')
|
||||
.where('id', runId)
|
||||
.update({
|
||||
@@ -887,11 +1133,14 @@ async function runBackupInternal(isManual = false) {
|
||||
total_files_checked: files.length,
|
||||
average_file_size: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0,
|
||||
destination: destinationType,
|
||||
// Per-Stage-B-path breakdown — { [pathKey]: { count, size } }
|
||||
per_path: perPath,
|
||||
// Keep camelCase for backward compatibility
|
||||
totalFilesChecked: files.length,
|
||||
filesBackedUp: result.backedUpCount,
|
||||
totalSize: result.backedUpSize,
|
||||
averageFileSize: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0
|
||||
averageFileSize: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0,
|
||||
perPath
|
||||
})
|
||||
});
|
||||
|
||||
@@ -1071,11 +1320,34 @@ async function getBackupStatus(limit = 10) {
|
||||
|
||||
const lastRunWithManifest = lastRun ? { ...lastRun, manifestValid } : null;
|
||||
|
||||
// Separate "most recent attempt" from "most recent SUCCESS" so the
|
||||
// dashboard widget can distinguish:
|
||||
// - last attempt failed → red, "Last attempt failed at X"
|
||||
// - last attempt running → blue spinner, "In progress since X"
|
||||
// - never succeeded → critical, "No successful backup yet"
|
||||
// - last attempt succeeded → green tick, "Last backup X ago"
|
||||
// Previously the widget showed the most-recent row with a generic
|
||||
// green tick regardless of status, so a crashed run from 5 minutes
|
||||
// ago looked identical to a successful one. Same "silent failure
|
||||
// not surfaced" class Stage A was designed to fight.
|
||||
const lastSuccessful = runs.find(r => r.status === 'completed') || null;
|
||||
// Detect zombie running rows (started >30min ago, never updated)
|
||||
// — these are processes that died without writing a completed_at.
|
||||
// Surface them so the admin can tell at a glance vs a live run.
|
||||
const ZOMBIE_THRESHOLD_MS = 30 * 60 * 1000;
|
||||
const zombieRuns = runs.filter(r =>
|
||||
r.status === 'running'
|
||||
&& r.started_at
|
||||
&& (Date.now() - new Date(r.started_at).getTime()) > ZOMBIE_THRESHOLD_MS
|
||||
);
|
||||
|
||||
return {
|
||||
isRunning,
|
||||
isHealthy: Boolean(lastRun && lastRun.status === 'completed'),
|
||||
lastRun: lastRunWithManifest,
|
||||
lastBackup: lastRunWithManifest, // Alias for frontend compatibility
|
||||
lastSuccessfulBackup: lastSuccessful, // NEW — see comment above
|
||||
zombieRuns, // NEW — running >30min, likely crashed
|
||||
recentRuns: runs,
|
||||
recentBackups: runs, // Alias for frontend compatibility
|
||||
totalBackups: runs.filter(r => r.status === 'completed').length,
|
||||
|
||||
@@ -206,12 +206,20 @@ class DatabaseBackupService {
|
||||
'--format=plain',
|
||||
'--encoding=UTF8'
|
||||
];
|
||||
|
||||
// Add transaction support for consistency
|
||||
if (!options.noTransaction) {
|
||||
pgDumpOptions.push('--single-transaction');
|
||||
}
|
||||
|
||||
|
||||
// NOTE: do NOT add `--single-transaction` here. It looks like
|
||||
// the right flag for "consistent snapshot" but it isn't a pg_dump
|
||||
// option — it belongs to pg_restore / psql and pg_dump rejects it
|
||||
// with `unrecognized option: single-transaction` (exit code 1).
|
||||
// pg_dump already wraps the entire export in a single REPEATABLE
|
||||
// READ snapshot automatically (since Postgres 9.x), so consistency
|
||||
// is built in. If we ever need stricter cross-pg-cluster snapshot
|
||||
// sharing, use `--snapshot=<id>` — but the typical inline-dump
|
||||
// path doesn't need it. Bug went undetected until Stage A wired
|
||||
// this code into the user-facing "Run Backup Now" path; prior
|
||||
// callers (scheduled cron, dedicated admin DB-backup page) hit
|
||||
// the same failure but on installs that had never exercised them.
|
||||
|
||||
// Add compression if not doing it separately
|
||||
if (options.compress && !options.separateCompression) {
|
||||
pgDumpOptions.push('--compress=6');
|
||||
@@ -309,8 +317,23 @@ class DatabaseBackupService {
|
||||
// Get current schema version
|
||||
const schemaVersion = await this.getCurrentSchemaVersion();
|
||||
|
||||
// Create backup run record with version info
|
||||
const [runId] = await db('database_backup_runs').insert({
|
||||
// Create backup run record with version info.
|
||||
//
|
||||
// Insert shape divergence between SQLite + Postgres made the old
|
||||
// `const [runId] = await db(...).insert({...})` form throw
|
||||
// "(intermediate value) is not iterable" on Postgres installs:
|
||||
//
|
||||
// - SQLite-via-knex: insert() returns `[lastInsertId]` (array)
|
||||
// - Postgres-via-knex: insert() without .returning() returns an
|
||||
// empty object / row count — not iterable
|
||||
//
|
||||
// Bug went undetected until Stage A wired this method into the
|
||||
// "Run Backup Now" inline-dump path — before that, only the
|
||||
// scheduled-cron + dedicated-admin-page callers exercised it,
|
||||
// and Ralf's install had never triggered either. Cure: same
|
||||
// explicit `.returning('id')` + dual-shape coalesce pattern that
|
||||
// `backupService.js:949` uses for its own `backup_runs` insert.
|
||||
const insertResult = await db('database_backup_runs').insert({
|
||||
started_at: startTime,
|
||||
status: 'running',
|
||||
backup_type: this.dbType,
|
||||
@@ -324,8 +347,9 @@ class DatabaseBackupService {
|
||||
node_env: process.env.NODE_ENV || 'production',
|
||||
db_type: this.dbType
|
||||
})
|
||||
});
|
||||
|
||||
}).returning('id');
|
||||
const runId = insertResult[0]?.id || insertResult[0];
|
||||
|
||||
backupRun = { id: runId };
|
||||
|
||||
// Get initial checksums
|
||||
|
||||
@@ -39,6 +39,12 @@ class RestoreService {
|
||||
this.currentProgress = null;
|
||||
this.restoreLog = [];
|
||||
this.preRestoreBackupPath = null;
|
||||
// Snapshot of operator-meta settings (e.g. `restore_allow_force`)
|
||||
// captured by performDatabaseRestore BEFORE the DROP DATABASE.
|
||||
// Drained by restore() AFTER post-restore verification passes so
|
||||
// the replay doesn't inflate the row-count check. Reset per run
|
||||
// via beforeRestore() to keep state from leaking across calls.
|
||||
this.preservedMetaSnapshot = [];
|
||||
this.dbType = knexConfig.client === 'pg' ? 'postgresql' : 'sqlite';
|
||||
this.tempDir = path.join(os.tmpdir(), 'picpeak-restore');
|
||||
}
|
||||
@@ -64,6 +70,7 @@ class RestoreService {
|
||||
|
||||
this.isRunning = true;
|
||||
this.restoreLog = [];
|
||||
this.preservedMetaSnapshot = []; // reset per run
|
||||
const startTime = new Date();
|
||||
let restoreRun = null;
|
||||
|
||||
@@ -154,9 +161,37 @@ class RestoreService {
|
||||
this.log('warn', 'Pre-restore backup skipped at user request');
|
||||
}
|
||||
|
||||
// Step 5: Download backup if from S3
|
||||
// Step 5: Download backup if from S3, or resolve the local root.
|
||||
//
|
||||
// The wizard passes `options.source = 'local'` (the SOURCE TYPE
|
||||
// string) — not a path. The old code assigned that string to
|
||||
// `localBackupPath` verbatim and every downstream `path.join(...)`
|
||||
// ended up with junk like `local/database/<file>.sql.gz`. Caused
|
||||
// the disaster-recovery restore flow to fail with
|
||||
// `Database backup file not found: local/database/...` even when
|
||||
// the manifest recorded the correct absolute path AND the file
|
||||
// existed at exactly that path on disk.
|
||||
//
|
||||
// Resolve `'local'` to the configured backup destination root by
|
||||
// reading `backup_destination_path` from app_settings. That's the
|
||||
// same root the file-backup walker writes to, so every relative
|
||||
// `file.path` in the manifest resolves correctly via
|
||||
// `path.join(localBackupPath, file.path)` further down.
|
||||
let localBackupPath = options.source;
|
||||
if (options.source.startsWith('s3://')) {
|
||||
if (options.source === 'local') {
|
||||
try {
|
||||
const row = await db('app_settings')
|
||||
.where('setting_key', 'backup_destination_path')
|
||||
.first();
|
||||
if (row?.setting_value) {
|
||||
let parsed;
|
||||
try { parsed = JSON.parse(row.setting_value); } catch (_) { parsed = row.setting_value; }
|
||||
if (parsed) localBackupPath = parsed;
|
||||
}
|
||||
} catch (err) {
|
||||
this.log('warn', `Could not resolve backup_destination_path: ${err.message}`);
|
||||
}
|
||||
} else if (options.source.startsWith('s3://')) {
|
||||
this.updateProgress('Downloading backup from S3...');
|
||||
localBackupPath = await this.downloadFromS3(options.source, manifest, options);
|
||||
}
|
||||
@@ -193,6 +228,89 @@ class RestoreService {
|
||||
throw new Error(`Post-restore verification failed: ${verification.errors.join(', ')}`);
|
||||
}
|
||||
|
||||
// Step 7b: Replay operator-meta settings AFTER verification.
|
||||
//
|
||||
// `performDatabaseRestore` stashed the pre-DROP snapshot of
|
||||
// operator-meta keys on `this.preservedMetaSnapshot`. We drain
|
||||
// it here, AFTER verification has already confirmed the
|
||||
// restored DB matches the backup's row counts. Running this
|
||||
// upsert sequence here instead of inside performDatabaseRestore
|
||||
// (where it used to live) prevents the replay from inflating
|
||||
// the post-restore row count and tripping the verification
|
||||
// check — see the round-3 PR #596 notes for the full story.
|
||||
//
|
||||
// UPSERT by setting_key: if the backup had the same key with a
|
||||
// different value, we overwrite; if the row doesn't exist in
|
||||
// the backup, we insert. Either way the operator's pre-restore
|
||||
// policy survives. SQLite branch leaves the snapshot empty so
|
||||
// this block is a no-op there.
|
||||
if (this.preservedMetaSnapshot && this.preservedMetaSnapshot.length > 0) {
|
||||
try {
|
||||
for (const row of this.preservedMetaSnapshot) {
|
||||
await db('app_settings')
|
||||
.insert({
|
||||
setting_key: row.setting_key,
|
||||
setting_value: row.setting_value,
|
||||
setting_type: row.setting_type || 'restore',
|
||||
updated_at: new Date(),
|
||||
})
|
||||
.onConflict('setting_key')
|
||||
.merge({
|
||||
setting_value: row.setting_value,
|
||||
updated_at: new Date(),
|
||||
});
|
||||
}
|
||||
this.log('info', `Replayed ${this.preservedMetaSnapshot.length} restore-meta setting(s) post-verification`);
|
||||
} catch (err) {
|
||||
this.log('warn', `Could not replay restore-meta settings (admin may need to re-set them): ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 7c: Apply any post-backup migrations to the restored DB.
|
||||
//
|
||||
// The backup carries the schema state of whatever migrations had
|
||||
// been applied at backup time. If the running image is NEWER —
|
||||
// because the admin upgraded picpeak between when the backup
|
||||
// was taken and when they restored — the restored DB ends up
|
||||
// mismatched against the running code: queries fail, new
|
||||
// columns are missing, new tables don't exist.
|
||||
//
|
||||
// Previously the comment said "deferred to next container
|
||||
// restart" — but that left the running process serving a
|
||||
// mismatched schema until the operator manually restarted.
|
||||
// Not acceptable per the "backup must restore completely even
|
||||
// when new features have been added in the meantime" contract.
|
||||
//
|
||||
// Implementation: shell out to `npm run migrate:safe`, which is
|
||||
// the EXACT script wait-for-db.sh runs on boot. Running it as a
|
||||
// subprocess means no risk to our reinit'd pool (subprocess gets
|
||||
// its own knex instance, destroys it on exit; our parent pool
|
||||
// is untouched). Idempotent — migrations already applied are
|
||||
// tracked in the restored `migrations` table and get skipped.
|
||||
//
|
||||
// Failure is non-fatal: the restore data itself is in place,
|
||||
// and the next container restart's wait-for-db.sh will retry.
|
||||
// Surfacing the error gives the operator a chance to investigate
|
||||
// proactively rather than discovering it on the next 500 from
|
||||
// a missing column.
|
||||
try {
|
||||
this.log('info', 'Applying post-restore migrations to restored database...');
|
||||
this.updateProgress('Applying any post-backup migrations...');
|
||||
const backendRoot = path.join(__dirname, '..', '..');
|
||||
const { stderr } = await spawnAsync('npm', ['run', 'migrate:safe'], {
|
||||
cwd: backendRoot,
|
||||
env: { ...process.env },
|
||||
});
|
||||
if (stderr && stderr.trim()) {
|
||||
this.log('info', `Post-restore migrate:safe stderr: ${stderr.slice(0, 500)}`);
|
||||
}
|
||||
this.log('info', 'Post-restore migrations applied');
|
||||
} catch (migErr) {
|
||||
this.log('warn',
|
||||
`Post-restore migrate:safe failed — restore data is in place but the schema may lag the running image. ` +
|
||||
`A container restart will retry via wait-for-db.sh. Error: ${migErr.message}`);
|
||||
}
|
||||
|
||||
// Step 8: Clean up temporary files
|
||||
if (localBackupPath !== options.source) {
|
||||
await fs.unlink(localBackupPath).catch(err =>
|
||||
@@ -208,6 +326,15 @@ class RestoreService {
|
||||
await db('restore_runs').where('id', runId).update({
|
||||
completed_at: endTime,
|
||||
status: 'completed',
|
||||
// Default for the column is `false`. Without this line, every
|
||||
// SUCCESSFUL restore ends up with `status='completed',
|
||||
// was_successful=false` — which the BackupDashboard "last
|
||||
// successful restore" widget then filters out, and any future
|
||||
// audit query that gates on was_successful misses the row
|
||||
// entirely. Cosmetic but enough to mislead an operator
|
||||
// scanning restore history. Catches Ralf 2026-06-01 + maintainer
|
||||
// PR #596 review note about the cosmetic.
|
||||
was_successful: true,
|
||||
duration_seconds: durationSeconds,
|
||||
pre_restore_backup_path: this.preRestoreBackupPath,
|
||||
statistics: JSON.stringify({
|
||||
@@ -242,12 +369,53 @@ class RestoreService {
|
||||
} catch (error) {
|
||||
this.log('error', 'Restore failed', { error: error.message, stack: error.stack });
|
||||
|
||||
// Update restore run record
|
||||
// Always attempt rollback when a pre-restore backup exists.
|
||||
// Historically rollback was only triggered when post-restore
|
||||
// verification failed (inside the try block) — anything that
|
||||
// threw earlier (path-resolution bugs, pg_restore failure, file
|
||||
// copy errors) left the destination half-clobbered and forced
|
||||
// the admin to do another reset-from-volume cycle before the
|
||||
// next attempt could be honest. Fixing the rollback here closes
|
||||
// the "every failed restore makes the next one worse" footgun.
|
||||
let rollbackAttempted = false;
|
||||
let rollbackSucceeded = false;
|
||||
let rollbackError = null;
|
||||
if (this.preRestoreBackupPath) {
|
||||
rollbackAttempted = true;
|
||||
try {
|
||||
this.log('info', 'Attempting rollback from pre-restore safety backup', {
|
||||
path: this.preRestoreBackupPath,
|
||||
});
|
||||
await this.attemptRollback(this.preRestoreBackupPath);
|
||||
rollbackSucceeded = true;
|
||||
this.log('info', 'Rollback completed');
|
||||
} catch (rbErr) {
|
||||
rollbackError = rbErr.message;
|
||||
this.log('error', 'Rollback FAILED — install may be in a partial state',
|
||||
{ error: rbErr.message, stack: rbErr.stack });
|
||||
}
|
||||
} else {
|
||||
this.log('warn', 'No pre-restore backup available — cannot auto-rollback. ' +
|
||||
'Destination may be in a partial state. Verify business-docs/ and the DB before retrying.');
|
||||
}
|
||||
|
||||
// Update restore run record. We persist BOTH the original
|
||||
// restore failure AND the rollback status so the admin can tell
|
||||
// from a single SQL query which scenario they're in:
|
||||
// - rollback succeeded → destination is back to pre-restore state, safe to retry
|
||||
// - rollback failed → partial state, admin must inspect before next attempt
|
||||
// - rollback skipped → user opted out via skipPreBackup; same as above
|
||||
if (restoreRun) {
|
||||
const failureMessage = rollbackAttempted
|
||||
? (rollbackSucceeded
|
||||
? `${error.message} (rolled back successfully to pre-restore state)`
|
||||
: `${error.message} | ROLLBACK ALSO FAILED: ${rollbackError} — destination is in a partial state, inspect before retrying`)
|
||||
: `${error.message} (no pre-restore backup available — destination may be partial)`;
|
||||
await db('restore_runs').where('id', restoreRun.id).update({
|
||||
completed_at: new Date(),
|
||||
status: 'failed',
|
||||
error_message: error.message,
|
||||
error_message: failureMessage,
|
||||
was_rollback_attempted: rollbackAttempted,
|
||||
restore_log: JSON.stringify(this.restoreLog)
|
||||
});
|
||||
}
|
||||
@@ -255,7 +423,9 @@ class RestoreService {
|
||||
// Send failure notification
|
||||
await this.sendRestoreNotification('failure', {
|
||||
error: error.message,
|
||||
restoreType: options.restoreType
|
||||
restoreType: options.restoreType,
|
||||
rollbackAttempted,
|
||||
rollbackSucceeded,
|
||||
});
|
||||
|
||||
throw error;
|
||||
@@ -371,21 +541,32 @@ class RestoreService {
|
||||
}
|
||||
}
|
||||
|
||||
// Check if restoring would overwrite existing data
|
||||
// Check if restoring would overwrite existing data.
|
||||
//
|
||||
// NOTE: pg-driver returns `count('* as count')` as a STRING (it
|
||||
// serialises `bigint` to string to avoid JS precision loss for
|
||||
// huge counts) — see PR #596 review for the `bigint`-as-string
|
||||
// discussion. Both blocks below coerce to `Number` before
|
||||
// comparing AND before interpolating into the warning text, so
|
||||
// the count renders as `5` not `"5"` regardless of DB driver.
|
||||
// Don't drop the `Number()` calls without also re-auditing the
|
||||
// strict-equality call sites flagged in the same review.
|
||||
if (options.restoreType === 'full' || options.restoreType === 'database') {
|
||||
const eventCount = await db('events').count('* as count').first();
|
||||
if (eventCount && eventCount.count > 0) {
|
||||
validation.warnings.push(`Database contains ${eventCount.count} existing events that will be overwritten`);
|
||||
const eventCountN = Number(eventCount?.count || 0);
|
||||
if (eventCountN > 0) {
|
||||
validation.warnings.push(`Database contains ${eventCountN} existing events that will be overwritten`);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for active users
|
||||
// Check for active users (same coercion contract as above).
|
||||
const activeUsers = await db('admin_users')
|
||||
.where('is_active', formatBoolean(true))
|
||||
.count('* as count')
|
||||
.first();
|
||||
if (activeUsers && activeUsers.count > 0) {
|
||||
validation.warnings.push(`There are ${activeUsers.count} active admin users`);
|
||||
const activeUsersN = Number(activeUsers?.count || 0);
|
||||
if (activeUsersN > 0) {
|
||||
validation.warnings.push(`There are ${activeUsersN} active admin users`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
@@ -661,13 +842,55 @@ class RestoreService {
|
||||
throw new Error('No database backup file found in manifest');
|
||||
}
|
||||
|
||||
const dbBackupPath = path.join(backupPath, 'database', path.basename(dbBackupFile));
|
||||
|
||||
// Check if backup file exists
|
||||
try {
|
||||
await fs.access(dbBackupPath);
|
||||
} catch (error) {
|
||||
throw new Error(`Database backup file not found: ${dbBackupPath}`);
|
||||
// Layered resolution for the database dump path:
|
||||
//
|
||||
// 1. Manifest stores the absolute path the dumper wrote to
|
||||
// (e.g. `/backup/database/picpeak-db-postgresql-<ts>.sql.gz`).
|
||||
// That's the truth — try it first.
|
||||
// 2. Some older manifests store a path RELATIVE to the file-backup
|
||||
// destination root (`database/<file>.sql.gz`). Reconstruct that
|
||||
// way as a fallback.
|
||||
// 3. Final fallback: `<backupPath>/database/<basename>`, the
|
||||
// historical reconstruction used before this fix. Preserved so
|
||||
// no existing valid path breaks.
|
||||
//
|
||||
// The original code used (3) exclusively, which meant the restore
|
||||
// service ignored the absolute path the manifest recorded and
|
||||
// looked under a synthetic `<backupPath>/database/<file>` root —
|
||||
// which on Ralf's install became `local/database/<file>` because
|
||||
// `backupPath` was the source type string, not a directory. Caused
|
||||
// the canonical disaster-recovery flow to fail with
|
||||
// `Database backup file not found: local/database/...sql.gz`
|
||||
// even though the file existed at exactly the path the manifest
|
||||
// recorded.
|
||||
const candidates = [
|
||||
// (1) Honour absolute paths recorded by the dumper.
|
||||
path.isAbsolute(dbBackupFile) ? dbBackupFile : null,
|
||||
// (2) Relative-to-backupPath as-stored (no basename munging).
|
||||
path.join(backupPath, dbBackupFile),
|
||||
// (3) Legacy reconstruct.
|
||||
path.join(backupPath, 'database', path.basename(dbBackupFile)),
|
||||
].filter(Boolean);
|
||||
|
||||
let dbBackupPath = null;
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
await fs.access(candidate);
|
||||
dbBackupPath = candidate;
|
||||
break;
|
||||
} catch (_) {
|
||||
// try next candidate
|
||||
}
|
||||
}
|
||||
|
||||
if (!dbBackupPath) {
|
||||
throw new Error(
|
||||
`Database backup file not found. Tried: ${candidates.join(', ')}. ` +
|
||||
`Manifest recorded path: ${dbBackupFile}. ` +
|
||||
`Hint: this usually means the manifest's database.backup_file path no longer ` +
|
||||
`exists on disk (deleted? moved? volume not mounted?). Check ` +
|
||||
`~/<your-compose-dir>/backup/database/ on the host.`
|
||||
);
|
||||
}
|
||||
|
||||
// Decompress if needed
|
||||
@@ -679,6 +902,31 @@ class RestoreService {
|
||||
restoreFile = decompressedPath;
|
||||
}
|
||||
|
||||
// Snapshot of operator-meta keys captured BEFORE the DROP.
|
||||
// Stashed onto `this.preservedMetaSnapshot` so the parent
|
||||
// `restore()` method can drain + apply it AFTER post-restore
|
||||
// verification passes. Order matters here:
|
||||
//
|
||||
// - PR #596 round 1: lifted the declaration above the
|
||||
// SQLite/PG split to fix a ReferenceError when the replay
|
||||
// was inline at the bottom of this method.
|
||||
// - PR #596 round 3: moved the REPLAY itself out of here and
|
||||
// into restore(), because the round-1 in-method replay ran
|
||||
// BEFORE post-restore verification — which then counted the
|
||||
// replayed row and flagged
|
||||
// Table app_settings row count mismatch: expected 190, got 191
|
||||
// as a verification failure even though both Stage A and
|
||||
// the replay had succeeded. Verification now sees the
|
||||
// as-restored DB (matches the backup exactly), replay layers
|
||||
// on top after verification has signed off.
|
||||
//
|
||||
// SQLite branch leaves `preservedMetaSnapshot` empty — verification
|
||||
// and replay both no-op for it, unchanged behaviour.
|
||||
const PRESERVED_META_KEYS = [
|
||||
'restore_allow_force',
|
||||
'restore_allow_force_auto_upgraded',
|
||||
];
|
||||
|
||||
try {
|
||||
if (this.dbType === 'sqlite') {
|
||||
// SQLite restore
|
||||
@@ -715,24 +963,210 @@ class RestoreService {
|
||||
// PostgreSQL restore
|
||||
const { host, port, user, password, database } = knexConfig.connection;
|
||||
const env = { ...process.env, PGPASSWORD: password };
|
||||
|
||||
|
||||
// Snapshot operator-meta settings BEFORE the DROP so we can
|
||||
// restore them after the psql load. These keys are about how
|
||||
// the operator wants the install to behave (force-restore
|
||||
// permission, auto-upgrade tracking), not user-facing state —
|
||||
// they should NOT be overwritten by whatever values the backup
|
||||
// happens to contain.
|
||||
//
|
||||
// Chicken-and-egg this closes: `restore_allow_force` defaults
|
||||
// to true (post tonight's migration 032 edit), but every
|
||||
// restore would overwrite it with whatever the backup carried.
|
||||
// Admin sets it to true → restores → wakes up with the row
|
||||
// back to whatever was in the backup. Two consecutive restores
|
||||
// needed the SQL workaround again. With this snapshot/replay,
|
||||
// the operator's policy persists across restores.
|
||||
//
|
||||
// PRESERVED_META_KEYS is declared above the SQLite/PG split
|
||||
// (~L820). The snapshot READ happens here in the PG branch
|
||||
// (must run before DROP), but is stashed on
|
||||
// `this.preservedMetaSnapshot` for the parent `restore()`
|
||||
// method to consume AFTER verification — see the round-3
|
||||
// notes there.
|
||||
try {
|
||||
this.preservedMetaSnapshot = await db('app_settings')
|
||||
.whereIn('setting_key', PRESERVED_META_KEYS)
|
||||
.select('setting_key', 'setting_value', 'setting_type');
|
||||
this.log('info', `Snapshotted ${this.preservedMetaSnapshot.length} restore-meta setting(s) for post-restore replay`, {
|
||||
keys: this.preservedMetaSnapshot.map(r => r.setting_key),
|
||||
});
|
||||
} catch (err) {
|
||||
this.log('warn', `Could not snapshot restore-meta settings (continuing): ${err.message}`);
|
||||
}
|
||||
|
||||
// `psql` with no `-d` defaults to a database whose name matches
|
||||
// the connecting user, NOT a maintenance DB. So on installs
|
||||
// where the user's home DB doesn't exist (e.g. user=`picpeak`,
|
||||
// target DB=`picpeak_prod`, no `picpeak` DB), the next two
|
||||
// statements failed with:
|
||||
// FATAL: database "picpeak" does not exist
|
||||
// even though the actual target DB was alive and connectable.
|
||||
//
|
||||
// Fix: explicitly connect to `postgres` (the maintenance DB
|
||||
// every PG cluster ships with) for the DROP/CREATE. We can't
|
||||
// connect to the target DB itself anyway — DROP DATABASE
|
||||
// refuses to run while a connection is open to it.
|
||||
//
|
||||
// Use `DB_CHECK_DB` env var as an override hook (matches the
|
||||
// pattern wait-for-db.sh already exposes) for installs where
|
||||
// the `postgres` DB is restricted to superusers.
|
||||
const maintenanceDb = process.env.DB_CHECK_DB || 'postgres';
|
||||
|
||||
// The backend's own knex pool holds N active connections to
|
||||
// the target database (default 5-25 per knexfile.js). PostgreSQL
|
||||
// refuses DROP DATABASE while any session is connected:
|
||||
// ERROR: database "X" is being accessed by other users
|
||||
// DETAIL: There are N other sessions using the database.
|
||||
// We have to evict those sessions ourselves before issuing the
|
||||
// DROP. Two-step approach:
|
||||
// 1. Close knex's own pool so we don't fight ourselves.
|
||||
// 2. pg_terminate_backend() the rest (other server replicas,
|
||||
// pg_stat_activity stragglers, leftover idle txns).
|
||||
//
|
||||
// After CREATE DATABASE, knex will lazily re-open the pool on
|
||||
// the next query — handled by db.js's connection retry logic.
|
||||
this.log('warn', 'Closing knex pool before dropping target database...');
|
||||
try { await db.destroy(); } catch (poolErr) {
|
||||
this.log('warn', `Pool destroy threw (continuing): ${poolErr.message}`);
|
||||
}
|
||||
|
||||
this.log('warn', 'Terminating any remaining sessions on target database...', {
|
||||
target: database,
|
||||
});
|
||||
// pg_terminate_backend takes a pid. Kill every session against
|
||||
// the target DB except our own connection (which is to the
|
||||
// maintenance DB anyway). Wrapped in `SELECT ... FROM ... WHERE`
|
||||
// so we get one psql round-trip instead of N.
|
||||
await spawnAsync('psql', [
|
||||
'-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb,
|
||||
'-c',
|
||||
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity ` +
|
||||
`WHERE datname = '${database.replace(/'/g, "''")}' AND pid <> pg_backend_pid()`,
|
||||
], { env });
|
||||
|
||||
// Drop and recreate database (extremely dangerous!)
|
||||
this.log('warn', 'Dropping and recreating PostgreSQL database...');
|
||||
|
||||
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-c', `DROP DATABASE IF EXISTS ${database}`], { env });
|
||||
this.log('warn', 'Dropping and recreating PostgreSQL database...', {
|
||||
target: database, via: maintenanceDb,
|
||||
});
|
||||
|
||||
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-c', `CREATE DATABASE ${database}`], { env });
|
||||
// WITH (FORCE) on Postgres 13+ kills any remaining connections
|
||||
// atomically with the DROP. On older Postgres the FORCE option
|
||||
// doesn't exist, so we fall back to plain DROP IF EXISTS — by
|
||||
// which point pg_terminate_backend should have cleared the
|
||||
// table. Try FORCE first, fall back to plain on syntax error.
|
||||
try {
|
||||
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb,
|
||||
'-c', `DROP DATABASE IF EXISTS "${database}" WITH (FORCE)`], { env });
|
||||
} catch (forceErr) {
|
||||
// PG < 13: WITH (FORCE) is a syntax error. Plain DROP after
|
||||
// our pg_terminate_backend pass should now succeed.
|
||||
this.log('info', 'DROP DATABASE WITH (FORCE) not supported — falling back to plain DROP', {
|
||||
error: forceErr.message,
|
||||
});
|
||||
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb,
|
||||
'-c', `DROP DATABASE IF EXISTS "${database}"`], { env });
|
||||
}
|
||||
|
||||
// Restore from backup
|
||||
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb, '-c', `CREATE DATABASE "${database}"`], { env });
|
||||
|
||||
// Restore from backup — this one DOES connect to the target DB.
|
||||
await spawnFromFile('psql', ['-h', host, '-p', String(port), '-U', user, '-d', database], restoreFile, { env });
|
||||
|
||||
// Re-sync every SERIAL / IDENTITY sequence in the public schema
|
||||
// to MAX(id)+1 of its owning table. pg_dump emits setval()
|
||||
// statements, but they don't always land cleanly when:
|
||||
// - the dump has `--clean` (the setval may execute before
|
||||
// the rebuilt rows, depending on dump ordering)
|
||||
// - the in-process knex pool had a cached sequence value
|
||||
// before db.destroy() (already mitigated, but defensive)
|
||||
// - rows were inserted mid-restore (the pre-restore safety
|
||||
// backup creates a database_backup_runs row before DROP)
|
||||
// Result if skipped: every subsequent INSERT into a serial-id
|
||||
// table fails with `duplicate key value violates unique
|
||||
// constraint "<table>_pkey"`. Surfaced on Ralf's install as
|
||||
// "A record with this value already exists" on every CRUD
|
||||
// action AND `database_backup_runs_pkey` violation on the
|
||||
// next Run Backup Now. Fix is a single DO block that walks
|
||||
// pg_class + pg_attribute and setval()s each sequence to
|
||||
// GREATEST(MAX(<col>), 1). Cheap (a few ms even on large
|
||||
// schemas), safe (doesn't touch row data), idempotent.
|
||||
this.log('info', 'Re-syncing PostgreSQL sequences to MAX(id) of each table...');
|
||||
await spawnAsync('psql', [
|
||||
'-h', host, '-p', String(port), '-U', user, '-d', database,
|
||||
'-c',
|
||||
`DO $$
|
||||
DECLARE
|
||||
r RECORD;
|
||||
max_id BIGINT;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT n.nspname AS schema_name, t.relname AS table_name, a.attname AS column_name,
|
||||
pg_get_serial_sequence(quote_ident(n.nspname) || '.' || quote_ident(t.relname), a.attname) AS seq_name
|
||||
FROM pg_class t
|
||||
JOIN pg_namespace n ON n.oid = t.relnamespace
|
||||
JOIN pg_attribute a ON a.attrelid = t.oid
|
||||
WHERE n.nspname = 'public'
|
||||
AND t.relkind = 'r'
|
||||
AND a.attnum > 0
|
||||
AND NOT a.attisdropped
|
||||
AND pg_get_serial_sequence(quote_ident(n.nspname) || '.' || quote_ident(t.relname), a.attname) IS NOT NULL
|
||||
LOOP
|
||||
EXECUTE format('SELECT COALESCE(MAX(%I), 0) FROM %I.%I', r.column_name, r.schema_name, r.table_name) INTO max_id;
|
||||
EXECUTE format('SELECT setval(%L, %s, true)', r.seq_name, GREATEST(max_id, 1));
|
||||
END LOOP;
|
||||
END $$;`
|
||||
], { env });
|
||||
this.log('info', 'Sequence resync completed');
|
||||
}
|
||||
|
||||
// Re-initialize database connection
|
||||
const { db: newDb } = require('../database/db');
|
||||
|
||||
// Run migrations to ensure schema is up to date
|
||||
this.log('info', 'Running database migrations...');
|
||||
await newDb.migrate.latest();
|
||||
// Re-initialize the in-process knex pool. The DROP/CREATE
|
||||
// DATABASE pair above destroyed our connections and the recreated
|
||||
// database has a different pg_database OID — any pooled
|
||||
// connection from before would either be dead or pointed at a
|
||||
// ghost. Without explicit reinit, every query in the process
|
||||
// after restore returns `Error: Unable to acquire a connection`
|
||||
// until the container is manually restarted (and admin sees
|
||||
// "An error occurred" on the login screen even after the restore
|
||||
// technically succeeded). reinitPool destroys + rebuilds the
|
||||
// pool and probes the new one with `SELECT 1` so failures here
|
||||
// surface immediately instead of polluting the next request.
|
||||
const { reinitPool } = require('../database/db');
|
||||
this.log('info', 'Re-initializing knex pool against the restored database...');
|
||||
await reinitPool();
|
||||
this.log('info', 'Knex pool re-initialized');
|
||||
|
||||
// NOTE: we deliberately do NOT call `db.migrate.latest()` here.
|
||||
//
|
||||
// The picpeak migrations directory contains `helpers.js` (a
|
||||
// shared helper module, not a migration), plus `core/` and
|
||||
// `legacy/` subdirectories. Knex's built-in migrator scans the
|
||||
// top-level directory and rejects any file without `up`/`down`
|
||||
// exports — so `db.migrate.latest()` throws
|
||||
// Invalid migration: helpers.js must have both an up and down function
|
||||
// every time it runs in this codebase. The production code path
|
||||
// uses `npm run migrate:safe` (run-migrations-safe.js) which
|
||||
// knows to skip helpers.js + walks core/ explicitly.
|
||||
//
|
||||
// The safe runner gets invoked AFTER verification in restore()
|
||||
// (see step 7c) to apply any post-backup migrations to the
|
||||
// restored DB. This closes the contract "backup must restore
|
||||
// completely even when new features have been added in the
|
||||
// meantime" — without this step, restoring an old backup on a
|
||||
// newer image would leave the running process serving a
|
||||
// mismatched schema until the next container restart.
|
||||
this.log('info', 'Schema migrations deferred to restore() step 7c (npm run migrate:safe subprocess)');
|
||||
|
||||
// NOTE: operator-meta REPLAY does NOT happen here any more.
|
||||
// PR #596 round 3: if the replay runs inside performDatabaseRestore,
|
||||
// it lands BEFORE post-restore verification — and verification
|
||||
// then counts the replayed row as a mismatch (e.g. "expected 190,
|
||||
// got 191" because the fresh-install seeded
|
||||
// `restore_allow_force_auto_upgraded` that wasn't in the backup).
|
||||
// Replay is now drained by the parent `restore()` method AFTER
|
||||
// verification passes. Snapshot lives on
|
||||
// `this.preservedMetaSnapshot` for that drain.
|
||||
|
||||
return { success: true };
|
||||
|
||||
@@ -905,9 +1339,20 @@ class RestoreService {
|
||||
for (const [table, expected] of Object.entries(manifest.database.row_counts)) {
|
||||
try {
|
||||
const result = await db(table).count('* as count').first();
|
||||
if (result.count !== expected.rowCount) {
|
||||
// pg-driver serialises `bigint` as string to preserve
|
||||
// precision for huge counts, so `result.count` on PG is
|
||||
// e.g. `"16"` while the manifest's `expected.rowCount`
|
||||
// is the JS number `16`. Strict `!==` flagged every
|
||||
// match as a mismatch on PG. Caught on PR #596 review:
|
||||
// `Table activity_logs row count mismatch:
|
||||
// expected 16, got 16`
|
||||
// every table, all "matching". Coerce both sides to
|
||||
// Number to compare reliably across SQLite (number) and
|
||||
// PG (string).
|
||||
const actual = Number(result.count);
|
||||
if (actual !== expected.rowCount) {
|
||||
verification.errors.push(
|
||||
`Table ${table} row count mismatch: expected ${expected.rowCount}, got ${result.count}`
|
||||
`Table ${table} row count mismatch: expected ${expected.rowCount}, got ${actual}`
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -41,60 +41,128 @@ function spawnAsync(cmd, args = [], options = {}) {
|
||||
|
||||
/**
|
||||
* Run a command and redirect stdout to a file (replaces shell `> file`).
|
||||
*
|
||||
* Historically this passed `fs.createWriteStream(outputPath)` directly as
|
||||
* `stdio[1]` to `child_process.spawn`. That relied on Node auto-extracting
|
||||
* the WriteStream's `.fd` — but the stream opens async, so on a fast call
|
||||
* `fd` is still `null` when `spawn()` reads it. Older Node releases would
|
||||
* tolerate this; Node 22 throws synchronously with
|
||||
* `The argument 'stdio' is invalid. Received WriteStream { fd: null, ... }`.
|
||||
*
|
||||
* Cure: use `stdio: ['ignore', 'pipe', 'pipe']` and wire the WriteStream
|
||||
* up via the streams API (`stdout.pipe(outStream)`). Works on every Node
|
||||
* version; also gives us a clean error bridge from both the WriteStream
|
||||
* AND the child process to the promise, instead of the previous code's
|
||||
* blind `outStream.destroy()` / `outStream.end()` calls that left stream
|
||||
* errors uncaught (Node 22 process-fatal — separate footgun this fixes
|
||||
* by the same change).
|
||||
*
|
||||
* Used by:
|
||||
* - databaseBackup.createPostgreSQLBackup (inline-dump path, the
|
||||
* thing that just bit Ralf's install)
|
||||
* - restoreService pre-restore safety snapshot (would have hit the
|
||||
* same on next restore attempt)
|
||||
*/
|
||||
function spawnToFile(cmd, args, outputPath, options = {}) {
|
||||
const fs = require('fs');
|
||||
return new Promise((resolve, reject) => {
|
||||
const outStream = fs.createWriteStream(outputPath);
|
||||
let settled = false;
|
||||
const settleReject = (err) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
try { outStream.destroy(); } catch (_) { /* best effort */ }
|
||||
reject(err);
|
||||
};
|
||||
const settleResolve = (value) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve(value);
|
||||
};
|
||||
|
||||
// Bridge WriteStream errors (EACCES, ENOSPC, etc.) to the promise.
|
||||
// Without this, an unhandled 'error' event on the stream is process-
|
||||
// fatal on Node 22 and bypasses the caller's try/catch entirely —
|
||||
// which is exactly the failure mode that crashed the picpeak
|
||||
// backend container on its first inline-dump attempt.
|
||||
outStream.on('error', settleReject);
|
||||
|
||||
const child = spawn(cmd, args, {
|
||||
shell: false,
|
||||
...options,
|
||||
stdio: ['ignore', outStream, 'pipe']
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
});
|
||||
|
||||
// Pipe stdout → file. The pipe call attaches its own 'error'
|
||||
// handlers on both ends so a child-stdout failure also reaches us.
|
||||
child.stdout.pipe(outStream);
|
||||
|
||||
const stderrChunks = [];
|
||||
child.stderr.on('data', chunk => stderrChunks.push(chunk));
|
||||
child.stderr.on('error', settleReject);
|
||||
|
||||
child.on('error', (err) => {
|
||||
outStream.destroy();
|
||||
reject(err);
|
||||
});
|
||||
child.on('error', settleReject);
|
||||
child.on('close', (code) => {
|
||||
outStream.end();
|
||||
const stderr = Buffer.concat(stderrChunks).toString();
|
||||
if (code !== 0) {
|
||||
const err = new Error(`${cmd} exited with code ${code}: ${stderr}`);
|
||||
err.code = code;
|
||||
err.stderr = stderr;
|
||||
return reject(err);
|
||||
}
|
||||
resolve({ stderr });
|
||||
// Wait for the file write to flush before resolving — otherwise
|
||||
// a fast 'close' could resolve while the WriteStream still has
|
||||
// buffered bytes, producing a truncated dump.
|
||||
outStream.end(() => {
|
||||
const stderr = Buffer.concat(stderrChunks).toString();
|
||||
if (code !== 0) {
|
||||
const err = new Error(`${cmd} exited with code ${code}: ${stderr}`);
|
||||
err.code = code;
|
||||
err.stderr = stderr;
|
||||
return settleReject(err);
|
||||
}
|
||||
settleResolve({ stderr });
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a command and pipe a file into stdin (replaces shell `< file`).
|
||||
*
|
||||
* Same Node 22 stdio strictness applies as for `spawnToFile` above — the
|
||||
* ReadStream `fd` is null at spawn time. Use `stdio[0] = 'pipe'` and pipe
|
||||
* the file stream into `child.stdin` via the streams API instead.
|
||||
*/
|
||||
function spawnFromFile(cmd, args, inputPath, options = {}) {
|
||||
const fs = require('fs');
|
||||
return new Promise((resolve, reject) => {
|
||||
const inStream = fs.createReadStream(inputPath);
|
||||
let settled = false;
|
||||
const settleReject = (err) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
try { inStream.destroy(); } catch (_) { /* best effort */ }
|
||||
reject(err);
|
||||
};
|
||||
const settleResolve = (value) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve(value);
|
||||
};
|
||||
|
||||
inStream.on('error', settleReject);
|
||||
|
||||
const child = spawn(cmd, args, {
|
||||
shell: false,
|
||||
...options,
|
||||
stdio: [inStream, 'pipe', 'pipe']
|
||||
stdio: ['pipe', 'pipe', 'pipe']
|
||||
});
|
||||
|
||||
inStream.pipe(child.stdin);
|
||||
|
||||
const stdoutChunks = [];
|
||||
const stderrChunks = [];
|
||||
child.stdout.on('data', chunk => stdoutChunks.push(chunk));
|
||||
child.stdout.on('error', settleReject);
|
||||
child.stderr.on('data', chunk => stderrChunks.push(chunk));
|
||||
child.stderr.on('error', settleReject);
|
||||
child.stdin.on('error', settleReject);
|
||||
|
||||
child.on('error', (err) => {
|
||||
inStream.destroy();
|
||||
reject(err);
|
||||
});
|
||||
child.on('error', settleReject);
|
||||
child.on('close', (code) => {
|
||||
const stdout = Buffer.concat(stdoutChunks).toString();
|
||||
const stderr = Buffer.concat(stderrChunks).toString();
|
||||
@@ -103,9 +171,9 @@ function spawnFromFile(cmd, args, inputPath, options = {}) {
|
||||
err.code = code;
|
||||
err.stdout = stdout;
|
||||
err.stderr = stderr;
|
||||
return reject(err);
|
||||
return settleReject(err);
|
||||
}
|
||||
resolve({ stdout, stderr });
|
||||
settleResolve({ stdout, stderr });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user