Merge pull request #596 from Luca-Timo/bugfix/crm-backup

Backup & Restore hardening — close the silent files-only data-loss class
This commit is contained in:
Paul Nothaft
2026-06-02 09:10:57 +02:00
committed by GitHub
38 changed files with 6042 additions and 251 deletions
+1 -1
View File
@@ -146,7 +146,7 @@ Full documentation lives at **[docs.picpeak.app](https://docs.picpeak.app)** —
- 🚀 [**Deployment**](https://docs.picpeak.app/deployment) - Docker, environment variables, reverse proxy, SSL
- ⚙️ [**Admin Settings**](https://docs.picpeak.app/guides/admin-settings) - Every tab in the Settings panel
- 🎯 [**Creating Events**](https://docs.picpeak.app/guides/creating-events) - Full event field reference
- 💾 [**Backup & Restore**](https://docs.picpeak.app/guides/backup-restore) - Local, S3, rsync destinations
- 💾 [**Backup & Restore**](https://docs.picpeak.app/guides/backup-restore) - Backup configuration, restore wizard, full disaster recovery
- 🔌 [**API Reference**](https://docs.picpeak.app/api) - REST endpoints, OpenAPI spec, webhooks
- 🪝 [**Webhooks**](https://docs.picpeak.app/features/webhooks) - Event payloads, signing, filters, templates
@@ -0,0 +1,242 @@
/**
* Integration test for GET /api/admin/system-health/backup-coverage.
*
* Pins the Stage C diagnostic that tells admins what the next
* "Run Backup Now" will include, skip, or silently miss.
*
* Test surface:
* 1. Empty / fresh install → default seed (7 paths), inline mode,
* no DB dump on file yet, no drift
* 2. Toggle `include_in_default=false` → coverage flips to
* 'skipped-by-toggle'
* 3. Feature_flag gating reflects the actual app_settings value
* (events/archived ⇄ backup_include_archived)
* 4. Drift detection: a top-level subdir on disk with no
* `backup_paths` row is flagged in `unconfiguredOnDisk`
* 5. Allow-list: `backups/` and `tmp/` are never flagged as drift
* 6. Scheduled-only mode + recent dump → `database.ok = true`
* 7. Scheduled-only mode + stale (>26h) dump → `database.ok = false`
* and `lastDumpStale = true`
*
* Same auth/permission pass-through strategy as
* adminBackupIntegrity.test.js — we exercise the route's logic,
* not the auth middleware.
*/
const fs = require('fs');
const path = require('path');
const express = require('express');
const request = require('supertest');
const { bootCrmDb } = require('./helpers/crmDb');
jest.mock('../../src/middleware/auth', () => ({
adminAuth: (req, _res, next) => { req.admin = { id: 1 }; next(); },
customerAuth: (_req, _res, next) => next(),
galleryAuth: (_req, _res, next) => next(),
}));
jest.mock('../../src/middleware/permissions', () => ({
requirePermission: () => (_req, _res, next) => next(),
}));
jest.setTimeout(30000);
describe('GET /api/admin/system-health/backup-coverage', () => {
let db;
let cleanup;
let storagePath;
let app;
beforeAll(async () => {
({ db, cleanup } = await bootCrmDb());
storagePath = process.env.STORAGE_PATH;
const route = require('../../src/routes/adminSystemHealth');
app = express();
app.use(express.json());
app.use('/api/admin/system-health', route);
}, 120000);
afterAll(async () => {
if (cleanup) await cleanup();
});
function mkdir(rel) {
fs.mkdirSync(path.join(storagePath, rel), { recursive: true });
}
function rmdir(rel) {
fs.rmSync(path.join(storagePath, rel), { recursive: true, force: true });
}
async function restoreDefaultPaths() {
await db('backup_paths').del();
const { DEFAULT_PATHS } = require('../../migrations/core/109_add_backup_paths');
await db('backup_paths').insert(DEFAULT_PATHS.map((row) => ({
...row,
created_at: new Date(),
updated_at: new Date(),
})));
}
beforeEach(async () => {
await restoreDefaultPaths();
await db('database_backup_runs').del().catch(() => {});
await db('app_settings').where('setting_type', 'backup').del().catch(() => {});
});
it('returns the canonical 7 paths + database block on a fresh install', async () => {
const res = await request(app).get('/api/admin/system-health/backup-coverage');
expect(res.status).toBe(200);
expect(res.body).toHaveProperty('report');
const { report } = res.body;
expect(report.paths.map((p) => p.path)).toEqual([
'events/active',
'events/archived',
'thumbnails',
'previews',
'heroes',
'uploads',
'business-docs',
]);
// Default mode is inline — no inline_dump setting present means
// "inline is ON" (matches ensureDatabaseDumpForBackup semantics).
expect(report.database.mode).toBe('inline');
expect(report.database.ok).toBe(true);
expect(report.summary).toMatchObject({
configuredCount: 7,
tableMissingFallbackInUse: false,
});
});
it('flips a path to skipped-by-toggle when include_in_default=false', async () => {
await db('backup_paths').where('path', 'thumbnails').update({
include_in_default: false,
});
const res = await request(app).get('/api/admin/system-health/backup-coverage');
const thumbnails = res.body.report.paths.find((p) => p.path === 'thumbnails');
expect(thumbnails.coverage).toBe('skipped-by-toggle');
expect(thumbnails.includeInDefault).toBe(false);
});
it('feature_flag gating reflects app_settings (archived path off vs on)', async () => {
// backup_include_archived not set → archived skipped via flag
const off = await request(app).get('/api/admin/system-health/backup-coverage');
const archivedOff = off.body.report.paths.find((p) => p.path === 'events/archived');
expect(archivedOff.coverage).toBe('skipped-by-feature-flag');
expect(archivedOff.featureFlag).toBe('backup_include_archived');
expect(archivedOff.featureFlagValue).toBe(null); // unset
// Now set the flag — but path is missing on disk, so coverage
// resolves to 'missing-on-disk', proving the flag was honoured.
await db('app_settings').insert({
setting_key: 'backup_include_archived',
setting_value: JSON.stringify(true),
setting_type: 'backup',
}).onConflict('setting_key').merge();
const on = await request(app).get('/api/admin/system-health/backup-coverage');
const archivedOn = on.body.report.paths.find((p) => p.path === 'events/archived');
expect(archivedOn.featureFlagValue).toBe(true);
// No on-disk dir → 'missing-on-disk' (not 'skipped-by-feature-flag')
expect(['missing-on-disk', 'will-scan']).toContain(archivedOn.coverage);
});
it('detects unconfigured top-level subdirs as drift', async () => {
mkdir('events/active'); // configured
mkdir('plugin-store/cache'); // DRIFT
mkdir('shiny-new-feature/data'); // DRIFT
const res = await request(app).get('/api/admin/system-health/backup-coverage');
expect(res.body.report.drift.unconfiguredOnDisk).toEqual(expect.arrayContaining([
'plugin-store',
'shiny-new-feature',
]));
expect(res.body.report.drift.unconfiguredOnDisk).not.toContain('events');
rmdir('plugin-store');
rmdir('shiny-new-feature');
});
it('never flags backups/ or tmp/ as drift (allow-list)', async () => {
mkdir('backups');
mkdir('tmp');
const res = await request(app).get('/api/admin/system-health/backup-coverage');
expect(res.body.report.drift.unconfiguredOnDisk).not.toContain('backups');
expect(res.body.report.drift.unconfiguredOnDisk).not.toContain('tmp');
expect(res.body.report.drift.expectedNonBackupDirs).toEqual(
expect.arrayContaining(['backups', 'tmp']),
);
rmdir('backups');
rmdir('tmp');
});
it('scheduled-only mode + recent dump → database.ok=true, not stale', async () => {
await db('app_settings').insert({
setting_key: 'backup_database_inline_dump',
setting_value: JSON.stringify(false),
setting_type: 'backup',
}).onConflict('setting_key').merge();
const recentDump = path.join(storagePath, 'backups', 'recent.sql.gz');
fs.mkdirSync(path.dirname(recentDump), { recursive: true });
fs.writeFileSync(recentDump, 'pretend dump');
await db('database_backup_runs').insert({
started_at: new Date(),
completed_at: new Date(), // just now
status: 'completed',
backup_type: 'pg',
file_path: recentDump,
file_size_bytes: fs.statSync(recentDump).size,
destination_path: recentDump,
});
const res = await request(app).get('/api/admin/system-health/backup-coverage');
expect(res.body.report.database.mode).toBe('scheduled-only');
expect(res.body.report.database.inlineDumpExplicitlyDisabled).toBe(true);
expect(res.body.report.database.lastDumpStale).toBe(false);
expect(res.body.report.database.ok).toBe(true);
});
it('scheduled-only mode + stale dump → database.ok=false, lastDumpStale=true', async () => {
await db('app_settings').insert({
setting_key: 'backup_database_inline_dump',
setting_value: JSON.stringify(false),
setting_type: 'backup',
}).onConflict('setting_key').merge();
const oldDump = path.join(storagePath, 'backups', 'old.sql.gz');
fs.mkdirSync(path.dirname(oldDump), { recursive: true });
fs.writeFileSync(oldDump, 'pretend old dump');
// 48 hours ago — well past the 26h staleness threshold. ISO
// string instead of a Date object because knex-sqlite's datetime
// serialisation has a quirk where some Date instances coerce to
// '[object Object]' on insert (the test 6 "recent dump" case
// passes only because `new Date()` happens to round-trip safely;
// arithmetic Dates don't).
const stale = new Date(Date.now() - 48 * 60 * 60 * 1000).toISOString();
await db('database_backup_runs').insert({
started_at: stale,
completed_at: stale,
status: 'completed',
backup_type: 'pg',
file_path: oldDump,
file_size_bytes: fs.statSync(oldDump).size,
destination_path: oldDump,
});
const res = await request(app).get('/api/admin/system-health/backup-coverage');
expect(res.body.report.database.lastDumpStale).toBe(true);
expect(res.body.report.database.ok).toBe(false);
// Top-level summary reflects the failed DB check.
expect(res.body.report.summary.databaseOk).toBe(false);
expect(res.body.report.summary.overallOk).toBe(false);
});
});
@@ -0,0 +1,140 @@
/**
* Integration test for GET /api/admin/system-health/backup-integrity.
*
* Auth + permission middleware are mocked to pass-through so the test
* focuses on the route's own behaviour: scope-param validation, the
* successResponse envelope, and that the underlying service report
* surfaces correctly in the JSON body.
*
* The verifier service itself is exercised against the real schema
* (bootCrmDb) and real filesystem — only the auth gate is stubbed.
*/
const fs = require('fs');
const path = require('path');
const express = require('express');
const request = require('supertest');
const { bootCrmDb, seedMinimal } = require('./helpers/crmDb');
// Pass-through auth so we don't need to mint JWTs.
jest.mock('../../src/middleware/auth', () => ({
adminAuth: (req, _res, next) => { req.admin = { id: 1 }; next(); },
customerAuth: (_req, _res, next) => next(),
galleryAuth: (_req, _res, next) => next(),
}));
// Pass-through permissions so settings.view always allows.
jest.mock('../../src/middleware/permissions', () => ({
requirePermission: () => (_req, _res, next) => next(),
}));
jest.setTimeout(30000);
describe('GET /api/admin/system-health/backup-integrity', () => {
let cleanup;
let db;
let customerId;
let app;
let storagePath;
beforeAll(async () => {
({ db, cleanup } = await bootCrmDb());
({ customerId } = await seedMinimal(db));
storagePath = process.env.STORAGE_PATH;
// Mount the route on a minimal Express app. Cold-require after
// bootCrmDb so the route's downstream `require('../database/db')`
// sees the same db instance.
const route = require('../../src/routes/adminSystemHealth');
app = express();
app.use(express.json());
app.use('/api/admin/system-health', route);
}, 120000);
afterAll(async () => {
if (cleanup) await cleanup();
});
beforeEach(async () => {
await db('contracts').del().catch(() => {});
await db('invoices').del().catch(() => {});
await db('quotes').del().catch(() => {});
});
it('returns a report envelope when nothing references any path', async () => {
const res = await request(app).get('/api/admin/system-health/backup-integrity');
expect(res.status).toBe(200);
expect(res.body).toHaveProperty('report');
expect(res.body.report.summary).toMatchObject({
totalRows: 0,
missingFiles: 0,
hashMismatches: 0,
verifiedOk: 0,
existsButNoHash: 0,
});
expect(res.body.report.scopes).toEqual(expect.arrayContaining([
'quote', 'contract', 'contract-signature', 'invoice',
]));
});
it('surfaces a missing file in the response payload', async () => {
await db('contracts').insert({
customer_account_id: customerId,
contract_number: 'C-B7-MISSING',
status: 'sent',
issue_date: '2026-01-01',
signed_pdf_path: 'business-docs/contract/2026/C-B7-MISSING.pdf',
created_at: new Date(),
});
const res = await request(app).get('/api/admin/system-health/backup-integrity');
expect(res.status).toBe(200);
expect(res.body.report.summary.missingFiles).toBe(1);
expect(res.body.report.missing[0]).toMatchObject({
table: 'contracts',
column: 'signed_pdf_path',
expectedPath: 'business-docs/contract/2026/C-B7-MISSING.pdf',
});
});
it('honours the ?scope=invoice filter', async () => {
// Seed both an invoice and a contract with missing files. With
// scope=invoice the contract row must not appear.
await db('invoices').insert({
customer_account_id: customerId,
invoice_number: 'INV-B7-SCOPE',
status: 'sent',
issue_date: '2026-01-01',
due_date: '2026-01-31',
pdf_path: 'business-docs/invoice/2026/INV-B7-SCOPE.pdf',
created_at: new Date(),
});
await db('contracts').insert({
customer_account_id: customerId,
contract_number: 'C-B7-SCOPE',
status: 'sent',
issue_date: '2026-01-01',
signed_pdf_path: 'business-docs/contract/2026/C-B7-SCOPE.pdf',
created_at: new Date(),
});
const res = await request(app)
.get('/api/admin/system-health/backup-integrity')
.query({ scope: 'invoice' });
expect(res.status).toBe(200);
expect(res.body.report.scopes).toEqual(['invoice']);
expect(res.body.report.missing.every((m) => m.table === 'invoices')).toBe(true);
});
it('rejects an unknown scope with 400 + a code', async () => {
const res = await request(app)
.get('/api/admin/system-health/backup-integrity')
.query({ scope: 'gallery' });
expect(res.status).toBe(400);
expect(res.body.code).toBe('BACKUP_INTEGRITY_UNKNOWN_SCOPE');
expect(res.body.validScopes).toEqual(expect.arrayContaining([
'quote', 'contract', 'contract-signature', 'invoice',
]));
});
});
@@ -0,0 +1,88 @@
/**
* Regression net for the business-docs coverage gap fixed in this PR.
*
* Prior to the fix, `getFilesToBackupInternal()` enumerated a fixed
* list of storage subdirectories (events/active, events/archived,
* thumbnails, previews, heroes, uploads) and silently omitted the
* entire `business-docs/` tree. That meant every CRM PDF + signature
* drawing — quotes, contracts (system-rendered + wet uploads),
* invoices, Storno, imported historical invoices, and the customer
* signature PNG/JPG drawn on the public signing page — fell outside
* the in-app scheduled backup, leaving every `*_path` column on
* `quotes` / `contracts` / `invoices` as a broken FK after restore.
*
* The fix is a single `scanDirectory(business-docs, ...)` call. This
* suite pins the contract so a future refactor of the walker cannot
* silently drop business-docs again.
*/
const fs = require('fs');
const path = require('path');
const { bootCrmDb } = require('./helpers/crmDb');
describe('backupService — business-docs is in the backup walker', () => {
let cleanup;
let backupService;
let storagePath;
beforeAll(async () => {
({ cleanup } = await bootCrmDb());
storagePath = process.env.STORAGE_PATH;
// Cold-require after bootCrmDb so backupService picks up the same
// db instance + STORAGE_PATH the test harness configured.
backupService = require('../../src/services/backupService');
}, 120000);
afterAll(async () => {
if (cleanup) await cleanup();
});
function seed(relPath, content = 'dummy bytes for backup test') {
const abs = path.join(storagePath, relPath);
fs.mkdirSync(path.dirname(abs), { recursive: true });
fs.writeFileSync(abs, content);
}
it('does not error when business-docs is absent', async () => {
// Fresh harness has no business-docs/ tree at all. The walker
// must short-circuit on ENOENT rather than throw — installs that
// never used CRM features have to keep backing up fine.
await expect(backupService.getFilesToBackup(false)).resolves.toEqual(expect.any(Array));
});
it('picks up every CRM-relevant business-docs subdirectory', async () => {
// Seed one file in each of the five subpaths the renderer + import
// routes write to. The signature path is the one most prone to be
// forgotten — it lives one level deeper than the others (per-
// contract subfolder, not per-year).
seed('business-docs/quote/2026/Q-001.pdf');
seed('business-docs/contract/2026/C-001.pdf');
seed('business-docs/contract/signatures/42/customer-1700000000000.png');
seed('business-docs/invoice/2026/INV-001.pdf');
seed('business-docs/invoice-imports/2026/scan.pdf');
const files = await backupService.getFilesToBackup(false);
const rels = files.map((f) => f.relativePath);
expect(rels).toEqual(expect.arrayContaining([
'business-docs/quote/2026/Q-001.pdf',
'business-docs/contract/2026/C-001.pdf',
'business-docs/contract/signatures/42/customer-1700000000000.png',
'business-docs/invoice/2026/INV-001.pdf',
'business-docs/invoice-imports/2026/scan.pdf',
]));
});
it('walks newly-created business-docs files without needing a restart', async () => {
// The walker reads the filesystem live on every call; this guards
// against a future "cache the scan result at boot" optimisation
// that would miss freshly-written PDFs (which is exactly what
// happens during normal operation — every send writes a new file).
seed('business-docs/invoice/2027/INV-NEW.pdf');
const files = await backupService.getFilesToBackup(false);
const rels = files.map((f) => f.relativePath);
expect(rels).toContain('business-docs/invoice/2027/INV-NEW.pdf');
});
});
@@ -0,0 +1,180 @@
/**
* Pins the Stage-B refactor that lifted the file-backup walker's
* subdirectory list out of hard-coded JS into the `backup_paths`
* table seeded by migration 109.
*
* Scenarios:
* 1. Walker reads canonical seed → all 7 default subdirs walked
* 2. include_in_default=false on one row → that subdir is skipped
* 3. New row inserted at runtime → walker picks it up without restart
* 4. feature_flag gating → row only walked when the named app_settings
* boolean is truthy (mirrors historical `includeArchived` behavior)
* 5. Empty table → walker falls back to LEGACY_BACKUP_PATHS (defense
* in depth — never silently scans nothing)
*
* Why not stub `db('backup_paths')`: the whole point of Stage B is
* that the walker is now data-driven, so the test has to actually
* mutate the table and observe the walker's output change. Stubs
* would re-introduce the hard-coding the refactor is meant to remove.
*/
const fs = require('fs');
const path = require('path');
const { bootCrmDb } = require('./helpers/crmDb');
jest.setTimeout(30000);
describe('backupService — configurable walker (backup_paths)', () => {
let db;
let cleanup;
let storagePath;
let backupService;
beforeAll(async () => {
({ db, cleanup } = await bootCrmDb());
storagePath = process.env.STORAGE_PATH;
backupService = require('../../src/services/backupService');
}, 120000);
afterAll(async () => {
if (cleanup) await cleanup();
});
function seedFile(relPath, content = 'dummy bytes') {
const abs = path.join(storagePath, relPath);
fs.mkdirSync(path.dirname(abs), { recursive: true });
fs.writeFileSync(abs, content);
}
beforeEach(async () => {
// Restore canonical seed before every test. Tests mutate this table
// freely; the next test starts from a known state.
await db('backup_paths').del();
const {
DEFAULT_PATHS,
} = require('../../migrations/core/109_add_backup_paths');
await db('backup_paths').insert(DEFAULT_PATHS.map((row) => ({
...row,
created_at: new Date(),
updated_at: new Date(),
})));
});
it('migration 109 seeds the canonical 7 paths', async () => {
const rows = await db('backup_paths').orderBy('display_order', 'asc').select();
expect(rows.map((r) => r.path)).toEqual([
'events/active',
'events/archived',
'thumbnails',
'previews',
'heroes',
'uploads',
'business-docs',
]);
// Only events/archived is gated by a feature flag.
expect(rows.filter((r) => r.feature_flag).map((r) => r.path)).toEqual([
'events/archived',
]);
});
it('walks every default subdir when files are present', async () => {
seedFile('events/active/E1/a.jpg');
seedFile('thumbnails/E1/a.jpg');
seedFile('previews/E1/a.jpg');
seedFile('heroes/E1/hero.jpg');
seedFile('uploads/intake/x.bin');
seedFile('business-docs/quote/2026/Q-001.pdf');
// events/archived is gated — left out of this test; covered below.
const files = await backupService.getFilesToBackup({ backup_include_archived: true });
const rels = files.map((f) => f.relativePath);
expect(rels).toEqual(expect.arrayContaining([
'events/active/E1/a.jpg',
'thumbnails/E1/a.jpg',
'previews/E1/a.jpg',
'heroes/E1/hero.jpg',
'uploads/intake/x.bin',
'business-docs/quote/2026/Q-001.pdf',
]));
});
it('skips a path when include_in_default is toggled off', async () => {
seedFile('thumbnails/E1/thumb.jpg');
seedFile('events/active/E1/photo.jpg');
await db('backup_paths').where('path', 'thumbnails').update({
include_in_default: false,
});
const files = await backupService.getFilesToBackup({ backup_include_archived: true });
const rels = files.map((f) => f.relativePath);
expect(rels).toContain('events/active/E1/photo.jpg');
expect(rels).not.toContain('thumbnails/E1/thumb.jpg');
});
it('picks up a new path inserted at runtime — no restart needed', async () => {
// Simulates a future feature shipping its own subdirectory and
// self-healing a `backup_paths` row at boot.
await db('backup_paths').insert({
path: 'plugin-store',
include_in_default: true,
feature_flag: null,
display_order: 200,
description: 'Hypothetical future feature payload',
created_at: new Date(),
updated_at: new Date(),
});
seedFile('plugin-store/cache/payload.bin');
const files = await backupService.getFilesToBackup({ backup_include_archived: true });
const rels = files.map((f) => f.relativePath);
expect(rels).toContain('plugin-store/cache/payload.bin');
});
it('respects feature_flag gating (events/archived ⇄ backup_include_archived)', async () => {
seedFile('events/active/E1/active.jpg');
seedFile('events/archived/E2/archived.jpg');
// backup_include_archived=false → archived/ is skipped.
const filesOff = await backupService.getFilesToBackup({ backup_include_archived: false });
const relsOff = filesOff.map((f) => f.relativePath);
expect(relsOff).toContain('events/active/E1/active.jpg');
expect(relsOff).not.toContain('events/archived/E2/archived.jpg');
// backup_include_archived=true → archived/ is included.
const filesOn = await backupService.getFilesToBackup({ backup_include_archived: true });
const relsOn = filesOn.map((f) => f.relativePath);
expect(relsOn).toContain('events/archived/E2/archived.jpg');
});
it('falls back to LEGACY_BACKUP_PATHS when the table is empty', async () => {
// Defense in depth: even if seed-and-self-heal both failed, the
// walker must still cover the historical set so "Run Backup Now"
// cannot silently degrade to no-op.
await db('backup_paths').del();
seedFile('events/active/E1/photo.jpg');
seedFile('business-docs/quote/2026/Q-002.pdf');
const files = await backupService.getFilesToBackup({ backup_include_archived: true });
const rels = files.map((f) => f.relativePath);
expect(rels).toContain('events/active/E1/photo.jpg');
expect(rels).toContain('business-docs/quote/2026/Q-002.pdf');
});
it('legacy boolean call signature still works (backward compat)', async () => {
// Existing call sites (and the businessDocs regression test) pass
// a boolean for `includeArchived`. Refactor must not break them.
seedFile('events/archived/E3/legacy.jpg');
const filesOff = await backupService.getFilesToBackup(false);
expect(filesOff.map((f) => f.relativePath)).not.toContain('events/archived/E3/legacy.jpg');
const filesOn = await backupService.getFilesToBackup(true);
expect(filesOn.map((f) => f.relativePath)).toContain('events/archived/E3/legacy.jpg');
});
});
@@ -0,0 +1,188 @@
/**
* Pins the inline-DB-dump + fail-loud guard added to `runBackupInternal`.
*
* The previous behaviour was: file-backup looked up an existing dump via
* `getDatabaseBackupInfo()` and silently shipped a files-only manifest
* when none was found. Admins clicking "Run Backup Now" got an apparent
* success that omitted every customer / quote / invoice / contract row —
* the data-loss footgun that this commit closes.
*
* Five scenarios under test:
* 1. Default (inline dump enabled), dump succeeds → backup proceeds
* 2. Default, dump throws → run aborts, backup_runs row marked failed
* 3. Opt-out + recent DB dump available → backup proceeds
* 4. Opt-out + no DB dump available → fail loud
* 5. Opt-out + DB dump file is 0 bytes on disk → fail loud
*
* Mocking strategy: the underlying `databaseBackupService.backup()` and
* the local-destination writer are stubbed so the test exercises just
* the new guard logic without depending on `pg_dump` / `sqlite3` CLI
* binaries being available in the test environment.
*/
const fs = require('fs');
const path = require('path');
const { bootCrmDb } = require('./helpers/crmDb');
// Set up mocks BEFORE bootCrmDb so backupService picks them up at require time.
const mockBackupFn = jest.fn();
jest.mock('../../src/services/databaseBackup', () => ({
databaseBackupService: { backup: mockBackupFn },
startScheduledBackups: jest.fn(),
stopScheduledBackups: jest.fn(),
DatabaseBackupService: class {},
}));
jest.setTimeout(30000);
describe('backupService — inline DB dump + fail-loud guard', () => {
let db;
let cleanup;
let storagePath;
let backupService;
let dumpFileAbs;
beforeAll(async () => {
({ db, cleanup } = await bootCrmDb());
storagePath = process.env.STORAGE_PATH;
backupService = require('../../src/services/backupService');
// Seed backup destination settings so the run can proceed past the
// "destination not configured" guard.
const dest = path.join(storagePath, 'backups');
fs.mkdirSync(dest, { recursive: true });
// getBackupConfigInternal filters by setting_type='backup', so the
// tests have to seed with that type or the resolver returns
// `{ ... }` with the keys missing — runBackup then sees
// `backup_destination_type === undefined` and bails before our
// new guard runs.
await db('app_settings').insert([
{ setting_key: 'backup_destination_type', setting_value: JSON.stringify('local'), setting_type: 'backup' },
{ setting_key: 'backup_destination_path', setting_value: JSON.stringify(dest), setting_type: 'backup' },
{ setting_key: 'backup_enabled', setting_value: JSON.stringify(true), setting_type: 'backup' },
{ setting_key: 'backup_email_on_failure', setting_value: JSON.stringify(false), setting_type: 'backup' },
]).onConflict('setting_key').merge();
// Pre-create a dump file that getDatabaseBackupInfo can resolve to.
// Reused/mutated per-test via the database_backup_runs seed below.
dumpFileAbs = path.join(storagePath, 'backups', 'fake-dump.sql.gz');
fs.writeFileSync(dumpFileAbs, 'pretend this is a pg_dump'.repeat(100));
// Neutralise the file-scan step: we don't care which files would
// be backed up, just whether the run reaches that stage at all.
backupService.getFilesToBackup = jest.fn(async () => []);
}, 120000);
afterAll(async () => {
if (cleanup) await cleanup();
});
beforeEach(async () => {
mockBackupFn.mockReset();
// Default to "dump produced this file with this size" — the per-test
// setup overrides as needed.
mockBackupFn.mockResolvedValue({
success: true,
path: dumpFileAbs,
size: fs.statSync(dumpFileAbs).size,
duration: 1,
checksum: 'abc',
});
// Re-seed the database_backup_runs row that getDatabaseBackupInfo
// resolves against (its query is `status='completed'` + most recent).
await db('database_backup_runs').del();
await db('database_backup_runs').insert({
started_at: new Date(),
completed_at: new Date(),
status: 'completed',
backup_type: 'pg',
file_path: dumpFileAbs,
file_size_bytes: fs.statSync(dumpFileAbs).size,
destination_path: dumpFileAbs,
});
});
it('default behaviour: inline dump runs, then file backup proceeds', async () => {
// Inline-dump setting is unset (undefined) — default is ON.
await db('app_settings').where('setting_key', 'backup_database_inline_dump').del();
await backupService.runBackup(true);
expect(mockBackupFn).toHaveBeenCalledTimes(1);
const run = await db('backup_runs').orderBy('id', 'desc').first();
expect(run.status).toBe('completed');
expect(run.error_message).toBeNull();
});
it('aborts the run when the inline dump throws', async () => {
await db('app_settings').where('setting_key', 'backup_database_inline_dump').del();
mockBackupFn.mockRejectedValueOnce(new Error('pg_dump segfaulted'));
await backupService.runBackup(true);
const run = await db('backup_runs').orderBy('id', 'desc').first();
expect(run.status).toBe('failed');
expect(run.error_message).toMatch(/pg_dump segfaulted/);
});
it('opt-out: skips inline dump but proceeds when a recent dump exists', async () => {
await db('app_settings').insert({
setting_key: 'backup_database_inline_dump',
setting_value: JSON.stringify(false),
setting_type: 'backup',
}).onConflict('setting_key').merge();
await backupService.runBackup(true);
expect(mockBackupFn).not.toHaveBeenCalled();
const run = await db('backup_runs').orderBy('id', 'desc').first();
expect(run.status).toBe('completed');
});
it('opt-out + no recent dump: fails loud with a clear error', async () => {
await db('app_settings').insert({
setting_key: 'backup_database_inline_dump',
setting_value: JSON.stringify(false),
setting_type: 'backup',
}).onConflict('setting_key').merge();
// Wipe the dump row so getDatabaseBackupInfo returns backupFile=null.
await db('database_backup_runs').del();
await backupService.runBackup(true);
const run = await db('backup_runs').orderBy('id', 'desc').first();
expect(run.status).toBe('failed');
expect(run.error_message).toMatch(/No database backup available/);
});
it('opt-out + 0-byte dump file: fails loud', async () => {
await db('app_settings').insert({
setting_key: 'backup_database_inline_dump',
setting_value: JSON.stringify(false),
setting_type: 'backup',
}).onConflict('setting_key').merge();
const emptyDump = path.join(storagePath, 'backups', 'empty-dump.sql.gz');
fs.writeFileSync(emptyDump, '');
await db('database_backup_runs').del();
await db('database_backup_runs').insert({
started_at: new Date(),
completed_at: new Date(),
status: 'completed',
backup_type: 'pg',
file_path: emptyDump,
file_size_bytes: 0,
destination_path: emptyDump,
});
await backupService.runBackup(true);
const run = await db('backup_runs').orderBy('id', 'desc').first();
expect(run.status).toBe('failed');
expect(run.error_message).toMatch(/is empty/);
});
});
@@ -0,0 +1,180 @@
/**
* Per-Stage-B-path tally — Tier 3 of tonight's backup hardening.
*
* Pins the new `computePerPathStats` logic that the Backup History
* "Content Backed Up" pane reads via `backup_runs.statistics.per_path`.
*
* Three scenarios:
* 1. Single file under one path — straightforward attribution
* 2. Multiple paths with overlapping prefixes — longest-prefix wins
* (e.g. `events/active/E1/x.jpg` should attribute to
* `events/active`, not `events`)
* 3. File outside any configured path — silently dropped, doesn't
* throw or contaminate other buckets
*
* Tests exercise the EXPORTED side: write a backup_runs row via the
* service entry point and assert the statistics JSON shape. We don't
* stub `computePerPathStats` directly — the integration view is what
* the frontend actually consumes.
*/
const fs = require('fs');
const path = require('path');
const { bootCrmDb } = require('./helpers/crmDb');
jest.setTimeout(30000);
describe('backupService — per-Stage-B-path statistics', () => {
let db;
let cleanup;
let storagePath;
let backupService;
beforeAll(async () => {
({ db, cleanup } = await bootCrmDb());
storagePath = process.env.STORAGE_PATH;
backupService = require('../../src/services/backupService');
}, 120000);
afterAll(async () => {
if (cleanup) await cleanup();
});
function mkFile(rel, content = 'x'.repeat(100)) {
const abs = path.join(storagePath, rel);
fs.mkdirSync(path.dirname(abs), { recursive: true });
fs.writeFileSync(abs, content);
}
beforeEach(async () => {
// Clean slate of any artefacts from prior tests
await db('backup_runs').del();
await db('app_settings').where('setting_type', 'backup').del();
await db('app_settings').insert([
{ setting_key: 'backup_destination_type', setting_value: JSON.stringify('local'), setting_type: 'backup' },
{ setting_key: 'backup_destination_path', setting_value: JSON.stringify(path.join(storagePath, 'destination')), setting_type: 'backup' },
{ setting_key: 'backup_enabled', setting_value: JSON.stringify(true), setting_type: 'backup' },
{ setting_key: 'backup_email_on_failure', setting_value: JSON.stringify(false), setting_type: 'backup' },
{ setting_key: 'backup_include_archived', setting_value: JSON.stringify(true), setting_type: 'backup' },
]).onConflict('setting_key').merge();
fs.mkdirSync(path.join(storagePath, 'destination'), { recursive: true });
// Restore canonical backup_paths from migration 109
const { DEFAULT_PATHS } = require('../../migrations/core/109_add_backup_paths');
await db('backup_paths').del();
await db('backup_paths').insert(DEFAULT_PATHS.map((row) => ({
...row,
created_at: new Date(),
updated_at: new Date(),
})));
// Wipe leftover files between tests
for (const dir of ['events', 'business-docs', 'thumbnails', 'previews', 'heroes', 'uploads']) {
const p = path.join(storagePath, dir);
if (fs.existsSync(p)) fs.rmSync(p, { recursive: true, force: true });
}
});
it('attributes files to their owning backup_paths row', async () => {
mkFile('events/active/E1/photo-a.jpg', 'X'.repeat(1000));
mkFile('events/active/E1/photo-b.jpg', 'X'.repeat(2000));
mkFile('business-docs/quote/2026/Q-1.pdf', 'X'.repeat(500));
mkFile('thumbnails/E1/photo-a.jpg', 'X'.repeat(50));
// Disable the inline DB dump so we don't need pg_dump in tests;
// the file walker is what produces per_path.
await db('app_settings').insert({
setting_key: 'backup_database_inline_dump',
setting_value: JSON.stringify(false),
setting_type: 'backup',
}).onConflict('setting_key').merge();
// Seed a fake DB-backup row so the fail-loud guard is satisfied.
const fakeDump = path.join(storagePath, 'destination', 'fake.sql.gz');
fs.writeFileSync(fakeDump, 'pretend dump');
await db('database_backup_runs').insert({
started_at: new Date(),
completed_at: new Date(),
status: 'completed',
backup_type: 'pg',
file_path: fakeDump,
file_size_bytes: fs.statSync(fakeDump).size,
destination_path: fakeDump,
});
await backupService.runBackup(true);
const run = await db('backup_runs').orderBy('id', 'desc').first();
expect(run.status).toBe('completed');
const statsRaw = typeof run.statistics === 'string'
? JSON.parse(run.statistics)
: run.statistics;
expect(statsRaw.per_path).toBeDefined();
// events/active should have 2 files (3000 bytes)
expect(statsRaw.per_path['events/active']).toEqual({ count: 2, size: 3000 });
// business-docs should have 1 file (500 bytes)
expect(statsRaw.per_path['business-docs']).toEqual({ count: 1, size: 500 });
// thumbnails should have 1 file (50 bytes)
expect(statsRaw.per_path['thumbnails']).toEqual({ count: 1, size: 50 });
// No spurious buckets for paths that had nothing
expect(statsRaw.per_path['previews']).toBeUndefined();
expect(statsRaw.per_path['heroes']).toBeUndefined();
});
it('archived path attributed separately from active when both have files', async () => {
mkFile('events/active/E1/active.jpg', 'X'.repeat(100));
mkFile('events/archived/E2/archived.jpg', 'X'.repeat(200));
// backup_include_archived already set true in beforeEach so the
// archived walker fires; same opt-out for inline DB dump.
await db('app_settings').insert({
setting_key: 'backup_database_inline_dump',
setting_value: JSON.stringify(false),
setting_type: 'backup',
}).onConflict('setting_key').merge();
const fakeDump = path.join(storagePath, 'destination', 'fake.sql.gz');
fs.writeFileSync(fakeDump, 'pretend dump');
await db('database_backup_runs').insert({
started_at: new Date(),
completed_at: new Date(),
status: 'completed',
backup_type: 'pg',
file_path: fakeDump,
file_size_bytes: fs.statSync(fakeDump).size,
destination_path: fakeDump,
});
await backupService.runBackup(true);
const run = await db('backup_runs').orderBy('id', 'desc').first();
const statsRaw = typeof run.statistics === 'string'
? JSON.parse(run.statistics)
: run.statistics;
// events/active and events/archived attribute separately —
// longest-prefix match prevents `events/active/...` from claiming
// an `events/archived/...` file or vice versa.
expect(statsRaw.per_path['events/active']).toEqual({ count: 1, size: 100 });
expect(statsRaw.per_path['events/archived']).toEqual({ count: 1, size: 200 });
});
});
// NOTE on walker duplication
//
// If two `backup_paths` rows overlap (e.g. one row at `events` AND
// another at `events/active`), the walker scans the same files twice
// — once via each path. Per-path stats then attribute the file to the
// longest-prefix-matching path BOTH times, producing inflated counts.
//
// The canonical seed in migration 109 contains no overlapping pairs,
// so this isn't exercised in practice. But an admin who hand-adds a
// broad row that overlaps an existing nested one will see double
// counts in their next backup's statistics + the destination will
// receive duplicate copies (wasting space). Worth flagging if anyone
// reports it — the fix is to de-dupe `files` in
// `getFilesToBackupInternal` before returning, OR to skip walking a
// path if a longer one has already covered it.
@@ -0,0 +1,214 @@
/**
* Install-from-backup boot hook — pins the trigger-file convention.
*
* The hook itself depends on `restoreService.restore`, which is hard
* to fully exercise in an integration test without a real PG cluster
* (sequence resync, DROP/CREATE, etc.). So we stub the actual restore
* and verify the BOOT HOOK logic:
*
* - No trigger file → no-op, ran=false
* - Empty trigger file → picks newest manifest from manifests/
* - Non-empty trigger file → uses the path inside
* - DB not empty → refuses (no restore call)
* - DB not empty + FORCE env → proceeds
* - Successful restore → deletes trigger file
* - Failed restore → leaves trigger file in place
*
* These are the surfaces an admin will hit when actually using the
* feature — the docker-compose-on-real-PG end-to-end test belongs in
* the follow-up CI work captured as task #7 earlier today.
*/
const fs = require('fs');
const path = require('path');
const { bootCrmDb } = require('./helpers/crmDb');
// Stub the heavy lifting so the test stays fast + portable.
const mockRestore = jest.fn();
jest.mock('../../src/services/restoreService', () => ({
restoreService: {
restore: (...args) => mockRestore(...args),
},
}));
jest.setTimeout(30000);
describe('installFromBackupBoot', () => {
let db;
let cleanup;
let storagePath;
let backupRoot;
let manifestsDir;
let tryInstallFromBackup;
let originalBackupRootEnv;
let originalForceEnv;
beforeAll(async () => {
({ db, cleanup } = await bootCrmDb());
storagePath = process.env.STORAGE_PATH;
backupRoot = path.join(storagePath, 'backup');
manifestsDir = path.join(backupRoot, 'manifests');
fs.mkdirSync(manifestsDir, { recursive: true });
originalBackupRootEnv = process.env.BACKUP_ROOT;
originalForceEnv = process.env.INSTALL_FROM_BACKUP_FORCE;
process.env.BACKUP_ROOT = backupRoot;
({ tryInstallFromBackup } = require('../../src/services/_installFromBackupBoot'));
}, 120000);
afterAll(async () => {
if (originalBackupRootEnv === undefined) {
delete process.env.BACKUP_ROOT;
} else {
process.env.BACKUP_ROOT = originalBackupRootEnv;
}
if (originalForceEnv === undefined) {
delete process.env.INSTALL_FROM_BACKUP_FORCE;
} else {
process.env.INSTALL_FROM_BACKUP_FORCE = originalForceEnv;
}
if (cleanup) await cleanup();
});
beforeEach(async () => {
mockRestore.mockReset();
mockRestore.mockResolvedValue({ success: true });
delete process.env.INSTALL_FROM_BACKUP_FORCE;
// Clean trigger files + manifests between tests
for (const name of ['RESTORE_ON_INSTALL', 'RESTORE_ON_INSTALL.txt']) {
const p = path.join(backupRoot, name);
if (fs.existsSync(p)) fs.unlinkSync(p);
}
for (const f of fs.readdirSync(manifestsDir)) {
fs.unlinkSync(path.join(manifestsDir, f));
}
// Reset DB to fresh-install state
await db('events').del();
// Leave admin_users alone — fresh-install state has 1 row.
});
it('no trigger file → no-op', async () => {
const result = await tryInstallFromBackup(db);
expect(result.ran).toBe(false);
expect(mockRestore).not.toHaveBeenCalled();
});
it('empty trigger file picks the newest manifest from manifests/', async () => {
const older = path.join(manifestsDir, 'backup-manifest-001.json');
const newer = path.join(manifestsDir, 'backup-manifest-002.json');
fs.writeFileSync(older, '{}');
// Set the newer file's mtime slightly later so it wins the sort
const past = new Date(Date.now() - 60_000);
fs.utimesSync(older, past, past);
fs.writeFileSync(newer, '{}');
// Empty trigger
fs.writeFileSync(path.join(backupRoot, 'RESTORE_ON_INSTALL'), '');
const result = await tryInstallFromBackup(db);
expect(result.ran).toBe(true);
expect(result.manifestPath).toBe(newer);
expect(mockRestore).toHaveBeenCalledWith(expect.objectContaining({
source: 'local',
manifestPath: newer,
restoreType: 'full',
force: true,
skipPreBackup: true,
}));
});
it('non-empty trigger file uses the path inside', async () => {
const specific = path.join(manifestsDir, 'backup-manifest-specific.json');
fs.writeFileSync(specific, '{}');
// Relative to backupRoot
fs.writeFileSync(
path.join(backupRoot, 'RESTORE_ON_INSTALL'),
'manifests/backup-manifest-specific.json\n',
);
const result = await tryInstallFromBackup(db);
expect(result.ran).toBe(true);
expect(result.manifestPath).toBe(specific);
});
it('deletes the trigger file after a successful restore', async () => {
const manifest = path.join(manifestsDir, 'backup-manifest-001.json');
fs.writeFileSync(manifest, '{}');
const triggerPath = path.join(backupRoot, 'RESTORE_ON_INSTALL');
fs.writeFileSync(triggerPath, '');
await tryInstallFromBackup(db);
expect(fs.existsSync(triggerPath)).toBe(false);
});
it('leaves the trigger file in place when restore throws', async () => {
mockRestore.mockRejectedValueOnce(new Error('restore exploded'));
const manifest = path.join(manifestsDir, 'backup-manifest-001.json');
fs.writeFileSync(manifest, '{}');
const triggerPath = path.join(backupRoot, 'RESTORE_ON_INSTALL');
fs.writeFileSync(triggerPath, '');
const result = await tryInstallFromBackup(db);
expect(result.ran).toBe(false);
expect(result.error).toMatch(/restore exploded/);
expect(fs.existsSync(triggerPath)).toBe(true);
});
it('refuses to run when the install already has events', async () => {
// Simulate an install with existing data
await db('events').insert({
slug: 'existing-event',
event_name: 'Existing Event',
event_type: 'wedding',
event_date: new Date(),
host_email: 'host@example.com',
admin_email: 'host@example.com',
expires_at: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000),
share_link: 'existing-event-token',
password_hash: 'dummy-hash-for-test',
created_at: new Date(),
});
const manifest = path.join(manifestsDir, 'backup-manifest-001.json');
fs.writeFileSync(manifest, '{}');
fs.writeFileSync(path.join(backupRoot, 'RESTORE_ON_INSTALL'), '');
const result = await tryInstallFromBackup(db);
expect(result.ran).toBe(false);
expect(result.error).toMatch(/Database not empty/);
expect(mockRestore).not.toHaveBeenCalled();
// Trigger file should NOT be deleted — admin needs to fix + retry
expect(fs.existsSync(path.join(backupRoot, 'RESTORE_ON_INSTALL'))).toBe(true);
});
it('proceeds when INSTALL_FROM_BACKUP_FORCE=true even with existing data', async () => {
await db('events').insert({
slug: 'existing-event-2',
event_name: 'Existing Event 2',
event_type: 'wedding',
event_date: new Date(),
host_email: 'host@example.com',
admin_email: 'host@example.com',
expires_at: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000),
share_link: 'existing-event-2-token',
password_hash: 'dummy-hash-for-test-2',
created_at: new Date(),
});
const manifest = path.join(manifestsDir, 'backup-manifest-001.json');
fs.writeFileSync(manifest, '{}');
fs.writeFileSync(path.join(backupRoot, 'RESTORE_ON_INSTALL'), '');
process.env.INSTALL_FROM_BACKUP_FORCE = 'true';
const result = await tryInstallFromBackup(db);
expect(result.ran).toBe(true);
expect(mockRestore).toHaveBeenCalled();
});
});
@@ -0,0 +1,259 @@
/**
* Pins the fix for the PR #596 review blocker.
*
* **The bug**
*
* `preservedMeta` was declared with `let` INSIDE the PostgreSQL
* `else` branch of `performDatabaseRestore`, then read AFTER the
* `else` block closed at the shared replay site (~L1030). On every
* real PG restore:
*
* ReferenceError: preservedMeta is not defined
*
* would fire — psql had already completed the data restore, but
* the operator-meta replay never ran, the trigger file was left
* in place by `_installFromBackupBoot.js` because the restore
* "failed", and `combined.log` got a loud FAILED line even though
* the data was back. Caught on PR #596 review by the maintainer.
*
* **Why CI missed it**
*
* The integration tests around `performFullRestore` only exercise
* the SQLite branch via `this.dbType === 'sqlite'`. The PG branch
* (~L827-984) requires a real PG connection + real `psql` binary,
* neither of which are in the test environment. So the scope leak
* sat untested until the maintainer ran a real DR cycle.
*
* **What this test does**
*
* Reads the source of `restoreService.js` and asserts the scope
* contract: the `preservedMeta` declaration sits ABOVE the
* SQLite/PG branch split, so the replay block at the bottom of the
* try{} can read it on either branch.
*
* Source-inspection is uglier than a runtime test but it has two
* advantages here: (a) it doesn't require a real PG cluster + psql
* binary in CI, (b) it pins the EXACT contract — "the declaration
* must be visible to the replay block" — which is the property
* that broke, more directly than a runtime test would.
*
* The follow-up "real-PG integration test in CI" (separate task)
* would replace this with an end-to-end exercise, at which point
* this can be deleted.
*/
const fs = require('fs');
const path = require('path');
describe('restoreService — PG branch scope contract (PR #596 review)', () => {
let src;
let lines;
beforeAll(() => {
src = fs.readFileSync(
path.join(__dirname, '..', '..', 'src', 'services', 'restoreService.js'),
'utf8',
);
lines = src.split(/\r?\n/);
});
/** Return the 1-based line number of the FIRST line matching `re`. */
function findFirst(re) {
const idx = lines.findIndex((l) => re.test(l));
return idx >= 0 ? idx + 1 : -1;
}
/** Return the 1-based line number of the LAST line matching `re`. */
function findLast(re) {
let last = -1;
lines.forEach((l, i) => { if (re.test(l)) last = i + 1; });
return last;
}
it('preservedMetaSnapshot lives on `this` and is initialised in the constructor', () => {
// PR #596 round 3 moved the snapshot from a block-scoped local to
// an instance variable so the replay can happen in `restore()`
// AFTER post-restore verification — preventing the replay row
// from inflating the row-count check.
//
// Contract:
// 1. The constructor initialises `this.preservedMetaSnapshot = []`
// 2. The `restore()` entry point resets it per call (no leak
// across consecutive runs in the singleton service instance)
// 3. `performDatabaseRestore` assigns to `this.preservedMetaSnapshot`
// inside the PG branch (must run before DROP)
// 4. The replay reads `this.preservedMetaSnapshot` — NOT a bare
// `preservedMeta` local — so a future refactor can't
// accidentally drop the snapshot half on the floor again.
const constructorInit = lines.some((l) =>
/this\.preservedMetaSnapshot\s*=\s*\[\s*\]/.test(l)
);
expect(constructorInit).toBe(true);
const assignmentSites = lines.filter((l) =>
/this\.preservedMetaSnapshot\s*=\s*(\[\s*\]|await\s+db)/.test(l)
);
// Constructor init + restore() per-run reset + the PG-branch
// assignment from db query. Three writes.
expect(assignmentSites.length).toBeGreaterThanOrEqual(3);
// No stray bare `preservedMeta` local-scoped declaration in
// performDatabaseRestore — would indicate someone re-introduced
// the round-1 footgun.
const dangerousLocalDecl = lines.filter((l) =>
/^\s*(let|const)\s+preservedMeta\s*=/.test(l)
);
expect(dangerousLocalDecl).toEqual([]);
});
it('every .count() result is coerced to Number before comparison', () => {
// PR #596 review caught a second PG-only landmine: pg-driver
// returns COUNT(*) as a string ("16" not 16) to preserve bigint
// precision. The original code compared `result.count !==
// expected.rowCount` and every match flagged as a mismatch on PG.
//
// The fix coerces with `Number(...)` at every comparison +
// interpolation site. This test catches a future regression where
// a refactor uses `.count` directly in a `===` / `!==` / `>` /
// `<` comparison without coercing.
//
// Heuristic: find every `.count` access in the file and make sure
// the line either:
// (a) wraps it in `Number(...)`, or
// (b) is purely an interpolation that already coerced upstream
// (e.g. `validation.warnings.push(`... ${eventCountN} ...`)`
// where eventCountN is the coerced local), or
// (c) is the docstring/comment line (filtered separately).
//
// We approximate this by listing every `.count` reference site
// and asserting that lines doing comparisons (`===`/`!==`/`>`/
// `<`/`>=`/`<=`) on a raw `.count` access without `Number(...)`
// around it are zero.
const dangerousLines = lines
.map((l, i) => ({ line: i + 1, text: l }))
// Filter to lines that compare a .count result
.filter(({ text }) => {
// Skip comments
if (/^\s*(\/\/|\*)/.test(text)) return false;
// Detect a `.count` (followed by `)` for `?.count` or by space/operator)
// being directly compared via ===/!==/>/<.
// Match the BAD pattern: `<something>.count <op> <something>`
// where <op> is === / !== / > / < / >= / <=
const bareCountInComparison = /\w+\??\.count\s*(?:!==|===|>=?|<=?)\s+/;
// ALLOW if the .count is preceded by `Number(` in the same line
const wrappedInNumber = /Number\(\s*\w+\??\.count/;
return bareCountInComparison.test(text) && !wrappedInNumber.test(text);
});
expect(dangerousLines).toEqual([]);
});
it('the completed-restore update sets was_successful=true', () => {
// Without this, every successful restore ends up with
// status='completed', was_successful=false — the dashboard's
// "last successful restore" widget then filters out the row +
// any future audit query gating on was_successful misses it.
// Caught locally + maintainer PR #596 review.
//
// Contract: the update payload that writes status='completed' on
// the SUCCESS branch ALSO includes was_successful: true. We pin
// it by source inspection so any future refactor of the success
// payload keeps both fields together.
// The success-branch update lives AFTER performPostRestoreVerification.
// There's also a `status: 'completed'` in the dry-run / early-return
// path (failure handling has its own block too) — we want the
// SUCCESS-branch one specifically.
const verifyLine = findFirst(/performPostRestoreVerification\s*\(/);
expect(verifyLine).toBeGreaterThan(0);
const completedStatusLineIdx = lines
.map((l, i) => ({ line: i + 1, text: l }))
.find(({ line, text }) =>
line > verifyLine && /status:\s*['"]completed['"]/.test(text)
);
expect(completedStatusLineIdx).toBeDefined();
// Look in the next ~10 lines for was_successful: true. The actual
// payload is small (no nested objects between status and the
// closing })), so a fixed-window search is reliable.
const window = lines.slice(
completedStatusLineIdx.line - 1,
completedStatusLineIdx.line + 10,
).join('\n');
expect(window).toMatch(/was_successful:\s*true/);
});
it('npm run migrate:safe is invoked after the replay in restore()', () => {
// Contract from PR #596 round 4: backups taken on older picpeak
// versions must restore COMPLETELY on a newer image — even if new
// migrations have been added since the backup was taken. The
// restore() flow shells out to `npm run migrate:safe` AFTER the
// operator-meta replay so the schema catches up to the running
// code WITHIN the restore boundary (not on the next container
// restart).
//
// Contract:
// 1. A `migrate:safe` shell-out exists somewhere in restoreService
// 2. It sits AFTER the replay drain — verification → replay →
// migrations is the documented order
// 3. It does NOT sit inside performDatabaseRestore (must run
// against the reinit'd pool from the parent restore())
const migrateLine = findFirst(/['"]migrate:safe['"]/);
expect(migrateLine).toBeGreaterThan(0);
const replayLine = findLast(/this\.preservedMetaSnapshot\.length\s*>\s*0/);
expect(replayLine).toBeGreaterThan(0);
expect(migrateLine).toBeGreaterThan(replayLine);
// Must NOT live inside performDatabaseRestore (same scope as the
// replay check above).
const dbRestoreStart = findFirst(/async\s+performDatabaseRestore\s*\(/);
let dbRestoreEnd = -1;
for (let i = dbRestoreStart; i < lines.length; i++) {
if (/^ \}\s*$/.test(lines[i])) {
dbRestoreEnd = i + 1;
break;
}
}
expect(migrateLine < dbRestoreStart || migrateLine > dbRestoreEnd).toBe(true);
});
it('the replay site lives in restore() AFTER performPostRestoreVerification', () => {
// PR #596 round 3 moved the replay out of performDatabaseRestore
// and into the parent restore() method, sequenced AFTER the
// post-restore verification. Otherwise the replay's upserted row
// count was being flagged as a verification mismatch (e.g.
// "expected 190, got 191" because the fresh-install seeded
// `restore_allow_force_auto_upgraded` that wasn't in the backup).
//
// Contract: the line that drains `this.preservedMetaSnapshot`
// must come AFTER `performPostRestoreVerification` AND must NOT
// sit inside `performDatabaseRestore`.
const verificationLine = findFirst(/performPostRestoreVerification\s*\(/);
expect(verificationLine).toBeGreaterThan(0);
const replayLine = findLast(/this\.preservedMetaSnapshot\.length\s*>\s*0/);
expect(replayLine).toBeGreaterThan(0);
expect(replayLine).toBeGreaterThan(verificationLine);
// `performDatabaseRestore` must not contain the replay drain.
// Find the function bounds + assert no drain line falls inside.
const dbRestoreStart = findFirst(/async\s+performDatabaseRestore\s*\(/);
expect(dbRestoreStart).toBeGreaterThan(0);
// Find the closing brace of performDatabaseRestore. Lazy heuristic:
// the first `^ \}\s*$` (two-space indent + }) after the function
// start. Brittle to indent changes but unambiguous in this codebase.
let dbRestoreEnd = -1;
for (let i = dbRestoreStart; i < lines.length; i++) {
if (/^ \}\s*$/.test(lines[i])) {
dbRestoreEnd = i + 1;
break;
}
}
expect(dbRestoreEnd).toBeGreaterThan(dbRestoreStart);
// The replay drain line must be OUTSIDE [dbRestoreStart, dbRestoreEnd].
expect(replayLine < dbRestoreStart || replayLine > dbRestoreEnd).toBe(true);
});
});
@@ -0,0 +1,219 @@
/**
* Verifies the backup-integrity check covers every CRM document
* artefact column and correctly buckets each row into:
* - verifiedOk — file exists AND hash matches (when hash is stored)
* - missing — `*_path` set but file is not on disk
* - hashMismatches — file exists but bytes don't hash to `*_sha256`
* - existsButNoHash — file exists, no `*_sha256` column for this row
*
* Uses the CRM integration harness (bootCrmDb) so the schema +
* STORAGE_PATH wiring exactly mirrors production behaviour.
*
* Background: this service is the diagnostic for the
* `storage/business-docs/` gap fixed in the same PR — without it,
* a restored install would have audit-trail columns referencing
* files that no longer exist, but admins would have no way to see
* the breakage until a customer asked for their contract back.
*/
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const { bootCrmDb, seedMinimal } = require('../integration/helpers/crmDb');
jest.setTimeout(30000);
describe('backupIntegrityService.verifyDocumentArtefacts', () => {
let db;
let cleanup;
let customerId;
let storagePath;
let backupIntegrityService;
function seedFile(relPath, content) {
const abs = path.join(storagePath, relPath);
fs.mkdirSync(path.dirname(abs), { recursive: true });
fs.writeFileSync(abs, content);
return { abs, relPath, sha: sha256(content) };
}
function sha256(content) {
return crypto.createHash('sha256').update(content).digest('hex');
}
beforeAll(async () => {
({ db, cleanup } = await bootCrmDb());
({ customerId } = await seedMinimal(db));
storagePath = process.env.STORAGE_PATH;
backupIntegrityService = require('../../src/services/backupIntegrityService');
}, 120000);
afterAll(async () => {
if (cleanup) await cleanup();
});
beforeEach(async () => {
// Wipe CRM rows between tests so each scenario sees a clean slate.
// Order matters: child tables before parents.
await db('invoice_line_items').del().catch(() => {});
await db('invoice_payment_log').del().catch(() => {});
await db('invoices').del().catch(() => {});
await db('quote_line_items').del().catch(() => {});
await db('quotes').del().catch(() => {});
await db('contracts').del().catch(() => {});
});
it('returns an empty report when no documents reference any path', async () => {
const report = await backupIntegrityService.verifyDocumentArtefacts();
expect(report.summary.totalRows).toBe(0);
expect(report.summary.verifiedOk).toBe(0);
expect(report.missing).toEqual([]);
expect(report.hashMismatches).toEqual([]);
expect(report.existsButNoHash).toEqual([]);
expect(report.scannedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/);
expect(report.scopes).toEqual(expect.arrayContaining(['quote', 'contract', 'contract-signature', 'invoice']));
});
it('flags a contract whose signed_pdf_path file is missing', async () => {
// Reference a file that we deliberately never create on disk.
// knex's `.returning('id')` returns `[{ id: N }]` on Postgres and
// newer SQLite, but `[N]` (plain int) on some SQLite versions —
// unwrap both shapes the same way the crmDb test harness does.
const inserted = await db('contracts').insert({
customer_account_id: customerId,
contract_number: 'C-2026-MISSING',
status: 'sent',
issue_date: '2026-01-01',
signed_pdf_path: 'business-docs/contract/2026/C-2026-MISSING.pdf',
created_at: new Date(),
}).returning('id');
const contractId = typeof inserted[0] === 'object' ? inserted[0].id : inserted[0];
const report = await backupIntegrityService.verifyDocumentArtefacts({ scope: ['contract'] });
const hit = report.missing.find((m) => m.rowId === contractId);
expect(hit).toMatchObject({
table: 'contracts',
column: 'signed_pdf_path',
expectedPath: 'business-docs/contract/2026/C-2026-MISSING.pdf',
});
expect(report.summary.missingFiles).toBe(1);
});
it('verifies a contract whose file exists AND hash matches', async () => {
const { relPath, sha } = seedFile(
'business-docs/contract/2026/C-2026-OK.pdf',
'this is the signed contract content',
);
await db('contracts').insert({
customer_account_id: customerId,
contract_number: 'C-2026-OK',
status: 'fully_signed',
issue_date: '2026-01-01',
signed_pdf_path: relPath,
signed_pdf_sha256: sha,
created_at: new Date(),
});
const report = await backupIntegrityService.verifyDocumentArtefacts({ scope: ['contract'] });
expect(report.summary.verifiedOk).toBeGreaterThanOrEqual(1);
expect(report.summary.missingFiles).toBe(0);
expect(report.summary.hashMismatches).toBe(0);
});
it('flags a hash mismatch when the file exists but bytes differ from signed_pdf_sha256', async () => {
const { relPath } = seedFile(
'business-docs/contract/2026/C-2026-TAMPER.pdf',
'tampered bytes on disk',
);
const inserted = await db('contracts').insert({
customer_account_id: customerId,
contract_number: 'C-2026-TAMPER',
status: 'fully_signed',
issue_date: '2026-01-01',
signed_pdf_path: relPath,
// Hash for completely different content — simulates tampering or
// bit-rot between sign-time and now.
signed_pdf_sha256: sha256('the ORIGINAL bytes the customer signed'),
created_at: new Date(),
}).returning('id');
const contractId = typeof inserted[0] === 'object' ? inserted[0].id : inserted[0];
const report = await backupIntegrityService.verifyDocumentArtefacts({ scope: ['contract'] });
const hit = report.hashMismatches.find((m) => m.rowId === contractId);
expect(hit).toBeDefined();
expect(hit.expectedSha).not.toBe(hit.actualSha);
expect(hit.column).toBe('signed_pdf_path');
});
it('buckets signature PNGs into existsButNoHash (no hash column)', async () => {
const { relPath } = seedFile(
'business-docs/contract/signatures/99/customer-1700000000000.png',
'\x89PNG\r\n\x1a\n', // doesn't have to be a real PNG, just bytes
);
await db('contracts').insert({
customer_account_id: customerId,
contract_number: 'C-2026-SIG',
status: 'fully_signed',
issue_date: '2026-01-01',
signed_customer_signature_path: relPath,
created_at: new Date(),
});
const report = await backupIntegrityService.verifyDocumentArtefacts({
scope: ['contract-signature'],
});
expect(report.summary.existsButNoHash).toBeGreaterThanOrEqual(1);
expect(report.summary.verifiedOk).toBe(0); // no hash → not "verified ok"
expect(report.summary.missingFiles).toBe(0);
const hit = report.existsButNoHash.find((r) => r.column === 'signed_customer_signature_path');
expect(hit).toBeDefined();
});
it('respects the scope filter — contract scope skips quote/invoice tables', async () => {
// Seed an invoice with a missing pdf_path AND a contract with a
// missing signed_pdf_path. Scoping to contract should only flag
// the contract.
await db('invoices').insert({
customer_account_id: customerId,
invoice_number: 'INV-2026-SCOPE',
status: 'sent',
pdf_path: 'business-docs/invoice/2026/INV-2026-SCOPE.pdf',
issue_date: '2026-01-01',
due_date: '2026-01-31',
created_at: new Date(),
});
await db('contracts').insert({
customer_account_id: customerId,
contract_number: 'C-2026-SCOPE',
status: 'sent',
issue_date: '2026-01-01',
signed_pdf_path: 'business-docs/contract/2026/C-2026-SCOPE.pdf',
created_at: new Date(),
});
const report = await backupIntegrityService.verifyDocumentArtefacts({ scope: ['contract'] });
expect(report.scopes).toEqual(['contract']);
expect(report.missing.every((m) => m.table === 'contracts')).toBe(true);
expect(report.missing.some((m) => m.table === 'invoices')).toBe(false);
});
it('covers invoices.imported_pdf_path (admin-uploaded historical scans)', async () => {
// Imported invoices are the most catastrophic case — there's no
// renderer that can reproduce them. Verifier must check this column
// alongside invoices.pdf_path.
await db('invoices').insert({
customer_account_id: customerId,
invoice_number: 'IMP-2025-001',
status: 'sent',
imported_pdf_path: 'business-docs/invoice-imports/2025/legacy.pdf',
issue_date: '2025-06-01',
due_date: '2025-07-01',
created_at: new Date(),
});
const report = await backupIntegrityService.verifyDocumentArtefacts({ scope: ['invoice'] });
const hit = report.missing.find((m) => m.column === 'imported_pdf_path');
expect(hit).toBeDefined();
});
});
+25 -2
View File
@@ -33,11 +33,34 @@ exports.up = async function(knex) {
// Try to save credentials to file, but don't fail if we can't
const dataDir = path.join(__dirname, '..', '..', 'data');
const setupInfoPath = path.join(dataDir, 'ADMIN_CREDENTIALS.txt');
// Detect a pending install-from-backup trigger. If one exists,
// these credentials are about to be obsoleted by the restore —
// the backup's admin row replaces this fresh-install one a few
// seconds from now. We still write the file (in case the
// restore fails and the fresh admin is the only way in) but
// annotate the top so admins reading the file after restore
// don't waste time trying credentials that no longer exist.
// Flagged on PR #596 review.
const fsSync = require('fs');
const backupRoot = process.env.BACKUP_ROOT || '/backup';
const triggerWillFire = fsSync.existsSync(path.join(backupRoot, 'RESTORE_ON_INSTALL'))
|| fsSync.existsSync(path.join(backupRoot, 'RESTORE_ON_INSTALL.txt'));
const restoreNotice = triggerWillFire ? `
⚠️ RESTORE_ON_INSTALL TRIGGER DETECTED ⚠️
These credentials are temporary. An install-from-backup run is queued
to fire on the next server start, which will REPLACE this admin row
with the one from the backup. After the restore completes, log in
with your ORIGINAL pre-disaster credentials — not the ones below.
If the restore fails for some reason, the credentials below remain
valid as a fallback recovery path.
` : '';
const setupInfo = `
========================================
PicPeak Admin Credentials
========================================
========================================${restoreNotice}
Your admin account has been created with these credentials:
@@ -100,8 +100,18 @@ exports.up = async function(knex) {
// Add restore-related settings to app_settings
const restoreSettings = [
{
// Default ON so fresh installs can recover from disaster
// without the catch-22 documented in _restoreSettingsBoot.js
// (fresh-install admin user trips the "1 active admin" warning,
// which can only be overridden with force=true, which the wizard
// refused if this setting was false — exactly the moment an
// admin can least afford a SQL incantation). Flipped from false
// to true 2026-05-30. Existing installs that ran this migration
// with the OLD value will get auto-upgraded once by the boot
// self-heal in _restoreSettingsBoot.js — see the
// `restore_allow_force_auto_upgraded` guard there.
setting_key: 'restore_allow_force',
setting_value: JSON.stringify(false),
setting_value: JSON.stringify(true),
setting_type: 'restore'
},
{
@@ -0,0 +1,139 @@
/**
* Migration 109 — config-driven backup walker.
*
* (Originally numbered 108 on bugfix/crm-backup. Renumbered to 109
* before merge because upstream/beta independently shipped
* 108_seed_sl_email_template_translations.js. The createTable is
* idempotent via `hasTable` guard and the seed uses
* `onConflict('path').ignore()`, so beta installs that ran the
* 108-named version of this file get a harmless no-op when 109
* runs against the already-seeded table.)
*
* Stage B of the three-stage backup-hardening plan. The file-backup
* walker (`getFilesToBackupInternal` in backupService.js) historically
* hard-coded its list of subdirectories: events/active, events/archived,
* thumbnails, previews, heroes, uploads, business-docs.
*
* That list is a footgun every time a new feature lands that drops
* artefacts under STORAGE_PATH/<something>/ — the maintainer has to
* remember to edit the walker, and there's no schema-level record of
* what *should* be backed up. The CRM rollout missed `business-docs`
* for ~6 months (#XXX) for exactly this reason.
*
* This migration introduces a `backup_paths` table that the walker
* reads at runtime. New features add a row; the walker picks them up
* automatically. The `feature_flag` column gates scans behind an
* existing app_settings boolean (e.g. `backup_include_archived`),
* mirroring how the previous `includeArchived` parameter worked.
*
* Columns:
* - path : relative to STORAGE_PATH, unique
* - include_in_default : on/off without deleting the row (so
* audit trail of "we used to back this up"
* is preserved)
* - feature_flag : nullable; when set, walker checks the
* same-named app_settings boolean before
* scanning. Matches the existing pattern
* used by `backup_include_archived`.
* - display_order : controls admin-UI listing order
* - description : human-readable purpose, shown in admin UI
*
* Defense-in-depth: the walker also keeps a hard-coded LEGACY_DEFAULTS
* fallback so that if this table is somehow empty (failed migration on
* an existing install, manual truncation), backups still cover the
* historical set instead of silently shipping nothing. The boot-time
* self-heal in `_backupPathsBoot.js` re-seeds missing default rows on
* every startup so newly-added defaults reach already-deployed
* installs without a follow-up migration.
*
* Idempotent: skips the createTable if it already exists, and the
* seed uses `onConflict('path').ignore()` so re-runs don't duplicate.
*/
const DEFAULT_PATHS = [
{
path: 'events/active',
include_in_default: true,
feature_flag: null,
display_order: 10,
description: 'Active gallery photo originals',
},
{
path: 'events/archived',
include_in_default: true,
feature_flag: 'backup_include_archived',
display_order: 20,
description: 'Archived gallery photo originals (gated by backup_include_archived)',
},
{
path: 'thumbnails',
include_in_default: true,
feature_flag: null,
display_order: 30,
description: 'Generated gallery thumbnails',
},
{
path: 'previews',
include_in_default: true,
feature_flag: null,
display_order: 40,
description: 'Lightbox preview tier (#492)',
},
{
path: 'heroes',
include_in_default: true,
feature_flag: null,
display_order: 50,
description: 'Gallery hero header images',
},
{
path: 'uploads',
include_in_default: true,
feature_flag: null,
display_order: 60,
description: 'Direct uploads root (wet-signature contracts, imported invoices, etc.)',
},
{
path: 'business-docs',
include_in_default: true,
feature_flag: null,
display_order: 70,
description: 'CRM PDFs, signature artefacts, admin-imported historical invoices',
},
];
exports.up = async function(knex) {
const exists = await knex.schema.hasTable('backup_paths');
if (!exists) {
await knex.schema.createTable('backup_paths', (t) => {
t.increments('id').primary();
t.string('path', 256).notNullable().unique();
t.boolean('include_in_default').notNullable().defaultTo(true);
t.string('feature_flag', 64).nullable();
t.integer('display_order').notNullable().defaultTo(100);
t.string('description', 256).nullable();
t.timestamp('created_at').notNullable().defaultTo(knex.fn.now());
t.timestamp('updated_at').notNullable().defaultTo(knex.fn.now());
});
}
// Seed defaults — `onConflict('path').ignore()` so already-seeded rows
// (manual edits by admins, prior partial runs) survive untouched.
await knex('backup_paths')
.insert(DEFAULT_PATHS.map((row) => ({
...row,
created_at: new Date(),
updated_at: new Date(),
})))
.onConflict('path')
.ignore();
};
exports.down = async function(knex) {
await knex.schema.dropTableIfExists('backup_paths');
};
// Exported so the self-heal boot helper can reuse the same authoritative
// list without re-declaring it. Tests also import this to assert the
// walker is reading from this source.
exports.DEFAULT_PATHS = DEFAULT_PATHS;
+43
View File
@@ -660,6 +660,7 @@ app.use('/api/admin/contracts', require('./src/routes/adminContracts'));
app.use('/api/admin/calendar', require('./src/routes/adminCalendar'));
app.use('/api/admin/deals', require('./src/routes/adminDeals'));
app.use('/api/admin/tax-report', require('./src/routes/adminTaxReport'));
app.use('/api/admin/system-health', require('./src/routes/adminSystemHealth'));
app.use('/api/admin/dev', require('./src/routes/adminDev'));
app.use('/api/public/quotes', require('./src/routes/publicQuotes'));
app.use('/api/public/contracts', require('./src/routes/publicContracts'));
@@ -799,6 +800,48 @@ async function startServer() {
const { startS3AutoImporter } = require('./src/services/s3AutoImporter');
startS3AutoImporter();
// Self-heal the `backup_paths` table before the backup service
// starts — the file-backup walker reads from it, so missing
// canonical rows (a new subdirectory shipped by a future feature)
// get re-seeded here on every boot. See _backupPathsBoot.js for
// the full rationale; pattern mirrors _emailTemplateBoot.js.
try {
const { seedBackupPathsAtBoot } = require('./src/services/_backupPathsBoot');
await seedBackupPathsAtBoot(db, logger);
} catch (err) {
logger.warn('backup_paths self-heal failed at boot:', err.message);
}
// Self-heal restore-meta settings — currently just
// `restore_allow_force` defaulting to ON so fresh installs can
// recover from disaster without a SQL incantation. Only seeds on
// FRESH installs (existing rows, true or false, are preserved).
// See _restoreSettingsBoot.js for the full rationale.
try {
const { seedRestoreSettingsAtBoot } = require('./src/services/_restoreSettingsBoot');
await seedRestoreSettingsAtBoot(db, logger);
} catch (err) {
logger.warn('restore-settings self-heal failed at boot:', err.message);
}
// Install-from-backup trigger. If `RESTORE_ON_INSTALL` (or
// `.txt`) exists in the /backup mount AND the DB is empty, run
// the restore HERE before any admin UI surfaces. Lets admins
// recover a picpeak install with: (a) place backup files in the
// bind mount, (b) drop the trigger file, (c) `docker compose up`.
// No onboarding wizard, no throwaway admin, no compose-file
// changes. See _installFromBackupBoot.js for the full rationale
// + the safety gates.
try {
const { tryInstallFromBackup } = require('./src/services/_installFromBackupBoot');
const result = await tryInstallFromBackup(db, logger);
if (result.ran) {
logger.info(`Install-from-backup: completed from ${result.manifestPath}. Server will start with restored state.`);
}
} catch (err) {
logger.warn('Install-from-backup hook threw:', err.message);
}
// Start backup service
await startBackupService();
+68 -3
View File
@@ -24,8 +24,73 @@ try {
try { logger.warn('SQLite directory ensure failed', { error: e.message }); } catch (_) {}
}
// Create database connection with built-in retry logic
const db = knex(knexConfig);
// Create database connection with built-in retry logic.
//
// The underlying knex instance is held in `_db` and reachable through a
// Proxy `db` that forwards every call to the current instance. This
// indirection exists so `reinitPool()` below can swap the live pool
// without breaking the thousands of existing `const { db } = require(...)`
// imports — they capture the Proxy once, and every subsequent
// `db('table')` / `db.schema.hasTable(...)` lookup goes through the
// Proxy to whatever `_db` currently points at.
//
// Used by the restore service after DROP/CREATE DATABASE: the old pool
// is destroyed during the drop (to release PG connections so the DROP
// can succeed), then `reinitPool()` opens a fresh pool against the
// recreated DB. Without this, every query in the process after a
// restore failed with "Unable to acquire a connection" until the
// container was manually restarted — exactly the footgun Ralf hit
// repeatedly on 2026-05-30.
let _db = knex(knexConfig);
const db = new Proxy(function knexCall() {}, {
// db('tableName') — knex's query builder entry point
apply(_target, _thisArg, args) {
return _db(...args);
},
// db.schema, db.raw, db.migrate, etc.
get(_target, prop) {
const v = _db[prop];
return typeof v === 'function' ? v.bind(_db) : v;
},
// Defensive: future code that does `if ('schema' in db)` works.
has(_target, prop) { return prop in _db; },
});
/**
* Tear down the current knex pool and open a fresh one.
*
* Idempotent — calling it twice in a row just destroys + recreates
* twice, no error. Throws if the new pool can't establish a
* connection (which surfaces a clear error rather than letting the
* caller proceed with a half-broken pool).
*
* Used by restoreService after the DROP/CREATE DATABASE pair.
*/
async function reinitPool() {
const prev = _db;
try {
await prev.destroy();
} catch (err) {
logger.warn(`Old pool destroy failed (continuing with reinit): ${err.message}`);
}
_db = knex(knexConfig);
// Probe the new pool with a no-op query so we fail loudly here if
// the new pool can't connect — better than silently handing the
// caller a broken pool and surfacing the error on the next admin
// request.
try {
if (knexConfig.client === 'pg') {
await _db.raw('SELECT 1');
} else {
await _db.raw('SELECT 1');
}
} catch (probeErr) {
logger.error('New pool failed health-check after reinit', { error: probeErr.message });
throw probeErr;
}
logger.info('knex pool re-initialized successfully');
}
// Connection retry configuration
const MAX_RETRIES = 3;
@@ -607,4 +672,4 @@ async function logActivity(activityType, metadata = {}, eventId = null, actor =
}
}
module.exports = { db, initializeDatabase, logActivity, withRetry };
module.exports = { db, initializeDatabase, logActivity, withRetry, reinitPool };
+13 -2
View File
@@ -13,7 +13,18 @@ let cachedTimeout = null;
let cacheExpiry = 0;
const CACHE_DURATION = 30 * 60 * 1000; // 30 minutes - reduced DB queries
// Clean up expired sessions every 5 minutes
// Clean up expired sessions every 5 minutes.
//
// `.unref()` so this timer doesn't keep the event loop alive on its
// own — without it, every jest worker that requires this module
// (directly or transitively via server.js / a middleware-importing
// route file) gets stuck and either prints the "worker failed to
// exit gracefully" warning or, under high CI load, force-kills mid-
// test and takes an unrelated suite down with it (we hit this with
// integration/storageBackend.test.js on PR #555). Production
// behaviour is unchanged: the timer fires every 5 min as long as
// the server has anything else keeping the loop alive (HTTP server,
// other intervals), which is always.
setInterval(() => {
const now = Date.now();
for (const [token, lastActivity] of sessions.entries()) {
@@ -21,7 +32,7 @@ setInterval(() => {
sessions.delete(token);
}
}
}, 5 * 60 * 1000);
}, 5 * 60 * 1000).unref();
async function getSessionTimeout() {
const now = Date.now();
+269 -101
View File
@@ -348,51 +348,7 @@ router.get('/run/:id/report', requirePermission('backup.view'), async (req, res)
*/
router.get('/available-backups', requirePermission('backup.view'), async (req, res) => {
try {
const backups = [];
// Get local file backups
const backupConfig = await getBackupConfig();
if (backupConfig.backup_destination_type === 'local' && backupConfig.backup_destination_path) {
try {
const files = await fs.readdir(backupConfig.backup_destination_path);
for (const file of files) {
if (file.endsWith('.json') || file.endsWith('.yaml')) {
const filePath = path.join(backupConfig.backup_destination_path, file);
const stats = await fs.stat(filePath);
backups.push({
type: 'local',
name: file,
path: filePath,
size: stats.size,
modified: stats.mtime
});
}
}
} catch (error) {
logger.warn('Failed to list local backups:', error);
}
}
// Get database backups from backup_runs table
const backupRuns = await db('backup_runs')
.where('status', 'completed')
.whereNotNull('manifest_path')
.orderBy('completed_at', 'desc')
.limit(20);
for (const run of backupRuns) {
backups.push({
type: run.manifest_path.startsWith('s3://') ? 's3' : 'local',
name: `Backup ${run.completed_at}`,
path: run.manifest_path,
manifestId: run.manifest_id,
size: run.total_size_bytes,
filesCount: run.files_backed_up,
duration: run.duration_seconds,
completed: run.completed_at
});
}
const backups = await discoverAvailableBackups();
res.json({
success: true,
data: backups
@@ -406,6 +362,213 @@ router.get('/available-backups', requirePermission('backup.view'), async (req, r
}
});
/**
* Discover restorable backups by walking the configured destination
* directory + harvesting the backup_runs table.
*
* **Why we recurse the disk first, DB second**
*
* The disk is the source of truth for restore. After a disaster
* (`docker compose down -v`, drive corruption, fresh install) the
* `backup_runs` table is empty — but the manifest JSONs are exactly
* what's left on disk for an admin to recover from. A wizard that
* only reads the DB shows "No backups found" precisely when the
* admin needs it most. So we walk first, dedupe-by-manifestId
* against any surviving DB rows, and present a unified list.
*
* Discovery rules:
* - Walks `backup_destination_path` AND `backup_manifest_path` if
* they're distinct (manifests can live in a sibling directory).
* - Recurses up to 3 levels deep — enough to find
* `<root>/manifests/backup-manifest-<id>.json` (the default
* layout) without scanning the entire photo tree.
* - Matches manifest files by glob: `backup-manifest-*.json`,
* `backup-manifest-*.yaml`, and the legacy bare `manifest.json`.
* - Parses each manifest to extract real metadata (timestamp,
* size, file count, source type) instead of showing the admin
* a list of opaque filenames.
*
* Returns: array of `{ type, name, path, manifestId, size,
* filesCount, completed, source: 'disk' | 'db' }`.
*/
async function discoverAvailableBackups() {
const backupConfig = await getBackupConfig();
const backups = [];
const seenManifestIds = new Set();
if (backupConfig.backup_destination_type === 'local') {
const roots = new Set();
if (backupConfig.backup_destination_path) roots.add(backupConfig.backup_destination_path);
if (backupConfig.backup_manifest_path) roots.add(backupConfig.backup_manifest_path);
for (const root of roots) {
try {
const manifestPaths = await walkForManifests(root, 3);
for (const filePath of manifestPaths) {
try {
const parsed = await parseManifestMetadata(filePath);
if (parsed.manifestId) seenManifestIds.add(parsed.manifestId);
backups.push(parsed);
} catch (err) {
// Don't fail discovery because ONE manifest is corrupt —
// surface the file with a note so the admin sees something
// is wrong and can investigate.
logger.warn(`Manifest unreadable at ${filePath}: ${err.message}`);
const stats = await fs.stat(filePath).catch(() => null);
backups.push({
type: 'local',
name: path.basename(filePath),
path: filePath,
manifestId: null,
size: stats?.size || 0,
filesCount: null,
completed: stats?.mtime || null,
source: 'disk',
corrupt: true,
error: err.message,
});
}
}
} catch (err) {
logger.warn(`Could not scan backup root ${root}: ${err.message}`);
}
}
}
// Layer in surviving DB rows, deduping by manifest_id so we don't
// show the same backup twice with different shapes.
const backupRuns = await db('backup_runs')
.where('status', 'completed')
.whereNotNull('manifest_path')
.orderBy('completed_at', 'desc')
.limit(50);
for (const run of backupRuns) {
if (run.manifest_id && seenManifestIds.has(run.manifest_id)) continue;
backups.push({
type: run.manifest_path.startsWith('s3://') ? 's3' : 'local',
name: `Backup ${run.completed_at}`,
path: run.manifest_path,
manifestId: run.manifest_id,
size: run.total_size_bytes,
filesCount: run.files_backed_up,
duration: run.duration_seconds,
completed: run.completed_at,
source: 'db',
});
}
// Most recent first.
backups.sort((a, b) => {
const aTime = a.completed ? new Date(a.completed).getTime() : 0;
const bTime = b.completed ? new Date(b.completed).getTime() : 0;
return bTime - aTime;
});
return backups;
}
/**
* Recursive manifest finder. Depth-limited so we don't enumerate
* thousands of photo files. Yields absolute paths.
*/
async function walkForManifests(dir, maxDepth, depth = 0) {
if (depth > maxDepth) return [];
let entries;
try {
entries = await fs.readdir(dir, { withFileTypes: true });
} catch (err) {
if (err.code === 'ENOENT') return [];
throw err;
}
const out = [];
for (const entry of entries) {
const full = path.join(dir, entry.name);
if (entry.isDirectory()) {
// Skip obvious noise: photo trees, node_modules, hidden dirs.
if (entry.name === 'events' || entry.name === 'business-docs'
|| entry.name === 'thumbnails' || entry.name === 'previews'
|| entry.name === 'heroes' || entry.name === 'uploads'
|| entry.name.startsWith('.')
|| entry.name === 'node_modules') continue;
out.push(...await walkForManifests(full, maxDepth, depth + 1));
} else if (entry.isFile() && isManifestFilename(entry.name)) {
out.push(full);
}
}
return out;
}
function isManifestFilename(name) {
// Canonical: backup-manifest-<id>.json / .yaml
// Legacy: manifest.json (inside backup-<id>/manifest.json layout)
// Be liberal in what we accept — admin may have renamed.
if (/^backup-manifest-.+\.(json|ya?ml)$/i.test(name)) return true;
if (/^manifest\.(json|ya?ml)$/i.test(name)) return true;
return false;
}
/**
* Parse a manifest file and pull out the fields the wizard wants.
* Tolerates schema drift across manifest versions (v1, v2) by
* checking multiple shapes.
*/
async function parseManifestMetadata(filePath) {
const raw = await fs.readFile(filePath, 'utf8');
let parsed;
if (filePath.toLowerCase().endsWith('.json')) {
parsed = JSON.parse(raw);
} else {
// Minimal YAML support — most admins use JSON; only do require()
// if a .yaml manifest is actually present.
const yaml = require('js-yaml');
parsed = yaml.load(raw);
}
const stats = await fs.stat(filePath);
// v2 shape: { manifest: { id, timestamp }, backup: { ... }, files: [...], database: { ... } }
// v1 shape: { backup_id, started_at, files: [...] } (older)
const manifestId =
parsed?.manifest?.id
|| parsed?.backup?.id
|| parsed?.backup_id
|| null;
const completed =
parsed?.backup?.completed_at
|| parsed?.manifest?.timestamp
|| parsed?.completed_at
|| stats.mtime;
const filesCount =
(Array.isArray(parsed?.files) ? parsed.files.length : null)
?? parsed?.backup?.total_files
?? null;
const totalSizeBytes =
parsed?.backup?.total_size_bytes
?? parsed?.total_size_bytes
?? null;
return {
type: 'local',
name: path.basename(filePath),
path: filePath,
manifestId,
size: totalSizeBytes ?? stats.size,
filesCount,
completed,
source: 'disk',
databaseIncluded: Boolean(parsed?.database?.backup_file),
// Helpful for the UI: lets it show "This backup has no DB" warning
// — exactly the surface that would have caught Ralf's original
// four files-only manifests if it had existed.
schemaVersion: parsed?.manifest?.version || parsed?.version || '1.0',
};
}
/**
* List backups for restore (POST version for frontend compatibility)
* Accepts source type in request body
@@ -413,59 +576,58 @@ router.get('/available-backups', requirePermission('backup.view'), async (req, r
router.post('/list-backups', requirePermission('backup.view'), async (req, res) => {
try {
const { source } = req.body; // 'local', 's3', or undefined for all
const backups = [];
// Get backup configuration
const backupConfig = await getBackupConfig();
// Use the same disk-first discovery the GET endpoint uses so that
// a fresh post-`docker compose down -v` install (empty backup_runs
// table) can still see what's on disk. The whole point of restore
// is "the DB is broken, rebuild it from disk" — a wizard that
// only queries the DB shows "No backups found" exactly when it's
// needed most. See discoverAvailableBackups for the full rationale.
const discovered = await discoverAvailableBackups();
// Get database backups from backup_runs table
const backupRuns = await db('backup_runs')
.where('status', 'completed')
.whereNotNull('manifest_path')
.orderBy('completed_at', 'desc')
.limit(20);
const filtered = source
? discovered.filter((b) => b.type === source)
: discovered;
for (const run of backupRuns) {
const isS3 = run.manifest_path.startsWith('s3://');
const backupType = isS3 ? 's3' : 'local';
// Filter by source if specified
if (source && source !== backupType) {
continue;
}
backups.push({
id: run.id,
type: backupType,
name: `Backup from ${new Date(run.completed_at).toLocaleString()}`,
path: run.manifest_path,
manifest_path: run.manifest_path,
manifestId: run.manifest_id,
manifestPath: run.manifest_path,
size: parseInt(run.total_size_bytes) || 0,
total_size: parseInt(run.total_size_bytes) || 0,
total_size_bytes: parseInt(run.total_size_bytes) || 0,
filesCount: run.files_backed_up || 0,
files_backed_up: run.files_backed_up || 0,
duration: run.duration_seconds,
duration_seconds: run.duration_seconds,
// Frontend expects snake_case date fields
created_at: run.completed_at,
completed_at: run.completed_at,
started_at: run.started_at,
// camelCase aliases
completedAt: run.completed_at,
startedAt: run.started_at,
// Backup metadata
status: run.status,
backup_type: run.backup_type,
backupType: run.backup_type,
backup_mode: run.backup_mode,
backupMode: run.backup_mode,
app_version: run.app_version,
appVersion: run.app_version
});
}
// Shape for frontend compatibility — preserves every alias the
// frontend was already reading (snake_case + camelCase), so the
// UI rendering doesn't have to change.
const backups = filtered.map((b) => ({
id: b.manifestId || null,
type: b.type,
name: b.completed
? `Backup from ${new Date(b.completed).toLocaleString()}`
: b.name,
path: b.path,
manifest_path: b.path,
manifestId: b.manifestId,
manifestPath: b.path,
size: parseInt(b.size) || 0,
total_size: parseInt(b.size) || 0,
total_size_bytes: parseInt(b.size) || 0,
filesCount: b.filesCount || 0,
files_backed_up: b.filesCount || 0,
duration: b.duration || null,
duration_seconds: b.duration || null,
created_at: b.completed,
completed_at: b.completed,
started_at: b.completed,
completedAt: b.completed,
startedAt: b.completed,
status: 'completed',
// Stage A-aware: when the source is a disk-scanned manifest we
// can tell the wizard whether the DB dump is present, so the
// UI can warn before the admin picks a files-only backup.
database_included: b.databaseIncluded,
databaseIncluded: b.databaseIncluded,
corrupt: b.corrupt || false,
// Provenance: 'disk' (manifest read from filesystem) vs 'db'
// (backup_runs row that the disk didn't surface) — useful for
// debugging which side is missing.
source: b.source,
schema_version: b.schemaVersion,
schemaVersion: b.schemaVersion,
}));
res.json({
success: true,
@@ -556,17 +718,23 @@ async function getRestoreSettings() {
const settings = await db('app_settings')
.where('setting_type', 'restore')
.select('setting_key', 'setting_value');
const result = {};
settings.forEach(setting => {
// Convert boolean strings to actual booleans
if (setting.setting_value === '1' || setting.setting_value === '0') {
result[setting.setting_key] = setting.setting_value === '1';
// Boolean-string normalization. Historically only handled '1'/'0',
// but other code paths (boot self-heal, admin UI, direct SQL) write
// 'true' / 'false' or JSON-encoded "true" / "false". Accept all four
// shapes so `!settings.<key>` evaluates correctly downstream.
const raw = setting.setting_value;
if (raw === '1' || raw === 'true' || raw === '"true"') {
result[setting.setting_key] = true;
} else if (raw === '0' || raw === 'false' || raw === '"false"') {
result[setting.setting_key] = false;
} else {
result[setting.setting_key] = setting.setting_value;
result[setting.setting_key] = raw;
}
});
return result;
}
+87
View File
@@ -0,0 +1,87 @@
/**
* Admin → System Health
*
* Endpoint mounted at /api/admin/system-health. The "Backup
* integrity" sub-endpoint is the on-demand verifier for CRM
* document artefacts — confirms every `*_path` column on quotes /
* contracts / invoices points at a file that actually exists on
* disk and (where a `*_sha256` column is set) the file's bytes
* still hash to the expected value.
*
* Per the design decisions locked with the maintainer:
* - On-demand only; no scheduler (D1)
* - Not auto-triggered after restore (D2)
* - Wet-upload contracts are hash-verified same as system-rendered (D3)
*
* Read-only. Returns a JSON report — never mutates DB or fs.
*/
const express = require('express');
const { query } = require('express-validator');
const { adminAuth } = require('../middleware/auth');
const { requirePermission } = require('../middleware/permissions');
const { handleAsync, validateRequest, successResponse } = require('../utils/routeHelpers');
const { verifyDocumentArtefacts } = require('../services/backupIntegrityService');
const { getCoverageReport } = require('../services/backupCoverageService');
const router = express.Router();
router.use(adminAuth);
const VALID_SCOPES = ['quote', 'contract', 'contract-signature', 'invoice'];
router.get(
'/backup-integrity',
requirePermission('settings.view'),
[
// CSV string like `?scope=contract,invoice`. Each member must be
// one of the four known scopes. Empty / omitted means full scan.
query('scope').optional({ values: 'falsy' }).isString().isLength({ max: 128 }),
],
handleAsync(async (req, res) => {
validateRequest(req);
let scope;
if (req.query.scope) {
scope = String(req.query.scope)
.split(',')
.map((s) => s.trim())
.filter(Boolean);
// Defense-in-depth: reject unknown scope tokens so a typo doesn't
// silently scan everything when the caller wanted just one slice.
const unknown = scope.filter((s) => !VALID_SCOPES.includes(s));
if (unknown.length > 0) {
return res.status(400).json({
error: `Unknown scope(s): ${unknown.join(', ')}`,
code: 'BACKUP_INTEGRITY_UNKNOWN_SCOPE',
validScopes: VALID_SCOPES,
});
}
}
const report = await verifyDocumentArtefacts({ scope });
return successResponse(res, { report });
}),
);
/**
* GET /api/admin/system-health/backup-coverage
*
* Stage C of the backup-hardening plan. Returns the data-driven
* coverage report — what the next "Run Backup Now" will include /
* skip / silently miss, plus the database-dump status block.
*
* Read-only, on-demand. No scope parameter — the report is cheap
* (only top-level directory listing under STORAGE_PATH, no recursion).
*
* See backupCoverageService.js for the full rationale and the
* coverage-classification rules.
*/
router.get(
'/backup-coverage',
requirePermission('settings.view'),
handleAsync(async (req, res) => {
const report = await getCoverageReport();
return successResponse(res, { report });
}),
);
module.exports = router;
@@ -201,14 +201,22 @@ describe('DatabaseBackupService', () => {
delete: jest.fn().mockResolvedValue(1)
});
// Mock fs.unlink
fs.unlink = jest.fn().mockResolvedValue(undefined);
// Stub fs.promises.unlink via jest.spyOn so the original is
// restored when the test finishes. The previous form
// (`fs.unlink = jest.fn()`) leaked into every test that ran
// after this one in the same jest worker — most visibly
// integration/storageBackend.test.js, whose LocalFsStorage
// delete() became a silent no-op and the subsequent
// exists() assertion flipped from false to true. spyOn +
// mockRestore in afterEach keeps the stub scoped to this test.
const unlinkSpy = jest.spyOn(fs, 'unlink').mockResolvedValue(undefined);
await service.cleanupOldBackups(30);
expect(fs.unlink).toHaveBeenCalledTimes(2);
expect(fs.unlink).toHaveBeenCalledWith('/backup/old1.sql.gz');
expect(fs.unlink).toHaveBeenCalledWith('/backup/old2.sql.gz');
expect(unlinkSpy).toHaveBeenCalledTimes(2);
expect(unlinkSpy).toHaveBeenCalledWith('/backup/old1.sql.gz');
expect(unlinkSpy).toHaveBeenCalledWith('/backup/old2.sql.gz');
unlinkSpy.mockRestore();
});
});
+90
View File
@@ -0,0 +1,90 @@
/**
* Boot-time self-heal for the `backup_paths` table.
*
* **Why this exists**
*
* Knex won't re-run an applied migration, so once migration
* 109_add_backup_paths.js has run, any later default we want to add
* (a new subdirectory shipped by a future feature) would never reach
* already-deployed installs. The historical fix for this kind of
* "schema is fine, seed drifted" problem is the boot-time self-heal
* pattern documented in [[feedback_self_heal_pattern]] — we just
* re-apply the canonical seed on every boot with `onConflict.ignore()`
* so admin edits stay intact and new rows trickle in.
*
* **Authoritative list**
*
* The list of defaults lives on migration 109 itself
* (`DEFAULT_PATHS` export) — one source of truth that both the
* migration and this seeder read. Tests assert these two stay in
* lockstep.
*
* **Failure semantics**
*
* If the table doesn't exist yet (migrations haven't run, fresh
* install before migration 109 lands, etc.) we no-op and log. The
* walker has a hard-coded `LEGACY_DEFAULTS` fallback for the same
* reason — defense in depth so "Run Backup Now" can never silently
* ship a files-only manifest because of a seed issue. See
* `backupService.js` getFilesToBackupInternal.
*/
const { DEFAULT_PATHS } = require('../../migrations/core/109_add_backup_paths');
let booted = false;
/**
* Idempotently re-seed `backup_paths` with the canonical defaults.
*
* @param {object} db knex instance
* @param {object} logger app logger (must expose .info / .warn)
* @returns {Promise<{ seeded: string[] }>} paths newly inserted on this boot.
*/
async function seedBackupPathsAtBoot(db, logger) {
const log = logger || { info: () => {}, warn: () => {} };
if (booted) return { seeded: [] };
if (!(await db.schema.hasTable('backup_paths'))) {
log.warn('backup_paths table missing at boot — self-heal skipped (migration 109 may not have run yet)');
return { seeded: [] };
}
// Diff: which canonical paths are missing from the table right now?
// We can't easily get "what got inserted by onConflict.ignore" out of
// knex on both backends, so we just compute the diff ourselves and log
// it — admins benefit from seeing exactly what got auto-added when a
// new feature ships.
const existing = await db('backup_paths').select('path');
const existingSet = new Set(existing.map((r) => r.path));
const missing = DEFAULT_PATHS.filter((p) => !existingSet.has(p.path));
if (missing.length === 0) {
booted = true;
return { seeded: [] };
}
try {
await db('backup_paths')
.insert(missing.map((row) => ({
...row,
created_at: new Date(),
updated_at: new Date(),
})))
.onConflict('path')
.ignore();
log.info(`backup_paths self-heal added ${missing.length} row(s): ${missing.map((m) => m.path).join(', ')}`);
} catch (err) {
log.warn(`backup_paths self-heal failed: ${err.message}`);
}
booted = true;
return { seeded: missing.map((m) => m.path) };
}
// Test-only: reset the module-level boot flag so jest can re-exercise
// the seeder against a fresh test DB inside a single worker.
function _resetBootForTests() {
booted = false;
}
module.exports = { seedBackupPathsAtBoot, _resetBootForTests };
@@ -0,0 +1,242 @@
/**
* Install-from-backup boot hook.
*
* **The problem this closes**
*
* Without this hook, recovering a picpeak install from a backup is a
* six-step process:
* 1. Stand up the compose stack with empty volumes
* 2. Wait for boot → land on the onboarding wizard
* 3. Create a throwaway fresh-install admin
* 4. Navigate to Backup → Restore
* 5. Walk through the wizard with Force Restore ticked
* 6. Log out, log back in with original (pre-disaster) credentials
*
* With this hook, admins skip steps 2-6. They place their backup
* artefacts in the existing bind-mounted /backup directory, drop a
* trigger file alongside, and the next container start runs the
* restore BEFORE creating the throwaway onboarding admin. Server
* comes up populated, original login works first try.
*
* **The trigger file convention (chosen for zero compose-file changes)**
*
* Drop a file named `RESTORE_ON_INSTALL` (no extension, or .txt) into
* the root of the `/backup` mount. Two payload variants:
*
* 1. EMPTY file (or pure whitespace) — auto-pick the newest
* `backup-manifest-*.json` from `/backup/manifests/`. Useful when
* the admin doesn't know or care which one is most recent.
*
* 2. NON-EMPTY file containing a relative or absolute path to a
* specific manifest. Trimmed; first line wins. Useful when the
* admin wants a specific older backup.
*
* After a successful restore the trigger file is DELETED so the next
* boot doesn't re-trigger. On failure the file is preserved + the
* error is logged, so the admin can fix the input and retry by just
* restarting the container.
*
* **Safety**
*
* Three layers gate this against accidental data loss:
* 1. Trigger file must exist (intentional admin action, not auto-magic)
* 2. DB must be empty — admin_users.count = 0 AND events.count = 0.
* If either is non-zero, the hook refuses to run.
* 3. The Stage A restore path (with all of tonight's fixes) handles
* the actual swap atomically. If anything fails, the rollback
* runs and the install stays in fresh-install state.
*
* Override: `INSTALL_FROM_BACKUP_FORCE=true` skips the empty-DB check
* for the "I know what I'm doing" edge case (e.g. dev environment
* rebuilds where there's leftover data that's safe to clobber).
*/
const fs = require('fs');
const path = require('path');
const TRIGGER_FILENAMES = ['RESTORE_ON_INSTALL', 'RESTORE_ON_INSTALL.txt'];
/**
* Resolve which file should be treated as the trigger. Returns
* `{ triggerPath, manifestPath }` if found, or null if no trigger
* file is present (the common case — most boots).
*/
async function findTrigger(backupRoot, logger) {
for (const name of TRIGGER_FILENAMES) {
const triggerPath = path.join(backupRoot, name);
if (fs.existsSync(triggerPath)) {
let payload;
try {
payload = fs.readFileSync(triggerPath, 'utf8').trim();
} catch (err) {
logger.warn(`Install-from-backup: trigger file ${triggerPath} is unreadable: ${err.message}`);
return null;
}
if (!payload) {
// Auto-pick the newest manifest
const manifestsDir = path.join(backupRoot, 'manifests');
if (!fs.existsSync(manifestsDir)) {
logger.warn(`Install-from-backup: trigger file found but ${manifestsDir} doesn't exist`);
return null;
}
const entries = fs.readdirSync(manifestsDir)
.filter((f) => /^backup-manifest-.+\.(json|ya?ml)$/i.test(f))
.map((f) => {
const full = path.join(manifestsDir, f);
return { full, mtime: fs.statSync(full).mtimeMs };
})
.sort((a, b) => b.mtime - a.mtime);
if (entries.length === 0) {
logger.warn(`Install-from-backup: no manifests found in ${manifestsDir}`);
return null;
}
return { triggerPath, manifestPath: entries[0].full };
}
// Take the first non-empty line as the manifest path
const firstLine = payload.split(/\r?\n/).find((l) => l.trim()) || '';
const manifestPath = path.isAbsolute(firstLine)
? firstLine
: path.join(backupRoot, firstLine);
if (!fs.existsSync(manifestPath)) {
logger.warn(`Install-from-backup: trigger file points at ${manifestPath} which doesn't exist`);
return null;
}
return { triggerPath, manifestPath };
}
}
return null;
}
/**
* Check the DB is empty enough that restoring on top is safe.
* Returns `true` if safe, `false` if there's existing data.
*/
async function isDatabaseFresh(db, logger) {
try {
if (!(await db.schema.hasTable('admin_users'))) {
// No admin_users table yet — schema is mid-migration or wholly
// empty. Definitely safe to restore on top.
return true;
}
const adminCount = await db('admin_users').count('* as c').first();
const adminN = Number(adminCount?.c || 0);
let eventN = 0;
if (await db.schema.hasTable('events')) {
const eventCount = await db('events').count('* as c').first();
eventN = Number(eventCount?.c || 0);
}
if (adminN > 1 || eventN > 0) {
logger.warn(
`Install-from-backup: refusing — install has ${adminN} admin(s) and ${eventN} event(s). `
+ 'This guard prevents accidental clobbering of production data. '
+ 'Override with INSTALL_FROM_BACKUP_FORCE=true if you really want to restore on top.'
);
return false;
}
// adminN === 1 is the "fresh install ran migration 001 and auto-created
// the default admin" case. That admin is throwaway — the restore will
// replace it with the backup's admin row. So we treat 1 admin + 0
// events as fresh.
return true;
} catch (err) {
logger.warn(`Install-from-backup: fresh-install check threw: ${err.message}. Assuming NOT fresh.`);
return false;
}
}
/**
* Public entry point — called from server.js after migrations and
* before startServer.
*
* @returns {Promise<{ ran: boolean, manifestPath?: string, error?: string }>}
*/
async function tryInstallFromBackup(db, logger) {
const log = logger || { info: () => {}, warn: () => {}, error: () => {} };
// The container's winston logger writes to `/app/logs/combined.log`
// by default and may not always tee to stdout, so admins running
// `docker logs picpeak-beta-backend` after a `compose up` would
// see no signal that a restore happened — flagged on PR #596
// review. We mirror the key trigger / start / end lines to
// console.log as well so the docker-logs surface tells the story
// without needing to exec into the container.
const announce = (msg) => {
try { console.log(`[install-from-backup] ${msg}`); } catch (_) { /* defensive */ }
};
const backupRoot = process.env.BACKUP_ROOT || '/backup';
if (!fs.existsSync(backupRoot)) {
return { ran: false };
}
const trigger = await findTrigger(backupRoot, log);
if (!trigger) {
return { ran: false };
}
log.info(`Install-from-backup: trigger file found at ${trigger.triggerPath}, target manifest ${trigger.manifestPath}`);
announce(`trigger file detected → ${trigger.manifestPath}`);
const forceOverride = process.env.INSTALL_FROM_BACKUP_FORCE === 'true';
const isFresh = await isDatabaseFresh(db, log);
if (!isFresh && !forceOverride) {
log.warn('Install-from-backup: skipping. Trigger file left in place so you can correct + retry.');
announce('skipping — install has existing data and INSTALL_FROM_BACKUP_FORCE is not set');
return { ran: false, error: 'Database not empty' };
}
log.info(`Install-from-backup: restoring from ${trigger.manifestPath}...`);
announce(`starting restore from ${trigger.manifestPath}`);
try {
const { restoreService } = require('./restoreService');
const result = await restoreService.restore({
source: 'local',
manifestPath: trigger.manifestPath,
restoreType: 'full',
// Force=true because the fresh-install admin auto-created by
// migration 001 trips the "1 active admin" warning — we WANT to
// override that warning, since replacing the throwaway admin
// with the backup's admin is exactly the goal.
force: true,
// SkipPreBackup=true because backing up an empty install is
// pointless. Saves a few seconds and reduces disk noise.
skipPreBackup: true,
operator: {
type: 'install-from-backup',
userId: null,
ip: null,
},
});
if (result?.success === false) {
throw new Error(result?.error || 'Restore service reported failure');
}
log.info(`Install-from-backup: restore completed successfully from ${trigger.manifestPath}`);
announce('restore completed successfully');
// Remove the trigger so the next boot doesn't redo it.
try {
fs.unlinkSync(trigger.triggerPath);
log.info(`Install-from-backup: removed trigger file ${trigger.triggerPath}`);
} catch (unlinkErr) {
log.warn(`Install-from-backup: could not remove trigger file (manual cleanup needed): ${unlinkErr.message}`);
}
return { ran: true, manifestPath: trigger.manifestPath };
} catch (err) {
log.error(`Install-from-backup: FAILED — ${err.message}`);
log.warn('Trigger file left in place so you can fix the input and retry by restarting the container.');
announce(`FAILED — ${err.message}. Trigger file left in place for retry.`);
return { ran: false, error: err.message };
}
}
module.exports = { tryInstallFromBackup };
@@ -0,0 +1,173 @@
/**
* Boot-time self-heal for restore-meta settings.
*
* **Why this exists**
*
* `restore_allow_force` gates whether the Restore wizard accepts a
* `force: true` payload. The flag exists to add admin friction
* before letting a restore override safety warnings (e.g. "1 active
* admin user — restoring would clobber the current install").
*
* In practice the friction lands at the worst possible moment: a
* fresh install (no app_settings row yet OR `restore_allow_force =
* false` by default) hits the wall on its very FIRST restore. The
* admin is mid disaster-recovery, panicked, and gets:
*
* "Force restore is not allowed by system settings"
*
* They then have to hand-craft SQL like
*
* INSERT INTO app_settings (setting_key, setting_value, ...)
* VALUES ('restore_allow_force', 'true', 'restore', NOW())
* ON CONFLICT ... SET setting_value = 'true';
*
* before they can recover their data. This isn't security — the
* admin who could run that SQL could also flip the setting via the
* UI. It's just a sharp edge that bites every new install once.
*
* Cure: seed the default ON at boot via `INSERT ... ON CONFLICT
* DO NOTHING`. New installs get force-allowed out of the box.
* Existing installs that have explicitly set the row (true OR
* false) are NOT overwritten — admin policy wins. Same pattern
* `_backupPathsBoot.js` uses for the canonical backup_paths rows.
*
* **Default-ON rationale (matches Stage A's principle)**
*
* Stage A defaulted inline DB dumps to ON because the cost of
* forgetting was data loss. By the same logic, `restore_allow_force`
* defaults ON because the cost of forgetting is being unable to
* recover from a disaster. Audit logging captures every forced
* restore so the accountability story stays intact.
*
* If/when the broader "exclude restore-meta settings from being
* overwritten by restore" follow-up lands (the second half of this
* chicken-and-egg), this self-heal becomes the safety net for
* fresh installs only — existing installs by that point have the
* row preserved across restores.
*/
const SEEDS = [
{
setting_key: 'restore_allow_force',
setting_value: 'true',
setting_type: 'restore',
rationale: 'Default ON so fresh installs can recover from disaster '
+ 'without a SQL incantation. Admins who want to require manual '
+ 'intervention can disable via the admin UI.',
},
];
/**
* Installs that ran migration 032 BEFORE the 2026-05-30 in-place edit
* have a `restore_allow_force` row with the deprecated `false` default
* (literal string `'false'` from `JSON.stringify(false)`). Per
* [[feedback_self_heal_pattern]] knex won't re-run the corrected
* migration on those installs, so we have to bump the row to `true`
* here ONCE at boot.
*
* The bump is guarded by a tracking row `restore_allow_force_auto_upgraded`
* so we don't fight an admin who explicitly disables force later:
* - Tracking row absent → bump if the value is the deprecated `'false'`
* - Tracking row present → never touch `restore_allow_force` again
*
* The bump applies ONLY when the existing value EXACTLY equals the old
* migration default. Any other value (`'true'`, admin-set anything,
* empty, null) is left alone — those reflect either the fixed
* migration's output or a deliberate admin choice.
*/
const DEPRECATED_DEFAULT_VALUE = 'false';
const AUTO_UPGRADE_FLAG_KEY = 'restore_allow_force_auto_upgraded';
let booted = false;
/**
* Seed the canonical restore-meta settings on fresh installs.
*
* @param {object} db knex instance
* @param {object} logger app logger (must expose .info / .warn)
* @returns {Promise<{ seeded: string[], upgraded: string[] }>}
*/
async function seedRestoreSettingsAtBoot(db, logger) {
const log = logger || { info: () => {}, warn: () => {} };
if (booted) return { seeded: [], upgraded: [] };
if (!(await db.schema.hasTable('app_settings'))) {
log.warn('app_settings table missing at boot — restore-settings self-heal skipped');
return { seeded: [], upgraded: [] };
}
const seeded = [];
const upgraded = [];
// Step 1: fresh-install seeding. Insert rows that don't exist at all.
for (const seed of SEEDS) {
try {
const existing = await db('app_settings')
.where('setting_key', seed.setting_key)
.first();
if (existing) continue;
await db('app_settings').insert({
setting_key: seed.setting_key,
setting_value: seed.setting_value,
setting_type: seed.setting_type,
updated_at: new Date(),
});
seeded.push(seed.setting_key);
log.info(`Seeded restore-meta setting ${seed.setting_key}=${seed.setting_value}`);
} catch (err) {
log.warn(`Failed to seed restore-meta setting ${seed.setting_key}: ${err.message}`);
}
}
// Step 2: one-time auto-upgrade for installs that ran the OLD
// migration 032 (which seeded restore_allow_force='false'). Bump
// to 'true' iff the value is still the deprecated default AND the
// auto-upgrade tracking flag hasn't already been set.
try {
const guard = await db('app_settings')
.where('setting_key', AUTO_UPGRADE_FLAG_KEY)
.first();
if (!guard) {
const row = await db('app_settings')
.where('setting_key', 'restore_allow_force')
.first();
if (row && row.setting_value === DEPRECATED_DEFAULT_VALUE) {
await db('app_settings')
.where('setting_key', 'restore_allow_force')
.update({
setting_value: 'true',
updated_at: new Date(),
});
upgraded.push('restore_allow_force');
log.info('Auto-upgraded restore_allow_force from deprecated migration-032 default \'false\' to \'true\' '
+ '(fresh-install disaster recovery now works without SQL incantation)');
}
// Always set the guard, even if no upgrade happened — prevents
// the bump from firing later if an admin sets the value to
// false on purpose.
await db('app_settings').insert({
setting_key: AUTO_UPGRADE_FLAG_KEY,
setting_value: 'true',
setting_type: 'restore',
updated_at: new Date(),
});
}
} catch (err) {
log.warn(`restore_allow_force auto-upgrade failed: ${err.message}`);
}
booted = true;
return { seeded, upgraded };
}
// Test-only: reset the module-level boot flag so jest can re-exercise
// the seeder against a fresh test DB inside a single worker.
function _resetBootForTests() {
booted = false;
}
module.exports = { seedRestoreSettingsAtBoot, _resetBootForTests, SEEDS };
@@ -0,0 +1,351 @@
/**
* Backup-coverage diagnostic — Stage C of the backup-hardening plan.
*
* **Why this is a separate service**
*
* Stage A (inline DB dump + fail-loud) and Stage B (config-driven
* walker via `backup_paths`) close the data-loss footgun, but they
* don't tell an admin *what* the next backup will actually cover.
* That's a separate question — and a particularly important one,
* because the whole reason Stage B exists is that the walker's
* subdirectory list used to silently fall behind reality every time
* a new feature dropped artefacts under STORAGE_PATH.
*
* This service answers two questions:
*
* 1. For every row in `backup_paths`, what will the next backup
* do with it? (scan / skip-via-feature-flag / skip-via-toggle /
* missing-on-disk)
* 2. What subdirectories EXIST under STORAGE_PATH but have NO row
* in `backup_paths` — i.e. drift the admin should know about
* before they lose data on a restore?
*
* Plus a top-level database-dump status block: are we configured
* for inline dump (default), or relying on the scheduled dump?
* When was the last successful dump? Is it stale?
*
* **What this service does NOT do**
*
* - Does not run the backup
* - Does not write anything (no DB mutations, no fs touches)
* - Does not auto-recover drift (it's a diagnostic — admins decide
* whether to add a `backup_paths` row, delete the orphan dir, etc.)
* - Does not walk file contents — only top-level directory entries
* under STORAGE_PATH are inspected (cheap; no recursion through
* potentially-millions of photos)
*
* Read-only. Returns a JSON report — same shape as
* backupIntegrityService.verifyDocumentArtefacts.
*/
const fs = require('fs').promises;
const path = require('path');
const { db } = require('../database/db');
const logger = require('../utils/logger');
const backupService = require('./backupService');
const STORAGE_ROOT = () => process.env.STORAGE_PATH || path.join(process.cwd(), 'storage');
/**
* Top-level subdirectories we expect to find under STORAGE_PATH but
* which are intentionally NOT in `backup_paths` — they're generated
* caches / runtime artefacts that the backup is supposed to skip.
* Listing them here keeps the drift detector from flagging them.
*
* `backups` — the destination directory the backup writer itself
* creates, plus the `database_backup_runs` dump files.
* Including it in the walker would create a recursive
* "backup of backups" feedback loop.
*
* `tmp` — short-lived scratch space (e.g. PDF render staging,
* S3 multipart uploads). Re-created on demand, never
* holds the only copy of anything.
*/
const EXPECTED_NON_BACKUP_DIRS = new Set([
'backups',
'tmp',
]);
/**
* How stale a database dump can be before we flag it. 26 hours so a
* daily scheduled dump is still considered "fresh" if it ran a few
* hours late.
*/
const DB_DUMP_STALE_AFTER_MS = 26 * 60 * 60 * 1000;
function parseSettingValue(raw) {
if (raw === null || raw === undefined) return null;
if (typeof raw !== 'string') return raw;
try { return JSON.parse(raw); } catch (_) {
if (raw === 'true') return true;
if (raw === 'false') return false;
const n = Number(raw);
return Number.isFinite(n) ? n : raw;
}
}
async function readBackupConfig() {
try {
const rows = await db('app_settings')
.where('setting_type', 'backup')
.select('setting_key', 'setting_value');
const cfg = {};
for (const row of rows) {
cfg[row.setting_key] = parseSettingValue(row.setting_value);
}
return cfg;
} catch (err) {
logger.warn(`backup-coverage: could not read backup config — ${err.message}`);
return {};
}
}
async function listConfiguredPaths() {
try {
if (!(await db.schema.hasTable('backup_paths'))) return null;
return await db('backup_paths')
.orderBy('display_order', 'asc')
.select('path', 'include_in_default', 'feature_flag', 'display_order', 'description');
} catch (err) {
logger.warn(`backup-coverage: could not read backup_paths — ${err.message}`);
return null;
}
}
async function listTopLevelStorageDirs() {
const root = STORAGE_ROOT();
try {
const entries = await fs.readdir(root, { withFileTypes: true });
return entries.filter((e) => e.isDirectory()).map((e) => e.name);
} catch (err) {
if (err.code === 'ENOENT') return [];
logger.warn(`backup-coverage: could not read STORAGE_PATH (${root}) — ${err.message}`);
return [];
}
}
async function statPath(absPath) {
try {
const st = await fs.stat(absPath);
return { exists: true, isDir: st.isDirectory() };
} catch (err) {
if (err.code === 'ENOENT') return { exists: false, isDir: false };
throw err;
}
}
/**
* Build the database-dump status block. Tells the admin whether
* "Run Backup Now" will inline-dump (default) or rely on the
* scheduled-dump path, plus how fresh the most recent dump is.
*/
async function buildDatabaseStatus(config) {
// normalizeBoolean(undefined) === false, so we have to gate on
// explicit-false the same way ensureDatabaseDumpForBackup does.
const inlineExplicitlyOff = config.backup_database_inline_dump !== undefined
&& config.backup_database_inline_dump !== null
&& config.backup_database_inline_dump === false;
const mode = inlineExplicitlyOff ? 'scheduled-only' : 'inline';
let recent = null;
try {
if (await db.schema.hasTable('database_backup_runs')) {
recent = await db('database_backup_runs')
.where('status', 'completed')
.orderBy('completed_at', 'desc')
.first();
}
} catch (err) {
logger.warn(`backup-coverage: could not read database_backup_runs — ${err.message}`);
}
const status = {
mode,
inlineDumpExplicitlyDisabled: inlineExplicitlyOff,
lastDumpAt: recent ? recent.completed_at : null,
lastDumpType: recent ? recent.backup_type : null,
lastDumpSizeBytes: recent ? Number(recent.file_size_bytes || 0) : 0,
lastDumpFilePath: recent ? recent.file_path : null,
lastDumpAgeMs: null,
lastDumpStale: null,
ok: null,
};
if (recent && recent.completed_at) {
const completedAt = recent.completed_at instanceof Date
? recent.completed_at
: new Date(recent.completed_at);
status.lastDumpAgeMs = Date.now() - completedAt.getTime();
status.lastDumpStale = status.lastDumpAgeMs > DB_DUMP_STALE_AFTER_MS;
}
// ok semantics:
// - inline mode: always ok=true (next backup will produce a fresh
// dump on demand, staleness is irrelevant)
// - scheduled-only: ok=true iff a recent non-stale dump exists,
// because the file-backup guard will fail-loud otherwise
if (mode === 'inline') {
status.ok = true;
} else {
status.ok = Boolean(recent && recent.file_path && status.lastDumpStale === false);
}
return status;
}
/**
* Per-path coverage:
* - configured + include_in_default + (no feature_flag OR flag truthy) → 'will-scan'
* - configured + include_in_default + flag falsey → 'skipped-by-feature-flag'
* - configured + include_in_default = false → 'skipped-by-toggle'
* - configured but missing on disk → 'missing-on-disk'
*
* Returns one entry per `backup_paths` row.
*/
async function buildConfiguredPathReport(configuredRows, config) {
const root = STORAGE_ROOT();
const result = [];
for (const row of configuredRows) {
const absPath = path.join(root, row.path);
const stat = await statPath(absPath);
const includedInDefault = Boolean(row.include_in_default);
let featureFlagValue = null;
if (row.feature_flag) {
const v = config[row.feature_flag];
featureFlagValue = v === undefined ? null : Boolean(v);
}
let coverage;
if (!includedInDefault) {
coverage = 'skipped-by-toggle';
} else if (row.feature_flag && featureFlagValue !== true) {
// null (unset) and explicit false both gate the path off — matches
// the walker's normalizeBoolean semantics
coverage = 'skipped-by-feature-flag';
} else if (!stat.exists) {
coverage = 'missing-on-disk';
} else {
coverage = 'will-scan';
}
result.push({
path: row.path,
includeInDefault: includedInDefault,
featureFlag: row.feature_flag || null,
featureFlagValue,
displayOrder: row.display_order,
description: row.description || null,
existsOnDisk: stat.exists,
coverage,
});
}
return result;
}
/**
* Drift detection: top-level subdirs under STORAGE_PATH that are not
* in `backup_paths` AND not in the `EXPECTED_NON_BACKUP_DIRS` allow-list.
*
* These are the directories that will be missed by "Run Backup Now"
* — either intentionally (a new feature drops cache files there and
* the admin doesn't want them backed up — they should add them to the
* allow-list) or accidentally (a feature shipped without a matching
* `backup_paths` row — the data-loss footgun this whole effort is
* designed to catch).
*/
function detectDrift(diskDirs, configuredPaths) {
// configured paths can be nested ('events/active'); we only diff the
// top-level segment ('events') because that's the granularity admins
// see in the storage tree. A path like 'events/active' implies the
// 'events' top-level is "known to the backup config".
const configuredTopLevels = new Set(
configuredPaths.map((p) => p.path.split('/')[0]),
);
return diskDirs
.filter((d) => !configuredTopLevels.has(d))
.filter((d) => !EXPECTED_NON_BACKUP_DIRS.has(d))
.sort();
}
/**
* Public entry point.
*
* @returns {Promise<{
* database: object,
* paths: Array<object>,
* drift: { unconfiguredOnDisk: string[], expectedNonBackupDirs: string[] },
* summary: object,
* generatedAt: string,
* }>}
*/
async function getCoverageReport() {
const config = await readBackupConfig();
const configuredRows = await listConfiguredPaths();
const diskDirs = await listTopLevelStorageDirs();
// Fallback when the table doesn't exist yet (migration 108 hasn't
// run for some reason). Mirrors the walker's LEGACY_BACKUP_PATHS
// contract — every other layer of this system uses the same
// belt-and-suspenders fallback.
const fallback = configuredRows === null;
const effectiveRows = configuredRows || [
{ path: 'events/active', include_in_default: true, feature_flag: null, display_order: 10, description: 'Legacy fallback (backup_paths missing)' },
{ path: 'events/archived', include_in_default: true, feature_flag: 'backup_include_archived', display_order: 20, description: 'Legacy fallback (backup_paths missing)' },
{ path: 'thumbnails', include_in_default: true, feature_flag: null, display_order: 30, description: 'Legacy fallback (backup_paths missing)' },
{ path: 'previews', include_in_default: true, feature_flag: null, display_order: 40, description: 'Legacy fallback (backup_paths missing)' },
{ path: 'heroes', include_in_default: true, feature_flag: null, display_order: 50, description: 'Legacy fallback (backup_paths missing)' },
{ path: 'uploads', include_in_default: true, feature_flag: null, display_order: 60, description: 'Legacy fallback (backup_paths missing)' },
{ path: 'business-docs', include_in_default: true, feature_flag: null, display_order: 70, description: 'Legacy fallback (backup_paths missing)' },
];
const [database, paths] = await Promise.all([
buildDatabaseStatus(config),
buildConfiguredPathReport(effectiveRows, config),
]);
const unconfiguredOnDisk = detectDrift(diskDirs, effectiveRows);
const summary = {
configuredCount: effectiveRows.length,
willScanCount: paths.filter((p) => p.coverage === 'will-scan').length,
skippedByToggleCount: paths.filter((p) => p.coverage === 'skipped-by-toggle').length,
skippedByFeatureFlagCount: paths.filter((p) => p.coverage === 'skipped-by-feature-flag').length,
missingOnDiskCount: paths.filter((p) => p.coverage === 'missing-on-disk').length,
driftCount: unconfiguredOnDisk.length,
tableMissingFallbackInUse: fallback,
databaseOk: database.ok,
// overall: green only when DB is ok AND there's at least one path
// that will actually be scanned AND no drift was found
overallOk: Boolean(
database.ok
&& paths.some((p) => p.coverage === 'will-scan')
&& unconfiguredOnDisk.length === 0,
),
};
return {
database,
paths,
drift: {
unconfiguredOnDisk,
expectedNonBackupDirs: Array.from(EXPECTED_NON_BACKUP_DIRS).sort(),
},
summary,
generatedAt: new Date().toISOString(),
};
}
module.exports = {
getCoverageReport,
// Exported for test introspection — the route doesn't use these.
EXPECTED_NON_BACKUP_DIRS,
DB_DUMP_STALE_AFTER_MS,
};
// Silence unused import lint warning — backupService is required so
// the module-graph cache primes (some tests jest.mock it before
// requiring this service).
void backupService;
@@ -0,0 +1,228 @@
/**
* Backup-integrity verifier — walks every CRM document-artefact path
* column and confirms (a) the file exists on disk, (b) when a SHA-256
* is stored, the file's actual bytes hash to the stored value.
*
* **Why this is a separate service**
*
* The audit trail captured at issue / sign time (signed_customer_ip,
* signed_by_customer_at, signed_pdf_sha256, signed_*_signature_path,
* issue_date, etc.) is worth exactly nothing on its own — what makes
* it legally meaningful is being able to produce the document the
* audit trail refers to. A backup that captures the DB but skips
* `storage/business-docs/` (the bug fixed in this same PR) leaves
* every `*_path` column a broken FK and every `*_sha256` column with
* nothing to verify against. This service is the diagnostic for
* exactly that drift — runs on demand, surfaces missing files +
* hash mismatches without making any changes.
*
* **Verification modes**
*
* - existence — file at `*_path` must exist on disk
* - sha256 — file at `*_path` must exist AND its sha256 must
* equal `*_sha256` column (when that column is set)
*
* Per-table coverage (lines reference migrations/core/107_crm_consolidated.js):
*
* quotes.pdf_path line 844 (existence)
* contracts.pdf_path line 1245 (existence + sha256 via contracts.pdf_sha256)
* contracts.signed_pdf_path line 1246 (existence + sha256 via contracts.signed_pdf_sha256)
* contracts.signed_customer_signature_path line 1269 (existence — drawn signatures, no hash column)
* contracts.signed_admin_signature_path line 1273 (existence — admin counter-signature drawing)
* invoices.pdf_path line 1026 (existence)
* invoices.imported_pdf_path line 1020 (existence — admin-uploaded scans)
*
* Wet uploads (`contracts.signed_pdf_is_wet_upload = true`) DO have a
* `signed_pdf_sha256` computed at upload time (contractService.js
* upload route), so they're hash-verified the same as system-rendered
* contracts — no special case here.
*
* **What this service does NOT do**
*
* - Does not write anything (no DB mutations, no fs touches)
* - Does not fail the request when a mismatch is found — the
* report shape carries the data, the caller decides what to do
* - Does not auto-trigger after restore (D2 decision: surface a
* CTA on the restore-completed screen instead)
* - Does not run on a schedule (D1 decision: on-demand v1; revisit
* once we have runtime data on large installs)
*/
const fs = require('fs');
const crypto = require('crypto');
const path = require('path');
const { db } = require('../database/db');
const logger = require('../utils/logger');
const STORAGE_ROOT = () => process.env.STORAGE_PATH || path.join(process.cwd(), 'storage');
/**
* Every column the verifier walks, declared once so the test suite
* and the service share a single source of truth. Order is the order
* the report lists rows in — table-major, then column-by-column.
*/
const CHECKS = [
{ table: 'quotes', pathColumn: 'pdf_path', shaColumn: null, scope: 'quote' },
{ table: 'contracts', pathColumn: 'pdf_path', shaColumn: 'pdf_sha256', scope: 'contract' },
{ table: 'contracts', pathColumn: 'signed_pdf_path', shaColumn: 'signed_pdf_sha256', scope: 'contract' },
{ table: 'contracts', pathColumn: 'signed_customer_signature_path', shaColumn: null, scope: 'contract-signature' },
{ table: 'contracts', pathColumn: 'signed_admin_signature_path', shaColumn: null, scope: 'contract-signature' },
{ table: 'invoices', pathColumn: 'pdf_path', shaColumn: null, scope: 'invoice' },
{ table: 'invoices', pathColumn: 'imported_pdf_path', shaColumn: null, scope: 'invoice' },
];
/** Stream-hash a file to sha256 hex without buffering the whole thing. */
function hashFile(absPath) {
return new Promise((resolve, reject) => {
const hash = crypto.createHash('sha256');
const stream = fs.createReadStream(absPath);
stream.on('error', reject);
stream.on('data', (chunk) => hash.update(chunk));
stream.on('end', () => resolve(hash.digest('hex')));
});
}
/**
* @param {object} [options]
* @param {string[]} [options.scope] Filter checks by scope tag:
* 'quote' | 'contract' | 'contract-signature' | 'invoice'.
* Defaults to all four (full scan).
* @returns {Promise<{
* scannedAt: string,
* scopes: string[],
* summary: {
* totalRows: number,
* verifiedOk: number,
* missingFiles: number,
* hashMismatches: number,
* existsButNoHash: number,
* },
* missing: Array<{ table, rowId, column, expectedPath }>,
* hashMismatches: Array<{ table, rowId, column, expectedPath, expectedSha, actualSha }>,
* existsButNoHash: Array<{ table, rowId, column, path }>,
* }>}
*
* `existsButNoHash` is the existence-only-verified bucket — the file
* was found but no `*_sha256` column exists for it (quote/invoice PDFs,
* signature PNGs). Surfaced separately so admins can distinguish
* "verified by hash" from "verified by existence only" — the latter
* is weaker evidence in a legal dispute.
*/
async function verifyDocumentArtefacts(options = {}) {
const scopes = Array.isArray(options.scope) && options.scope.length > 0
? options.scope.slice()
: Array.from(new Set(CHECKS.map((c) => c.scope)));
const checksToRun = CHECKS.filter((c) => scopes.includes(c.scope));
const storageRoot = STORAGE_ROOT();
const missing = [];
const hashMismatches = [];
const existsButNoHash = [];
let totalRows = 0;
let verifiedOk = 0;
for (const check of checksToRun) {
// Skip the check cleanly when the column or table doesn't exist
// on this install — keeps the verifier safe to run on partial
// migrations or installs that have features disabled.
if (!(await db.schema.hasTable(check.table))) continue;
if (!(await db.schema.hasColumn(check.table, check.pathColumn))) continue;
const select = ['id', check.pathColumn];
const hasHashColumn = check.shaColumn
&& (await db.schema.hasColumn(check.table, check.shaColumn));
if (hasHashColumn) select.push(check.shaColumn);
const rows = await db(check.table)
.whereNotNull(check.pathColumn)
.select(...select);
for (const row of rows) {
totalRows += 1;
const storedPath = row[check.pathColumn];
// Stored paths can be absolute (older rows) or relative-to-
// storage (newer rows). Normalize: resolve relative paths
// against STORAGE_PATH; absolute paths are used verbatim.
const absPath = path.isAbsolute(storedPath)
? storedPath
: path.join(storageRoot, storedPath);
let exists = false;
try {
exists = fs.existsSync(absPath);
} catch (_) { exists = false; }
if (!exists) {
missing.push({
table: check.table,
rowId: row.id,
column: check.pathColumn,
expectedPath: storedPath,
});
continue;
}
const expectedSha = hasHashColumn ? row[check.shaColumn] : null;
if (!expectedSha) {
// File exists but we have no hash to verify it against.
existsButNoHash.push({
table: check.table,
rowId: row.id,
column: check.pathColumn,
path: storedPath,
});
continue;
}
let actualSha;
try {
actualSha = await hashFile(absPath);
} catch (err) {
logger.warn(`backupIntegrity: failed to hash ${absPath}: ${err.message}`);
missing.push({
table: check.table,
rowId: row.id,
column: check.pathColumn,
expectedPath: storedPath,
});
continue;
}
if (actualSha !== expectedSha) {
hashMismatches.push({
table: check.table,
rowId: row.id,
column: check.pathColumn,
expectedPath: storedPath,
expectedSha,
actualSha,
});
continue;
}
verifiedOk += 1;
}
}
return {
scannedAt: new Date().toISOString(),
scopes,
summary: {
totalRows,
verifiedOk,
missingFiles: missing.length,
hashMismatches: hashMismatches.length,
existsButNoHash: existsButNoHash.length,
},
missing,
hashMismatches,
existsButNoHash,
};
}
module.exports = {
verifyDocumentArtefacts,
// Exported for tests; not part of the route API.
_internal: { CHECKS, hashFile },
};
+292 -20
View File
@@ -259,6 +259,73 @@ async function hasDatabaseChanged(sinceTime) {
}
}
/**
* Run an inline database dump (default ON) and then verify a usable dump
* is actually on disk before letting the file-backup proceed. Returns the
* verified `databaseInfo` so the caller can pass it straight into the
* manifest builder without re-querying.
*
* Why this lives here and not inline in `runBackupInternal`:
* - Encapsulates the "Run Backup Now must include DB" guarantee
* introduced when the silent files-only bug was discovered
* (2026-05-29 — admin lost CRM after `docker compose down -v`)
* - Lets the manifest path share the same `databaseInfo` object
* instead of doing a second `getDatabaseBackupInfo()` round-trip
* - Thrown errors bubble up to `runBackupInternal`'s catch, which
* marks the `backup_runs` row failed and queues the admin email
*
* Default-ON semantics: `backup_database_inline_dump` is only treated
* as disabled when explicitly set to false. `undefined` (the case on
* every existing install that predates the setting) falls through to
* the safe-default ON branch. `normalizeBoolean(undefined)` returns
* false, so a naive `!== false` check would silently disable the
* inline dump for every upgrading install.
*/
async function ensureDatabaseDumpForBackup(config) {
const inlineDumpExplicitlyOff = config.backup_database_inline_dump !== undefined
&& config.backup_database_inline_dump !== null
&& normalizeBoolean(config.backup_database_inline_dump) === false;
if (!inlineDumpExplicitlyOff) {
logger.info('Running inline database dump before file backup...');
const { databaseBackupService } = require('./databaseBackup');
const dumpResult = await databaseBackupService.backup({});
logger.info(`Inline database dump completed: ${dumpResult.path} ` +
`(${(dumpResult.size / 1024 / 1024).toFixed(2)} MB)`);
}
const databaseInfo = await service.getDatabaseBackupInfo();
if (!databaseInfo.backupFile) {
throw new Error(
'No database backup available to include in this file backup. ' +
'Either keep backup_database_inline_dump enabled (default) or configure ' +
'backup_database_schedule and let it run at least once first.'
);
}
let dumpStat;
try {
dumpStat = await fs.stat(databaseInfo.backupFile);
} catch (statErr) {
if (statErr.code === 'ENOENT') {
throw new Error(
`Database backup file at ${databaseInfo.backupFile} is missing from disk. ` +
'Refusing to proceed with file backup; configure backup_database_schedule or ' +
'keep backup_database_inline_dump enabled.'
);
}
throw statErr;
}
if (!dumpStat.size) {
throw new Error(
`Database backup file at ${databaseInfo.backupFile} is empty (0 bytes). ` +
'Refusing to proceed with file backup to avoid shipping a manifest with no DB content.'
);
}
return databaseInfo;
}
async function getDatabaseBackupInfoInternal() {
try {
const recent = await db('database_backup_runs')
@@ -350,28 +417,179 @@ async function scanDirectory(dirPath, fileList, basePath, excludePatterns = [])
}
}
async function getFilesToBackupInternal(includeArchived = true) {
/**
* Hard-coded fallback when `backup_paths` is missing/empty. Mirrors
* the canonical seed in migration 109 — kept here as defense in depth
* so the walker can never silently degrade to "no directories scanned"
* because of a seed problem.
*
* Order matches the legacy behavior of the inlined sequence this
* function used to contain.
*/
const LEGACY_BACKUP_PATHS = [
{ path: 'events/active', feature_flag: null },
{ path: 'events/archived', feature_flag: 'backup_include_archived' },
{ path: 'thumbnails', feature_flag: null },
{ path: 'previews', feature_flag: null },
{ path: 'heroes', feature_flag: null },
{ path: 'uploads', feature_flag: null },
{ path: 'business-docs', feature_flag: null },
];
/**
* Resolve the walker's target subdirectories from `backup_paths`.
*
* Layered fallback (defense in depth — no scenario where the walker
* silently scans nothing):
* 1. Read `backup_paths` rows where include_in_default = true,
* ordered by display_order.
* 2. If the table is missing OR returns zero rows, fall back to
* LEGACY_BACKUP_PATHS. Logged loudly so the admin sees it.
*
* Per-row gating: when `feature_flag` is set, the corresponding
* config key in `app_settings` must resolve truthy for that path to
* be included. Mirrors the historical `includeArchived` parameter,
* but now driven by data instead of a hard-coded boolean.
*
* @param {object} config resolved backup config (parseSettingValue'd).
* Used to evaluate feature_flag gates.
* @returns {Promise<Array<{ path: string, feature_flag: string|null }>>}
*/
async function resolveBackupPaths(config) {
let rows;
try {
if (!(await db.schema.hasTable('backup_paths'))) {
logger.warn('backup_paths table missing — falling back to LEGACY_BACKUP_PATHS');
rows = LEGACY_BACKUP_PATHS;
} else {
rows = await db('backup_paths')
.where('include_in_default', formatBoolean(true))
.orderBy('display_order', 'asc')
.select('path', 'feature_flag');
if (!rows.length) {
logger.warn('backup_paths has no rows with include_in_default=true — falling back to LEGACY_BACKUP_PATHS');
rows = LEGACY_BACKUP_PATHS;
}
}
} catch (err) {
logger.warn(`Failed to query backup_paths (${err.message}) — falling back to LEGACY_BACKUP_PATHS`);
rows = LEGACY_BACKUP_PATHS;
}
// Apply feature_flag gating. A row with feature_flag='backup_include_archived'
// requires config.backup_include_archived to be truthy (same semantics as
// the historical `includeArchived` parameter).
return rows.filter((row) => {
if (!row.feature_flag) return true;
const flagValue = config ? config[row.feature_flag] : undefined;
return normalizeBoolean(flagValue);
});
}
/**
* Bucket actually-backed-up files into their owning `backup_paths` row.
*
* Uses longest-prefix match — e.g. `events/active/E1/photo.jpg` matches
* `events/active` (length 13) rather than `events` (length 6, if that
* row existed). This handles the case where a future feature ships a
* nested backup_paths row that overlaps an existing one.
*
* @param {string[]} backedUpRelativePaths — paths that actually got
* copied / uploaded (post-incremental-filter). The exact list
* the destination implementation reports back.
* @param {Array<{relativePath, size}>} allFiles — the full file
* catalogue from the walker, used as a size lookup table.
* @returns {Promise<Record<string, { count: number, size: number }>>}
* keyed by `backup_paths.path` (e.g. 'events/active'). Paths
* with zero matches are omitted to keep the manifest compact.
*/
async function computePerPathStats(backedUpRelativePaths, allFiles) {
if (!backedUpRelativePaths || backedUpRelativePaths.length === 0) {
return {};
}
// Reuse the same source of truth the walker uses, so a row toggled
// off by include_in_default doesn't appear in the breakdown either.
let configuredPaths;
try {
if (await db.schema.hasTable('backup_paths')) {
configuredPaths = await db('backup_paths')
.where('include_in_default', formatBoolean(true))
.orderBy('display_order', 'asc')
.select('path');
}
} catch (err) {
logger.warn(`Could not load backup_paths for per-path stats — falling back to legacy: ${err.message}`);
}
if (!configuredPaths || configuredPaths.length === 0) {
configuredPaths = LEGACY_BACKUP_PATHS.map((p) => ({ path: p.path }));
}
// Longest-prefix-first so nested paths win over their parents.
const sortedPaths = configuredPaths
.map((row) => row.path)
.sort((a, b) => b.length - a.length);
// Size lookup. relativePath uses OS path separators in `allFiles`
// (whatever scanDirectory built); the backup_paths rows always use
// forward slashes. Normalize the lookup key once.
const sizeByPath = new Map();
for (const f of allFiles || []) {
sizeByPath.set(f.relativePath.split(path.sep).join('/'), f.size || 0);
}
const stats = {};
for (const relativePath of backedUpRelativePaths) {
const norm = relativePath.split(path.sep).join('/');
// Find the longest configured path that this file's relativePath starts with.
const match = sortedPaths.find(
(p) => norm === p || norm.startsWith(`${p}/`)
);
if (!match) continue; // file outside any configured path (shouldn't happen)
if (!stats[match]) stats[match] = { count: 0, size: 0 };
stats[match].count += 1;
stats[match].size += sizeByPath.get(norm) || 0;
}
return stats;
}
async function getFilesToBackupInternal(configOrIncludeArchived = true) {
const files = [];
const storagePath = getStoragePath();
await scanDirectory(path.join(storagePath, 'events/active'), files, storagePath);
if (normalizeBoolean(includeArchived)) {
await scanDirectory(path.join(storagePath, 'events/archived'), files, storagePath);
// Backward-compatible call signature:
// - Boolean `true|false` → legacy `includeArchived` argument. We
// forge a config-shaped object so the feature-flag gating
// resolves the same way the old code path did.
// - Object → full resolved backup config (preferred).
// - Anything else → treated as "include archived" (truthy).
let config;
if (typeof configOrIncludeArchived === 'object' && configOrIncludeArchived !== null) {
config = configOrIncludeArchived;
} else {
config = { backup_include_archived: normalizeBoolean(configOrIncludeArchived) };
}
await scanDirectory(path.join(storagePath, 'thumbnails'), files, storagePath);
// Lightbox preview tier (#492). Cheap to back up — typically a few
// hundred KB per photo — and saves admins the regenerate cycle on
// a restore. Tolerated when missing (admins who never enabled the
// feature won't have the folder; scanDirectory short-circuits on
// ENOENT cleanly).
await scanDirectory(path.join(storagePath, 'previews'), files, storagePath);
// Heroes too — same logic; admins who picked a hero photo for the
// gallery header had its 1920x1080 file generated and was missed
// by the original backup walk before this addition.
await scanDirectory(path.join(storagePath, 'heroes'), files, storagePath);
await scanDirectory(path.join(storagePath, 'uploads'), files, storagePath);
const targets = await resolveBackupPaths(config);
for (const target of targets) {
// CRM document estate is special-cased in the comment block below
// because it's the most expensive omission to recover from:
// - business-docs/quote/<year>/*.pdf
// - business-docs/contract/<year>/*.pdf (system-rendered + wet uploads)
// - business-docs/contract/signatures/<contract_id>/*.{png,jpg}
// (drawn signatures, forensic-preserved per Date.now() filename)
// - business-docs/invoice/<year>/*.pdf (issued invoices + Storno)
// - business-docs/invoice-imports/<year>/*.pdf (admin-imported
// historical invoices — irrecoverable if not backed up)
// Without this scan, the audit trail (signed_pdf_sha256, signed_*
// _ip, accepted_at, etc.) survives the restore but the documents
// those values refer to do not, leaving every CRM *_path column a
// broken FK. scanDirectory short-circuits on ENOENT so installs
// that never used CRM features won't error.
await scanDirectory(path.join(storagePath, target.path), files, storagePath);
}
return files;
}
@@ -799,7 +1017,17 @@ async function runBackupInternal(isManual = false) {
}).returning('id');
runId = insertResult[0]?.id || insertResult[0];
const files = await service.getFilesToBackup(config.backup_include_archived);
// Inline DB dump + fail-loud verification. The returned `databaseInfo`
// is reused at manifest-build time below so we don't pay a second
// `getDatabaseBackupInfo()` round-trip — see `ensureDatabaseDumpForBackup`
// for the full rationale.
const verifiedDatabaseInfo = await ensureDatabaseDumpForBackup(config);
// Pass the full config so the walker can evaluate any feature_flag
// gates declared in the backup_paths table (e.g. `events/archived`
// gated by `backup_include_archived`). Boolean signature is still
// supported for legacy callers and tests — see getFilesToBackupInternal.
const files = await service.getFilesToBackup(config);
logger.info(`Found ${files.length} files to check for backup`);
let result;
@@ -826,7 +1054,13 @@ async function runBackupInternal(isManual = false) {
const previousBackup = await getPreviousSuccessfulBackup(runId);
const manifestFiles = buildManifestFiles(result.backedUpFiles, files);
const databaseInfo = result.databaseInfo || await service.getDatabaseBackupInfo();
// `verifiedDatabaseInfo` came from ensureDatabaseDumpForBackup at the
// top of this run — reuse it so manifest building doesn't pay a
// second `getDatabaseBackupInfo()` round-trip. The
// `result.databaseInfo` branch is kept for destination implementations
// (S3, future destinations) that override the local info on the result
// object; falls back to the verified copy otherwise.
const databaseInfo = result.databaseInfo || verifiedDatabaseInfo;
const manifestOptions = {
backupType: previousBackup ? 'incremental' : 'full',
@@ -869,6 +1103,18 @@ async function runBackupInternal(isManual = false) {
logger.error('Failed to generate backup manifest:', error);
}
// Per-Stage-B-path stats — bucket the actually-backed-up files
// into their owning backup_paths row by longest-prefix match. Lets
// the Backup History detail pane render a true breakdown
// events/active: 142 files (3.2 GB)
// business-docs: 17 files (4.5 MB)
// thumbnails: 142 files (12.4 MB)
// instead of the legacy "Photos + Archives + Other" categorization
// that didn't reflect Stage B's data-driven walker. Falls back to
// an empty map if backup_paths is missing (defense in depth — the
// walker has the same fallback).
const perPath = await computePerPathStats(result.backedUpFiles, files);
await db('backup_runs')
.where('id', runId)
.update({
@@ -887,11 +1133,14 @@ async function runBackupInternal(isManual = false) {
total_files_checked: files.length,
average_file_size: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0,
destination: destinationType,
// Per-Stage-B-path breakdown — { [pathKey]: { count, size } }
per_path: perPath,
// Keep camelCase for backward compatibility
totalFilesChecked: files.length,
filesBackedUp: result.backedUpCount,
totalSize: result.backedUpSize,
averageFileSize: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0
averageFileSize: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0,
perPath
})
});
@@ -1071,11 +1320,34 @@ async function getBackupStatus(limit = 10) {
const lastRunWithManifest = lastRun ? { ...lastRun, manifestValid } : null;
// Separate "most recent attempt" from "most recent SUCCESS" so the
// dashboard widget can distinguish:
// - last attempt failed → red, "Last attempt failed at X"
// - last attempt running → blue spinner, "In progress since X"
// - never succeeded → critical, "No successful backup yet"
// - last attempt succeeded → green tick, "Last backup X ago"
// Previously the widget showed the most-recent row with a generic
// green tick regardless of status, so a crashed run from 5 minutes
// ago looked identical to a successful one. Same "silent failure
// not surfaced" class Stage A was designed to fight.
const lastSuccessful = runs.find(r => r.status === 'completed') || null;
// Detect zombie running rows (started >30min ago, never updated)
// — these are processes that died without writing a completed_at.
// Surface them so the admin can tell at a glance vs a live run.
const ZOMBIE_THRESHOLD_MS = 30 * 60 * 1000;
const zombieRuns = runs.filter(r =>
r.status === 'running'
&& r.started_at
&& (Date.now() - new Date(r.started_at).getTime()) > ZOMBIE_THRESHOLD_MS
);
return {
isRunning,
isHealthy: Boolean(lastRun && lastRun.status === 'completed'),
lastRun: lastRunWithManifest,
lastBackup: lastRunWithManifest, // Alias for frontend compatibility
lastSuccessfulBackup: lastSuccessful, // NEW — see comment above
zombieRuns, // NEW — running >30min, likely crashed
recentRuns: runs,
recentBackups: runs, // Alias for frontend compatibility
totalBackups: runs.filter(r => r.status === 'completed').length,
+34 -10
View File
@@ -206,12 +206,20 @@ class DatabaseBackupService {
'--format=plain',
'--encoding=UTF8'
];
// Add transaction support for consistency
if (!options.noTransaction) {
pgDumpOptions.push('--single-transaction');
}
// NOTE: do NOT add `--single-transaction` here. It looks like
// the right flag for "consistent snapshot" but it isn't a pg_dump
// option — it belongs to pg_restore / psql and pg_dump rejects it
// with `unrecognized option: single-transaction` (exit code 1).
// pg_dump already wraps the entire export in a single REPEATABLE
// READ snapshot automatically (since Postgres 9.x), so consistency
// is built in. If we ever need stricter cross-pg-cluster snapshot
// sharing, use `--snapshot=<id>` — but the typical inline-dump
// path doesn't need it. Bug went undetected until Stage A wired
// this code into the user-facing "Run Backup Now" path; prior
// callers (scheduled cron, dedicated admin DB-backup page) hit
// the same failure but on installs that had never exercised them.
// Add compression if not doing it separately
if (options.compress && !options.separateCompression) {
pgDumpOptions.push('--compress=6');
@@ -309,8 +317,23 @@ class DatabaseBackupService {
// Get current schema version
const schemaVersion = await this.getCurrentSchemaVersion();
// Create backup run record with version info
const [runId] = await db('database_backup_runs').insert({
// Create backup run record with version info.
//
// Insert shape divergence between SQLite + Postgres made the old
// `const [runId] = await db(...).insert({...})` form throw
// "(intermediate value) is not iterable" on Postgres installs:
//
// - SQLite-via-knex: insert() returns `[lastInsertId]` (array)
// - Postgres-via-knex: insert() without .returning() returns an
// empty object / row count — not iterable
//
// Bug went undetected until Stage A wired this method into the
// "Run Backup Now" inline-dump path — before that, only the
// scheduled-cron + dedicated-admin-page callers exercised it,
// and Ralf's install had never triggered either. Cure: same
// explicit `.returning('id')` + dual-shape coalesce pattern that
// `backupService.js:949` uses for its own `backup_runs` insert.
const insertResult = await db('database_backup_runs').insert({
started_at: startTime,
status: 'running',
backup_type: this.dbType,
@@ -324,8 +347,9 @@ class DatabaseBackupService {
node_env: process.env.NODE_ENV || 'production',
db_type: this.dbType
})
});
}).returning('id');
const runId = insertResult[0]?.id || insertResult[0];
backupRun = { id: runId };
// Get initial checksums
+477 -32
View File
@@ -39,6 +39,12 @@ class RestoreService {
this.currentProgress = null;
this.restoreLog = [];
this.preRestoreBackupPath = null;
// Snapshot of operator-meta settings (e.g. `restore_allow_force`)
// captured by performDatabaseRestore BEFORE the DROP DATABASE.
// Drained by restore() AFTER post-restore verification passes so
// the replay doesn't inflate the row-count check. Reset per run
// via beforeRestore() to keep state from leaking across calls.
this.preservedMetaSnapshot = [];
this.dbType = knexConfig.client === 'pg' ? 'postgresql' : 'sqlite';
this.tempDir = path.join(os.tmpdir(), 'picpeak-restore');
}
@@ -64,6 +70,7 @@ class RestoreService {
this.isRunning = true;
this.restoreLog = [];
this.preservedMetaSnapshot = []; // reset per run
const startTime = new Date();
let restoreRun = null;
@@ -154,9 +161,37 @@ class RestoreService {
this.log('warn', 'Pre-restore backup skipped at user request');
}
// Step 5: Download backup if from S3
// Step 5: Download backup if from S3, or resolve the local root.
//
// The wizard passes `options.source = 'local'` (the SOURCE TYPE
// string) — not a path. The old code assigned that string to
// `localBackupPath` verbatim and every downstream `path.join(...)`
// ended up with junk like `local/database/<file>.sql.gz`. Caused
// the disaster-recovery restore flow to fail with
// `Database backup file not found: local/database/...` even when
// the manifest recorded the correct absolute path AND the file
// existed at exactly that path on disk.
//
// Resolve `'local'` to the configured backup destination root by
// reading `backup_destination_path` from app_settings. That's the
// same root the file-backup walker writes to, so every relative
// `file.path` in the manifest resolves correctly via
// `path.join(localBackupPath, file.path)` further down.
let localBackupPath = options.source;
if (options.source.startsWith('s3://')) {
if (options.source === 'local') {
try {
const row = await db('app_settings')
.where('setting_key', 'backup_destination_path')
.first();
if (row?.setting_value) {
let parsed;
try { parsed = JSON.parse(row.setting_value); } catch (_) { parsed = row.setting_value; }
if (parsed) localBackupPath = parsed;
}
} catch (err) {
this.log('warn', `Could not resolve backup_destination_path: ${err.message}`);
}
} else if (options.source.startsWith('s3://')) {
this.updateProgress('Downloading backup from S3...');
localBackupPath = await this.downloadFromS3(options.source, manifest, options);
}
@@ -193,6 +228,89 @@ class RestoreService {
throw new Error(`Post-restore verification failed: ${verification.errors.join(', ')}`);
}
// Step 7b: Replay operator-meta settings AFTER verification.
//
// `performDatabaseRestore` stashed the pre-DROP snapshot of
// operator-meta keys on `this.preservedMetaSnapshot`. We drain
// it here, AFTER verification has already confirmed the
// restored DB matches the backup's row counts. Running this
// upsert sequence here instead of inside performDatabaseRestore
// (where it used to live) prevents the replay from inflating
// the post-restore row count and tripping the verification
// check — see the round-3 PR #596 notes for the full story.
//
// UPSERT by setting_key: if the backup had the same key with a
// different value, we overwrite; if the row doesn't exist in
// the backup, we insert. Either way the operator's pre-restore
// policy survives. SQLite branch leaves the snapshot empty so
// this block is a no-op there.
if (this.preservedMetaSnapshot && this.preservedMetaSnapshot.length > 0) {
try {
for (const row of this.preservedMetaSnapshot) {
await db('app_settings')
.insert({
setting_key: row.setting_key,
setting_value: row.setting_value,
setting_type: row.setting_type || 'restore',
updated_at: new Date(),
})
.onConflict('setting_key')
.merge({
setting_value: row.setting_value,
updated_at: new Date(),
});
}
this.log('info', `Replayed ${this.preservedMetaSnapshot.length} restore-meta setting(s) post-verification`);
} catch (err) {
this.log('warn', `Could not replay restore-meta settings (admin may need to re-set them): ${err.message}`);
}
}
// Step 7c: Apply any post-backup migrations to the restored DB.
//
// The backup carries the schema state of whatever migrations had
// been applied at backup time. If the running image is NEWER —
// because the admin upgraded picpeak between when the backup
// was taken and when they restored — the restored DB ends up
// mismatched against the running code: queries fail, new
// columns are missing, new tables don't exist.
//
// Previously the comment said "deferred to next container
// restart" — but that left the running process serving a
// mismatched schema until the operator manually restarted.
// Not acceptable per the "backup must restore completely even
// when new features have been added in the meantime" contract.
//
// Implementation: shell out to `npm run migrate:safe`, which is
// the EXACT script wait-for-db.sh runs on boot. Running it as a
// subprocess means no risk to our reinit'd pool (subprocess gets
// its own knex instance, destroys it on exit; our parent pool
// is untouched). Idempotent — migrations already applied are
// tracked in the restored `migrations` table and get skipped.
//
// Failure is non-fatal: the restore data itself is in place,
// and the next container restart's wait-for-db.sh will retry.
// Surfacing the error gives the operator a chance to investigate
// proactively rather than discovering it on the next 500 from
// a missing column.
try {
this.log('info', 'Applying post-restore migrations to restored database...');
this.updateProgress('Applying any post-backup migrations...');
const backendRoot = path.join(__dirname, '..', '..');
const { stderr } = await spawnAsync('npm', ['run', 'migrate:safe'], {
cwd: backendRoot,
env: { ...process.env },
});
if (stderr && stderr.trim()) {
this.log('info', `Post-restore migrate:safe stderr: ${stderr.slice(0, 500)}`);
}
this.log('info', 'Post-restore migrations applied');
} catch (migErr) {
this.log('warn',
`Post-restore migrate:safe failed — restore data is in place but the schema may lag the running image. ` +
`A container restart will retry via wait-for-db.sh. Error: ${migErr.message}`);
}
// Step 8: Clean up temporary files
if (localBackupPath !== options.source) {
await fs.unlink(localBackupPath).catch(err =>
@@ -208,6 +326,15 @@ class RestoreService {
await db('restore_runs').where('id', runId).update({
completed_at: endTime,
status: 'completed',
// Default for the column is `false`. Without this line, every
// SUCCESSFUL restore ends up with `status='completed',
// was_successful=false` — which the BackupDashboard "last
// successful restore" widget then filters out, and any future
// audit query that gates on was_successful misses the row
// entirely. Cosmetic but enough to mislead an operator
// scanning restore history. Catches Ralf 2026-06-01 + maintainer
// PR #596 review note about the cosmetic.
was_successful: true,
duration_seconds: durationSeconds,
pre_restore_backup_path: this.preRestoreBackupPath,
statistics: JSON.stringify({
@@ -242,12 +369,53 @@ class RestoreService {
} catch (error) {
this.log('error', 'Restore failed', { error: error.message, stack: error.stack });
// Update restore run record
// Always attempt rollback when a pre-restore backup exists.
// Historically rollback was only triggered when post-restore
// verification failed (inside the try block) — anything that
// threw earlier (path-resolution bugs, pg_restore failure, file
// copy errors) left the destination half-clobbered and forced
// the admin to do another reset-from-volume cycle before the
// next attempt could be honest. Fixing the rollback here closes
// the "every failed restore makes the next one worse" footgun.
let rollbackAttempted = false;
let rollbackSucceeded = false;
let rollbackError = null;
if (this.preRestoreBackupPath) {
rollbackAttempted = true;
try {
this.log('info', 'Attempting rollback from pre-restore safety backup', {
path: this.preRestoreBackupPath,
});
await this.attemptRollback(this.preRestoreBackupPath);
rollbackSucceeded = true;
this.log('info', 'Rollback completed');
} catch (rbErr) {
rollbackError = rbErr.message;
this.log('error', 'Rollback FAILED — install may be in a partial state',
{ error: rbErr.message, stack: rbErr.stack });
}
} else {
this.log('warn', 'No pre-restore backup available — cannot auto-rollback. ' +
'Destination may be in a partial state. Verify business-docs/ and the DB before retrying.');
}
// Update restore run record. We persist BOTH the original
// restore failure AND the rollback status so the admin can tell
// from a single SQL query which scenario they're in:
// - rollback succeeded → destination is back to pre-restore state, safe to retry
// - rollback failed → partial state, admin must inspect before next attempt
// - rollback skipped → user opted out via skipPreBackup; same as above
if (restoreRun) {
const failureMessage = rollbackAttempted
? (rollbackSucceeded
? `${error.message} (rolled back successfully to pre-restore state)`
: `${error.message} | ROLLBACK ALSO FAILED: ${rollbackError} — destination is in a partial state, inspect before retrying`)
: `${error.message} (no pre-restore backup available — destination may be partial)`;
await db('restore_runs').where('id', restoreRun.id).update({
completed_at: new Date(),
status: 'failed',
error_message: error.message,
error_message: failureMessage,
was_rollback_attempted: rollbackAttempted,
restore_log: JSON.stringify(this.restoreLog)
});
}
@@ -255,7 +423,9 @@ class RestoreService {
// Send failure notification
await this.sendRestoreNotification('failure', {
error: error.message,
restoreType: options.restoreType
restoreType: options.restoreType,
rollbackAttempted,
rollbackSucceeded,
});
throw error;
@@ -371,21 +541,32 @@ class RestoreService {
}
}
// Check if restoring would overwrite existing data
// Check if restoring would overwrite existing data.
//
// NOTE: pg-driver returns `count('* as count')` as a STRING (it
// serialises `bigint` to string to avoid JS precision loss for
// huge counts) — see PR #596 review for the `bigint`-as-string
// discussion. Both blocks below coerce to `Number` before
// comparing AND before interpolating into the warning text, so
// the count renders as `5` not `"5"` regardless of DB driver.
// Don't drop the `Number()` calls without also re-auditing the
// strict-equality call sites flagged in the same review.
if (options.restoreType === 'full' || options.restoreType === 'database') {
const eventCount = await db('events').count('* as count').first();
if (eventCount && eventCount.count > 0) {
validation.warnings.push(`Database contains ${eventCount.count} existing events that will be overwritten`);
const eventCountN = Number(eventCount?.count || 0);
if (eventCountN > 0) {
validation.warnings.push(`Database contains ${eventCountN} existing events that will be overwritten`);
}
}
// Check for active users
// Check for active users (same coercion contract as above).
const activeUsers = await db('admin_users')
.where('is_active', formatBoolean(true))
.count('* as count')
.first();
if (activeUsers && activeUsers.count > 0) {
validation.warnings.push(`There are ${activeUsers.count} active admin users`);
const activeUsersN = Number(activeUsers?.count || 0);
if (activeUsersN > 0) {
validation.warnings.push(`There are ${activeUsersN} active admin users`);
}
} catch (error) {
@@ -661,13 +842,55 @@ class RestoreService {
throw new Error('No database backup file found in manifest');
}
const dbBackupPath = path.join(backupPath, 'database', path.basename(dbBackupFile));
// Check if backup file exists
try {
await fs.access(dbBackupPath);
} catch (error) {
throw new Error(`Database backup file not found: ${dbBackupPath}`);
// Layered resolution for the database dump path:
//
// 1. Manifest stores the absolute path the dumper wrote to
// (e.g. `/backup/database/picpeak-db-postgresql-<ts>.sql.gz`).
// That's the truth — try it first.
// 2. Some older manifests store a path RELATIVE to the file-backup
// destination root (`database/<file>.sql.gz`). Reconstruct that
// way as a fallback.
// 3. Final fallback: `<backupPath>/database/<basename>`, the
// historical reconstruction used before this fix. Preserved so
// no existing valid path breaks.
//
// The original code used (3) exclusively, which meant the restore
// service ignored the absolute path the manifest recorded and
// looked under a synthetic `<backupPath>/database/<file>` root —
// which on Ralf's install became `local/database/<file>` because
// `backupPath` was the source type string, not a directory. Caused
// the canonical disaster-recovery flow to fail with
// `Database backup file not found: local/database/...sql.gz`
// even though the file existed at exactly the path the manifest
// recorded.
const candidates = [
// (1) Honour absolute paths recorded by the dumper.
path.isAbsolute(dbBackupFile) ? dbBackupFile : null,
// (2) Relative-to-backupPath as-stored (no basename munging).
path.join(backupPath, dbBackupFile),
// (3) Legacy reconstruct.
path.join(backupPath, 'database', path.basename(dbBackupFile)),
].filter(Boolean);
let dbBackupPath = null;
for (const candidate of candidates) {
try {
await fs.access(candidate);
dbBackupPath = candidate;
break;
} catch (_) {
// try next candidate
}
}
if (!dbBackupPath) {
throw new Error(
`Database backup file not found. Tried: ${candidates.join(', ')}. ` +
`Manifest recorded path: ${dbBackupFile}. ` +
`Hint: this usually means the manifest's database.backup_file path no longer ` +
`exists on disk (deleted? moved? volume not mounted?). Check ` +
`~/<your-compose-dir>/backup/database/ on the host.`
);
}
// Decompress if needed
@@ -679,6 +902,31 @@ class RestoreService {
restoreFile = decompressedPath;
}
// Snapshot of operator-meta keys captured BEFORE the DROP.
// Stashed onto `this.preservedMetaSnapshot` so the parent
// `restore()` method can drain + apply it AFTER post-restore
// verification passes. Order matters here:
//
// - PR #596 round 1: lifted the declaration above the
// SQLite/PG split to fix a ReferenceError when the replay
// was inline at the bottom of this method.
// - PR #596 round 3: moved the REPLAY itself out of here and
// into restore(), because the round-1 in-method replay ran
// BEFORE post-restore verification — which then counted the
// replayed row and flagged
// Table app_settings row count mismatch: expected 190, got 191
// as a verification failure even though both Stage A and
// the replay had succeeded. Verification now sees the
// as-restored DB (matches the backup exactly), replay layers
// on top after verification has signed off.
//
// SQLite branch leaves `preservedMetaSnapshot` empty — verification
// and replay both no-op for it, unchanged behaviour.
const PRESERVED_META_KEYS = [
'restore_allow_force',
'restore_allow_force_auto_upgraded',
];
try {
if (this.dbType === 'sqlite') {
// SQLite restore
@@ -715,24 +963,210 @@ class RestoreService {
// PostgreSQL restore
const { host, port, user, password, database } = knexConfig.connection;
const env = { ...process.env, PGPASSWORD: password };
// Snapshot operator-meta settings BEFORE the DROP so we can
// restore them after the psql load. These keys are about how
// the operator wants the install to behave (force-restore
// permission, auto-upgrade tracking), not user-facing state —
// they should NOT be overwritten by whatever values the backup
// happens to contain.
//
// Chicken-and-egg this closes: `restore_allow_force` defaults
// to true (post tonight's migration 032 edit), but every
// restore would overwrite it with whatever the backup carried.
// Admin sets it to true → restores → wakes up with the row
// back to whatever was in the backup. Two consecutive restores
// needed the SQL workaround again. With this snapshot/replay,
// the operator's policy persists across restores.
//
// PRESERVED_META_KEYS is declared above the SQLite/PG split
// (~L820). The snapshot READ happens here in the PG branch
// (must run before DROP), but is stashed on
// `this.preservedMetaSnapshot` for the parent `restore()`
// method to consume AFTER verification — see the round-3
// notes there.
try {
this.preservedMetaSnapshot = await db('app_settings')
.whereIn('setting_key', PRESERVED_META_KEYS)
.select('setting_key', 'setting_value', 'setting_type');
this.log('info', `Snapshotted ${this.preservedMetaSnapshot.length} restore-meta setting(s) for post-restore replay`, {
keys: this.preservedMetaSnapshot.map(r => r.setting_key),
});
} catch (err) {
this.log('warn', `Could not snapshot restore-meta settings (continuing): ${err.message}`);
}
// `psql` with no `-d` defaults to a database whose name matches
// the connecting user, NOT a maintenance DB. So on installs
// where the user's home DB doesn't exist (e.g. user=`picpeak`,
// target DB=`picpeak_prod`, no `picpeak` DB), the next two
// statements failed with:
// FATAL: database "picpeak" does not exist
// even though the actual target DB was alive and connectable.
//
// Fix: explicitly connect to `postgres` (the maintenance DB
// every PG cluster ships with) for the DROP/CREATE. We can't
// connect to the target DB itself anyway — DROP DATABASE
// refuses to run while a connection is open to it.
//
// Use `DB_CHECK_DB` env var as an override hook (matches the
// pattern wait-for-db.sh already exposes) for installs where
// the `postgres` DB is restricted to superusers.
const maintenanceDb = process.env.DB_CHECK_DB || 'postgres';
// The backend's own knex pool holds N active connections to
// the target database (default 5-25 per knexfile.js). PostgreSQL
// refuses DROP DATABASE while any session is connected:
// ERROR: database "X" is being accessed by other users
// DETAIL: There are N other sessions using the database.
// We have to evict those sessions ourselves before issuing the
// DROP. Two-step approach:
// 1. Close knex's own pool so we don't fight ourselves.
// 2. pg_terminate_backend() the rest (other server replicas,
// pg_stat_activity stragglers, leftover idle txns).
//
// After CREATE DATABASE, knex will lazily re-open the pool on
// the next query — handled by db.js's connection retry logic.
this.log('warn', 'Closing knex pool before dropping target database...');
try { await db.destroy(); } catch (poolErr) {
this.log('warn', `Pool destroy threw (continuing): ${poolErr.message}`);
}
this.log('warn', 'Terminating any remaining sessions on target database...', {
target: database,
});
// pg_terminate_backend takes a pid. Kill every session against
// the target DB except our own connection (which is to the
// maintenance DB anyway). Wrapped in `SELECT ... FROM ... WHERE`
// so we get one psql round-trip instead of N.
await spawnAsync('psql', [
'-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb,
'-c',
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity ` +
`WHERE datname = '${database.replace(/'/g, "''")}' AND pid <> pg_backend_pid()`,
], { env });
// Drop and recreate database (extremely dangerous!)
this.log('warn', 'Dropping and recreating PostgreSQL database...');
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-c', `DROP DATABASE IF EXISTS ${database}`], { env });
this.log('warn', 'Dropping and recreating PostgreSQL database...', {
target: database, via: maintenanceDb,
});
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-c', `CREATE DATABASE ${database}`], { env });
// WITH (FORCE) on Postgres 13+ kills any remaining connections
// atomically with the DROP. On older Postgres the FORCE option
// doesn't exist, so we fall back to plain DROP IF EXISTS — by
// which point pg_terminate_backend should have cleared the
// table. Try FORCE first, fall back to plain on syntax error.
try {
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb,
'-c', `DROP DATABASE IF EXISTS "${database}" WITH (FORCE)`], { env });
} catch (forceErr) {
// PG < 13: WITH (FORCE) is a syntax error. Plain DROP after
// our pg_terminate_backend pass should now succeed.
this.log('info', 'DROP DATABASE WITH (FORCE) not supported — falling back to plain DROP', {
error: forceErr.message,
});
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb,
'-c', `DROP DATABASE IF EXISTS "${database}"`], { env });
}
// Restore from backup
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb, '-c', `CREATE DATABASE "${database}"`], { env });
// Restore from backup — this one DOES connect to the target DB.
await spawnFromFile('psql', ['-h', host, '-p', String(port), '-U', user, '-d', database], restoreFile, { env });
// Re-sync every SERIAL / IDENTITY sequence in the public schema
// to MAX(id)+1 of its owning table. pg_dump emits setval()
// statements, but they don't always land cleanly when:
// - the dump has `--clean` (the setval may execute before
// the rebuilt rows, depending on dump ordering)
// - the in-process knex pool had a cached sequence value
// before db.destroy() (already mitigated, but defensive)
// - rows were inserted mid-restore (the pre-restore safety
// backup creates a database_backup_runs row before DROP)
// Result if skipped: every subsequent INSERT into a serial-id
// table fails with `duplicate key value violates unique
// constraint "<table>_pkey"`. Surfaced on Ralf's install as
// "A record with this value already exists" on every CRUD
// action AND `database_backup_runs_pkey` violation on the
// next Run Backup Now. Fix is a single DO block that walks
// pg_class + pg_attribute and setval()s each sequence to
// GREATEST(MAX(<col>), 1). Cheap (a few ms even on large
// schemas), safe (doesn't touch row data), idempotent.
this.log('info', 'Re-syncing PostgreSQL sequences to MAX(id) of each table...');
await spawnAsync('psql', [
'-h', host, '-p', String(port), '-U', user, '-d', database,
'-c',
`DO $$
DECLARE
r RECORD;
max_id BIGINT;
BEGIN
FOR r IN
SELECT n.nspname AS schema_name, t.relname AS table_name, a.attname AS column_name,
pg_get_serial_sequence(quote_ident(n.nspname) || '.' || quote_ident(t.relname), a.attname) AS seq_name
FROM pg_class t
JOIN pg_namespace n ON n.oid = t.relnamespace
JOIN pg_attribute a ON a.attrelid = t.oid
WHERE n.nspname = 'public'
AND t.relkind = 'r'
AND a.attnum > 0
AND NOT a.attisdropped
AND pg_get_serial_sequence(quote_ident(n.nspname) || '.' || quote_ident(t.relname), a.attname) IS NOT NULL
LOOP
EXECUTE format('SELECT COALESCE(MAX(%I), 0) FROM %I.%I', r.column_name, r.schema_name, r.table_name) INTO max_id;
EXECUTE format('SELECT setval(%L, %s, true)', r.seq_name, GREATEST(max_id, 1));
END LOOP;
END $$;`
], { env });
this.log('info', 'Sequence resync completed');
}
// Re-initialize database connection
const { db: newDb } = require('../database/db');
// Run migrations to ensure schema is up to date
this.log('info', 'Running database migrations...');
await newDb.migrate.latest();
// Re-initialize the in-process knex pool. The DROP/CREATE
// DATABASE pair above destroyed our connections and the recreated
// database has a different pg_database OID — any pooled
// connection from before would either be dead or pointed at a
// ghost. Without explicit reinit, every query in the process
// after restore returns `Error: Unable to acquire a connection`
// until the container is manually restarted (and admin sees
// "An error occurred" on the login screen even after the restore
// technically succeeded). reinitPool destroys + rebuilds the
// pool and probes the new one with `SELECT 1` so failures here
// surface immediately instead of polluting the next request.
const { reinitPool } = require('../database/db');
this.log('info', 'Re-initializing knex pool against the restored database...');
await reinitPool();
this.log('info', 'Knex pool re-initialized');
// NOTE: we deliberately do NOT call `db.migrate.latest()` here.
//
// The picpeak migrations directory contains `helpers.js` (a
// shared helper module, not a migration), plus `core/` and
// `legacy/` subdirectories. Knex's built-in migrator scans the
// top-level directory and rejects any file without `up`/`down`
// exports — so `db.migrate.latest()` throws
// Invalid migration: helpers.js must have both an up and down function
// every time it runs in this codebase. The production code path
// uses `npm run migrate:safe` (run-migrations-safe.js) which
// knows to skip helpers.js + walks core/ explicitly.
//
// The safe runner gets invoked AFTER verification in restore()
// (see step 7c) to apply any post-backup migrations to the
// restored DB. This closes the contract "backup must restore
// completely even when new features have been added in the
// meantime" — without this step, restoring an old backup on a
// newer image would leave the running process serving a
// mismatched schema until the next container restart.
this.log('info', 'Schema migrations deferred to restore() step 7c (npm run migrate:safe subprocess)');
// NOTE: operator-meta REPLAY does NOT happen here any more.
// PR #596 round 3: if the replay runs inside performDatabaseRestore,
// it lands BEFORE post-restore verification — and verification
// then counts the replayed row as a mismatch (e.g. "expected 190,
// got 191" because the fresh-install seeded
// `restore_allow_force_auto_upgraded` that wasn't in the backup).
// Replay is now drained by the parent `restore()` method AFTER
// verification passes. Snapshot lives on
// `this.preservedMetaSnapshot` for that drain.
return { success: true };
@@ -905,9 +1339,20 @@ class RestoreService {
for (const [table, expected] of Object.entries(manifest.database.row_counts)) {
try {
const result = await db(table).count('* as count').first();
if (result.count !== expected.rowCount) {
// pg-driver serialises `bigint` as string to preserve
// precision for huge counts, so `result.count` on PG is
// e.g. `"16"` while the manifest's `expected.rowCount`
// is the JS number `16`. Strict `!==` flagged every
// match as a mismatch on PG. Caught on PR #596 review:
// `Table activity_logs row count mismatch:
// expected 16, got 16`
// every table, all "matching". Coerce both sides to
// Number to compare reliably across SQLite (number) and
// PG (string).
const actual = Number(result.count);
if (actual !== expected.rowCount) {
verification.errors.push(
`Table ${table} row count mismatch: expected ${expected.rowCount}, got ${result.count}`
`Table ${table} row count mismatch: expected ${expected.rowCount}, got ${actual}`
);
}
} catch (error) {
+89 -21
View File
@@ -41,60 +41,128 @@ function spawnAsync(cmd, args = [], options = {}) {
/**
* Run a command and redirect stdout to a file (replaces shell `> file`).
*
* Historically this passed `fs.createWriteStream(outputPath)` directly as
* `stdio[1]` to `child_process.spawn`. That relied on Node auto-extracting
* the WriteStream's `.fd` but the stream opens async, so on a fast call
* `fd` is still `null` when `spawn()` reads it. Older Node releases would
* tolerate this; Node 22 throws synchronously with
* `The argument 'stdio' is invalid. Received WriteStream { fd: null, ... }`.
*
* Cure: use `stdio: ['ignore', 'pipe', 'pipe']` and wire the WriteStream
* up via the streams API (`stdout.pipe(outStream)`). Works on every Node
* version; also gives us a clean error bridge from both the WriteStream
* AND the child process to the promise, instead of the previous code's
* blind `outStream.destroy()` / `outStream.end()` calls that left stream
* errors uncaught (Node 22 process-fatal separate footgun this fixes
* by the same change).
*
* Used by:
* - databaseBackup.createPostgreSQLBackup (inline-dump path, the
* thing that just bit Ralf's install)
* - restoreService pre-restore safety snapshot (would have hit the
* same on next restore attempt)
*/
function spawnToFile(cmd, args, outputPath, options = {}) {
const fs = require('fs');
return new Promise((resolve, reject) => {
const outStream = fs.createWriteStream(outputPath);
let settled = false;
const settleReject = (err) => {
if (settled) return;
settled = true;
try { outStream.destroy(); } catch (_) { /* best effort */ }
reject(err);
};
const settleResolve = (value) => {
if (settled) return;
settled = true;
resolve(value);
};
// Bridge WriteStream errors (EACCES, ENOSPC, etc.) to the promise.
// Without this, an unhandled 'error' event on the stream is process-
// fatal on Node 22 and bypasses the caller's try/catch entirely —
// which is exactly the failure mode that crashed the picpeak
// backend container on its first inline-dump attempt.
outStream.on('error', settleReject);
const child = spawn(cmd, args, {
shell: false,
...options,
stdio: ['ignore', outStream, 'pipe']
stdio: ['ignore', 'pipe', 'pipe']
});
// Pipe stdout → file. The pipe call attaches its own 'error'
// handlers on both ends so a child-stdout failure also reaches us.
child.stdout.pipe(outStream);
const stderrChunks = [];
child.stderr.on('data', chunk => stderrChunks.push(chunk));
child.stderr.on('error', settleReject);
child.on('error', (err) => {
outStream.destroy();
reject(err);
});
child.on('error', settleReject);
child.on('close', (code) => {
outStream.end();
const stderr = Buffer.concat(stderrChunks).toString();
if (code !== 0) {
const err = new Error(`${cmd} exited with code ${code}: ${stderr}`);
err.code = code;
err.stderr = stderr;
return reject(err);
}
resolve({ stderr });
// Wait for the file write to flush before resolving — otherwise
// a fast 'close' could resolve while the WriteStream still has
// buffered bytes, producing a truncated dump.
outStream.end(() => {
const stderr = Buffer.concat(stderrChunks).toString();
if (code !== 0) {
const err = new Error(`${cmd} exited with code ${code}: ${stderr}`);
err.code = code;
err.stderr = stderr;
return settleReject(err);
}
settleResolve({ stderr });
});
});
});
}
/**
* Run a command and pipe a file into stdin (replaces shell `< file`).
*
* Same Node 22 stdio strictness applies as for `spawnToFile` above the
* ReadStream `fd` is null at spawn time. Use `stdio[0] = 'pipe'` and pipe
* the file stream into `child.stdin` via the streams API instead.
*/
function spawnFromFile(cmd, args, inputPath, options = {}) {
const fs = require('fs');
return new Promise((resolve, reject) => {
const inStream = fs.createReadStream(inputPath);
let settled = false;
const settleReject = (err) => {
if (settled) return;
settled = true;
try { inStream.destroy(); } catch (_) { /* best effort */ }
reject(err);
};
const settleResolve = (value) => {
if (settled) return;
settled = true;
resolve(value);
};
inStream.on('error', settleReject);
const child = spawn(cmd, args, {
shell: false,
...options,
stdio: [inStream, 'pipe', 'pipe']
stdio: ['pipe', 'pipe', 'pipe']
});
inStream.pipe(child.stdin);
const stdoutChunks = [];
const stderrChunks = [];
child.stdout.on('data', chunk => stdoutChunks.push(chunk));
child.stdout.on('error', settleReject);
child.stderr.on('data', chunk => stderrChunks.push(chunk));
child.stderr.on('error', settleReject);
child.stdin.on('error', settleReject);
child.on('error', (err) => {
inStream.destroy();
reject(err);
});
child.on('error', settleReject);
child.on('close', (code) => {
const stdout = Buffer.concat(stdoutChunks).toString();
const stderr = Buffer.concat(stderrChunks).toString();
@@ -103,9 +171,9 @@ function spawnFromFile(cmd, args, inputPath, options = {}) {
err.code = code;
err.stdout = stdout;
err.stderr = stderr;
return reject(err);
return settleReject(err);
}
resolve({ stdout, stderr });
settleResolve({ stdout, stderr });
});
});
}
+350
View File
@@ -0,0 +1,350 @@
---
title: Backup & Restore
description: How picpeak captures your install, where backups land, and how to recover from them — including full disaster recovery.
sidebar_position: 2
---
# Backup & Restore
picpeak's backup system captures your entire install — database, photos, CRM documents, gallery archives, and configuration — to a destination of your choice. Recovery happens through one of two paths depending on how badly things went wrong:
- **The install is alive** → use the **Restore wizard** in the admin UI to roll back to a chosen backup.
- **The install is gone** (host migration, `docker compose down -v`, drive replacement) → use the **install-from-backup** trigger file convention to rebuild in one boot, with no onboarding wizard and no temporary admin step.
This guide covers both.
## Table of contents
- [What gets backed up](#what-gets-backed-up)
- [Destinations](#destinations)
- [Inline DB dump](#inline-db-dump)
- [Custom backup paths](#custom-backup-paths)
- [The Coverage tab](#the-coverage-tab)
- [The Integrity tab](#the-integrity-tab)
- [Restoring on a live install](#restoring-on-a-live-install)
- [Disaster recovery (install from a backup)](#disaster-recovery-install-from-a-backup)
- [Backup History detail](#backup-history-detail)
- [Settings reference](#settings-reference)
- [Troubleshooting](#troubleshooting)
## What gets backed up
Every "Run Backup Now" (manual or scheduled) produces:
1. **A database dump** captured inline at the start of the run. Always included by default. picpeak refuses to ship a backup without a database dump unless the operator has explicitly opted out via the `backup_database_inline_dump` setting — see [Inline DB dump](#inline-db-dump) below.
2. **Files from a configurable list of paths**, declared in the `backup_paths` table:
| Path | Default | Notes |
| --- | --- | --- |
| `events/active` | ✓ | Live gallery photo originals |
| `events/archived` | gated by `backup_include_archived` | Long-term archive |
| `thumbnails` | ✓ | Generated thumbnails |
| `previews` | ✓ | Lightbox preview tier |
| `heroes` | ✓ | Gallery hero images |
| `uploads` | ✓ | Wet-signature contracts, imported invoices, etc. |
| `business-docs` | ✓ | CRM PDFs, signature artefacts, imported historical invoices |
Admins can add or remove rows from `backup_paths` to teach the walker about new feature directories — see [Custom backup paths](#custom-backup-paths).
3. **A manifest JSON** describing the run, written to `<destination>/manifests/backup-manifest-<id>.json`. The manifest carries the database dump path, the file inventory, checksums, and per-path counters.
## Destinations
picpeak supports three destination types, configured via **Backup → Configuration**:
- **Local** — files copied to a directory on the same host (default: `/backup` inside the container, which is typically a bind mount).
- **S3 / MinIO** — files uploaded via the S3 API. Supports custom endpoints (for MinIO, Backblaze B2, Wasabi, etc.).
- **rsync** — synchronised to a remote host over SSH.
Direct download from the admin UI is supported for local destinations; S3 backups can be retrieved via pre-signed URLs.
## Inline DB dump
Every "Run Backup Now" runs `pg_dump` (or `sqlite3 .backup`) inline before walking files. This guarantees the manifest's `database.backup_file` is always a fresh capture, never a stale reference to a previously-scheduled dump that may not exist.
If the inline dump fails (disk full, pg_dump crash, permission error), the run aborts and writes the error to `backup_runs.error_message`. The UI surfaces this as a failed run — no more silent files-only manifests.
**To opt out** (e.g. if you have a separately-orchestrated DB backup that you trust more):
```sql
INSERT INTO app_settings (setting_key, setting_value, setting_type, updated_at)
VALUES ('backup_database_inline_dump', 'false', 'backup', NOW())
ON CONFLICT (setting_key) DO UPDATE
SET setting_value = 'false', updated_at = NOW();
```
With inline-dump opted out, picpeak's fail-loud guard still applies: a file backup with no recent DB dump on file (within 26 hours) will fail rather than ship a files-only manifest.
## Custom backup paths
To add a new directory to the backup walker (e.g. you've shipped a feature that drops artefacts under `storage/my-feature/`):
```sql
INSERT INTO backup_paths (path, include_in_default, display_order, description, created_at, updated_at)
VALUES ('my-feature', true, 100, 'My new feature artefacts', NOW(), NOW());
```
Next "Run Backup Now" picks it up — no restart, no migration. Set `include_in_default = false` to temporarily disable a path without dropping the row.
The `feature_flag` column gates a path behind an app_settings boolean (matches how `events/archived` is gated by `backup_include_archived`). Useful when a backup path corresponds to an optional feature.
## The Coverage tab
**Backup → Coverage** answers "what will the next backup actually include?" without having to run it:
- **Database** — inline-dump mode + last dump timestamp + staleness check
- **Configured paths** — one row per `backup_paths` entry with its current coverage status (`will-scan` / `skipped-by-toggle` / `skipped-by-feature-flag` / `missing-on-disk`)
- **Drift detection** — flags top-level directories under `STORAGE_PATH` that exist on disk but have NO matching `backup_paths` row. This is the canary for "a feature shipped without a matching backup row" — the most common cause of silent data loss in pre-2026-05 picpeak.
The Coverage tab auto-fetches on open. If everything is green, your next backup will capture what you'd expect.
## The Integrity tab
**Backup → Integrity** verifies that every `*_path` column on quotes / contracts / invoices / signatures actually resolves to a file on disk, AND that files with a stored `*_sha256` still hash to the same value. Read-only, on-demand. Useful for:
- Post-restore validation
- Detecting bit-rot
- Auditing legal-evidence artefacts before a tax review or dispute
## Restoring on a live install
Use this when picpeak is running and you want to roll back to a specific backup point — e.g. recovering accidentally-deleted records, reverting a bad migration, or testing a restore drill.
**Backup → Restore** walks you through:
1. **Source** — Local, S3, or rsync
2. **Choose Backup** — manifests discovered from disk (works after a fresh install where `backup_runs` is empty) or from the database history
3. **Restore Options** — Full / Database only / Files only / Selective + Force + Skip Pre-Restore
4. **Review** — surfaces validation warnings before you commit
5. **Restore Progress** — real-time stream of the actual steps
Failures during restore trigger an automatic rollback from the pre-restore safety snapshot. The destination ends up either as the restored state OR as the original pre-restore state — never as a half-clobbered mix.
### Restoring an older backup on a newer image
picpeak's restore path is forward-compatible: a backup taken on an older version restores cleanly onto a newer image without any manual schema work. After loading the dump, the restore service runs the same `npm run migrate:safe` script that `wait-for-db.sh` uses on every container boot. Any migrations that have been added between the backup's snapshot and the current image are applied inline, against the freshly-restored DB, before the restore is reported as complete.
Net effect: even if `bugfix/cool-new-feature` shipped a migration that adds a `widgets` table and your backup predates that branch, after restore your install has the `widgets` table (empty), the right indexes, and any seed rows the migration emits. No "you'll need to restart the container once" footnote.
The same applies to the install-from-backup trigger — migrations land inside the restore boundary, so the moment the server prints `Server running on port 3000`, the schema matches the running image. Log in and use the install immediately.
## Disaster recovery (install from a backup)
For full DR after `docker compose down -v`, host migration, drive replacement, or moving an install between hosts. picpeak detects a trigger file on first boot and runs the restore before the admin UI surfaces. You open the browser, log in with your original credentials, and the install is fully populated.
### Prerequisites
- Your install's backup files must already be present in the `/backup` mount. They survive `docker compose down -v` because `/backup` is a bind mount, not a Docker-managed volume.
- The image must include the install-from-backup feature (shipped 2026-05-31 on `beta`; available in `main` after the next stable release).
- The backup must contain a database dump. The wizard cannot reconstruct your CRM data, customers, quotes, invoices, or admin users from a files-only backup. Confirm by inspecting any `backup-manifest-*.json` and checking that `database.backup_file` is non-null.
### How the trigger works
On every container start, picpeak's boot sequence checks for a trigger file in the root of the `/backup` mount. If found AND the destination database is empty, the restore runs automatically. After a successful restore the trigger file is deleted so the next boot doesn't redo the work. On failure the trigger file is preserved — fix the input and restart the container to retry.
The trigger file is named **`RESTORE_ON_INSTALL`** (no extension) or **`RESTORE_ON_INSTALL.txt`** — either is accepted.
### Two trigger flavors
#### Auto-pick the newest backup
Create an empty trigger file:
```sh
touch /path/to/backup/RESTORE_ON_INSTALL
```
The boot hook will scan `/backup/manifests/` for files matching `backup-manifest-*.json` (or `.yaml`) and pick the one with the most recent modification time. Best for the common DR case where you simply want the latest snapshot.
#### Use a specific backup
Write the path of the manifest you want — either relative to the `/backup` mount root or an absolute path — into the trigger file:
```sh
# Relative path (recommended)
echo "manifests/backup-manifest-backup-20260530-190617-e9be97b3.json" \
> /path/to/backup/RESTORE_ON_INSTALL
# Or absolute path inside the container
echo "/backup/manifests/backup-manifest-backup-20260530-190617-e9be97b3.json" \
> /path/to/backup/RESTORE_ON_INSTALL
```
Use this when you need to restore an older backup (e.g. rolling back a data corruption that happened after the most recent backup ran).
### Full DR walkthrough
```sh
# 1. Snapshot the backup outside the compose directory (belt + suspenders).
# This is a docker-compose-down-v-proof copy in case anything goes wrong.
SNAPSHOT=~/picpeak-snapshots/$(date +%Y%m%d-%H%M%S)
mkdir -p "$SNAPSHOT" && cp -av /path/to/picpeak/backup/. "$SNAPSHOT/"
# 2. Verify the backup is restorable. database.backup_file must be non-null.
LATEST=$(ls -t /path/to/picpeak/backup/manifests/*.json | head -1)
docker compose exec backend cat "/backup/manifests/$(basename $LATEST)" \
| python3 -c "import json,sys; m=json.load(sys.stdin); \
print('DB included:', bool(m.get('database',{}).get('backup_file')))"
# 3. Drop the trigger file. Two variants — pick one:
# (a) auto-pick newest
touch /path/to/picpeak/backup/RESTORE_ON_INSTALL
# (b) specific manifest
echo "manifests/backup-manifest-backup-20260530-190617-e9be97b3.json" \
> /path/to/picpeak/backup/RESTORE_ON_INSTALL
# 4. Boot.
docker compose down -v
docker compose up -d
docker compose logs -f backend --tail=100
```
When the boot log shows `Install-from-backup: restore completed successfully` followed by `Server running on port 3000`, the install is ready. Open the admin UI and log in with your original (pre-disaster) credentials.
### Safety gates
Three layers prevent accidental data loss:
1. **The trigger file must exist.** No auto-magic — an admin explicitly drops the file to signal intent.
2. **The destination database must be empty.** If the database contains any events, the install-from-backup hook refuses to run. The fresh-install default admin (auto-created by migration 001) is treated as throwaway and replaced by the backup's admin row, so a single admin user does not block the restore.
3. **Failed restores roll back to the pre-restore state.** If anything fails after the DROP DATABASE step, picpeak's automatic rollback restores the destination from the pre-restore safety backup it took before starting.
#### Override for advanced cases
If you have a populated install you intentionally want to clobber (dev rebuilds, staging refresh, etc.):
```yaml
# In docker-compose.yml
backend:
environment:
- INSTALL_FROM_BACKUP_FORCE=true
```
Or via the CLI:
```sh
INSTALL_FROM_BACKUP_FORCE=true docker compose up -d backend
```
With this set, gate #2 is skipped and the restore proceeds even with existing data. Gates #1 (trigger file presence) and #3 (rollback on failure) still apply.
### Verifying DR success
After the boot log shows `Install-from-backup: restore completed successfully`:
```sh
# Trigger should be gone (deleted on successful restore)
ls /path/to/picpeak/backup/RESTORE_ON_INSTALL 2>/dev/null \
|| echo "Trigger cleaned up — restore succeeded."
# Inspect the restore_runs row
docker compose exec -T postgres psql -U picpeak -d picpeak_prod -c \
"SELECT id, status, was_successful, was_rollback_attempted FROM restore_runs ORDER BY id DESC LIMIT 1;"
# Confirm data is back
docker compose exec -T postgres psql -U picpeak -d picpeak_prod -c "
SELECT 'admin' AS t, COUNT(*) FROM admin_users
UNION ALL SELECT 'events', COUNT(*) FROM events
UNION ALL SELECT 'invoices', COUNT(*) FROM invoices
UNION ALL SELECT 'app_settings', COUNT(*) FROM app_settings;"
```
Then open the admin login and use your **original** pre-disaster credentials.
## Backup History detail
Each row in **Backup → Backup History** expands to show:
- **Database** — whether the dump was included
- **Per-path file counts** — one row per `backup_paths` entry that contributed files, with count + total size. e.g.:
```
events/active 142 (3.2 GB)
business-docs 17 (4.5 MB)
thumbnails 142 (12.4 MB)
```
- **Total files** + total bytes
- Error message if the run failed
This breakdown reflects Stage B's data-driven walker, so admins can see at a glance which paths contributed how much.
## Settings reference
Backup-related settings live in `app_settings` with `setting_type = 'backup'` or `setting_type = 'restore'`:
| Setting | Default | Notes |
| --- | --- | --- |
| `backup_enabled` | true | Master scheduler switch |
| `backup_destination_type` | `'local'` | `'local'`, `'s3'`, or `'rsync'` |
| `backup_destination_path` | `'/backup'` | Local destination |
| `backup_database_inline_dump` | true | Inline DB dump on every run |
| `backup_include_archived` | false | Gate `events/archived` |
| `backup_incremental` | true | Skip unchanged files (checksum-tracked) |
| `restore_allow_force` | true | Permit Force Restore via the wizard |
| `restore_require_pre_backup` | true | Take a pre-restore safety snapshot |
| `restore_verify_checksums` | true | Verify file checksums after restore |
| `restore_email_on_completion` | true | Notify admin when restore finishes |
| `restore_retention_days` | 30 | How long pre-restore snapshots survive |
Most settings are exposed in the **Backup → Configuration** tab. Less common ones can be set via SQL.
## Troubleshooting
### "Run Backup Now" fails with `No database backup available`
The inline DB dump was disabled AND no recent scheduled dump exists. Either re-enable inline dumps (set `backup_database_inline_dump = 'true'`) or configure a scheduled DB dump that completes within the staleness window (default 26h).
### Backup History row says "completed" but shows 0 files
This is the legacy of a pre-2026-05 install where the walker was hard-coded and missed paths. After upgrading, the new walker captures everything per `backup_paths`. The 0-file row is historical — new backups will count correctly.
### Restore wizard shows "No backups found"
The wizard's disk discovery looks in `backup_destination_path` + its `manifests/` subdirectory. If you moved manifests elsewhere or your bind mount points at a different host directory than expected, the discovery won't find them. Check `backup_destination_path` in the Configuration tab matches reality.
### Restore completes but login fails
Caused by the pre-2026-05 dead-pool bug — fixed in the current image. If you're on a stale image and still see this, restart the backend container once:
```sh
docker compose restart backend
```
Then try logging in again.
### Install-from-backup: boot log shows no `Install-from-backup:` lines
Check that the trigger file is actually visible from inside the container:
```sh
docker compose exec backend ls -la /backup/RESTORE_ON_INSTALL
docker compose exec backend cat /backup/RESTORE_ON_INSTALL
```
If `ls` reports the file but the hook didn't run, the most likely cause is that the trigger pointed at a manifest that doesn't exist. The hook silently returns when the manifest path can't be resolved. Verify the path inside the file matches an actual manifest:
```sh
docker compose exec backend ls -la /backup/manifests/
```
### Install-from-backup: restore fails and leaves the trigger file in place
This is the intentional behavior — fix the input, then restart the container to retry. The boot log will surface the underlying error (e.g. corrupt manifest, missing database dump file, validator warnings without the force override).
Common failure modes:
- **Backup is files-only** (`database.backup_file = null` in the manifest). Pick a different backup or proceed with caution via the admin wizard, knowing you will restore files only.
- **Manifest path mismatch.** The path in the trigger file points at a manifest that doesn't exist. Either fix the path or use the empty-file auto-pick variant.
- **Existing data** without the force override. Either start from a truly empty install (`docker compose down -v`) or set `INSTALL_FROM_BACKUP_FORCE=true`.
### How do I disable install-from-backup entirely?
Don't create a `RESTORE_ON_INSTALL` file. Without the trigger, the hook is a no-op on every boot. There is no separate "off switch" because the feature is opt-in by design.
## See also
- [Deployment](/deployment) — Docker, environment variables, volumes
- [Admin Settings](/guides/admin-settings) — Configuration tab walkthrough
@@ -0,0 +1,433 @@
import React from 'react';
import { useTranslation } from 'react-i18next';
import {
ShieldCheck,
ShieldAlert,
Database,
FolderTree,
AlertTriangle,
CheckCircle2,
XCircle,
EyeOff,
Clock,
RefreshCw,
Loader2,
} from 'lucide-react';
import { useQuery } from '@tanstack/react-query';
// Locale-aware formatters per [[feedback_respect_general_format_settings]].
import { useLocalizedDate } from '../../hooks/useLocalizedDate';
import { Card, Button } from '../common';
import {
adminService,
BackupCoverageReport,
BackupPathCoverage,
} from '../../services/admin.service';
/**
* BackupCoverageCard Stage C of the backup-hardening plan.
*
* Tells the admin what the next "Run Backup Now" will actually do:
*
* - Database: inline-dump or scheduled, last dump age, staleness
* - Configured paths: per-row coverage (will-scan / skipped by
* toggle / skipped by feature flag / missing on disk)
* - Drift: top-level subdirs under STORAGE_PATH that have no
* `backup_paths` row (the "feature shipped without a backup row"
* footgun this whole effort is designed to catch)
*
* Auto-fetches on mount unlike the integrity verifier, this is
* a cheap query (no recursion) so admins should always see the
* current state when they open the tab.
*/
export const BackupCoverageCard: React.FC = () => {
const { t } = useTranslation();
const { formatDateTime } = useLocalizedDate();
const { data, isLoading, isError, error, refetch, isFetching } = useQuery({
queryKey: ['backup-coverage'],
queryFn: () => adminService.getBackupCoverage(),
// The report changes only when (a) backup_paths is edited or
// (b) a new scheduled dump completes. Stale time of 30s keeps
// the UI snappy without hammering the endpoint.
staleTime: 30_000,
});
return (
<Card className="p-6">
<Header report={data} loading={isLoading} onRefresh={() => refetch()} refreshing={isFetching} />
{isError && (
<ErrorBanner message={(error as Error)?.message ?? 'unknown error'} />
)}
{data && (
<>
{data.summary.tableMissingFallbackInUse && (
<FallbackWarning />
)}
<SectionGrid>
<DatabaseStatusCard database={data.database} />
<SummaryCard summary={data.summary} />
</SectionGrid>
<PathsTable paths={data.paths} />
<DriftSection drift={data.drift} />
<p className="text-xs text-neutral-500 dark:text-neutral-400 mt-4">
{t('backup.coverage.generatedAt', 'Coverage generated: {{when}}', {
when: formatDateTime(new Date(data.generatedAt)),
})}
</p>
</>
)}
</Card>
);
};
const Header: React.FC<{
report: BackupCoverageReport | undefined;
loading: boolean;
onRefresh: () => void;
refreshing: boolean;
}> = ({ report, loading, onRefresh, refreshing }) => {
const { t } = useTranslation();
const healthy = report?.summary.overallOk;
return (
<div className="flex items-start justify-between mb-4">
<div>
<div className="flex items-center gap-2 mb-1">
{loading || refreshing ? (
<Loader2 className="w-5 h-5 text-neutral-400 animate-spin" />
) : healthy ? (
<ShieldCheck className="w-5 h-5 text-green-600 dark:text-green-400" />
) : report ? (
<ShieldAlert className="w-5 h-5 text-amber-600 dark:text-amber-400" />
) : (
<ShieldCheck className="w-5 h-5 text-neutral-400" />
)}
<h3 className="text-lg font-semibold text-neutral-900 dark:text-neutral-100">
{t('backup.coverage.title', 'Backup coverage')}
</h3>
</div>
<p className="text-sm text-neutral-600 dark:text-neutral-400 max-w-2xl">
{t(
'backup.coverage.description',
'Shows what the next backup will include, skip, or silently miss. The database block confirms the dump strategy. The "drift" section flags subdirectories that exist on disk but are not in the backup configuration — usually a sign that a new feature shipped without a matching backup_paths row.',
)}
</p>
</div>
<Button
variant="ghost"
onClick={onRefresh}
disabled={loading || refreshing}
leftIcon={
refreshing
? <Loader2 className="w-4 h-4 animate-spin" />
: <RefreshCw className="w-4 h-4" />
}
>
{t('backup.coverage.refresh', 'Refresh')}
</Button>
</div>
);
};
const ErrorBanner: React.FC<{ message: string }> = ({ message }) => {
const { t } = useTranslation();
return (
<div className="mb-4 p-3 rounded-lg bg-red-50 dark:bg-red-900/30 text-sm text-red-700 dark:text-red-300">
{t('backup.coverage.error', 'Could not load coverage report: {{message}}', { message })}
</div>
);
};
const FallbackWarning: React.FC = () => {
const { t } = useTranslation();
return (
<div className="mb-4 p-3 rounded-lg bg-amber-50 dark:bg-amber-900/30 text-sm text-amber-800 dark:text-amber-200 flex items-start gap-2">
<AlertTriangle className="w-4 h-4 flex-shrink-0 mt-0.5" />
<span>
{t(
'backup.coverage.fallbackInUse',
'The backup_paths table is missing. The walker is using its legacy hard-coded fallback. Migration 108 may not have run — check server logs and re-run migrations.',
)}
</span>
</div>
);
};
const SectionGrid: React.FC<{ children: React.ReactNode }> = ({ children }) => (
<div className="grid grid-cols-1 md:grid-cols-2 gap-3 mb-4">{children}</div>
);
const DatabaseStatusCard: React.FC<{
database: BackupCoverageReport['database'];
}> = ({ database }) => {
const { t } = useTranslation();
const { formatDateTime } = useLocalizedDate();
const isInline = database.mode === 'inline';
const tone: Tone = database.ok ? 'green' : 'red';
const dumpAge = database.lastDumpAgeMs !== null
? formatAge(database.lastDumpAgeMs)
: null;
return (
<div className={`rounded-lg p-4 ${TONE_BG[tone]}`}>
<div className="flex items-center gap-2 mb-2">
<Database className="w-4 h-4" />
<h4 className="font-semibold text-sm uppercase tracking-wide">
{t('backup.coverage.database.title', 'Database')}
</h4>
{database.ok ? (
<CheckCircle2 className="w-4 h-4 ml-auto" />
) : (
<XCircle className="w-4 h-4 ml-auto" />
)}
</div>
<dl className="space-y-1 text-sm">
<Row
label={t('backup.coverage.database.mode', 'Mode')}
value={isInline
? t('backup.coverage.database.modeInline', 'Inline dump on every backup')
: t('backup.coverage.database.modeScheduled', 'Scheduled-only (inline opted out)')}
/>
{database.lastDumpAt ? (
<>
<Row
label={t('backup.coverage.database.lastDump', 'Last dump')}
value={`${formatDateTime(new Date(database.lastDumpAt))}${
dumpAge ? ` (${dumpAge})` : ''
}`}
/>
<Row
label={t('backup.coverage.database.lastDumpSize', 'Size')}
value={formatBytes(database.lastDumpSizeBytes)}
/>
</>
) : (
<Row
label={t('backup.coverage.database.lastDump', 'Last dump')}
value={t('backup.coverage.database.noDump', 'No dump on file yet')}
/>
)}
{database.lastDumpStale && (
<Row
label={t('backup.coverage.database.staleLabel', 'Status')}
value={t('backup.coverage.database.stale', 'Stale — older than 26h')}
icon={<Clock className="w-3.5 h-3.5" />}
/>
)}
</dl>
</div>
);
};
const SummaryCard: React.FC<{
summary: BackupCoverageReport['summary'];
}> = ({ summary }) => {
const { t } = useTranslation();
const tone: Tone = summary.overallOk
? 'green'
: summary.driftCount > 0 || !summary.databaseOk
? 'amber'
: 'neutral';
return (
<div className={`rounded-lg p-4 ${TONE_BG[tone]}`}>
<div className="flex items-center gap-2 mb-2">
<FolderTree className="w-4 h-4" />
<h4 className="font-semibold text-sm uppercase tracking-wide">
{t('backup.coverage.summary.title', 'Summary')}
</h4>
</div>
<dl className="space-y-1 text-sm">
<Row
label={t('backup.coverage.summary.willScan', 'Will scan')}
value={`${summary.willScanCount} / ${summary.configuredCount}`}
/>
{summary.skippedByToggleCount > 0 && (
<Row
label={t('backup.coverage.summary.skippedByToggle', 'Skipped (toggle off)')}
value={String(summary.skippedByToggleCount)}
/>
)}
{summary.skippedByFeatureFlagCount > 0 && (
<Row
label={t('backup.coverage.summary.skippedByFlag', 'Skipped (feature flag)')}
value={String(summary.skippedByFeatureFlagCount)}
/>
)}
{summary.missingOnDiskCount > 0 && (
<Row
label={t('backup.coverage.summary.missingOnDisk', 'Missing on disk')}
value={String(summary.missingOnDiskCount)}
/>
)}
<Row
label={t('backup.coverage.summary.drift', 'Unconfigured on disk (drift)')}
value={String(summary.driftCount)}
/>
</dl>
</div>
);
};
const PathsTable: React.FC<{ paths: BackupCoverageReport['paths'] }> = ({ paths }) => {
const { t } = useTranslation();
return (
<div className="border border-neutral-200 dark:border-neutral-700 rounded-lg overflow-hidden">
<div className="px-3 py-2 bg-neutral-50 dark:bg-neutral-800/50 border-b border-neutral-200 dark:border-neutral-700">
<h4 className="text-sm font-semibold text-neutral-900 dark:text-neutral-100">
{t('backup.coverage.paths.heading', 'Configured paths')}
</h4>
</div>
<div className="overflow-x-auto">
<table className="w-full text-sm">
<thead className="bg-neutral-50 dark:bg-neutral-800/30">
<tr className="text-left text-xs uppercase tracking-wide text-neutral-500 dark:text-neutral-400">
<th className="px-3 py-2">{t('backup.coverage.paths.path', 'Path')}</th>
<th className="px-3 py-2">{t('backup.coverage.paths.coverage', 'Coverage')}</th>
<th className="px-3 py-2">{t('backup.coverage.paths.featureFlag', 'Feature flag')}</th>
<th className="px-3 py-2">{t('backup.coverage.paths.description', 'Description')}</th>
</tr>
</thead>
<tbody>
{paths.map((p) => (
<tr
key={p.path}
className="border-t border-neutral-200 dark:border-neutral-700"
>
<td className="px-3 py-2 font-mono text-xs text-neutral-700 dark:text-neutral-300">
{p.path}
</td>
<td className="px-3 py-2">
<CoverageBadge coverage={p.coverage} />
</td>
<td className="px-3 py-2 text-xs text-neutral-600 dark:text-neutral-400">
{p.featureFlag
? `${p.featureFlag} = ${p.featureFlagValue === null ? '∅' : String(p.featureFlagValue)}`
: '—'}
</td>
<td className="px-3 py-2 text-xs text-neutral-600 dark:text-neutral-400">
{p.description ?? '—'}
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
);
};
const DriftSection: React.FC<{ drift: BackupCoverageReport['drift'] }> = ({ drift }) => {
const { t } = useTranslation();
if (drift.unconfiguredOnDisk.length === 0) {
return (
<div className="mt-4 p-3 rounded-lg bg-green-50 dark:bg-green-900/30 text-sm text-green-700 dark:text-green-300 flex items-center gap-2">
<CheckCircle2 className="w-4 h-4" />
{t(
'backup.coverage.drift.none',
'No drift detected — every top-level subdirectory under STORAGE_PATH is either in backup_paths or in the expected non-backup allow-list.',
)}
</div>
);
}
return (
<div className="mt-4 border border-amber-300 dark:border-amber-700 rounded-lg overflow-hidden">
<div className="px-3 py-2 bg-amber-50 dark:bg-amber-900/30 border-b border-amber-300 dark:border-amber-700">
<div className="flex items-center gap-2">
<AlertTriangle className="w-4 h-4 text-amber-700 dark:text-amber-300" />
<h4 className="text-sm font-semibold text-amber-800 dark:text-amber-200">
{t('backup.coverage.drift.heading', 'Drift detected: subdirectories not covered by any backup_paths row')}
</h4>
</div>
<p className="text-xs text-amber-700 dark:text-amber-300 mt-1">
{t(
'backup.coverage.drift.caption',
'These directories exist on disk but the walker will skip them. Either add a backup_paths row, move the files into a covered location, or — if they are runtime caches — confirm they are safe to exclude.',
)}
</p>
</div>
<ul className="divide-y divide-amber-200 dark:divide-amber-800">
{drift.unconfiguredOnDisk.map((d) => (
<li
key={d}
className="px-3 py-2 font-mono text-xs text-amber-900 dark:text-amber-100 flex items-center gap-2"
>
<EyeOff className="w-3.5 h-3.5" />
{d}
</li>
))}
</ul>
</div>
);
};
const CoverageBadge: React.FC<{ coverage: BackupPathCoverage }> = ({ coverage }) => {
const { t } = useTranslation();
const map: Record<BackupPathCoverage, { tone: Tone; label: string }> = {
'will-scan': {
tone: 'green',
label: t('backup.coverage.coverage.willScan', 'Will scan'),
},
'skipped-by-toggle': {
tone: 'neutral',
label: t('backup.coverage.coverage.skippedByToggle', 'Off'),
},
'skipped-by-feature-flag': {
tone: 'neutral',
label: t('backup.coverage.coverage.skippedByFlag', 'Gated off'),
},
'missing-on-disk': {
tone: 'amber',
label: t('backup.coverage.coverage.missingOnDisk', 'Missing on disk'),
},
};
const { tone, label } = map[coverage];
return (
<span className={`inline-block px-2 py-0.5 rounded text-xs font-medium ${TONE_BG[tone]}`}>
{label}
</span>
);
};
const Row: React.FC<{ label: string; value: string; icon?: React.ReactNode }> = ({
label, value, icon,
}) => (
<div className="flex justify-between items-center gap-3">
<dt className="text-xs uppercase tracking-wide opacity-80 flex items-center gap-1">
{icon}
{label}
</dt>
<dd className="text-sm font-medium text-right">{value}</dd>
</div>
);
type Tone = 'neutral' | 'green' | 'amber' | 'red';
const TONE_BG: Record<Tone, string> = {
neutral: 'bg-neutral-100 dark:bg-neutral-800 text-neutral-700 dark:text-neutral-200',
green: 'bg-green-50 dark:bg-green-900/30 text-green-700 dark:text-green-300',
amber: 'bg-amber-50 dark:bg-amber-900/30 text-amber-700 dark:text-amber-300',
red: 'bg-red-50 dark:bg-red-900/30 text-red-700 dark:text-red-300',
};
function formatBytes(bytes: number): string {
if (!bytes) return '0 B';
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
const i = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1);
return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 1)} ${units[i]}`;
}
function formatAge(ms: number): string {
const sec = Math.floor(ms / 1000);
if (sec < 60) return `${sec}s ago`;
const min = Math.floor(sec / 60);
if (min < 60) return `${min}m ago`;
const hr = Math.floor(min / 60);
if (hr < 48) return `${hr}h ago`;
const day = Math.floor(hr / 24);
return `${day}d ago`;
}
@@ -16,7 +16,10 @@ import {
AlertTriangle,
Info
} from 'lucide-react';
import { format, formatDistanceToNow } from 'date-fns';
// Per [[feedback_respect_general_format_settings]] — route every
// displayed date/time through useLocalizedDate so general_date_format
// and general_time_format settings apply uniformly.
import { useLocalizedDate } from '../../hooks/useLocalizedDate';
import { Card, Button } from '../common';
export type HealthStatus = 'excellent' | 'good' | 'warning' | 'critical';
@@ -38,11 +41,15 @@ interface BackupRecord {
backup_type: string;
created_at: string;
duration_seconds: number;
started_at?: string;
error_message?: string;
statistics?: BackupStatistics;
}
interface BackupStatus {
lastBackup?: BackupRecord;
lastBackup?: BackupRecord; // most recent attempt, any status
lastSuccessfulBackup?: BackupRecord; // most recent completed run
zombieRuns?: BackupRecord[]; // running > 30min, likely crashed
totalBackups?: number;
recentBackups?: BackupRecord[];
}
@@ -105,18 +112,41 @@ const healthColors: Record<HealthStatus, string> = {
export const BackupDashboard: React.FC<BackupDashboardProps> = ({ status, config, onRunBackup, isBackupRunning }) => {
const { t } = useTranslation();
const lastBackup = status?.lastBackup;
const statistics = lastBackup?.statistics ?? {};
const { format, formatTime, formatDateTime, formatDistanceToNow } = useLocalizedDate();
const lastBackup = status?.lastBackup; // any status
const lastSuccessfulBackup = status?.lastSuccessfulBackup; // status='completed' only
const zombieRuns = status?.zombieRuns ?? [];
const statistics = lastSuccessfulBackup?.statistics ?? lastBackup?.statistics ?? {};
const isConfigured = config && config.backup_destination_type;
const isEnabled = config?.backup_enabled;
// Use the most recent SUCCESSFUL backup as the "age" reference for
// health, so a transient failure doesn't immediately drop the score
// — but call out failed/running/zombie attempts explicitly so the
// admin sees them at a glance.
const getHealthScore = (): { score: number; status: HealthStatus; message: string } => {
if (!lastBackup) return { score: 0, status: 'critical', message: t('backup.dashboard.healthMessages.noBackups') };
if (!lastSuccessfulBackup) {
// No successful backup ever recorded.
if (lastBackup?.status === 'failed') {
return { score: 0, status: 'critical', message: t('backup.dashboard.healthMessages.lastBackupFailed') };
}
return { score: 0, status: 'critical', message: t('backup.dashboard.healthMessages.noBackups') };
}
const hoursSinceBackup = (Date.now() - new Date(lastBackup.created_at).getTime()) / (1000 * 60 * 60);
const hoursSinceBackup = (Date.now() - new Date(lastSuccessfulBackup.created_at).getTime()) / (1000 * 60 * 60);
if (lastBackup.status === 'failed') {
return { score: 0, status: 'critical', message: t('backup.dashboard.healthMessages.lastBackupFailed') };
// A successful backup exists. Bias the score on its age, but if
// the MOST RECENT attempt failed, downgrade the message so the
// admin sees the regression even though older backups are fine.
const latestAttemptFailed = lastBackup && lastBackup.id !== lastSuccessfulBackup.id
&& lastBackup.status === 'failed';
if (latestAttemptFailed) {
return {
score: 50,
status: 'warning',
message: t('backup.dashboard.healthMessages.lastBackupFailed'),
};
}
if (hoursSinceBackup < 24) {
@@ -190,9 +220,43 @@ export const BackupDashboard: React.FC<BackupDashboardProps> = ({ status, config
<div className="flex-1">
<p className="text-neutral-700 dark:text-neutral-300 font-medium">{health.message}</p>
{lastBackup && (
{/* Show the last successful backup explicitly previously
this read `lastBackup.created_at` which silently rendered
a failed/running row as if it were the last success. */}
{lastSuccessfulBackup && (
<p className="text-sm text-neutral-500 dark:text-neutral-400 mt-1">
Last successful backup: {formatDistanceToNow(new Date(lastBackup.created_at), { addSuffix: true })}
{t('backup.dashboard.lastSuccessful', 'Last successful backup')}: {formatDistanceToNow(new Date(lastSuccessfulBackup.created_at), { addSuffix: true })}
</p>
)}
{/* If the most recent attempt is NOT the last successful
run, surface it separately so the admin sees the
divergence (latest attempt failed or running). */}
{lastBackup && lastBackup.id !== lastSuccessfulBackup?.id && (
<p className={`text-sm mt-1 ${
lastBackup.status === 'failed'
? 'text-red-600 dark:text-red-400 font-medium'
: lastBackup.status === 'running'
? 'text-blue-600 dark:text-blue-400'
: 'text-neutral-500 dark:text-neutral-400'
}`}>
{t('backup.dashboard.lastAttempt', 'Last attempt')}: {formatDistanceToNow(new Date(lastBackup.created_at), { addSuffix: true })}
{' · '}
{t(`backup.dashboard.status.${lastBackup.status}`, lastBackup.status)}
{lastBackup.status === 'failed' && lastBackup.error_message && (
<span className="block text-xs text-red-600 dark:text-red-400 mt-0.5">
{lastBackup.error_message.split('\n')[0].slice(0, 200)}
</span>
)}
</p>
)}
{/* Zombie warning running >30min, almost certainly crashed.
Admin needs to know they may be looking at a hung row
that won't ever flip to completed. */}
{zombieRuns.length > 0 && (
<p className="text-sm mt-1 text-amber-700 dark:text-amber-300 font-medium">
{t('backup.dashboard.zombieRuns',
'{{count}} backup(s) running >30min — may have crashed without completing',
{ count: zombieRuns.length })}
</p>
)}
@@ -225,7 +289,7 @@ export const BackupDashboard: React.FC<BackupDashboardProps> = ({ status, config
label={t('backup.dashboard.stats.totalBackups')}
value={status?.totalBackups || 0}
color="blue"
subtext={lastBackup ? `${t('backup.dashboard.stats.last')}: ${format(new Date(lastBackup.created_at), 'PP')}` : t('backup.dashboard.stats.noBackupsYet')}
subtext={lastBackup ? `${t('backup.dashboard.stats.last')}: ${format(new Date(lastBackup.created_at))}` : t('backup.dashboard.stats.noBackupsYet')}
/>
<StatCard
@@ -241,7 +305,7 @@ export const BackupDashboard: React.FC<BackupDashboardProps> = ({ status, config
label={t('backup.dashboard.stats.lastDuration')}
value={lastBackup ? `${Math.round(lastBackup.duration_seconds / 60)}m` : 'N/A'}
color="purple"
subtext={lastBackup ? format(new Date(lastBackup.created_at), 'p') : ''}
subtext={lastBackup ? formatTime(new Date(lastBackup.created_at)) : ''}
/>
<StatCard
@@ -273,7 +337,7 @@ export const BackupDashboard: React.FC<BackupDashboardProps> = ({ status, config
{t('backup.dashboard.backupType', { type: backup.backup_type })}
</p>
<p className="text-sm text-neutral-500 dark:text-neutral-400">
{format(new Date(backup.created_at), 'PPp')}
{formatDateTime(new Date(backup.created_at))}
</p>
</div>
</div>
+101 -20
View File
@@ -20,11 +20,17 @@ import {
RefreshCw,
Loader2
} from 'lucide-react';
import { format, formatDistanceToNow } from 'date-fns';
import { toast } from 'react-toastify';
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
import { Button, Card, Input, Loading } from '../common';
import { api } from '../../config/api';
// Per [[feedback_respect_general_format_settings]]: route every displayed
// date/time through useLocalizedDate so the admin's general_date_format +
// general_time_format settings apply uniformly. Previously the backup
// History pane used raw date-fns format() with hard-coded 'p' (12-hour
// AM/PM) and 'PPP' (US-locale long date), which ignored the settings
// Ralf 2026-05-31 flagged "11:25 PM" on a 24h-configured install.
import { useLocalizedDate } from '../../hooks/useLocalizedDate';
const statusIcons = {
completed: { icon: CheckCircle, color: 'text-green-500' },
@@ -48,6 +54,12 @@ export const BackupHistory = () => {
const [filterStatus, setFilterStatus] = useState('all');
const [currentPage, setCurrentPage] = useState(1);
const queryClient = useQueryClient();
// Locale-aware formatters that respect admin's general_date_format +
// general_time_format settings. See useLocalizedDate.ts for the full
// contract; formatTime gives "HH:mm" (24h) or "h:mm a" (12h) based on
// the setting, format(date) honors general_date_format, and
// formatDistanceToNow returns "2 minutes ago" in the admin's i18n locale.
const { format, formatTime, formatDistanceToNow } = useLocalizedDate();
// Fetch backup history
const { data, isLoading, refetch } = useQuery({
@@ -90,7 +102,7 @@ export const BackupHistory = () => {
};
const handleDelete = (backup) => {
if (window.confirm(`Are you sure you want to delete this backup from ${format(new Date(backup.created_at), 'PPP')}?`)) {
if (window.confirm(`Are you sure you want to delete this backup from ${format(new Date(backup.created_at))}?`)) {
deleteMutation.mutate(backup.id);
}
};
@@ -200,10 +212,10 @@ export const BackupHistory = () => {
<td className="px-6 py-4 whitespace-nowrap">
<div>
<p className="text-sm font-medium text-neutral-900 dark:text-neutral-100">
{format(new Date(backup.created_at), 'PPP')}
{format(new Date(backup.created_at))}
</p>
<p className="text-xs text-neutral-500 dark:text-neutral-400">
{format(new Date(backup.created_at), 'p')} {formatDistanceToNow(new Date(backup.created_at), { addSuffix: true })}
{formatTime(new Date(backup.created_at))} {formatDistanceToNow(new Date(backup.created_at), { addSuffix: true })}
</p>
</div>
</td>
@@ -236,7 +248,7 @@ export const BackupHistory = () => {
</button>
{backup.manifest_path && (
<button
onClick={() => window.open(`/admin/backup/download/${backup.id}`, '_blank')}
onClick={() => window.open(`/api/admin/backup/download/${backup.id}`, '_blank')}
className="text-neutral-400 hover:text-neutral-600 dark:hover:text-neutral-300"
title={t('backup.actions.download')}
>
@@ -270,18 +282,27 @@ export const BackupHistory = () => {
</div>
<div className="flex justify-between">
<span className="text-neutral-500 dark:text-neutral-400">{t('backup.history.details.started')}:</span>
<span className="text-neutral-900 dark:text-neutral-100">{format(new Date(backup.created_at), 'p')}</span>
<span className="text-neutral-900 dark:text-neutral-100">{formatTime(new Date(backup.created_at))}</span>
</div>
{backup.completed_at && (
<div className="flex justify-between">
<span className="text-neutral-500 dark:text-neutral-400">{t('backup.history.details.completed')}:</span>
<span className="text-neutral-900 dark:text-neutral-100">{format(new Date(backup.completed_at), 'p')}</span>
<span className="text-neutral-900 dark:text-neutral-100">{formatTime(new Date(backup.completed_at))}</span>
</div>
)}
</div>
</div>
{/* Content Backed Up */}
{/* Content Backed Up
Two render paths depending on what the backend
provided:
- NEW: per_path map { "events/active": {count, size}, ... }
from Stage B's walker. One row per path,
ordered by display_order.
- LEGACY: fall back to Photos + Archives +
"Other" bucket so the arithmetic still adds
up when restoring a backup taken before this
change shipped. */}
<div className="space-y-2">
<h4 className="font-medium text-neutral-900 dark:text-neutral-100">{t('backup.history.details.contentBackedUp')}</h4>
<div className="space-y-2">
@@ -289,18 +310,78 @@ export const BackupHistory = () => {
<Database className={`h-4 w-4 ${stats.database_backed_up ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
<span className="text-sm text-neutral-700 dark:text-neutral-300">{t('backup.configuration.whatToBackup.database')}</span>
</div>
<div className="flex items-center space-x-2">
<Image className={`h-4 w-4 ${stats.photos_backed_up > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
<span className="text-sm text-neutral-700 dark:text-neutral-300">
Photos ({stats.photos_backed_up || 0} of {stats.total_photos || 0})
</span>
</div>
<div className="flex items-center space-x-2">
<FileArchive className={`h-4 w-4 ${stats.archives_backed_up > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
<span className="text-sm text-neutral-700 dark:text-neutral-300">
Archives ({stats.archives_backed_up || 0})
</span>
</div>
{(() => {
// Per-path breakdown when present
const perPath = stats.per_path || stats.perPath;
if (perPath && Object.keys(perPath).length > 0) {
const formatSize = (bytes) => {
if (!bytes) return '0 B';
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
const i = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1);
return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 1)} ${units[i]}`;
};
// Sort by path string so the order is stable across renders;
// backend uses backup_paths.display_order to drive the walker
// but doesn't carry order into per_path map alphabetic is
// fine for the display.
const entries = Object.entries(perPath).sort(([a], [b]) => a.localeCompare(b));
return (
<>
{entries.map(([pathKey, info]) => (
<div key={pathKey} className="flex items-center space-x-2">
<FileArchive className={`h-4 w-4 ${info.count > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
<span className="text-sm text-neutral-700 dark:text-neutral-300 font-mono">
{pathKey}
</span>
<span className="text-sm text-neutral-500 dark:text-neutral-400 ml-auto">
{info.count} {info.size ? `(${formatSize(info.size)})` : ''}
</span>
</div>
))}
<div className="flex items-center space-x-2 pt-1 border-t border-neutral-200 dark:border-neutral-700">
<span className="text-xs uppercase tracking-wide text-neutral-500 dark:text-neutral-400">
{t('backup.history.details.totalFiles', 'Total files')}: {stats.files_processed || 0}
</span>
</div>
</>
);
}
// LEGACY rendering for backups taken before
// per_path was emitted.
const total = Number(stats.files_processed) || 0;
const accounted =
(Number(stats.photos_backed_up) || 0)
+ (Number(stats.archives_backed_up) || 0);
const other = Math.max(total - accounted, 0);
return (
<>
<div className="flex items-center space-x-2">
<Image className={`h-4 w-4 ${stats.photos_backed_up > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
<span className="text-sm text-neutral-700 dark:text-neutral-300">
Photos ({stats.photos_backed_up || 0} of {stats.total_photos || 0})
</span>
</div>
<div className="flex items-center space-x-2">
<FileArchive className={`h-4 w-4 ${stats.archives_backed_up > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
<span className="text-sm text-neutral-700 dark:text-neutral-300">
Archives ({stats.archives_backed_up || 0})
</span>
</div>
<div className="flex items-center space-x-2">
<FileArchive className={`h-4 w-4 ${other > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
<span className="text-sm text-neutral-700 dark:text-neutral-300">
{t('backup.history.details.otherFiles', 'Business documents & other')} ({other})
</span>
</div>
<div className="flex items-center space-x-2 pt-1 border-t border-neutral-200 dark:border-neutral-700">
<span className="text-xs uppercase tracking-wide text-neutral-500 dark:text-neutral-400">
{t('backup.history.details.totalFiles', 'Total files')}: {total}
</span>
</div>
</>
);
})()}
</div>
</div>
@@ -0,0 +1,287 @@
import React, { useState } from 'react';
import { useTranslation } from 'react-i18next';
import {
ShieldCheck,
ShieldAlert,
FileX,
Hash,
HelpCircle,
Play,
Loader2,
} from 'lucide-react';
import { useMutation } from '@tanstack/react-query';
// Locale-aware formatters per [[feedback_respect_general_format_settings]].
import { useLocalizedDate } from '../../hooks/useLocalizedDate';
import { Card, Button } from '../common';
import { adminService, BackupIntegrityReport } from '../../services/admin.service';
/**
* BackupIntegrityCard on-demand verifier for CRM document artefacts.
*
* Walks every `*_path` column on quotes / contracts / invoices and
* confirms (a) the referenced file exists on disk, (b) where a SHA-256
* is stored, the file's bytes hash to the expected value. Surfaces
* three failure buckets:
*
* - missing `*_path` set, file not on disk (broken FK)
* - hashMismatches file exists but bytes don't match the stored hash
* - existsButNoHash verified by existence only; weaker evidence
*
* Designed to be portable. Currently embedded as a tab on
* `BackupManagement.tsx`; when the System Health page (backlog item)
* lands, this same component can be lifted there without changes.
*/
export const BackupIntegrityCard: React.FC = () => {
const { t } = useTranslation();
const { formatDateTime } = useLocalizedDate();
const [report, setReport] = useState<BackupIntegrityReport | null>(null);
const [expanded, setExpanded] = useState<'missing' | 'hashMismatches' | null>(null);
const runCheck = useMutation({
mutationFn: () => adminService.getBackupIntegrity(),
onSuccess: (data) => {
setReport(data);
// Auto-expand whichever failure bucket has entries, prioritising
// the more severe one (missing > hashMismatches).
if (data.summary.missingFiles > 0) setExpanded('missing');
else if (data.summary.hashMismatches > 0) setExpanded('hashMismatches');
else setExpanded(null);
},
});
const summary = report?.summary;
const isHealthy = report
&& summary
&& summary.missingFiles === 0
&& summary.hashMismatches === 0;
return (
<Card className="p-6">
<div className="flex items-start justify-between mb-4">
<div>
<div className="flex items-center gap-2 mb-1">
{isHealthy ? (
<ShieldCheck className="w-5 h-5 text-green-600 dark:text-green-400" />
) : report ? (
<ShieldAlert className="w-5 h-5 text-red-600 dark:text-red-400" />
) : (
<ShieldCheck className="w-5 h-5 text-neutral-400" />
)}
<h3 className="text-lg font-semibold text-neutral-900 dark:text-neutral-100">
{t('backup.integrity.title', 'Document integrity')}
</h3>
</div>
<p className="text-sm text-neutral-600 dark:text-neutral-400 max-w-2xl">
{t(
'backup.integrity.description',
'Verifies every CRM document (quote / contract / invoice / signature) referenced from the database actually exists on disk and — where a hash is stored — its bytes still match. Read-only, on-demand.',
)}
</p>
</div>
<Button
variant="primary"
onClick={() => runCheck.mutate()}
disabled={runCheck.isPending}
leftIcon={
runCheck.isPending
? <Loader2 className="w-4 h-4 animate-spin" />
: <Play className="w-4 h-4" />
}
>
{runCheck.isPending
? t('backup.integrity.running', 'Checking…')
: t('backup.integrity.runNow', 'Run check now')}
</Button>
</div>
{runCheck.isError && (
<div className="mb-4 p-3 rounded-lg bg-red-50 dark:bg-red-900/30 text-sm text-red-700 dark:text-red-300">
{t('backup.integrity.error', 'Check failed: {{message}}', {
message: (runCheck.error as Error)?.message ?? 'unknown error',
})}
</div>
)}
{report && summary && (
<>
<div className="grid grid-cols-2 md:grid-cols-5 gap-3 mb-4">
<Counter
label={t('backup.integrity.summary.total', 'Total')}
value={summary.totalRows}
tone="neutral"
/>
<Counter
label={t('backup.integrity.summary.verifiedOk', 'Hash-verified')}
value={summary.verifiedOk}
tone="green"
icon={<Hash className="w-4 h-4" />}
/>
<Counter
label={t('backup.integrity.summary.existsButNoHash', 'Exists only')}
value={summary.existsButNoHash}
tone="amber"
icon={<HelpCircle className="w-4 h-4" />}
tooltip={t(
'backup.integrity.summary.existsButNoHashHint',
'File found, but no SHA-256 is stored for it (quote/invoice PDFs, signature drawings). Existence-only is weaker evidence in a dispute.',
)}
/>
<Counter
label={t('backup.integrity.summary.missingFiles', 'Missing')}
value={summary.missingFiles}
tone={summary.missingFiles > 0 ? 'red' : 'neutral'}
icon={<FileX className="w-4 h-4" />}
onClick={summary.missingFiles > 0
? () => setExpanded(expanded === 'missing' ? null : 'missing')
: undefined}
/>
<Counter
label={t('backup.integrity.summary.hashMismatches', 'Hash mismatches')}
value={summary.hashMismatches}
tone={summary.hashMismatches > 0 ? 'red' : 'neutral'}
icon={<ShieldAlert className="w-4 h-4" />}
onClick={summary.hashMismatches > 0
? () => setExpanded(expanded === 'hashMismatches' ? null : 'hashMismatches')
: undefined}
/>
</div>
<p className="text-xs text-neutral-500 dark:text-neutral-400 mb-3">
{t('backup.integrity.scannedAt', 'Last checked: {{when}}', {
when: formatDateTime(new Date(report.scannedAt)),
})}
</p>
{expanded === 'missing' && summary.missingFiles > 0 && (
<ResultTable
title={t('backup.integrity.missing.heading', 'Missing files')}
caption={t(
'backup.integrity.missing.caption',
'These rows reference a path that does not exist on disk. After a restore, this means the artefact was lost from the backup chain; for fresh installs, it usually means the file was deleted manually.',
)}
rows={report.missing.map((m) => ({
table: m.table,
rowId: m.rowId,
column: m.column,
detail: m.expectedPath,
}))}
/>
)}
{expanded === 'hashMismatches' && summary.hashMismatches > 0 && (
<ResultTable
title={t('backup.integrity.hashMismatches.heading', 'Hash mismatches')}
caption={t(
'backup.integrity.hashMismatches.caption',
'The file exists but its current bytes do not match the SHA-256 captured at issue / sign time. Indicates tampering, bit-rot, or a restore that pulled in a different copy than the original.',
)}
rows={report.hashMismatches.map((m) => ({
table: m.table,
rowId: m.rowId,
column: m.column,
detail: `${m.expectedPath} (expected ${m.expectedSha.slice(0, 12)}…, got ${m.actualSha.slice(0, 12)}…)`,
}))}
/>
)}
</>
)}
{!report && !runCheck.isPending && (
<p className="text-sm text-neutral-500 dark:text-neutral-400 italic">
{t(
'backup.integrity.emptyState',
'No check has been run yet in this session. Click "Run check now" to scan the document estate.',
)}
</p>
)}
</Card>
);
};
type Tone = 'neutral' | 'green' | 'amber' | 'red';
const TONE_CLASSES: Record<Tone, string> = {
neutral: 'bg-neutral-100 dark:bg-neutral-800 text-neutral-700 dark:text-neutral-200',
green: 'bg-green-50 dark:bg-green-900/30 text-green-700 dark:text-green-300',
amber: 'bg-amber-50 dark:bg-amber-900/30 text-amber-700 dark:text-amber-300',
red: 'bg-red-50 dark:bg-red-900/30 text-red-700 dark:text-red-300',
};
const Counter: React.FC<{
label: string;
value: number;
tone: Tone;
icon?: React.ReactNode;
tooltip?: string;
onClick?: () => void;
}> = ({ label, value, tone, icon, tooltip, onClick }) => {
const interactive = Boolean(onClick);
const classes = `rounded-lg p-3 ${TONE_CLASSES[tone]} ${
interactive ? 'cursor-pointer hover:ring-2 hover:ring-offset-1 hover:ring-current/30 transition' : ''
}`;
return (
<div
className={classes}
onClick={onClick}
title={tooltip}
role={interactive ? 'button' : undefined}
tabIndex={interactive ? 0 : undefined}
>
<div className="flex items-center gap-1.5 text-xs font-medium uppercase tracking-wide opacity-80">
{icon}
<span>{label}</span>
</div>
<div className="text-2xl font-semibold mt-1 tabular-nums">{value}</div>
</div>
);
};
const ResultTable: React.FC<{
title: string;
caption: string;
rows: Array<{ table: string; rowId: number; column: string; detail: string }>;
}> = ({ title, caption, rows }) => {
const { t } = useTranslation();
return (
<div className="mt-4 border border-neutral-200 dark:border-neutral-700 rounded-lg overflow-hidden">
<div className="p-3 bg-neutral-50 dark:bg-neutral-800/50 border-b border-neutral-200 dark:border-neutral-700">
<h4 className="text-sm font-semibold text-neutral-900 dark:text-neutral-100">{title}</h4>
<p className="text-xs text-neutral-600 dark:text-neutral-400 mt-1">{caption}</p>
</div>
<div className="overflow-x-auto">
<table className="w-full text-sm">
<thead className="bg-neutral-50 dark:bg-neutral-800/30">
<tr className="text-left text-xs uppercase tracking-wide text-neutral-500 dark:text-neutral-400">
<th className="px-3 py-2">{t('backup.integrity.results.table', 'Table')}</th>
<th className="px-3 py-2">{t('backup.integrity.results.rowId', 'Row id')}</th>
<th className="px-3 py-2">{t('backup.integrity.results.column', 'Column')}</th>
<th className="px-3 py-2">{t('backup.integrity.results.detail', 'Detail')}</th>
</tr>
</thead>
<tbody>
{rows.map((r, i) => (
<tr
key={`${r.table}-${r.rowId}-${r.column}-${i}`}
className="border-t border-neutral-200 dark:border-neutral-700"
>
<td className="px-3 py-2 font-mono text-xs text-neutral-700 dark:text-neutral-300">
{r.table}
</td>
<td className="px-3 py-2 tabular-nums text-neutral-700 dark:text-neutral-300">
{r.rowId}
</td>
<td className="px-3 py-2 font-mono text-xs text-neutral-700 dark:text-neutral-300">
{r.column}
</td>
<td className="px-3 py-2 font-mono text-xs text-neutral-700 dark:text-neutral-300 break-all">
{r.detail}
</td>
</tr>
))}
</tbody>
</table>
</div>
</div>
);
};
+134 -10
View File
@@ -21,7 +21,8 @@ import {
Eye,
Calendar,
Clock,
AlertCircle
AlertCircle,
ShieldCheck
} from 'lucide-react';
import { format } from 'date-fns';
import { toast } from 'react-toastify';
@@ -29,7 +30,7 @@ import { useQuery, useMutation } from '@tanstack/react-query';
import { Button, Card, Input, Loading } from '../common';
import { api } from '../../config/api';
export const RestoreWizard = () => {
export const RestoreWizard = ({ onVerifyIntegrity } = {}) => {
const { t } = useTranslation();
const [currentStep, setCurrentStep] = useState(0);
@@ -337,15 +338,68 @@ export const RestoreWizard = () => {
</p>
</div>
</div>
{backup.encrypted && (
<Shield className="h-5 w-5 text-neutral-400" />
)}
<div className="flex items-center space-x-2">
{/* Files-only warning backend's /list-backups now
returns `database_included: boolean` parsed from
the manifest's database.backup_file field. A row
where this is false is exactly the data-loss
scenario the Stage A guard prevents going forward:
a manifest written without an inline DB dump.
Restoring it would NOT bring CRM data back. */}
{backup.database_included === false && (
<span
className="inline-flex items-center gap-1 px-2 py-0.5 rounded-md text-xs font-medium bg-red-100 dark:bg-red-900/40 text-red-700 dark:text-red-300 border border-red-300 dark:border-red-700"
title={t('backup.restore.backup.filesOnlyHint',
'This backup has no database dump — restoring it will NOT recover the database (CRM data, customers, quotes, invoices, contracts will be empty after restore).')}
>
<AlertCircle className="h-3 w-3" />
{t('backup.restore.backup.filesOnlyBadge', 'No DB')}
</span>
)}
{backup.corrupt && (
<span
className="inline-flex items-center gap-1 px-2 py-0.5 rounded-md text-xs font-medium bg-amber-100 dark:bg-amber-900/40 text-amber-700 dark:text-amber-300 border border-amber-300 dark:border-amber-700"
title={t('backup.restore.backup.corruptHint',
'The manifest file is unreadable — the backup may be incomplete or damaged.')}
>
<AlertCircle className="h-3 w-3" />
{t('backup.restore.backup.corruptBadge', 'Corrupt')}
</span>
)}
{backup.encrypted && (
<Shield className="h-5 w-5 text-neutral-400" />
)}
</div>
</div>
</Card>
))}
</div>
)}
{/* Files-only callout below the selected card. Reinforces the
badge with a longer explanation + reminds the admin that
restoring this WILL still proceed they just won't get the
DB back. Stops the silent-failure class that originally
caused Ralf's 2026-05-29 data loss (four files-only manifests
mistaken for full backups). */}
{restoreData.selectedBackup && restoreData.selectedBackup.database_included === false && (
<Card className="p-4 bg-red-50 dark:bg-red-900/30 border-red-300 dark:border-red-700">
<div className="flex items-start space-x-3">
<AlertCircle className="h-5 w-5 text-red-600 dark:text-red-400 mt-0.5" />
<div className="flex-1">
<p className="text-sm font-semibold text-red-800 dark:text-red-200">
{t('backup.restore.backup.filesOnlyWarning.title',
'Selected backup has no database dump')}
</p>
<p className="mt-1 text-sm text-red-700 dark:text-red-300">
{t('backup.restore.backup.filesOnlyWarning.message',
'Restoring this backup will recover files (photos, PDFs) but the database — including admin users, customers, quotes, invoices, contracts, and settings — will NOT come back. Pick a different backup if you have one with a database dump, or proceed only if files-only is what you want.')}
</p>
</div>
</div>
</Card>
)}
{restoreData.selectedBackup?.encrypted && (
<Card className="p-4 bg-amber-50 dark:bg-amber-900/30 border-amber-200 dark:border-amber-800">
<div className="flex items-start space-x-3">
@@ -573,16 +627,63 @@ export const RestoreWizard = () => {
const renderProgress = () => {
const progress = restoreStatus?.currentProgress || {};
const isRunning = restoreStatus?.isRunning;
// Pull the most recent restore_runs row from history so we can
// tell whether the "not running" state means success, failure, or
// never-started. The history endpoint already returns rows newest
// first.
const lastRun = restoreStatus?.history?.[0];
const lastRunFailed =
!isRunning && lastRun && (lastRun.status === 'failed' || lastRun.was_successful === false);
const lastRunSucceeded =
!isRunning && lastRun && lastRun.status === 'completed' && lastRun.was_successful === true;
// Strip the noisy stack-trace tail from the error message so the
// user sees the actionable line first.
const lastRunError = lastRun?.error_message
? lastRun.error_message.split('\n')[0].slice(0, 500)
: null;
const subtitle = isRunning
? t('backup.restore.progress.inProgress')
: lastRunFailed
? t('backup.restore.progress.failedSubtitle', 'Restore failed — see error below. Destination has been rolled back to its pre-restore state.')
: lastRunSucceeded
? t('backup.restore.progress.completed')
: t('backup.restore.progress.idle', 'No restore in progress.');
return (
<div className="space-y-6">
<div>
<h3 className="text-lg font-semibold text-neutral-900 dark:text-neutral-100 mb-2">{t('backup.restore.progress.title')}</h3>
<p className="text-sm text-neutral-600 dark:text-neutral-400">
{isRunning ? t('backup.restore.progress.inProgress') : t('backup.restore.progress.completed')}
<p className={`text-sm ${
lastRunFailed
? 'text-red-700 dark:text-red-300 font-medium'
: 'text-neutral-600 dark:text-neutral-400'
}`}>
{subtitle}
</p>
</div>
{lastRunFailed && (
<div className="bg-red-50 dark:bg-red-900/30 border border-red-300 dark:border-red-700 rounded-lg p-4">
<div className="flex items-start gap-3">
<XCircle className="h-5 w-5 text-red-500 flex-shrink-0 mt-0.5" />
<div className="flex-1">
<h4 className="text-sm font-semibold text-red-800 dark:text-red-200 mb-1">
{t('backup.restore.progress.errorTitle', 'Restore did not complete')}
</h4>
<p className="text-sm text-red-700 dark:text-red-300 font-mono break-all">
{lastRunError || t('backup.restore.progress.errorUnknown', 'No error message recorded.')}
</p>
{lastRun.was_rollback_attempted && (
<p className="mt-2 text-xs text-red-600 dark:text-red-400">
{t('backup.restore.progress.rolledBack',
'Pre-restore safety backup was used to roll back. Destination is in its pre-restore state — safe to retry once the issue above is resolved.')}
</p>
)}
</div>
</div>
</div>
)}
{/* Progress Bar */}
<Card className="p-6">
<div className="space-y-4">
@@ -645,18 +746,41 @@ export const RestoreWizard = () => {
</Card>
)}
{/* Completion Actions */}
{!isRunning && progress.status === 'completed' && (
{/* Completion Actions only when the most recent run actually
succeeded. Previously this gated on `progress.status` which
could be null between runs, so the green "Restore completed
successfully" banner could render alongside a silent failure. */}
{lastRunSucceeded && (
<div className="bg-green-50 dark:bg-green-900/30 border border-green-200 dark:border-green-800 rounded-lg p-4">
<div className="flex">
<CheckCircle className="h-5 w-5 text-green-400 mt-0.5" />
<div className="ml-3">
<div className="ml-3 flex-1">
<h3 className="text-sm font-medium text-green-800 dark:text-green-200">
{t('backup.restore.progress.success.title')}
</h3>
<p className="mt-1 text-sm text-green-700 dark:text-green-300">
{t('backup.restore.progress.success.message')}
</p>
{/* Post-restore CTA: jump to the integrity check (D2). The
audit trail captured at sign / issue time is worth
nothing if the documents it refers to are missing
from the restored copy verifier surfaces that
drift in one click before the admin trusts the
restored state. */}
{onVerifyIntegrity && (
<Button
variant="outline"
size="sm"
className="mt-3"
onClick={onVerifyIntegrity}
leftIcon={<ShieldCheck className="w-4 h-4" />}
>
{t(
'backup.restore.progress.success.verifyIntegrity',
'Verify document integrity now',
)}
</Button>
)}
</div>
</div>
</div>
+84 -3
View File
@@ -230,7 +230,85 @@
"dashboard": "Dashboard",
"configuration": "Konfiguration",
"history": "Backup-Verlauf",
"restore": "Wiederherstellung"
"restore": "Wiederherstellung",
"integrity": "Integrität",
"coverage": "Abdeckung"
},
"integrity": {
"title": "Dokumentintegrität",
"description": "Prüft, ob jedes in der Datenbank referenzierte CRM-Dokument (Angebot / Vertrag / Rechnung / Unterschrift) tatsächlich auf der Festplatte existiert und — sofern ein Hash gespeichert ist — die Datei-Bytes weiterhin übereinstimmen. Nur lesend, auf Abruf.",
"runNow": "Prüfung starten",
"running": "Prüfe…",
"error": "Prüfung fehlgeschlagen: {{message}}",
"scannedAt": "Zuletzt geprüft: {{when}}",
"emptyState": "In dieser Sitzung wurde noch keine Prüfung ausgeführt. Klicke auf \"Prüfung starten\", um den Dokumentbestand zu durchsuchen.",
"summary": {
"total": "Gesamt",
"verifiedOk": "Hash-verifiziert",
"existsButNoHash": "Nur Existenz",
"existsButNoHashHint": "Datei gefunden, aber kein SHA-256 gespeichert (Angebots-/Rechnungs-PDFs, Unterschrifts-Zeichnungen). Nur-Existenz ist im Streitfall schwächeres Beweismaterial.",
"missingFiles": "Fehlend",
"hashMismatches": "Hash-Abweichungen"
},
"missing": {
"heading": "Fehlende Dateien",
"caption": "Diese Zeilen verweisen auf einen Pfad, der nicht auf der Festplatte existiert. Nach einer Wiederherstellung bedeutet das, dass das Artefakt aus der Sicherungskette verloren ging; bei frischen Installationen wurde die Datei meist manuell gelöscht."
},
"hashMismatches": {
"heading": "Hash-Abweichungen",
"caption": "Die Datei existiert, aber ihre aktuellen Bytes stimmen nicht mit dem zum Ausstellungs-/Signierzeitpunkt erfassten SHA-256 überein. Deutet auf Manipulation, Bit-Rot oder eine Wiederherstellung mit einer abweichenden Kopie hin."
},
"results": {
"table": "Tabelle",
"rowId": "Zeilen-ID",
"column": "Spalte",
"detail": "Detail"
}
},
"coverage": {
"title": "Backup-Abdeckung",
"description": "Zeigt, was das nächste Backup einschließt, überspringt oder stillschweigend verfehlt. Der Datenbank-Block bestätigt die Dump-Strategie. Der \"Drift\"-Abschnitt markiert Unterverzeichnisse, die auf der Festplatte existieren, aber nicht in der Backup-Konfiguration stehen — meist ein Hinweis darauf, dass eine neue Funktion ohne passende backup_paths-Zeile ausgeliefert wurde.",
"refresh": "Aktualisieren",
"error": "Abdeckungs-Bericht konnte nicht geladen werden: {{message}}",
"generatedAt": "Abdeckung erstellt: {{when}}",
"fallbackInUse": "Die Tabelle backup_paths fehlt. Der Walker nutzt seine fest verdrahtete Legacy-Fallback-Liste. Migration 108 wurde möglicherweise nicht ausgeführt — prüfen Sie die Server-Logs und führen Sie die Migrationen erneut aus.",
"database": {
"title": "Datenbank",
"mode": "Modus",
"modeInline": "Inline-Dump bei jedem Backup",
"modeScheduled": "Nur geplant (Inline abgewählt)",
"lastDump": "Letzter Dump",
"lastDumpSize": "Größe",
"noDump": "Noch kein Dump vorhanden",
"staleLabel": "Status",
"stale": "Veraltet — älter als 26 Std."
},
"summary": {
"title": "Zusammenfassung",
"willScan": "Wird gescannt",
"skippedByToggle": "Übersprungen (Schalter aus)",
"skippedByFlag": "Übersprungen (Feature-Flag)",
"missingOnDisk": "Auf Festplatte fehlend",
"drift": "Nicht konfiguriert auf Festplatte (Drift)"
},
"paths": {
"heading": "Konfigurierte Pfade",
"path": "Pfad",
"coverage": "Abdeckung",
"featureFlag": "Feature-Flag",
"description": "Beschreibung"
},
"coverage": {
"willScan": "Wird gescannt",
"skippedByToggle": "Aus",
"skippedByFlag": "Per Flag aus",
"missingOnDisk": "Auf Festplatte fehlend"
},
"drift": {
"heading": "Drift erkannt: Unterverzeichnisse ohne backup_paths-Zeile",
"caption": "Diese Verzeichnisse existieren auf der Festplatte, werden vom Walker aber übersprungen. Entweder eine backup_paths-Zeile hinzufügen, die Dateien in ein abgedecktes Verzeichnis verschieben, oder — wenn es sich um Laufzeit-Caches handelt — bestätigen, dass der Ausschluss sicher ist.",
"none": "Kein Drift erkannt — jedes Top-Level-Unterverzeichnis unter STORAGE_PATH steht entweder in backup_paths oder in der erwarteten Nicht-Backup-Allow-List."
}
},
"status": {
"inProgress": "Backup läuft...",
@@ -397,7 +475,9 @@
"completed": "Abgeschlossen",
"contentBackedUp": "Gesicherter Inhalt",
"errorDetails": "Fehlerdetails",
"manifest": "Manifest"
"manifest": "Manifest",
"otherFiles": "Geschäftsdokumente & sonstige",
"totalFiles": "Gesamtdateien"
},
"pagination": {
"showing": "Zeige {{from}}-{{to}} von {{total}} Backups",
@@ -517,7 +597,8 @@
"restoreLogs": "Wiederherstellungsprotokolle",
"success": {
"title": "Wiederherstellung erfolgreich abgeschlossen",
"message": "Ihre Daten wurden wiederhergestellt. Bitte überprüfen Sie, ob alles korrekt funktioniert."
"message": "Ihre Daten wurden wiederhergestellt. Bitte überprüfen Sie, ob alles korrekt funktioniert.",
"verifyIntegrity": "Dokumentintegrität jetzt prüfen"
}
},
"actions": {
+84 -3
View File
@@ -2121,7 +2121,85 @@
"dashboard": "Dashboard",
"configuration": "Configuration",
"history": "Backup History",
"restore": "Restore"
"restore": "Restore",
"integrity": "Integrity",
"coverage": "Coverage"
},
"integrity": {
"title": "Document integrity",
"description": "Verifies every CRM document (quote / contract / invoice / signature) referenced from the database actually exists on disk and — where a hash is stored — its bytes still match. Read-only, on-demand.",
"runNow": "Run check now",
"running": "Checking…",
"error": "Check failed: {{message}}",
"scannedAt": "Last checked: {{when}}",
"emptyState": "No check has been run yet in this session. Click \"Run check now\" to scan the document estate.",
"summary": {
"total": "Total",
"verifiedOk": "Hash-verified",
"existsButNoHash": "Exists only",
"existsButNoHashHint": "File found, but no SHA-256 is stored for it (quote/invoice PDFs, signature drawings). Existence-only is weaker evidence in a dispute.",
"missingFiles": "Missing",
"hashMismatches": "Hash mismatches"
},
"missing": {
"heading": "Missing files",
"caption": "These rows reference a path that does not exist on disk. After a restore, this means the artefact was lost from the backup chain; for fresh installs, it usually means the file was deleted manually."
},
"hashMismatches": {
"heading": "Hash mismatches",
"caption": "The file exists but its current bytes do not match the SHA-256 captured at issue / sign time. Indicates tampering, bit-rot, or a restore that pulled in a different copy than the original."
},
"results": {
"table": "Table",
"rowId": "Row id",
"column": "Column",
"detail": "Detail"
}
},
"coverage": {
"title": "Backup coverage",
"description": "Shows what the next backup will include, skip, or silently miss. The database block confirms the dump strategy. The \"drift\" section flags subdirectories that exist on disk but are not in the backup configuration — usually a sign that a new feature shipped without a matching backup_paths row.",
"refresh": "Refresh",
"error": "Could not load coverage report: {{message}}",
"generatedAt": "Coverage generated: {{when}}",
"fallbackInUse": "The backup_paths table is missing. The walker is using its legacy hard-coded fallback. Migration 108 may not have run — check server logs and re-run migrations.",
"database": {
"title": "Database",
"mode": "Mode",
"modeInline": "Inline dump on every backup",
"modeScheduled": "Scheduled-only (inline opted out)",
"lastDump": "Last dump",
"lastDumpSize": "Size",
"noDump": "No dump on file yet",
"staleLabel": "Status",
"stale": "Stale — older than 26h"
},
"summary": {
"title": "Summary",
"willScan": "Will scan",
"skippedByToggle": "Skipped (toggle off)",
"skippedByFlag": "Skipped (feature flag)",
"missingOnDisk": "Missing on disk",
"drift": "Unconfigured on disk (drift)"
},
"paths": {
"heading": "Configured paths",
"path": "Path",
"coverage": "Coverage",
"featureFlag": "Feature flag",
"description": "Description"
},
"coverage": {
"willScan": "Will scan",
"skippedByToggle": "Off",
"skippedByFlag": "Gated off",
"missingOnDisk": "Missing on disk"
},
"drift": {
"heading": "Drift detected: subdirectories not covered by any backup_paths row",
"caption": "These directories exist on disk but the walker will skip them. Either add a backup_paths row, move the files into a covered location, or — if they are runtime caches — confirm they are safe to exclude.",
"none": "No drift detected — every top-level subdirectory under STORAGE_PATH is either in backup_paths or in the expected non-backup allow-list."
}
},
"status": {
"inProgress": "Backup in progress...",
@@ -2288,7 +2366,9 @@
"completed": "Completed",
"contentBackedUp": "Content Backed Up",
"errorDetails": "Error Details",
"manifest": "Manifest"
"manifest": "Manifest",
"otherFiles": "Business documents & other",
"totalFiles": "Total files"
},
"pagination": {
"showing": "Showing {{from}}-{{to}} of {{total}} backups",
@@ -2408,7 +2488,8 @@
"restoreLogs": "Restore Logs",
"success": {
"title": "Restore Completed Successfully",
"message": "Your data has been restored. Please verify everything is working correctly."
"message": "Your data has been restored. Please verify everything is working correctly.",
"verifyIntegrity": "Verify document integrity now"
}
},
"actions": {
+16 -2
View File
@@ -10,6 +10,8 @@ import {
Clock,
Loader2,
Shield,
ShieldCheck,
FolderTree,
} from 'lucide-react';
import { toast } from 'react-toastify';
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
@@ -21,9 +23,11 @@ import { BackupDashboard } from '../../components/admin/BackupDashboard';
import { BackupConfiguration } from '../../components/admin/BackupConfiguration';
import { BackupHistory } from '../../components/admin/BackupHistory';
import { RestoreWizard } from '../../components/admin/RestoreWizard';
import { BackupIntegrityCard } from '../../components/admin/BackupIntegrityCard';
import { BackupCoverageCard } from '../../components/admin/BackupCoverageCard';
import { api } from '../../config/api';
type TabId = 'dashboard' | 'configuration' | 'history' | 'restore';
type TabId = 'dashboard' | 'configuration' | 'history' | 'restore' | 'integrity' | 'coverage';
export const BackupManagement: React.FC = () => {
const [activeTab, setActiveTab] = useState<TabId>('dashboard');
@@ -35,6 +39,8 @@ export const BackupManagement: React.FC = () => {
{ id: 'configuration' as const, label: t('backup.tabs.configuration'), icon: Settings },
{ id: 'history' as const, label: t('backup.tabs.history'), icon: History },
{ id: 'restore' as const, label: t('backup.tabs.restore'), icon: RefreshCw },
{ id: 'integrity' as const, label: t('backup.tabs.integrity', 'Integrity'), icon: ShieldCheck },
{ id: 'coverage' as const, label: t('backup.tabs.coverage', 'Coverage'), icon: FolderTree },
];
const { data: backupStatus, isLoading: statusLoading } = useQuery({
@@ -218,7 +224,15 @@ export const BackupManagement: React.FC = () => {
)}
{activeTab === 'restore' && (
<RestoreWizard />
<RestoreWizard onVerifyIntegrity={() => setActiveTab('integrity')} />
)}
{activeTab === 'integrity' && (
<BackupIntegrityCard />
)}
{activeTab === 'coverage' && (
<BackupCoverageCard />
)}
</div>
</div>
+119
View File
@@ -201,6 +201,102 @@ export interface Activity {
createdAt: string;
}
// Backup-integrity verifier — diagnostic endpoint that walks every
// CRM document-artefact path column and confirms files exist on disk
// (plus SHA-256 match where the schema stores one). Mirrors the
// shape returned by backupIntegrityService.verifyDocumentArtefacts.
export type BackupIntegrityScope =
| 'quote'
| 'contract'
| 'contract-signature'
| 'invoice';
export interface BackupIntegrityMissingRow {
table: string;
rowId: number;
column: string;
expectedPath: string;
}
export interface BackupIntegrityHashMismatchRow extends BackupIntegrityMissingRow {
expectedSha: string;
actualSha: string;
}
export interface BackupIntegrityExistsButNoHashRow {
table: string;
rowId: number;
column: string;
path: string;
}
export interface BackupIntegrityReport {
scannedAt: string;
scopes: BackupIntegrityScope[];
summary: {
totalRows: number;
verifiedOk: number;
missingFiles: number;
hashMismatches: number;
existsButNoHash: number;
};
missing: BackupIntegrityMissingRow[];
hashMismatches: BackupIntegrityHashMismatchRow[];
existsButNoHash: BackupIntegrityExistsButNoHashRow[];
}
// ---- Backup-coverage (Stage C of backup-hardening plan) -----------------
export type BackupPathCoverage =
| 'will-scan'
| 'skipped-by-toggle'
| 'skipped-by-feature-flag'
| 'missing-on-disk';
export interface BackupCoveragePath {
path: string;
includeInDefault: boolean;
featureFlag: string | null;
featureFlagValue: boolean | null;
displayOrder: number;
description: string | null;
existsOnDisk: boolean;
coverage: BackupPathCoverage;
}
export interface BackupCoverageDatabase {
mode: 'inline' | 'scheduled-only';
inlineDumpExplicitlyDisabled: boolean;
lastDumpAt: string | null;
lastDumpType: string | null;
lastDumpSizeBytes: number;
lastDumpFilePath: string | null;
lastDumpAgeMs: number | null;
lastDumpStale: boolean | null;
ok: boolean;
}
export interface BackupCoverageReport {
generatedAt: string;
database: BackupCoverageDatabase;
paths: BackupCoveragePath[];
drift: {
unconfiguredOnDisk: string[];
expectedNonBackupDirs: string[];
};
summary: {
configuredCount: number;
willScanCount: number;
skippedByToggleCount: number;
skippedByFeatureFlagCount: number;
missingOnDiskCount: number;
driftCount: number;
tableMissingFallbackInUse: boolean;
databaseOk: boolean;
overallOk: boolean;
};
}
export interface AdminProfile {
id: number;
username: string;
@@ -262,6 +358,29 @@ export const adminService = {
return response.data;
},
// Backup-integrity verifier (read-only diagnostic). `scope` filters
// which document classes to walk; omit for a full scan.
async getBackupIntegrity(
scope?: BackupIntegrityScope[],
): Promise<BackupIntegrityReport> {
const params = scope && scope.length > 0 ? { scope: scope.join(',') } : undefined;
const response = await api.get<{ report: BackupIntegrityReport }>(
'/admin/system-health/backup-integrity',
{ params },
);
return response.data.report;
},
// Backup-coverage diagnostic (Stage C). Answers "what will the
// next backup actually include / skip / silently miss?" — read-only,
// no parameters. See backupCoverageService.js for the full report shape.
async getBackupCoverage(): Promise<BackupCoverageReport> {
const response = await api.get<{ report: BackupCoverageReport }>(
'/admin/system-health/backup-coverage',
);
return response.data.report;
},
// Format activity message
formatActivityMessage(activity: Activity): string {
// Feature-flag toggles carry a `changed` diff in metadata. Render