Merge branch 'beta' of https://github.com/the-luap/picpeak into feat/crm-improvements
This commit is contained in:
Luca
@@ -0,0 +1,242 @@
|
||||
/**
|
||||
* Integration test for GET /api/admin/system-health/backup-coverage.
|
||||
*
|
||||
* Pins the Stage C diagnostic that tells admins what the next
|
||||
* "Run Backup Now" will include, skip, or silently miss.
|
||||
*
|
||||
* Test surface:
|
||||
* 1. Empty / fresh install → default seed (7 paths), inline mode,
|
||||
* no DB dump on file yet, no drift
|
||||
* 2. Toggle `include_in_default=false` → coverage flips to
|
||||
* 'skipped-by-toggle'
|
||||
* 3. Feature_flag gating reflects the actual app_settings value
|
||||
* (events/archived ⇄ backup_include_archived)
|
||||
* 4. Drift detection: a top-level subdir on disk with no
|
||||
* `backup_paths` row is flagged in `unconfiguredOnDisk`
|
||||
* 5. Allow-list: `backups/` and `tmp/` are never flagged as drift
|
||||
* 6. Scheduled-only mode + recent dump → `database.ok = true`
|
||||
* 7. Scheduled-only mode + stale (>26h) dump → `database.ok = false`
|
||||
* and `lastDumpStale = true`
|
||||
*
|
||||
* Same auth/permission pass-through strategy as
|
||||
* adminBackupIntegrity.test.js — we exercise the route's logic,
|
||||
* not the auth middleware.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const express = require('express');
|
||||
const request = require('supertest');
|
||||
|
||||
const { bootCrmDb } = require('./helpers/crmDb');
|
||||
|
||||
jest.mock('../../src/middleware/auth', () => ({
|
||||
adminAuth: (req, _res, next) => { req.admin = { id: 1 }; next(); },
|
||||
customerAuth: (_req, _res, next) => next(),
|
||||
galleryAuth: (_req, _res, next) => next(),
|
||||
}));
|
||||
|
||||
jest.mock('../../src/middleware/permissions', () => ({
|
||||
requirePermission: () => (_req, _res, next) => next(),
|
||||
}));
|
||||
|
||||
jest.setTimeout(30000);
|
||||
|
||||
describe('GET /api/admin/system-health/backup-coverage', () => {
|
||||
let db;
|
||||
let cleanup;
|
||||
let storagePath;
|
||||
let app;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup } = await bootCrmDb());
|
||||
storagePath = process.env.STORAGE_PATH;
|
||||
|
||||
const route = require('../../src/routes/adminSystemHealth');
|
||||
app = express();
|
||||
app.use(express.json());
|
||||
app.use('/api/admin/system-health', route);
|
||||
}, 120000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (cleanup) await cleanup();
|
||||
});
|
||||
|
||||
function mkdir(rel) {
|
||||
fs.mkdirSync(path.join(storagePath, rel), { recursive: true });
|
||||
}
|
||||
|
||||
function rmdir(rel) {
|
||||
fs.rmSync(path.join(storagePath, rel), { recursive: true, force: true });
|
||||
}
|
||||
|
||||
async function restoreDefaultPaths() {
|
||||
await db('backup_paths').del();
|
||||
const { DEFAULT_PATHS } = require('../../migrations/core/109_add_backup_paths');
|
||||
await db('backup_paths').insert(DEFAULT_PATHS.map((row) => ({
|
||||
...row,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
})));
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
await restoreDefaultPaths();
|
||||
await db('database_backup_runs').del().catch(() => {});
|
||||
await db('app_settings').where('setting_type', 'backup').del().catch(() => {});
|
||||
});
|
||||
|
||||
it('returns the canonical 7 paths + database block on a fresh install', async () => {
|
||||
const res = await request(app).get('/api/admin/system-health/backup-coverage');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toHaveProperty('report');
|
||||
|
||||
const { report } = res.body;
|
||||
expect(report.paths.map((p) => p.path)).toEqual([
|
||||
'events/active',
|
||||
'events/archived',
|
||||
'thumbnails',
|
||||
'previews',
|
||||
'heroes',
|
||||
'uploads',
|
||||
'business-docs',
|
||||
]);
|
||||
|
||||
// Default mode is inline — no inline_dump setting present means
|
||||
// "inline is ON" (matches ensureDatabaseDumpForBackup semantics).
|
||||
expect(report.database.mode).toBe('inline');
|
||||
expect(report.database.ok).toBe(true);
|
||||
|
||||
expect(report.summary).toMatchObject({
|
||||
configuredCount: 7,
|
||||
tableMissingFallbackInUse: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('flips a path to skipped-by-toggle when include_in_default=false', async () => {
|
||||
await db('backup_paths').where('path', 'thumbnails').update({
|
||||
include_in_default: false,
|
||||
});
|
||||
|
||||
const res = await request(app).get('/api/admin/system-health/backup-coverage');
|
||||
const thumbnails = res.body.report.paths.find((p) => p.path === 'thumbnails');
|
||||
expect(thumbnails.coverage).toBe('skipped-by-toggle');
|
||||
expect(thumbnails.includeInDefault).toBe(false);
|
||||
});
|
||||
|
||||
it('feature_flag gating reflects app_settings (archived path off vs on)', async () => {
|
||||
// backup_include_archived not set → archived skipped via flag
|
||||
const off = await request(app).get('/api/admin/system-health/backup-coverage');
|
||||
const archivedOff = off.body.report.paths.find((p) => p.path === 'events/archived');
|
||||
expect(archivedOff.coverage).toBe('skipped-by-feature-flag');
|
||||
expect(archivedOff.featureFlag).toBe('backup_include_archived');
|
||||
expect(archivedOff.featureFlagValue).toBe(null); // unset
|
||||
|
||||
// Now set the flag — but path is missing on disk, so coverage
|
||||
// resolves to 'missing-on-disk', proving the flag was honoured.
|
||||
await db('app_settings').insert({
|
||||
setting_key: 'backup_include_archived',
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'backup',
|
||||
}).onConflict('setting_key').merge();
|
||||
|
||||
const on = await request(app).get('/api/admin/system-health/backup-coverage');
|
||||
const archivedOn = on.body.report.paths.find((p) => p.path === 'events/archived');
|
||||
expect(archivedOn.featureFlagValue).toBe(true);
|
||||
// No on-disk dir → 'missing-on-disk' (not 'skipped-by-feature-flag')
|
||||
expect(['missing-on-disk', 'will-scan']).toContain(archivedOn.coverage);
|
||||
});
|
||||
|
||||
it('detects unconfigured top-level subdirs as drift', async () => {
|
||||
mkdir('events/active'); // configured
|
||||
mkdir('plugin-store/cache'); // DRIFT
|
||||
mkdir('shiny-new-feature/data'); // DRIFT
|
||||
|
||||
const res = await request(app).get('/api/admin/system-health/backup-coverage');
|
||||
expect(res.body.report.drift.unconfiguredOnDisk).toEqual(expect.arrayContaining([
|
||||
'plugin-store',
|
||||
'shiny-new-feature',
|
||||
]));
|
||||
expect(res.body.report.drift.unconfiguredOnDisk).not.toContain('events');
|
||||
|
||||
rmdir('plugin-store');
|
||||
rmdir('shiny-new-feature');
|
||||
});
|
||||
|
||||
it('never flags backups/ or tmp/ as drift (allow-list)', async () => {
|
||||
mkdir('backups');
|
||||
mkdir('tmp');
|
||||
|
||||
const res = await request(app).get('/api/admin/system-health/backup-coverage');
|
||||
expect(res.body.report.drift.unconfiguredOnDisk).not.toContain('backups');
|
||||
expect(res.body.report.drift.unconfiguredOnDisk).not.toContain('tmp');
|
||||
expect(res.body.report.drift.expectedNonBackupDirs).toEqual(
|
||||
expect.arrayContaining(['backups', 'tmp']),
|
||||
);
|
||||
|
||||
rmdir('backups');
|
||||
rmdir('tmp');
|
||||
});
|
||||
|
||||
it('scheduled-only mode + recent dump → database.ok=true, not stale', async () => {
|
||||
await db('app_settings').insert({
|
||||
setting_key: 'backup_database_inline_dump',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'backup',
|
||||
}).onConflict('setting_key').merge();
|
||||
|
||||
const recentDump = path.join(storagePath, 'backups', 'recent.sql.gz');
|
||||
fs.mkdirSync(path.dirname(recentDump), { recursive: true });
|
||||
fs.writeFileSync(recentDump, 'pretend dump');
|
||||
await db('database_backup_runs').insert({
|
||||
started_at: new Date(),
|
||||
completed_at: new Date(), // just now
|
||||
status: 'completed',
|
||||
backup_type: 'pg',
|
||||
file_path: recentDump,
|
||||
file_size_bytes: fs.statSync(recentDump).size,
|
||||
destination_path: recentDump,
|
||||
});
|
||||
|
||||
const res = await request(app).get('/api/admin/system-health/backup-coverage');
|
||||
expect(res.body.report.database.mode).toBe('scheduled-only');
|
||||
expect(res.body.report.database.inlineDumpExplicitlyDisabled).toBe(true);
|
||||
expect(res.body.report.database.lastDumpStale).toBe(false);
|
||||
expect(res.body.report.database.ok).toBe(true);
|
||||
});
|
||||
|
||||
it('scheduled-only mode + stale dump → database.ok=false, lastDumpStale=true', async () => {
|
||||
await db('app_settings').insert({
|
||||
setting_key: 'backup_database_inline_dump',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'backup',
|
||||
}).onConflict('setting_key').merge();
|
||||
|
||||
const oldDump = path.join(storagePath, 'backups', 'old.sql.gz');
|
||||
fs.mkdirSync(path.dirname(oldDump), { recursive: true });
|
||||
fs.writeFileSync(oldDump, 'pretend old dump');
|
||||
// 48 hours ago — well past the 26h staleness threshold. ISO
|
||||
// string instead of a Date object because knex-sqlite's datetime
|
||||
// serialisation has a quirk where some Date instances coerce to
|
||||
// '[object Object]' on insert (the test 6 "recent dump" case
|
||||
// passes only because `new Date()` happens to round-trip safely;
|
||||
// arithmetic Dates don't).
|
||||
const stale = new Date(Date.now() - 48 * 60 * 60 * 1000).toISOString();
|
||||
await db('database_backup_runs').insert({
|
||||
started_at: stale,
|
||||
completed_at: stale,
|
||||
status: 'completed',
|
||||
backup_type: 'pg',
|
||||
file_path: oldDump,
|
||||
file_size_bytes: fs.statSync(oldDump).size,
|
||||
destination_path: oldDump,
|
||||
});
|
||||
|
||||
const res = await request(app).get('/api/admin/system-health/backup-coverage');
|
||||
expect(res.body.report.database.lastDumpStale).toBe(true);
|
||||
expect(res.body.report.database.ok).toBe(false);
|
||||
// Top-level summary reflects the failed DB check.
|
||||
expect(res.body.report.summary.databaseOk).toBe(false);
|
||||
expect(res.body.report.summary.overallOk).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Integration test for GET /api/admin/system-health/backup-integrity.
|
||||
*
|
||||
* Auth + permission middleware are mocked to pass-through so the test
|
||||
* focuses on the route's own behaviour: scope-param validation, the
|
||||
* successResponse envelope, and that the underlying service report
|
||||
* surfaces correctly in the JSON body.
|
||||
*
|
||||
* The verifier service itself is exercised against the real schema
|
||||
* (bootCrmDb) and real filesystem — only the auth gate is stubbed.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const express = require('express');
|
||||
const request = require('supertest');
|
||||
|
||||
const { bootCrmDb, seedMinimal } = require('./helpers/crmDb');
|
||||
|
||||
// Pass-through auth so we don't need to mint JWTs.
|
||||
jest.mock('../../src/middleware/auth', () => ({
|
||||
adminAuth: (req, _res, next) => { req.admin = { id: 1 }; next(); },
|
||||
customerAuth: (_req, _res, next) => next(),
|
||||
galleryAuth: (_req, _res, next) => next(),
|
||||
}));
|
||||
|
||||
// Pass-through permissions so settings.view always allows.
|
||||
jest.mock('../../src/middleware/permissions', () => ({
|
||||
requirePermission: () => (_req, _res, next) => next(),
|
||||
}));
|
||||
|
||||
jest.setTimeout(30000);
|
||||
|
||||
describe('GET /api/admin/system-health/backup-integrity', () => {
|
||||
let cleanup;
|
||||
let db;
|
||||
let customerId;
|
||||
let app;
|
||||
let storagePath;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup } = await bootCrmDb());
|
||||
({ customerId } = await seedMinimal(db));
|
||||
storagePath = process.env.STORAGE_PATH;
|
||||
|
||||
// Mount the route on a minimal Express app. Cold-require after
|
||||
// bootCrmDb so the route's downstream `require('../database/db')`
|
||||
// sees the same db instance.
|
||||
const route = require('../../src/routes/adminSystemHealth');
|
||||
app = express();
|
||||
app.use(express.json());
|
||||
app.use('/api/admin/system-health', route);
|
||||
}, 120000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (cleanup) await cleanup();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await db('contracts').del().catch(() => {});
|
||||
await db('invoices').del().catch(() => {});
|
||||
await db('quotes').del().catch(() => {});
|
||||
});
|
||||
|
||||
it('returns a report envelope when nothing references any path', async () => {
|
||||
const res = await request(app).get('/api/admin/system-health/backup-integrity');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toHaveProperty('report');
|
||||
expect(res.body.report.summary).toMatchObject({
|
||||
totalRows: 0,
|
||||
missingFiles: 0,
|
||||
hashMismatches: 0,
|
||||
verifiedOk: 0,
|
||||
existsButNoHash: 0,
|
||||
});
|
||||
expect(res.body.report.scopes).toEqual(expect.arrayContaining([
|
||||
'quote', 'contract', 'contract-signature', 'invoice',
|
||||
]));
|
||||
});
|
||||
|
||||
it('surfaces a missing file in the response payload', async () => {
|
||||
await db('contracts').insert({
|
||||
customer_account_id: customerId,
|
||||
contract_number: 'C-B7-MISSING',
|
||||
status: 'sent',
|
||||
issue_date: '2026-01-01',
|
||||
signed_pdf_path: 'business-docs/contract/2026/C-B7-MISSING.pdf',
|
||||
created_at: new Date(),
|
||||
});
|
||||
|
||||
const res = await request(app).get('/api/admin/system-health/backup-integrity');
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.report.summary.missingFiles).toBe(1);
|
||||
expect(res.body.report.missing[0]).toMatchObject({
|
||||
table: 'contracts',
|
||||
column: 'signed_pdf_path',
|
||||
expectedPath: 'business-docs/contract/2026/C-B7-MISSING.pdf',
|
||||
});
|
||||
});
|
||||
|
||||
it('honours the ?scope=invoice filter', async () => {
|
||||
// Seed both an invoice and a contract with missing files. With
|
||||
// scope=invoice the contract row must not appear.
|
||||
await db('invoices').insert({
|
||||
customer_account_id: customerId,
|
||||
invoice_number: 'INV-B7-SCOPE',
|
||||
status: 'sent',
|
||||
issue_date: '2026-01-01',
|
||||
due_date: '2026-01-31',
|
||||
pdf_path: 'business-docs/invoice/2026/INV-B7-SCOPE.pdf',
|
||||
created_at: new Date(),
|
||||
});
|
||||
await db('contracts').insert({
|
||||
customer_account_id: customerId,
|
||||
contract_number: 'C-B7-SCOPE',
|
||||
status: 'sent',
|
||||
issue_date: '2026-01-01',
|
||||
signed_pdf_path: 'business-docs/contract/2026/C-B7-SCOPE.pdf',
|
||||
created_at: new Date(),
|
||||
});
|
||||
|
||||
const res = await request(app)
|
||||
.get('/api/admin/system-health/backup-integrity')
|
||||
.query({ scope: 'invoice' });
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.report.scopes).toEqual(['invoice']);
|
||||
expect(res.body.report.missing.every((m) => m.table === 'invoices')).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects an unknown scope with 400 + a code', async () => {
|
||||
const res = await request(app)
|
||||
.get('/api/admin/system-health/backup-integrity')
|
||||
.query({ scope: 'gallery' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(res.body.code).toBe('BACKUP_INTEGRITY_UNKNOWN_SCOPE');
|
||||
expect(res.body.validScopes).toEqual(expect.arrayContaining([
|
||||
'quote', 'contract', 'contract-signature', 'invoice',
|
||||
]));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* Regression net for the business-docs coverage gap fixed in this PR.
|
||||
*
|
||||
* Prior to the fix, `getFilesToBackupInternal()` enumerated a fixed
|
||||
* list of storage subdirectories (events/active, events/archived,
|
||||
* thumbnails, previews, heroes, uploads) and silently omitted the
|
||||
* entire `business-docs/` tree. That meant every CRM PDF + signature
|
||||
* drawing — quotes, contracts (system-rendered + wet uploads),
|
||||
* invoices, Storno, imported historical invoices, and the customer
|
||||
* signature PNG/JPG drawn on the public signing page — fell outside
|
||||
* the in-app scheduled backup, leaving every `*_path` column on
|
||||
* `quotes` / `contracts` / `invoices` as a broken FK after restore.
|
||||
*
|
||||
* The fix is a single `scanDirectory(business-docs, ...)` call. This
|
||||
* suite pins the contract so a future refactor of the walker cannot
|
||||
* silently drop business-docs again.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const { bootCrmDb } = require('./helpers/crmDb');
|
||||
|
||||
describe('backupService — business-docs is in the backup walker', () => {
|
||||
let cleanup;
|
||||
let backupService;
|
||||
let storagePath;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ cleanup } = await bootCrmDb());
|
||||
storagePath = process.env.STORAGE_PATH;
|
||||
// Cold-require after bootCrmDb so backupService picks up the same
|
||||
// db instance + STORAGE_PATH the test harness configured.
|
||||
backupService = require('../../src/services/backupService');
|
||||
}, 120000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (cleanup) await cleanup();
|
||||
});
|
||||
|
||||
function seed(relPath, content = 'dummy bytes for backup test') {
|
||||
const abs = path.join(storagePath, relPath);
|
||||
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
||||
fs.writeFileSync(abs, content);
|
||||
}
|
||||
|
||||
it('does not error when business-docs is absent', async () => {
|
||||
// Fresh harness has no business-docs/ tree at all. The walker
|
||||
// must short-circuit on ENOENT rather than throw — installs that
|
||||
// never used CRM features have to keep backing up fine.
|
||||
await expect(backupService.getFilesToBackup(false)).resolves.toEqual(expect.any(Array));
|
||||
});
|
||||
|
||||
it('picks up every CRM-relevant business-docs subdirectory', async () => {
|
||||
// Seed one file in each of the five subpaths the renderer + import
|
||||
// routes write to. The signature path is the one most prone to be
|
||||
// forgotten — it lives one level deeper than the others (per-
|
||||
// contract subfolder, not per-year).
|
||||
seed('business-docs/quote/2026/Q-001.pdf');
|
||||
seed('business-docs/contract/2026/C-001.pdf');
|
||||
seed('business-docs/contract/signatures/42/customer-1700000000000.png');
|
||||
seed('business-docs/invoice/2026/INV-001.pdf');
|
||||
seed('business-docs/invoice-imports/2026/scan.pdf');
|
||||
|
||||
const files = await backupService.getFilesToBackup(false);
|
||||
const rels = files.map((f) => f.relativePath);
|
||||
|
||||
expect(rels).toEqual(expect.arrayContaining([
|
||||
'business-docs/quote/2026/Q-001.pdf',
|
||||
'business-docs/contract/2026/C-001.pdf',
|
||||
'business-docs/contract/signatures/42/customer-1700000000000.png',
|
||||
'business-docs/invoice/2026/INV-001.pdf',
|
||||
'business-docs/invoice-imports/2026/scan.pdf',
|
||||
]));
|
||||
});
|
||||
|
||||
it('walks newly-created business-docs files without needing a restart', async () => {
|
||||
// The walker reads the filesystem live on every call; this guards
|
||||
// against a future "cache the scan result at boot" optimisation
|
||||
// that would miss freshly-written PDFs (which is exactly what
|
||||
// happens during normal operation — every send writes a new file).
|
||||
seed('business-docs/invoice/2027/INV-NEW.pdf');
|
||||
|
||||
const files = await backupService.getFilesToBackup(false);
|
||||
const rels = files.map((f) => f.relativePath);
|
||||
expect(rels).toContain('business-docs/invoice/2027/INV-NEW.pdf');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* Pins the Stage-B refactor that lifted the file-backup walker's
|
||||
* subdirectory list out of hard-coded JS into the `backup_paths`
|
||||
* table seeded by migration 109.
|
||||
*
|
||||
* Scenarios:
|
||||
* 1. Walker reads canonical seed → all 7 default subdirs walked
|
||||
* 2. include_in_default=false on one row → that subdir is skipped
|
||||
* 3. New row inserted at runtime → walker picks it up without restart
|
||||
* 4. feature_flag gating → row only walked when the named app_settings
|
||||
* boolean is truthy (mirrors historical `includeArchived` behavior)
|
||||
* 5. Empty table → walker falls back to LEGACY_BACKUP_PATHS (defense
|
||||
* in depth — never silently scans nothing)
|
||||
*
|
||||
* Why not stub `db('backup_paths')`: the whole point of Stage B is
|
||||
* that the walker is now data-driven, so the test has to actually
|
||||
* mutate the table and observe the walker's output change. Stubs
|
||||
* would re-introduce the hard-coding the refactor is meant to remove.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const { bootCrmDb } = require('./helpers/crmDb');
|
||||
|
||||
jest.setTimeout(30000);
|
||||
|
||||
describe('backupService — configurable walker (backup_paths)', () => {
|
||||
let db;
|
||||
let cleanup;
|
||||
let storagePath;
|
||||
let backupService;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup } = await bootCrmDb());
|
||||
storagePath = process.env.STORAGE_PATH;
|
||||
backupService = require('../../src/services/backupService');
|
||||
}, 120000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (cleanup) await cleanup();
|
||||
});
|
||||
|
||||
function seedFile(relPath, content = 'dummy bytes') {
|
||||
const abs = path.join(storagePath, relPath);
|
||||
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
||||
fs.writeFileSync(abs, content);
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
// Restore canonical seed before every test. Tests mutate this table
|
||||
// freely; the next test starts from a known state.
|
||||
await db('backup_paths').del();
|
||||
const {
|
||||
DEFAULT_PATHS,
|
||||
} = require('../../migrations/core/109_add_backup_paths');
|
||||
await db('backup_paths').insert(DEFAULT_PATHS.map((row) => ({
|
||||
...row,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
})));
|
||||
});
|
||||
|
||||
it('migration 109 seeds the canonical 7 paths', async () => {
|
||||
const rows = await db('backup_paths').orderBy('display_order', 'asc').select();
|
||||
expect(rows.map((r) => r.path)).toEqual([
|
||||
'events/active',
|
||||
'events/archived',
|
||||
'thumbnails',
|
||||
'previews',
|
||||
'heroes',
|
||||
'uploads',
|
||||
'business-docs',
|
||||
]);
|
||||
// Only events/archived is gated by a feature flag.
|
||||
expect(rows.filter((r) => r.feature_flag).map((r) => r.path)).toEqual([
|
||||
'events/archived',
|
||||
]);
|
||||
});
|
||||
|
||||
it('walks every default subdir when files are present', async () => {
|
||||
seedFile('events/active/E1/a.jpg');
|
||||
seedFile('thumbnails/E1/a.jpg');
|
||||
seedFile('previews/E1/a.jpg');
|
||||
seedFile('heroes/E1/hero.jpg');
|
||||
seedFile('uploads/intake/x.bin');
|
||||
seedFile('business-docs/quote/2026/Q-001.pdf');
|
||||
// events/archived is gated — left out of this test; covered below.
|
||||
|
||||
const files = await backupService.getFilesToBackup({ backup_include_archived: true });
|
||||
const rels = files.map((f) => f.relativePath);
|
||||
|
||||
expect(rels).toEqual(expect.arrayContaining([
|
||||
'events/active/E1/a.jpg',
|
||||
'thumbnails/E1/a.jpg',
|
||||
'previews/E1/a.jpg',
|
||||
'heroes/E1/hero.jpg',
|
||||
'uploads/intake/x.bin',
|
||||
'business-docs/quote/2026/Q-001.pdf',
|
||||
]));
|
||||
});
|
||||
|
||||
it('skips a path when include_in_default is toggled off', async () => {
|
||||
seedFile('thumbnails/E1/thumb.jpg');
|
||||
seedFile('events/active/E1/photo.jpg');
|
||||
|
||||
await db('backup_paths').where('path', 'thumbnails').update({
|
||||
include_in_default: false,
|
||||
});
|
||||
|
||||
const files = await backupService.getFilesToBackup({ backup_include_archived: true });
|
||||
const rels = files.map((f) => f.relativePath);
|
||||
|
||||
expect(rels).toContain('events/active/E1/photo.jpg');
|
||||
expect(rels).not.toContain('thumbnails/E1/thumb.jpg');
|
||||
});
|
||||
|
||||
it('picks up a new path inserted at runtime — no restart needed', async () => {
|
||||
// Simulates a future feature shipping its own subdirectory and
|
||||
// self-healing a `backup_paths` row at boot.
|
||||
await db('backup_paths').insert({
|
||||
path: 'plugin-store',
|
||||
include_in_default: true,
|
||||
feature_flag: null,
|
||||
display_order: 200,
|
||||
description: 'Hypothetical future feature payload',
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
});
|
||||
seedFile('plugin-store/cache/payload.bin');
|
||||
|
||||
const files = await backupService.getFilesToBackup({ backup_include_archived: true });
|
||||
const rels = files.map((f) => f.relativePath);
|
||||
|
||||
expect(rels).toContain('plugin-store/cache/payload.bin');
|
||||
});
|
||||
|
||||
it('respects feature_flag gating (events/archived ⇄ backup_include_archived)', async () => {
|
||||
seedFile('events/active/E1/active.jpg');
|
||||
seedFile('events/archived/E2/archived.jpg');
|
||||
|
||||
// backup_include_archived=false → archived/ is skipped.
|
||||
const filesOff = await backupService.getFilesToBackup({ backup_include_archived: false });
|
||||
const relsOff = filesOff.map((f) => f.relativePath);
|
||||
expect(relsOff).toContain('events/active/E1/active.jpg');
|
||||
expect(relsOff).not.toContain('events/archived/E2/archived.jpg');
|
||||
|
||||
// backup_include_archived=true → archived/ is included.
|
||||
const filesOn = await backupService.getFilesToBackup({ backup_include_archived: true });
|
||||
const relsOn = filesOn.map((f) => f.relativePath);
|
||||
expect(relsOn).toContain('events/archived/E2/archived.jpg');
|
||||
});
|
||||
|
||||
it('falls back to LEGACY_BACKUP_PATHS when the table is empty', async () => {
|
||||
// Defense in depth: even if seed-and-self-heal both failed, the
|
||||
// walker must still cover the historical set so "Run Backup Now"
|
||||
// cannot silently degrade to no-op.
|
||||
await db('backup_paths').del();
|
||||
seedFile('events/active/E1/photo.jpg');
|
||||
seedFile('business-docs/quote/2026/Q-002.pdf');
|
||||
|
||||
const files = await backupService.getFilesToBackup({ backup_include_archived: true });
|
||||
const rels = files.map((f) => f.relativePath);
|
||||
|
||||
expect(rels).toContain('events/active/E1/photo.jpg');
|
||||
expect(rels).toContain('business-docs/quote/2026/Q-002.pdf');
|
||||
});
|
||||
|
||||
it('legacy boolean call signature still works (backward compat)', async () => {
|
||||
// Existing call sites (and the businessDocs regression test) pass
|
||||
// a boolean for `includeArchived`. Refactor must not break them.
|
||||
seedFile('events/archived/E3/legacy.jpg');
|
||||
|
||||
const filesOff = await backupService.getFilesToBackup(false);
|
||||
expect(filesOff.map((f) => f.relativePath)).not.toContain('events/archived/E3/legacy.jpg');
|
||||
|
||||
const filesOn = await backupService.getFilesToBackup(true);
|
||||
expect(filesOn.map((f) => f.relativePath)).toContain('events/archived/E3/legacy.jpg');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,188 @@
|
||||
/**
|
||||
* Pins the inline-DB-dump + fail-loud guard added to `runBackupInternal`.
|
||||
*
|
||||
* The previous behaviour was: file-backup looked up an existing dump via
|
||||
* `getDatabaseBackupInfo()` and silently shipped a files-only manifest
|
||||
* when none was found. Admins clicking "Run Backup Now" got an apparent
|
||||
* success that omitted every customer / quote / invoice / contract row —
|
||||
* the data-loss footgun that this commit closes.
|
||||
*
|
||||
* Five scenarios under test:
|
||||
* 1. Default (inline dump enabled), dump succeeds → backup proceeds
|
||||
* 2. Default, dump throws → run aborts, backup_runs row marked failed
|
||||
* 3. Opt-out + recent DB dump available → backup proceeds
|
||||
* 4. Opt-out + no DB dump available → fail loud
|
||||
* 5. Opt-out + DB dump file is 0 bytes on disk → fail loud
|
||||
*
|
||||
* Mocking strategy: the underlying `databaseBackupService.backup()` and
|
||||
* the local-destination writer are stubbed so the test exercises just
|
||||
* the new guard logic without depending on `pg_dump` / `sqlite3` CLI
|
||||
* binaries being available in the test environment.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const { bootCrmDb } = require('./helpers/crmDb');
|
||||
|
||||
// Set up mocks BEFORE bootCrmDb so backupService picks them up at require time.
|
||||
const mockBackupFn = jest.fn();
|
||||
jest.mock('../../src/services/databaseBackup', () => ({
|
||||
databaseBackupService: { backup: mockBackupFn },
|
||||
startScheduledBackups: jest.fn(),
|
||||
stopScheduledBackups: jest.fn(),
|
||||
DatabaseBackupService: class {},
|
||||
}));
|
||||
|
||||
jest.setTimeout(30000);
|
||||
|
||||
describe('backupService — inline DB dump + fail-loud guard', () => {
|
||||
let db;
|
||||
let cleanup;
|
||||
let storagePath;
|
||||
let backupService;
|
||||
let dumpFileAbs;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup } = await bootCrmDb());
|
||||
storagePath = process.env.STORAGE_PATH;
|
||||
backupService = require('../../src/services/backupService');
|
||||
|
||||
// Seed backup destination settings so the run can proceed past the
|
||||
// "destination not configured" guard.
|
||||
const dest = path.join(storagePath, 'backups');
|
||||
fs.mkdirSync(dest, { recursive: true });
|
||||
// getBackupConfigInternal filters by setting_type='backup', so the
|
||||
// tests have to seed with that type or the resolver returns
|
||||
// `{ ... }` with the keys missing — runBackup then sees
|
||||
// `backup_destination_type === undefined` and bails before our
|
||||
// new guard runs.
|
||||
await db('app_settings').insert([
|
||||
{ setting_key: 'backup_destination_type', setting_value: JSON.stringify('local'), setting_type: 'backup' },
|
||||
{ setting_key: 'backup_destination_path', setting_value: JSON.stringify(dest), setting_type: 'backup' },
|
||||
{ setting_key: 'backup_enabled', setting_value: JSON.stringify(true), setting_type: 'backup' },
|
||||
{ setting_key: 'backup_email_on_failure', setting_value: JSON.stringify(false), setting_type: 'backup' },
|
||||
]).onConflict('setting_key').merge();
|
||||
|
||||
// Pre-create a dump file that getDatabaseBackupInfo can resolve to.
|
||||
// Reused/mutated per-test via the database_backup_runs seed below.
|
||||
dumpFileAbs = path.join(storagePath, 'backups', 'fake-dump.sql.gz');
|
||||
fs.writeFileSync(dumpFileAbs, 'pretend this is a pg_dump'.repeat(100));
|
||||
|
||||
// Neutralise the file-scan step: we don't care which files would
|
||||
// be backed up, just whether the run reaches that stage at all.
|
||||
backupService.getFilesToBackup = jest.fn(async () => []);
|
||||
}, 120000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (cleanup) await cleanup();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
mockBackupFn.mockReset();
|
||||
// Default to "dump produced this file with this size" — the per-test
|
||||
// setup overrides as needed.
|
||||
mockBackupFn.mockResolvedValue({
|
||||
success: true,
|
||||
path: dumpFileAbs,
|
||||
size: fs.statSync(dumpFileAbs).size,
|
||||
duration: 1,
|
||||
checksum: 'abc',
|
||||
});
|
||||
|
||||
// Re-seed the database_backup_runs row that getDatabaseBackupInfo
|
||||
// resolves against (its query is `status='completed'` + most recent).
|
||||
await db('database_backup_runs').del();
|
||||
await db('database_backup_runs').insert({
|
||||
started_at: new Date(),
|
||||
completed_at: new Date(),
|
||||
status: 'completed',
|
||||
backup_type: 'pg',
|
||||
file_path: dumpFileAbs,
|
||||
file_size_bytes: fs.statSync(dumpFileAbs).size,
|
||||
destination_path: dumpFileAbs,
|
||||
});
|
||||
});
|
||||
|
||||
it('default behaviour: inline dump runs, then file backup proceeds', async () => {
|
||||
// Inline-dump setting is unset (undefined) — default is ON.
|
||||
await db('app_settings').where('setting_key', 'backup_database_inline_dump').del();
|
||||
|
||||
await backupService.runBackup(true);
|
||||
|
||||
expect(mockBackupFn).toHaveBeenCalledTimes(1);
|
||||
|
||||
const run = await db('backup_runs').orderBy('id', 'desc').first();
|
||||
expect(run.status).toBe('completed');
|
||||
expect(run.error_message).toBeNull();
|
||||
});
|
||||
|
||||
it('aborts the run when the inline dump throws', async () => {
|
||||
await db('app_settings').where('setting_key', 'backup_database_inline_dump').del();
|
||||
mockBackupFn.mockRejectedValueOnce(new Error('pg_dump segfaulted'));
|
||||
|
||||
await backupService.runBackup(true);
|
||||
|
||||
const run = await db('backup_runs').orderBy('id', 'desc').first();
|
||||
expect(run.status).toBe('failed');
|
||||
expect(run.error_message).toMatch(/pg_dump segfaulted/);
|
||||
});
|
||||
|
||||
it('opt-out: skips inline dump but proceeds when a recent dump exists', async () => {
|
||||
await db('app_settings').insert({
|
||||
setting_key: 'backup_database_inline_dump',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'backup',
|
||||
}).onConflict('setting_key').merge();
|
||||
|
||||
await backupService.runBackup(true);
|
||||
|
||||
expect(mockBackupFn).not.toHaveBeenCalled();
|
||||
|
||||
const run = await db('backup_runs').orderBy('id', 'desc').first();
|
||||
expect(run.status).toBe('completed');
|
||||
});
|
||||
|
||||
it('opt-out + no recent dump: fails loud with a clear error', async () => {
|
||||
await db('app_settings').insert({
|
||||
setting_key: 'backup_database_inline_dump',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'backup',
|
||||
}).onConflict('setting_key').merge();
|
||||
// Wipe the dump row so getDatabaseBackupInfo returns backupFile=null.
|
||||
await db('database_backup_runs').del();
|
||||
|
||||
await backupService.runBackup(true);
|
||||
|
||||
const run = await db('backup_runs').orderBy('id', 'desc').first();
|
||||
expect(run.status).toBe('failed');
|
||||
expect(run.error_message).toMatch(/No database backup available/);
|
||||
});
|
||||
|
||||
it('opt-out + 0-byte dump file: fails loud', async () => {
|
||||
await db('app_settings').insert({
|
||||
setting_key: 'backup_database_inline_dump',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'backup',
|
||||
}).onConflict('setting_key').merge();
|
||||
|
||||
const emptyDump = path.join(storagePath, 'backups', 'empty-dump.sql.gz');
|
||||
fs.writeFileSync(emptyDump, '');
|
||||
await db('database_backup_runs').del();
|
||||
await db('database_backup_runs').insert({
|
||||
started_at: new Date(),
|
||||
completed_at: new Date(),
|
||||
status: 'completed',
|
||||
backup_type: 'pg',
|
||||
file_path: emptyDump,
|
||||
file_size_bytes: 0,
|
||||
destination_path: emptyDump,
|
||||
});
|
||||
|
||||
await backupService.runBackup(true);
|
||||
|
||||
const run = await db('backup_runs').orderBy('id', 'desc').first();
|
||||
expect(run.status).toBe('failed');
|
||||
expect(run.error_message).toMatch(/is empty/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* Per-Stage-B-path tally — Tier 3 of tonight's backup hardening.
|
||||
*
|
||||
* Pins the new `computePerPathStats` logic that the Backup History
|
||||
* "Content Backed Up" pane reads via `backup_runs.statistics.per_path`.
|
||||
*
|
||||
* Three scenarios:
|
||||
* 1. Single file under one path — straightforward attribution
|
||||
* 2. Multiple paths with overlapping prefixes — longest-prefix wins
|
||||
* (e.g. `events/active/E1/x.jpg` should attribute to
|
||||
* `events/active`, not `events`)
|
||||
* 3. File outside any configured path — silently dropped, doesn't
|
||||
* throw or contaminate other buckets
|
||||
*
|
||||
* Tests exercise the EXPORTED side: write a backup_runs row via the
|
||||
* service entry point and assert the statistics JSON shape. We don't
|
||||
* stub `computePerPathStats` directly — the integration view is what
|
||||
* the frontend actually consumes.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const { bootCrmDb } = require('./helpers/crmDb');
|
||||
|
||||
jest.setTimeout(30000);
|
||||
|
||||
describe('backupService — per-Stage-B-path statistics', () => {
|
||||
let db;
|
||||
let cleanup;
|
||||
let storagePath;
|
||||
let backupService;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup } = await bootCrmDb());
|
||||
storagePath = process.env.STORAGE_PATH;
|
||||
backupService = require('../../src/services/backupService');
|
||||
}, 120000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (cleanup) await cleanup();
|
||||
});
|
||||
|
||||
function mkFile(rel, content = 'x'.repeat(100)) {
|
||||
const abs = path.join(storagePath, rel);
|
||||
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
||||
fs.writeFileSync(abs, content);
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean slate of any artefacts from prior tests
|
||||
await db('backup_runs').del();
|
||||
await db('app_settings').where('setting_type', 'backup').del();
|
||||
await db('app_settings').insert([
|
||||
{ setting_key: 'backup_destination_type', setting_value: JSON.stringify('local'), setting_type: 'backup' },
|
||||
{ setting_key: 'backup_destination_path', setting_value: JSON.stringify(path.join(storagePath, 'destination')), setting_type: 'backup' },
|
||||
{ setting_key: 'backup_enabled', setting_value: JSON.stringify(true), setting_type: 'backup' },
|
||||
{ setting_key: 'backup_email_on_failure', setting_value: JSON.stringify(false), setting_type: 'backup' },
|
||||
{ setting_key: 'backup_include_archived', setting_value: JSON.stringify(true), setting_type: 'backup' },
|
||||
]).onConflict('setting_key').merge();
|
||||
fs.mkdirSync(path.join(storagePath, 'destination'), { recursive: true });
|
||||
|
||||
// Restore canonical backup_paths from migration 109
|
||||
const { DEFAULT_PATHS } = require('../../migrations/core/109_add_backup_paths');
|
||||
await db('backup_paths').del();
|
||||
await db('backup_paths').insert(DEFAULT_PATHS.map((row) => ({
|
||||
...row,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
})));
|
||||
|
||||
// Wipe leftover files between tests
|
||||
for (const dir of ['events', 'business-docs', 'thumbnails', 'previews', 'heroes', 'uploads']) {
|
||||
const p = path.join(storagePath, dir);
|
||||
if (fs.existsSync(p)) fs.rmSync(p, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('attributes files to their owning backup_paths row', async () => {
|
||||
mkFile('events/active/E1/photo-a.jpg', 'X'.repeat(1000));
|
||||
mkFile('events/active/E1/photo-b.jpg', 'X'.repeat(2000));
|
||||
mkFile('business-docs/quote/2026/Q-1.pdf', 'X'.repeat(500));
|
||||
mkFile('thumbnails/E1/photo-a.jpg', 'X'.repeat(50));
|
||||
|
||||
// Disable the inline DB dump so we don't need pg_dump in tests;
|
||||
// the file walker is what produces per_path.
|
||||
await db('app_settings').insert({
|
||||
setting_key: 'backup_database_inline_dump',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'backup',
|
||||
}).onConflict('setting_key').merge();
|
||||
|
||||
// Seed a fake DB-backup row so the fail-loud guard is satisfied.
|
||||
const fakeDump = path.join(storagePath, 'destination', 'fake.sql.gz');
|
||||
fs.writeFileSync(fakeDump, 'pretend dump');
|
||||
await db('database_backup_runs').insert({
|
||||
started_at: new Date(),
|
||||
completed_at: new Date(),
|
||||
status: 'completed',
|
||||
backup_type: 'pg',
|
||||
file_path: fakeDump,
|
||||
file_size_bytes: fs.statSync(fakeDump).size,
|
||||
destination_path: fakeDump,
|
||||
});
|
||||
|
||||
await backupService.runBackup(true);
|
||||
|
||||
const run = await db('backup_runs').orderBy('id', 'desc').first();
|
||||
expect(run.status).toBe('completed');
|
||||
|
||||
const statsRaw = typeof run.statistics === 'string'
|
||||
? JSON.parse(run.statistics)
|
||||
: run.statistics;
|
||||
expect(statsRaw.per_path).toBeDefined();
|
||||
|
||||
// events/active should have 2 files (3000 bytes)
|
||||
expect(statsRaw.per_path['events/active']).toEqual({ count: 2, size: 3000 });
|
||||
// business-docs should have 1 file (500 bytes)
|
||||
expect(statsRaw.per_path['business-docs']).toEqual({ count: 1, size: 500 });
|
||||
// thumbnails should have 1 file (50 bytes)
|
||||
expect(statsRaw.per_path['thumbnails']).toEqual({ count: 1, size: 50 });
|
||||
|
||||
// No spurious buckets for paths that had nothing
|
||||
expect(statsRaw.per_path['previews']).toBeUndefined();
|
||||
expect(statsRaw.per_path['heroes']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('archived path attributed separately from active when both have files', async () => {
|
||||
mkFile('events/active/E1/active.jpg', 'X'.repeat(100));
|
||||
mkFile('events/archived/E2/archived.jpg', 'X'.repeat(200));
|
||||
|
||||
// backup_include_archived already set true in beforeEach so the
|
||||
// archived walker fires; same opt-out for inline DB dump.
|
||||
await db('app_settings').insert({
|
||||
setting_key: 'backup_database_inline_dump',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'backup',
|
||||
}).onConflict('setting_key').merge();
|
||||
const fakeDump = path.join(storagePath, 'destination', 'fake.sql.gz');
|
||||
fs.writeFileSync(fakeDump, 'pretend dump');
|
||||
await db('database_backup_runs').insert({
|
||||
started_at: new Date(),
|
||||
completed_at: new Date(),
|
||||
status: 'completed',
|
||||
backup_type: 'pg',
|
||||
file_path: fakeDump,
|
||||
file_size_bytes: fs.statSync(fakeDump).size,
|
||||
destination_path: fakeDump,
|
||||
});
|
||||
|
||||
await backupService.runBackup(true);
|
||||
|
||||
const run = await db('backup_runs').orderBy('id', 'desc').first();
|
||||
const statsRaw = typeof run.statistics === 'string'
|
||||
? JSON.parse(run.statistics)
|
||||
: run.statistics;
|
||||
|
||||
// events/active and events/archived attribute separately —
|
||||
// longest-prefix match prevents `events/active/...` from claiming
|
||||
// an `events/archived/...` file or vice versa.
|
||||
expect(statsRaw.per_path['events/active']).toEqual({ count: 1, size: 100 });
|
||||
expect(statsRaw.per_path['events/archived']).toEqual({ count: 1, size: 200 });
|
||||
});
|
||||
});
|
||||
|
||||
// NOTE on walker duplication
|
||||
//
|
||||
// If two `backup_paths` rows overlap (e.g. one row at `events` AND
|
||||
// another at `events/active`), the walker scans the same files twice
|
||||
// — once via each path. Per-path stats then attribute the file to the
|
||||
// longest-prefix-matching path BOTH times, producing inflated counts.
|
||||
//
|
||||
// The canonical seed in migration 109 contains no overlapping pairs,
|
||||
// so this isn't exercised in practice. But an admin who hand-adds a
|
||||
// broad row that overlaps an existing nested one will see double
|
||||
// counts in their next backup's statistics + the destination will
|
||||
// receive duplicate copies (wasting space). Worth flagging if anyone
|
||||
// reports it — the fix is to de-dupe `files` in
|
||||
// `getFilesToBackupInternal` before returning, OR to skip walking a
|
||||
// path if a longer one has already covered it.
|
||||
@@ -0,0 +1,214 @@
|
||||
/**
|
||||
* Install-from-backup boot hook — pins the trigger-file convention.
|
||||
*
|
||||
* The hook itself depends on `restoreService.restore`, which is hard
|
||||
* to fully exercise in an integration test without a real PG cluster
|
||||
* (sequence resync, DROP/CREATE, etc.). So we stub the actual restore
|
||||
* and verify the BOOT HOOK logic:
|
||||
*
|
||||
* - No trigger file → no-op, ran=false
|
||||
* - Empty trigger file → picks newest manifest from manifests/
|
||||
* - Non-empty trigger file → uses the path inside
|
||||
* - DB not empty → refuses (no restore call)
|
||||
* - DB not empty + FORCE env → proceeds
|
||||
* - Successful restore → deletes trigger file
|
||||
* - Failed restore → leaves trigger file in place
|
||||
*
|
||||
* These are the surfaces an admin will hit when actually using the
|
||||
* feature — the docker-compose-on-real-PG end-to-end test belongs in
|
||||
* the follow-up CI work captured as task #7 earlier today.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const { bootCrmDb } = require('./helpers/crmDb');
|
||||
|
||||
// Stub the heavy lifting so the test stays fast + portable.
|
||||
const mockRestore = jest.fn();
|
||||
jest.mock('../../src/services/restoreService', () => ({
|
||||
restoreService: {
|
||||
restore: (...args) => mockRestore(...args),
|
||||
},
|
||||
}));
|
||||
|
||||
jest.setTimeout(30000);
|
||||
|
||||
describe('installFromBackupBoot', () => {
|
||||
let db;
|
||||
let cleanup;
|
||||
let storagePath;
|
||||
let backupRoot;
|
||||
let manifestsDir;
|
||||
let tryInstallFromBackup;
|
||||
let originalBackupRootEnv;
|
||||
let originalForceEnv;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup } = await bootCrmDb());
|
||||
storagePath = process.env.STORAGE_PATH;
|
||||
backupRoot = path.join(storagePath, 'backup');
|
||||
manifestsDir = path.join(backupRoot, 'manifests');
|
||||
fs.mkdirSync(manifestsDir, { recursive: true });
|
||||
|
||||
originalBackupRootEnv = process.env.BACKUP_ROOT;
|
||||
originalForceEnv = process.env.INSTALL_FROM_BACKUP_FORCE;
|
||||
process.env.BACKUP_ROOT = backupRoot;
|
||||
|
||||
({ tryInstallFromBackup } = require('../../src/services/_installFromBackupBoot'));
|
||||
}, 120000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (originalBackupRootEnv === undefined) {
|
||||
delete process.env.BACKUP_ROOT;
|
||||
} else {
|
||||
process.env.BACKUP_ROOT = originalBackupRootEnv;
|
||||
}
|
||||
if (originalForceEnv === undefined) {
|
||||
delete process.env.INSTALL_FROM_BACKUP_FORCE;
|
||||
} else {
|
||||
process.env.INSTALL_FROM_BACKUP_FORCE = originalForceEnv;
|
||||
}
|
||||
if (cleanup) await cleanup();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
mockRestore.mockReset();
|
||||
mockRestore.mockResolvedValue({ success: true });
|
||||
delete process.env.INSTALL_FROM_BACKUP_FORCE;
|
||||
|
||||
// Clean trigger files + manifests between tests
|
||||
for (const name of ['RESTORE_ON_INSTALL', 'RESTORE_ON_INSTALL.txt']) {
|
||||
const p = path.join(backupRoot, name);
|
||||
if (fs.existsSync(p)) fs.unlinkSync(p);
|
||||
}
|
||||
for (const f of fs.readdirSync(manifestsDir)) {
|
||||
fs.unlinkSync(path.join(manifestsDir, f));
|
||||
}
|
||||
|
||||
// Reset DB to fresh-install state
|
||||
await db('events').del();
|
||||
// Leave admin_users alone — fresh-install state has 1 row.
|
||||
});
|
||||
|
||||
it('no trigger file → no-op', async () => {
|
||||
const result = await tryInstallFromBackup(db);
|
||||
expect(result.ran).toBe(false);
|
||||
expect(mockRestore).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('empty trigger file picks the newest manifest from manifests/', async () => {
|
||||
const older = path.join(manifestsDir, 'backup-manifest-001.json');
|
||||
const newer = path.join(manifestsDir, 'backup-manifest-002.json');
|
||||
fs.writeFileSync(older, '{}');
|
||||
// Set the newer file's mtime slightly later so it wins the sort
|
||||
const past = new Date(Date.now() - 60_000);
|
||||
fs.utimesSync(older, past, past);
|
||||
fs.writeFileSync(newer, '{}');
|
||||
|
||||
// Empty trigger
|
||||
fs.writeFileSync(path.join(backupRoot, 'RESTORE_ON_INSTALL'), '');
|
||||
|
||||
const result = await tryInstallFromBackup(db);
|
||||
expect(result.ran).toBe(true);
|
||||
expect(result.manifestPath).toBe(newer);
|
||||
expect(mockRestore).toHaveBeenCalledWith(expect.objectContaining({
|
||||
source: 'local',
|
||||
manifestPath: newer,
|
||||
restoreType: 'full',
|
||||
force: true,
|
||||
skipPreBackup: true,
|
||||
}));
|
||||
});
|
||||
|
||||
it('non-empty trigger file uses the path inside', async () => {
|
||||
const specific = path.join(manifestsDir, 'backup-manifest-specific.json');
|
||||
fs.writeFileSync(specific, '{}');
|
||||
|
||||
// Relative to backupRoot
|
||||
fs.writeFileSync(
|
||||
path.join(backupRoot, 'RESTORE_ON_INSTALL'),
|
||||
'manifests/backup-manifest-specific.json\n',
|
||||
);
|
||||
|
||||
const result = await tryInstallFromBackup(db);
|
||||
expect(result.ran).toBe(true);
|
||||
expect(result.manifestPath).toBe(specific);
|
||||
});
|
||||
|
||||
it('deletes the trigger file after a successful restore', async () => {
|
||||
const manifest = path.join(manifestsDir, 'backup-manifest-001.json');
|
||||
fs.writeFileSync(manifest, '{}');
|
||||
const triggerPath = path.join(backupRoot, 'RESTORE_ON_INSTALL');
|
||||
fs.writeFileSync(triggerPath, '');
|
||||
|
||||
await tryInstallFromBackup(db);
|
||||
expect(fs.existsSync(triggerPath)).toBe(false);
|
||||
});
|
||||
|
||||
it('leaves the trigger file in place when restore throws', async () => {
|
||||
mockRestore.mockRejectedValueOnce(new Error('restore exploded'));
|
||||
const manifest = path.join(manifestsDir, 'backup-manifest-001.json');
|
||||
fs.writeFileSync(manifest, '{}');
|
||||
const triggerPath = path.join(backupRoot, 'RESTORE_ON_INSTALL');
|
||||
fs.writeFileSync(triggerPath, '');
|
||||
|
||||
const result = await tryInstallFromBackup(db);
|
||||
expect(result.ran).toBe(false);
|
||||
expect(result.error).toMatch(/restore exploded/);
|
||||
expect(fs.existsSync(triggerPath)).toBe(true);
|
||||
});
|
||||
|
||||
it('refuses to run when the install already has events', async () => {
|
||||
// Simulate an install with existing data
|
||||
await db('events').insert({
|
||||
slug: 'existing-event',
|
||||
event_name: 'Existing Event',
|
||||
event_type: 'wedding',
|
||||
event_date: new Date(),
|
||||
host_email: '[email protected]',
|
||||
admin_email: '[email protected]',
|
||||
expires_at: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000),
|
||||
share_link: 'existing-event-token',
|
||||
password_hash: 'dummy-hash-for-test',
|
||||
created_at: new Date(),
|
||||
});
|
||||
|
||||
const manifest = path.join(manifestsDir, 'backup-manifest-001.json');
|
||||
fs.writeFileSync(manifest, '{}');
|
||||
fs.writeFileSync(path.join(backupRoot, 'RESTORE_ON_INSTALL'), '');
|
||||
|
||||
const result = await tryInstallFromBackup(db);
|
||||
expect(result.ran).toBe(false);
|
||||
expect(result.error).toMatch(/Database not empty/);
|
||||
expect(mockRestore).not.toHaveBeenCalled();
|
||||
|
||||
// Trigger file should NOT be deleted — admin needs to fix + retry
|
||||
expect(fs.existsSync(path.join(backupRoot, 'RESTORE_ON_INSTALL'))).toBe(true);
|
||||
});
|
||||
|
||||
it('proceeds when INSTALL_FROM_BACKUP_FORCE=true even with existing data', async () => {
|
||||
await db('events').insert({
|
||||
slug: 'existing-event-2',
|
||||
event_name: 'Existing Event 2',
|
||||
event_type: 'wedding',
|
||||
event_date: new Date(),
|
||||
host_email: '[email protected]',
|
||||
admin_email: '[email protected]',
|
||||
expires_at: new Date(Date.now() + 7 * 24 * 60 * 60 * 1000),
|
||||
share_link: 'existing-event-2-token',
|
||||
password_hash: 'dummy-hash-for-test-2',
|
||||
created_at: new Date(),
|
||||
});
|
||||
|
||||
const manifest = path.join(manifestsDir, 'backup-manifest-001.json');
|
||||
fs.writeFileSync(manifest, '{}');
|
||||
fs.writeFileSync(path.join(backupRoot, 'RESTORE_ON_INSTALL'), '');
|
||||
|
||||
process.env.INSTALL_FROM_BACKUP_FORCE = 'true';
|
||||
const result = await tryInstallFromBackup(db);
|
||||
|
||||
expect(result.ran).toBe(true);
|
||||
expect(mockRestore).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,259 @@
|
||||
/**
|
||||
* Pins the fix for the PR #596 review blocker.
|
||||
*
|
||||
* **The bug**
|
||||
*
|
||||
* `preservedMeta` was declared with `let` INSIDE the PostgreSQL
|
||||
* `else` branch of `performDatabaseRestore`, then read AFTER the
|
||||
* `else` block closed at the shared replay site (~L1030). On every
|
||||
* real PG restore:
|
||||
*
|
||||
* ReferenceError: preservedMeta is not defined
|
||||
*
|
||||
* would fire — psql had already completed the data restore, but
|
||||
* the operator-meta replay never ran, the trigger file was left
|
||||
* in place by `_installFromBackupBoot.js` because the restore
|
||||
* "failed", and `combined.log` got a loud FAILED line even though
|
||||
* the data was back. Caught on PR #596 review by the maintainer.
|
||||
*
|
||||
* **Why CI missed it**
|
||||
*
|
||||
* The integration tests around `performFullRestore` only exercise
|
||||
* the SQLite branch via `this.dbType === 'sqlite'`. The PG branch
|
||||
* (~L827-984) requires a real PG connection + real `psql` binary,
|
||||
* neither of which are in the test environment. So the scope leak
|
||||
* sat untested until the maintainer ran a real DR cycle.
|
||||
*
|
||||
* **What this test does**
|
||||
*
|
||||
* Reads the source of `restoreService.js` and asserts the scope
|
||||
* contract: the `preservedMeta` declaration sits ABOVE the
|
||||
* SQLite/PG branch split, so the replay block at the bottom of the
|
||||
* try{} can read it on either branch.
|
||||
*
|
||||
* Source-inspection is uglier than a runtime test but it has two
|
||||
* advantages here: (a) it doesn't require a real PG cluster + psql
|
||||
* binary in CI, (b) it pins the EXACT contract — "the declaration
|
||||
* must be visible to the replay block" — which is the property
|
||||
* that broke, more directly than a runtime test would.
|
||||
*
|
||||
* The follow-up "real-PG integration test in CI" (separate task)
|
||||
* would replace this with an end-to-end exercise, at which point
|
||||
* this can be deleted.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
describe('restoreService — PG branch scope contract (PR #596 review)', () => {
|
||||
let src;
|
||||
let lines;
|
||||
|
||||
beforeAll(() => {
|
||||
src = fs.readFileSync(
|
||||
path.join(__dirname, '..', '..', 'src', 'services', 'restoreService.js'),
|
||||
'utf8',
|
||||
);
|
||||
lines = src.split(/\r?\n/);
|
||||
});
|
||||
|
||||
/** Return the 1-based line number of the FIRST line matching `re`. */
|
||||
function findFirst(re) {
|
||||
const idx = lines.findIndex((l) => re.test(l));
|
||||
return idx >= 0 ? idx + 1 : -1;
|
||||
}
|
||||
|
||||
/** Return the 1-based line number of the LAST line matching `re`. */
|
||||
function findLast(re) {
|
||||
let last = -1;
|
||||
lines.forEach((l, i) => { if (re.test(l)) last = i + 1; });
|
||||
return last;
|
||||
}
|
||||
|
||||
it('preservedMetaSnapshot lives on `this` and is initialised in the constructor', () => {
|
||||
// PR #596 round 3 moved the snapshot from a block-scoped local to
|
||||
// an instance variable so the replay can happen in `restore()`
|
||||
// AFTER post-restore verification — preventing the replay row
|
||||
// from inflating the row-count check.
|
||||
//
|
||||
// Contract:
|
||||
// 1. The constructor initialises `this.preservedMetaSnapshot = []`
|
||||
// 2. The `restore()` entry point resets it per call (no leak
|
||||
// across consecutive runs in the singleton service instance)
|
||||
// 3. `performDatabaseRestore` assigns to `this.preservedMetaSnapshot`
|
||||
// inside the PG branch (must run before DROP)
|
||||
// 4. The replay reads `this.preservedMetaSnapshot` — NOT a bare
|
||||
// `preservedMeta` local — so a future refactor can't
|
||||
// accidentally drop the snapshot half on the floor again.
|
||||
const constructorInit = lines.some((l) =>
|
||||
/this\.preservedMetaSnapshot\s*=\s*\[\s*\]/.test(l)
|
||||
);
|
||||
expect(constructorInit).toBe(true);
|
||||
|
||||
const assignmentSites = lines.filter((l) =>
|
||||
/this\.preservedMetaSnapshot\s*=\s*(\[\s*\]|await\s+db)/.test(l)
|
||||
);
|
||||
// Constructor init + restore() per-run reset + the PG-branch
|
||||
// assignment from db query. Three writes.
|
||||
expect(assignmentSites.length).toBeGreaterThanOrEqual(3);
|
||||
|
||||
// No stray bare `preservedMeta` local-scoped declaration in
|
||||
// performDatabaseRestore — would indicate someone re-introduced
|
||||
// the round-1 footgun.
|
||||
const dangerousLocalDecl = lines.filter((l) =>
|
||||
/^\s*(let|const)\s+preservedMeta\s*=/.test(l)
|
||||
);
|
||||
expect(dangerousLocalDecl).toEqual([]);
|
||||
});
|
||||
|
||||
it('every .count() result is coerced to Number before comparison', () => {
|
||||
// PR #596 review caught a second PG-only landmine: pg-driver
|
||||
// returns COUNT(*) as a string ("16" not 16) to preserve bigint
|
||||
// precision. The original code compared `result.count !==
|
||||
// expected.rowCount` and every match flagged as a mismatch on PG.
|
||||
//
|
||||
// The fix coerces with `Number(...)` at every comparison +
|
||||
// interpolation site. This test catches a future regression where
|
||||
// a refactor uses `.count` directly in a `===` / `!==` / `>` /
|
||||
// `<` comparison without coercing.
|
||||
//
|
||||
// Heuristic: find every `.count` access in the file and make sure
|
||||
// the line either:
|
||||
// (a) wraps it in `Number(...)`, or
|
||||
// (b) is purely an interpolation that already coerced upstream
|
||||
// (e.g. `validation.warnings.push(`... ${eventCountN} ...`)`
|
||||
// where eventCountN is the coerced local), or
|
||||
// (c) is the docstring/comment line (filtered separately).
|
||||
//
|
||||
// We approximate this by listing every `.count` reference site
|
||||
// and asserting that lines doing comparisons (`===`/`!==`/`>`/
|
||||
// `<`/`>=`/`<=`) on a raw `.count` access without `Number(...)`
|
||||
// around it are zero.
|
||||
const dangerousLines = lines
|
||||
.map((l, i) => ({ line: i + 1, text: l }))
|
||||
// Filter to lines that compare a .count result
|
||||
.filter(({ text }) => {
|
||||
// Skip comments
|
||||
if (/^\s*(\/\/|\*)/.test(text)) return false;
|
||||
// Detect a `.count` (followed by `)` for `?.count` or by space/operator)
|
||||
// being directly compared via ===/!==/>/<.
|
||||
// Match the BAD pattern: `<something>.count <op> <something>`
|
||||
// where <op> is === / !== / > / < / >= / <=
|
||||
const bareCountInComparison = /\w+\??\.count\s*(?:!==|===|>=?|<=?)\s+/;
|
||||
// ALLOW if the .count is preceded by `Number(` in the same line
|
||||
const wrappedInNumber = /Number\(\s*\w+\??\.count/;
|
||||
return bareCountInComparison.test(text) && !wrappedInNumber.test(text);
|
||||
});
|
||||
|
||||
expect(dangerousLines).toEqual([]);
|
||||
});
|
||||
|
||||
it('the completed-restore update sets was_successful=true', () => {
|
||||
// Without this, every successful restore ends up with
|
||||
// status='completed', was_successful=false — the dashboard's
|
||||
// "last successful restore" widget then filters out the row +
|
||||
// any future audit query gating on was_successful misses it.
|
||||
// Caught locally + maintainer PR #596 review.
|
||||
//
|
||||
// Contract: the update payload that writes status='completed' on
|
||||
// the SUCCESS branch ALSO includes was_successful: true. We pin
|
||||
// it by source inspection so any future refactor of the success
|
||||
// payload keeps both fields together.
|
||||
// The success-branch update lives AFTER performPostRestoreVerification.
|
||||
// There's also a `status: 'completed'` in the dry-run / early-return
|
||||
// path (failure handling has its own block too) — we want the
|
||||
// SUCCESS-branch one specifically.
|
||||
const verifyLine = findFirst(/performPostRestoreVerification\s*\(/);
|
||||
expect(verifyLine).toBeGreaterThan(0);
|
||||
|
||||
const completedStatusLineIdx = lines
|
||||
.map((l, i) => ({ line: i + 1, text: l }))
|
||||
.find(({ line, text }) =>
|
||||
line > verifyLine && /status:\s*['"]completed['"]/.test(text)
|
||||
);
|
||||
expect(completedStatusLineIdx).toBeDefined();
|
||||
|
||||
// Look in the next ~10 lines for was_successful: true. The actual
|
||||
// payload is small (no nested objects between status and the
|
||||
// closing })), so a fixed-window search is reliable.
|
||||
const window = lines.slice(
|
||||
completedStatusLineIdx.line - 1,
|
||||
completedStatusLineIdx.line + 10,
|
||||
).join('\n');
|
||||
expect(window).toMatch(/was_successful:\s*true/);
|
||||
});
|
||||
|
||||
it('npm run migrate:safe is invoked after the replay in restore()', () => {
|
||||
// Contract from PR #596 round 4: backups taken on older picpeak
|
||||
// versions must restore COMPLETELY on a newer image — even if new
|
||||
// migrations have been added since the backup was taken. The
|
||||
// restore() flow shells out to `npm run migrate:safe` AFTER the
|
||||
// operator-meta replay so the schema catches up to the running
|
||||
// code WITHIN the restore boundary (not on the next container
|
||||
// restart).
|
||||
//
|
||||
// Contract:
|
||||
// 1. A `migrate:safe` shell-out exists somewhere in restoreService
|
||||
// 2. It sits AFTER the replay drain — verification → replay →
|
||||
// migrations is the documented order
|
||||
// 3. It does NOT sit inside performDatabaseRestore (must run
|
||||
// against the reinit'd pool from the parent restore())
|
||||
const migrateLine = findFirst(/['"]migrate:safe['"]/);
|
||||
expect(migrateLine).toBeGreaterThan(0);
|
||||
|
||||
const replayLine = findLast(/this\.preservedMetaSnapshot\.length\s*>\s*0/);
|
||||
expect(replayLine).toBeGreaterThan(0);
|
||||
expect(migrateLine).toBeGreaterThan(replayLine);
|
||||
|
||||
// Must NOT live inside performDatabaseRestore (same scope as the
|
||||
// replay check above).
|
||||
const dbRestoreStart = findFirst(/async\s+performDatabaseRestore\s*\(/);
|
||||
let dbRestoreEnd = -1;
|
||||
for (let i = dbRestoreStart; i < lines.length; i++) {
|
||||
if (/^ \}\s*$/.test(lines[i])) {
|
||||
dbRestoreEnd = i + 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
expect(migrateLine < dbRestoreStart || migrateLine > dbRestoreEnd).toBe(true);
|
||||
});
|
||||
|
||||
it('the replay site lives in restore() AFTER performPostRestoreVerification', () => {
|
||||
// PR #596 round 3 moved the replay out of performDatabaseRestore
|
||||
// and into the parent restore() method, sequenced AFTER the
|
||||
// post-restore verification. Otherwise the replay's upserted row
|
||||
// count was being flagged as a verification mismatch (e.g.
|
||||
// "expected 190, got 191" because the fresh-install seeded
|
||||
// `restore_allow_force_auto_upgraded` that wasn't in the backup).
|
||||
//
|
||||
// Contract: the line that drains `this.preservedMetaSnapshot`
|
||||
// must come AFTER `performPostRestoreVerification` AND must NOT
|
||||
// sit inside `performDatabaseRestore`.
|
||||
const verificationLine = findFirst(/performPostRestoreVerification\s*\(/);
|
||||
expect(verificationLine).toBeGreaterThan(0);
|
||||
|
||||
const replayLine = findLast(/this\.preservedMetaSnapshot\.length\s*>\s*0/);
|
||||
expect(replayLine).toBeGreaterThan(0);
|
||||
expect(replayLine).toBeGreaterThan(verificationLine);
|
||||
|
||||
// `performDatabaseRestore` must not contain the replay drain.
|
||||
// Find the function bounds + assert no drain line falls inside.
|
||||
const dbRestoreStart = findFirst(/async\s+performDatabaseRestore\s*\(/);
|
||||
expect(dbRestoreStart).toBeGreaterThan(0);
|
||||
|
||||
// Find the closing brace of performDatabaseRestore. Lazy heuristic:
|
||||
// the first `^ \}\s*$` (two-space indent + }) after the function
|
||||
// start. Brittle to indent changes but unambiguous in this codebase.
|
||||
let dbRestoreEnd = -1;
|
||||
for (let i = dbRestoreStart; i < lines.length; i++) {
|
||||
if (/^ \}\s*$/.test(lines[i])) {
|
||||
dbRestoreEnd = i + 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
expect(dbRestoreEnd).toBeGreaterThan(dbRestoreStart);
|
||||
|
||||
// The replay drain line must be OUTSIDE [dbRestoreStart, dbRestoreEnd].
|
||||
expect(replayLine < dbRestoreStart || replayLine > dbRestoreEnd).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,219 @@
|
||||
/**
|
||||
* Verifies the backup-integrity check covers every CRM document
|
||||
* artefact column and correctly buckets each row into:
|
||||
* - verifiedOk — file exists AND hash matches (when hash is stored)
|
||||
* - missing — `*_path` set but file is not on disk
|
||||
* - hashMismatches — file exists but bytes don't hash to `*_sha256`
|
||||
* - existsButNoHash — file exists, no `*_sha256` column for this row
|
||||
*
|
||||
* Uses the CRM integration harness (bootCrmDb) so the schema +
|
||||
* STORAGE_PATH wiring exactly mirrors production behaviour.
|
||||
*
|
||||
* Background: this service is the diagnostic for the
|
||||
* `storage/business-docs/` gap fixed in the same PR — without it,
|
||||
* a restored install would have audit-trail columns referencing
|
||||
* files that no longer exist, but admins would have no way to see
|
||||
* the breakage until a customer asked for their contract back.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
|
||||
const { bootCrmDb, seedMinimal } = require('../integration/helpers/crmDb');
|
||||
|
||||
jest.setTimeout(30000);
|
||||
|
||||
describe('backupIntegrityService.verifyDocumentArtefacts', () => {
|
||||
let db;
|
||||
let cleanup;
|
||||
let customerId;
|
||||
let storagePath;
|
||||
let backupIntegrityService;
|
||||
|
||||
function seedFile(relPath, content) {
|
||||
const abs = path.join(storagePath, relPath);
|
||||
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
||||
fs.writeFileSync(abs, content);
|
||||
return { abs, relPath, sha: sha256(content) };
|
||||
}
|
||||
|
||||
function sha256(content) {
|
||||
return crypto.createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup } = await bootCrmDb());
|
||||
({ customerId } = await seedMinimal(db));
|
||||
storagePath = process.env.STORAGE_PATH;
|
||||
backupIntegrityService = require('../../src/services/backupIntegrityService');
|
||||
}, 120000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (cleanup) await cleanup();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Wipe CRM rows between tests so each scenario sees a clean slate.
|
||||
// Order matters: child tables before parents.
|
||||
await db('invoice_line_items').del().catch(() => {});
|
||||
await db('invoice_payment_log').del().catch(() => {});
|
||||
await db('invoices').del().catch(() => {});
|
||||
await db('quote_line_items').del().catch(() => {});
|
||||
await db('quotes').del().catch(() => {});
|
||||
await db('contracts').del().catch(() => {});
|
||||
});
|
||||
|
||||
it('returns an empty report when no documents reference any path', async () => {
|
||||
const report = await backupIntegrityService.verifyDocumentArtefacts();
|
||||
expect(report.summary.totalRows).toBe(0);
|
||||
expect(report.summary.verifiedOk).toBe(0);
|
||||
expect(report.missing).toEqual([]);
|
||||
expect(report.hashMismatches).toEqual([]);
|
||||
expect(report.existsButNoHash).toEqual([]);
|
||||
expect(report.scannedAt).toMatch(/^\d{4}-\d{2}-\d{2}T/);
|
||||
expect(report.scopes).toEqual(expect.arrayContaining(['quote', 'contract', 'contract-signature', 'invoice']));
|
||||
});
|
||||
|
||||
it('flags a contract whose signed_pdf_path file is missing', async () => {
|
||||
// Reference a file that we deliberately never create on disk.
|
||||
// knex's `.returning('id')` returns `[{ id: N }]` on Postgres and
|
||||
// newer SQLite, but `[N]` (plain int) on some SQLite versions —
|
||||
// unwrap both shapes the same way the crmDb test harness does.
|
||||
const inserted = await db('contracts').insert({
|
||||
customer_account_id: customerId,
|
||||
contract_number: 'C-2026-MISSING',
|
||||
status: 'sent',
|
||||
issue_date: '2026-01-01',
|
||||
signed_pdf_path: 'business-docs/contract/2026/C-2026-MISSING.pdf',
|
||||
created_at: new Date(),
|
||||
}).returning('id');
|
||||
const contractId = typeof inserted[0] === 'object' ? inserted[0].id : inserted[0];
|
||||
|
||||
const report = await backupIntegrityService.verifyDocumentArtefacts({ scope: ['contract'] });
|
||||
const hit = report.missing.find((m) => m.rowId === contractId);
|
||||
expect(hit).toMatchObject({
|
||||
table: 'contracts',
|
||||
column: 'signed_pdf_path',
|
||||
expectedPath: 'business-docs/contract/2026/C-2026-MISSING.pdf',
|
||||
});
|
||||
expect(report.summary.missingFiles).toBe(1);
|
||||
});
|
||||
|
||||
it('verifies a contract whose file exists AND hash matches', async () => {
|
||||
const { relPath, sha } = seedFile(
|
||||
'business-docs/contract/2026/C-2026-OK.pdf',
|
||||
'this is the signed contract content',
|
||||
);
|
||||
await db('contracts').insert({
|
||||
customer_account_id: customerId,
|
||||
contract_number: 'C-2026-OK',
|
||||
status: 'fully_signed',
|
||||
issue_date: '2026-01-01',
|
||||
signed_pdf_path: relPath,
|
||||
signed_pdf_sha256: sha,
|
||||
created_at: new Date(),
|
||||
});
|
||||
|
||||
const report = await backupIntegrityService.verifyDocumentArtefacts({ scope: ['contract'] });
|
||||
expect(report.summary.verifiedOk).toBeGreaterThanOrEqual(1);
|
||||
expect(report.summary.missingFiles).toBe(0);
|
||||
expect(report.summary.hashMismatches).toBe(0);
|
||||
});
|
||||
|
||||
it('flags a hash mismatch when the file exists but bytes differ from signed_pdf_sha256', async () => {
|
||||
const { relPath } = seedFile(
|
||||
'business-docs/contract/2026/C-2026-TAMPER.pdf',
|
||||
'tampered bytes on disk',
|
||||
);
|
||||
const inserted = await db('contracts').insert({
|
||||
customer_account_id: customerId,
|
||||
contract_number: 'C-2026-TAMPER',
|
||||
status: 'fully_signed',
|
||||
issue_date: '2026-01-01',
|
||||
signed_pdf_path: relPath,
|
||||
// Hash for completely different content — simulates tampering or
|
||||
// bit-rot between sign-time and now.
|
||||
signed_pdf_sha256: sha256('the ORIGINAL bytes the customer signed'),
|
||||
created_at: new Date(),
|
||||
}).returning('id');
|
||||
const contractId = typeof inserted[0] === 'object' ? inserted[0].id : inserted[0];
|
||||
|
||||
const report = await backupIntegrityService.verifyDocumentArtefacts({ scope: ['contract'] });
|
||||
const hit = report.hashMismatches.find((m) => m.rowId === contractId);
|
||||
expect(hit).toBeDefined();
|
||||
expect(hit.expectedSha).not.toBe(hit.actualSha);
|
||||
expect(hit.column).toBe('signed_pdf_path');
|
||||
});
|
||||
|
||||
it('buckets signature PNGs into existsButNoHash (no hash column)', async () => {
|
||||
const { relPath } = seedFile(
|
||||
'business-docs/contract/signatures/99/customer-1700000000000.png',
|
||||
'\x89PNG\r\n\x1a\n', // doesn't have to be a real PNG, just bytes
|
||||
);
|
||||
await db('contracts').insert({
|
||||
customer_account_id: customerId,
|
||||
contract_number: 'C-2026-SIG',
|
||||
status: 'fully_signed',
|
||||
issue_date: '2026-01-01',
|
||||
signed_customer_signature_path: relPath,
|
||||
created_at: new Date(),
|
||||
});
|
||||
|
||||
const report = await backupIntegrityService.verifyDocumentArtefacts({
|
||||
scope: ['contract-signature'],
|
||||
});
|
||||
expect(report.summary.existsButNoHash).toBeGreaterThanOrEqual(1);
|
||||
expect(report.summary.verifiedOk).toBe(0); // no hash → not "verified ok"
|
||||
expect(report.summary.missingFiles).toBe(0);
|
||||
const hit = report.existsButNoHash.find((r) => r.column === 'signed_customer_signature_path');
|
||||
expect(hit).toBeDefined();
|
||||
});
|
||||
|
||||
it('respects the scope filter — contract scope skips quote/invoice tables', async () => {
|
||||
// Seed an invoice with a missing pdf_path AND a contract with a
|
||||
// missing signed_pdf_path. Scoping to contract should only flag
|
||||
// the contract.
|
||||
await db('invoices').insert({
|
||||
customer_account_id: customerId,
|
||||
invoice_number: 'INV-2026-SCOPE',
|
||||
status: 'sent',
|
||||
pdf_path: 'business-docs/invoice/2026/INV-2026-SCOPE.pdf',
|
||||
issue_date: '2026-01-01',
|
||||
due_date: '2026-01-31',
|
||||
created_at: new Date(),
|
||||
});
|
||||
await db('contracts').insert({
|
||||
customer_account_id: customerId,
|
||||
contract_number: 'C-2026-SCOPE',
|
||||
status: 'sent',
|
||||
issue_date: '2026-01-01',
|
||||
signed_pdf_path: 'business-docs/contract/2026/C-2026-SCOPE.pdf',
|
||||
created_at: new Date(),
|
||||
});
|
||||
|
||||
const report = await backupIntegrityService.verifyDocumentArtefacts({ scope: ['contract'] });
|
||||
expect(report.scopes).toEqual(['contract']);
|
||||
expect(report.missing.every((m) => m.table === 'contracts')).toBe(true);
|
||||
expect(report.missing.some((m) => m.table === 'invoices')).toBe(false);
|
||||
});
|
||||
|
||||
it('covers invoices.imported_pdf_path (admin-uploaded historical scans)', async () => {
|
||||
// Imported invoices are the most catastrophic case — there's no
|
||||
// renderer that can reproduce them. Verifier must check this column
|
||||
// alongside invoices.pdf_path.
|
||||
await db('invoices').insert({
|
||||
customer_account_id: customerId,
|
||||
invoice_number: 'IMP-2025-001',
|
||||
status: 'sent',
|
||||
imported_pdf_path: 'business-docs/invoice-imports/2025/legacy.pdf',
|
||||
issue_date: '2025-06-01',
|
||||
due_date: '2025-07-01',
|
||||
created_at: new Date(),
|
||||
});
|
||||
|
||||
const report = await backupIntegrityService.verifyDocumentArtefacts({ scope: ['invoice'] });
|
||||
const hit = report.missing.find((m) => m.column === 'imported_pdf_path');
|
||||
expect(hit).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -33,11 +33,34 @@ exports.up = async function(knex) {
|
||||
// Try to save credentials to file, but don't fail if we can't
|
||||
const dataDir = path.join(__dirname, '..', '..', 'data');
|
||||
const setupInfoPath = path.join(dataDir, 'ADMIN_CREDENTIALS.txt');
|
||||
|
||||
|
||||
// Detect a pending install-from-backup trigger. If one exists,
|
||||
// these credentials are about to be obsoleted by the restore —
|
||||
// the backup's admin row replaces this fresh-install one a few
|
||||
// seconds from now. We still write the file (in case the
|
||||
// restore fails and the fresh admin is the only way in) but
|
||||
// annotate the top so admins reading the file after restore
|
||||
// don't waste time trying credentials that no longer exist.
|
||||
// Flagged on PR #596 review.
|
||||
const fsSync = require('fs');
|
||||
const backupRoot = process.env.BACKUP_ROOT || '/backup';
|
||||
const triggerWillFire = fsSync.existsSync(path.join(backupRoot, 'RESTORE_ON_INSTALL'))
|
||||
|| fsSync.existsSync(path.join(backupRoot, 'RESTORE_ON_INSTALL.txt'));
|
||||
const restoreNotice = triggerWillFire ? `
|
||||
|
||||
⚠️ RESTORE_ON_INSTALL TRIGGER DETECTED ⚠️
|
||||
These credentials are temporary. An install-from-backup run is queued
|
||||
to fire on the next server start, which will REPLACE this admin row
|
||||
with the one from the backup. After the restore completes, log in
|
||||
with your ORIGINAL pre-disaster credentials — not the ones below.
|
||||
If the restore fails for some reason, the credentials below remain
|
||||
valid as a fallback recovery path.
|
||||
` : '';
|
||||
|
||||
const setupInfo = `
|
||||
========================================
|
||||
PicPeak Admin Credentials
|
||||
========================================
|
||||
========================================${restoreNotice}
|
||||
|
||||
Your admin account has been created with these credentials:
|
||||
|
||||
|
||||
@@ -100,8 +100,18 @@ exports.up = async function(knex) {
|
||||
// Add restore-related settings to app_settings
|
||||
const restoreSettings = [
|
||||
{
|
||||
// Default ON so fresh installs can recover from disaster
|
||||
// without the catch-22 documented in _restoreSettingsBoot.js
|
||||
// (fresh-install admin user trips the "1 active admin" warning,
|
||||
// which can only be overridden with force=true, which the wizard
|
||||
// refused if this setting was false — exactly the moment an
|
||||
// admin can least afford a SQL incantation). Flipped from false
|
||||
// to true 2026-05-30. Existing installs that ran this migration
|
||||
// with the OLD value will get auto-upgraded once by the boot
|
||||
// self-heal in _restoreSettingsBoot.js — see the
|
||||
// `restore_allow_force_auto_upgraded` guard there.
|
||||
setting_key: 'restore_allow_force',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'restore'
|
||||
},
|
||||
{
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Migration 109 — config-driven backup walker.
|
||||
*
|
||||
* (Originally numbered 108 on bugfix/crm-backup. Renumbered to 109
|
||||
* before merge because upstream/beta independently shipped
|
||||
* 108_seed_sl_email_template_translations.js. The createTable is
|
||||
* idempotent via `hasTable` guard and the seed uses
|
||||
* `onConflict('path').ignore()`, so beta installs that ran the
|
||||
* 108-named version of this file get a harmless no-op when 109
|
||||
* runs against the already-seeded table.)
|
||||
*
|
||||
* Stage B of the three-stage backup-hardening plan. The file-backup
|
||||
* walker (`getFilesToBackupInternal` in backupService.js) historically
|
||||
* hard-coded its list of subdirectories: events/active, events/archived,
|
||||
* thumbnails, previews, heroes, uploads, business-docs.
|
||||
*
|
||||
* That list is a footgun every time a new feature lands that drops
|
||||
* artefacts under STORAGE_PATH/<something>/ — the maintainer has to
|
||||
* remember to edit the walker, and there's no schema-level record of
|
||||
* what *should* be backed up. The CRM rollout missed `business-docs`
|
||||
* for ~6 months (#XXX) for exactly this reason.
|
||||
*
|
||||
* This migration introduces a `backup_paths` table that the walker
|
||||
* reads at runtime. New features add a row; the walker picks them up
|
||||
* automatically. The `feature_flag` column gates scans behind an
|
||||
* existing app_settings boolean (e.g. `backup_include_archived`),
|
||||
* mirroring how the previous `includeArchived` parameter worked.
|
||||
*
|
||||
* Columns:
|
||||
* - path : relative to STORAGE_PATH, unique
|
||||
* - include_in_default : on/off without deleting the row (so
|
||||
* audit trail of "we used to back this up"
|
||||
* is preserved)
|
||||
* - feature_flag : nullable; when set, walker checks the
|
||||
* same-named app_settings boolean before
|
||||
* scanning. Matches the existing pattern
|
||||
* used by `backup_include_archived`.
|
||||
* - display_order : controls admin-UI listing order
|
||||
* - description : human-readable purpose, shown in admin UI
|
||||
*
|
||||
* Defense-in-depth: the walker also keeps a hard-coded LEGACY_DEFAULTS
|
||||
* fallback so that if this table is somehow empty (failed migration on
|
||||
* an existing install, manual truncation), backups still cover the
|
||||
* historical set instead of silently shipping nothing. The boot-time
|
||||
* self-heal in `_backupPathsBoot.js` re-seeds missing default rows on
|
||||
* every startup so newly-added defaults reach already-deployed
|
||||
* installs without a follow-up migration.
|
||||
*
|
||||
* Idempotent: skips the createTable if it already exists, and the
|
||||
* seed uses `onConflict('path').ignore()` so re-runs don't duplicate.
|
||||
*/
|
||||
|
||||
const DEFAULT_PATHS = [
|
||||
{
|
||||
path: 'events/active',
|
||||
include_in_default: true,
|
||||
feature_flag: null,
|
||||
display_order: 10,
|
||||
description: 'Active gallery photo originals',
|
||||
},
|
||||
{
|
||||
path: 'events/archived',
|
||||
include_in_default: true,
|
||||
feature_flag: 'backup_include_archived',
|
||||
display_order: 20,
|
||||
description: 'Archived gallery photo originals (gated by backup_include_archived)',
|
||||
},
|
||||
{
|
||||
path: 'thumbnails',
|
||||
include_in_default: true,
|
||||
feature_flag: null,
|
||||
display_order: 30,
|
||||
description: 'Generated gallery thumbnails',
|
||||
},
|
||||
{
|
||||
path: 'previews',
|
||||
include_in_default: true,
|
||||
feature_flag: null,
|
||||
display_order: 40,
|
||||
description: 'Lightbox preview tier (#492)',
|
||||
},
|
||||
{
|
||||
path: 'heroes',
|
||||
include_in_default: true,
|
||||
feature_flag: null,
|
||||
display_order: 50,
|
||||
description: 'Gallery hero header images',
|
||||
},
|
||||
{
|
||||
path: 'uploads',
|
||||
include_in_default: true,
|
||||
feature_flag: null,
|
||||
display_order: 60,
|
||||
description: 'Direct uploads root (wet-signature contracts, imported invoices, etc.)',
|
||||
},
|
||||
{
|
||||
path: 'business-docs',
|
||||
include_in_default: true,
|
||||
feature_flag: null,
|
||||
display_order: 70,
|
||||
description: 'CRM PDFs, signature artefacts, admin-imported historical invoices',
|
||||
},
|
||||
];
|
||||
|
||||
exports.up = async function(knex) {
|
||||
const exists = await knex.schema.hasTable('backup_paths');
|
||||
if (!exists) {
|
||||
await knex.schema.createTable('backup_paths', (t) => {
|
||||
t.increments('id').primary();
|
||||
t.string('path', 256).notNullable().unique();
|
||||
t.boolean('include_in_default').notNullable().defaultTo(true);
|
||||
t.string('feature_flag', 64).nullable();
|
||||
t.integer('display_order').notNullable().defaultTo(100);
|
||||
t.string('description', 256).nullable();
|
||||
t.timestamp('created_at').notNullable().defaultTo(knex.fn.now());
|
||||
t.timestamp('updated_at').notNullable().defaultTo(knex.fn.now());
|
||||
});
|
||||
}
|
||||
|
||||
// Seed defaults — `onConflict('path').ignore()` so already-seeded rows
|
||||
// (manual edits by admins, prior partial runs) survive untouched.
|
||||
await knex('backup_paths')
|
||||
.insert(DEFAULT_PATHS.map((row) => ({
|
||||
...row,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
})))
|
||||
.onConflict('path')
|
||||
.ignore();
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.dropTableIfExists('backup_paths');
|
||||
};
|
||||
|
||||
// Exported so the self-heal boot helper can reuse the same authoritative
|
||||
// list without re-declaring it. Tests also import this to assert the
|
||||
// walker is reading from this source.
|
||||
exports.DEFAULT_PATHS = DEFAULT_PATHS;
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "picpeak-backend",
|
||||
"version": "3.59.1-beta.0",
|
||||
"version": "3.60.1-beta.0",
|
||||
"description": "Backend for PicPeak event photo sharing platform",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
|
||||
@@ -660,6 +660,7 @@ app.use('/api/admin/contracts', require('./src/routes/adminContracts'));
|
||||
app.use('/api/admin/calendar', require('./src/routes/adminCalendar'));
|
||||
app.use('/api/admin/deals', require('./src/routes/adminDeals'));
|
||||
app.use('/api/admin/tax-report', require('./src/routes/adminTaxReport'));
|
||||
app.use('/api/admin/system-health', require('./src/routes/adminSystemHealth'));
|
||||
app.use('/api/admin/dev', require('./src/routes/adminDev'));
|
||||
app.use('/api/public/quotes', require('./src/routes/publicQuotes'));
|
||||
app.use('/api/public/contracts', require('./src/routes/publicContracts'));
|
||||
@@ -799,6 +800,48 @@ async function startServer() {
|
||||
const { startS3AutoImporter } = require('./src/services/s3AutoImporter');
|
||||
startS3AutoImporter();
|
||||
|
||||
// Self-heal the `backup_paths` table before the backup service
|
||||
// starts — the file-backup walker reads from it, so missing
|
||||
// canonical rows (a new subdirectory shipped by a future feature)
|
||||
// get re-seeded here on every boot. See _backupPathsBoot.js for
|
||||
// the full rationale; pattern mirrors _emailTemplateBoot.js.
|
||||
try {
|
||||
const { seedBackupPathsAtBoot } = require('./src/services/_backupPathsBoot');
|
||||
await seedBackupPathsAtBoot(db, logger);
|
||||
} catch (err) {
|
||||
logger.warn('backup_paths self-heal failed at boot:', err.message);
|
||||
}
|
||||
|
||||
// Self-heal restore-meta settings — currently just
|
||||
// `restore_allow_force` defaulting to ON so fresh installs can
|
||||
// recover from disaster without a SQL incantation. Only seeds on
|
||||
// FRESH installs (existing rows, true or false, are preserved).
|
||||
// See _restoreSettingsBoot.js for the full rationale.
|
||||
try {
|
||||
const { seedRestoreSettingsAtBoot } = require('./src/services/_restoreSettingsBoot');
|
||||
await seedRestoreSettingsAtBoot(db, logger);
|
||||
} catch (err) {
|
||||
logger.warn('restore-settings self-heal failed at boot:', err.message);
|
||||
}
|
||||
|
||||
// Install-from-backup trigger. If `RESTORE_ON_INSTALL` (or
|
||||
// `.txt`) exists in the /backup mount AND the DB is empty, run
|
||||
// the restore HERE before any admin UI surfaces. Lets admins
|
||||
// recover a picpeak install with: (a) place backup files in the
|
||||
// bind mount, (b) drop the trigger file, (c) `docker compose up`.
|
||||
// No onboarding wizard, no throwaway admin, no compose-file
|
||||
// changes. See _installFromBackupBoot.js for the full rationale
|
||||
// + the safety gates.
|
||||
try {
|
||||
const { tryInstallFromBackup } = require('./src/services/_installFromBackupBoot');
|
||||
const result = await tryInstallFromBackup(db, logger);
|
||||
if (result.ran) {
|
||||
logger.info(`Install-from-backup: completed from ${result.manifestPath}. Server will start with restored state.`);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('Install-from-backup hook threw:', err.message);
|
||||
}
|
||||
|
||||
// Start backup service
|
||||
await startBackupService();
|
||||
|
||||
|
||||
@@ -24,8 +24,73 @@ try {
|
||||
try { logger.warn('SQLite directory ensure failed', { error: e.message }); } catch (_) {}
|
||||
}
|
||||
|
||||
// Create database connection with built-in retry logic
|
||||
const db = knex(knexConfig);
|
||||
// Create database connection with built-in retry logic.
|
||||
//
|
||||
// The underlying knex instance is held in `_db` and reachable through a
|
||||
// Proxy `db` that forwards every call to the current instance. This
|
||||
// indirection exists so `reinitPool()` below can swap the live pool
|
||||
// without breaking the thousands of existing `const { db } = require(...)`
|
||||
// imports — they capture the Proxy once, and every subsequent
|
||||
// `db('table')` / `db.schema.hasTable(...)` lookup goes through the
|
||||
// Proxy to whatever `_db` currently points at.
|
||||
//
|
||||
// Used by the restore service after DROP/CREATE DATABASE: the old pool
|
||||
// is destroyed during the drop (to release PG connections so the DROP
|
||||
// can succeed), then `reinitPool()` opens a fresh pool against the
|
||||
// recreated DB. Without this, every query in the process after a
|
||||
// restore failed with "Unable to acquire a connection" until the
|
||||
// container was manually restarted — exactly the footgun Ralf hit
|
||||
// repeatedly on 2026-05-30.
|
||||
let _db = knex(knexConfig);
|
||||
|
||||
const db = new Proxy(function knexCall() {}, {
|
||||
// db('tableName') — knex's query builder entry point
|
||||
apply(_target, _thisArg, args) {
|
||||
return _db(...args);
|
||||
},
|
||||
// db.schema, db.raw, db.migrate, etc.
|
||||
get(_target, prop) {
|
||||
const v = _db[prop];
|
||||
return typeof v === 'function' ? v.bind(_db) : v;
|
||||
},
|
||||
// Defensive: future code that does `if ('schema' in db)` works.
|
||||
has(_target, prop) { return prop in _db; },
|
||||
});
|
||||
|
||||
/**
|
||||
* Tear down the current knex pool and open a fresh one.
|
||||
*
|
||||
* Idempotent — calling it twice in a row just destroys + recreates
|
||||
* twice, no error. Throws if the new pool can't establish a
|
||||
* connection (which surfaces a clear error rather than letting the
|
||||
* caller proceed with a half-broken pool).
|
||||
*
|
||||
* Used by restoreService after the DROP/CREATE DATABASE pair.
|
||||
*/
|
||||
async function reinitPool() {
|
||||
const prev = _db;
|
||||
try {
|
||||
await prev.destroy();
|
||||
} catch (err) {
|
||||
logger.warn(`Old pool destroy failed (continuing with reinit): ${err.message}`);
|
||||
}
|
||||
_db = knex(knexConfig);
|
||||
// Probe the new pool with a no-op query so we fail loudly here if
|
||||
// the new pool can't connect — better than silently handing the
|
||||
// caller a broken pool and surfacing the error on the next admin
|
||||
// request.
|
||||
try {
|
||||
if (knexConfig.client === 'pg') {
|
||||
await _db.raw('SELECT 1');
|
||||
} else {
|
||||
await _db.raw('SELECT 1');
|
||||
}
|
||||
} catch (probeErr) {
|
||||
logger.error('New pool failed health-check after reinit', { error: probeErr.message });
|
||||
throw probeErr;
|
||||
}
|
||||
logger.info('knex pool re-initialized successfully');
|
||||
}
|
||||
|
||||
// Connection retry configuration
|
||||
const MAX_RETRIES = 3;
|
||||
@@ -607,4 +672,4 @@ async function logActivity(activityType, metadata = {}, eventId = null, actor =
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { db, initializeDatabase, logActivity, withRetry };
|
||||
module.exports = { db, initializeDatabase, logActivity, withRetry, reinitPool };
|
||||
|
||||
@@ -13,7 +13,18 @@ let cachedTimeout = null;
|
||||
let cacheExpiry = 0;
|
||||
const CACHE_DURATION = 30 * 60 * 1000; // 30 minutes - reduced DB queries
|
||||
|
||||
// Clean up expired sessions every 5 minutes
|
||||
// Clean up expired sessions every 5 minutes.
|
||||
//
|
||||
// `.unref()` so this timer doesn't keep the event loop alive on its
|
||||
// own — without it, every jest worker that requires this module
|
||||
// (directly or transitively via server.js / a middleware-importing
|
||||
// route file) gets stuck and either prints the "worker failed to
|
||||
// exit gracefully" warning or, under high CI load, force-kills mid-
|
||||
// test and takes an unrelated suite down with it (we hit this with
|
||||
// integration/storageBackend.test.js on PR #555). Production
|
||||
// behaviour is unchanged: the timer fires every 5 min as long as
|
||||
// the server has anything else keeping the loop alive (HTTP server,
|
||||
// other intervals), which is always.
|
||||
setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const [token, lastActivity] of sessions.entries()) {
|
||||
@@ -21,7 +32,7 @@ setInterval(() => {
|
||||
sessions.delete(token);
|
||||
}
|
||||
}
|
||||
}, 5 * 60 * 1000);
|
||||
}, 5 * 60 * 1000).unref();
|
||||
|
||||
async function getSessionTimeout() {
|
||||
const now = Date.now();
|
||||
|
||||
@@ -28,6 +28,13 @@ jest.mock('../../middleware/auth', () => ({
|
||||
adminAuth: (_req, _res, next) => next(),
|
||||
}));
|
||||
|
||||
// requirePermission is its own module — without this mock the real
|
||||
// implementation runs, queries role_permissions on the mocked db, and
|
||||
// 403s before we ever reach the handler.
|
||||
jest.mock('../../middleware/permissions', () => ({
|
||||
requirePermission: () => (_req, _res, next) => next(),
|
||||
}));
|
||||
|
||||
const { db } = require('../../database/db');
|
||||
const notificationsRouter = require('../adminNotifications');
|
||||
|
||||
|
||||
@@ -98,62 +98,22 @@ router.put('/read-all', adminAuth, requirePermission('settings.edit'), async (re
|
||||
}
|
||||
});
|
||||
|
||||
// Delete old notifications (older than 30 days and read)
|
||||
router.delete('/clear-old', adminAuth, requirePermission('settings.edit'), async (req, res) => {
|
||||
// Clear all notifications (#597).
|
||||
//
|
||||
// The frontend AdminHeader "Clear All" button hits this — its service
|
||||
// at `notifications.service.ts` does DELETE /admin/notifications/clear-all.
|
||||
// The previous /clear-old route was named for an "older than 30 days
|
||||
// and read" semantic but had a fallback that deleted EVERYTHING when
|
||||
// nothing matched the date filter, so it was effectively a confusingly
|
||||
// named Clear All anyway. Drop the rename and the branching, return
|
||||
// the simple deletedCount the existing test (and frontend toast) expect.
|
||||
router.delete('/clear-all', adminAuth, requirePermission('settings.edit'), async (req, res) => {
|
||||
try {
|
||||
// Use database-agnostic date calculation
|
||||
const thirtyDaysAgo = new Date();
|
||||
thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
|
||||
|
||||
let deletedCount = 0;
|
||||
const client = db?.client?.config?.client;
|
||||
|
||||
if (client === 'pg') {
|
||||
const primaryResult = await db.raw(
|
||||
`
|
||||
WITH deleted AS (
|
||||
DELETE FROM activity_logs
|
||||
WHERE read_at IS NOT NULL OR created_at < ?
|
||||
RETURNING id
|
||||
)
|
||||
SELECT COUNT(*)::int AS count FROM deleted
|
||||
`,
|
||||
[thirtyDaysAgo.toISOString()]
|
||||
);
|
||||
deletedCount = primaryResult.rows?.[0]?.count || 0;
|
||||
|
||||
if (deletedCount === 0) {
|
||||
const fallbackResult = await db.raw(
|
||||
`
|
||||
WITH deleted AS (
|
||||
DELETE FROM activity_logs
|
||||
RETURNING id
|
||||
)
|
||||
SELECT COUNT(*)::int AS count FROM deleted
|
||||
`
|
||||
);
|
||||
deletedCount = fallbackResult.rows?.[0]?.count || 0;
|
||||
}
|
||||
} else {
|
||||
deletedCount = await db('activity_logs')
|
||||
.where(function () {
|
||||
this.whereNotNull('read_at')
|
||||
.orWhere('created_at', '<', thirtyDaysAgo);
|
||||
})
|
||||
.delete();
|
||||
|
||||
if (deletedCount === 0) {
|
||||
deletedCount = await db('activity_logs').delete();
|
||||
}
|
||||
}
|
||||
|
||||
res.json({
|
||||
message: deletedCount > 0 ? 'Old notifications cleared' : 'No notifications to clear',
|
||||
deletedCount
|
||||
});
|
||||
const deletedCount = await db('activity_logs').delete();
|
||||
res.json({ message: 'All notifications cleared', deletedCount });
|
||||
} catch (error) {
|
||||
console.error('Clear old notifications error:', error);
|
||||
res.status(500).json({ error: 'Failed to clear old notifications' });
|
||||
console.error('Clear notifications error:', error);
|
||||
res.status(500).json({ error: 'Failed to clear notifications' });
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
+269
-101
@@ -348,51 +348,7 @@ router.get('/run/:id/report', requirePermission('backup.view'), async (req, res)
|
||||
*/
|
||||
router.get('/available-backups', requirePermission('backup.view'), async (req, res) => {
|
||||
try {
|
||||
const backups = [];
|
||||
|
||||
// Get local file backups
|
||||
const backupConfig = await getBackupConfig();
|
||||
if (backupConfig.backup_destination_type === 'local' && backupConfig.backup_destination_path) {
|
||||
try {
|
||||
const files = await fs.readdir(backupConfig.backup_destination_path);
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json') || file.endsWith('.yaml')) {
|
||||
const filePath = path.join(backupConfig.backup_destination_path, file);
|
||||
const stats = await fs.stat(filePath);
|
||||
backups.push({
|
||||
type: 'local',
|
||||
name: file,
|
||||
path: filePath,
|
||||
size: stats.size,
|
||||
modified: stats.mtime
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to list local backups:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Get database backups from backup_runs table
|
||||
const backupRuns = await db('backup_runs')
|
||||
.where('status', 'completed')
|
||||
.whereNotNull('manifest_path')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.limit(20);
|
||||
|
||||
for (const run of backupRuns) {
|
||||
backups.push({
|
||||
type: run.manifest_path.startsWith('s3://') ? 's3' : 'local',
|
||||
name: `Backup ${run.completed_at}`,
|
||||
path: run.manifest_path,
|
||||
manifestId: run.manifest_id,
|
||||
size: run.total_size_bytes,
|
||||
filesCount: run.files_backed_up,
|
||||
duration: run.duration_seconds,
|
||||
completed: run.completed_at
|
||||
});
|
||||
}
|
||||
|
||||
const backups = await discoverAvailableBackups();
|
||||
res.json({
|
||||
success: true,
|
||||
data: backups
|
||||
@@ -406,6 +362,213 @@ router.get('/available-backups', requirePermission('backup.view'), async (req, r
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Discover restorable backups by walking the configured destination
|
||||
* directory + harvesting the backup_runs table.
|
||||
*
|
||||
* **Why we recurse the disk first, DB second**
|
||||
*
|
||||
* The disk is the source of truth for restore. After a disaster
|
||||
* (`docker compose down -v`, drive corruption, fresh install) the
|
||||
* `backup_runs` table is empty — but the manifest JSONs are exactly
|
||||
* what's left on disk for an admin to recover from. A wizard that
|
||||
* only reads the DB shows "No backups found" precisely when the
|
||||
* admin needs it most. So we walk first, dedupe-by-manifestId
|
||||
* against any surviving DB rows, and present a unified list.
|
||||
*
|
||||
* Discovery rules:
|
||||
* - Walks `backup_destination_path` AND `backup_manifest_path` if
|
||||
* they're distinct (manifests can live in a sibling directory).
|
||||
* - Recurses up to 3 levels deep — enough to find
|
||||
* `<root>/manifests/backup-manifest-<id>.json` (the default
|
||||
* layout) without scanning the entire photo tree.
|
||||
* - Matches manifest files by glob: `backup-manifest-*.json`,
|
||||
* `backup-manifest-*.yaml`, and the legacy bare `manifest.json`.
|
||||
* - Parses each manifest to extract real metadata (timestamp,
|
||||
* size, file count, source type) instead of showing the admin
|
||||
* a list of opaque filenames.
|
||||
*
|
||||
* Returns: array of `{ type, name, path, manifestId, size,
|
||||
* filesCount, completed, source: 'disk' | 'db' }`.
|
||||
*/
|
||||
async function discoverAvailableBackups() {
|
||||
const backupConfig = await getBackupConfig();
|
||||
const backups = [];
|
||||
const seenManifestIds = new Set();
|
||||
|
||||
if (backupConfig.backup_destination_type === 'local') {
|
||||
const roots = new Set();
|
||||
if (backupConfig.backup_destination_path) roots.add(backupConfig.backup_destination_path);
|
||||
if (backupConfig.backup_manifest_path) roots.add(backupConfig.backup_manifest_path);
|
||||
|
||||
for (const root of roots) {
|
||||
try {
|
||||
const manifestPaths = await walkForManifests(root, 3);
|
||||
for (const filePath of manifestPaths) {
|
||||
try {
|
||||
const parsed = await parseManifestMetadata(filePath);
|
||||
if (parsed.manifestId) seenManifestIds.add(parsed.manifestId);
|
||||
backups.push(parsed);
|
||||
} catch (err) {
|
||||
// Don't fail discovery because ONE manifest is corrupt —
|
||||
// surface the file with a note so the admin sees something
|
||||
// is wrong and can investigate.
|
||||
logger.warn(`Manifest unreadable at ${filePath}: ${err.message}`);
|
||||
const stats = await fs.stat(filePath).catch(() => null);
|
||||
backups.push({
|
||||
type: 'local',
|
||||
name: path.basename(filePath),
|
||||
path: filePath,
|
||||
manifestId: null,
|
||||
size: stats?.size || 0,
|
||||
filesCount: null,
|
||||
completed: stats?.mtime || null,
|
||||
source: 'disk',
|
||||
corrupt: true,
|
||||
error: err.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`Could not scan backup root ${root}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Layer in surviving DB rows, deduping by manifest_id so we don't
|
||||
// show the same backup twice with different shapes.
|
||||
const backupRuns = await db('backup_runs')
|
||||
.where('status', 'completed')
|
||||
.whereNotNull('manifest_path')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.limit(50);
|
||||
|
||||
for (const run of backupRuns) {
|
||||
if (run.manifest_id && seenManifestIds.has(run.manifest_id)) continue;
|
||||
backups.push({
|
||||
type: run.manifest_path.startsWith('s3://') ? 's3' : 'local',
|
||||
name: `Backup ${run.completed_at}`,
|
||||
path: run.manifest_path,
|
||||
manifestId: run.manifest_id,
|
||||
size: run.total_size_bytes,
|
||||
filesCount: run.files_backed_up,
|
||||
duration: run.duration_seconds,
|
||||
completed: run.completed_at,
|
||||
source: 'db',
|
||||
});
|
||||
}
|
||||
|
||||
// Most recent first.
|
||||
backups.sort((a, b) => {
|
||||
const aTime = a.completed ? new Date(a.completed).getTime() : 0;
|
||||
const bTime = b.completed ? new Date(b.completed).getTime() : 0;
|
||||
return bTime - aTime;
|
||||
});
|
||||
|
||||
return backups;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursive manifest finder. Depth-limited so we don't enumerate
|
||||
* thousands of photo files. Yields absolute paths.
|
||||
*/
|
||||
async function walkForManifests(dir, maxDepth, depth = 0) {
|
||||
if (depth > maxDepth) return [];
|
||||
let entries;
|
||||
try {
|
||||
entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') return [];
|
||||
throw err;
|
||||
}
|
||||
|
||||
const out = [];
|
||||
for (const entry of entries) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
// Skip obvious noise: photo trees, node_modules, hidden dirs.
|
||||
if (entry.name === 'events' || entry.name === 'business-docs'
|
||||
|| entry.name === 'thumbnails' || entry.name === 'previews'
|
||||
|| entry.name === 'heroes' || entry.name === 'uploads'
|
||||
|| entry.name.startsWith('.')
|
||||
|| entry.name === 'node_modules') continue;
|
||||
out.push(...await walkForManifests(full, maxDepth, depth + 1));
|
||||
} else if (entry.isFile() && isManifestFilename(entry.name)) {
|
||||
out.push(full);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function isManifestFilename(name) {
|
||||
// Canonical: backup-manifest-<id>.json / .yaml
|
||||
// Legacy: manifest.json (inside backup-<id>/manifest.json layout)
|
||||
// Be liberal in what we accept — admin may have renamed.
|
||||
if (/^backup-manifest-.+\.(json|ya?ml)$/i.test(name)) return true;
|
||||
if (/^manifest\.(json|ya?ml)$/i.test(name)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a manifest file and pull out the fields the wizard wants.
|
||||
* Tolerates schema drift across manifest versions (v1, v2) by
|
||||
* checking multiple shapes.
|
||||
*/
|
||||
async function parseManifestMetadata(filePath) {
|
||||
const raw = await fs.readFile(filePath, 'utf8');
|
||||
let parsed;
|
||||
if (filePath.toLowerCase().endsWith('.json')) {
|
||||
parsed = JSON.parse(raw);
|
||||
} else {
|
||||
// Minimal YAML support — most admins use JSON; only do require()
|
||||
// if a .yaml manifest is actually present.
|
||||
const yaml = require('js-yaml');
|
||||
parsed = yaml.load(raw);
|
||||
}
|
||||
|
||||
const stats = await fs.stat(filePath);
|
||||
|
||||
// v2 shape: { manifest: { id, timestamp }, backup: { ... }, files: [...], database: { ... } }
|
||||
// v1 shape: { backup_id, started_at, files: [...] } (older)
|
||||
const manifestId =
|
||||
parsed?.manifest?.id
|
||||
|| parsed?.backup?.id
|
||||
|| parsed?.backup_id
|
||||
|| null;
|
||||
|
||||
const completed =
|
||||
parsed?.backup?.completed_at
|
||||
|| parsed?.manifest?.timestamp
|
||||
|| parsed?.completed_at
|
||||
|| stats.mtime;
|
||||
|
||||
const filesCount =
|
||||
(Array.isArray(parsed?.files) ? parsed.files.length : null)
|
||||
?? parsed?.backup?.total_files
|
||||
?? null;
|
||||
|
||||
const totalSizeBytes =
|
||||
parsed?.backup?.total_size_bytes
|
||||
?? parsed?.total_size_bytes
|
||||
?? null;
|
||||
|
||||
return {
|
||||
type: 'local',
|
||||
name: path.basename(filePath),
|
||||
path: filePath,
|
||||
manifestId,
|
||||
size: totalSizeBytes ?? stats.size,
|
||||
filesCount,
|
||||
completed,
|
||||
source: 'disk',
|
||||
databaseIncluded: Boolean(parsed?.database?.backup_file),
|
||||
// Helpful for the UI: lets it show "This backup has no DB" warning
|
||||
// — exactly the surface that would have caught Ralf's original
|
||||
// four files-only manifests if it had existed.
|
||||
schemaVersion: parsed?.manifest?.version || parsed?.version || '1.0',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* List backups for restore (POST version for frontend compatibility)
|
||||
* Accepts source type in request body
|
||||
@@ -413,59 +576,58 @@ router.get('/available-backups', requirePermission('backup.view'), async (req, r
|
||||
router.post('/list-backups', requirePermission('backup.view'), async (req, res) => {
|
||||
try {
|
||||
const { source } = req.body; // 'local', 's3', or undefined for all
|
||||
const backups = [];
|
||||
|
||||
// Get backup configuration
|
||||
const backupConfig = await getBackupConfig();
|
||||
// Use the same disk-first discovery the GET endpoint uses so that
|
||||
// a fresh post-`docker compose down -v` install (empty backup_runs
|
||||
// table) can still see what's on disk. The whole point of restore
|
||||
// is "the DB is broken, rebuild it from disk" — a wizard that
|
||||
// only queries the DB shows "No backups found" exactly when it's
|
||||
// needed most. See discoverAvailableBackups for the full rationale.
|
||||
const discovered = await discoverAvailableBackups();
|
||||
|
||||
// Get database backups from backup_runs table
|
||||
const backupRuns = await db('backup_runs')
|
||||
.where('status', 'completed')
|
||||
.whereNotNull('manifest_path')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.limit(20);
|
||||
const filtered = source
|
||||
? discovered.filter((b) => b.type === source)
|
||||
: discovered;
|
||||
|
||||
for (const run of backupRuns) {
|
||||
const isS3 = run.manifest_path.startsWith('s3://');
|
||||
const backupType = isS3 ? 's3' : 'local';
|
||||
|
||||
// Filter by source if specified
|
||||
if (source && source !== backupType) {
|
||||
continue;
|
||||
}
|
||||
|
||||
backups.push({
|
||||
id: run.id,
|
||||
type: backupType,
|
||||
name: `Backup from ${new Date(run.completed_at).toLocaleString()}`,
|
||||
path: run.manifest_path,
|
||||
manifest_path: run.manifest_path,
|
||||
manifestId: run.manifest_id,
|
||||
manifestPath: run.manifest_path,
|
||||
size: parseInt(run.total_size_bytes) || 0,
|
||||
total_size: parseInt(run.total_size_bytes) || 0,
|
||||
total_size_bytes: parseInt(run.total_size_bytes) || 0,
|
||||
filesCount: run.files_backed_up || 0,
|
||||
files_backed_up: run.files_backed_up || 0,
|
||||
duration: run.duration_seconds,
|
||||
duration_seconds: run.duration_seconds,
|
||||
// Frontend expects snake_case date fields
|
||||
created_at: run.completed_at,
|
||||
completed_at: run.completed_at,
|
||||
started_at: run.started_at,
|
||||
// camelCase aliases
|
||||
completedAt: run.completed_at,
|
||||
startedAt: run.started_at,
|
||||
// Backup metadata
|
||||
status: run.status,
|
||||
backup_type: run.backup_type,
|
||||
backupType: run.backup_type,
|
||||
backup_mode: run.backup_mode,
|
||||
backupMode: run.backup_mode,
|
||||
app_version: run.app_version,
|
||||
appVersion: run.app_version
|
||||
});
|
||||
}
|
||||
// Shape for frontend compatibility — preserves every alias the
|
||||
// frontend was already reading (snake_case + camelCase), so the
|
||||
// UI rendering doesn't have to change.
|
||||
const backups = filtered.map((b) => ({
|
||||
id: b.manifestId || null,
|
||||
type: b.type,
|
||||
name: b.completed
|
||||
? `Backup from ${new Date(b.completed).toLocaleString()}`
|
||||
: b.name,
|
||||
path: b.path,
|
||||
manifest_path: b.path,
|
||||
manifestId: b.manifestId,
|
||||
manifestPath: b.path,
|
||||
size: parseInt(b.size) || 0,
|
||||
total_size: parseInt(b.size) || 0,
|
||||
total_size_bytes: parseInt(b.size) || 0,
|
||||
filesCount: b.filesCount || 0,
|
||||
files_backed_up: b.filesCount || 0,
|
||||
duration: b.duration || null,
|
||||
duration_seconds: b.duration || null,
|
||||
created_at: b.completed,
|
||||
completed_at: b.completed,
|
||||
started_at: b.completed,
|
||||
completedAt: b.completed,
|
||||
startedAt: b.completed,
|
||||
status: 'completed',
|
||||
// Stage A-aware: when the source is a disk-scanned manifest we
|
||||
// can tell the wizard whether the DB dump is present, so the
|
||||
// UI can warn before the admin picks a files-only backup.
|
||||
database_included: b.databaseIncluded,
|
||||
databaseIncluded: b.databaseIncluded,
|
||||
corrupt: b.corrupt || false,
|
||||
// Provenance: 'disk' (manifest read from filesystem) vs 'db'
|
||||
// (backup_runs row that the disk didn't surface) — useful for
|
||||
// debugging which side is missing.
|
||||
source: b.source,
|
||||
schema_version: b.schemaVersion,
|
||||
schemaVersion: b.schemaVersion,
|
||||
}));
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
@@ -556,17 +718,23 @@ async function getRestoreSettings() {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'restore')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
|
||||
const result = {};
|
||||
settings.forEach(setting => {
|
||||
// Convert boolean strings to actual booleans
|
||||
if (setting.setting_value === '1' || setting.setting_value === '0') {
|
||||
result[setting.setting_key] = setting.setting_value === '1';
|
||||
// Boolean-string normalization. Historically only handled '1'/'0',
|
||||
// but other code paths (boot self-heal, admin UI, direct SQL) write
|
||||
// 'true' / 'false' or JSON-encoded "true" / "false". Accept all four
|
||||
// shapes so `!settings.<key>` evaluates correctly downstream.
|
||||
const raw = setting.setting_value;
|
||||
if (raw === '1' || raw === 'true' || raw === '"true"') {
|
||||
result[setting.setting_key] = true;
|
||||
} else if (raw === '0' || raw === 'false' || raw === '"false"') {
|
||||
result[setting.setting_key] = false;
|
||||
} else {
|
||||
result[setting.setting_key] = setting.setting_value;
|
||||
result[setting.setting_key] = raw;
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
* Admin → System Health
|
||||
*
|
||||
* Endpoint mounted at /api/admin/system-health. The "Backup
|
||||
* integrity" sub-endpoint is the on-demand verifier for CRM
|
||||
* document artefacts — confirms every `*_path` column on quotes /
|
||||
* contracts / invoices points at a file that actually exists on
|
||||
* disk and (where a `*_sha256` column is set) the file's bytes
|
||||
* still hash to the expected value.
|
||||
*
|
||||
* Per the design decisions locked with the maintainer:
|
||||
* - On-demand only; no scheduler (D1)
|
||||
* - Not auto-triggered after restore (D2)
|
||||
* - Wet-upload contracts are hash-verified same as system-rendered (D3)
|
||||
*
|
||||
* Read-only. Returns a JSON report — never mutates DB or fs.
|
||||
*/
|
||||
|
||||
const express = require('express');
|
||||
const { query } = require('express-validator');
|
||||
const { adminAuth } = require('../middleware/auth');
|
||||
const { requirePermission } = require('../middleware/permissions');
|
||||
const { handleAsync, validateRequest, successResponse } = require('../utils/routeHelpers');
|
||||
const { verifyDocumentArtefacts } = require('../services/backupIntegrityService');
|
||||
const { getCoverageReport } = require('../services/backupCoverageService');
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
router.use(adminAuth);
|
||||
|
||||
const VALID_SCOPES = ['quote', 'contract', 'contract-signature', 'invoice'];
|
||||
|
||||
router.get(
|
||||
'/backup-integrity',
|
||||
requirePermission('settings.view'),
|
||||
[
|
||||
// CSV string like `?scope=contract,invoice`. Each member must be
|
||||
// one of the four known scopes. Empty / omitted means full scan.
|
||||
query('scope').optional({ values: 'falsy' }).isString().isLength({ max: 128 }),
|
||||
],
|
||||
handleAsync(async (req, res) => {
|
||||
validateRequest(req);
|
||||
let scope;
|
||||
if (req.query.scope) {
|
||||
scope = String(req.query.scope)
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
// Defense-in-depth: reject unknown scope tokens so a typo doesn't
|
||||
// silently scan everything when the caller wanted just one slice.
|
||||
const unknown = scope.filter((s) => !VALID_SCOPES.includes(s));
|
||||
if (unknown.length > 0) {
|
||||
return res.status(400).json({
|
||||
error: `Unknown scope(s): ${unknown.join(', ')}`,
|
||||
code: 'BACKUP_INTEGRITY_UNKNOWN_SCOPE',
|
||||
validScopes: VALID_SCOPES,
|
||||
});
|
||||
}
|
||||
}
|
||||
const report = await verifyDocumentArtefacts({ scope });
|
||||
return successResponse(res, { report });
|
||||
}),
|
||||
);
|
||||
|
||||
/**
|
||||
* GET /api/admin/system-health/backup-coverage
|
||||
*
|
||||
* Stage C of the backup-hardening plan. Returns the data-driven
|
||||
* coverage report — what the next "Run Backup Now" will include /
|
||||
* skip / silently miss, plus the database-dump status block.
|
||||
*
|
||||
* Read-only, on-demand. No scope parameter — the report is cheap
|
||||
* (only top-level directory listing under STORAGE_PATH, no recursion).
|
||||
*
|
||||
* See backupCoverageService.js for the full rationale and the
|
||||
* coverage-classification rules.
|
||||
*/
|
||||
router.get(
|
||||
'/backup-coverage',
|
||||
requirePermission('settings.view'),
|
||||
handleAsync(async (req, res) => {
|
||||
const report = await getCoverageReport();
|
||||
return successResponse(res, { report });
|
||||
}),
|
||||
);
|
||||
|
||||
module.exports = router;
|
||||
@@ -201,14 +201,22 @@ describe('DatabaseBackupService', () => {
|
||||
delete: jest.fn().mockResolvedValue(1)
|
||||
});
|
||||
|
||||
// Mock fs.unlink
|
||||
fs.unlink = jest.fn().mockResolvedValue(undefined);
|
||||
|
||||
// Stub fs.promises.unlink via jest.spyOn so the original is
|
||||
// restored when the test finishes. The previous form
|
||||
// (`fs.unlink = jest.fn()`) leaked into every test that ran
|
||||
// after this one in the same jest worker — most visibly
|
||||
// integration/storageBackend.test.js, whose LocalFsStorage
|
||||
// delete() became a silent no-op and the subsequent
|
||||
// exists() assertion flipped from false to true. spyOn +
|
||||
// mockRestore in afterEach keeps the stub scoped to this test.
|
||||
const unlinkSpy = jest.spyOn(fs, 'unlink').mockResolvedValue(undefined);
|
||||
|
||||
await service.cleanupOldBackups(30);
|
||||
|
||||
expect(fs.unlink).toHaveBeenCalledTimes(2);
|
||||
expect(fs.unlink).toHaveBeenCalledWith('/backup/old1.sql.gz');
|
||||
expect(fs.unlink).toHaveBeenCalledWith('/backup/old2.sql.gz');
|
||||
|
||||
expect(unlinkSpy).toHaveBeenCalledTimes(2);
|
||||
expect(unlinkSpy).toHaveBeenCalledWith('/backup/old1.sql.gz');
|
||||
expect(unlinkSpy).toHaveBeenCalledWith('/backup/old2.sql.gz');
|
||||
unlinkSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* Boot-time self-heal for the `backup_paths` table.
|
||||
*
|
||||
* **Why this exists**
|
||||
*
|
||||
* Knex won't re-run an applied migration, so once migration
|
||||
* 109_add_backup_paths.js has run, any later default we want to add
|
||||
* (a new subdirectory shipped by a future feature) would never reach
|
||||
* already-deployed installs. The historical fix for this kind of
|
||||
* "schema is fine, seed drifted" problem is the boot-time self-heal
|
||||
* pattern documented in [[feedback_self_heal_pattern]] — we just
|
||||
* re-apply the canonical seed on every boot with `onConflict.ignore()`
|
||||
* so admin edits stay intact and new rows trickle in.
|
||||
*
|
||||
* **Authoritative list**
|
||||
*
|
||||
* The list of defaults lives on migration 109 itself
|
||||
* (`DEFAULT_PATHS` export) — one source of truth that both the
|
||||
* migration and this seeder read. Tests assert these two stay in
|
||||
* lockstep.
|
||||
*
|
||||
* **Failure semantics**
|
||||
*
|
||||
* If the table doesn't exist yet (migrations haven't run, fresh
|
||||
* install before migration 109 lands, etc.) we no-op and log. The
|
||||
* walker has a hard-coded `LEGACY_DEFAULTS` fallback for the same
|
||||
* reason — defense in depth so "Run Backup Now" can never silently
|
||||
* ship a files-only manifest because of a seed issue. See
|
||||
* `backupService.js` getFilesToBackupInternal.
|
||||
*/
|
||||
|
||||
const { DEFAULT_PATHS } = require('../../migrations/core/109_add_backup_paths');
|
||||
|
||||
let booted = false;
|
||||
|
||||
/**
|
||||
* Idempotently re-seed `backup_paths` with the canonical defaults.
|
||||
*
|
||||
* @param {object} db knex instance
|
||||
* @param {object} logger app logger (must expose .info / .warn)
|
||||
* @returns {Promise<{ seeded: string[] }>} paths newly inserted on this boot.
|
||||
*/
|
||||
async function seedBackupPathsAtBoot(db, logger) {
|
||||
const log = logger || { info: () => {}, warn: () => {} };
|
||||
if (booted) return { seeded: [] };
|
||||
|
||||
if (!(await db.schema.hasTable('backup_paths'))) {
|
||||
log.warn('backup_paths table missing at boot — self-heal skipped (migration 109 may not have run yet)');
|
||||
return { seeded: [] };
|
||||
}
|
||||
|
||||
// Diff: which canonical paths are missing from the table right now?
|
||||
// We can't easily get "what got inserted by onConflict.ignore" out of
|
||||
// knex on both backends, so we just compute the diff ourselves and log
|
||||
// it — admins benefit from seeing exactly what got auto-added when a
|
||||
// new feature ships.
|
||||
const existing = await db('backup_paths').select('path');
|
||||
const existingSet = new Set(existing.map((r) => r.path));
|
||||
const missing = DEFAULT_PATHS.filter((p) => !existingSet.has(p.path));
|
||||
|
||||
if (missing.length === 0) {
|
||||
booted = true;
|
||||
return { seeded: [] };
|
||||
}
|
||||
|
||||
try {
|
||||
await db('backup_paths')
|
||||
.insert(missing.map((row) => ({
|
||||
...row,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
})))
|
||||
.onConflict('path')
|
||||
.ignore();
|
||||
log.info(`backup_paths self-heal added ${missing.length} row(s): ${missing.map((m) => m.path).join(', ')}`);
|
||||
} catch (err) {
|
||||
log.warn(`backup_paths self-heal failed: ${err.message}`);
|
||||
}
|
||||
|
||||
booted = true;
|
||||
return { seeded: missing.map((m) => m.path) };
|
||||
}
|
||||
|
||||
// Test-only: reset the module-level boot flag so jest can re-exercise
|
||||
// the seeder against a fresh test DB inside a single worker.
|
||||
function _resetBootForTests() {
|
||||
booted = false;
|
||||
}
|
||||
|
||||
module.exports = { seedBackupPathsAtBoot, _resetBootForTests };
|
||||
@@ -0,0 +1,242 @@
|
||||
/**
|
||||
* Install-from-backup boot hook.
|
||||
*
|
||||
* **The problem this closes**
|
||||
*
|
||||
* Without this hook, recovering a picpeak install from a backup is a
|
||||
* six-step process:
|
||||
* 1. Stand up the compose stack with empty volumes
|
||||
* 2. Wait for boot → land on the onboarding wizard
|
||||
* 3. Create a throwaway fresh-install admin
|
||||
* 4. Navigate to Backup → Restore
|
||||
* 5. Walk through the wizard with Force Restore ticked
|
||||
* 6. Log out, log back in with original (pre-disaster) credentials
|
||||
*
|
||||
* With this hook, admins skip steps 2-6. They place their backup
|
||||
* artefacts in the existing bind-mounted /backup directory, drop a
|
||||
* trigger file alongside, and the next container start runs the
|
||||
* restore BEFORE creating the throwaway onboarding admin. Server
|
||||
* comes up populated, original login works first try.
|
||||
*
|
||||
* **The trigger file convention (chosen for zero compose-file changes)**
|
||||
*
|
||||
* Drop a file named `RESTORE_ON_INSTALL` (no extension, or .txt) into
|
||||
* the root of the `/backup` mount. Two payload variants:
|
||||
*
|
||||
* 1. EMPTY file (or pure whitespace) — auto-pick the newest
|
||||
* `backup-manifest-*.json` from `/backup/manifests/`. Useful when
|
||||
* the admin doesn't know or care which one is most recent.
|
||||
*
|
||||
* 2. NON-EMPTY file containing a relative or absolute path to a
|
||||
* specific manifest. Trimmed; first line wins. Useful when the
|
||||
* admin wants a specific older backup.
|
||||
*
|
||||
* After a successful restore the trigger file is DELETED so the next
|
||||
* boot doesn't re-trigger. On failure the file is preserved + the
|
||||
* error is logged, so the admin can fix the input and retry by just
|
||||
* restarting the container.
|
||||
*
|
||||
* **Safety**
|
||||
*
|
||||
* Three layers gate this against accidental data loss:
|
||||
* 1. Trigger file must exist (intentional admin action, not auto-magic)
|
||||
* 2. DB must be empty — admin_users.count = 0 AND events.count = 0.
|
||||
* If either is non-zero, the hook refuses to run.
|
||||
* 3. The Stage A restore path (with all of tonight's fixes) handles
|
||||
* the actual swap atomically. If anything fails, the rollback
|
||||
* runs and the install stays in fresh-install state.
|
||||
*
|
||||
* Override: `INSTALL_FROM_BACKUP_FORCE=true` skips the empty-DB check
|
||||
* for the "I know what I'm doing" edge case (e.g. dev environment
|
||||
* rebuilds where there's leftover data that's safe to clobber).
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const TRIGGER_FILENAMES = ['RESTORE_ON_INSTALL', 'RESTORE_ON_INSTALL.txt'];
|
||||
|
||||
/**
|
||||
* Resolve which file should be treated as the trigger. Returns
|
||||
* `{ triggerPath, manifestPath }` if found, or null if no trigger
|
||||
* file is present (the common case — most boots).
|
||||
*/
|
||||
async function findTrigger(backupRoot, logger) {
|
||||
for (const name of TRIGGER_FILENAMES) {
|
||||
const triggerPath = path.join(backupRoot, name);
|
||||
if (fs.existsSync(triggerPath)) {
|
||||
let payload;
|
||||
try {
|
||||
payload = fs.readFileSync(triggerPath, 'utf8').trim();
|
||||
} catch (err) {
|
||||
logger.warn(`Install-from-backup: trigger file ${triggerPath} is unreadable: ${err.message}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!payload) {
|
||||
// Auto-pick the newest manifest
|
||||
const manifestsDir = path.join(backupRoot, 'manifests');
|
||||
if (!fs.existsSync(manifestsDir)) {
|
||||
logger.warn(`Install-from-backup: trigger file found but ${manifestsDir} doesn't exist`);
|
||||
return null;
|
||||
}
|
||||
const entries = fs.readdirSync(manifestsDir)
|
||||
.filter((f) => /^backup-manifest-.+\.(json|ya?ml)$/i.test(f))
|
||||
.map((f) => {
|
||||
const full = path.join(manifestsDir, f);
|
||||
return { full, mtime: fs.statSync(full).mtimeMs };
|
||||
})
|
||||
.sort((a, b) => b.mtime - a.mtime);
|
||||
if (entries.length === 0) {
|
||||
logger.warn(`Install-from-backup: no manifests found in ${manifestsDir}`);
|
||||
return null;
|
||||
}
|
||||
return { triggerPath, manifestPath: entries[0].full };
|
||||
}
|
||||
|
||||
// Take the first non-empty line as the manifest path
|
||||
const firstLine = payload.split(/\r?\n/).find((l) => l.trim()) || '';
|
||||
const manifestPath = path.isAbsolute(firstLine)
|
||||
? firstLine
|
||||
: path.join(backupRoot, firstLine);
|
||||
|
||||
if (!fs.existsSync(manifestPath)) {
|
||||
logger.warn(`Install-from-backup: trigger file points at ${manifestPath} which doesn't exist`);
|
||||
return null;
|
||||
}
|
||||
return { triggerPath, manifestPath };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the DB is empty enough that restoring on top is safe.
|
||||
* Returns `true` if safe, `false` if there's existing data.
|
||||
*/
|
||||
async function isDatabaseFresh(db, logger) {
|
||||
try {
|
||||
if (!(await db.schema.hasTable('admin_users'))) {
|
||||
// No admin_users table yet — schema is mid-migration or wholly
|
||||
// empty. Definitely safe to restore on top.
|
||||
return true;
|
||||
}
|
||||
const adminCount = await db('admin_users').count('* as c').first();
|
||||
const adminN = Number(adminCount?.c || 0);
|
||||
|
||||
let eventN = 0;
|
||||
if (await db.schema.hasTable('events')) {
|
||||
const eventCount = await db('events').count('* as c').first();
|
||||
eventN = Number(eventCount?.c || 0);
|
||||
}
|
||||
|
||||
if (adminN > 1 || eventN > 0) {
|
||||
logger.warn(
|
||||
`Install-from-backup: refusing — install has ${adminN} admin(s) and ${eventN} event(s). `
|
||||
+ 'This guard prevents accidental clobbering of production data. '
|
||||
+ 'Override with INSTALL_FROM_BACKUP_FORCE=true if you really want to restore on top.'
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
// adminN === 1 is the "fresh install ran migration 001 and auto-created
|
||||
// the default admin" case. That admin is throwaway — the restore will
|
||||
// replace it with the backup's admin row. So we treat 1 admin + 0
|
||||
// events as fresh.
|
||||
return true;
|
||||
} catch (err) {
|
||||
logger.warn(`Install-from-backup: fresh-install check threw: ${err.message}. Assuming NOT fresh.`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Public entry point — called from server.js after migrations and
|
||||
* before startServer.
|
||||
*
|
||||
* @returns {Promise<{ ran: boolean, manifestPath?: string, error?: string }>}
|
||||
*/
|
||||
async function tryInstallFromBackup(db, logger) {
|
||||
const log = logger || { info: () => {}, warn: () => {}, error: () => {} };
|
||||
|
||||
// The container's winston logger writes to `/app/logs/combined.log`
|
||||
// by default and may not always tee to stdout, so admins running
|
||||
// `docker logs picpeak-beta-backend` after a `compose up` would
|
||||
// see no signal that a restore happened — flagged on PR #596
|
||||
// review. We mirror the key trigger / start / end lines to
|
||||
// console.log as well so the docker-logs surface tells the story
|
||||
// without needing to exec into the container.
|
||||
const announce = (msg) => {
|
||||
try { console.log(`[install-from-backup] ${msg}`); } catch (_) { /* defensive */ }
|
||||
};
|
||||
|
||||
const backupRoot = process.env.BACKUP_ROOT || '/backup';
|
||||
if (!fs.existsSync(backupRoot)) {
|
||||
return { ran: false };
|
||||
}
|
||||
|
||||
const trigger = await findTrigger(backupRoot, log);
|
||||
if (!trigger) {
|
||||
return { ran: false };
|
||||
}
|
||||
|
||||
log.info(`Install-from-backup: trigger file found at ${trigger.triggerPath}, target manifest ${trigger.manifestPath}`);
|
||||
announce(`trigger file detected → ${trigger.manifestPath}`);
|
||||
|
||||
const forceOverride = process.env.INSTALL_FROM_BACKUP_FORCE === 'true';
|
||||
const isFresh = await isDatabaseFresh(db, log);
|
||||
if (!isFresh && !forceOverride) {
|
||||
log.warn('Install-from-backup: skipping. Trigger file left in place so you can correct + retry.');
|
||||
announce('skipping — install has existing data and INSTALL_FROM_BACKUP_FORCE is not set');
|
||||
return { ran: false, error: 'Database not empty' };
|
||||
}
|
||||
|
||||
log.info(`Install-from-backup: restoring from ${trigger.manifestPath}...`);
|
||||
announce(`starting restore from ${trigger.manifestPath}`);
|
||||
|
||||
try {
|
||||
const { restoreService } = require('./restoreService');
|
||||
const result = await restoreService.restore({
|
||||
source: 'local',
|
||||
manifestPath: trigger.manifestPath,
|
||||
restoreType: 'full',
|
||||
// Force=true because the fresh-install admin auto-created by
|
||||
// migration 001 trips the "1 active admin" warning — we WANT to
|
||||
// override that warning, since replacing the throwaway admin
|
||||
// with the backup's admin is exactly the goal.
|
||||
force: true,
|
||||
// SkipPreBackup=true because backing up an empty install is
|
||||
// pointless. Saves a few seconds and reduces disk noise.
|
||||
skipPreBackup: true,
|
||||
operator: {
|
||||
type: 'install-from-backup',
|
||||
userId: null,
|
||||
ip: null,
|
||||
},
|
||||
});
|
||||
|
||||
if (result?.success === false) {
|
||||
throw new Error(result?.error || 'Restore service reported failure');
|
||||
}
|
||||
|
||||
log.info(`Install-from-backup: restore completed successfully from ${trigger.manifestPath}`);
|
||||
announce('restore completed successfully');
|
||||
|
||||
// Remove the trigger so the next boot doesn't redo it.
|
||||
try {
|
||||
fs.unlinkSync(trigger.triggerPath);
|
||||
log.info(`Install-from-backup: removed trigger file ${trigger.triggerPath}`);
|
||||
} catch (unlinkErr) {
|
||||
log.warn(`Install-from-backup: could not remove trigger file (manual cleanup needed): ${unlinkErr.message}`);
|
||||
}
|
||||
|
||||
return { ran: true, manifestPath: trigger.manifestPath };
|
||||
} catch (err) {
|
||||
log.error(`Install-from-backup: FAILED — ${err.message}`);
|
||||
log.warn('Trigger file left in place so you can fix the input and retry by restarting the container.');
|
||||
announce(`FAILED — ${err.message}. Trigger file left in place for retry.`);
|
||||
return { ran: false, error: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { tryInstallFromBackup };
|
||||
@@ -0,0 +1,173 @@
|
||||
/**
|
||||
* Boot-time self-heal for restore-meta settings.
|
||||
*
|
||||
* **Why this exists**
|
||||
*
|
||||
* `restore_allow_force` gates whether the Restore wizard accepts a
|
||||
* `force: true` payload. The flag exists to add admin friction
|
||||
* before letting a restore override safety warnings (e.g. "1 active
|
||||
* admin user — restoring would clobber the current install").
|
||||
*
|
||||
* In practice the friction lands at the worst possible moment: a
|
||||
* fresh install (no app_settings row yet OR `restore_allow_force =
|
||||
* false` by default) hits the wall on its very FIRST restore. The
|
||||
* admin is mid disaster-recovery, panicked, and gets:
|
||||
*
|
||||
* "Force restore is not allowed by system settings"
|
||||
*
|
||||
* They then have to hand-craft SQL like
|
||||
*
|
||||
* INSERT INTO app_settings (setting_key, setting_value, ...)
|
||||
* VALUES ('restore_allow_force', 'true', 'restore', NOW())
|
||||
* ON CONFLICT ... SET setting_value = 'true';
|
||||
*
|
||||
* before they can recover their data. This isn't security — the
|
||||
* admin who could run that SQL could also flip the setting via the
|
||||
* UI. It's just a sharp edge that bites every new install once.
|
||||
*
|
||||
* Cure: seed the default ON at boot via `INSERT ... ON CONFLICT
|
||||
* DO NOTHING`. New installs get force-allowed out of the box.
|
||||
* Existing installs that have explicitly set the row (true OR
|
||||
* false) are NOT overwritten — admin policy wins. Same pattern
|
||||
* `_backupPathsBoot.js` uses for the canonical backup_paths rows.
|
||||
*
|
||||
* **Default-ON rationale (matches Stage A's principle)**
|
||||
*
|
||||
* Stage A defaulted inline DB dumps to ON because the cost of
|
||||
* forgetting was data loss. By the same logic, `restore_allow_force`
|
||||
* defaults ON because the cost of forgetting is being unable to
|
||||
* recover from a disaster. Audit logging captures every forced
|
||||
* restore so the accountability story stays intact.
|
||||
*
|
||||
* If/when the broader "exclude restore-meta settings from being
|
||||
* overwritten by restore" follow-up lands (the second half of this
|
||||
* chicken-and-egg), this self-heal becomes the safety net for
|
||||
* fresh installs only — existing installs by that point have the
|
||||
* row preserved across restores.
|
||||
*/
|
||||
|
||||
const SEEDS = [
|
||||
{
|
||||
setting_key: 'restore_allow_force',
|
||||
setting_value: 'true',
|
||||
setting_type: 'restore',
|
||||
rationale: 'Default ON so fresh installs can recover from disaster '
|
||||
+ 'without a SQL incantation. Admins who want to require manual '
|
||||
+ 'intervention can disable via the admin UI.',
|
||||
},
|
||||
];
|
||||
|
||||
/**
|
||||
* Installs that ran migration 032 BEFORE the 2026-05-30 in-place edit
|
||||
* have a `restore_allow_force` row with the deprecated `false` default
|
||||
* (literal string `'false'` from `JSON.stringify(false)`). Per
|
||||
* [[feedback_self_heal_pattern]] knex won't re-run the corrected
|
||||
* migration on those installs, so we have to bump the row to `true`
|
||||
* here ONCE at boot.
|
||||
*
|
||||
* The bump is guarded by a tracking row `restore_allow_force_auto_upgraded`
|
||||
* so we don't fight an admin who explicitly disables force later:
|
||||
* - Tracking row absent → bump if the value is the deprecated `'false'`
|
||||
* - Tracking row present → never touch `restore_allow_force` again
|
||||
*
|
||||
* The bump applies ONLY when the existing value EXACTLY equals the old
|
||||
* migration default. Any other value (`'true'`, admin-set anything,
|
||||
* empty, null) is left alone — those reflect either the fixed
|
||||
* migration's output or a deliberate admin choice.
|
||||
*/
|
||||
const DEPRECATED_DEFAULT_VALUE = 'false';
|
||||
const AUTO_UPGRADE_FLAG_KEY = 'restore_allow_force_auto_upgraded';
|
||||
|
||||
let booted = false;
|
||||
|
||||
/**
|
||||
* Seed the canonical restore-meta settings on fresh installs.
|
||||
*
|
||||
* @param {object} db knex instance
|
||||
* @param {object} logger app logger (must expose .info / .warn)
|
||||
* @returns {Promise<{ seeded: string[], upgraded: string[] }>}
|
||||
*/
|
||||
async function seedRestoreSettingsAtBoot(db, logger) {
|
||||
const log = logger || { info: () => {}, warn: () => {} };
|
||||
if (booted) return { seeded: [], upgraded: [] };
|
||||
|
||||
if (!(await db.schema.hasTable('app_settings'))) {
|
||||
log.warn('app_settings table missing at boot — restore-settings self-heal skipped');
|
||||
return { seeded: [], upgraded: [] };
|
||||
}
|
||||
|
||||
const seeded = [];
|
||||
const upgraded = [];
|
||||
|
||||
// Step 1: fresh-install seeding. Insert rows that don't exist at all.
|
||||
for (const seed of SEEDS) {
|
||||
try {
|
||||
const existing = await db('app_settings')
|
||||
.where('setting_key', seed.setting_key)
|
||||
.first();
|
||||
if (existing) continue;
|
||||
|
||||
await db('app_settings').insert({
|
||||
setting_key: seed.setting_key,
|
||||
setting_value: seed.setting_value,
|
||||
setting_type: seed.setting_type,
|
||||
updated_at: new Date(),
|
||||
});
|
||||
seeded.push(seed.setting_key);
|
||||
log.info(`Seeded restore-meta setting ${seed.setting_key}=${seed.setting_value}`);
|
||||
} catch (err) {
|
||||
log.warn(`Failed to seed restore-meta setting ${seed.setting_key}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: one-time auto-upgrade for installs that ran the OLD
|
||||
// migration 032 (which seeded restore_allow_force='false'). Bump
|
||||
// to 'true' iff the value is still the deprecated default AND the
|
||||
// auto-upgrade tracking flag hasn't already been set.
|
||||
try {
|
||||
const guard = await db('app_settings')
|
||||
.where('setting_key', AUTO_UPGRADE_FLAG_KEY)
|
||||
.first();
|
||||
|
||||
if (!guard) {
|
||||
const row = await db('app_settings')
|
||||
.where('setting_key', 'restore_allow_force')
|
||||
.first();
|
||||
|
||||
if (row && row.setting_value === DEPRECATED_DEFAULT_VALUE) {
|
||||
await db('app_settings')
|
||||
.where('setting_key', 'restore_allow_force')
|
||||
.update({
|
||||
setting_value: 'true',
|
||||
updated_at: new Date(),
|
||||
});
|
||||
upgraded.push('restore_allow_force');
|
||||
log.info('Auto-upgraded restore_allow_force from deprecated migration-032 default \'false\' to \'true\' '
|
||||
+ '(fresh-install disaster recovery now works without SQL incantation)');
|
||||
}
|
||||
|
||||
// Always set the guard, even if no upgrade happened — prevents
|
||||
// the bump from firing later if an admin sets the value to
|
||||
// false on purpose.
|
||||
await db('app_settings').insert({
|
||||
setting_key: AUTO_UPGRADE_FLAG_KEY,
|
||||
setting_value: 'true',
|
||||
setting_type: 'restore',
|
||||
updated_at: new Date(),
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
log.warn(`restore_allow_force auto-upgrade failed: ${err.message}`);
|
||||
}
|
||||
|
||||
booted = true;
|
||||
return { seeded, upgraded };
|
||||
}
|
||||
|
||||
// Test-only: reset the module-level boot flag so jest can re-exercise
|
||||
// the seeder against a fresh test DB inside a single worker.
|
||||
function _resetBootForTests() {
|
||||
booted = false;
|
||||
}
|
||||
|
||||
module.exports = { seedRestoreSettingsAtBoot, _resetBootForTests, SEEDS };
|
||||
@@ -0,0 +1,351 @@
|
||||
/**
|
||||
* Backup-coverage diagnostic — Stage C of the backup-hardening plan.
|
||||
*
|
||||
* **Why this is a separate service**
|
||||
*
|
||||
* Stage A (inline DB dump + fail-loud) and Stage B (config-driven
|
||||
* walker via `backup_paths`) close the data-loss footgun, but they
|
||||
* don't tell an admin *what* the next backup will actually cover.
|
||||
* That's a separate question — and a particularly important one,
|
||||
* because the whole reason Stage B exists is that the walker's
|
||||
* subdirectory list used to silently fall behind reality every time
|
||||
* a new feature dropped artefacts under STORAGE_PATH.
|
||||
*
|
||||
* This service answers two questions:
|
||||
*
|
||||
* 1. For every row in `backup_paths`, what will the next backup
|
||||
* do with it? (scan / skip-via-feature-flag / skip-via-toggle /
|
||||
* missing-on-disk)
|
||||
* 2. What subdirectories EXIST under STORAGE_PATH but have NO row
|
||||
* in `backup_paths` — i.e. drift the admin should know about
|
||||
* before they lose data on a restore?
|
||||
*
|
||||
* Plus a top-level database-dump status block: are we configured
|
||||
* for inline dump (default), or relying on the scheduled dump?
|
||||
* When was the last successful dump? Is it stale?
|
||||
*
|
||||
* **What this service does NOT do**
|
||||
*
|
||||
* - Does not run the backup
|
||||
* - Does not write anything (no DB mutations, no fs touches)
|
||||
* - Does not auto-recover drift (it's a diagnostic — admins decide
|
||||
* whether to add a `backup_paths` row, delete the orphan dir, etc.)
|
||||
* - Does not walk file contents — only top-level directory entries
|
||||
* under STORAGE_PATH are inspected (cheap; no recursion through
|
||||
* potentially-millions of photos)
|
||||
*
|
||||
* Read-only. Returns a JSON report — same shape as
|
||||
* backupIntegrityService.verifyDocumentArtefacts.
|
||||
*/
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { db } = require('../database/db');
|
||||
const logger = require('../utils/logger');
|
||||
const backupService = require('./backupService');
|
||||
|
||||
const STORAGE_ROOT = () => process.env.STORAGE_PATH || path.join(process.cwd(), 'storage');
|
||||
|
||||
/**
|
||||
* Top-level subdirectories we expect to find under STORAGE_PATH but
|
||||
* which are intentionally NOT in `backup_paths` — they're generated
|
||||
* caches / runtime artefacts that the backup is supposed to skip.
|
||||
* Listing them here keeps the drift detector from flagging them.
|
||||
*
|
||||
* `backups` — the destination directory the backup writer itself
|
||||
* creates, plus the `database_backup_runs` dump files.
|
||||
* Including it in the walker would create a recursive
|
||||
* "backup of backups" feedback loop.
|
||||
*
|
||||
* `tmp` — short-lived scratch space (e.g. PDF render staging,
|
||||
* S3 multipart uploads). Re-created on demand, never
|
||||
* holds the only copy of anything.
|
||||
*/
|
||||
const EXPECTED_NON_BACKUP_DIRS = new Set([
|
||||
'backups',
|
||||
'tmp',
|
||||
]);
|
||||
|
||||
/**
|
||||
* How stale a database dump can be before we flag it. 26 hours so a
|
||||
* daily scheduled dump is still considered "fresh" if it ran a few
|
||||
* hours late.
|
||||
*/
|
||||
const DB_DUMP_STALE_AFTER_MS = 26 * 60 * 60 * 1000;
|
||||
|
||||
function parseSettingValue(raw) {
|
||||
if (raw === null || raw === undefined) return null;
|
||||
if (typeof raw !== 'string') return raw;
|
||||
try { return JSON.parse(raw); } catch (_) {
|
||||
if (raw === 'true') return true;
|
||||
if (raw === 'false') return false;
|
||||
const n = Number(raw);
|
||||
return Number.isFinite(n) ? n : raw;
|
||||
}
|
||||
}
|
||||
|
||||
async function readBackupConfig() {
|
||||
try {
|
||||
const rows = await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
const cfg = {};
|
||||
for (const row of rows) {
|
||||
cfg[row.setting_key] = parseSettingValue(row.setting_value);
|
||||
}
|
||||
return cfg;
|
||||
} catch (err) {
|
||||
logger.warn(`backup-coverage: could not read backup config — ${err.message}`);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
async function listConfiguredPaths() {
|
||||
try {
|
||||
if (!(await db.schema.hasTable('backup_paths'))) return null;
|
||||
return await db('backup_paths')
|
||||
.orderBy('display_order', 'asc')
|
||||
.select('path', 'include_in_default', 'feature_flag', 'display_order', 'description');
|
||||
} catch (err) {
|
||||
logger.warn(`backup-coverage: could not read backup_paths — ${err.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function listTopLevelStorageDirs() {
|
||||
const root = STORAGE_ROOT();
|
||||
try {
|
||||
const entries = await fs.readdir(root, { withFileTypes: true });
|
||||
return entries.filter((e) => e.isDirectory()).map((e) => e.name);
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') return [];
|
||||
logger.warn(`backup-coverage: could not read STORAGE_PATH (${root}) — ${err.message}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function statPath(absPath) {
|
||||
try {
|
||||
const st = await fs.stat(absPath);
|
||||
return { exists: true, isDir: st.isDirectory() };
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') return { exists: false, isDir: false };
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the database-dump status block. Tells the admin whether
|
||||
* "Run Backup Now" will inline-dump (default) or rely on the
|
||||
* scheduled-dump path, plus how fresh the most recent dump is.
|
||||
*/
|
||||
async function buildDatabaseStatus(config) {
|
||||
// normalizeBoolean(undefined) === false, so we have to gate on
|
||||
// explicit-false the same way ensureDatabaseDumpForBackup does.
|
||||
const inlineExplicitlyOff = config.backup_database_inline_dump !== undefined
|
||||
&& config.backup_database_inline_dump !== null
|
||||
&& config.backup_database_inline_dump === false;
|
||||
const mode = inlineExplicitlyOff ? 'scheduled-only' : 'inline';
|
||||
|
||||
let recent = null;
|
||||
try {
|
||||
if (await db.schema.hasTable('database_backup_runs')) {
|
||||
recent = await db('database_backup_runs')
|
||||
.where('status', 'completed')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.first();
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`backup-coverage: could not read database_backup_runs — ${err.message}`);
|
||||
}
|
||||
|
||||
const status = {
|
||||
mode,
|
||||
inlineDumpExplicitlyDisabled: inlineExplicitlyOff,
|
||||
lastDumpAt: recent ? recent.completed_at : null,
|
||||
lastDumpType: recent ? recent.backup_type : null,
|
||||
lastDumpSizeBytes: recent ? Number(recent.file_size_bytes || 0) : 0,
|
||||
lastDumpFilePath: recent ? recent.file_path : null,
|
||||
lastDumpAgeMs: null,
|
||||
lastDumpStale: null,
|
||||
ok: null,
|
||||
};
|
||||
|
||||
if (recent && recent.completed_at) {
|
||||
const completedAt = recent.completed_at instanceof Date
|
||||
? recent.completed_at
|
||||
: new Date(recent.completed_at);
|
||||
status.lastDumpAgeMs = Date.now() - completedAt.getTime();
|
||||
status.lastDumpStale = status.lastDumpAgeMs > DB_DUMP_STALE_AFTER_MS;
|
||||
}
|
||||
|
||||
// ok semantics:
|
||||
// - inline mode: always ok=true (next backup will produce a fresh
|
||||
// dump on demand, staleness is irrelevant)
|
||||
// - scheduled-only: ok=true iff a recent non-stale dump exists,
|
||||
// because the file-backup guard will fail-loud otherwise
|
||||
if (mode === 'inline') {
|
||||
status.ok = true;
|
||||
} else {
|
||||
status.ok = Boolean(recent && recent.file_path && status.lastDumpStale === false);
|
||||
}
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-path coverage:
|
||||
* - configured + include_in_default + (no feature_flag OR flag truthy) → 'will-scan'
|
||||
* - configured + include_in_default + flag falsey → 'skipped-by-feature-flag'
|
||||
* - configured + include_in_default = false → 'skipped-by-toggle'
|
||||
* - configured but missing on disk → 'missing-on-disk'
|
||||
*
|
||||
* Returns one entry per `backup_paths` row.
|
||||
*/
|
||||
async function buildConfiguredPathReport(configuredRows, config) {
|
||||
const root = STORAGE_ROOT();
|
||||
const result = [];
|
||||
|
||||
for (const row of configuredRows) {
|
||||
const absPath = path.join(root, row.path);
|
||||
const stat = await statPath(absPath);
|
||||
const includedInDefault = Boolean(row.include_in_default);
|
||||
let featureFlagValue = null;
|
||||
if (row.feature_flag) {
|
||||
const v = config[row.feature_flag];
|
||||
featureFlagValue = v === undefined ? null : Boolean(v);
|
||||
}
|
||||
|
||||
let coverage;
|
||||
if (!includedInDefault) {
|
||||
coverage = 'skipped-by-toggle';
|
||||
} else if (row.feature_flag && featureFlagValue !== true) {
|
||||
// null (unset) and explicit false both gate the path off — matches
|
||||
// the walker's normalizeBoolean semantics
|
||||
coverage = 'skipped-by-feature-flag';
|
||||
} else if (!stat.exists) {
|
||||
coverage = 'missing-on-disk';
|
||||
} else {
|
||||
coverage = 'will-scan';
|
||||
}
|
||||
|
||||
result.push({
|
||||
path: row.path,
|
||||
includeInDefault: includedInDefault,
|
||||
featureFlag: row.feature_flag || null,
|
||||
featureFlagValue,
|
||||
displayOrder: row.display_order,
|
||||
description: row.description || null,
|
||||
existsOnDisk: stat.exists,
|
||||
coverage,
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drift detection: top-level subdirs under STORAGE_PATH that are not
|
||||
* in `backup_paths` AND not in the `EXPECTED_NON_BACKUP_DIRS` allow-list.
|
||||
*
|
||||
* These are the directories that will be missed by "Run Backup Now"
|
||||
* — either intentionally (a new feature drops cache files there and
|
||||
* the admin doesn't want them backed up — they should add them to the
|
||||
* allow-list) or accidentally (a feature shipped without a matching
|
||||
* `backup_paths` row — the data-loss footgun this whole effort is
|
||||
* designed to catch).
|
||||
*/
|
||||
function detectDrift(diskDirs, configuredPaths) {
|
||||
// configured paths can be nested ('events/active'); we only diff the
|
||||
// top-level segment ('events') because that's the granularity admins
|
||||
// see in the storage tree. A path like 'events/active' implies the
|
||||
// 'events' top-level is "known to the backup config".
|
||||
const configuredTopLevels = new Set(
|
||||
configuredPaths.map((p) => p.path.split('/')[0]),
|
||||
);
|
||||
|
||||
return diskDirs
|
||||
.filter((d) => !configuredTopLevels.has(d))
|
||||
.filter((d) => !EXPECTED_NON_BACKUP_DIRS.has(d))
|
||||
.sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Public entry point.
|
||||
*
|
||||
* @returns {Promise<{
|
||||
* database: object,
|
||||
* paths: Array<object>,
|
||||
* drift: { unconfiguredOnDisk: string[], expectedNonBackupDirs: string[] },
|
||||
* summary: object,
|
||||
* generatedAt: string,
|
||||
* }>}
|
||||
*/
|
||||
async function getCoverageReport() {
|
||||
const config = await readBackupConfig();
|
||||
const configuredRows = await listConfiguredPaths();
|
||||
const diskDirs = await listTopLevelStorageDirs();
|
||||
|
||||
// Fallback when the table doesn't exist yet (migration 108 hasn't
|
||||
// run for some reason). Mirrors the walker's LEGACY_BACKUP_PATHS
|
||||
// contract — every other layer of this system uses the same
|
||||
// belt-and-suspenders fallback.
|
||||
const fallback = configuredRows === null;
|
||||
const effectiveRows = configuredRows || [
|
||||
{ path: 'events/active', include_in_default: true, feature_flag: null, display_order: 10, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'events/archived', include_in_default: true, feature_flag: 'backup_include_archived', display_order: 20, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'thumbnails', include_in_default: true, feature_flag: null, display_order: 30, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'previews', include_in_default: true, feature_flag: null, display_order: 40, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'heroes', include_in_default: true, feature_flag: null, display_order: 50, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'uploads', include_in_default: true, feature_flag: null, display_order: 60, description: 'Legacy fallback (backup_paths missing)' },
|
||||
{ path: 'business-docs', include_in_default: true, feature_flag: null, display_order: 70, description: 'Legacy fallback (backup_paths missing)' },
|
||||
];
|
||||
|
||||
const [database, paths] = await Promise.all([
|
||||
buildDatabaseStatus(config),
|
||||
buildConfiguredPathReport(effectiveRows, config),
|
||||
]);
|
||||
|
||||
const unconfiguredOnDisk = detectDrift(diskDirs, effectiveRows);
|
||||
|
||||
const summary = {
|
||||
configuredCount: effectiveRows.length,
|
||||
willScanCount: paths.filter((p) => p.coverage === 'will-scan').length,
|
||||
skippedByToggleCount: paths.filter((p) => p.coverage === 'skipped-by-toggle').length,
|
||||
skippedByFeatureFlagCount: paths.filter((p) => p.coverage === 'skipped-by-feature-flag').length,
|
||||
missingOnDiskCount: paths.filter((p) => p.coverage === 'missing-on-disk').length,
|
||||
driftCount: unconfiguredOnDisk.length,
|
||||
tableMissingFallbackInUse: fallback,
|
||||
databaseOk: database.ok,
|
||||
// overall: green only when DB is ok AND there's at least one path
|
||||
// that will actually be scanned AND no drift was found
|
||||
overallOk: Boolean(
|
||||
database.ok
|
||||
&& paths.some((p) => p.coverage === 'will-scan')
|
||||
&& unconfiguredOnDisk.length === 0,
|
||||
),
|
||||
};
|
||||
|
||||
return {
|
||||
database,
|
||||
paths,
|
||||
drift: {
|
||||
unconfiguredOnDisk,
|
||||
expectedNonBackupDirs: Array.from(EXPECTED_NON_BACKUP_DIRS).sort(),
|
||||
},
|
||||
summary,
|
||||
generatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getCoverageReport,
|
||||
// Exported for test introspection — the route doesn't use these.
|
||||
EXPECTED_NON_BACKUP_DIRS,
|
||||
DB_DUMP_STALE_AFTER_MS,
|
||||
};
|
||||
|
||||
// Silence unused import lint warning — backupService is required so
|
||||
// the module-graph cache primes (some tests jest.mock it before
|
||||
// requiring this service).
|
||||
void backupService;
|
||||
@@ -0,0 +1,228 @@
|
||||
/**
|
||||
* Backup-integrity verifier — walks every CRM document-artefact path
|
||||
* column and confirms (a) the file exists on disk, (b) when a SHA-256
|
||||
* is stored, the file's actual bytes hash to the stored value.
|
||||
*
|
||||
* **Why this is a separate service**
|
||||
*
|
||||
* The audit trail captured at issue / sign time (signed_customer_ip,
|
||||
* signed_by_customer_at, signed_pdf_sha256, signed_*_signature_path,
|
||||
* issue_date, etc.) is worth exactly nothing on its own — what makes
|
||||
* it legally meaningful is being able to produce the document the
|
||||
* audit trail refers to. A backup that captures the DB but skips
|
||||
* `storage/business-docs/` (the bug fixed in this same PR) leaves
|
||||
* every `*_path` column a broken FK and every `*_sha256` column with
|
||||
* nothing to verify against. This service is the diagnostic for
|
||||
* exactly that drift — runs on demand, surfaces missing files +
|
||||
* hash mismatches without making any changes.
|
||||
*
|
||||
* **Verification modes**
|
||||
*
|
||||
* - existence — file at `*_path` must exist on disk
|
||||
* - sha256 — file at `*_path` must exist AND its sha256 must
|
||||
* equal `*_sha256` column (when that column is set)
|
||||
*
|
||||
* Per-table coverage (lines reference migrations/core/107_crm_consolidated.js):
|
||||
*
|
||||
* quotes.pdf_path line 844 (existence)
|
||||
* contracts.pdf_path line 1245 (existence + sha256 via contracts.pdf_sha256)
|
||||
* contracts.signed_pdf_path line 1246 (existence + sha256 via contracts.signed_pdf_sha256)
|
||||
* contracts.signed_customer_signature_path line 1269 (existence — drawn signatures, no hash column)
|
||||
* contracts.signed_admin_signature_path line 1273 (existence — admin counter-signature drawing)
|
||||
* invoices.pdf_path line 1026 (existence)
|
||||
* invoices.imported_pdf_path line 1020 (existence — admin-uploaded scans)
|
||||
*
|
||||
* Wet uploads (`contracts.signed_pdf_is_wet_upload = true`) DO have a
|
||||
* `signed_pdf_sha256` computed at upload time (contractService.js
|
||||
* upload route), so they're hash-verified the same as system-rendered
|
||||
* contracts — no special case here.
|
||||
*
|
||||
* **What this service does NOT do**
|
||||
*
|
||||
* - Does not write anything (no DB mutations, no fs touches)
|
||||
* - Does not fail the request when a mismatch is found — the
|
||||
* report shape carries the data, the caller decides what to do
|
||||
* - Does not auto-trigger after restore (D2 decision: surface a
|
||||
* CTA on the restore-completed screen instead)
|
||||
* - Does not run on a schedule (D1 decision: on-demand v1; revisit
|
||||
* once we have runtime data on large installs)
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const crypto = require('crypto');
|
||||
const path = require('path');
|
||||
const { db } = require('../database/db');
|
||||
const logger = require('../utils/logger');
|
||||
|
||||
const STORAGE_ROOT = () => process.env.STORAGE_PATH || path.join(process.cwd(), 'storage');
|
||||
|
||||
/**
|
||||
* Every column the verifier walks, declared once so the test suite
|
||||
* and the service share a single source of truth. Order is the order
|
||||
* the report lists rows in — table-major, then column-by-column.
|
||||
*/
|
||||
const CHECKS = [
|
||||
{ table: 'quotes', pathColumn: 'pdf_path', shaColumn: null, scope: 'quote' },
|
||||
{ table: 'contracts', pathColumn: 'pdf_path', shaColumn: 'pdf_sha256', scope: 'contract' },
|
||||
{ table: 'contracts', pathColumn: 'signed_pdf_path', shaColumn: 'signed_pdf_sha256', scope: 'contract' },
|
||||
{ table: 'contracts', pathColumn: 'signed_customer_signature_path', shaColumn: null, scope: 'contract-signature' },
|
||||
{ table: 'contracts', pathColumn: 'signed_admin_signature_path', shaColumn: null, scope: 'contract-signature' },
|
||||
{ table: 'invoices', pathColumn: 'pdf_path', shaColumn: null, scope: 'invoice' },
|
||||
{ table: 'invoices', pathColumn: 'imported_pdf_path', shaColumn: null, scope: 'invoice' },
|
||||
];
|
||||
|
||||
/** Stream-hash a file to sha256 hex without buffering the whole thing. */
|
||||
function hashFile(absPath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = crypto.createHash('sha256');
|
||||
const stream = fs.createReadStream(absPath);
|
||||
stream.on('error', reject);
|
||||
stream.on('data', (chunk) => hash.update(chunk));
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {object} [options]
|
||||
* @param {string[]} [options.scope] Filter checks by scope tag:
|
||||
* 'quote' | 'contract' | 'contract-signature' | 'invoice'.
|
||||
* Defaults to all four (full scan).
|
||||
* @returns {Promise<{
|
||||
* scannedAt: string,
|
||||
* scopes: string[],
|
||||
* summary: {
|
||||
* totalRows: number,
|
||||
* verifiedOk: number,
|
||||
* missingFiles: number,
|
||||
* hashMismatches: number,
|
||||
* existsButNoHash: number,
|
||||
* },
|
||||
* missing: Array<{ table, rowId, column, expectedPath }>,
|
||||
* hashMismatches: Array<{ table, rowId, column, expectedPath, expectedSha, actualSha }>,
|
||||
* existsButNoHash: Array<{ table, rowId, column, path }>,
|
||||
* }>}
|
||||
*
|
||||
* `existsButNoHash` is the existence-only-verified bucket — the file
|
||||
* was found but no `*_sha256` column exists for it (quote/invoice PDFs,
|
||||
* signature PNGs). Surfaced separately so admins can distinguish
|
||||
* "verified by hash" from "verified by existence only" — the latter
|
||||
* is weaker evidence in a legal dispute.
|
||||
*/
|
||||
async function verifyDocumentArtefacts(options = {}) {
|
||||
const scopes = Array.isArray(options.scope) && options.scope.length > 0
|
||||
? options.scope.slice()
|
||||
: Array.from(new Set(CHECKS.map((c) => c.scope)));
|
||||
|
||||
const checksToRun = CHECKS.filter((c) => scopes.includes(c.scope));
|
||||
const storageRoot = STORAGE_ROOT();
|
||||
|
||||
const missing = [];
|
||||
const hashMismatches = [];
|
||||
const existsButNoHash = [];
|
||||
let totalRows = 0;
|
||||
let verifiedOk = 0;
|
||||
|
||||
for (const check of checksToRun) {
|
||||
// Skip the check cleanly when the column or table doesn't exist
|
||||
// on this install — keeps the verifier safe to run on partial
|
||||
// migrations or installs that have features disabled.
|
||||
if (!(await db.schema.hasTable(check.table))) continue;
|
||||
if (!(await db.schema.hasColumn(check.table, check.pathColumn))) continue;
|
||||
|
||||
const select = ['id', check.pathColumn];
|
||||
const hasHashColumn = check.shaColumn
|
||||
&& (await db.schema.hasColumn(check.table, check.shaColumn));
|
||||
if (hasHashColumn) select.push(check.shaColumn);
|
||||
|
||||
const rows = await db(check.table)
|
||||
.whereNotNull(check.pathColumn)
|
||||
.select(...select);
|
||||
|
||||
for (const row of rows) {
|
||||
totalRows += 1;
|
||||
const storedPath = row[check.pathColumn];
|
||||
// Stored paths can be absolute (older rows) or relative-to-
|
||||
// storage (newer rows). Normalize: resolve relative paths
|
||||
// against STORAGE_PATH; absolute paths are used verbatim.
|
||||
const absPath = path.isAbsolute(storedPath)
|
||||
? storedPath
|
||||
: path.join(storageRoot, storedPath);
|
||||
|
||||
let exists = false;
|
||||
try {
|
||||
exists = fs.existsSync(absPath);
|
||||
} catch (_) { exists = false; }
|
||||
|
||||
if (!exists) {
|
||||
missing.push({
|
||||
table: check.table,
|
||||
rowId: row.id,
|
||||
column: check.pathColumn,
|
||||
expectedPath: storedPath,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const expectedSha = hasHashColumn ? row[check.shaColumn] : null;
|
||||
if (!expectedSha) {
|
||||
// File exists but we have no hash to verify it against.
|
||||
existsButNoHash.push({
|
||||
table: check.table,
|
||||
rowId: row.id,
|
||||
column: check.pathColumn,
|
||||
path: storedPath,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let actualSha;
|
||||
try {
|
||||
actualSha = await hashFile(absPath);
|
||||
} catch (err) {
|
||||
logger.warn(`backupIntegrity: failed to hash ${absPath}: ${err.message}`);
|
||||
missing.push({
|
||||
table: check.table,
|
||||
rowId: row.id,
|
||||
column: check.pathColumn,
|
||||
expectedPath: storedPath,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (actualSha !== expectedSha) {
|
||||
hashMismatches.push({
|
||||
table: check.table,
|
||||
rowId: row.id,
|
||||
column: check.pathColumn,
|
||||
expectedPath: storedPath,
|
||||
expectedSha,
|
||||
actualSha,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
verifiedOk += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
scannedAt: new Date().toISOString(),
|
||||
scopes,
|
||||
summary: {
|
||||
totalRows,
|
||||
verifiedOk,
|
||||
missingFiles: missing.length,
|
||||
hashMismatches: hashMismatches.length,
|
||||
existsButNoHash: existsButNoHash.length,
|
||||
},
|
||||
missing,
|
||||
hashMismatches,
|
||||
existsButNoHash,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
verifyDocumentArtefacts,
|
||||
// Exported for tests; not part of the route API.
|
||||
_internal: { CHECKS, hashFile },
|
||||
};
|
||||
@@ -259,6 +259,73 @@ async function hasDatabaseChanged(sinceTime) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run an inline database dump (default ON) and then verify a usable dump
|
||||
* is actually on disk before letting the file-backup proceed. Returns the
|
||||
* verified `databaseInfo` so the caller can pass it straight into the
|
||||
* manifest builder without re-querying.
|
||||
*
|
||||
* Why this lives here and not inline in `runBackupInternal`:
|
||||
* - Encapsulates the "Run Backup Now must include DB" guarantee
|
||||
* introduced when the silent files-only bug was discovered
|
||||
* (2026-05-29 — admin lost CRM after `docker compose down -v`)
|
||||
* - Lets the manifest path share the same `databaseInfo` object
|
||||
* instead of doing a second `getDatabaseBackupInfo()` round-trip
|
||||
* - Thrown errors bubble up to `runBackupInternal`'s catch, which
|
||||
* marks the `backup_runs` row failed and queues the admin email
|
||||
*
|
||||
* Default-ON semantics: `backup_database_inline_dump` is only treated
|
||||
* as disabled when explicitly set to false. `undefined` (the case on
|
||||
* every existing install that predates the setting) falls through to
|
||||
* the safe-default ON branch. `normalizeBoolean(undefined)` returns
|
||||
* false, so a naive `!== false` check would silently disable the
|
||||
* inline dump for every upgrading install.
|
||||
*/
|
||||
async function ensureDatabaseDumpForBackup(config) {
|
||||
const inlineDumpExplicitlyOff = config.backup_database_inline_dump !== undefined
|
||||
&& config.backup_database_inline_dump !== null
|
||||
&& normalizeBoolean(config.backup_database_inline_dump) === false;
|
||||
|
||||
if (!inlineDumpExplicitlyOff) {
|
||||
logger.info('Running inline database dump before file backup...');
|
||||
const { databaseBackupService } = require('./databaseBackup');
|
||||
const dumpResult = await databaseBackupService.backup({});
|
||||
logger.info(`Inline database dump completed: ${dumpResult.path} ` +
|
||||
`(${(dumpResult.size / 1024 / 1024).toFixed(2)} MB)`);
|
||||
}
|
||||
|
||||
const databaseInfo = await service.getDatabaseBackupInfo();
|
||||
if (!databaseInfo.backupFile) {
|
||||
throw new Error(
|
||||
'No database backup available to include in this file backup. ' +
|
||||
'Either keep backup_database_inline_dump enabled (default) or configure ' +
|
||||
'backup_database_schedule and let it run at least once first.'
|
||||
);
|
||||
}
|
||||
|
||||
let dumpStat;
|
||||
try {
|
||||
dumpStat = await fs.stat(databaseInfo.backupFile);
|
||||
} catch (statErr) {
|
||||
if (statErr.code === 'ENOENT') {
|
||||
throw new Error(
|
||||
`Database backup file at ${databaseInfo.backupFile} is missing from disk. ` +
|
||||
'Refusing to proceed with file backup; configure backup_database_schedule or ' +
|
||||
'keep backup_database_inline_dump enabled.'
|
||||
);
|
||||
}
|
||||
throw statErr;
|
||||
}
|
||||
if (!dumpStat.size) {
|
||||
throw new Error(
|
||||
`Database backup file at ${databaseInfo.backupFile} is empty (0 bytes). ` +
|
||||
'Refusing to proceed with file backup to avoid shipping a manifest with no DB content.'
|
||||
);
|
||||
}
|
||||
|
||||
return databaseInfo;
|
||||
}
|
||||
|
||||
async function getDatabaseBackupInfoInternal() {
|
||||
try {
|
||||
const recent = await db('database_backup_runs')
|
||||
@@ -350,28 +417,179 @@ async function scanDirectory(dirPath, fileList, basePath, excludePatterns = [])
|
||||
}
|
||||
}
|
||||
|
||||
async function getFilesToBackupInternal(includeArchived = true) {
|
||||
/**
|
||||
* Hard-coded fallback when `backup_paths` is missing/empty. Mirrors
|
||||
* the canonical seed in migration 109 — kept here as defense in depth
|
||||
* so the walker can never silently degrade to "no directories scanned"
|
||||
* because of a seed problem.
|
||||
*
|
||||
* Order matches the legacy behavior of the inlined sequence this
|
||||
* function used to contain.
|
||||
*/
|
||||
const LEGACY_BACKUP_PATHS = [
|
||||
{ path: 'events/active', feature_flag: null },
|
||||
{ path: 'events/archived', feature_flag: 'backup_include_archived' },
|
||||
{ path: 'thumbnails', feature_flag: null },
|
||||
{ path: 'previews', feature_flag: null },
|
||||
{ path: 'heroes', feature_flag: null },
|
||||
{ path: 'uploads', feature_flag: null },
|
||||
{ path: 'business-docs', feature_flag: null },
|
||||
];
|
||||
|
||||
/**
|
||||
* Resolve the walker's target subdirectories from `backup_paths`.
|
||||
*
|
||||
* Layered fallback (defense in depth — no scenario where the walker
|
||||
* silently scans nothing):
|
||||
* 1. Read `backup_paths` rows where include_in_default = true,
|
||||
* ordered by display_order.
|
||||
* 2. If the table is missing OR returns zero rows, fall back to
|
||||
* LEGACY_BACKUP_PATHS. Logged loudly so the admin sees it.
|
||||
*
|
||||
* Per-row gating: when `feature_flag` is set, the corresponding
|
||||
* config key in `app_settings` must resolve truthy for that path to
|
||||
* be included. Mirrors the historical `includeArchived` parameter,
|
||||
* but now driven by data instead of a hard-coded boolean.
|
||||
*
|
||||
* @param {object} config resolved backup config (parseSettingValue'd).
|
||||
* Used to evaluate feature_flag gates.
|
||||
* @returns {Promise<Array<{ path: string, feature_flag: string|null }>>}
|
||||
*/
|
||||
async function resolveBackupPaths(config) {
|
||||
let rows;
|
||||
try {
|
||||
if (!(await db.schema.hasTable('backup_paths'))) {
|
||||
logger.warn('backup_paths table missing — falling back to LEGACY_BACKUP_PATHS');
|
||||
rows = LEGACY_BACKUP_PATHS;
|
||||
} else {
|
||||
rows = await db('backup_paths')
|
||||
.where('include_in_default', formatBoolean(true))
|
||||
.orderBy('display_order', 'asc')
|
||||
.select('path', 'feature_flag');
|
||||
if (!rows.length) {
|
||||
logger.warn('backup_paths has no rows with include_in_default=true — falling back to LEGACY_BACKUP_PATHS');
|
||||
rows = LEGACY_BACKUP_PATHS;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`Failed to query backup_paths (${err.message}) — falling back to LEGACY_BACKUP_PATHS`);
|
||||
rows = LEGACY_BACKUP_PATHS;
|
||||
}
|
||||
|
||||
// Apply feature_flag gating. A row with feature_flag='backup_include_archived'
|
||||
// requires config.backup_include_archived to be truthy (same semantics as
|
||||
// the historical `includeArchived` parameter).
|
||||
return rows.filter((row) => {
|
||||
if (!row.feature_flag) return true;
|
||||
const flagValue = config ? config[row.feature_flag] : undefined;
|
||||
return normalizeBoolean(flagValue);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Bucket actually-backed-up files into their owning `backup_paths` row.
|
||||
*
|
||||
* Uses longest-prefix match — e.g. `events/active/E1/photo.jpg` matches
|
||||
* `events/active` (length 13) rather than `events` (length 6, if that
|
||||
* row existed). This handles the case where a future feature ships a
|
||||
* nested backup_paths row that overlaps an existing one.
|
||||
*
|
||||
* @param {string[]} backedUpRelativePaths — paths that actually got
|
||||
* copied / uploaded (post-incremental-filter). The exact list
|
||||
* the destination implementation reports back.
|
||||
* @param {Array<{relativePath, size}>} allFiles — the full file
|
||||
* catalogue from the walker, used as a size lookup table.
|
||||
* @returns {Promise<Record<string, { count: number, size: number }>>}
|
||||
* keyed by `backup_paths.path` (e.g. 'events/active'). Paths
|
||||
* with zero matches are omitted to keep the manifest compact.
|
||||
*/
|
||||
async function computePerPathStats(backedUpRelativePaths, allFiles) {
|
||||
if (!backedUpRelativePaths || backedUpRelativePaths.length === 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Reuse the same source of truth the walker uses, so a row toggled
|
||||
// off by include_in_default doesn't appear in the breakdown either.
|
||||
let configuredPaths;
|
||||
try {
|
||||
if (await db.schema.hasTable('backup_paths')) {
|
||||
configuredPaths = await db('backup_paths')
|
||||
.where('include_in_default', formatBoolean(true))
|
||||
.orderBy('display_order', 'asc')
|
||||
.select('path');
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`Could not load backup_paths for per-path stats — falling back to legacy: ${err.message}`);
|
||||
}
|
||||
if (!configuredPaths || configuredPaths.length === 0) {
|
||||
configuredPaths = LEGACY_BACKUP_PATHS.map((p) => ({ path: p.path }));
|
||||
}
|
||||
|
||||
// Longest-prefix-first so nested paths win over their parents.
|
||||
const sortedPaths = configuredPaths
|
||||
.map((row) => row.path)
|
||||
.sort((a, b) => b.length - a.length);
|
||||
|
||||
// Size lookup. relativePath uses OS path separators in `allFiles`
|
||||
// (whatever scanDirectory built); the backup_paths rows always use
|
||||
// forward slashes. Normalize the lookup key once.
|
||||
const sizeByPath = new Map();
|
||||
for (const f of allFiles || []) {
|
||||
sizeByPath.set(f.relativePath.split(path.sep).join('/'), f.size || 0);
|
||||
}
|
||||
|
||||
const stats = {};
|
||||
for (const relativePath of backedUpRelativePaths) {
|
||||
const norm = relativePath.split(path.sep).join('/');
|
||||
// Find the longest configured path that this file's relativePath starts with.
|
||||
const match = sortedPaths.find(
|
||||
(p) => norm === p || norm.startsWith(`${p}/`)
|
||||
);
|
||||
if (!match) continue; // file outside any configured path (shouldn't happen)
|
||||
if (!stats[match]) stats[match] = { count: 0, size: 0 };
|
||||
stats[match].count += 1;
|
||||
stats[match].size += sizeByPath.get(norm) || 0;
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
async function getFilesToBackupInternal(configOrIncludeArchived = true) {
|
||||
const files = [];
|
||||
const storagePath = getStoragePath();
|
||||
|
||||
await scanDirectory(path.join(storagePath, 'events/active'), files, storagePath);
|
||||
|
||||
if (normalizeBoolean(includeArchived)) {
|
||||
await scanDirectory(path.join(storagePath, 'events/archived'), files, storagePath);
|
||||
// Backward-compatible call signature:
|
||||
// - Boolean `true|false` → legacy `includeArchived` argument. We
|
||||
// forge a config-shaped object so the feature-flag gating
|
||||
// resolves the same way the old code path did.
|
||||
// - Object → full resolved backup config (preferred).
|
||||
// - Anything else → treated as "include archived" (truthy).
|
||||
let config;
|
||||
if (typeof configOrIncludeArchived === 'object' && configOrIncludeArchived !== null) {
|
||||
config = configOrIncludeArchived;
|
||||
} else {
|
||||
config = { backup_include_archived: normalizeBoolean(configOrIncludeArchived) };
|
||||
}
|
||||
|
||||
await scanDirectory(path.join(storagePath, 'thumbnails'), files, storagePath);
|
||||
// Lightbox preview tier (#492). Cheap to back up — typically a few
|
||||
// hundred KB per photo — and saves admins the regenerate cycle on
|
||||
// a restore. Tolerated when missing (admins who never enabled the
|
||||
// feature won't have the folder; scanDirectory short-circuits on
|
||||
// ENOENT cleanly).
|
||||
await scanDirectory(path.join(storagePath, 'previews'), files, storagePath);
|
||||
// Heroes too — same logic; admins who picked a hero photo for the
|
||||
// gallery header had its 1920x1080 file generated and was missed
|
||||
// by the original backup walk before this addition.
|
||||
await scanDirectory(path.join(storagePath, 'heroes'), files, storagePath);
|
||||
await scanDirectory(path.join(storagePath, 'uploads'), files, storagePath);
|
||||
const targets = await resolveBackupPaths(config);
|
||||
|
||||
for (const target of targets) {
|
||||
// CRM document estate is special-cased in the comment block below
|
||||
// because it's the most expensive omission to recover from:
|
||||
// - business-docs/quote/<year>/*.pdf
|
||||
// - business-docs/contract/<year>/*.pdf (system-rendered + wet uploads)
|
||||
// - business-docs/contract/signatures/<contract_id>/*.{png,jpg}
|
||||
// (drawn signatures, forensic-preserved per Date.now() filename)
|
||||
// - business-docs/invoice/<year>/*.pdf (issued invoices + Storno)
|
||||
// - business-docs/invoice-imports/<year>/*.pdf (admin-imported
|
||||
// historical invoices — irrecoverable if not backed up)
|
||||
// Without this scan, the audit trail (signed_pdf_sha256, signed_*
|
||||
// _ip, accepted_at, etc.) survives the restore but the documents
|
||||
// those values refer to do not, leaving every CRM *_path column a
|
||||
// broken FK. scanDirectory short-circuits on ENOENT so installs
|
||||
// that never used CRM features won't error.
|
||||
await scanDirectory(path.join(storagePath, target.path), files, storagePath);
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
@@ -799,7 +1017,17 @@ async function runBackupInternal(isManual = false) {
|
||||
}).returning('id');
|
||||
runId = insertResult[0]?.id || insertResult[0];
|
||||
|
||||
const files = await service.getFilesToBackup(config.backup_include_archived);
|
||||
// Inline DB dump + fail-loud verification. The returned `databaseInfo`
|
||||
// is reused at manifest-build time below so we don't pay a second
|
||||
// `getDatabaseBackupInfo()` round-trip — see `ensureDatabaseDumpForBackup`
|
||||
// for the full rationale.
|
||||
const verifiedDatabaseInfo = await ensureDatabaseDumpForBackup(config);
|
||||
|
||||
// Pass the full config so the walker can evaluate any feature_flag
|
||||
// gates declared in the backup_paths table (e.g. `events/archived`
|
||||
// gated by `backup_include_archived`). Boolean signature is still
|
||||
// supported for legacy callers and tests — see getFilesToBackupInternal.
|
||||
const files = await service.getFilesToBackup(config);
|
||||
logger.info(`Found ${files.length} files to check for backup`);
|
||||
|
||||
let result;
|
||||
@@ -826,7 +1054,13 @@ async function runBackupInternal(isManual = false) {
|
||||
|
||||
const previousBackup = await getPreviousSuccessfulBackup(runId);
|
||||
const manifestFiles = buildManifestFiles(result.backedUpFiles, files);
|
||||
const databaseInfo = result.databaseInfo || await service.getDatabaseBackupInfo();
|
||||
// `verifiedDatabaseInfo` came from ensureDatabaseDumpForBackup at the
|
||||
// top of this run — reuse it so manifest building doesn't pay a
|
||||
// second `getDatabaseBackupInfo()` round-trip. The
|
||||
// `result.databaseInfo` branch is kept for destination implementations
|
||||
// (S3, future destinations) that override the local info on the result
|
||||
// object; falls back to the verified copy otherwise.
|
||||
const databaseInfo = result.databaseInfo || verifiedDatabaseInfo;
|
||||
|
||||
const manifestOptions = {
|
||||
backupType: previousBackup ? 'incremental' : 'full',
|
||||
@@ -869,6 +1103,18 @@ async function runBackupInternal(isManual = false) {
|
||||
logger.error('Failed to generate backup manifest:', error);
|
||||
}
|
||||
|
||||
// Per-Stage-B-path stats — bucket the actually-backed-up files
|
||||
// into their owning backup_paths row by longest-prefix match. Lets
|
||||
// the Backup History detail pane render a true breakdown
|
||||
// events/active: 142 files (3.2 GB)
|
||||
// business-docs: 17 files (4.5 MB)
|
||||
// thumbnails: 142 files (12.4 MB)
|
||||
// instead of the legacy "Photos + Archives + Other" categorization
|
||||
// that didn't reflect Stage B's data-driven walker. Falls back to
|
||||
// an empty map if backup_paths is missing (defense in depth — the
|
||||
// walker has the same fallback).
|
||||
const perPath = await computePerPathStats(result.backedUpFiles, files);
|
||||
|
||||
await db('backup_runs')
|
||||
.where('id', runId)
|
||||
.update({
|
||||
@@ -887,11 +1133,14 @@ async function runBackupInternal(isManual = false) {
|
||||
total_files_checked: files.length,
|
||||
average_file_size: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0,
|
||||
destination: destinationType,
|
||||
// Per-Stage-B-path breakdown — { [pathKey]: { count, size } }
|
||||
per_path: perPath,
|
||||
// Keep camelCase for backward compatibility
|
||||
totalFilesChecked: files.length,
|
||||
filesBackedUp: result.backedUpCount,
|
||||
totalSize: result.backedUpSize,
|
||||
averageFileSize: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0
|
||||
averageFileSize: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0,
|
||||
perPath
|
||||
})
|
||||
});
|
||||
|
||||
@@ -1071,11 +1320,34 @@ async function getBackupStatus(limit = 10) {
|
||||
|
||||
const lastRunWithManifest = lastRun ? { ...lastRun, manifestValid } : null;
|
||||
|
||||
// Separate "most recent attempt" from "most recent SUCCESS" so the
|
||||
// dashboard widget can distinguish:
|
||||
// - last attempt failed → red, "Last attempt failed at X"
|
||||
// - last attempt running → blue spinner, "In progress since X"
|
||||
// - never succeeded → critical, "No successful backup yet"
|
||||
// - last attempt succeeded → green tick, "Last backup X ago"
|
||||
// Previously the widget showed the most-recent row with a generic
|
||||
// green tick regardless of status, so a crashed run from 5 minutes
|
||||
// ago looked identical to a successful one. Same "silent failure
|
||||
// not surfaced" class Stage A was designed to fight.
|
||||
const lastSuccessful = runs.find(r => r.status === 'completed') || null;
|
||||
// Detect zombie running rows (started >30min ago, never updated)
|
||||
// — these are processes that died without writing a completed_at.
|
||||
// Surface them so the admin can tell at a glance vs a live run.
|
||||
const ZOMBIE_THRESHOLD_MS = 30 * 60 * 1000;
|
||||
const zombieRuns = runs.filter(r =>
|
||||
r.status === 'running'
|
||||
&& r.started_at
|
||||
&& (Date.now() - new Date(r.started_at).getTime()) > ZOMBIE_THRESHOLD_MS
|
||||
);
|
||||
|
||||
return {
|
||||
isRunning,
|
||||
isHealthy: Boolean(lastRun && lastRun.status === 'completed'),
|
||||
lastRun: lastRunWithManifest,
|
||||
lastBackup: lastRunWithManifest, // Alias for frontend compatibility
|
||||
lastSuccessfulBackup: lastSuccessful, // NEW — see comment above
|
||||
zombieRuns, // NEW — running >30min, likely crashed
|
||||
recentRuns: runs,
|
||||
recentBackups: runs, // Alias for frontend compatibility
|
||||
totalBackups: runs.filter(r => r.status === 'completed').length,
|
||||
|
||||
@@ -206,12 +206,20 @@ class DatabaseBackupService {
|
||||
'--format=plain',
|
||||
'--encoding=UTF8'
|
||||
];
|
||||
|
||||
// Add transaction support for consistency
|
||||
if (!options.noTransaction) {
|
||||
pgDumpOptions.push('--single-transaction');
|
||||
}
|
||||
|
||||
|
||||
// NOTE: do NOT add `--single-transaction` here. It looks like
|
||||
// the right flag for "consistent snapshot" but it isn't a pg_dump
|
||||
// option — it belongs to pg_restore / psql and pg_dump rejects it
|
||||
// with `unrecognized option: single-transaction` (exit code 1).
|
||||
// pg_dump already wraps the entire export in a single REPEATABLE
|
||||
// READ snapshot automatically (since Postgres 9.x), so consistency
|
||||
// is built in. If we ever need stricter cross-pg-cluster snapshot
|
||||
// sharing, use `--snapshot=<id>` — but the typical inline-dump
|
||||
// path doesn't need it. Bug went undetected until Stage A wired
|
||||
// this code into the user-facing "Run Backup Now" path; prior
|
||||
// callers (scheduled cron, dedicated admin DB-backup page) hit
|
||||
// the same failure but on installs that had never exercised them.
|
||||
|
||||
// Add compression if not doing it separately
|
||||
if (options.compress && !options.separateCompression) {
|
||||
pgDumpOptions.push('--compress=6');
|
||||
@@ -309,8 +317,23 @@ class DatabaseBackupService {
|
||||
// Get current schema version
|
||||
const schemaVersion = await this.getCurrentSchemaVersion();
|
||||
|
||||
// Create backup run record with version info
|
||||
const [runId] = await db('database_backup_runs').insert({
|
||||
// Create backup run record with version info.
|
||||
//
|
||||
// Insert shape divergence between SQLite + Postgres made the old
|
||||
// `const [runId] = await db(...).insert({...})` form throw
|
||||
// "(intermediate value) is not iterable" on Postgres installs:
|
||||
//
|
||||
// - SQLite-via-knex: insert() returns `[lastInsertId]` (array)
|
||||
// - Postgres-via-knex: insert() without .returning() returns an
|
||||
// empty object / row count — not iterable
|
||||
//
|
||||
// Bug went undetected until Stage A wired this method into the
|
||||
// "Run Backup Now" inline-dump path — before that, only the
|
||||
// scheduled-cron + dedicated-admin-page callers exercised it,
|
||||
// and Ralf's install had never triggered either. Cure: same
|
||||
// explicit `.returning('id')` + dual-shape coalesce pattern that
|
||||
// `backupService.js:949` uses for its own `backup_runs` insert.
|
||||
const insertResult = await db('database_backup_runs').insert({
|
||||
started_at: startTime,
|
||||
status: 'running',
|
||||
backup_type: this.dbType,
|
||||
@@ -324,8 +347,9 @@ class DatabaseBackupService {
|
||||
node_env: process.env.NODE_ENV || 'production',
|
||||
db_type: this.dbType
|
||||
})
|
||||
});
|
||||
|
||||
}).returning('id');
|
||||
const runId = insertResult[0]?.id || insertResult[0];
|
||||
|
||||
backupRun = { id: runId };
|
||||
|
||||
// Get initial checksums
|
||||
|
||||
@@ -39,6 +39,12 @@ class RestoreService {
|
||||
this.currentProgress = null;
|
||||
this.restoreLog = [];
|
||||
this.preRestoreBackupPath = null;
|
||||
// Snapshot of operator-meta settings (e.g. `restore_allow_force`)
|
||||
// captured by performDatabaseRestore BEFORE the DROP DATABASE.
|
||||
// Drained by restore() AFTER post-restore verification passes so
|
||||
// the replay doesn't inflate the row-count check. Reset per run
|
||||
// via beforeRestore() to keep state from leaking across calls.
|
||||
this.preservedMetaSnapshot = [];
|
||||
this.dbType = knexConfig.client === 'pg' ? 'postgresql' : 'sqlite';
|
||||
this.tempDir = path.join(os.tmpdir(), 'picpeak-restore');
|
||||
}
|
||||
@@ -64,6 +70,7 @@ class RestoreService {
|
||||
|
||||
this.isRunning = true;
|
||||
this.restoreLog = [];
|
||||
this.preservedMetaSnapshot = []; // reset per run
|
||||
const startTime = new Date();
|
||||
let restoreRun = null;
|
||||
|
||||
@@ -154,9 +161,37 @@ class RestoreService {
|
||||
this.log('warn', 'Pre-restore backup skipped at user request');
|
||||
}
|
||||
|
||||
// Step 5: Download backup if from S3
|
||||
// Step 5: Download backup if from S3, or resolve the local root.
|
||||
//
|
||||
// The wizard passes `options.source = 'local'` (the SOURCE TYPE
|
||||
// string) — not a path. The old code assigned that string to
|
||||
// `localBackupPath` verbatim and every downstream `path.join(...)`
|
||||
// ended up with junk like `local/database/<file>.sql.gz`. Caused
|
||||
// the disaster-recovery restore flow to fail with
|
||||
// `Database backup file not found: local/database/...` even when
|
||||
// the manifest recorded the correct absolute path AND the file
|
||||
// existed at exactly that path on disk.
|
||||
//
|
||||
// Resolve `'local'` to the configured backup destination root by
|
||||
// reading `backup_destination_path` from app_settings. That's the
|
||||
// same root the file-backup walker writes to, so every relative
|
||||
// `file.path` in the manifest resolves correctly via
|
||||
// `path.join(localBackupPath, file.path)` further down.
|
||||
let localBackupPath = options.source;
|
||||
if (options.source.startsWith('s3://')) {
|
||||
if (options.source === 'local') {
|
||||
try {
|
||||
const row = await db('app_settings')
|
||||
.where('setting_key', 'backup_destination_path')
|
||||
.first();
|
||||
if (row?.setting_value) {
|
||||
let parsed;
|
||||
try { parsed = JSON.parse(row.setting_value); } catch (_) { parsed = row.setting_value; }
|
||||
if (parsed) localBackupPath = parsed;
|
||||
}
|
||||
} catch (err) {
|
||||
this.log('warn', `Could not resolve backup_destination_path: ${err.message}`);
|
||||
}
|
||||
} else if (options.source.startsWith('s3://')) {
|
||||
this.updateProgress('Downloading backup from S3...');
|
||||
localBackupPath = await this.downloadFromS3(options.source, manifest, options);
|
||||
}
|
||||
@@ -193,6 +228,89 @@ class RestoreService {
|
||||
throw new Error(`Post-restore verification failed: ${verification.errors.join(', ')}`);
|
||||
}
|
||||
|
||||
// Step 7b: Replay operator-meta settings AFTER verification.
|
||||
//
|
||||
// `performDatabaseRestore` stashed the pre-DROP snapshot of
|
||||
// operator-meta keys on `this.preservedMetaSnapshot`. We drain
|
||||
// it here, AFTER verification has already confirmed the
|
||||
// restored DB matches the backup's row counts. Running this
|
||||
// upsert sequence here instead of inside performDatabaseRestore
|
||||
// (where it used to live) prevents the replay from inflating
|
||||
// the post-restore row count and tripping the verification
|
||||
// check — see the round-3 PR #596 notes for the full story.
|
||||
//
|
||||
// UPSERT by setting_key: if the backup had the same key with a
|
||||
// different value, we overwrite; if the row doesn't exist in
|
||||
// the backup, we insert. Either way the operator's pre-restore
|
||||
// policy survives. SQLite branch leaves the snapshot empty so
|
||||
// this block is a no-op there.
|
||||
if (this.preservedMetaSnapshot && this.preservedMetaSnapshot.length > 0) {
|
||||
try {
|
||||
for (const row of this.preservedMetaSnapshot) {
|
||||
await db('app_settings')
|
||||
.insert({
|
||||
setting_key: row.setting_key,
|
||||
setting_value: row.setting_value,
|
||||
setting_type: row.setting_type || 'restore',
|
||||
updated_at: new Date(),
|
||||
})
|
||||
.onConflict('setting_key')
|
||||
.merge({
|
||||
setting_value: row.setting_value,
|
||||
updated_at: new Date(),
|
||||
});
|
||||
}
|
||||
this.log('info', `Replayed ${this.preservedMetaSnapshot.length} restore-meta setting(s) post-verification`);
|
||||
} catch (err) {
|
||||
this.log('warn', `Could not replay restore-meta settings (admin may need to re-set them): ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 7c: Apply any post-backup migrations to the restored DB.
|
||||
//
|
||||
// The backup carries the schema state of whatever migrations had
|
||||
// been applied at backup time. If the running image is NEWER —
|
||||
// because the admin upgraded picpeak between when the backup
|
||||
// was taken and when they restored — the restored DB ends up
|
||||
// mismatched against the running code: queries fail, new
|
||||
// columns are missing, new tables don't exist.
|
||||
//
|
||||
// Previously the comment said "deferred to next container
|
||||
// restart" — but that left the running process serving a
|
||||
// mismatched schema until the operator manually restarted.
|
||||
// Not acceptable per the "backup must restore completely even
|
||||
// when new features have been added in the meantime" contract.
|
||||
//
|
||||
// Implementation: shell out to `npm run migrate:safe`, which is
|
||||
// the EXACT script wait-for-db.sh runs on boot. Running it as a
|
||||
// subprocess means no risk to our reinit'd pool (subprocess gets
|
||||
// its own knex instance, destroys it on exit; our parent pool
|
||||
// is untouched). Idempotent — migrations already applied are
|
||||
// tracked in the restored `migrations` table and get skipped.
|
||||
//
|
||||
// Failure is non-fatal: the restore data itself is in place,
|
||||
// and the next container restart's wait-for-db.sh will retry.
|
||||
// Surfacing the error gives the operator a chance to investigate
|
||||
// proactively rather than discovering it on the next 500 from
|
||||
// a missing column.
|
||||
try {
|
||||
this.log('info', 'Applying post-restore migrations to restored database...');
|
||||
this.updateProgress('Applying any post-backup migrations...');
|
||||
const backendRoot = path.join(__dirname, '..', '..');
|
||||
const { stderr } = await spawnAsync('npm', ['run', 'migrate:safe'], {
|
||||
cwd: backendRoot,
|
||||
env: { ...process.env },
|
||||
});
|
||||
if (stderr && stderr.trim()) {
|
||||
this.log('info', `Post-restore migrate:safe stderr: ${stderr.slice(0, 500)}`);
|
||||
}
|
||||
this.log('info', 'Post-restore migrations applied');
|
||||
} catch (migErr) {
|
||||
this.log('warn',
|
||||
`Post-restore migrate:safe failed — restore data is in place but the schema may lag the running image. ` +
|
||||
`A container restart will retry via wait-for-db.sh. Error: ${migErr.message}`);
|
||||
}
|
||||
|
||||
// Step 8: Clean up temporary files
|
||||
if (localBackupPath !== options.source) {
|
||||
await fs.unlink(localBackupPath).catch(err =>
|
||||
@@ -208,6 +326,15 @@ class RestoreService {
|
||||
await db('restore_runs').where('id', runId).update({
|
||||
completed_at: endTime,
|
||||
status: 'completed',
|
||||
// Default for the column is `false`. Without this line, every
|
||||
// SUCCESSFUL restore ends up with `status='completed',
|
||||
// was_successful=false` — which the BackupDashboard "last
|
||||
// successful restore" widget then filters out, and any future
|
||||
// audit query that gates on was_successful misses the row
|
||||
// entirely. Cosmetic but enough to mislead an operator
|
||||
// scanning restore history. Catches Ralf 2026-06-01 + maintainer
|
||||
// PR #596 review note about the cosmetic.
|
||||
was_successful: true,
|
||||
duration_seconds: durationSeconds,
|
||||
pre_restore_backup_path: this.preRestoreBackupPath,
|
||||
statistics: JSON.stringify({
|
||||
@@ -242,12 +369,53 @@ class RestoreService {
|
||||
} catch (error) {
|
||||
this.log('error', 'Restore failed', { error: error.message, stack: error.stack });
|
||||
|
||||
// Update restore run record
|
||||
// Always attempt rollback when a pre-restore backup exists.
|
||||
// Historically rollback was only triggered when post-restore
|
||||
// verification failed (inside the try block) — anything that
|
||||
// threw earlier (path-resolution bugs, pg_restore failure, file
|
||||
// copy errors) left the destination half-clobbered and forced
|
||||
// the admin to do another reset-from-volume cycle before the
|
||||
// next attempt could be honest. Fixing the rollback here closes
|
||||
// the "every failed restore makes the next one worse" footgun.
|
||||
let rollbackAttempted = false;
|
||||
let rollbackSucceeded = false;
|
||||
let rollbackError = null;
|
||||
if (this.preRestoreBackupPath) {
|
||||
rollbackAttempted = true;
|
||||
try {
|
||||
this.log('info', 'Attempting rollback from pre-restore safety backup', {
|
||||
path: this.preRestoreBackupPath,
|
||||
});
|
||||
await this.attemptRollback(this.preRestoreBackupPath);
|
||||
rollbackSucceeded = true;
|
||||
this.log('info', 'Rollback completed');
|
||||
} catch (rbErr) {
|
||||
rollbackError = rbErr.message;
|
||||
this.log('error', 'Rollback FAILED — install may be in a partial state',
|
||||
{ error: rbErr.message, stack: rbErr.stack });
|
||||
}
|
||||
} else {
|
||||
this.log('warn', 'No pre-restore backup available — cannot auto-rollback. ' +
|
||||
'Destination may be in a partial state. Verify business-docs/ and the DB before retrying.');
|
||||
}
|
||||
|
||||
// Update restore run record. We persist BOTH the original
|
||||
// restore failure AND the rollback status so the admin can tell
|
||||
// from a single SQL query which scenario they're in:
|
||||
// - rollback succeeded → destination is back to pre-restore state, safe to retry
|
||||
// - rollback failed → partial state, admin must inspect before next attempt
|
||||
// - rollback skipped → user opted out via skipPreBackup; same as above
|
||||
if (restoreRun) {
|
||||
const failureMessage = rollbackAttempted
|
||||
? (rollbackSucceeded
|
||||
? `${error.message} (rolled back successfully to pre-restore state)`
|
||||
: `${error.message} | ROLLBACK ALSO FAILED: ${rollbackError} — destination is in a partial state, inspect before retrying`)
|
||||
: `${error.message} (no pre-restore backup available — destination may be partial)`;
|
||||
await db('restore_runs').where('id', restoreRun.id).update({
|
||||
completed_at: new Date(),
|
||||
status: 'failed',
|
||||
error_message: error.message,
|
||||
error_message: failureMessage,
|
||||
was_rollback_attempted: rollbackAttempted,
|
||||
restore_log: JSON.stringify(this.restoreLog)
|
||||
});
|
||||
}
|
||||
@@ -255,7 +423,9 @@ class RestoreService {
|
||||
// Send failure notification
|
||||
await this.sendRestoreNotification('failure', {
|
||||
error: error.message,
|
||||
restoreType: options.restoreType
|
||||
restoreType: options.restoreType,
|
||||
rollbackAttempted,
|
||||
rollbackSucceeded,
|
||||
});
|
||||
|
||||
throw error;
|
||||
@@ -371,21 +541,32 @@ class RestoreService {
|
||||
}
|
||||
}
|
||||
|
||||
// Check if restoring would overwrite existing data
|
||||
// Check if restoring would overwrite existing data.
|
||||
//
|
||||
// NOTE: pg-driver returns `count('* as count')` as a STRING (it
|
||||
// serialises `bigint` to string to avoid JS precision loss for
|
||||
// huge counts) — see PR #596 review for the `bigint`-as-string
|
||||
// discussion. Both blocks below coerce to `Number` before
|
||||
// comparing AND before interpolating into the warning text, so
|
||||
// the count renders as `5` not `"5"` regardless of DB driver.
|
||||
// Don't drop the `Number()` calls without also re-auditing the
|
||||
// strict-equality call sites flagged in the same review.
|
||||
if (options.restoreType === 'full' || options.restoreType === 'database') {
|
||||
const eventCount = await db('events').count('* as count').first();
|
||||
if (eventCount && eventCount.count > 0) {
|
||||
validation.warnings.push(`Database contains ${eventCount.count} existing events that will be overwritten`);
|
||||
const eventCountN = Number(eventCount?.count || 0);
|
||||
if (eventCountN > 0) {
|
||||
validation.warnings.push(`Database contains ${eventCountN} existing events that will be overwritten`);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for active users
|
||||
// Check for active users (same coercion contract as above).
|
||||
const activeUsers = await db('admin_users')
|
||||
.where('is_active', formatBoolean(true))
|
||||
.count('* as count')
|
||||
.first();
|
||||
if (activeUsers && activeUsers.count > 0) {
|
||||
validation.warnings.push(`There are ${activeUsers.count} active admin users`);
|
||||
const activeUsersN = Number(activeUsers?.count || 0);
|
||||
if (activeUsersN > 0) {
|
||||
validation.warnings.push(`There are ${activeUsersN} active admin users`);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
@@ -661,13 +842,55 @@ class RestoreService {
|
||||
throw new Error('No database backup file found in manifest');
|
||||
}
|
||||
|
||||
const dbBackupPath = path.join(backupPath, 'database', path.basename(dbBackupFile));
|
||||
|
||||
// Check if backup file exists
|
||||
try {
|
||||
await fs.access(dbBackupPath);
|
||||
} catch (error) {
|
||||
throw new Error(`Database backup file not found: ${dbBackupPath}`);
|
||||
// Layered resolution for the database dump path:
|
||||
//
|
||||
// 1. Manifest stores the absolute path the dumper wrote to
|
||||
// (e.g. `/backup/database/picpeak-db-postgresql-<ts>.sql.gz`).
|
||||
// That's the truth — try it first.
|
||||
// 2. Some older manifests store a path RELATIVE to the file-backup
|
||||
// destination root (`database/<file>.sql.gz`). Reconstruct that
|
||||
// way as a fallback.
|
||||
// 3. Final fallback: `<backupPath>/database/<basename>`, the
|
||||
// historical reconstruction used before this fix. Preserved so
|
||||
// no existing valid path breaks.
|
||||
//
|
||||
// The original code used (3) exclusively, which meant the restore
|
||||
// service ignored the absolute path the manifest recorded and
|
||||
// looked under a synthetic `<backupPath>/database/<file>` root —
|
||||
// which on Ralf's install became `local/database/<file>` because
|
||||
// `backupPath` was the source type string, not a directory. Caused
|
||||
// the canonical disaster-recovery flow to fail with
|
||||
// `Database backup file not found: local/database/...sql.gz`
|
||||
// even though the file existed at exactly the path the manifest
|
||||
// recorded.
|
||||
const candidates = [
|
||||
// (1) Honour absolute paths recorded by the dumper.
|
||||
path.isAbsolute(dbBackupFile) ? dbBackupFile : null,
|
||||
// (2) Relative-to-backupPath as-stored (no basename munging).
|
||||
path.join(backupPath, dbBackupFile),
|
||||
// (3) Legacy reconstruct.
|
||||
path.join(backupPath, 'database', path.basename(dbBackupFile)),
|
||||
].filter(Boolean);
|
||||
|
||||
let dbBackupPath = null;
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
await fs.access(candidate);
|
||||
dbBackupPath = candidate;
|
||||
break;
|
||||
} catch (_) {
|
||||
// try next candidate
|
||||
}
|
||||
}
|
||||
|
||||
if (!dbBackupPath) {
|
||||
throw new Error(
|
||||
`Database backup file not found. Tried: ${candidates.join(', ')}. ` +
|
||||
`Manifest recorded path: ${dbBackupFile}. ` +
|
||||
`Hint: this usually means the manifest's database.backup_file path no longer ` +
|
||||
`exists on disk (deleted? moved? volume not mounted?). Check ` +
|
||||
`~/<your-compose-dir>/backup/database/ on the host.`
|
||||
);
|
||||
}
|
||||
|
||||
// Decompress if needed
|
||||
@@ -679,6 +902,31 @@ class RestoreService {
|
||||
restoreFile = decompressedPath;
|
||||
}
|
||||
|
||||
// Snapshot of operator-meta keys captured BEFORE the DROP.
|
||||
// Stashed onto `this.preservedMetaSnapshot` so the parent
|
||||
// `restore()` method can drain + apply it AFTER post-restore
|
||||
// verification passes. Order matters here:
|
||||
//
|
||||
// - PR #596 round 1: lifted the declaration above the
|
||||
// SQLite/PG split to fix a ReferenceError when the replay
|
||||
// was inline at the bottom of this method.
|
||||
// - PR #596 round 3: moved the REPLAY itself out of here and
|
||||
// into restore(), because the round-1 in-method replay ran
|
||||
// BEFORE post-restore verification — which then counted the
|
||||
// replayed row and flagged
|
||||
// Table app_settings row count mismatch: expected 190, got 191
|
||||
// as a verification failure even though both Stage A and
|
||||
// the replay had succeeded. Verification now sees the
|
||||
// as-restored DB (matches the backup exactly), replay layers
|
||||
// on top after verification has signed off.
|
||||
//
|
||||
// SQLite branch leaves `preservedMetaSnapshot` empty — verification
|
||||
// and replay both no-op for it, unchanged behaviour.
|
||||
const PRESERVED_META_KEYS = [
|
||||
'restore_allow_force',
|
||||
'restore_allow_force_auto_upgraded',
|
||||
];
|
||||
|
||||
try {
|
||||
if (this.dbType === 'sqlite') {
|
||||
// SQLite restore
|
||||
@@ -715,24 +963,210 @@ class RestoreService {
|
||||
// PostgreSQL restore
|
||||
const { host, port, user, password, database } = knexConfig.connection;
|
||||
const env = { ...process.env, PGPASSWORD: password };
|
||||
|
||||
|
||||
// Snapshot operator-meta settings BEFORE the DROP so we can
|
||||
// restore them after the psql load. These keys are about how
|
||||
// the operator wants the install to behave (force-restore
|
||||
// permission, auto-upgrade tracking), not user-facing state —
|
||||
// they should NOT be overwritten by whatever values the backup
|
||||
// happens to contain.
|
||||
//
|
||||
// Chicken-and-egg this closes: `restore_allow_force` defaults
|
||||
// to true (post tonight's migration 032 edit), but every
|
||||
// restore would overwrite it with whatever the backup carried.
|
||||
// Admin sets it to true → restores → wakes up with the row
|
||||
// back to whatever was in the backup. Two consecutive restores
|
||||
// needed the SQL workaround again. With this snapshot/replay,
|
||||
// the operator's policy persists across restores.
|
||||
//
|
||||
// PRESERVED_META_KEYS is declared above the SQLite/PG split
|
||||
// (~L820). The snapshot READ happens here in the PG branch
|
||||
// (must run before DROP), but is stashed on
|
||||
// `this.preservedMetaSnapshot` for the parent `restore()`
|
||||
// method to consume AFTER verification — see the round-3
|
||||
// notes there.
|
||||
try {
|
||||
this.preservedMetaSnapshot = await db('app_settings')
|
||||
.whereIn('setting_key', PRESERVED_META_KEYS)
|
||||
.select('setting_key', 'setting_value', 'setting_type');
|
||||
this.log('info', `Snapshotted ${this.preservedMetaSnapshot.length} restore-meta setting(s) for post-restore replay`, {
|
||||
keys: this.preservedMetaSnapshot.map(r => r.setting_key),
|
||||
});
|
||||
} catch (err) {
|
||||
this.log('warn', `Could not snapshot restore-meta settings (continuing): ${err.message}`);
|
||||
}
|
||||
|
||||
// `psql` with no `-d` defaults to a database whose name matches
|
||||
// the connecting user, NOT a maintenance DB. So on installs
|
||||
// where the user's home DB doesn't exist (e.g. user=`picpeak`,
|
||||
// target DB=`picpeak_prod`, no `picpeak` DB), the next two
|
||||
// statements failed with:
|
||||
// FATAL: database "picpeak" does not exist
|
||||
// even though the actual target DB was alive and connectable.
|
||||
//
|
||||
// Fix: explicitly connect to `postgres` (the maintenance DB
|
||||
// every PG cluster ships with) for the DROP/CREATE. We can't
|
||||
// connect to the target DB itself anyway — DROP DATABASE
|
||||
// refuses to run while a connection is open to it.
|
||||
//
|
||||
// Use `DB_CHECK_DB` env var as an override hook (matches the
|
||||
// pattern wait-for-db.sh already exposes) for installs where
|
||||
// the `postgres` DB is restricted to superusers.
|
||||
const maintenanceDb = process.env.DB_CHECK_DB || 'postgres';
|
||||
|
||||
// The backend's own knex pool holds N active connections to
|
||||
// the target database (default 5-25 per knexfile.js). PostgreSQL
|
||||
// refuses DROP DATABASE while any session is connected:
|
||||
// ERROR: database "X" is being accessed by other users
|
||||
// DETAIL: There are N other sessions using the database.
|
||||
// We have to evict those sessions ourselves before issuing the
|
||||
// DROP. Two-step approach:
|
||||
// 1. Close knex's own pool so we don't fight ourselves.
|
||||
// 2. pg_terminate_backend() the rest (other server replicas,
|
||||
// pg_stat_activity stragglers, leftover idle txns).
|
||||
//
|
||||
// After CREATE DATABASE, knex will lazily re-open the pool on
|
||||
// the next query — handled by db.js's connection retry logic.
|
||||
this.log('warn', 'Closing knex pool before dropping target database...');
|
||||
try { await db.destroy(); } catch (poolErr) {
|
||||
this.log('warn', `Pool destroy threw (continuing): ${poolErr.message}`);
|
||||
}
|
||||
|
||||
this.log('warn', 'Terminating any remaining sessions on target database...', {
|
||||
target: database,
|
||||
});
|
||||
// pg_terminate_backend takes a pid. Kill every session against
|
||||
// the target DB except our own connection (which is to the
|
||||
// maintenance DB anyway). Wrapped in `SELECT ... FROM ... WHERE`
|
||||
// so we get one psql round-trip instead of N.
|
||||
await spawnAsync('psql', [
|
||||
'-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb,
|
||||
'-c',
|
||||
`SELECT pg_terminate_backend(pid) FROM pg_stat_activity ` +
|
||||
`WHERE datname = '${database.replace(/'/g, "''")}' AND pid <> pg_backend_pid()`,
|
||||
], { env });
|
||||
|
||||
// Drop and recreate database (extremely dangerous!)
|
||||
this.log('warn', 'Dropping and recreating PostgreSQL database...');
|
||||
|
||||
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-c', `DROP DATABASE IF EXISTS ${database}`], { env });
|
||||
this.log('warn', 'Dropping and recreating PostgreSQL database...', {
|
||||
target: database, via: maintenanceDb,
|
||||
});
|
||||
|
||||
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-c', `CREATE DATABASE ${database}`], { env });
|
||||
// WITH (FORCE) on Postgres 13+ kills any remaining connections
|
||||
// atomically with the DROP. On older Postgres the FORCE option
|
||||
// doesn't exist, so we fall back to plain DROP IF EXISTS — by
|
||||
// which point pg_terminate_backend should have cleared the
|
||||
// table. Try FORCE first, fall back to plain on syntax error.
|
||||
try {
|
||||
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb,
|
||||
'-c', `DROP DATABASE IF EXISTS "${database}" WITH (FORCE)`], { env });
|
||||
} catch (forceErr) {
|
||||
// PG < 13: WITH (FORCE) is a syntax error. Plain DROP after
|
||||
// our pg_terminate_backend pass should now succeed.
|
||||
this.log('info', 'DROP DATABASE WITH (FORCE) not supported — falling back to plain DROP', {
|
||||
error: forceErr.message,
|
||||
});
|
||||
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb,
|
||||
'-c', `DROP DATABASE IF EXISTS "${database}"`], { env });
|
||||
}
|
||||
|
||||
// Restore from backup
|
||||
await spawnAsync('psql', ['-h', host, '-p', String(port), '-U', user, '-d', maintenanceDb, '-c', `CREATE DATABASE "${database}"`], { env });
|
||||
|
||||
// Restore from backup — this one DOES connect to the target DB.
|
||||
await spawnFromFile('psql', ['-h', host, '-p', String(port), '-U', user, '-d', database], restoreFile, { env });
|
||||
|
||||
// Re-sync every SERIAL / IDENTITY sequence in the public schema
|
||||
// to MAX(id)+1 of its owning table. pg_dump emits setval()
|
||||
// statements, but they don't always land cleanly when:
|
||||
// - the dump has `--clean` (the setval may execute before
|
||||
// the rebuilt rows, depending on dump ordering)
|
||||
// - the in-process knex pool had a cached sequence value
|
||||
// before db.destroy() (already mitigated, but defensive)
|
||||
// - rows were inserted mid-restore (the pre-restore safety
|
||||
// backup creates a database_backup_runs row before DROP)
|
||||
// Result if skipped: every subsequent INSERT into a serial-id
|
||||
// table fails with `duplicate key value violates unique
|
||||
// constraint "<table>_pkey"`. Surfaced on Ralf's install as
|
||||
// "A record with this value already exists" on every CRUD
|
||||
// action AND `database_backup_runs_pkey` violation on the
|
||||
// next Run Backup Now. Fix is a single DO block that walks
|
||||
// pg_class + pg_attribute and setval()s each sequence to
|
||||
// GREATEST(MAX(<col>), 1). Cheap (a few ms even on large
|
||||
// schemas), safe (doesn't touch row data), idempotent.
|
||||
this.log('info', 'Re-syncing PostgreSQL sequences to MAX(id) of each table...');
|
||||
await spawnAsync('psql', [
|
||||
'-h', host, '-p', String(port), '-U', user, '-d', database,
|
||||
'-c',
|
||||
`DO $$
|
||||
DECLARE
|
||||
r RECORD;
|
||||
max_id BIGINT;
|
||||
BEGIN
|
||||
FOR r IN
|
||||
SELECT n.nspname AS schema_name, t.relname AS table_name, a.attname AS column_name,
|
||||
pg_get_serial_sequence(quote_ident(n.nspname) || '.' || quote_ident(t.relname), a.attname) AS seq_name
|
||||
FROM pg_class t
|
||||
JOIN pg_namespace n ON n.oid = t.relnamespace
|
||||
JOIN pg_attribute a ON a.attrelid = t.oid
|
||||
WHERE n.nspname = 'public'
|
||||
AND t.relkind = 'r'
|
||||
AND a.attnum > 0
|
||||
AND NOT a.attisdropped
|
||||
AND pg_get_serial_sequence(quote_ident(n.nspname) || '.' || quote_ident(t.relname), a.attname) IS NOT NULL
|
||||
LOOP
|
||||
EXECUTE format('SELECT COALESCE(MAX(%I), 0) FROM %I.%I', r.column_name, r.schema_name, r.table_name) INTO max_id;
|
||||
EXECUTE format('SELECT setval(%L, %s, true)', r.seq_name, GREATEST(max_id, 1));
|
||||
END LOOP;
|
||||
END $$;`
|
||||
], { env });
|
||||
this.log('info', 'Sequence resync completed');
|
||||
}
|
||||
|
||||
// Re-initialize database connection
|
||||
const { db: newDb } = require('../database/db');
|
||||
|
||||
// Run migrations to ensure schema is up to date
|
||||
this.log('info', 'Running database migrations...');
|
||||
await newDb.migrate.latest();
|
||||
// Re-initialize the in-process knex pool. The DROP/CREATE
|
||||
// DATABASE pair above destroyed our connections and the recreated
|
||||
// database has a different pg_database OID — any pooled
|
||||
// connection from before would either be dead or pointed at a
|
||||
// ghost. Without explicit reinit, every query in the process
|
||||
// after restore returns `Error: Unable to acquire a connection`
|
||||
// until the container is manually restarted (and admin sees
|
||||
// "An error occurred" on the login screen even after the restore
|
||||
// technically succeeded). reinitPool destroys + rebuilds the
|
||||
// pool and probes the new one with `SELECT 1` so failures here
|
||||
// surface immediately instead of polluting the next request.
|
||||
const { reinitPool } = require('../database/db');
|
||||
this.log('info', 'Re-initializing knex pool against the restored database...');
|
||||
await reinitPool();
|
||||
this.log('info', 'Knex pool re-initialized');
|
||||
|
||||
// NOTE: we deliberately do NOT call `db.migrate.latest()` here.
|
||||
//
|
||||
// The picpeak migrations directory contains `helpers.js` (a
|
||||
// shared helper module, not a migration), plus `core/` and
|
||||
// `legacy/` subdirectories. Knex's built-in migrator scans the
|
||||
// top-level directory and rejects any file without `up`/`down`
|
||||
// exports — so `db.migrate.latest()` throws
|
||||
// Invalid migration: helpers.js must have both an up and down function
|
||||
// every time it runs in this codebase. The production code path
|
||||
// uses `npm run migrate:safe` (run-migrations-safe.js) which
|
||||
// knows to skip helpers.js + walks core/ explicitly.
|
||||
//
|
||||
// The safe runner gets invoked AFTER verification in restore()
|
||||
// (see step 7c) to apply any post-backup migrations to the
|
||||
// restored DB. This closes the contract "backup must restore
|
||||
// completely even when new features have been added in the
|
||||
// meantime" — without this step, restoring an old backup on a
|
||||
// newer image would leave the running process serving a
|
||||
// mismatched schema until the next container restart.
|
||||
this.log('info', 'Schema migrations deferred to restore() step 7c (npm run migrate:safe subprocess)');
|
||||
|
||||
// NOTE: operator-meta REPLAY does NOT happen here any more.
|
||||
// PR #596 round 3: if the replay runs inside performDatabaseRestore,
|
||||
// it lands BEFORE post-restore verification — and verification
|
||||
// then counts the replayed row as a mismatch (e.g. "expected 190,
|
||||
// got 191" because the fresh-install seeded
|
||||
// `restore_allow_force_auto_upgraded` that wasn't in the backup).
|
||||
// Replay is now drained by the parent `restore()` method AFTER
|
||||
// verification passes. Snapshot lives on
|
||||
// `this.preservedMetaSnapshot` for that drain.
|
||||
|
||||
return { success: true };
|
||||
|
||||
@@ -905,9 +1339,20 @@ class RestoreService {
|
||||
for (const [table, expected] of Object.entries(manifest.database.row_counts)) {
|
||||
try {
|
||||
const result = await db(table).count('* as count').first();
|
||||
if (result.count !== expected.rowCount) {
|
||||
// pg-driver serialises `bigint` as string to preserve
|
||||
// precision for huge counts, so `result.count` on PG is
|
||||
// e.g. `"16"` while the manifest's `expected.rowCount`
|
||||
// is the JS number `16`. Strict `!==` flagged every
|
||||
// match as a mismatch on PG. Caught on PR #596 review:
|
||||
// `Table activity_logs row count mismatch:
|
||||
// expected 16, got 16`
|
||||
// every table, all "matching". Coerce both sides to
|
||||
// Number to compare reliably across SQLite (number) and
|
||||
// PG (string).
|
||||
const actual = Number(result.count);
|
||||
if (actual !== expected.rowCount) {
|
||||
verification.errors.push(
|
||||
`Table ${table} row count mismatch: expected ${expected.rowCount}, got ${result.count}`
|
||||
`Table ${table} row count mismatch: expected ${expected.rowCount}, got ${actual}`
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -41,60 +41,128 @@ function spawnAsync(cmd, args = [], options = {}) {
|
||||
|
||||
/**
|
||||
* Run a command and redirect stdout to a file (replaces shell `> file`).
|
||||
*
|
||||
* Historically this passed `fs.createWriteStream(outputPath)` directly as
|
||||
* `stdio[1]` to `child_process.spawn`. That relied on Node auto-extracting
|
||||
* the WriteStream's `.fd` — but the stream opens async, so on a fast call
|
||||
* `fd` is still `null` when `spawn()` reads it. Older Node releases would
|
||||
* tolerate this; Node 22 throws synchronously with
|
||||
* `The argument 'stdio' is invalid. Received WriteStream { fd: null, ... }`.
|
||||
*
|
||||
* Cure: use `stdio: ['ignore', 'pipe', 'pipe']` and wire the WriteStream
|
||||
* up via the streams API (`stdout.pipe(outStream)`). Works on every Node
|
||||
* version; also gives us a clean error bridge from both the WriteStream
|
||||
* AND the child process to the promise, instead of the previous code's
|
||||
* blind `outStream.destroy()` / `outStream.end()` calls that left stream
|
||||
* errors uncaught (Node 22 process-fatal — separate footgun this fixes
|
||||
* by the same change).
|
||||
*
|
||||
* Used by:
|
||||
* - databaseBackup.createPostgreSQLBackup (inline-dump path, the
|
||||
* thing that just bit Ralf's install)
|
||||
* - restoreService pre-restore safety snapshot (would have hit the
|
||||
* same on next restore attempt)
|
||||
*/
|
||||
function spawnToFile(cmd, args, outputPath, options = {}) {
|
||||
const fs = require('fs');
|
||||
return new Promise((resolve, reject) => {
|
||||
const outStream = fs.createWriteStream(outputPath);
|
||||
let settled = false;
|
||||
const settleReject = (err) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
try { outStream.destroy(); } catch (_) { /* best effort */ }
|
||||
reject(err);
|
||||
};
|
||||
const settleResolve = (value) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve(value);
|
||||
};
|
||||
|
||||
// Bridge WriteStream errors (EACCES, ENOSPC, etc.) to the promise.
|
||||
// Without this, an unhandled 'error' event on the stream is process-
|
||||
// fatal on Node 22 and bypasses the caller's try/catch entirely —
|
||||
// which is exactly the failure mode that crashed the picpeak
|
||||
// backend container on its first inline-dump attempt.
|
||||
outStream.on('error', settleReject);
|
||||
|
||||
const child = spawn(cmd, args, {
|
||||
shell: false,
|
||||
...options,
|
||||
stdio: ['ignore', outStream, 'pipe']
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
});
|
||||
|
||||
// Pipe stdout → file. The pipe call attaches its own 'error'
|
||||
// handlers on both ends so a child-stdout failure also reaches us.
|
||||
child.stdout.pipe(outStream);
|
||||
|
||||
const stderrChunks = [];
|
||||
child.stderr.on('data', chunk => stderrChunks.push(chunk));
|
||||
child.stderr.on('error', settleReject);
|
||||
|
||||
child.on('error', (err) => {
|
||||
outStream.destroy();
|
||||
reject(err);
|
||||
});
|
||||
child.on('error', settleReject);
|
||||
child.on('close', (code) => {
|
||||
outStream.end();
|
||||
const stderr = Buffer.concat(stderrChunks).toString();
|
||||
if (code !== 0) {
|
||||
const err = new Error(`${cmd} exited with code ${code}: ${stderr}`);
|
||||
err.code = code;
|
||||
err.stderr = stderr;
|
||||
return reject(err);
|
||||
}
|
||||
resolve({ stderr });
|
||||
// Wait for the file write to flush before resolving — otherwise
|
||||
// a fast 'close' could resolve while the WriteStream still has
|
||||
// buffered bytes, producing a truncated dump.
|
||||
outStream.end(() => {
|
||||
const stderr = Buffer.concat(stderrChunks).toString();
|
||||
if (code !== 0) {
|
||||
const err = new Error(`${cmd} exited with code ${code}: ${stderr}`);
|
||||
err.code = code;
|
||||
err.stderr = stderr;
|
||||
return settleReject(err);
|
||||
}
|
||||
settleResolve({ stderr });
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a command and pipe a file into stdin (replaces shell `< file`).
|
||||
*
|
||||
* Same Node 22 stdio strictness applies as for `spawnToFile` above — the
|
||||
* ReadStream `fd` is null at spawn time. Use `stdio[0] = 'pipe'` and pipe
|
||||
* the file stream into `child.stdin` via the streams API instead.
|
||||
*/
|
||||
function spawnFromFile(cmd, args, inputPath, options = {}) {
|
||||
const fs = require('fs');
|
||||
return new Promise((resolve, reject) => {
|
||||
const inStream = fs.createReadStream(inputPath);
|
||||
let settled = false;
|
||||
const settleReject = (err) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
try { inStream.destroy(); } catch (_) { /* best effort */ }
|
||||
reject(err);
|
||||
};
|
||||
const settleResolve = (value) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve(value);
|
||||
};
|
||||
|
||||
inStream.on('error', settleReject);
|
||||
|
||||
const child = spawn(cmd, args, {
|
||||
shell: false,
|
||||
...options,
|
||||
stdio: [inStream, 'pipe', 'pipe']
|
||||
stdio: ['pipe', 'pipe', 'pipe']
|
||||
});
|
||||
|
||||
inStream.pipe(child.stdin);
|
||||
|
||||
const stdoutChunks = [];
|
||||
const stderrChunks = [];
|
||||
child.stdout.on('data', chunk => stdoutChunks.push(chunk));
|
||||
child.stdout.on('error', settleReject);
|
||||
child.stderr.on('data', chunk => stderrChunks.push(chunk));
|
||||
child.stderr.on('error', settleReject);
|
||||
child.stdin.on('error', settleReject);
|
||||
|
||||
child.on('error', (err) => {
|
||||
inStream.destroy();
|
||||
reject(err);
|
||||
});
|
||||
child.on('error', settleReject);
|
||||
child.on('close', (code) => {
|
||||
const stdout = Buffer.concat(stdoutChunks).toString();
|
||||
const stderr = Buffer.concat(stderrChunks).toString();
|
||||
@@ -103,9 +171,9 @@ function spawnFromFile(cmd, args, inputPath, options = {}) {
|
||||
err.code = code;
|
||||
err.stdout = stdout;
|
||||
err.stderr = stderr;
|
||||
return reject(err);
|
||||
return settleReject(err);
|
||||
}
|
||||
resolve({ stdout, stderr });
|
||||
settleResolve({ stdout, stderr });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user