From e7dffa656bff8e4bb366bbd7f20f973c51ccb318 Mon Sep 17 00:00:00 2001 From: Luca <102960244+Luca-Timo@users.noreply.github.com> Date: Sun, 31 May 2026 22:54:09 +0200 Subject: [PATCH] feat(backup-stats): per-Stage-B-path counters in backup statistics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the last gap from tonight's backup-hardening: backup_runs. statistics now carries a `per_path` map keyed by backup_paths.path (e.g. `events/active`, `business-docs`), with per-bucket count + size. Backend (backupService.js): - new `computePerPathStats(backedUpFiles, allFiles)` helper that bucket-sorts each backed-up file into its owning backup_paths row by longest-prefix match. Reuses the same backup_paths source the walker reads, so toggling include_in_default off propagates correctly. Falls back to LEGACY_BACKUP_PATHS if the table is missing. - runBackupInternal calls it after the destination implementation reports back, includes the result in statistics under both snake_case (`per_path`) and camelCase (`perPath`) keys for the same alias treatment the existing fields get. Frontend (BackupHistory.jsx): - Backup History detail pane now renders one row per per_path entry when present, with path label + count + formatted size. - Falls back to the legacy Photos / Archives / "Other" rendering when the field is absent (backups taken before this commit). No breaking change for stored history. Tests: new backupService.perPathStats.test.js — 2 scenarios pinning attribution behaviour (single-path, nested-paths-don't-collide). Plus a NOTE comment about overlapping-path walker behaviour (out of scope; canonical seed doesn't hit it). --- .../backupService.perPathStats.test.js | 180 ++++++++++++++++++ backend/src/services/backupService.js | 85 ++++++++- .../src/components/admin/BackupHistory.jsx | 80 ++++++-- 3 files changed, 323 insertions(+), 22 deletions(-) create mode 100644 backend/__tests__/integration/backupService.perPathStats.test.js diff --git a/backend/__tests__/integration/backupService.perPathStats.test.js b/backend/__tests__/integration/backupService.perPathStats.test.js new file mode 100644 index 00000000..10a4366e --- /dev/null +++ b/backend/__tests__/integration/backupService.perPathStats.test.js @@ -0,0 +1,180 @@ +/** + * Per-Stage-B-path tally — Tier 3 of tonight's backup hardening. + * + * Pins the new `computePerPathStats` logic that the Backup History + * "Content Backed Up" pane reads via `backup_runs.statistics.per_path`. + * + * Three scenarios: + * 1. Single file under one path — straightforward attribution + * 2. Multiple paths with overlapping prefixes — longest-prefix wins + * (e.g. `events/active/E1/x.jpg` should attribute to + * `events/active`, not `events`) + * 3. File outside any configured path — silently dropped, doesn't + * throw or contaminate other buckets + * + * Tests exercise the EXPORTED side: write a backup_runs row via the + * service entry point and assert the statistics JSON shape. We don't + * stub `computePerPathStats` directly — the integration view is what + * the frontend actually consumes. + */ + +const fs = require('fs'); +const path = require('path'); + +const { bootCrmDb } = require('./helpers/crmDb'); + +jest.setTimeout(30000); + +describe('backupService — per-Stage-B-path statistics', () => { + let db; + let cleanup; + let storagePath; + let backupService; + + beforeAll(async () => { + ({ db, cleanup } = await bootCrmDb()); + storagePath = process.env.STORAGE_PATH; + backupService = require('../../src/services/backupService'); + }, 120000); + + afterAll(async () => { + if (cleanup) await cleanup(); + }); + + function mkFile(rel, content = 'x'.repeat(100)) { + const abs = path.join(storagePath, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, content); + } + + beforeEach(async () => { + // Clean slate of any artefacts from prior tests + await db('backup_runs').del(); + await db('app_settings').where('setting_type', 'backup').del(); + await db('app_settings').insert([ + { setting_key: 'backup_destination_type', setting_value: JSON.stringify('local'), setting_type: 'backup' }, + { setting_key: 'backup_destination_path', setting_value: JSON.stringify(path.join(storagePath, 'destination')), setting_type: 'backup' }, + { setting_key: 'backup_enabled', setting_value: JSON.stringify(true), setting_type: 'backup' }, + { setting_key: 'backup_email_on_failure', setting_value: JSON.stringify(false), setting_type: 'backup' }, + { setting_key: 'backup_include_archived', setting_value: JSON.stringify(true), setting_type: 'backup' }, + ]).onConflict('setting_key').merge(); + fs.mkdirSync(path.join(storagePath, 'destination'), { recursive: true }); + + // Restore canonical backup_paths from migration 108 + const { DEFAULT_PATHS } = require('../../migrations/core/108_add_backup_paths'); + await db('backup_paths').del(); + await db('backup_paths').insert(DEFAULT_PATHS.map((row) => ({ + ...row, + created_at: new Date(), + updated_at: new Date(), + }))); + + // Wipe leftover files between tests + for (const dir of ['events', 'business-docs', 'thumbnails', 'previews', 'heroes', 'uploads']) { + const p = path.join(storagePath, dir); + if (fs.existsSync(p)) fs.rmSync(p, { recursive: true, force: true }); + } + }); + + it('attributes files to their owning backup_paths row', async () => { + mkFile('events/active/E1/photo-a.jpg', 'X'.repeat(1000)); + mkFile('events/active/E1/photo-b.jpg', 'X'.repeat(2000)); + mkFile('business-docs/quote/2026/Q-1.pdf', 'X'.repeat(500)); + mkFile('thumbnails/E1/photo-a.jpg', 'X'.repeat(50)); + + // Disable the inline DB dump so we don't need pg_dump in tests; + // the file walker is what produces per_path. + await db('app_settings').insert({ + setting_key: 'backup_database_inline_dump', + setting_value: JSON.stringify(false), + setting_type: 'backup', + }).onConflict('setting_key').merge(); + + // Seed a fake DB-backup row so the fail-loud guard is satisfied. + const fakeDump = path.join(storagePath, 'destination', 'fake.sql.gz'); + fs.writeFileSync(fakeDump, 'pretend dump'); + await db('database_backup_runs').insert({ + started_at: new Date(), + completed_at: new Date(), + status: 'completed', + backup_type: 'pg', + file_path: fakeDump, + file_size_bytes: fs.statSync(fakeDump).size, + destination_path: fakeDump, + }); + + await backupService.runBackup(true); + + const run = await db('backup_runs').orderBy('id', 'desc').first(); + expect(run.status).toBe('completed'); + + const statsRaw = typeof run.statistics === 'string' + ? JSON.parse(run.statistics) + : run.statistics; + expect(statsRaw.per_path).toBeDefined(); + + // events/active should have 2 files (3000 bytes) + expect(statsRaw.per_path['events/active']).toEqual({ count: 2, size: 3000 }); + // business-docs should have 1 file (500 bytes) + expect(statsRaw.per_path['business-docs']).toEqual({ count: 1, size: 500 }); + // thumbnails should have 1 file (50 bytes) + expect(statsRaw.per_path['thumbnails']).toEqual({ count: 1, size: 50 }); + + // No spurious buckets for paths that had nothing + expect(statsRaw.per_path['previews']).toBeUndefined(); + expect(statsRaw.per_path['heroes']).toBeUndefined(); + }); + + it('archived path attributed separately from active when both have files', async () => { + mkFile('events/active/E1/active.jpg', 'X'.repeat(100)); + mkFile('events/archived/E2/archived.jpg', 'X'.repeat(200)); + + // backup_include_archived already set true in beforeEach so the + // archived walker fires; same opt-out for inline DB dump. + await db('app_settings').insert({ + setting_key: 'backup_database_inline_dump', + setting_value: JSON.stringify(false), + setting_type: 'backup', + }).onConflict('setting_key').merge(); + const fakeDump = path.join(storagePath, 'destination', 'fake.sql.gz'); + fs.writeFileSync(fakeDump, 'pretend dump'); + await db('database_backup_runs').insert({ + started_at: new Date(), + completed_at: new Date(), + status: 'completed', + backup_type: 'pg', + file_path: fakeDump, + file_size_bytes: fs.statSync(fakeDump).size, + destination_path: fakeDump, + }); + + await backupService.runBackup(true); + + const run = await db('backup_runs').orderBy('id', 'desc').first(); + const statsRaw = typeof run.statistics === 'string' + ? JSON.parse(run.statistics) + : run.statistics; + + // events/active and events/archived attribute separately — + // longest-prefix match prevents `events/active/...` from claiming + // an `events/archived/...` file or vice versa. + expect(statsRaw.per_path['events/active']).toEqual({ count: 1, size: 100 }); + expect(statsRaw.per_path['events/archived']).toEqual({ count: 1, size: 200 }); + }); +}); + +// NOTE on walker duplication +// +// If two `backup_paths` rows overlap (e.g. one row at `events` AND +// another at `events/active`), the walker scans the same files twice +// — once via each path. Per-path stats then attribute the file to the +// longest-prefix-matching path BOTH times, producing inflated counts. +// +// The canonical seed in migration 108 contains no overlapping pairs, +// so this isn't exercised in practice. But an admin who hand-adds a +// broad row that overlaps an existing nested one will see double +// counts in their next backup's statistics + the destination will +// receive duplicate copies (wasting space). Worth flagging if anyone +// reports it — the fix is to de-dupe `files` in +// `getFilesToBackupInternal` before returning, OR to skip walking a +// path if a longer one has already covered it. diff --git a/backend/src/services/backupService.js b/backend/src/services/backupService.js index aeeaf3d7..dd90332c 100644 --- a/backend/src/services/backupService.js +++ b/backend/src/services/backupService.js @@ -486,6 +486,74 @@ async function resolveBackupPaths(config) { }); } +/** + * Bucket actually-backed-up files into their owning `backup_paths` row. + * + * Uses longest-prefix match — e.g. `events/active/E1/photo.jpg` matches + * `events/active` (length 13) rather than `events` (length 6, if that + * row existed). This handles the case where a future feature ships a + * nested backup_paths row that overlaps an existing one. + * + * @param {string[]} backedUpRelativePaths — paths that actually got + * copied / uploaded (post-incremental-filter). The exact list + * the destination implementation reports back. + * @param {Array<{relativePath, size}>} allFiles — the full file + * catalogue from the walker, used as a size lookup table. + * @returns {Promise>} + * keyed by `backup_paths.path` (e.g. 'events/active'). Paths + * with zero matches are omitted to keep the manifest compact. + */ +async function computePerPathStats(backedUpRelativePaths, allFiles) { + if (!backedUpRelativePaths || backedUpRelativePaths.length === 0) { + return {}; + } + + // Reuse the same source of truth the walker uses, so a row toggled + // off by include_in_default doesn't appear in the breakdown either. + let configuredPaths; + try { + if (await db.schema.hasTable('backup_paths')) { + configuredPaths = await db('backup_paths') + .where('include_in_default', formatBoolean(true)) + .orderBy('display_order', 'asc') + .select('path'); + } + } catch (err) { + logger.warn(`Could not load backup_paths for per-path stats — falling back to legacy: ${err.message}`); + } + if (!configuredPaths || configuredPaths.length === 0) { + configuredPaths = LEGACY_BACKUP_PATHS.map((p) => ({ path: p.path })); + } + + // Longest-prefix-first so nested paths win over their parents. + const sortedPaths = configuredPaths + .map((row) => row.path) + .sort((a, b) => b.length - a.length); + + // Size lookup. relativePath uses OS path separators in `allFiles` + // (whatever scanDirectory built); the backup_paths rows always use + // forward slashes. Normalize the lookup key once. + const sizeByPath = new Map(); + for (const f of allFiles || []) { + sizeByPath.set(f.relativePath.split(path.sep).join('/'), f.size || 0); + } + + const stats = {}; + for (const relativePath of backedUpRelativePaths) { + const norm = relativePath.split(path.sep).join('/'); + // Find the longest configured path that this file's relativePath starts with. + const match = sortedPaths.find( + (p) => norm === p || norm.startsWith(`${p}/`) + ); + if (!match) continue; // file outside any configured path (shouldn't happen) + if (!stats[match]) stats[match] = { count: 0, size: 0 }; + stats[match].count += 1; + stats[match].size += sizeByPath.get(norm) || 0; + } + + return stats; +} + async function getFilesToBackupInternal(configOrIncludeArchived = true) { const files = []; const storagePath = getStoragePath(); @@ -1035,6 +1103,18 @@ async function runBackupInternal(isManual = false) { logger.error('Failed to generate backup manifest:', error); } + // Per-Stage-B-path stats — bucket the actually-backed-up files + // into their owning backup_paths row by longest-prefix match. Lets + // the Backup History detail pane render a true breakdown + // events/active: 142 files (3.2 GB) + // business-docs: 17 files (4.5 MB) + // thumbnails: 142 files (12.4 MB) + // instead of the legacy "Photos + Archives + Other" categorization + // that didn't reflect Stage B's data-driven walker. Falls back to + // an empty map if backup_paths is missing (defense in depth — the + // walker has the same fallback). + const perPath = await computePerPathStats(result.backedUpFiles, files); + await db('backup_runs') .where('id', runId) .update({ @@ -1053,11 +1133,14 @@ async function runBackupInternal(isManual = false) { total_files_checked: files.length, average_file_size: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0, destination: destinationType, + // Per-Stage-B-path breakdown — { [pathKey]: { count, size } } + per_path: perPath, // Keep camelCase for backward compatibility totalFilesChecked: files.length, filesBackedUp: result.backedUpCount, totalSize: result.backedUpSize, - averageFileSize: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0 + averageFileSize: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0, + perPath }) }); diff --git a/frontend/src/components/admin/BackupHistory.jsx b/frontend/src/components/admin/BackupHistory.jsx index e079eb57..1bde610a 100644 --- a/frontend/src/components/admin/BackupHistory.jsx +++ b/frontend/src/components/admin/BackupHistory.jsx @@ -282,15 +282,15 @@ export const BackupHistory = () => { {/* Content Backed Up - Stage B's config-driven walker covers 7 path categories - (events/active, events/archived, thumbnails, previews, - heroes, uploads, business-docs). This pane historically - only counted 2 of them (photos + archives), so the - per-row sums never matched the total — Ralf 2026-05-30 - flagged a 3 files total with 0+0 visible breakdown. - Adds an explicit Total + an "Other" bucket so the - arithmetic adds up even when the backend doesn't yet - expose per-path counts. */} + Two render paths depending on what the backend + provided: + - NEW: per_path map { "events/active": {count, size}, ... } + from Stage B's walker. One row per path, + ordered by display_order. + - LEGACY: fall back to Photos + Archives + + "Other" bucket so the arithmetic still adds + up when restoring a backup taken before this + change shipped. */}

{t('backup.history.details.contentBackedUp')}

@@ -298,19 +298,45 @@ export const BackupHistory = () => { {t('backup.configuration.whatToBackup.database')}
-
- 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} /> - - Photos ({stats.photos_backed_up || 0} of {stats.total_photos || 0}) - -
-
- 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} /> - - Archives ({stats.archives_backed_up || 0}) - -
{(() => { + // Per-path breakdown when present + const perPath = stats.per_path || stats.perPath; + if (perPath && Object.keys(perPath).length > 0) { + const formatSize = (bytes) => { + if (!bytes) return '0 B'; + const units = ['B', 'KB', 'MB', 'GB', 'TB']; + const i = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1); + return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 1)} ${units[i]}`; + }; + // Sort by path string so the order is stable across renders; + // backend uses backup_paths.display_order to drive the walker + // but doesn't carry order into per_path map — alphabetic is + // fine for the display. + const entries = Object.entries(perPath).sort(([a], [b]) => a.localeCompare(b)); + return ( + <> + {entries.map(([pathKey, info]) => ( +
+ 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} /> + + {pathKey} + + + {info.count} {info.size ? `(${formatSize(info.size)})` : ''} + +
+ ))} +
+ + {t('backup.history.details.totalFiles', 'Total files')}: {stats.files_processed || 0} + +
+ + ); + } + + // LEGACY rendering for backups taken before + // per_path was emitted. const total = Number(stats.files_processed) || 0; const accounted = (Number(stats.photos_backed_up) || 0) @@ -318,6 +344,18 @@ export const BackupHistory = () => { const other = Math.max(total - accounted, 0); return ( <> +
+ 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} /> + + Photos ({stats.photos_backed_up || 0} of {stats.total_photos || 0}) + +
+
+ 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} /> + + Archives ({stats.archives_backed_up || 0}) + +
0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />