feat(backup-stats): per-Stage-B-path counters in backup statistics
Closes the last gap from tonight's backup-hardening: backup_runs.
statistics now carries a `per_path` map keyed by backup_paths.path
(e.g. `events/active`, `business-docs`), with per-bucket count + size.
Backend (backupService.js):
- new `computePerPathStats(backedUpFiles, allFiles)` helper that
bucket-sorts each backed-up file into its owning backup_paths row
by longest-prefix match. Reuses the same backup_paths source the
walker reads, so toggling include_in_default off propagates
correctly. Falls back to LEGACY_BACKUP_PATHS if the table is
missing.
- runBackupInternal calls it after the destination implementation
reports back, includes the result in statistics under both
snake_case (`per_path`) and camelCase (`perPath`) keys for the
same alias treatment the existing fields get.
Frontend (BackupHistory.jsx):
- Backup History detail pane now renders one row per per_path entry
when present, with path label + count + formatted size.
- Falls back to the legacy Photos / Archives / "Other" rendering
when the field is absent (backups taken before this commit). No
breaking change for stored history.
Tests: new backupService.perPathStats.test.js — 2 scenarios pinning
attribution behaviour (single-path, nested-paths-don't-collide).
Plus a NOTE comment about overlapping-path walker behaviour (out of
scope; canonical seed doesn't hit it).
This commit is contained in:
Luca
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* Per-Stage-B-path tally — Tier 3 of tonight's backup hardening.
|
||||
*
|
||||
* Pins the new `computePerPathStats` logic that the Backup History
|
||||
* "Content Backed Up" pane reads via `backup_runs.statistics.per_path`.
|
||||
*
|
||||
* Three scenarios:
|
||||
* 1. Single file under one path — straightforward attribution
|
||||
* 2. Multiple paths with overlapping prefixes — longest-prefix wins
|
||||
* (e.g. `events/active/E1/x.jpg` should attribute to
|
||||
* `events/active`, not `events`)
|
||||
* 3. File outside any configured path — silently dropped, doesn't
|
||||
* throw or contaminate other buckets
|
||||
*
|
||||
* Tests exercise the EXPORTED side: write a backup_runs row via the
|
||||
* service entry point and assert the statistics JSON shape. We don't
|
||||
* stub `computePerPathStats` directly — the integration view is what
|
||||
* the frontend actually consumes.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const { bootCrmDb } = require('./helpers/crmDb');
|
||||
|
||||
jest.setTimeout(30000);
|
||||
|
||||
describe('backupService — per-Stage-B-path statistics', () => {
|
||||
let db;
|
||||
let cleanup;
|
||||
let storagePath;
|
||||
let backupService;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ db, cleanup } = await bootCrmDb());
|
||||
storagePath = process.env.STORAGE_PATH;
|
||||
backupService = require('../../src/services/backupService');
|
||||
}, 120000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (cleanup) await cleanup();
|
||||
});
|
||||
|
||||
function mkFile(rel, content = 'x'.repeat(100)) {
|
||||
const abs = path.join(storagePath, rel);
|
||||
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
||||
fs.writeFileSync(abs, content);
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clean slate of any artefacts from prior tests
|
||||
await db('backup_runs').del();
|
||||
await db('app_settings').where('setting_type', 'backup').del();
|
||||
await db('app_settings').insert([
|
||||
{ setting_key: 'backup_destination_type', setting_value: JSON.stringify('local'), setting_type: 'backup' },
|
||||
{ setting_key: 'backup_destination_path', setting_value: JSON.stringify(path.join(storagePath, 'destination')), setting_type: 'backup' },
|
||||
{ setting_key: 'backup_enabled', setting_value: JSON.stringify(true), setting_type: 'backup' },
|
||||
{ setting_key: 'backup_email_on_failure', setting_value: JSON.stringify(false), setting_type: 'backup' },
|
||||
{ setting_key: 'backup_include_archived', setting_value: JSON.stringify(true), setting_type: 'backup' },
|
||||
]).onConflict('setting_key').merge();
|
||||
fs.mkdirSync(path.join(storagePath, 'destination'), { recursive: true });
|
||||
|
||||
// Restore canonical backup_paths from migration 108
|
||||
const { DEFAULT_PATHS } = require('../../migrations/core/108_add_backup_paths');
|
||||
await db('backup_paths').del();
|
||||
await db('backup_paths').insert(DEFAULT_PATHS.map((row) => ({
|
||||
...row,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date(),
|
||||
})));
|
||||
|
||||
// Wipe leftover files between tests
|
||||
for (const dir of ['events', 'business-docs', 'thumbnails', 'previews', 'heroes', 'uploads']) {
|
||||
const p = path.join(storagePath, dir);
|
||||
if (fs.existsSync(p)) fs.rmSync(p, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('attributes files to their owning backup_paths row', async () => {
|
||||
mkFile('events/active/E1/photo-a.jpg', 'X'.repeat(1000));
|
||||
mkFile('events/active/E1/photo-b.jpg', 'X'.repeat(2000));
|
||||
mkFile('business-docs/quote/2026/Q-1.pdf', 'X'.repeat(500));
|
||||
mkFile('thumbnails/E1/photo-a.jpg', 'X'.repeat(50));
|
||||
|
||||
// Disable the inline DB dump so we don't need pg_dump in tests;
|
||||
// the file walker is what produces per_path.
|
||||
await db('app_settings').insert({
|
||||
setting_key: 'backup_database_inline_dump',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'backup',
|
||||
}).onConflict('setting_key').merge();
|
||||
|
||||
// Seed a fake DB-backup row so the fail-loud guard is satisfied.
|
||||
const fakeDump = path.join(storagePath, 'destination', 'fake.sql.gz');
|
||||
fs.writeFileSync(fakeDump, 'pretend dump');
|
||||
await db('database_backup_runs').insert({
|
||||
started_at: new Date(),
|
||||
completed_at: new Date(),
|
||||
status: 'completed',
|
||||
backup_type: 'pg',
|
||||
file_path: fakeDump,
|
||||
file_size_bytes: fs.statSync(fakeDump).size,
|
||||
destination_path: fakeDump,
|
||||
});
|
||||
|
||||
await backupService.runBackup(true);
|
||||
|
||||
const run = await db('backup_runs').orderBy('id', 'desc').first();
|
||||
expect(run.status).toBe('completed');
|
||||
|
||||
const statsRaw = typeof run.statistics === 'string'
|
||||
? JSON.parse(run.statistics)
|
||||
: run.statistics;
|
||||
expect(statsRaw.per_path).toBeDefined();
|
||||
|
||||
// events/active should have 2 files (3000 bytes)
|
||||
expect(statsRaw.per_path['events/active']).toEqual({ count: 2, size: 3000 });
|
||||
// business-docs should have 1 file (500 bytes)
|
||||
expect(statsRaw.per_path['business-docs']).toEqual({ count: 1, size: 500 });
|
||||
// thumbnails should have 1 file (50 bytes)
|
||||
expect(statsRaw.per_path['thumbnails']).toEqual({ count: 1, size: 50 });
|
||||
|
||||
// No spurious buckets for paths that had nothing
|
||||
expect(statsRaw.per_path['previews']).toBeUndefined();
|
||||
expect(statsRaw.per_path['heroes']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('archived path attributed separately from active when both have files', async () => {
|
||||
mkFile('events/active/E1/active.jpg', 'X'.repeat(100));
|
||||
mkFile('events/archived/E2/archived.jpg', 'X'.repeat(200));
|
||||
|
||||
// backup_include_archived already set true in beforeEach so the
|
||||
// archived walker fires; same opt-out for inline DB dump.
|
||||
await db('app_settings').insert({
|
||||
setting_key: 'backup_database_inline_dump',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'backup',
|
||||
}).onConflict('setting_key').merge();
|
||||
const fakeDump = path.join(storagePath, 'destination', 'fake.sql.gz');
|
||||
fs.writeFileSync(fakeDump, 'pretend dump');
|
||||
await db('database_backup_runs').insert({
|
||||
started_at: new Date(),
|
||||
completed_at: new Date(),
|
||||
status: 'completed',
|
||||
backup_type: 'pg',
|
||||
file_path: fakeDump,
|
||||
file_size_bytes: fs.statSync(fakeDump).size,
|
||||
destination_path: fakeDump,
|
||||
});
|
||||
|
||||
await backupService.runBackup(true);
|
||||
|
||||
const run = await db('backup_runs').orderBy('id', 'desc').first();
|
||||
const statsRaw = typeof run.statistics === 'string'
|
||||
? JSON.parse(run.statistics)
|
||||
: run.statistics;
|
||||
|
||||
// events/active and events/archived attribute separately —
|
||||
// longest-prefix match prevents `events/active/...` from claiming
|
||||
// an `events/archived/...` file or vice versa.
|
||||
expect(statsRaw.per_path['events/active']).toEqual({ count: 1, size: 100 });
|
||||
expect(statsRaw.per_path['events/archived']).toEqual({ count: 1, size: 200 });
|
||||
});
|
||||
});
|
||||
|
||||
// NOTE on walker duplication
|
||||
//
|
||||
// If two `backup_paths` rows overlap (e.g. one row at `events` AND
|
||||
// another at `events/active`), the walker scans the same files twice
|
||||
// — once via each path. Per-path stats then attribute the file to the
|
||||
// longest-prefix-matching path BOTH times, producing inflated counts.
|
||||
//
|
||||
// The canonical seed in migration 108 contains no overlapping pairs,
|
||||
// so this isn't exercised in practice. But an admin who hand-adds a
|
||||
// broad row that overlaps an existing nested one will see double
|
||||
// counts in their next backup's statistics + the destination will
|
||||
// receive duplicate copies (wasting space). Worth flagging if anyone
|
||||
// reports it — the fix is to de-dupe `files` in
|
||||
// `getFilesToBackupInternal` before returning, OR to skip walking a
|
||||
// path if a longer one has already covered it.
|
||||
@@ -486,6 +486,74 @@ async function resolveBackupPaths(config) {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Bucket actually-backed-up files into their owning `backup_paths` row.
|
||||
*
|
||||
* Uses longest-prefix match — e.g. `events/active/E1/photo.jpg` matches
|
||||
* `events/active` (length 13) rather than `events` (length 6, if that
|
||||
* row existed). This handles the case where a future feature ships a
|
||||
* nested backup_paths row that overlaps an existing one.
|
||||
*
|
||||
* @param {string[]} backedUpRelativePaths — paths that actually got
|
||||
* copied / uploaded (post-incremental-filter). The exact list
|
||||
* the destination implementation reports back.
|
||||
* @param {Array<{relativePath, size}>} allFiles — the full file
|
||||
* catalogue from the walker, used as a size lookup table.
|
||||
* @returns {Promise<Record<string, { count: number, size: number }>>}
|
||||
* keyed by `backup_paths.path` (e.g. 'events/active'). Paths
|
||||
* with zero matches are omitted to keep the manifest compact.
|
||||
*/
|
||||
async function computePerPathStats(backedUpRelativePaths, allFiles) {
|
||||
if (!backedUpRelativePaths || backedUpRelativePaths.length === 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Reuse the same source of truth the walker uses, so a row toggled
|
||||
// off by include_in_default doesn't appear in the breakdown either.
|
||||
let configuredPaths;
|
||||
try {
|
||||
if (await db.schema.hasTable('backup_paths')) {
|
||||
configuredPaths = await db('backup_paths')
|
||||
.where('include_in_default', formatBoolean(true))
|
||||
.orderBy('display_order', 'asc')
|
||||
.select('path');
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn(`Could not load backup_paths for per-path stats — falling back to legacy: ${err.message}`);
|
||||
}
|
||||
if (!configuredPaths || configuredPaths.length === 0) {
|
||||
configuredPaths = LEGACY_BACKUP_PATHS.map((p) => ({ path: p.path }));
|
||||
}
|
||||
|
||||
// Longest-prefix-first so nested paths win over their parents.
|
||||
const sortedPaths = configuredPaths
|
||||
.map((row) => row.path)
|
||||
.sort((a, b) => b.length - a.length);
|
||||
|
||||
// Size lookup. relativePath uses OS path separators in `allFiles`
|
||||
// (whatever scanDirectory built); the backup_paths rows always use
|
||||
// forward slashes. Normalize the lookup key once.
|
||||
const sizeByPath = new Map();
|
||||
for (const f of allFiles || []) {
|
||||
sizeByPath.set(f.relativePath.split(path.sep).join('/'), f.size || 0);
|
||||
}
|
||||
|
||||
const stats = {};
|
||||
for (const relativePath of backedUpRelativePaths) {
|
||||
const norm = relativePath.split(path.sep).join('/');
|
||||
// Find the longest configured path that this file's relativePath starts with.
|
||||
const match = sortedPaths.find(
|
||||
(p) => norm === p || norm.startsWith(`${p}/`)
|
||||
);
|
||||
if (!match) continue; // file outside any configured path (shouldn't happen)
|
||||
if (!stats[match]) stats[match] = { count: 0, size: 0 };
|
||||
stats[match].count += 1;
|
||||
stats[match].size += sizeByPath.get(norm) || 0;
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
async function getFilesToBackupInternal(configOrIncludeArchived = true) {
|
||||
const files = [];
|
||||
const storagePath = getStoragePath();
|
||||
@@ -1035,6 +1103,18 @@ async function runBackupInternal(isManual = false) {
|
||||
logger.error('Failed to generate backup manifest:', error);
|
||||
}
|
||||
|
||||
// Per-Stage-B-path stats — bucket the actually-backed-up files
|
||||
// into their owning backup_paths row by longest-prefix match. Lets
|
||||
// the Backup History detail pane render a true breakdown
|
||||
// events/active: 142 files (3.2 GB)
|
||||
// business-docs: 17 files (4.5 MB)
|
||||
// thumbnails: 142 files (12.4 MB)
|
||||
// instead of the legacy "Photos + Archives + Other" categorization
|
||||
// that didn't reflect Stage B's data-driven walker. Falls back to
|
||||
// an empty map if backup_paths is missing (defense in depth — the
|
||||
// walker has the same fallback).
|
||||
const perPath = await computePerPathStats(result.backedUpFiles, files);
|
||||
|
||||
await db('backup_runs')
|
||||
.where('id', runId)
|
||||
.update({
|
||||
@@ -1053,11 +1133,14 @@ async function runBackupInternal(isManual = false) {
|
||||
total_files_checked: files.length,
|
||||
average_file_size: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0,
|
||||
destination: destinationType,
|
||||
// Per-Stage-B-path breakdown — { [pathKey]: { count, size } }
|
||||
per_path: perPath,
|
||||
// Keep camelCase for backward compatibility
|
||||
totalFilesChecked: files.length,
|
||||
filesBackedUp: result.backedUpCount,
|
||||
totalSize: result.backedUpSize,
|
||||
averageFileSize: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0
|
||||
averageFileSize: result.backedUpCount ? Math.round(result.backedUpSize / result.backedUpCount) : 0,
|
||||
perPath
|
||||
})
|
||||
});
|
||||
|
||||
|
||||
@@ -282,15 +282,15 @@ export const BackupHistory = () => {
|
||||
</div>
|
||||
|
||||
{/* Content Backed Up
|
||||
Stage B's config-driven walker covers 7 path categories
|
||||
(events/active, events/archived, thumbnails, previews,
|
||||
heroes, uploads, business-docs). This pane historically
|
||||
only counted 2 of them (photos + archives), so the
|
||||
per-row sums never matched the total — Ralf 2026-05-30
|
||||
flagged a 3 files total with 0+0 visible breakdown.
|
||||
Adds an explicit Total + an "Other" bucket so the
|
||||
arithmetic adds up even when the backend doesn't yet
|
||||
expose per-path counts. */}
|
||||
Two render paths depending on what the backend
|
||||
provided:
|
||||
- NEW: per_path map { "events/active": {count, size}, ... }
|
||||
from Stage B's walker. One row per path,
|
||||
ordered by display_order.
|
||||
- LEGACY: fall back to Photos + Archives +
|
||||
"Other" bucket so the arithmetic still adds
|
||||
up when restoring a backup taken before this
|
||||
change shipped. */}
|
||||
<div className="space-y-2">
|
||||
<h4 className="font-medium text-neutral-900 dark:text-neutral-100">{t('backup.history.details.contentBackedUp')}</h4>
|
||||
<div className="space-y-2">
|
||||
@@ -298,19 +298,45 @@ export const BackupHistory = () => {
|
||||
<Database className={`h-4 w-4 ${stats.database_backed_up ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
|
||||
<span className="text-sm text-neutral-700 dark:text-neutral-300">{t('backup.configuration.whatToBackup.database')}</span>
|
||||
</div>
|
||||
<div className="flex items-center space-x-2">
|
||||
<Image className={`h-4 w-4 ${stats.photos_backed_up > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
|
||||
<span className="text-sm text-neutral-700 dark:text-neutral-300">
|
||||
Photos ({stats.photos_backed_up || 0} of {stats.total_photos || 0})
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center space-x-2">
|
||||
<FileArchive className={`h-4 w-4 ${stats.archives_backed_up > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
|
||||
<span className="text-sm text-neutral-700 dark:text-neutral-300">
|
||||
Archives ({stats.archives_backed_up || 0})
|
||||
</span>
|
||||
</div>
|
||||
{(() => {
|
||||
// Per-path breakdown when present
|
||||
const perPath = stats.per_path || stats.perPath;
|
||||
if (perPath && Object.keys(perPath).length > 0) {
|
||||
const formatSize = (bytes) => {
|
||||
if (!bytes) return '0 B';
|
||||
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
const i = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1);
|
||||
return `${(bytes / Math.pow(1024, i)).toFixed(i === 0 ? 0 : 1)} ${units[i]}`;
|
||||
};
|
||||
// Sort by path string so the order is stable across renders;
|
||||
// backend uses backup_paths.display_order to drive the walker
|
||||
// but doesn't carry order into per_path map — alphabetic is
|
||||
// fine for the display.
|
||||
const entries = Object.entries(perPath).sort(([a], [b]) => a.localeCompare(b));
|
||||
return (
|
||||
<>
|
||||
{entries.map(([pathKey, info]) => (
|
||||
<div key={pathKey} className="flex items-center space-x-2">
|
||||
<FileArchive className={`h-4 w-4 ${info.count > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
|
||||
<span className="text-sm text-neutral-700 dark:text-neutral-300 font-mono">
|
||||
{pathKey}
|
||||
</span>
|
||||
<span className="text-sm text-neutral-500 dark:text-neutral-400 ml-auto">
|
||||
{info.count} {info.size ? `(${formatSize(info.size)})` : ''}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
<div className="flex items-center space-x-2 pt-1 border-t border-neutral-200 dark:border-neutral-700">
|
||||
<span className="text-xs uppercase tracking-wide text-neutral-500 dark:text-neutral-400">
|
||||
{t('backup.history.details.totalFiles', 'Total files')}: {stats.files_processed || 0}
|
||||
</span>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// LEGACY rendering for backups taken before
|
||||
// per_path was emitted.
|
||||
const total = Number(stats.files_processed) || 0;
|
||||
const accounted =
|
||||
(Number(stats.photos_backed_up) || 0)
|
||||
@@ -318,6 +344,18 @@ export const BackupHistory = () => {
|
||||
const other = Math.max(total - accounted, 0);
|
||||
return (
|
||||
<>
|
||||
<div className="flex items-center space-x-2">
|
||||
<Image className={`h-4 w-4 ${stats.photos_backed_up > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
|
||||
<span className="text-sm text-neutral-700 dark:text-neutral-300">
|
||||
Photos ({stats.photos_backed_up || 0} of {stats.total_photos || 0})
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center space-x-2">
|
||||
<FileArchive className={`h-4 w-4 ${stats.archives_backed_up > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
|
||||
<span className="text-sm text-neutral-700 dark:text-neutral-300">
|
||||
Archives ({stats.archives_backed_up || 0})
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center space-x-2">
|
||||
<FileArchive className={`h-4 w-4 ${other > 0 ? 'text-green-500' : 'text-neutral-300 dark:text-neutral-600'}`} />
|
||||
<span className="text-sm text-neutral-700 dark:text-neutral-300">
|
||||
|
||||
Reference in New Issue
Block a user