From f664fea60ca223479b917abb7a256d4320f089e3 Mon Sep 17 00:00:00 2001 From: Luca <102960244+Luca-Timo@users.noreply.github.com> Date: Sat, 30 May 2026 04:07:40 +0200 Subject: [PATCH] fix(restore): discover backups from disk, not just the DB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Restore wizard's "Choose Backup to Restore" list was driven only by the backup_runs table. After `docker compose down -v` (the disaster this whole hardening effort is designed to recover from), the DB is empty and the wizard shows "No backups found in selected source" — exactly when it's needed most. The manifest JSONs are still on disk; the wizard just can't see them. Adds disk-first discovery: - Walks backup_destination_path AND backup_manifest_path (manifests can live in a sibling directory under the canonical /manifests/backup-manifest-.json layout). Depth-limited recursion (3 levels) so the scan doesn't enumerate the photo tree. - Matches backup-manifest-*.json|yaml AND legacy bare manifest.json. - Parses each manifest for real metadata (timestamp, size, file count, database.backup_file presence) instead of showing the admin opaque filenames. - Layers in surviving backup_runs rows, deduping by manifest_id. Applied to both GET /available-backups (legacy) and POST /list-backups (the one the frontend actually calls). Same helper, two call sites. Side benefit: each returned row now carries `databaseIncluded` — so a future Restore UI iteration can show a "this backup has no DB dump" warning before the admin picks a files-only backup. Exactly the surface that would have caught Ralf's original four files-only manifests if it had existed. --- backend/src/routes/adminRestore.js | 352 +++++++++++++++++++++-------- 1 file changed, 257 insertions(+), 95 deletions(-) diff --git a/backend/src/routes/adminRestore.js b/backend/src/routes/adminRestore.js index f6c53c4e..305c6b91 100644 --- a/backend/src/routes/adminRestore.js +++ b/backend/src/routes/adminRestore.js @@ -348,51 +348,7 @@ router.get('/run/:id/report', requirePermission('backup.view'), async (req, res) */ router.get('/available-backups', requirePermission('backup.view'), async (req, res) => { try { - const backups = []; - - // Get local file backups - const backupConfig = await getBackupConfig(); - if (backupConfig.backup_destination_type === 'local' && backupConfig.backup_destination_path) { - try { - const files = await fs.readdir(backupConfig.backup_destination_path); - for (const file of files) { - if (file.endsWith('.json') || file.endsWith('.yaml')) { - const filePath = path.join(backupConfig.backup_destination_path, file); - const stats = await fs.stat(filePath); - backups.push({ - type: 'local', - name: file, - path: filePath, - size: stats.size, - modified: stats.mtime - }); - } - } - } catch (error) { - logger.warn('Failed to list local backups:', error); - } - } - - // Get database backups from backup_runs table - const backupRuns = await db('backup_runs') - .where('status', 'completed') - .whereNotNull('manifest_path') - .orderBy('completed_at', 'desc') - .limit(20); - - for (const run of backupRuns) { - backups.push({ - type: run.manifest_path.startsWith('s3://') ? 's3' : 'local', - name: `Backup ${run.completed_at}`, - path: run.manifest_path, - manifestId: run.manifest_id, - size: run.total_size_bytes, - filesCount: run.files_backed_up, - duration: run.duration_seconds, - completed: run.completed_at - }); - } - + const backups = await discoverAvailableBackups(); res.json({ success: true, data: backups @@ -406,6 +362,213 @@ router.get('/available-backups', requirePermission('backup.view'), async (req, r } }); +/** + * Discover restorable backups by walking the configured destination + * directory + harvesting the backup_runs table. + * + * **Why we recurse the disk first, DB second** + * + * The disk is the source of truth for restore. After a disaster + * (`docker compose down -v`, drive corruption, fresh install) the + * `backup_runs` table is empty — but the manifest JSONs are exactly + * what's left on disk for an admin to recover from. A wizard that + * only reads the DB shows "No backups found" precisely when the + * admin needs it most. So we walk first, dedupe-by-manifestId + * against any surviving DB rows, and present a unified list. + * + * Discovery rules: + * - Walks `backup_destination_path` AND `backup_manifest_path` if + * they're distinct (manifests can live in a sibling directory). + * - Recurses up to 3 levels deep — enough to find + * `/manifests/backup-manifest-.json` (the default + * layout) without scanning the entire photo tree. + * - Matches manifest files by glob: `backup-manifest-*.json`, + * `backup-manifest-*.yaml`, and the legacy bare `manifest.json`. + * - Parses each manifest to extract real metadata (timestamp, + * size, file count, source type) instead of showing the admin + * a list of opaque filenames. + * + * Returns: array of `{ type, name, path, manifestId, size, + * filesCount, completed, source: 'disk' | 'db' }`. + */ +async function discoverAvailableBackups() { + const backupConfig = await getBackupConfig(); + const backups = []; + const seenManifestIds = new Set(); + + if (backupConfig.backup_destination_type === 'local') { + const roots = new Set(); + if (backupConfig.backup_destination_path) roots.add(backupConfig.backup_destination_path); + if (backupConfig.backup_manifest_path) roots.add(backupConfig.backup_manifest_path); + + for (const root of roots) { + try { + const manifestPaths = await walkForManifests(root, 3); + for (const filePath of manifestPaths) { + try { + const parsed = await parseManifestMetadata(filePath); + if (parsed.manifestId) seenManifestIds.add(parsed.manifestId); + backups.push(parsed); + } catch (err) { + // Don't fail discovery because ONE manifest is corrupt — + // surface the file with a note so the admin sees something + // is wrong and can investigate. + logger.warn(`Manifest unreadable at ${filePath}: ${err.message}`); + const stats = await fs.stat(filePath).catch(() => null); + backups.push({ + type: 'local', + name: path.basename(filePath), + path: filePath, + manifestId: null, + size: stats?.size || 0, + filesCount: null, + completed: stats?.mtime || null, + source: 'disk', + corrupt: true, + error: err.message, + }); + } + } + } catch (err) { + logger.warn(`Could not scan backup root ${root}: ${err.message}`); + } + } + } + + // Layer in surviving DB rows, deduping by manifest_id so we don't + // show the same backup twice with different shapes. + const backupRuns = await db('backup_runs') + .where('status', 'completed') + .whereNotNull('manifest_path') + .orderBy('completed_at', 'desc') + .limit(50); + + for (const run of backupRuns) { + if (run.manifest_id && seenManifestIds.has(run.manifest_id)) continue; + backups.push({ + type: run.manifest_path.startsWith('s3://') ? 's3' : 'local', + name: `Backup ${run.completed_at}`, + path: run.manifest_path, + manifestId: run.manifest_id, + size: run.total_size_bytes, + filesCount: run.files_backed_up, + duration: run.duration_seconds, + completed: run.completed_at, + source: 'db', + }); + } + + // Most recent first. + backups.sort((a, b) => { + const aTime = a.completed ? new Date(a.completed).getTime() : 0; + const bTime = b.completed ? new Date(b.completed).getTime() : 0; + return bTime - aTime; + }); + + return backups; +} + +/** + * Recursive manifest finder. Depth-limited so we don't enumerate + * thousands of photo files. Yields absolute paths. + */ +async function walkForManifests(dir, maxDepth, depth = 0) { + if (depth > maxDepth) return []; + let entries; + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch (err) { + if (err.code === 'ENOENT') return []; + throw err; + } + + const out = []; + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + // Skip obvious noise: photo trees, node_modules, hidden dirs. + if (entry.name === 'events' || entry.name === 'business-docs' + || entry.name === 'thumbnails' || entry.name === 'previews' + || entry.name === 'heroes' || entry.name === 'uploads' + || entry.name.startsWith('.') + || entry.name === 'node_modules') continue; + out.push(...await walkForManifests(full, maxDepth, depth + 1)); + } else if (entry.isFile() && isManifestFilename(entry.name)) { + out.push(full); + } + } + return out; +} + +function isManifestFilename(name) { + // Canonical: backup-manifest-.json / .yaml + // Legacy: manifest.json (inside backup-/manifest.json layout) + // Be liberal in what we accept — admin may have renamed. + if (/^backup-manifest-.+\.(json|ya?ml)$/i.test(name)) return true; + if (/^manifest\.(json|ya?ml)$/i.test(name)) return true; + return false; +} + +/** + * Parse a manifest file and pull out the fields the wizard wants. + * Tolerates schema drift across manifest versions (v1, v2) by + * checking multiple shapes. + */ +async function parseManifestMetadata(filePath) { + const raw = await fs.readFile(filePath, 'utf8'); + let parsed; + if (filePath.toLowerCase().endsWith('.json')) { + parsed = JSON.parse(raw); + } else { + // Minimal YAML support — most admins use JSON; only do require() + // if a .yaml manifest is actually present. + const yaml = require('js-yaml'); + parsed = yaml.load(raw); + } + + const stats = await fs.stat(filePath); + + // v2 shape: { manifest: { id, timestamp }, backup: { ... }, files: [...], database: { ... } } + // v1 shape: { backup_id, started_at, files: [...] } (older) + const manifestId = + parsed?.manifest?.id + || parsed?.backup?.id + || parsed?.backup_id + || null; + + const completed = + parsed?.backup?.completed_at + || parsed?.manifest?.timestamp + || parsed?.completed_at + || stats.mtime; + + const filesCount = + (Array.isArray(parsed?.files) ? parsed.files.length : null) + ?? parsed?.backup?.total_files + ?? null; + + const totalSizeBytes = + parsed?.backup?.total_size_bytes + ?? parsed?.total_size_bytes + ?? null; + + return { + type: 'local', + name: path.basename(filePath), + path: filePath, + manifestId, + size: totalSizeBytes ?? stats.size, + filesCount, + completed, + source: 'disk', + databaseIncluded: Boolean(parsed?.database?.backup_file), + // Helpful for the UI: lets it show "This backup has no DB" warning + // — exactly the surface that would have caught Ralf's original + // four files-only manifests if it had existed. + schemaVersion: parsed?.manifest?.version || parsed?.version || '1.0', + }; +} + /** * List backups for restore (POST version for frontend compatibility) * Accepts source type in request body @@ -413,59 +576,58 @@ router.get('/available-backups', requirePermission('backup.view'), async (req, r router.post('/list-backups', requirePermission('backup.view'), async (req, res) => { try { const { source } = req.body; // 'local', 's3', or undefined for all - const backups = []; - // Get backup configuration - const backupConfig = await getBackupConfig(); + // Use the same disk-first discovery the GET endpoint uses so that + // a fresh post-`docker compose down -v` install (empty backup_runs + // table) can still see what's on disk. The whole point of restore + // is "the DB is broken, rebuild it from disk" — a wizard that + // only queries the DB shows "No backups found" exactly when it's + // needed most. See discoverAvailableBackups for the full rationale. + const discovered = await discoverAvailableBackups(); - // Get database backups from backup_runs table - const backupRuns = await db('backup_runs') - .where('status', 'completed') - .whereNotNull('manifest_path') - .orderBy('completed_at', 'desc') - .limit(20); + const filtered = source + ? discovered.filter((b) => b.type === source) + : discovered; - for (const run of backupRuns) { - const isS3 = run.manifest_path.startsWith('s3://'); - const backupType = isS3 ? 's3' : 'local'; - - // Filter by source if specified - if (source && source !== backupType) { - continue; - } - - backups.push({ - id: run.id, - type: backupType, - name: `Backup from ${new Date(run.completed_at).toLocaleString()}`, - path: run.manifest_path, - manifest_path: run.manifest_path, - manifestId: run.manifest_id, - manifestPath: run.manifest_path, - size: parseInt(run.total_size_bytes) || 0, - total_size: parseInt(run.total_size_bytes) || 0, - total_size_bytes: parseInt(run.total_size_bytes) || 0, - filesCount: run.files_backed_up || 0, - files_backed_up: run.files_backed_up || 0, - duration: run.duration_seconds, - duration_seconds: run.duration_seconds, - // Frontend expects snake_case date fields - created_at: run.completed_at, - completed_at: run.completed_at, - started_at: run.started_at, - // camelCase aliases - completedAt: run.completed_at, - startedAt: run.started_at, - // Backup metadata - status: run.status, - backup_type: run.backup_type, - backupType: run.backup_type, - backup_mode: run.backup_mode, - backupMode: run.backup_mode, - app_version: run.app_version, - appVersion: run.app_version - }); - } + // Shape for frontend compatibility — preserves every alias the + // frontend was already reading (snake_case + camelCase), so the + // UI rendering doesn't have to change. + const backups = filtered.map((b) => ({ + id: b.manifestId || null, + type: b.type, + name: b.completed + ? `Backup from ${new Date(b.completed).toLocaleString()}` + : b.name, + path: b.path, + manifest_path: b.path, + manifestId: b.manifestId, + manifestPath: b.path, + size: parseInt(b.size) || 0, + total_size: parseInt(b.size) || 0, + total_size_bytes: parseInt(b.size) || 0, + filesCount: b.filesCount || 0, + files_backed_up: b.filesCount || 0, + duration: b.duration || null, + duration_seconds: b.duration || null, + created_at: b.completed, + completed_at: b.completed, + started_at: b.completed, + completedAt: b.completed, + startedAt: b.completed, + status: 'completed', + // Stage A-aware: when the source is a disk-scanned manifest we + // can tell the wizard whether the DB dump is present, so the + // UI can warn before the admin picks a files-only backup. + database_included: b.databaseIncluded, + databaseIncluded: b.databaseIncluded, + corrupt: b.corrupt || false, + // Provenance: 'disk' (manifest read from filesystem) vs 'db' + // (backup_runs row that the disk didn't surface) — useful for + // debugging which side is missing. + source: b.source, + schema_version: b.schemaVersion, + schemaVersion: b.schemaVersion, + })); res.json({ success: true,