fix(restore): discover backups from disk, not just the DB
The Restore wizard's "Choose Backup to Restore" list was driven only
by the backup_runs table. After `docker compose down -v` (the disaster
this whole hardening effort is designed to recover from), the DB is
empty and the wizard shows "No backups found in selected source" —
exactly when it's needed most. The manifest JSONs are still on disk;
the wizard just can't see them.
Adds disk-first discovery:
- Walks backup_destination_path AND backup_manifest_path (manifests
can live in a sibling directory under the canonical
<root>/manifests/backup-manifest-<id>.json layout). Depth-limited
recursion (3 levels) so the scan doesn't enumerate the photo tree.
- Matches backup-manifest-*.json|yaml AND legacy bare manifest.json.
- Parses each manifest for real metadata (timestamp, size, file
count, database.backup_file presence) instead of showing the
admin opaque filenames.
- Layers in surviving backup_runs rows, deduping by manifest_id.
Applied to both GET /available-backups (legacy) and POST /list-backups
(the one the frontend actually calls). Same helper, two call sites.
Side benefit: each returned row now carries `databaseIncluded` — so a
future Restore UI iteration can show a "this backup has no DB dump"
warning before the admin picks a files-only backup. Exactly the
surface that would have caught Ralf's original four files-only
manifests if it had existed.
This commit is contained in:
Luca
@@ -348,51 +348,7 @@ router.get('/run/:id/report', requirePermission('backup.view'), async (req, res)
|
|||||||
*/
|
*/
|
||||||
router.get('/available-backups', requirePermission('backup.view'), async (req, res) => {
|
router.get('/available-backups', requirePermission('backup.view'), async (req, res) => {
|
||||||
try {
|
try {
|
||||||
const backups = [];
|
const backups = await discoverAvailableBackups();
|
||||||
|
|
||||||
// Get local file backups
|
|
||||||
const backupConfig = await getBackupConfig();
|
|
||||||
if (backupConfig.backup_destination_type === 'local' && backupConfig.backup_destination_path) {
|
|
||||||
try {
|
|
||||||
const files = await fs.readdir(backupConfig.backup_destination_path);
|
|
||||||
for (const file of files) {
|
|
||||||
if (file.endsWith('.json') || file.endsWith('.yaml')) {
|
|
||||||
const filePath = path.join(backupConfig.backup_destination_path, file);
|
|
||||||
const stats = await fs.stat(filePath);
|
|
||||||
backups.push({
|
|
||||||
type: 'local',
|
|
||||||
name: file,
|
|
||||||
path: filePath,
|
|
||||||
size: stats.size,
|
|
||||||
modified: stats.mtime
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
logger.warn('Failed to list local backups:', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get database backups from backup_runs table
|
|
||||||
const backupRuns = await db('backup_runs')
|
|
||||||
.where('status', 'completed')
|
|
||||||
.whereNotNull('manifest_path')
|
|
||||||
.orderBy('completed_at', 'desc')
|
|
||||||
.limit(20);
|
|
||||||
|
|
||||||
for (const run of backupRuns) {
|
|
||||||
backups.push({
|
|
||||||
type: run.manifest_path.startsWith('s3://') ? 's3' : 'local',
|
|
||||||
name: `Backup ${run.completed_at}`,
|
|
||||||
path: run.manifest_path,
|
|
||||||
manifestId: run.manifest_id,
|
|
||||||
size: run.total_size_bytes,
|
|
||||||
filesCount: run.files_backed_up,
|
|
||||||
duration: run.duration_seconds,
|
|
||||||
completed: run.completed_at
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
res.json({
|
res.json({
|
||||||
success: true,
|
success: true,
|
||||||
data: backups
|
data: backups
|
||||||
@@ -406,6 +362,213 @@ router.get('/available-backups', requirePermission('backup.view'), async (req, r
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Discover restorable backups by walking the configured destination
|
||||||
|
* directory + harvesting the backup_runs table.
|
||||||
|
*
|
||||||
|
* **Why we recurse the disk first, DB second**
|
||||||
|
*
|
||||||
|
* The disk is the source of truth for restore. After a disaster
|
||||||
|
* (`docker compose down -v`, drive corruption, fresh install) the
|
||||||
|
* `backup_runs` table is empty — but the manifest JSONs are exactly
|
||||||
|
* what's left on disk for an admin to recover from. A wizard that
|
||||||
|
* only reads the DB shows "No backups found" precisely when the
|
||||||
|
* admin needs it most. So we walk first, dedupe-by-manifestId
|
||||||
|
* against any surviving DB rows, and present a unified list.
|
||||||
|
*
|
||||||
|
* Discovery rules:
|
||||||
|
* - Walks `backup_destination_path` AND `backup_manifest_path` if
|
||||||
|
* they're distinct (manifests can live in a sibling directory).
|
||||||
|
* - Recurses up to 3 levels deep — enough to find
|
||||||
|
* `<root>/manifests/backup-manifest-<id>.json` (the default
|
||||||
|
* layout) without scanning the entire photo tree.
|
||||||
|
* - Matches manifest files by glob: `backup-manifest-*.json`,
|
||||||
|
* `backup-manifest-*.yaml`, and the legacy bare `manifest.json`.
|
||||||
|
* - Parses each manifest to extract real metadata (timestamp,
|
||||||
|
* size, file count, source type) instead of showing the admin
|
||||||
|
* a list of opaque filenames.
|
||||||
|
*
|
||||||
|
* Returns: array of `{ type, name, path, manifestId, size,
|
||||||
|
* filesCount, completed, source: 'disk' | 'db' }`.
|
||||||
|
*/
|
||||||
|
async function discoverAvailableBackups() {
|
||||||
|
const backupConfig = await getBackupConfig();
|
||||||
|
const backups = [];
|
||||||
|
const seenManifestIds = new Set();
|
||||||
|
|
||||||
|
if (backupConfig.backup_destination_type === 'local') {
|
||||||
|
const roots = new Set();
|
||||||
|
if (backupConfig.backup_destination_path) roots.add(backupConfig.backup_destination_path);
|
||||||
|
if (backupConfig.backup_manifest_path) roots.add(backupConfig.backup_manifest_path);
|
||||||
|
|
||||||
|
for (const root of roots) {
|
||||||
|
try {
|
||||||
|
const manifestPaths = await walkForManifests(root, 3);
|
||||||
|
for (const filePath of manifestPaths) {
|
||||||
|
try {
|
||||||
|
const parsed = await parseManifestMetadata(filePath);
|
||||||
|
if (parsed.manifestId) seenManifestIds.add(parsed.manifestId);
|
||||||
|
backups.push(parsed);
|
||||||
|
} catch (err) {
|
||||||
|
// Don't fail discovery because ONE manifest is corrupt —
|
||||||
|
// surface the file with a note so the admin sees something
|
||||||
|
// is wrong and can investigate.
|
||||||
|
logger.warn(`Manifest unreadable at ${filePath}: ${err.message}`);
|
||||||
|
const stats = await fs.stat(filePath).catch(() => null);
|
||||||
|
backups.push({
|
||||||
|
type: 'local',
|
||||||
|
name: path.basename(filePath),
|
||||||
|
path: filePath,
|
||||||
|
manifestId: null,
|
||||||
|
size: stats?.size || 0,
|
||||||
|
filesCount: null,
|
||||||
|
completed: stats?.mtime || null,
|
||||||
|
source: 'disk',
|
||||||
|
corrupt: true,
|
||||||
|
error: err.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
logger.warn(`Could not scan backup root ${root}: ${err.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Layer in surviving DB rows, deduping by manifest_id so we don't
|
||||||
|
// show the same backup twice with different shapes.
|
||||||
|
const backupRuns = await db('backup_runs')
|
||||||
|
.where('status', 'completed')
|
||||||
|
.whereNotNull('manifest_path')
|
||||||
|
.orderBy('completed_at', 'desc')
|
||||||
|
.limit(50);
|
||||||
|
|
||||||
|
for (const run of backupRuns) {
|
||||||
|
if (run.manifest_id && seenManifestIds.has(run.manifest_id)) continue;
|
||||||
|
backups.push({
|
||||||
|
type: run.manifest_path.startsWith('s3://') ? 's3' : 'local',
|
||||||
|
name: `Backup ${run.completed_at}`,
|
||||||
|
path: run.manifest_path,
|
||||||
|
manifestId: run.manifest_id,
|
||||||
|
size: run.total_size_bytes,
|
||||||
|
filesCount: run.files_backed_up,
|
||||||
|
duration: run.duration_seconds,
|
||||||
|
completed: run.completed_at,
|
||||||
|
source: 'db',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Most recent first.
|
||||||
|
backups.sort((a, b) => {
|
||||||
|
const aTime = a.completed ? new Date(a.completed).getTime() : 0;
|
||||||
|
const bTime = b.completed ? new Date(b.completed).getTime() : 0;
|
||||||
|
return bTime - aTime;
|
||||||
|
});
|
||||||
|
|
||||||
|
return backups;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recursive manifest finder. Depth-limited so we don't enumerate
|
||||||
|
* thousands of photo files. Yields absolute paths.
|
||||||
|
*/
|
||||||
|
async function walkForManifests(dir, maxDepth, depth = 0) {
|
||||||
|
if (depth > maxDepth) return [];
|
||||||
|
let entries;
|
||||||
|
try {
|
||||||
|
entries = await fs.readdir(dir, { withFileTypes: true });
|
||||||
|
} catch (err) {
|
||||||
|
if (err.code === 'ENOENT') return [];
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
const out = [];
|
||||||
|
for (const entry of entries) {
|
||||||
|
const full = path.join(dir, entry.name);
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
// Skip obvious noise: photo trees, node_modules, hidden dirs.
|
||||||
|
if (entry.name === 'events' || entry.name === 'business-docs'
|
||||||
|
|| entry.name === 'thumbnails' || entry.name === 'previews'
|
||||||
|
|| entry.name === 'heroes' || entry.name === 'uploads'
|
||||||
|
|| entry.name.startsWith('.')
|
||||||
|
|| entry.name === 'node_modules') continue;
|
||||||
|
out.push(...await walkForManifests(full, maxDepth, depth + 1));
|
||||||
|
} else if (entry.isFile() && isManifestFilename(entry.name)) {
|
||||||
|
out.push(full);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isManifestFilename(name) {
|
||||||
|
// Canonical: backup-manifest-<id>.json / .yaml
|
||||||
|
// Legacy: manifest.json (inside backup-<id>/manifest.json layout)
|
||||||
|
// Be liberal in what we accept — admin may have renamed.
|
||||||
|
if (/^backup-manifest-.+\.(json|ya?ml)$/i.test(name)) return true;
|
||||||
|
if (/^manifest\.(json|ya?ml)$/i.test(name)) return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse a manifest file and pull out the fields the wizard wants.
|
||||||
|
* Tolerates schema drift across manifest versions (v1, v2) by
|
||||||
|
* checking multiple shapes.
|
||||||
|
*/
|
||||||
|
async function parseManifestMetadata(filePath) {
|
||||||
|
const raw = await fs.readFile(filePath, 'utf8');
|
||||||
|
let parsed;
|
||||||
|
if (filePath.toLowerCase().endsWith('.json')) {
|
||||||
|
parsed = JSON.parse(raw);
|
||||||
|
} else {
|
||||||
|
// Minimal YAML support — most admins use JSON; only do require()
|
||||||
|
// if a .yaml manifest is actually present.
|
||||||
|
const yaml = require('js-yaml');
|
||||||
|
parsed = yaml.load(raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
const stats = await fs.stat(filePath);
|
||||||
|
|
||||||
|
// v2 shape: { manifest: { id, timestamp }, backup: { ... }, files: [...], database: { ... } }
|
||||||
|
// v1 shape: { backup_id, started_at, files: [...] } (older)
|
||||||
|
const manifestId =
|
||||||
|
parsed?.manifest?.id
|
||||||
|
|| parsed?.backup?.id
|
||||||
|
|| parsed?.backup_id
|
||||||
|
|| null;
|
||||||
|
|
||||||
|
const completed =
|
||||||
|
parsed?.backup?.completed_at
|
||||||
|
|| parsed?.manifest?.timestamp
|
||||||
|
|| parsed?.completed_at
|
||||||
|
|| stats.mtime;
|
||||||
|
|
||||||
|
const filesCount =
|
||||||
|
(Array.isArray(parsed?.files) ? parsed.files.length : null)
|
||||||
|
?? parsed?.backup?.total_files
|
||||||
|
?? null;
|
||||||
|
|
||||||
|
const totalSizeBytes =
|
||||||
|
parsed?.backup?.total_size_bytes
|
||||||
|
?? parsed?.total_size_bytes
|
||||||
|
?? null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
type: 'local',
|
||||||
|
name: path.basename(filePath),
|
||||||
|
path: filePath,
|
||||||
|
manifestId,
|
||||||
|
size: totalSizeBytes ?? stats.size,
|
||||||
|
filesCount,
|
||||||
|
completed,
|
||||||
|
source: 'disk',
|
||||||
|
databaseIncluded: Boolean(parsed?.database?.backup_file),
|
||||||
|
// Helpful for the UI: lets it show "This backup has no DB" warning
|
||||||
|
// — exactly the surface that would have caught Ralf's original
|
||||||
|
// four files-only manifests if it had existed.
|
||||||
|
schemaVersion: parsed?.manifest?.version || parsed?.version || '1.0',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* List backups for restore (POST version for frontend compatibility)
|
* List backups for restore (POST version for frontend compatibility)
|
||||||
* Accepts source type in request body
|
* Accepts source type in request body
|
||||||
@@ -413,59 +576,58 @@ router.get('/available-backups', requirePermission('backup.view'), async (req, r
|
|||||||
router.post('/list-backups', requirePermission('backup.view'), async (req, res) => {
|
router.post('/list-backups', requirePermission('backup.view'), async (req, res) => {
|
||||||
try {
|
try {
|
||||||
const { source } = req.body; // 'local', 's3', or undefined for all
|
const { source } = req.body; // 'local', 's3', or undefined for all
|
||||||
const backups = [];
|
|
||||||
|
|
||||||
// Get backup configuration
|
// Use the same disk-first discovery the GET endpoint uses so that
|
||||||
const backupConfig = await getBackupConfig();
|
// a fresh post-`docker compose down -v` install (empty backup_runs
|
||||||
|
// table) can still see what's on disk. The whole point of restore
|
||||||
|
// is "the DB is broken, rebuild it from disk" — a wizard that
|
||||||
|
// only queries the DB shows "No backups found" exactly when it's
|
||||||
|
// needed most. See discoverAvailableBackups for the full rationale.
|
||||||
|
const discovered = await discoverAvailableBackups();
|
||||||
|
|
||||||
// Get database backups from backup_runs table
|
const filtered = source
|
||||||
const backupRuns = await db('backup_runs')
|
? discovered.filter((b) => b.type === source)
|
||||||
.where('status', 'completed')
|
: discovered;
|
||||||
.whereNotNull('manifest_path')
|
|
||||||
.orderBy('completed_at', 'desc')
|
|
||||||
.limit(20);
|
|
||||||
|
|
||||||
for (const run of backupRuns) {
|
// Shape for frontend compatibility — preserves every alias the
|
||||||
const isS3 = run.manifest_path.startsWith('s3://');
|
// frontend was already reading (snake_case + camelCase), so the
|
||||||
const backupType = isS3 ? 's3' : 'local';
|
// UI rendering doesn't have to change.
|
||||||
|
const backups = filtered.map((b) => ({
|
||||||
// Filter by source if specified
|
id: b.manifestId || null,
|
||||||
if (source && source !== backupType) {
|
type: b.type,
|
||||||
continue;
|
name: b.completed
|
||||||
}
|
? `Backup from ${new Date(b.completed).toLocaleString()}`
|
||||||
|
: b.name,
|
||||||
backups.push({
|
path: b.path,
|
||||||
id: run.id,
|
manifest_path: b.path,
|
||||||
type: backupType,
|
manifestId: b.manifestId,
|
||||||
name: `Backup from ${new Date(run.completed_at).toLocaleString()}`,
|
manifestPath: b.path,
|
||||||
path: run.manifest_path,
|
size: parseInt(b.size) || 0,
|
||||||
manifest_path: run.manifest_path,
|
total_size: parseInt(b.size) || 0,
|
||||||
manifestId: run.manifest_id,
|
total_size_bytes: parseInt(b.size) || 0,
|
||||||
manifestPath: run.manifest_path,
|
filesCount: b.filesCount || 0,
|
||||||
size: parseInt(run.total_size_bytes) || 0,
|
files_backed_up: b.filesCount || 0,
|
||||||
total_size: parseInt(run.total_size_bytes) || 0,
|
duration: b.duration || null,
|
||||||
total_size_bytes: parseInt(run.total_size_bytes) || 0,
|
duration_seconds: b.duration || null,
|
||||||
filesCount: run.files_backed_up || 0,
|
created_at: b.completed,
|
||||||
files_backed_up: run.files_backed_up || 0,
|
completed_at: b.completed,
|
||||||
duration: run.duration_seconds,
|
started_at: b.completed,
|
||||||
duration_seconds: run.duration_seconds,
|
completedAt: b.completed,
|
||||||
// Frontend expects snake_case date fields
|
startedAt: b.completed,
|
||||||
created_at: run.completed_at,
|
status: 'completed',
|
||||||
completed_at: run.completed_at,
|
// Stage A-aware: when the source is a disk-scanned manifest we
|
||||||
started_at: run.started_at,
|
// can tell the wizard whether the DB dump is present, so the
|
||||||
// camelCase aliases
|
// UI can warn before the admin picks a files-only backup.
|
||||||
completedAt: run.completed_at,
|
database_included: b.databaseIncluded,
|
||||||
startedAt: run.started_at,
|
databaseIncluded: b.databaseIncluded,
|
||||||
// Backup metadata
|
corrupt: b.corrupt || false,
|
||||||
status: run.status,
|
// Provenance: 'disk' (manifest read from filesystem) vs 'db'
|
||||||
backup_type: run.backup_type,
|
// (backup_runs row that the disk didn't surface) — useful for
|
||||||
backupType: run.backup_type,
|
// debugging which side is missing.
|
||||||
backup_mode: run.backup_mode,
|
source: b.source,
|
||||||
backupMode: run.backup_mode,
|
schema_version: b.schemaVersion,
|
||||||
app_version: run.app_version,
|
schemaVersion: b.schemaVersion,
|
||||||
appVersion: run.app_version
|
}));
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
res.json({
|
res.json({
|
||||||
success: true,
|
success: true,
|
||||||
|
|||||||
Reference in New Issue
Block a user