fix(restore): resolve local source + always rollback on failure
Two changes that close the disaster-recovery loop the Stage A-B-C backup-hardening plan opened: 1. Resolve 'local' source to backup_destination_path The wizard passes options.source = 'local' (the SOURCE TYPE string). The old code assigned that verbatim to localBackupPath and every downstream path.join() ended up with junk like 'local/database/<file>.sql.gz'. Fixed by looking up backup_destination_path from app_settings when source='local', plus a layered candidate fallback in performDatabaseRestore so absolute paths in manifests are honoured first. 2. Auto-rollback on ANY failure during restore Previously rollback only fired when post-restore VERIFICATION failed (inside the try block). Anything that threw earlier — path bugs, pg_restore failure, file copy errors — left the destination half-clobbered with no automatic recovery. Now the catch block always invokes attemptRollback if a pre-restore backup exists, and persists rollback status in was_rollback_attempted + an enriched error_message so the admin can tell at a glance whether the destination is safe to retry on top of or needs manual inspection first. Surfaced during Ralf's validation of the end-to-end backup + restore cycle (`docker compose down -v` then restore from disk). Every prior failed attempt left stray PDFs behind that the next attempt had to navigate around — exactly the "every failure makes the next worse" pattern this fix kills.
This commit is contained in:
@@ -270,12 +270,53 @@ class RestoreService {
|
||||
} catch (error) {
|
||||
this.log('error', 'Restore failed', { error: error.message, stack: error.stack });
|
||||
|
||||
// Update restore run record
|
||||
// Always attempt rollback when a pre-restore backup exists.
|
||||
// Historically rollback was only triggered when post-restore
|
||||
// verification failed (inside the try block) — anything that
|
||||
// threw earlier (path-resolution bugs, pg_restore failure, file
|
||||
// copy errors) left the destination half-clobbered and forced
|
||||
// the admin to do another reset-from-volume cycle before the
|
||||
// next attempt could be honest. Fixing the rollback here closes
|
||||
// the "every failed restore makes the next one worse" footgun.
|
||||
let rollbackAttempted = false;
|
||||
let rollbackSucceeded = false;
|
||||
let rollbackError = null;
|
||||
if (this.preRestoreBackupPath) {
|
||||
rollbackAttempted = true;
|
||||
try {
|
||||
this.log('info', 'Attempting rollback from pre-restore safety backup', {
|
||||
path: this.preRestoreBackupPath,
|
||||
});
|
||||
await this.attemptRollback(this.preRestoreBackupPath);
|
||||
rollbackSucceeded = true;
|
||||
this.log('info', 'Rollback completed');
|
||||
} catch (rbErr) {
|
||||
rollbackError = rbErr.message;
|
||||
this.log('error', 'Rollback FAILED — install may be in a partial state',
|
||||
{ error: rbErr.message, stack: rbErr.stack });
|
||||
}
|
||||
} else {
|
||||
this.log('warn', 'No pre-restore backup available — cannot auto-rollback. ' +
|
||||
'Destination may be in a partial state. Verify business-docs/ and the DB before retrying.');
|
||||
}
|
||||
|
||||
// Update restore run record. We persist BOTH the original
|
||||
// restore failure AND the rollback status so the admin can tell
|
||||
// from a single SQL query which scenario they're in:
|
||||
// - rollback succeeded → destination is back to pre-restore state, safe to retry
|
||||
// - rollback failed → partial state, admin must inspect before next attempt
|
||||
// - rollback skipped → user opted out via skipPreBackup; same as above
|
||||
if (restoreRun) {
|
||||
const failureMessage = rollbackAttempted
|
||||
? (rollbackSucceeded
|
||||
? `${error.message} (rolled back successfully to pre-restore state)`
|
||||
: `${error.message} | ROLLBACK ALSO FAILED: ${rollbackError} — destination is in a partial state, inspect before retrying`)
|
||||
: `${error.message} (no pre-restore backup available — destination may be partial)`;
|
||||
await db('restore_runs').where('id', restoreRun.id).update({
|
||||
completed_at: new Date(),
|
||||
status: 'failed',
|
||||
error_message: error.message,
|
||||
error_message: failureMessage,
|
||||
was_rollback_attempted: rollbackAttempted,
|
||||
restore_log: JSON.stringify(this.restoreLog)
|
||||
});
|
||||
}
|
||||
@@ -283,7 +324,9 @@ class RestoreService {
|
||||
// Send failure notification
|
||||
await this.sendRestoreNotification('failure', {
|
||||
error: error.message,
|
||||
restoreType: options.restoreType
|
||||
restoreType: options.restoreType,
|
||||
rollbackAttempted,
|
||||
rollbackSucceeded,
|
||||
});
|
||||
|
||||
throw error;
|
||||
|
||||
Reference in New Issue
Block a user