fix(restore): resolve local source + always rollback on failure
Two changes that close the disaster-recovery loop the Stage A-B-C backup-hardening plan opened: 1. Resolve 'local' source to backup_destination_path The wizard passes options.source = 'local' (the SOURCE TYPE string). The old code assigned that verbatim to localBackupPath and every downstream path.join() ended up with junk like 'local/database/<file>.sql.gz'. Fixed by looking up backup_destination_path from app_settings when source='local', plus a layered candidate fallback in performDatabaseRestore so absolute paths in manifests are honoured first. 2. Auto-rollback on ANY failure during restore Previously rollback only fired when post-restore VERIFICATION failed (inside the try block). Anything that threw earlier — path bugs, pg_restore failure, file copy errors — left the destination half-clobbered with no automatic recovery. Now the catch block always invokes attemptRollback if a pre-restore backup exists, and persists rollback status in was_rollback_attempted + an enriched error_message so the admin can tell at a glance whether the destination is safe to retry on top of or needs manual inspection first. Surfaced during Ralf's validation of the end-to-end backup + restore cycle (`docker compose down -v` then restore from disk). Every prior failed attempt left stray PDFs behind that the next attempt had to navigate around — exactly the "every failure makes the next worse" pattern this fix kills.
This commit is contained in:
@@ -270,12 +270,53 @@ class RestoreService {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.log('error', 'Restore failed', { error: error.message, stack: error.stack });
|
this.log('error', 'Restore failed', { error: error.message, stack: error.stack });
|
||||||
|
|
||||||
// Update restore run record
|
// Always attempt rollback when a pre-restore backup exists.
|
||||||
|
// Historically rollback was only triggered when post-restore
|
||||||
|
// verification failed (inside the try block) — anything that
|
||||||
|
// threw earlier (path-resolution bugs, pg_restore failure, file
|
||||||
|
// copy errors) left the destination half-clobbered and forced
|
||||||
|
// the admin to do another reset-from-volume cycle before the
|
||||||
|
// next attempt could be honest. Fixing the rollback here closes
|
||||||
|
// the "every failed restore makes the next one worse" footgun.
|
||||||
|
let rollbackAttempted = false;
|
||||||
|
let rollbackSucceeded = false;
|
||||||
|
let rollbackError = null;
|
||||||
|
if (this.preRestoreBackupPath) {
|
||||||
|
rollbackAttempted = true;
|
||||||
|
try {
|
||||||
|
this.log('info', 'Attempting rollback from pre-restore safety backup', {
|
||||||
|
path: this.preRestoreBackupPath,
|
||||||
|
});
|
||||||
|
await this.attemptRollback(this.preRestoreBackupPath);
|
||||||
|
rollbackSucceeded = true;
|
||||||
|
this.log('info', 'Rollback completed');
|
||||||
|
} catch (rbErr) {
|
||||||
|
rollbackError = rbErr.message;
|
||||||
|
this.log('error', 'Rollback FAILED — install may be in a partial state',
|
||||||
|
{ error: rbErr.message, stack: rbErr.stack });
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
this.log('warn', 'No pre-restore backup available — cannot auto-rollback. ' +
|
||||||
|
'Destination may be in a partial state. Verify business-docs/ and the DB before retrying.');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update restore run record. We persist BOTH the original
|
||||||
|
// restore failure AND the rollback status so the admin can tell
|
||||||
|
// from a single SQL query which scenario they're in:
|
||||||
|
// - rollback succeeded → destination is back to pre-restore state, safe to retry
|
||||||
|
// - rollback failed → partial state, admin must inspect before next attempt
|
||||||
|
// - rollback skipped → user opted out via skipPreBackup; same as above
|
||||||
if (restoreRun) {
|
if (restoreRun) {
|
||||||
|
const failureMessage = rollbackAttempted
|
||||||
|
? (rollbackSucceeded
|
||||||
|
? `${error.message} (rolled back successfully to pre-restore state)`
|
||||||
|
: `${error.message} | ROLLBACK ALSO FAILED: ${rollbackError} — destination is in a partial state, inspect before retrying`)
|
||||||
|
: `${error.message} (no pre-restore backup available — destination may be partial)`;
|
||||||
await db('restore_runs').where('id', restoreRun.id).update({
|
await db('restore_runs').where('id', restoreRun.id).update({
|
||||||
completed_at: new Date(),
|
completed_at: new Date(),
|
||||||
status: 'failed',
|
status: 'failed',
|
||||||
error_message: error.message,
|
error_message: failureMessage,
|
||||||
|
was_rollback_attempted: rollbackAttempted,
|
||||||
restore_log: JSON.stringify(this.restoreLog)
|
restore_log: JSON.stringify(this.restoreLog)
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -283,7 +324,9 @@ class RestoreService {
|
|||||||
// Send failure notification
|
// Send failure notification
|
||||||
await this.sendRestoreNotification('failure', {
|
await this.sendRestoreNotification('failure', {
|
||||||
error: error.message,
|
error: error.message,
|
||||||
restoreType: options.restoreType
|
restoreType: options.restoreType,
|
||||||
|
rollbackAttempted,
|
||||||
|
rollbackSucceeded,
|
||||||
});
|
});
|
||||||
|
|
||||||
throw error;
|
throw error;
|
||||||
|
|||||||
Reference in New Issue
Block a user