feat: implement comprehensive backup and restore system with S3 support
- Add S3/MinIO storage adapter with multipart upload support - Implement database backup service for SQLite and PostgreSQL - Create backup manifest generator for tracking backup contents - Enhance backup service with S3 integration and incremental backups - Add restore service with safety measures and rollback capability - Create comprehensive test suite for all backup functionality - Add admin API endpoints for backup/restore management - Implement frontend UI with dashboard, configuration, and restore wizard - Add roadmap section to README with implemented backup feature This implementation provides: - Multiple backup destinations (local, rsync, S3/MinIO) - Intelligent change detection to minimize backup frequency - Full database backups with compression - Manifest-based restore with integrity validation - Pre-restore safety backups with rollback - Comprehensive error handling and monitoring - User-friendly admin interface 🤖 Generated with Claude Code Co-Authored-By: Claude <[email protected]>
This commit is contained in:
Claude
@@ -0,0 +1,243 @@
|
||||
const { db } = require('../src/database/db');
|
||||
|
||||
async function up() {
|
||||
console.log('Adding backup service tables and settings...');
|
||||
|
||||
// Create backup_runs table to track backup history
|
||||
const hasBackupRunsTable = await db.schema.hasTable('backup_runs');
|
||||
if (!hasBackupRunsTable) {
|
||||
await db.schema.createTable('backup_runs', (table) => {
|
||||
table.increments('id').primary();
|
||||
table.datetime('started_at').notNullable();
|
||||
table.datetime('completed_at');
|
||||
table.string('status').defaultTo('running'); // running, completed, failed
|
||||
table.string('backup_type'); // full, incremental
|
||||
table.integer('files_backed_up').defaultTo(0);
|
||||
table.bigInteger('total_size_bytes').defaultTo(0);
|
||||
table.integer('duration_seconds');
|
||||
table.text('error_message');
|
||||
table.json('statistics'); // Detailed stats about the backup
|
||||
table.json('file_checksums'); // Store checksums for change detection
|
||||
});
|
||||
}
|
||||
|
||||
// Create backup_file_states table to track individual file states
|
||||
const hasBackupFileStatesTable = await db.schema.hasTable('backup_file_states');
|
||||
if (!hasBackupFileStatesTable) {
|
||||
await db.schema.createTable('backup_file_states', (table) => {
|
||||
table.increments('id').primary();
|
||||
table.string('file_path').notNullable();
|
||||
table.string('checksum').notNullable();
|
||||
table.bigInteger('size_bytes');
|
||||
table.datetime('last_modified');
|
||||
table.datetime('last_backed_up');
|
||||
table.boolean('is_archived').defaultTo(false);
|
||||
table.index(['file_path'], 'idx_backup_file_path');
|
||||
table.index(['checksum'], 'idx_backup_checksum');
|
||||
});
|
||||
}
|
||||
|
||||
// Add backup-related settings to app_settings
|
||||
const backupSettings = [
|
||||
{
|
||||
setting_key: 'backup_enabled',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_schedule',
|
||||
setting_value: JSON.stringify('0 2 * * *'), // Default: 2 AM daily
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_destination_type',
|
||||
setting_value: JSON.stringify('local'), // local, rsync, s3
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_destination_path',
|
||||
setting_value: JSON.stringify('/backup/picpeak'),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_rsync_host',
|
||||
setting_value: JSON.stringify(''),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_rsync_user',
|
||||
setting_value: JSON.stringify(''),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_rsync_path',
|
||||
setting_value: JSON.stringify(''),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_rsync_ssh_key',
|
||||
setting_value: JSON.stringify(''),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_s3_endpoint',
|
||||
setting_value: JSON.stringify(''),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_s3_bucket',
|
||||
setting_value: JSON.stringify(''),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_s3_access_key',
|
||||
setting_value: JSON.stringify(''),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_s3_secret_key',
|
||||
setting_value: JSON.stringify(''),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_s3_region',
|
||||
setting_value: JSON.stringify('us-east-1'),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_retention_days',
|
||||
setting_value: JSON.stringify(30),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_include_archived',
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_compression',
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_email_on_failure',
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_email_on_success',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_max_file_size_mb',
|
||||
setting_value: JSON.stringify(5000), // Skip files larger than 5GB
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_exclude_patterns',
|
||||
setting_value: JSON.stringify(['*.tmp', '.DS_Store', 'Thumbs.db']),
|
||||
setting_type: 'backup'
|
||||
}
|
||||
];
|
||||
|
||||
// Insert backup settings if they don't exist
|
||||
for (const setting of backupSettings) {
|
||||
const exists = await db('app_settings')
|
||||
.where('setting_key', setting.setting_key)
|
||||
.first();
|
||||
|
||||
if (!exists) {
|
||||
await db('app_settings').insert(setting);
|
||||
}
|
||||
}
|
||||
|
||||
// Add backup-related email templates
|
||||
const backupEmailTemplates = [
|
||||
{
|
||||
template_key: 'backup_failed',
|
||||
subject_en: 'Backup Failed - Immediate Attention Required',
|
||||
subject_de: 'Backup fehlgeschlagen - Sofortige Aufmerksamkeit erforderlich',
|
||||
body_html_en: `<h2>Backup Failed</h2>
|
||||
<p>The scheduled backup has failed and requires immediate attention.</p>
|
||||
<p><strong>Error Details:</strong></p>
|
||||
<ul>
|
||||
<li>Start Time: {{start_time}}</li>
|
||||
<li>Backup Type: {{backup_type}}</li>
|
||||
<li>Error: {{error_message}}</li>
|
||||
</ul>
|
||||
<p>Please check the system logs for more details and resolve the issue as soon as possible.</p>`,
|
||||
body_html_de: `<h2>Backup fehlgeschlagen</h2>
|
||||
<p>Das geplante Backup ist fehlgeschlagen und erfordert sofortige Aufmerksamkeit.</p>
|
||||
<p><strong>Fehlerdetails:</strong></p>
|
||||
<ul>
|
||||
<li>Startzeit: {{start_time}}</li>
|
||||
<li>Backup-Typ: {{backup_type}}</li>
|
||||
<li>Fehler: {{error_message}}</li>
|
||||
</ul>
|
||||
<p>Bitte überprüfen Sie die Systemprotokolle für weitere Details und beheben Sie das Problem so schnell wie möglich.</p>`,
|
||||
body_text_en: 'Backup Failed\n\nThe scheduled backup has failed and requires immediate attention.\n\nStart Time: {{start_time}}\nBackup Type: {{backup_type}}\nError: {{error_message}}\n\nPlease check the system logs for more details.',
|
||||
body_text_de: 'Backup fehlgeschlagen\n\nDas geplante Backup ist fehlgeschlagen und erfordert sofortige Aufmerksamkeit.\n\nStartzeit: {{start_time}}\nBackup-Typ: {{backup_type}}\nFehler: {{error_message}}\n\nBitte überprüfen Sie die Systemprotokolle für weitere Details.',
|
||||
variables: JSON.stringify(['start_time', 'backup_type', 'error_message'])
|
||||
},
|
||||
{
|
||||
template_key: 'backup_completed',
|
||||
subject_en: 'Backup Completed Successfully',
|
||||
subject_de: 'Backup erfolgreich abgeschlossen',
|
||||
body_html_en: `<h2>Backup Completed</h2>
|
||||
<p>The scheduled backup has been completed successfully.</p>
|
||||
<p><strong>Backup Summary:</strong></p>
|
||||
<ul>
|
||||
<li>Start Time: {{start_time}}</li>
|
||||
<li>Duration: {{duration}}</li>
|
||||
<li>Files Backed Up: {{files_count}}</li>
|
||||
<li>Total Size: {{total_size}}</li>
|
||||
<li>Backup Type: {{backup_type}}</li>
|
||||
</ul>`,
|
||||
body_html_de: `<h2>Backup abgeschlossen</h2>
|
||||
<p>Das geplante Backup wurde erfolgreich abgeschlossen.</p>
|
||||
<p><strong>Backup-Zusammenfassung:</strong></p>
|
||||
<ul>
|
||||
<li>Startzeit: {{start_time}}</li>
|
||||
<li>Dauer: {{duration}}</li>
|
||||
<li>Gesicherte Dateien: {{files_count}}</li>
|
||||
<li>Gesamtgröße: {{total_size}}</li>
|
||||
<li>Backup-Typ: {{backup_type}}</li>
|
||||
</ul>`,
|
||||
body_text_en: 'Backup Completed\n\nThe scheduled backup has been completed successfully.\n\nStart Time: {{start_time}}\nDuration: {{duration}}\nFiles Backed Up: {{files_count}}\nTotal Size: {{total_size}}\nBackup Type: {{backup_type}}',
|
||||
body_text_de: 'Backup abgeschlossen\n\nDas geplante Backup wurde erfolgreich abgeschlossen.\n\nStartzeit: {{start_time}}\nDauer: {{duration}}\nGesicherte Dateien: {{files_count}}\nGesamtgröße: {{total_size}}\nBackup-Typ: {{backup_type}}',
|
||||
variables: JSON.stringify(['start_time', 'duration', 'files_count', 'total_size', 'backup_type'])
|
||||
}
|
||||
];
|
||||
|
||||
// Insert backup email templates if they don't exist
|
||||
for (const template of backupEmailTemplates) {
|
||||
const exists = await db('email_templates')
|
||||
.where('template_key', template.template_key)
|
||||
.first();
|
||||
|
||||
if (!exists) {
|
||||
await db('email_templates').insert(template);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Backup service tables and settings added successfully');
|
||||
}
|
||||
|
||||
async function down() {
|
||||
// Remove backup tables
|
||||
await db.schema.dropTableIfExists('backup_file_states');
|
||||
await db.schema.dropTableIfExists('backup_runs');
|
||||
|
||||
// Remove backup settings
|
||||
await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.delete();
|
||||
|
||||
// Remove backup email templates
|
||||
await db('email_templates')
|
||||
.whereIn('template_key', ['backup_failed', 'backup_completed'])
|
||||
.delete();
|
||||
}
|
||||
|
||||
module.exports = { up, down };
|
||||
@@ -0,0 +1,182 @@
|
||||
const { db } = require('../src/database/db');
|
||||
|
||||
async function up() {
|
||||
console.log('Adding database backup tables and settings...');
|
||||
|
||||
// Create database_backup_runs table to track database backup history
|
||||
const hasDatabaseBackupRunsTable = await db.schema.hasTable('database_backup_runs');
|
||||
if (!hasDatabaseBackupRunsTable) {
|
||||
await db.schema.createTable('database_backup_runs', (table) => {
|
||||
table.increments('id').primary();
|
||||
table.datetime('started_at').notNullable();
|
||||
table.datetime('completed_at');
|
||||
table.string('status').defaultTo('running'); // running, completed, failed
|
||||
table.string('backup_type'); // sqlite, postgresql
|
||||
table.string('destination_path');
|
||||
table.string('file_path');
|
||||
table.bigInteger('file_size_bytes').defaultTo(0);
|
||||
table.bigInteger('original_size_bytes').defaultTo(0);
|
||||
table.integer('duration_seconds');
|
||||
table.string('checksum'); // SHA256 checksum of backup file
|
||||
table.float('compression_ratio'); // Compression percentage
|
||||
table.json('table_checksums'); // Individual table checksums
|
||||
table.text('error_message');
|
||||
table.json('statistics'); // Detailed stats about the backup
|
||||
table.index(['started_at'], 'idx_db_backup_started');
|
||||
table.index(['status'], 'idx_db_backup_status');
|
||||
});
|
||||
}
|
||||
|
||||
// Add database backup-related settings to app_settings
|
||||
const databaseBackupSettings = [
|
||||
{
|
||||
setting_key: 'database_backup_enabled',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'database_backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'database_backup_schedule',
|
||||
setting_value: JSON.stringify('0 3 * * *'), // Default: 3 AM daily
|
||||
setting_type: 'database_backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'database_backup_destination_path',
|
||||
setting_value: JSON.stringify('/backup/database'),
|
||||
setting_type: 'database_backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'database_backup_compress',
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'database_backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'database_backup_validate_integrity',
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'database_backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'database_backup_include_checksums',
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'database_backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'database_backup_retention_days',
|
||||
setting_value: JSON.stringify(30),
|
||||
setting_type: 'database_backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'database_backup_email_on_failure',
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'database_backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'database_backup_email_on_success',
|
||||
setting_value: JSON.stringify(false),
|
||||
setting_type: 'database_backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'database_backup_max_retries',
|
||||
setting_value: JSON.stringify(3),
|
||||
setting_type: 'database_backup'
|
||||
}
|
||||
];
|
||||
|
||||
// Insert database backup settings if they don't exist
|
||||
for (const setting of databaseBackupSettings) {
|
||||
const exists = await db('app_settings')
|
||||
.where('setting_key', setting.setting_key)
|
||||
.first();
|
||||
|
||||
if (!exists) {
|
||||
await db('app_settings').insert(setting);
|
||||
}
|
||||
}
|
||||
|
||||
// Add database backup-related email templates
|
||||
const databaseBackupEmailTemplates = [
|
||||
{
|
||||
template_key: 'database_backup_failed',
|
||||
subject_en: 'Database Backup Failed - Critical Alert',
|
||||
subject_de: 'Datenbank-Backup fehlgeschlagen - Kritische Warnung',
|
||||
body_html_en: `<h2>Database Backup Failed</h2>
|
||||
<p>The scheduled database backup has failed and requires immediate attention.</p>
|
||||
<p><strong>Error Details:</strong></p>
|
||||
<ul>
|
||||
<li>Backup Type: {{backup_type}}</li>
|
||||
<li>Timestamp: {{timestamp}}</li>
|
||||
<li>Error: {{error_message}}</li>
|
||||
</ul>
|
||||
<p>This is a critical issue that could affect disaster recovery. Please investigate immediately.</p>`,
|
||||
body_html_de: `<h2>Datenbank-Backup fehlgeschlagen</h2>
|
||||
<p>Das geplante Datenbank-Backup ist fehlgeschlagen und erfordert sofortige Aufmerksamkeit.</p>
|
||||
<p><strong>Fehlerdetails:</strong></p>
|
||||
<ul>
|
||||
<li>Backup-Typ: {{backup_type}}</li>
|
||||
<li>Zeitstempel: {{timestamp}}</li>
|
||||
<li>Fehler: {{error_message}}</li>
|
||||
</ul>
|
||||
<p>Dies ist ein kritisches Problem, das die Disaster-Recovery beeinträchtigen könnte. Bitte untersuchen Sie es sofort.</p>`,
|
||||
body_text_en: 'Database Backup Failed\n\nThe scheduled database backup has failed.\n\nBackup Type: {{backup_type}}\nTimestamp: {{timestamp}}\nError: {{error_message}}\n\nThis is critical - please investigate immediately.',
|
||||
body_text_de: 'Datenbank-Backup fehlgeschlagen\n\nDas geplante Datenbank-Backup ist fehlgeschlagen.\n\nBackup-Typ: {{backup_type}}\nZeitstempel: {{timestamp}}\nFehler: {{error_message}}\n\nDies ist kritisch - bitte sofort untersuchen.',
|
||||
variables: JSON.stringify(['backup_type', 'timestamp', 'error_message'])
|
||||
},
|
||||
{
|
||||
template_key: 'database_backup_completed',
|
||||
subject_en: 'Database Backup Completed Successfully',
|
||||
subject_de: 'Datenbank-Backup erfolgreich abgeschlossen',
|
||||
body_html_en: `<h2>Database Backup Completed</h2>
|
||||
<p>The scheduled database backup has been completed successfully.</p>
|
||||
<p><strong>Backup Summary:</strong></p>
|
||||
<ul>
|
||||
<li>Backup Type: {{backup_type}}</li>
|
||||
<li>Duration: {{duration}}</li>
|
||||
<li>File Size: {{file_size}}</li>
|
||||
<li>Compression Ratio: {{compression_ratio}}</li>
|
||||
<li>File Path: {{file_path}}</li>
|
||||
</ul>`,
|
||||
body_html_de: `<h2>Datenbank-Backup abgeschlossen</h2>
|
||||
<p>Das geplante Datenbank-Backup wurde erfolgreich abgeschlossen.</p>
|
||||
<p><strong>Backup-Zusammenfassung:</strong></p>
|
||||
<ul>
|
||||
<li>Backup-Typ: {{backup_type}}</li>
|
||||
<li>Dauer: {{duration}}</li>
|
||||
<li>Dateigröße: {{file_size}}</li>
|
||||
<li>Komprimierungsverhältnis: {{compression_ratio}}</li>
|
||||
<li>Dateipfad: {{file_path}}</li>
|
||||
</ul>`,
|
||||
body_text_en: 'Database Backup Completed\n\nThe scheduled database backup has been completed successfully.\n\nBackup Type: {{backup_type}}\nDuration: {{duration}}\nFile Size: {{file_size}}\nCompression Ratio: {{compression_ratio}}\nFile Path: {{file_path}}',
|
||||
body_text_de: 'Datenbank-Backup abgeschlossen\n\nDas geplante Datenbank-Backup wurde erfolgreich abgeschlossen.\n\nBackup-Typ: {{backup_type}}\nDauer: {{duration}}\nDateigröße: {{file_size}}\nKomprimierungsverhältnis: {{compression_ratio}}\nDateipfad: {{file_path}}',
|
||||
variables: JSON.stringify(['backup_type', 'duration', 'file_size', 'compression_ratio', 'file_path'])
|
||||
}
|
||||
];
|
||||
|
||||
// Insert database backup email templates if they don't exist
|
||||
for (const template of databaseBackupEmailTemplates) {
|
||||
const exists = await db('email_templates')
|
||||
.where('template_key', template.template_key)
|
||||
.first();
|
||||
|
||||
if (!exists) {
|
||||
await db('email_templates').insert(template);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Database backup tables and settings added successfully');
|
||||
}
|
||||
|
||||
async function down() {
|
||||
// Remove database backup tables
|
||||
await db.schema.dropTableIfExists('database_backup_runs');
|
||||
|
||||
// Remove database backup settings
|
||||
await db('app_settings')
|
||||
.where('setting_type', 'database_backup')
|
||||
.delete();
|
||||
|
||||
// Remove database backup email templates
|
||||
await db('email_templates')
|
||||
.whereIn('template_key', ['database_backup_failed', 'database_backup_completed'])
|
||||
.delete();
|
||||
}
|
||||
|
||||
module.exports = { up, down };
|
||||
@@ -0,0 +1,86 @@
|
||||
const { db } = require('../src/database/db');
|
||||
const logger = require('../src/utils/logger');
|
||||
|
||||
async function up() {
|
||||
console.log('Adding backup manifest columns...');
|
||||
|
||||
// Add manifest columns to backup_runs table
|
||||
const hasManifestPath = await db.schema.hasColumn('backup_runs', 'manifest_path');
|
||||
if (!hasManifestPath) {
|
||||
await db.schema.alterTable('backup_runs', (table) => {
|
||||
table.string('manifest_path'); // Path to the manifest file
|
||||
table.string('manifest_id'); // Unique manifest ID
|
||||
table.string('manifest_format').defaultTo('json'); // json or yaml
|
||||
});
|
||||
}
|
||||
|
||||
// Add backup manifest-related settings to app_settings
|
||||
const manifestSettings = [
|
||||
{
|
||||
setting_key: 'backup_manifest_enabled',
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_manifest_format',
|
||||
setting_value: JSON.stringify('json'), // json or yaml
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_manifest_path',
|
||||
setting_value: JSON.stringify('/backup/manifests'),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_manifest_validate',
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'backup'
|
||||
},
|
||||
{
|
||||
setting_key: 'backup_manifest_include_checksums',
|
||||
setting_value: JSON.stringify(true),
|
||||
setting_type: 'backup'
|
||||
}
|
||||
];
|
||||
|
||||
// Insert manifest settings if they don't exist
|
||||
for (const setting of manifestSettings) {
|
||||
const exists = await db('app_settings')
|
||||
.where('setting_key', setting.setting_key)
|
||||
.first();
|
||||
|
||||
if (!exists) {
|
||||
await db('app_settings').insert(setting);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('✓ Backup manifest columns and settings added');
|
||||
}
|
||||
|
||||
async function down() {
|
||||
// Remove manifest columns from backup_runs table
|
||||
const hasManifestPath = await db.schema.hasColumn('backup_runs', 'manifest_path');
|
||||
if (hasManifestPath) {
|
||||
await db.schema.alterTable('backup_runs', (table) => {
|
||||
table.dropColumn('manifest_path');
|
||||
table.dropColumn('manifest_id');
|
||||
table.dropColumn('manifest_format');
|
||||
});
|
||||
}
|
||||
|
||||
// Remove manifest settings
|
||||
await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.whereIn('setting_key', [
|
||||
'backup_manifest_enabled',
|
||||
'backup_manifest_format',
|
||||
'backup_manifest_path',
|
||||
'backup_manifest_validate',
|
||||
'backup_manifest_include_checksums'
|
||||
])
|
||||
.delete();
|
||||
|
||||
console.log('✓ Backup manifest columns and settings removed');
|
||||
}
|
||||
|
||||
module.exports = { up, down };
|
||||
@@ -0,0 +1,131 @@
|
||||
exports.up = function(knex) {
|
||||
return knex.schema
|
||||
// Add new settings to app_settings table
|
||||
.table('app_settings', table => {
|
||||
// S3 configuration enhancements
|
||||
table.boolean('backup_s3_force_path_style').defaultTo(false).comment('Force path-style S3 URLs (for MinIO/self-hosted)');
|
||||
table.boolean('backup_s3_ssl_enabled').defaultTo(true).comment('Enable SSL/TLS for S3 connections');
|
||||
table.string('backup_s3_prefix', 255).comment('S3 key prefix for organizing backups');
|
||||
|
||||
// Backup features
|
||||
table.boolean('backup_incremental').defaultTo(false).comment('Enable incremental backups');
|
||||
table.boolean('backup_include_database').defaultTo(true).comment('Include database dumps in backups');
|
||||
table.boolean('backup_manifest_enabled').defaultTo(true).comment('Generate backup manifests');
|
||||
table.enum('backup_manifest_format', ['json', 'yaml']).defaultTo('json').comment('Manifest file format');
|
||||
table.boolean('backup_encryption_enabled').defaultTo(false).comment('Enable backup encryption');
|
||||
table.string('backup_database_schedule', 100).comment('Separate cron schedule for database-only backups');
|
||||
})
|
||||
|
||||
// Enhance backup_runs table
|
||||
.table('backup_runs', table => {
|
||||
// Manifest tracking
|
||||
table.string('manifest_path', 500).comment('Path to backup manifest file');
|
||||
table.uuid('manifest_id').comment('Unique identifier for the manifest');
|
||||
table.enum('manifest_format', ['json', 'yaml']).comment('Format of the manifest file');
|
||||
|
||||
// Incremental backup support
|
||||
table.integer('parent_backup_id').unsigned().references('id').inTable('backup_runs').onDelete('SET NULL').comment('Parent backup for incremental backups');
|
||||
table.enum('backup_mode', ['full', 'incremental', 'database']).defaultTo('full').comment('Type of backup performed');
|
||||
|
||||
// Add indexes for better query performance
|
||||
table.index(['backup_mode', 'status'], 'idx_backup_runs_mode_status');
|
||||
table.index(['parent_backup_id'], 'idx_backup_runs_parent');
|
||||
table.index(['created_at', 'backup_mode'], 'idx_backup_runs_created_mode');
|
||||
})
|
||||
|
||||
// Create backup_manifest table for storing detailed manifest metadata
|
||||
.createTable('backup_manifest', table => {
|
||||
table.increments('id').primary();
|
||||
table.integer('backup_run_id').unsigned().notNullable().references('id').inTable('backup_runs').onDelete('CASCADE');
|
||||
table.uuid('manifest_id').notNullable().unique().comment('Unique identifier matching backup_runs.manifest_id');
|
||||
table.string('version', 20).notNullable().defaultTo('1.0.0').comment('Manifest schema version');
|
||||
table.enum('format', ['json', 'yaml']).notNullable().defaultTo('json');
|
||||
|
||||
// Backup metadata
|
||||
table.timestamp('backup_start').notNullable();
|
||||
table.timestamp('backup_end').notNullable();
|
||||
table.bigInteger('total_size').unsigned().comment('Total size of backup in bytes');
|
||||
table.integer('file_count').unsigned().comment('Number of files in backup');
|
||||
table.integer('photo_count').unsigned().comment('Number of photos backed up');
|
||||
table.integer('event_count').unsigned().comment('Number of events backed up');
|
||||
|
||||
// Incremental backup metadata
|
||||
table.boolean('is_incremental').defaultTo(false);
|
||||
table.uuid('parent_manifest_id').comment('Parent manifest ID for incremental backups');
|
||||
table.timestamp('incremental_since').comment('Timestamp for incremental backup baseline');
|
||||
|
||||
// Content checksums
|
||||
table.string('checksum_algorithm', 50).defaultTo('sha256').comment('Algorithm used for checksums');
|
||||
table.text('manifest_checksum').comment('Checksum of the manifest file itself');
|
||||
|
||||
// Storage information
|
||||
table.string('storage_location', 500).comment('Primary storage location (local path or S3 URI)');
|
||||
table.string('storage_provider', 50).comment('Storage provider (local, s3, etc.)');
|
||||
|
||||
// Encryption metadata
|
||||
table.boolean('is_encrypted').defaultTo(false);
|
||||
table.string('encryption_algorithm', 100).comment('Encryption algorithm used');
|
||||
table.string('encryption_key_id', 255).comment('ID of encryption key used');
|
||||
|
||||
// Additional metadata as JSON
|
||||
table.json('metadata').comment('Additional metadata as JSON');
|
||||
|
||||
// Timestamps
|
||||
table.timestamps(true, true);
|
||||
|
||||
// Indexes
|
||||
table.index(['backup_run_id'], 'idx_manifest_backup_run');
|
||||
table.index(['manifest_id'], 'idx_manifest_uuid');
|
||||
table.index(['parent_manifest_id'], 'idx_manifest_parent');
|
||||
table.index(['backup_start', 'backup_end'], 'idx_manifest_time_range');
|
||||
table.index(['is_incremental', 'created_at'], 'idx_manifest_incremental_created');
|
||||
})
|
||||
|
||||
// Add composite indexes for common query patterns
|
||||
.raw(`
|
||||
CREATE INDEX IF NOT EXISTS idx_backup_runs_recent_successful
|
||||
ON backup_runs(created_at DESC)
|
||||
WHERE status = 'completed' AND backup_mode = 'full';
|
||||
`)
|
||||
.raw(`
|
||||
CREATE INDEX IF NOT EXISTS idx_backup_runs_incremental_chain
|
||||
ON backup_runs(parent_backup_id, created_at)
|
||||
WHERE backup_mode = 'incremental';
|
||||
`);
|
||||
};
|
||||
|
||||
exports.down = function(knex) {
|
||||
return knex.schema
|
||||
// Drop indexes first
|
||||
.raw('DROP INDEX IF EXISTS idx_backup_runs_incremental_chain;')
|
||||
.raw('DROP INDEX IF EXISTS idx_backup_runs_recent_successful;')
|
||||
|
||||
// Drop backup_manifest table
|
||||
.dropTableIfExists('backup_manifest')
|
||||
|
||||
// Remove columns from backup_runs table
|
||||
.table('backup_runs', table => {
|
||||
table.dropIndex(['backup_mode', 'status'], 'idx_backup_runs_mode_status');
|
||||
table.dropIndex(['parent_backup_id'], 'idx_backup_runs_parent');
|
||||
table.dropIndex(['created_at', 'backup_mode'], 'idx_backup_runs_created_mode');
|
||||
|
||||
table.dropColumn('manifest_path');
|
||||
table.dropColumn('manifest_id');
|
||||
table.dropColumn('manifest_format');
|
||||
table.dropColumn('parent_backup_id');
|
||||
table.dropColumn('backup_mode');
|
||||
})
|
||||
|
||||
// Remove columns from app_settings table
|
||||
.table('app_settings', table => {
|
||||
table.dropColumn('backup_s3_force_path_style');
|
||||
table.dropColumn('backup_s3_ssl_enabled');
|
||||
table.dropColumn('backup_s3_prefix');
|
||||
table.dropColumn('backup_incremental');
|
||||
table.dropColumn('backup_include_database');
|
||||
table.dropColumn('backup_manifest_enabled');
|
||||
table.dropColumn('backup_manifest_format');
|
||||
table.dropColumn('backup_encryption_enabled');
|
||||
table.dropColumn('backup_database_schedule');
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,249 @@
|
||||
const { formatBoolean, parseBoolean } = require('./helpers');
|
||||
|
||||
/**
|
||||
* Add restore_runs table for tracking restore operations
|
||||
*/
|
||||
exports.up = async function(knex) {
|
||||
// Create restore_runs table
|
||||
await knex.schema.createTable('restore_runs', table => {
|
||||
table.increments('id').primary();
|
||||
|
||||
// Timing
|
||||
table.timestamp('started_at').notNullable().defaultTo(knex.fn.now());
|
||||
table.timestamp('completed_at');
|
||||
table.integer('duration_seconds');
|
||||
|
||||
// Status and type
|
||||
table.string('status', 50).notNullable().defaultTo('running');
|
||||
table.string('restore_type', 50).notNullable(); // full, database, files, selective
|
||||
|
||||
// Source information
|
||||
table.string('source', 500).notNullable(); // Backup source path or S3 URL
|
||||
table.string('manifest_path', 500); // Path to manifest file
|
||||
|
||||
// Results
|
||||
table.text('error_message');
|
||||
table.text('statistics'); // JSON object with detailed statistics
|
||||
table.text('restore_log'); // JSON array of log entries
|
||||
|
||||
// Safety backup
|
||||
table.string('pre_restore_backup_path', 500); // Path to pre-restore safety backup
|
||||
|
||||
// Flags
|
||||
table.boolean('is_dry_run').defaultTo(formatBoolean(false));
|
||||
table.boolean('was_rollback_attempted').defaultTo(formatBoolean(false));
|
||||
table.boolean('was_successful').defaultTo(formatBoolean(false));
|
||||
|
||||
// Operator information
|
||||
table.string('operator_type', 50).defaultTo('manual'); // manual, scheduled, api
|
||||
table.integer('operator_user_id').references('id').inTable('admin_users').onDelete('SET NULL');
|
||||
table.string('operator_ip', 50);
|
||||
|
||||
// Metadata
|
||||
table.text('metadata'); // JSON object for additional data
|
||||
|
||||
table.index(['status', 'started_at']);
|
||||
table.index(['restore_type', 'started_at']);
|
||||
});
|
||||
|
||||
// Create restore_file_operations table for tracking individual file operations
|
||||
await knex.schema.createTable('restore_file_operations', table => {
|
||||
table.increments('id').primary();
|
||||
|
||||
table.integer('restore_run_id').notNullable()
|
||||
.references('id').inTable('restore_runs').onDelete('CASCADE');
|
||||
|
||||
table.string('file_path', 500).notNullable();
|
||||
table.string('operation', 50).notNullable(); // restore, skip, error
|
||||
table.string('status', 50).notNullable(); // pending, in_progress, completed, failed
|
||||
|
||||
table.bigInteger('file_size');
|
||||
table.string('checksum', 64);
|
||||
table.boolean('checksum_verified').defaultTo(formatBoolean(false));
|
||||
|
||||
table.text('error_message');
|
||||
table.timestamp('started_at');
|
||||
table.timestamp('completed_at');
|
||||
|
||||
table.index(['restore_run_id', 'status']);
|
||||
table.index(['file_path']);
|
||||
});
|
||||
|
||||
// Create restore_validation_results table
|
||||
await knex.schema.createTable('restore_validation_results', table => {
|
||||
table.increments('id').primary();
|
||||
|
||||
table.integer('restore_run_id').notNullable()
|
||||
.references('id').inTable('restore_runs').onDelete('CASCADE');
|
||||
|
||||
table.string('validation_type', 50).notNullable(); // pre-restore, post-restore
|
||||
table.boolean('is_valid').notNullable();
|
||||
|
||||
table.text('errors'); // JSON array of errors
|
||||
table.text('warnings'); // JSON array of warnings
|
||||
table.text('checksums'); // JSON object with checksum comparisons
|
||||
|
||||
table.timestamp('validated_at').notNullable().defaultTo(knex.fn.now());
|
||||
|
||||
table.index(['restore_run_id', 'validation_type']);
|
||||
});
|
||||
|
||||
// Add restore-related settings to app_settings
|
||||
await knex('app_settings').insert([
|
||||
{
|
||||
setting_key: 'restore_allow_force',
|
||||
setting_value: formatBoolean(false),
|
||||
setting_type: 'restore',
|
||||
description: 'Allow force restore with warnings',
|
||||
created_at: knex.fn.now(),
|
||||
updated_at: knex.fn.now()
|
||||
},
|
||||
{
|
||||
setting_key: 'restore_require_pre_backup',
|
||||
setting_value: formatBoolean(true),
|
||||
setting_type: 'restore',
|
||||
description: 'Require pre-restore backup',
|
||||
created_at: knex.fn.now(),
|
||||
updated_at: knex.fn.now()
|
||||
},
|
||||
{
|
||||
setting_key: 'restore_max_file_size_mb',
|
||||
setting_value: '5000',
|
||||
setting_type: 'restore',
|
||||
description: 'Maximum file size for restore (MB)',
|
||||
created_at: knex.fn.now(),
|
||||
updated_at: knex.fn.now()
|
||||
},
|
||||
{
|
||||
setting_key: 'restore_verify_checksums',
|
||||
setting_value: formatBoolean(true),
|
||||
setting_type: 'restore',
|
||||
description: 'Verify file checksums during restore',
|
||||
created_at: knex.fn.now(),
|
||||
updated_at: knex.fn.now()
|
||||
},
|
||||
{
|
||||
setting_key: 'restore_email_on_completion',
|
||||
setting_value: formatBoolean(true),
|
||||
setting_type: 'restore',
|
||||
description: 'Send email on restore completion',
|
||||
created_at: knex.fn.now(),
|
||||
updated_at: knex.fn.now()
|
||||
},
|
||||
{
|
||||
setting_key: 'restore_retention_days',
|
||||
setting_value: '30',
|
||||
setting_type: 'restore',
|
||||
description: 'Days to retain restore history',
|
||||
created_at: knex.fn.now(),
|
||||
updated_at: knex.fn.now()
|
||||
}
|
||||
]);
|
||||
|
||||
// Add new email templates for restore notifications
|
||||
const emailTemplates = [
|
||||
{
|
||||
name: 'restore_completed',
|
||||
subject: '✅ Restore Completed Successfully',
|
||||
body: `<h2>Restore Operation Completed</h2>
|
||||
<p>A restore operation has completed successfully.</p>
|
||||
|
||||
<h3>Details:</h3>
|
||||
<ul>
|
||||
<li><strong>Restore Type:</strong> {{restore_type}}</li>
|
||||
<li><strong>Duration:</strong> {{duration}}</li>
|
||||
<li><strong>Files Restored:</strong> {{files_restored}}</li>
|
||||
<li><strong>Backup ID:</strong> {{backup_id}}</li>
|
||||
<li><strong>Timestamp:</strong> {{timestamp}}</li>
|
||||
</ul>
|
||||
|
||||
<p>Please verify that all systems are functioning correctly after the restore.</p>`,
|
||||
language: 'en',
|
||||
is_active: formatBoolean(true),
|
||||
created_at: knex.fn.now(),
|
||||
updated_at: knex.fn.now()
|
||||
},
|
||||
{
|
||||
name: 'restore_failed',
|
||||
subject: '❌ Restore Operation Failed',
|
||||
body: `<h2>Restore Operation Failed</h2>
|
||||
<p>A restore operation has failed and requires attention.</p>
|
||||
|
||||
<h3>Details:</h3>
|
||||
<ul>
|
||||
<li><strong>Restore Type:</strong> {{restore_type}}</li>
|
||||
<li><strong>Error:</strong> {{error_message}}</li>
|
||||
<li><strong>Timestamp:</strong> {{timestamp}}</li>
|
||||
</ul>
|
||||
|
||||
<p>Please check the system logs for more details and take appropriate action.</p>
|
||||
|
||||
<p><strong>Important:</strong> If a pre-restore backup was created, it may be used for recovery.</p>`,
|
||||
language: 'en',
|
||||
is_active: formatBoolean(true),
|
||||
created_at: knex.fn.now(),
|
||||
updated_at: knex.fn.now()
|
||||
},
|
||||
{
|
||||
name: 'restore_completed',
|
||||
subject: '✅ Wiederherstellung erfolgreich abgeschlossen',
|
||||
body: `<h2>Wiederherstellungsvorgang abgeschlossen</h2>
|
||||
<p>Ein Wiederherstellungsvorgang wurde erfolgreich abgeschlossen.</p>
|
||||
|
||||
<h3>Details:</h3>
|
||||
<ul>
|
||||
<li><strong>Wiederherstellungstyp:</strong> {{restore_type}}</li>
|
||||
<li><strong>Dauer:</strong> {{duration}}</li>
|
||||
<li><strong>Wiederhergestellte Dateien:</strong> {{files_restored}}</li>
|
||||
<li><strong>Backup-ID:</strong> {{backup_id}}</li>
|
||||
<li><strong>Zeitstempel:</strong> {{timestamp}}</li>
|
||||
</ul>
|
||||
|
||||
<p>Bitte überprüfen Sie, ob alle Systeme nach der Wiederherstellung ordnungsgemäß funktionieren.</p>`,
|
||||
language: 'de',
|
||||
is_active: formatBoolean(true),
|
||||
created_at: knex.fn.now(),
|
||||
updated_at: knex.fn.now()
|
||||
},
|
||||
{
|
||||
name: 'restore_failed',
|
||||
subject: '❌ Wiederherstellungsvorgang fehlgeschlagen',
|
||||
body: `<h2>Wiederherstellungsvorgang fehlgeschlagen</h2>
|
||||
<p>Ein Wiederherstellungsvorgang ist fehlgeschlagen und erfordert Ihre Aufmerksamkeit.</p>
|
||||
|
||||
<h3>Details:</h3>
|
||||
<ul>
|
||||
<li><strong>Wiederherstellungstyp:</strong> {{restore_type}}</li>
|
||||
<li><strong>Fehler:</strong> {{error_message}}</li>
|
||||
<li><strong>Zeitstempel:</strong> {{timestamp}}</li>
|
||||
</ul>
|
||||
|
||||
<p>Bitte überprüfen Sie die Systemprotokolle für weitere Details und ergreifen Sie entsprechende Maßnahmen.</p>
|
||||
|
||||
<p><strong>Wichtig:</strong> Falls ein Backup vor der Wiederherstellung erstellt wurde, kann es zur Wiederherstellung verwendet werden.</p>`,
|
||||
language: 'de',
|
||||
is_active: formatBoolean(true),
|
||||
created_at: knex.fn.now(),
|
||||
updated_at: knex.fn.now()
|
||||
}
|
||||
];
|
||||
|
||||
await knex('email_templates').insert(emailTemplates);
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
// Remove email templates
|
||||
await knex('email_templates')
|
||||
.whereIn('name', ['restore_completed', 'restore_failed'])
|
||||
.delete();
|
||||
|
||||
// Remove settings
|
||||
await knex('app_settings')
|
||||
.where('setting_type', 'restore')
|
||||
.delete();
|
||||
|
||||
// Drop tables
|
||||
await knex.schema.dropTableIfExists('restore_validation_results');
|
||||
await knex.schema.dropTableIfExists('restore_file_operations');
|
||||
await knex.schema.dropTableIfExists('restore_runs');
|
||||
};
|
||||
Reference in New Issue
Block a user