feat: implement comprehensive backup and restore system with S3 support

- Add S3/MinIO storage adapter with multipart upload support
- Implement database backup service for SQLite and PostgreSQL
- Create backup manifest generator for tracking backup contents
- Enhance backup service with S3 integration and incremental backups
- Add restore service with safety measures and rollback capability
- Create comprehensive test suite for all backup functionality
- Add admin API endpoints for backup/restore management
- Implement frontend UI with dashboard, configuration, and restore wizard
- Add roadmap section to README with implemented backup feature

This implementation provides:
- Multiple backup destinations (local, rsync, S3/MinIO)
- Intelligent change detection to minimize backup frequency
- Full database backups with compression
- Manifest-based restore with integrity validation
- Pre-restore safety backups with rollback
- Comprehensive error handling and monitoring
- User-friendly admin interface

🤖 Generated with Claude Code

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
2025-07-22 09:05:52 +02:00
parent 3c6837bd90
commit f6a79c815e
44 changed files with 15839 additions and 14 deletions
+577
View File
@@ -0,0 +1,577 @@
#!/usr/bin/env node
/**
* Manual Integration Test Script for Enhanced Backup System
*
* This script provides a comprehensive test of the backup system with real services.
* It can be used to test against MinIO, AWS S3, or other S3-compatible services.
*
* Usage:
* node scripts/test-backup-integration.js [options]
*
* Options:
* --endpoint <url> S3 endpoint URL (default: http://localhost:9000)
* --access-key <key> S3 access key (default: minioadmin)
* --secret-key <key> S3 secret key (default: minioadmin)
* --bucket <name> S3 bucket name (default: test-backup-<timestamp>)
* --type <type> Backup type: s3, local, rsync (default: s3)
* --cleanup Clean up test data after completion
* --verbose Enable verbose logging
* --help Show this help message
*
* Examples:
* # Test with local MinIO
* node scripts/test-backup-integration.js
*
* # Test with AWS S3
* node scripts/test-backup-integration.js \
* --endpoint https://s3.amazonaws.com \
* --access-key AKIAIOSFODNN7EXAMPLE \
* --secret-key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \
* --bucket my-test-bucket
*
* # Test local backup
* node scripts/test-backup-integration.js --type local
*/
const path = require('path');
const fs = require('fs').promises;
const crypto = require('crypto');
const { S3Client, CreateBucketCommand, HeadBucketCommand, ListObjectsV2Command, GetObjectCommand, DeleteObjectsCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3');
// Parse command line arguments
const args = process.argv.slice(2);
const options = {
endpoint: 'http://localhost:9000',
accessKey: 'minioadmin',
secretKey: 'minioadmin',
bucket: `test-backup-${Date.now()}`,
type: 's3',
cleanup: false,
verbose: false
};
for (let i = 0; i < args.length; i++) {
switch (args[i]) {
case '--endpoint':
options.endpoint = args[++i];
break;
case '--access-key':
options.accessKey = args[++i];
break;
case '--secret-key':
options.secretKey = args[++i];
break;
case '--bucket':
options.bucket = args[++i];
break;
case '--type':
options.type = args[++i];
break;
case '--cleanup':
options.cleanup = true;
break;
case '--verbose':
options.verbose = true;
break;
case '--help':
console.log(module.exports.description || 'Manual Integration Test Script');
process.exit(0);
}
}
// Load environment and services
require('dotenv').config();
const { db, initialize: initDb } = require('../src/database/db');
const backupService = require('../src/services/backupService');
const S3StorageAdapter = require('../src/services/storage/s3Storage');
const logger = require('../src/utils/logger');
// Configure logger based on verbose flag
if (!options.verbose) {
logger.info = () => {};
logger.debug = () => {};
}
// Test results
const results = {
passed: 0,
failed: 0,
skipped: 0,
tests: []
};
// Test utilities
async function runTest(name, testFn) {
console.log(`\n📋 Running: ${name}`);
try {
const startTime = Date.now();
await testFn();
const duration = Date.now() - startTime;
console.log(`✅ PASSED: ${name} (${duration}ms)`);
results.passed++;
results.tests.push({ name, status: 'passed', duration });
} catch (error) {
console.error(`❌ FAILED: ${name}`);
console.error(` Error: ${error.message}`);
if (options.verbose) {
console.error(error.stack);
}
results.failed++;
results.tests.push({ name, status: 'failed', error: error.message });
}
}
async function skipTest(name, reason) {
console.log(`\n⏭️ Skipping: ${name}`);
console.log(` Reason: ${reason}`);
results.skipped++;
results.tests.push({ name, status: 'skipped', reason });
}
// Test functions
async function testS3Connection() {
const s3Adapter = new S3StorageAdapter({
bucket: options.bucket,
endpoint: options.endpoint,
accessKeyId: options.accessKey,
secretAccessKey: options.secretKey,
region: 'us-east-1',
forcePathStyle: true,
sslEnabled: options.endpoint.startsWith('https')
});
await s3Adapter.testConnection();
console.log(` ✓ Connected to S3 endpoint: ${options.endpoint}`);
console.log(` ✓ Bucket accessible: ${options.bucket}`);
}
async function setupTestData() {
const storagePath = path.join(__dirname, '../test-storage');
process.env.STORAGE_PATH = storagePath;
// Create directory structure
const dirs = [
'events/active/wedding-2024',
'events/active/birthday-2024',
'events/archived',
'thumbnails',
'uploads',
'backups'
];
for (const dir of dirs) {
await fs.mkdir(path.join(storagePath, dir), { recursive: true });
}
// Create test files with various sizes
const files = [
{ path: 'events/active/wedding-2024/photo1.jpg', size: 1024 * 1024 }, // 1MB
{ path: 'events/active/wedding-2024/photo2.jpg', size: 512 * 1024 }, // 512KB
{ path: 'events/active/birthday-2024/photo1.jpg', size: 2 * 1024 * 1024 }, // 2MB
{ path: 'events/archived/old-event.zip', size: 5 * 1024 * 1024 }, // 5MB
{ path: 'thumbnails/thumb1.jpg', size: 50 * 1024 }, // 50KB
{ path: 'uploads/logo.png', size: 100 * 1024 } // 100KB
];
let totalSize = 0;
for (const file of files) {
const content = crypto.randomBytes(file.size);
await fs.writeFile(path.join(storagePath, file.path), content);
totalSize += file.size;
}
console.log(` ✓ Created ${files.length} test files`);
console.log(` ✓ Total size: ${(totalSize / 1024 / 1024).toFixed(2)} MB`);
return { storagePath, fileCount: files.length, totalSize };
}
async function configureBackup(type) {
const baseSettings = [
{ setting_key: 'backup_enabled', setting_value: 'true' },
{ setting_key: 'backup_destination_type', setting_value: `"${type}"` },
{ setting_key: 'backup_include_archived', setting_value: 'true' },
{ setting_key: 'backup_include_database', setting_value: 'true' },
{ setting_key: 'backup_incremental', setting_value: 'true' },
{ setting_key: 'backup_manifest_format', setting_value: '"json"' },
{ setting_key: 'backup_max_file_size_mb', setting_value: '100' }
];
const typeSpecificSettings = {
s3: [
{ setting_key: 'backup_s3_bucket', setting_value: `"${options.bucket}"` },
{ setting_key: 'backup_s3_endpoint', setting_value: `"${options.endpoint}"` },
{ setting_key: 'backup_s3_access_key', setting_value: `"${options.accessKey}"` },
{ setting_key: 'backup_s3_secret_key', setting_value: `"${options.secretKey}"` },
{ setting_key: 'backup_s3_region', setting_value: '"us-east-1"' },
{ setting_key: 'backup_s3_force_path_style', setting_value: 'true' },
{ setting_key: 'backup_s3_ssl_enabled', setting_value: options.endpoint.startsWith('https') ? 'true' : 'false' }
],
local: [
{ setting_key: 'backup_destination_path', setting_value: `"${path.join(__dirname, '../test-backup')}"` }
],
rsync: [
{ setting_key: 'backup_rsync_host', setting_value: '"localhost"' },
{ setting_key: 'backup_rsync_path', setting_value: `"${path.join(__dirname, '../test-backup-rsync')}"` }
]
};
const settings = [...baseSettings, ...(typeSpecificSettings[type] || [])];
// Clear existing settings
await db('app_settings').where('setting_type', 'backup').del();
// Insert new settings
for (const setting of settings) {
await db('app_settings').insert({
setting_type: 'backup',
...setting,
created_at: new Date(),
updated_at: new Date()
});
}
console.log(` ✓ Configured ${type} backup with ${settings.length} settings`);
}
async function performBackup() {
const startTime = Date.now();
// Run the backup
await backupService.runBackup();
// Get backup results
const backupRun = await db('backup_runs')
.orderBy('started_at', 'desc')
.first();
if (!backupRun) {
throw new Error('No backup run found');
}
if (backupRun.status !== 'completed') {
throw new Error(`Backup failed with status: ${backupRun.status}, error: ${backupRun.error_message}`);
}
const duration = Date.now() - startTime;
console.log(` ✓ Backup completed in ${duration}ms`);
console.log(` ✓ Files backed up: ${backupRun.files_backed_up}`);
console.log(` ✓ Total size: ${(backupRun.total_size_bytes / 1024 / 1024).toFixed(2)} MB`);
console.log(` ✓ Manifest: ${backupRun.manifest_path ? 'Generated' : 'Not generated'}`);
return backupRun;
}
async function verifyS3Backup(backupRun) {
const s3Client = new S3Client({
endpoint: options.endpoint,
region: 'us-east-1',
credentials: {
accessKeyId: options.accessKey,
secretAccessKey: options.secretKey
},
forcePathStyle: true
});
// List objects in bucket
const listResponse = await s3Client.send(new ListObjectsV2Command({
Bucket: options.bucket
}));
const objects = listResponse.Contents || [];
console.log(` ✓ Objects in S3: ${objects.length}`);
// Verify key components
const hasBackupFolder = objects.some(obj => obj.Key.includes('backup-'));
const hasManifest = objects.some(obj => obj.Key.includes('backup-manifest'));
const hasSummary = objects.some(obj => obj.Key.includes('backup-summary.json'));
const hasPhotos = objects.some(obj => obj.Key.includes('events/active'));
if (!hasBackupFolder) throw new Error('No backup folder found in S3');
if (!hasManifest) throw new Error('No manifest found in S3');
if (!hasSummary) throw new Error('No summary found in S3');
if (!hasPhotos) throw new Error('No photos found in S3');
console.log(` ✓ Backup structure verified`);
// Download and verify a file
const photoObject = objects.find(obj => obj.Key.includes('photo1.jpg'));
if (photoObject) {
const getResponse = await s3Client.send(new GetObjectCommand({
Bucket: options.bucket,
Key: photoObject.Key
}));
const chunks = [];
for await (const chunk of getResponse.Body) {
chunks.push(chunk);
}
const content = Buffer.concat(chunks);
console.log(` ✓ Downloaded test file: ${photoObject.Key} (${content.length} bytes)`);
}
}
async function testIncrementalBackup(testData) {
// Modify a file
const modifiedFile = path.join(testData.storagePath, 'events/active/wedding-2024/photo1.jpg');
const newContent = crypto.randomBytes(1024 * 1024 + 100); // Slightly larger
await fs.writeFile(modifiedFile, newContent);
console.log(` ✓ Modified test file`);
// Perform incremental backup
const backupRun = await performBackup();
if (backupRun.files_backed_up !== 1) {
throw new Error(`Expected 1 file in incremental backup, got ${backupRun.files_backed_up}`);
}
console.log(` ✓ Incremental backup correctly identified changed file`);
// Verify manifest indicates incremental
if (backupRun.manifest_path) {
const { manifest } = await backupService.getBackupManifest(backupRun.id);
if (!manifest.incremental) {
throw new Error('Manifest does not indicate incremental backup');
}
console.log(` ✓ Manifest correctly marked as incremental`);
}
return backupRun;
}
async function testManifestValidation(backupRun) {
if (!backupRun.manifest_path) {
throw new Error('No manifest path in backup run');
}
const result = await backupService.validateBackupManifest(backupRun.manifest_path);
if (!result.valid) {
throw new Error(`Manifest validation failed: ${result.error}`);
}
console.log(` ✓ Manifest validation passed`);
console.log(` ✓ Manifest version: ${result.manifest.manifest.version}`);
console.log(` ✓ Files in manifest: ${result.manifest.files.count}`);
}
async function testBackupStatus() {
const status = await backupService.getBackupStatus(5);
console.log(` ✓ Backup service running: ${status.isRunning}`);
console.log(` ✓ Backup service healthy: ${status.isHealthy}`);
console.log(` ✓ Recent runs: ${status.recentRuns.length}`);
if (status.lastRun) {
console.log(` ✓ Last run status: ${status.lastRun.status}`);
console.log(` ✓ Manifest valid: ${status.lastRun.manifestValid}`);
}
}
async function cleanupTestData() {
if (!options.cleanup) {
console.log('\n📌 Test data retained for inspection');
console.log(` Storage: ${process.env.STORAGE_PATH}`);
if (options.type === 's3') {
console.log(` S3 Bucket: ${options.bucket}`);
}
return;
}
console.log('\n🧹 Cleaning up test data...');
// Clean storage directory
if (process.env.STORAGE_PATH) {
await fs.rm(process.env.STORAGE_PATH, { recursive: true, force: true });
console.log(' ✓ Removed test storage directory');
}
// Clean S3 bucket if used
if (options.type === 's3') {
const s3Client = new S3Client({
endpoint: options.endpoint,
region: 'us-east-1',
credentials: {
accessKeyId: options.accessKey,
secretAccessKey: options.secretKey
},
forcePathStyle: true
});
try {
// List and delete all objects
const listResponse = await s3Client.send(new ListObjectsV2Command({
Bucket: options.bucket
}));
if (listResponse.Contents && listResponse.Contents.length > 0) {
await s3Client.send(new DeleteObjectsCommand({
Bucket: options.bucket,
Delete: {
Objects: listResponse.Contents.map(obj => ({ Key: obj.Key }))
}
}));
console.log(` ✓ Deleted ${listResponse.Contents.length} objects from S3`);
}
// Delete bucket
await s3Client.send(new DeleteBucketCommand({
Bucket: options.bucket
}));
console.log(` ✓ Deleted S3 bucket: ${options.bucket}`);
} catch (error) {
console.error(` ⚠️ Failed to cleanup S3: ${error.message}`);
}
}
// Clean backup directories
const backupDirs = [
path.join(__dirname, '../test-backup'),
path.join(__dirname, '../test-backup-rsync')
];
for (const dir of backupDirs) {
await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
}
console.log(' ✓ Removed backup directories');
}
// Main test runner
async function main() {
console.log('🚀 Enhanced Backup System Integration Test');
console.log('==========================================');
console.log(`Type: ${options.type}`);
console.log(`Endpoint: ${options.endpoint}`);
console.log(`Bucket: ${options.bucket}`);
console.log('');
let s3Client;
let testData;
try {
// Initialize database
console.log('📦 Initializing database...');
await initDb();
await db.migrate.latest();
console.log(' ✓ Database initialized');
// S3-specific setup
if (options.type === 's3') {
// Test S3 connection
await runTest('S3 Connection Test', testS3Connection);
// Create S3 bucket if needed
s3Client = new S3Client({
endpoint: options.endpoint,
region: 'us-east-1',
credentials: {
accessKeyId: options.accessKey,
secretAccessKey: options.secretKey
},
forcePathStyle: true
});
try {
await s3Client.send(new HeadBucketCommand({ Bucket: options.bucket }));
console.log(`\n📦 Using existing bucket: ${options.bucket}`);
} catch (error) {
if (error.name === 'NotFound') {
await s3Client.send(new CreateBucketCommand({ Bucket: options.bucket }));
console.log(`\n📦 Created new bucket: ${options.bucket}`);
} else {
throw error;
}
}
}
// Setup test data
console.log('\n📁 Setting up test data...');
testData = await setupTestData();
// Configure backup
console.log(`\n⚙️ Configuring ${options.type} backup...`);
await configureBackup(options.type);
// Run tests based on backup type
await runTest('Initial Full Backup', performBackup);
if (options.type === 's3') {
await runTest('Verify S3 Backup Contents', async () => {
const lastRun = await db('backup_runs').orderBy('started_at', 'desc').first();
await verifyS3Backup(lastRun);
});
}
await runTest('Incremental Backup', () => testIncrementalBackup(testData));
await runTest('Manifest Validation', async () => {
const lastRun = await db('backup_runs').orderBy('started_at', 'desc').first();
await testManifestValidation(lastRun);
});
await runTest('Backup Status Check', testBackupStatus);
// Performance test with larger files
if (options.type === 's3') {
await runTest('Large File Backup (10MB)', async () => {
const largeFile = path.join(testData.storagePath, 'events/active/large.jpg');
await fs.writeFile(largeFile, crypto.randomBytes(10 * 1024 * 1024));
await performBackup();
});
}
// Test backup service lifecycle
await runTest('Backup Service Start/Stop', async () => {
await backupService.startBackupService();
console.log(' ✓ Service started');
backupService.stopBackupService();
console.log(' ✓ Service stopped');
});
// Print results summary
console.log('\n📊 Test Results Summary');
console.log('======================');
console.log(`✅ Passed: ${results.passed}`);
console.log(`❌ Failed: ${results.failed}`);
console.log(`⏭️ Skipped: ${results.skipped}`);
console.log(`📋 Total: ${results.tests.length}`);
if (results.failed > 0) {
console.log('\nFailed Tests:');
results.tests
.filter(t => t.status === 'failed')
.forEach(t => console.log(` - ${t.name}: ${t.error}`));
}
} catch (error) {
console.error('\n💥 Fatal error:', error.message);
if (options.verbose) {
console.error(error.stack);
}
results.failed++;
} finally {
// Cleanup
await cleanupTestData();
// Close database
await db.destroy();
// Exit with appropriate code
process.exit(results.failed > 0 ? 1 : 0);
}
}
// Run if called directly
if (require.main === module) {
main().catch(error => {
console.error('Unhandled error:', error);
process.exit(1);
});
}
module.exports = { runTest, skipTest };
+192
View File
@@ -0,0 +1,192 @@
#!/usr/bin/env node
/**
* Test script for backup manifest generator
* Demonstrates all features of the manifest generator
*/
const path = require('path');
const fs = require('fs').promises;
const backupManifest = require('../src/services/backupManifest');
const logger = require('../src/utils/logger');
async function testManifestGeneration() {
console.log('=== Testing Backup Manifest Generator ===\n');
try {
// 1. Generate a full backup manifest
console.log('1. Generating full backup manifest...');
const fullManifestOptions = {
backupType: 'full',
backupPath: '/backup/full/2025-01-21',
files: [
{
path: '/storage/events/active/wedding-smith-2025/DSC_001.jpg',
relativePath: 'events/active/wedding-smith-2025/DSC_001.jpg',
size: 2456789,
modified: new Date('2025-01-20T10:30:00Z'),
checksum: 'a1b2c3d4e5f6789012345678901234567890123456789012345678901234567890',
permissions: '644'
},
{
path: '/storage/events/active/wedding-smith-2025/DSC_002.jpg',
relativePath: 'events/active/wedding-smith-2025/DSC_002.jpg',
size: 2156789,
modified: new Date('2025-01-20T10:31:00Z'),
checksum: 'b2c3d4e5f67890123456789012345678901234567890123456789012345678901',
permissions: '644'
},
{
path: '/storage/thumbnails/wedding-smith-2025/thumb_DSC_001.jpg',
relativePath: 'thumbnails/wedding-smith-2025/thumb_DSC_001.jpg',
size: 45678,
modified: new Date('2025-01-20T10:35:00Z'),
checksum: 'c3d4e5f678901234567890123456789012345678901234567890123456789012',
permissions: '644'
}
],
databaseInfo: {
type: 'sqlite',
backupFile: 'database-backup-20250121-103000.sql.gz',
size: 1048576,
checksum: 'd4e5f6789012345678901234567890123456789012345678901234567890123',
tables: {
events: 156,
photos: 4523,
access_logs: 12456,
admin_users: 3
},
rowCounts: {
events: 156,
photos: 4523,
access_logs: 12456,
admin_users: 3
}
},
format: 'json',
customMetadata: {
operator: 'admin@example.com',
reason: 'Scheduled daily backup',
retentionDays: 30,
compressionType: 'gzip'
}
};
const fullManifest = await backupManifest.generateManifest(fullManifestOptions);
// Save in both formats
const jsonPath = path.join(__dirname, 'test-manifest-full.json');
const yamlPath = path.join(__dirname, 'test-manifest-full.yaml');
await backupManifest.saveManifest(fullManifest, jsonPath, 'json');
await backupManifest.saveManifest(fullManifest, yamlPath, 'yaml');
console.log('✓ Full backup manifest generated and saved\n');
// 2. Generate summary report
console.log('2. Generating summary report...');
const summaryReport = backupManifest.generateSummaryReport(fullManifest);
console.log(summaryReport);
console.log('\n');
// 3. Load and validate manifest
console.log('3. Loading and validating manifest...');
const loadedManifest = await backupManifest.loadManifest(jsonPath);
console.log('✓ Manifest loaded and validated successfully\n');
// 4. Generate incremental backup manifest
console.log('4. Generating incremental backup manifest...');
const incrementalOptions = {
backupType: 'incremental',
backupPath: '/backup/incremental/2025-01-22',
parentBackupId: fullManifest.backup.id,
files: [
// Original files with same checksums (unchanged)
fullManifestOptions.files[0],
fullManifestOptions.files[2],
// Modified file
{
...fullManifestOptions.files[1],
size: 2256789,
modified: new Date('2025-01-21T14:00:00Z'),
checksum: 'e5f678901234567890123456789012345678901234567890123456789012345'
},
// New file
{
path: '/storage/events/active/wedding-smith-2025/DSC_003.jpg',
relativePath: 'events/active/wedding-smith-2025/DSC_003.jpg',
size: 2356789,
modified: new Date('2025-01-21T14:30:00Z'),
checksum: 'f6789012345678901234567890123456789012345678901234567890123456',
permissions: '644'
}
],
databaseInfo: {
...fullManifestOptions.databaseInfo,
size: 1148576,
checksum: 'g7890123456789012345678901234567890123456789012345678901234567',
rowCounts: {
events: 158,
photos: 4567,
access_logs: 12789,
admin_users: 3
}
}
};
const incrementalManifest = await backupManifest.generateIncrementalManifest(
incrementalOptions,
fullManifest
);
const incrementalJsonPath = path.join(__dirname, 'test-manifest-incremental.json');
await backupManifest.saveManifest(incrementalManifest, incrementalJsonPath, 'json');
console.log('✓ Incremental backup manifest generated');
console.log(` - Added files: ${incrementalManifest.incremental.changes.added_files_count}`);
console.log(` - Modified files: ${incrementalManifest.incremental.changes.modified_files_count}`);
console.log(` - Deleted files: ${incrementalManifest.incremental.changes.deleted_files_count}`);
console.log(` - Size difference: ${(incrementalManifest.incremental.changes.size_difference / 1024).toFixed(2)} KB\n`);
// 5. Compare manifests
console.log('5. Comparing manifests...');
const comparison = backupManifest.compareManifests(incrementalManifest, fullManifest);
console.log('Comparison results:');
console.log(` - Added: ${comparison.added_files.length} files`);
console.log(` - Modified: ${comparison.modified_files.length} files`);
console.log(` - Deleted: ${comparison.deleted_files.length} files`);
console.log(` - Unchanged: ${comparison.unchanged_files.length} files`);
console.log(` - Database changed: ${comparison.database_changes.checksum_changed ? 'Yes' : 'No'}\n`);
// 6. Test manifest integrity
console.log('6. Testing manifest integrity...');
// Corrupt the manifest
const corruptedManifest = JSON.parse(JSON.stringify(incrementalManifest));
corruptedManifest.files.manifest[0].size = 9999999; // Change a file size
try {
backupManifest.validateManifest(corruptedManifest);
console.log('✗ Validation should have failed for corrupted manifest');
} catch (error) {
console.log('✓ Correctly detected corrupted manifest:', error.message);
}
console.log('\n=== All tests completed successfully! ===');
// Clean up test files
await fs.unlink(jsonPath).catch(() => {});
await fs.unlink(yamlPath).catch(() => {});
await fs.unlink(incrementalJsonPath).catch(() => {});
} catch (error) {
console.error('Test failed:', error);
logger.error('Manifest test failed:', error);
process.exit(1);
}
}
// Run tests
testManifestGeneration().catch(console.error);
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env node
require('dotenv').config();
const { initializeDatabase } = require('../src/database/db');
const { runBackup, getBackupStatus } = require('../src/services/backupService');
const logger = require('../src/utils/logger');
async function testBackupService() {
try {
console.log('Testing backup service...\n');
// Initialize database
await initializeDatabase();
// Get current backup status
console.log('Getting backup status...');
const statusBefore = await getBackupStatus();
console.log('Last run:', statusBefore.lastRun ? statusBefore.lastRun.started_at : 'Never');
console.log('Is healthy:', statusBefore.isHealthy);
console.log('');
// Run backup
console.log('Running backup...');
await runBackup();
// Get status after backup
console.log('\nGetting status after backup...');
const statusAfter = await getBackupStatus();
console.log('Last run:', statusAfter.lastRun ? statusAfter.lastRun.started_at : 'Never');
console.log('Status:', statusAfter.lastRun ? statusAfter.lastRun.status : 'Unknown');
console.log('Files backed up:', statusAfter.lastRun ? statusAfter.lastRun.files_backed_up : 0);
console.log('Total size:', statusAfter.lastRun ? `${(statusAfter.lastRun.total_size_bytes / 1024 / 1024).toFixed(2)} MB` : '0 MB');
if (statusAfter.lastRun && statusAfter.lastRun.error_message) {
console.log('Error:', statusAfter.lastRun.error_message);
}
console.log('\nBackup test completed!');
process.exit(0);
} catch (error) {
console.error('Test failed:', error);
process.exit(1);
}
}
testBackupService();
+325
View File
@@ -0,0 +1,325 @@
/**
* Test script for the restore service
*
* This script demonstrates the restore service functionality with safety checks
*
* Usage:
* node scripts/test-restore-service.js [options]
*
* Options:
* --dry-run Perform validation only without actual restore
* --force Force restore even with warnings
* --type Restore type: full, database, files, selective (default: full)
* --source Backup source path or S3 URL
* --manifest Path to backup manifest
*/
require('dotenv').config();
const { restoreService } = require('../src/services/restoreService');
const { db } = require('../src/database/db');
const logger = require('../src/utils/logger');
const path = require('path');
const fs = require('fs').promises;
// Parse command line arguments
const args = process.argv.slice(2);
const options = {
dryRun: args.includes('--dry-run'),
force: args.includes('--force'),
restoreType: 'full',
source: null,
manifestPath: null
};
// Parse restore type
const typeIndex = args.indexOf('--type');
if (typeIndex !== -1 && args[typeIndex + 1]) {
options.restoreType = args[typeIndex + 1];
}
// Parse source
const sourceIndex = args.indexOf('--source');
if (sourceIndex !== -1 && args[sourceIndex + 1]) {
options.source = args[sourceIndex + 1];
}
// Parse manifest
const manifestIndex = args.indexOf('--manifest');
if (manifestIndex !== -1 && args[manifestIndex + 1]) {
options.manifestPath = args[manifestIndex + 1];
}
async function testRestore() {
console.log('=== PicPeak Restore Service Test ===\n');
try {
// If no source/manifest provided, try to find a recent backup
if (!options.source || !options.manifestPath) {
console.log('No backup source specified. Looking for recent backups...\n');
const recentBackup = await db('backup_runs')
.where('status', 'completed')
.whereNotNull('manifest_path')
.orderBy('completed_at', 'desc')
.first();
if (!recentBackup) {
console.error('❌ No completed backups found in the database');
console.log('\nPlease run a backup first or specify --source and --manifest');
process.exit(1);
}
console.log(`Found recent backup from ${recentBackup.completed_at}`);
console.log(`Backup ID: ${recentBackup.manifest_id}`);
console.log(`Files backed up: ${recentBackup.files_backed_up}`);
console.log(`Total size: ${(recentBackup.total_size_bytes / 1024 / 1024).toFixed(2)} MB`);
console.log(`Manifest: ${recentBackup.manifest_path}\n`);
// For this test, we'll create a mock scenario
console.log('⚠️ This is a TEST MODE - using mock data for safety\n');
// Create test backup directory
const testBackupDir = path.join(__dirname, '../temp/test-backup');
await fs.mkdir(testBackupDir, { recursive: true });
// Create test manifest
const testManifest = {
manifest: {
version: '2.0',
created: new Date().toISOString(),
generator: 'Test Script',
format: 'json'
},
backup: {
id: 'test-backup-' + Date.now(),
type: 'full',
timestamp: new Date().toISOString(),
path: testBackupDir,
parent_backup_id: null,
retention_days: 30
},
system: {
hostname: require('os').hostname(),
platform: process.platform,
os_release: require('os').release(),
architecture: require('os').arch()
},
application: {
name: 'PicPeak',
version: require('../package.json').version,
node_version: process.version,
environment: 'test'
},
files: {
count: 0,
total_size: 0,
checksums: {},
manifest: []
},
database: {
type: process.env.DB_TYPE === 'postgresql' ? 'postgresql' : 'sqlite',
backup_file: null,
size: 0,
checksum: null,
tables: {},
row_counts: {}
},
verification: {
total_checksum: null,
file_count_check: 0,
size_check: 0,
integrity_timestamp: new Date().toISOString()
},
metadata: {
test_mode: true
}
};
// Calculate checksum
const crypto = require('crypto');
const manifestCopy = JSON.parse(JSON.stringify(testManifest));
delete manifestCopy.verification.total_checksum;
testManifest.verification.total_checksum = crypto
.createHash('sha256')
.update(JSON.stringify(manifestCopy, Object.keys(manifestCopy).sort()))
.digest('hex');
// Save test manifest
const testManifestPath = path.join(testBackupDir, 'test-manifest.json');
await fs.writeFile(testManifestPath, JSON.stringify(testManifest, null, 2));
options.source = testBackupDir;
options.manifestPath = testManifestPath;
}
// Display restore options
console.log('Restore Options:');
console.log(`- Type: ${options.restoreType}`);
console.log(`- Source: ${options.source}`);
console.log(`- Manifest: ${options.manifestPath}`);
console.log(`- Dry Run: ${options.dryRun ? 'Yes' : 'No'}`);
console.log(`- Force: ${options.force ? 'Yes' : 'No'}`);
console.log('');
// Add S3 config if source is S3
if (options.source.startsWith('s3://')) {
options.s3Config = {
accessKeyId: process.env.BACKUP_S3_ACCESS_KEY,
secretAccessKey: process.env.BACKUP_S3_SECRET_KEY,
region: process.env.BACKUP_S3_REGION || 'us-east-1',
endpoint: process.env.BACKUP_S3_ENDPOINT
};
if (!options.s3Config.accessKeyId || !options.s3Config.secretAccessKey) {
console.error('❌ S3 credentials not configured in environment');
process.exit(1);
}
}
// Confirm before proceeding (unless dry run)
if (!options.dryRun) {
console.log('⚠️ WARNING: This will restore data from the backup!');
console.log('⚠️ Current data may be overwritten!');
console.log('');
console.log('Press Ctrl+C to cancel, or wait 5 seconds to continue...');
await new Promise(resolve => setTimeout(resolve, 5000));
}
console.log('\nStarting restore operation...\n');
// Perform restore
const result = await restoreService.restore(options);
if (options.dryRun) {
console.log('\n=== DRY RUN RESULTS ===\n');
console.log('Validation:');
console.log(`- Valid: ${result.validation.isValid ? '✅ Yes' : '❌ No'}`);
if (result.validation.errors.length > 0) {
console.log('- Errors:');
result.validation.errors.forEach(err => console.log(`${err}`));
}
if (result.validation.warnings.length > 0) {
console.log('- Warnings:');
result.validation.warnings.forEach(warn => console.log(` ⚠️ ${warn}`));
}
console.log('\nDisk Space:');
console.log(`- Required: ${result.spaceCheck.requiredFormatted}`);
console.log(`- Available: ${result.spaceCheck.availableFormatted}`);
console.log(`- Sufficient: ${result.spaceCheck.hasEnoughSpace ? '✅ Yes' : '❌ No'}`);
} else {
console.log('\n=== RESTORE RESULTS ===\n');
console.log(`Status: ${result.success ? '✅ SUCCESS' : '❌ FAILED'}`);
console.log(`Duration: ${result.duration}s`);
if (result.result) {
console.log('\nItems Restored:');
if (result.result.databaseRestored !== undefined) {
console.log(`- Database: ${result.result.databaseRestored ? '✅' : '❌'}`);
}
if (result.result.filesRestored !== undefined) {
console.log(`- Files: ${result.result.filesRestored}`);
}
if (result.result.errors && result.result.errors.length > 0) {
console.log('- Errors:');
result.result.errors.forEach(err => console.log(`${err}`));
}
}
if (result.verification) {
console.log('\nVerification:');
console.log(`- Valid: ${result.verification.isValid ? '✅ Yes' : '❌ No'}`);
if (result.verification.errors.length > 0) {
console.log('- Errors:');
result.verification.errors.forEach(err => console.log(`${err}`));
}
}
if (result.preRestoreBackup) {
console.log('\nSafety Backup:');
console.log(`- Location: ${result.preRestoreBackup}`);
console.log('- This backup can be used to rollback if needed');
}
}
// Show recent log entries
console.log('\nRecent Log Entries:');
result.logs.slice(-10).forEach(log => {
const icon = log.level === 'error' ? '❌' : log.level === 'warn' ? '⚠️ ' : '️ ';
console.log(`${icon} [${log.timestamp}] ${log.message}`);
});
// Clean up test files
if (options.source && options.source.includes('test-backup')) {
await fs.rmdir(path.dirname(options.source), { recursive: true }).catch(() => {});
}
} catch (error) {
console.error('\n❌ Restore operation failed:', error.message);
// Show logs if available
if (restoreService.restoreLog && restoreService.restoreLog.length > 0) {
console.log('\nError Log:');
restoreService.restoreLog.slice(-10).forEach(log => {
if (log.level === 'error' || log.level === 'warn') {
console.log(`[${log.timestamp}] ${log.level.toUpperCase()}: ${log.message}`);
}
});
}
process.exit(1);
}
// Cleanup
await db.destroy();
process.exit(0);
}
// Show help if requested
if (args.includes('--help') || args.includes('-h')) {
console.log(`
PicPeak Restore Service Test
This script tests the restore service functionality with safety checks.
Usage:
node scripts/test-restore-service.js [options]
Options:
--dry-run Perform validation only without actual restore
--force Force restore even with warnings
--type Restore type: full, database, files, selective (default: full)
--source Backup source path or S3 URL
--manifest Path to backup manifest
--help Show this help message
Examples:
# Dry run with automatic backup selection
node scripts/test-restore-service.js --dry-run
# Full restore from specific backup
node scripts/test-restore-service.js --source /backup/2024-01-20 --manifest /backup/2024-01-20/manifest.json
# Database-only restore with force
node scripts/test-restore-service.js --type database --force --source /backup/2024-01-20 --manifest /backup/2024-01-20/manifest.json
# Restore from S3
node scripts/test-restore-service.js --source s3://my-bucket/backups/2024-01-20 --manifest s3://my-bucket/backups/2024-01-20/manifest.json
Safety Features:
- Pre-restore validation checks compatibility and warns about potential issues
- Automatic pre-restore backup is created (unless skipped)
- Post-restore verification ensures data integrity
- Rollback capability if restore fails
- Detailed logging of all operations
`);
process.exit(0);
}
// Run the test
testRestore();