feat: implement comprehensive backup and restore system with S3 support
- Add S3/MinIO storage adapter with multipart upload support - Implement database backup service for SQLite and PostgreSQL - Create backup manifest generator for tracking backup contents - Enhance backup service with S3 integration and incremental backups - Add restore service with safety measures and rollback capability - Create comprehensive test suite for all backup functionality - Add admin API endpoints for backup/restore management - Implement frontend UI with dashboard, configuration, and restore wizard - Add roadmap section to README with implemented backup feature This implementation provides: - Multiple backup destinations (local, rsync, S3/MinIO) - Intelligent change detection to minimize backup frequency - Full database backups with compression - Manifest-based restore with integrity validation - Pre-restore safety backups with rollback - Comprehensive error handling and monitoring - User-friendly admin interface 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Executable
+577
@@ -0,0 +1,577 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Manual Integration Test Script for Enhanced Backup System
|
||||
*
|
||||
* This script provides a comprehensive test of the backup system with real services.
|
||||
* It can be used to test against MinIO, AWS S3, or other S3-compatible services.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/test-backup-integration.js [options]
|
||||
*
|
||||
* Options:
|
||||
* --endpoint <url> S3 endpoint URL (default: http://localhost:9000)
|
||||
* --access-key <key> S3 access key (default: minioadmin)
|
||||
* --secret-key <key> S3 secret key (default: minioadmin)
|
||||
* --bucket <name> S3 bucket name (default: test-backup-<timestamp>)
|
||||
* --type <type> Backup type: s3, local, rsync (default: s3)
|
||||
* --cleanup Clean up test data after completion
|
||||
* --verbose Enable verbose logging
|
||||
* --help Show this help message
|
||||
*
|
||||
* Examples:
|
||||
* # Test with local MinIO
|
||||
* node scripts/test-backup-integration.js
|
||||
*
|
||||
* # Test with AWS S3
|
||||
* node scripts/test-backup-integration.js \
|
||||
* --endpoint https://s3.amazonaws.com \
|
||||
* --access-key AKIAIOSFODNN7EXAMPLE \
|
||||
* --secret-key wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \
|
||||
* --bucket my-test-bucket
|
||||
*
|
||||
* # Test local backup
|
||||
* node scripts/test-backup-integration.js --type local
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
const fs = require('fs').promises;
|
||||
const crypto = require('crypto');
|
||||
const { S3Client, CreateBucketCommand, HeadBucketCommand, ListObjectsV2Command, GetObjectCommand, DeleteObjectsCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3');
|
||||
|
||||
// Parse command line arguments
|
||||
const args = process.argv.slice(2);
|
||||
const options = {
|
||||
endpoint: 'http://localhost:9000',
|
||||
accessKey: 'minioadmin',
|
||||
secretKey: 'minioadmin',
|
||||
bucket: `test-backup-${Date.now()}`,
|
||||
type: 's3',
|
||||
cleanup: false,
|
||||
verbose: false
|
||||
};
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
switch (args[i]) {
|
||||
case '--endpoint':
|
||||
options.endpoint = args[++i];
|
||||
break;
|
||||
case '--access-key':
|
||||
options.accessKey = args[++i];
|
||||
break;
|
||||
case '--secret-key':
|
||||
options.secretKey = args[++i];
|
||||
break;
|
||||
case '--bucket':
|
||||
options.bucket = args[++i];
|
||||
break;
|
||||
case '--type':
|
||||
options.type = args[++i];
|
||||
break;
|
||||
case '--cleanup':
|
||||
options.cleanup = true;
|
||||
break;
|
||||
case '--verbose':
|
||||
options.verbose = true;
|
||||
break;
|
||||
case '--help':
|
||||
console.log(module.exports.description || 'Manual Integration Test Script');
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Load environment and services
|
||||
require('dotenv').config();
|
||||
const { db, initialize: initDb } = require('../src/database/db');
|
||||
const backupService = require('../src/services/backupService');
|
||||
const S3StorageAdapter = require('../src/services/storage/s3Storage');
|
||||
const logger = require('../src/utils/logger');
|
||||
|
||||
// Configure logger based on verbose flag
|
||||
if (!options.verbose) {
|
||||
logger.info = () => {};
|
||||
logger.debug = () => {};
|
||||
}
|
||||
|
||||
// Test results
|
||||
const results = {
|
||||
passed: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
tests: []
|
||||
};
|
||||
|
||||
// Test utilities
|
||||
async function runTest(name, testFn) {
|
||||
console.log(`\n📋 Running: ${name}`);
|
||||
try {
|
||||
const startTime = Date.now();
|
||||
await testFn();
|
||||
const duration = Date.now() - startTime;
|
||||
console.log(`✅ PASSED: ${name} (${duration}ms)`);
|
||||
results.passed++;
|
||||
results.tests.push({ name, status: 'passed', duration });
|
||||
} catch (error) {
|
||||
console.error(`❌ FAILED: ${name}`);
|
||||
console.error(` Error: ${error.message}`);
|
||||
if (options.verbose) {
|
||||
console.error(error.stack);
|
||||
}
|
||||
results.failed++;
|
||||
results.tests.push({ name, status: 'failed', error: error.message });
|
||||
}
|
||||
}
|
||||
|
||||
async function skipTest(name, reason) {
|
||||
console.log(`\n⏭️ Skipping: ${name}`);
|
||||
console.log(` Reason: ${reason}`);
|
||||
results.skipped++;
|
||||
results.tests.push({ name, status: 'skipped', reason });
|
||||
}
|
||||
|
||||
// Test functions
|
||||
async function testS3Connection() {
|
||||
const s3Adapter = new S3StorageAdapter({
|
||||
bucket: options.bucket,
|
||||
endpoint: options.endpoint,
|
||||
accessKeyId: options.accessKey,
|
||||
secretAccessKey: options.secretKey,
|
||||
region: 'us-east-1',
|
||||
forcePathStyle: true,
|
||||
sslEnabled: options.endpoint.startsWith('https')
|
||||
});
|
||||
|
||||
await s3Adapter.testConnection();
|
||||
console.log(` ✓ Connected to S3 endpoint: ${options.endpoint}`);
|
||||
console.log(` ✓ Bucket accessible: ${options.bucket}`);
|
||||
}
|
||||
|
||||
async function setupTestData() {
|
||||
const storagePath = path.join(__dirname, '../test-storage');
|
||||
process.env.STORAGE_PATH = storagePath;
|
||||
|
||||
// Create directory structure
|
||||
const dirs = [
|
||||
'events/active/wedding-2024',
|
||||
'events/active/birthday-2024',
|
||||
'events/archived',
|
||||
'thumbnails',
|
||||
'uploads',
|
||||
'backups'
|
||||
];
|
||||
|
||||
for (const dir of dirs) {
|
||||
await fs.mkdir(path.join(storagePath, dir), { recursive: true });
|
||||
}
|
||||
|
||||
// Create test files with various sizes
|
||||
const files = [
|
||||
{ path: 'events/active/wedding-2024/photo1.jpg', size: 1024 * 1024 }, // 1MB
|
||||
{ path: 'events/active/wedding-2024/photo2.jpg', size: 512 * 1024 }, // 512KB
|
||||
{ path: 'events/active/birthday-2024/photo1.jpg', size: 2 * 1024 * 1024 }, // 2MB
|
||||
{ path: 'events/archived/old-event.zip', size: 5 * 1024 * 1024 }, // 5MB
|
||||
{ path: 'thumbnails/thumb1.jpg', size: 50 * 1024 }, // 50KB
|
||||
{ path: 'uploads/logo.png', size: 100 * 1024 } // 100KB
|
||||
];
|
||||
|
||||
let totalSize = 0;
|
||||
for (const file of files) {
|
||||
const content = crypto.randomBytes(file.size);
|
||||
await fs.writeFile(path.join(storagePath, file.path), content);
|
||||
totalSize += file.size;
|
||||
}
|
||||
|
||||
console.log(` ✓ Created ${files.length} test files`);
|
||||
console.log(` ✓ Total size: ${(totalSize / 1024 / 1024).toFixed(2)} MB`);
|
||||
|
||||
return { storagePath, fileCount: files.length, totalSize };
|
||||
}
|
||||
|
||||
async function configureBackup(type) {
|
||||
const baseSettings = [
|
||||
{ setting_key: 'backup_enabled', setting_value: 'true' },
|
||||
{ setting_key: 'backup_destination_type', setting_value: `"${type}"` },
|
||||
{ setting_key: 'backup_include_archived', setting_value: 'true' },
|
||||
{ setting_key: 'backup_include_database', setting_value: 'true' },
|
||||
{ setting_key: 'backup_incremental', setting_value: 'true' },
|
||||
{ setting_key: 'backup_manifest_format', setting_value: '"json"' },
|
||||
{ setting_key: 'backup_max_file_size_mb', setting_value: '100' }
|
||||
];
|
||||
|
||||
const typeSpecificSettings = {
|
||||
s3: [
|
||||
{ setting_key: 'backup_s3_bucket', setting_value: `"${options.bucket}"` },
|
||||
{ setting_key: 'backup_s3_endpoint', setting_value: `"${options.endpoint}"` },
|
||||
{ setting_key: 'backup_s3_access_key', setting_value: `"${options.accessKey}"` },
|
||||
{ setting_key: 'backup_s3_secret_key', setting_value: `"${options.secretKey}"` },
|
||||
{ setting_key: 'backup_s3_region', setting_value: '"us-east-1"' },
|
||||
{ setting_key: 'backup_s3_force_path_style', setting_value: 'true' },
|
||||
{ setting_key: 'backup_s3_ssl_enabled', setting_value: options.endpoint.startsWith('https') ? 'true' : 'false' }
|
||||
],
|
||||
local: [
|
||||
{ setting_key: 'backup_destination_path', setting_value: `"${path.join(__dirname, '../test-backup')}"` }
|
||||
],
|
||||
rsync: [
|
||||
{ setting_key: 'backup_rsync_host', setting_value: '"localhost"' },
|
||||
{ setting_key: 'backup_rsync_path', setting_value: `"${path.join(__dirname, '../test-backup-rsync')}"` }
|
||||
]
|
||||
};
|
||||
|
||||
const settings = [...baseSettings, ...(typeSpecificSettings[type] || [])];
|
||||
|
||||
// Clear existing settings
|
||||
await db('app_settings').where('setting_type', 'backup').del();
|
||||
|
||||
// Insert new settings
|
||||
for (const setting of settings) {
|
||||
await db('app_settings').insert({
|
||||
setting_type: 'backup',
|
||||
...setting,
|
||||
created_at: new Date(),
|
||||
updated_at: new Date()
|
||||
});
|
||||
}
|
||||
|
||||
console.log(` ✓ Configured ${type} backup with ${settings.length} settings`);
|
||||
}
|
||||
|
||||
async function performBackup() {
|
||||
const startTime = Date.now();
|
||||
|
||||
// Run the backup
|
||||
await backupService.runBackup();
|
||||
|
||||
// Get backup results
|
||||
const backupRun = await db('backup_runs')
|
||||
.orderBy('started_at', 'desc')
|
||||
.first();
|
||||
|
||||
if (!backupRun) {
|
||||
throw new Error('No backup run found');
|
||||
}
|
||||
|
||||
if (backupRun.status !== 'completed') {
|
||||
throw new Error(`Backup failed with status: ${backupRun.status}, error: ${backupRun.error_message}`);
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
console.log(` ✓ Backup completed in ${duration}ms`);
|
||||
console.log(` ✓ Files backed up: ${backupRun.files_backed_up}`);
|
||||
console.log(` ✓ Total size: ${(backupRun.total_size_bytes / 1024 / 1024).toFixed(2)} MB`);
|
||||
console.log(` ✓ Manifest: ${backupRun.manifest_path ? 'Generated' : 'Not generated'}`);
|
||||
|
||||
return backupRun;
|
||||
}
|
||||
|
||||
async function verifyS3Backup(backupRun) {
|
||||
const s3Client = new S3Client({
|
||||
endpoint: options.endpoint,
|
||||
region: 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: options.accessKey,
|
||||
secretAccessKey: options.secretKey
|
||||
},
|
||||
forcePathStyle: true
|
||||
});
|
||||
|
||||
// List objects in bucket
|
||||
const listResponse = await s3Client.send(new ListObjectsV2Command({
|
||||
Bucket: options.bucket
|
||||
}));
|
||||
|
||||
const objects = listResponse.Contents || [];
|
||||
console.log(` ✓ Objects in S3: ${objects.length}`);
|
||||
|
||||
// Verify key components
|
||||
const hasBackupFolder = objects.some(obj => obj.Key.includes('backup-'));
|
||||
const hasManifest = objects.some(obj => obj.Key.includes('backup-manifest'));
|
||||
const hasSummary = objects.some(obj => obj.Key.includes('backup-summary.json'));
|
||||
const hasPhotos = objects.some(obj => obj.Key.includes('events/active'));
|
||||
|
||||
if (!hasBackupFolder) throw new Error('No backup folder found in S3');
|
||||
if (!hasManifest) throw new Error('No manifest found in S3');
|
||||
if (!hasSummary) throw new Error('No summary found in S3');
|
||||
if (!hasPhotos) throw new Error('No photos found in S3');
|
||||
|
||||
console.log(` ✓ Backup structure verified`);
|
||||
|
||||
// Download and verify a file
|
||||
const photoObject = objects.find(obj => obj.Key.includes('photo1.jpg'));
|
||||
if (photoObject) {
|
||||
const getResponse = await s3Client.send(new GetObjectCommand({
|
||||
Bucket: options.bucket,
|
||||
Key: photoObject.Key
|
||||
}));
|
||||
|
||||
const chunks = [];
|
||||
for await (const chunk of getResponse.Body) {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
const content = Buffer.concat(chunks);
|
||||
|
||||
console.log(` ✓ Downloaded test file: ${photoObject.Key} (${content.length} bytes)`);
|
||||
}
|
||||
}
|
||||
|
||||
async function testIncrementalBackup(testData) {
|
||||
// Modify a file
|
||||
const modifiedFile = path.join(testData.storagePath, 'events/active/wedding-2024/photo1.jpg');
|
||||
const newContent = crypto.randomBytes(1024 * 1024 + 100); // Slightly larger
|
||||
await fs.writeFile(modifiedFile, newContent);
|
||||
|
||||
console.log(` ✓ Modified test file`);
|
||||
|
||||
// Perform incremental backup
|
||||
const backupRun = await performBackup();
|
||||
|
||||
if (backupRun.files_backed_up !== 1) {
|
||||
throw new Error(`Expected 1 file in incremental backup, got ${backupRun.files_backed_up}`);
|
||||
}
|
||||
|
||||
console.log(` ✓ Incremental backup correctly identified changed file`);
|
||||
|
||||
// Verify manifest indicates incremental
|
||||
if (backupRun.manifest_path) {
|
||||
const { manifest } = await backupService.getBackupManifest(backupRun.id);
|
||||
if (!manifest.incremental) {
|
||||
throw new Error('Manifest does not indicate incremental backup');
|
||||
}
|
||||
console.log(` ✓ Manifest correctly marked as incremental`);
|
||||
}
|
||||
|
||||
return backupRun;
|
||||
}
|
||||
|
||||
async function testManifestValidation(backupRun) {
|
||||
if (!backupRun.manifest_path) {
|
||||
throw new Error('No manifest path in backup run');
|
||||
}
|
||||
|
||||
const result = await backupService.validateBackupManifest(backupRun.manifest_path);
|
||||
|
||||
if (!result.valid) {
|
||||
throw new Error(`Manifest validation failed: ${result.error}`);
|
||||
}
|
||||
|
||||
console.log(` ✓ Manifest validation passed`);
|
||||
console.log(` ✓ Manifest version: ${result.manifest.manifest.version}`);
|
||||
console.log(` ✓ Files in manifest: ${result.manifest.files.count}`);
|
||||
}
|
||||
|
||||
async function testBackupStatus() {
|
||||
const status = await backupService.getBackupStatus(5);
|
||||
|
||||
console.log(` ✓ Backup service running: ${status.isRunning}`);
|
||||
console.log(` ✓ Backup service healthy: ${status.isHealthy}`);
|
||||
console.log(` ✓ Recent runs: ${status.recentRuns.length}`);
|
||||
|
||||
if (status.lastRun) {
|
||||
console.log(` ✓ Last run status: ${status.lastRun.status}`);
|
||||
console.log(` ✓ Manifest valid: ${status.lastRun.manifestValid}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanupTestData() {
|
||||
if (!options.cleanup) {
|
||||
console.log('\n📌 Test data retained for inspection');
|
||||
console.log(` Storage: ${process.env.STORAGE_PATH}`);
|
||||
if (options.type === 's3') {
|
||||
console.log(` S3 Bucket: ${options.bucket}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
console.log('\n🧹 Cleaning up test data...');
|
||||
|
||||
// Clean storage directory
|
||||
if (process.env.STORAGE_PATH) {
|
||||
await fs.rm(process.env.STORAGE_PATH, { recursive: true, force: true });
|
||||
console.log(' ✓ Removed test storage directory');
|
||||
}
|
||||
|
||||
// Clean S3 bucket if used
|
||||
if (options.type === 's3') {
|
||||
const s3Client = new S3Client({
|
||||
endpoint: options.endpoint,
|
||||
region: 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: options.accessKey,
|
||||
secretAccessKey: options.secretKey
|
||||
},
|
||||
forcePathStyle: true
|
||||
});
|
||||
|
||||
try {
|
||||
// List and delete all objects
|
||||
const listResponse = await s3Client.send(new ListObjectsV2Command({
|
||||
Bucket: options.bucket
|
||||
}));
|
||||
|
||||
if (listResponse.Contents && listResponse.Contents.length > 0) {
|
||||
await s3Client.send(new DeleteObjectsCommand({
|
||||
Bucket: options.bucket,
|
||||
Delete: {
|
||||
Objects: listResponse.Contents.map(obj => ({ Key: obj.Key }))
|
||||
}
|
||||
}));
|
||||
console.log(` ✓ Deleted ${listResponse.Contents.length} objects from S3`);
|
||||
}
|
||||
|
||||
// Delete bucket
|
||||
await s3Client.send(new DeleteBucketCommand({
|
||||
Bucket: options.bucket
|
||||
}));
|
||||
console.log(` ✓ Deleted S3 bucket: ${options.bucket}`);
|
||||
} catch (error) {
|
||||
console.error(` ⚠️ Failed to cleanup S3: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean backup directories
|
||||
const backupDirs = [
|
||||
path.join(__dirname, '../test-backup'),
|
||||
path.join(__dirname, '../test-backup-rsync')
|
||||
];
|
||||
|
||||
for (const dir of backupDirs) {
|
||||
await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
console.log(' ✓ Removed backup directories');
|
||||
}
|
||||
|
||||
// Main test runner
|
||||
async function main() {
|
||||
console.log('🚀 Enhanced Backup System Integration Test');
|
||||
console.log('==========================================');
|
||||
console.log(`Type: ${options.type}`);
|
||||
console.log(`Endpoint: ${options.endpoint}`);
|
||||
console.log(`Bucket: ${options.bucket}`);
|
||||
console.log('');
|
||||
|
||||
let s3Client;
|
||||
let testData;
|
||||
|
||||
try {
|
||||
// Initialize database
|
||||
console.log('📦 Initializing database...');
|
||||
await initDb();
|
||||
await db.migrate.latest();
|
||||
console.log(' ✓ Database initialized');
|
||||
|
||||
// S3-specific setup
|
||||
if (options.type === 's3') {
|
||||
// Test S3 connection
|
||||
await runTest('S3 Connection Test', testS3Connection);
|
||||
|
||||
// Create S3 bucket if needed
|
||||
s3Client = new S3Client({
|
||||
endpoint: options.endpoint,
|
||||
region: 'us-east-1',
|
||||
credentials: {
|
||||
accessKeyId: options.accessKey,
|
||||
secretAccessKey: options.secretKey
|
||||
},
|
||||
forcePathStyle: true
|
||||
});
|
||||
|
||||
try {
|
||||
await s3Client.send(new HeadBucketCommand({ Bucket: options.bucket }));
|
||||
console.log(`\n📦 Using existing bucket: ${options.bucket}`);
|
||||
} catch (error) {
|
||||
if (error.name === 'NotFound') {
|
||||
await s3Client.send(new CreateBucketCommand({ Bucket: options.bucket }));
|
||||
console.log(`\n📦 Created new bucket: ${options.bucket}`);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Setup test data
|
||||
console.log('\n📁 Setting up test data...');
|
||||
testData = await setupTestData();
|
||||
|
||||
// Configure backup
|
||||
console.log(`\n⚙️ Configuring ${options.type} backup...`);
|
||||
await configureBackup(options.type);
|
||||
|
||||
// Run tests based on backup type
|
||||
await runTest('Initial Full Backup', performBackup);
|
||||
|
||||
if (options.type === 's3') {
|
||||
await runTest('Verify S3 Backup Contents', async () => {
|
||||
const lastRun = await db('backup_runs').orderBy('started_at', 'desc').first();
|
||||
await verifyS3Backup(lastRun);
|
||||
});
|
||||
}
|
||||
|
||||
await runTest('Incremental Backup', () => testIncrementalBackup(testData));
|
||||
|
||||
await runTest('Manifest Validation', async () => {
|
||||
const lastRun = await db('backup_runs').orderBy('started_at', 'desc').first();
|
||||
await testManifestValidation(lastRun);
|
||||
});
|
||||
|
||||
await runTest('Backup Status Check', testBackupStatus);
|
||||
|
||||
// Performance test with larger files
|
||||
if (options.type === 's3') {
|
||||
await runTest('Large File Backup (10MB)', async () => {
|
||||
const largeFile = path.join(testData.storagePath, 'events/active/large.jpg');
|
||||
await fs.writeFile(largeFile, crypto.randomBytes(10 * 1024 * 1024));
|
||||
await performBackup();
|
||||
});
|
||||
}
|
||||
|
||||
// Test backup service lifecycle
|
||||
await runTest('Backup Service Start/Stop', async () => {
|
||||
await backupService.startBackupService();
|
||||
console.log(' ✓ Service started');
|
||||
|
||||
backupService.stopBackupService();
|
||||
console.log(' ✓ Service stopped');
|
||||
});
|
||||
|
||||
// Print results summary
|
||||
console.log('\n📊 Test Results Summary');
|
||||
console.log('======================');
|
||||
console.log(`✅ Passed: ${results.passed}`);
|
||||
console.log(`❌ Failed: ${results.failed}`);
|
||||
console.log(`⏭️ Skipped: ${results.skipped}`);
|
||||
console.log(`📋 Total: ${results.tests.length}`);
|
||||
|
||||
if (results.failed > 0) {
|
||||
console.log('\nFailed Tests:');
|
||||
results.tests
|
||||
.filter(t => t.status === 'failed')
|
||||
.forEach(t => console.log(` - ${t.name}: ${t.error}`));
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('\n💥 Fatal error:', error.message);
|
||||
if (options.verbose) {
|
||||
console.error(error.stack);
|
||||
}
|
||||
results.failed++;
|
||||
} finally {
|
||||
// Cleanup
|
||||
await cleanupTestData();
|
||||
|
||||
// Close database
|
||||
await db.destroy();
|
||||
|
||||
// Exit with appropriate code
|
||||
process.exit(results.failed > 0 ? 1 : 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
main().catch(error => {
|
||||
console.error('Unhandled error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { runTest, skipTest };
|
||||
@@ -0,0 +1,192 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Test script for backup manifest generator
|
||||
* Demonstrates all features of the manifest generator
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
const fs = require('fs').promises;
|
||||
const backupManifest = require('../src/services/backupManifest');
|
||||
const logger = require('../src/utils/logger');
|
||||
|
||||
async function testManifestGeneration() {
|
||||
console.log('=== Testing Backup Manifest Generator ===\n');
|
||||
|
||||
try {
|
||||
// 1. Generate a full backup manifest
|
||||
console.log('1. Generating full backup manifest...');
|
||||
|
||||
const fullManifestOptions = {
|
||||
backupType: 'full',
|
||||
backupPath: '/backup/full/2025-01-21',
|
||||
files: [
|
||||
{
|
||||
path: '/storage/events/active/wedding-smith-2025/DSC_001.jpg',
|
||||
relativePath: 'events/active/wedding-smith-2025/DSC_001.jpg',
|
||||
size: 2456789,
|
||||
modified: new Date('2025-01-20T10:30:00Z'),
|
||||
checksum: 'a1b2c3d4e5f6789012345678901234567890123456789012345678901234567890',
|
||||
permissions: '644'
|
||||
},
|
||||
{
|
||||
path: '/storage/events/active/wedding-smith-2025/DSC_002.jpg',
|
||||
relativePath: 'events/active/wedding-smith-2025/DSC_002.jpg',
|
||||
size: 2156789,
|
||||
modified: new Date('2025-01-20T10:31:00Z'),
|
||||
checksum: 'b2c3d4e5f67890123456789012345678901234567890123456789012345678901',
|
||||
permissions: '644'
|
||||
},
|
||||
{
|
||||
path: '/storage/thumbnails/wedding-smith-2025/thumb_DSC_001.jpg',
|
||||
relativePath: 'thumbnails/wedding-smith-2025/thumb_DSC_001.jpg',
|
||||
size: 45678,
|
||||
modified: new Date('2025-01-20T10:35:00Z'),
|
||||
checksum: 'c3d4e5f678901234567890123456789012345678901234567890123456789012',
|
||||
permissions: '644'
|
||||
}
|
||||
],
|
||||
databaseInfo: {
|
||||
type: 'sqlite',
|
||||
backupFile: 'database-backup-20250121-103000.sql.gz',
|
||||
size: 1048576,
|
||||
checksum: 'd4e5f6789012345678901234567890123456789012345678901234567890123',
|
||||
tables: {
|
||||
events: 156,
|
||||
photos: 4523,
|
||||
access_logs: 12456,
|
||||
admin_users: 3
|
||||
},
|
||||
rowCounts: {
|
||||
events: 156,
|
||||
photos: 4523,
|
||||
access_logs: 12456,
|
||||
admin_users: 3
|
||||
}
|
||||
},
|
||||
format: 'json',
|
||||
customMetadata: {
|
||||
operator: 'admin@example.com',
|
||||
reason: 'Scheduled daily backup',
|
||||
retentionDays: 30,
|
||||
compressionType: 'gzip'
|
||||
}
|
||||
};
|
||||
|
||||
const fullManifest = await backupManifest.generateManifest(fullManifestOptions);
|
||||
|
||||
// Save in both formats
|
||||
const jsonPath = path.join(__dirname, 'test-manifest-full.json');
|
||||
const yamlPath = path.join(__dirname, 'test-manifest-full.yaml');
|
||||
|
||||
await backupManifest.saveManifest(fullManifest, jsonPath, 'json');
|
||||
await backupManifest.saveManifest(fullManifest, yamlPath, 'yaml');
|
||||
|
||||
console.log('✓ Full backup manifest generated and saved\n');
|
||||
|
||||
// 2. Generate summary report
|
||||
console.log('2. Generating summary report...');
|
||||
const summaryReport = backupManifest.generateSummaryReport(fullManifest);
|
||||
console.log(summaryReport);
|
||||
console.log('\n');
|
||||
|
||||
// 3. Load and validate manifest
|
||||
console.log('3. Loading and validating manifest...');
|
||||
const loadedManifest = await backupManifest.loadManifest(jsonPath);
|
||||
console.log('✓ Manifest loaded and validated successfully\n');
|
||||
|
||||
// 4. Generate incremental backup manifest
|
||||
console.log('4. Generating incremental backup manifest...');
|
||||
|
||||
const incrementalOptions = {
|
||||
backupType: 'incremental',
|
||||
backupPath: '/backup/incremental/2025-01-22',
|
||||
parentBackupId: fullManifest.backup.id,
|
||||
files: [
|
||||
// Original files with same checksums (unchanged)
|
||||
fullManifestOptions.files[0],
|
||||
fullManifestOptions.files[2],
|
||||
// Modified file
|
||||
{
|
||||
...fullManifestOptions.files[1],
|
||||
size: 2256789,
|
||||
modified: new Date('2025-01-21T14:00:00Z'),
|
||||
checksum: 'e5f678901234567890123456789012345678901234567890123456789012345'
|
||||
},
|
||||
// New file
|
||||
{
|
||||
path: '/storage/events/active/wedding-smith-2025/DSC_003.jpg',
|
||||
relativePath: 'events/active/wedding-smith-2025/DSC_003.jpg',
|
||||
size: 2356789,
|
||||
modified: new Date('2025-01-21T14:30:00Z'),
|
||||
checksum: 'f6789012345678901234567890123456789012345678901234567890123456',
|
||||
permissions: '644'
|
||||
}
|
||||
],
|
||||
databaseInfo: {
|
||||
...fullManifestOptions.databaseInfo,
|
||||
size: 1148576,
|
||||
checksum: 'g7890123456789012345678901234567890123456789012345678901234567',
|
||||
rowCounts: {
|
||||
events: 158,
|
||||
photos: 4567,
|
||||
access_logs: 12789,
|
||||
admin_users: 3
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const incrementalManifest = await backupManifest.generateIncrementalManifest(
|
||||
incrementalOptions,
|
||||
fullManifest
|
||||
);
|
||||
|
||||
const incrementalJsonPath = path.join(__dirname, 'test-manifest-incremental.json');
|
||||
await backupManifest.saveManifest(incrementalManifest, incrementalJsonPath, 'json');
|
||||
|
||||
console.log('✓ Incremental backup manifest generated');
|
||||
console.log(` - Added files: ${incrementalManifest.incremental.changes.added_files_count}`);
|
||||
console.log(` - Modified files: ${incrementalManifest.incremental.changes.modified_files_count}`);
|
||||
console.log(` - Deleted files: ${incrementalManifest.incremental.changes.deleted_files_count}`);
|
||||
console.log(` - Size difference: ${(incrementalManifest.incremental.changes.size_difference / 1024).toFixed(2)} KB\n`);
|
||||
|
||||
// 5. Compare manifests
|
||||
console.log('5. Comparing manifests...');
|
||||
const comparison = backupManifest.compareManifests(incrementalManifest, fullManifest);
|
||||
console.log('Comparison results:');
|
||||
console.log(` - Added: ${comparison.added_files.length} files`);
|
||||
console.log(` - Modified: ${comparison.modified_files.length} files`);
|
||||
console.log(` - Deleted: ${comparison.deleted_files.length} files`);
|
||||
console.log(` - Unchanged: ${comparison.unchanged_files.length} files`);
|
||||
console.log(` - Database changed: ${comparison.database_changes.checksum_changed ? 'Yes' : 'No'}\n`);
|
||||
|
||||
// 6. Test manifest integrity
|
||||
console.log('6. Testing manifest integrity...');
|
||||
|
||||
// Corrupt the manifest
|
||||
const corruptedManifest = JSON.parse(JSON.stringify(incrementalManifest));
|
||||
corruptedManifest.files.manifest[0].size = 9999999; // Change a file size
|
||||
|
||||
try {
|
||||
backupManifest.validateManifest(corruptedManifest);
|
||||
console.log('✗ Validation should have failed for corrupted manifest');
|
||||
} catch (error) {
|
||||
console.log('✓ Correctly detected corrupted manifest:', error.message);
|
||||
}
|
||||
|
||||
console.log('\n=== All tests completed successfully! ===');
|
||||
|
||||
// Clean up test files
|
||||
await fs.unlink(jsonPath).catch(() => {});
|
||||
await fs.unlink(yamlPath).catch(() => {});
|
||||
await fs.unlink(incrementalJsonPath).catch(() => {});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Test failed:', error);
|
||||
logger.error('Manifest test failed:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run tests
|
||||
testManifestGeneration().catch(console.error);
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
require('dotenv').config();
|
||||
const { initializeDatabase } = require('../src/database/db');
|
||||
const { runBackup, getBackupStatus } = require('../src/services/backupService');
|
||||
const logger = require('../src/utils/logger');
|
||||
|
||||
async function testBackupService() {
|
||||
try {
|
||||
console.log('Testing backup service...\n');
|
||||
|
||||
// Initialize database
|
||||
await initializeDatabase();
|
||||
|
||||
// Get current backup status
|
||||
console.log('Getting backup status...');
|
||||
const statusBefore = await getBackupStatus();
|
||||
console.log('Last run:', statusBefore.lastRun ? statusBefore.lastRun.started_at : 'Never');
|
||||
console.log('Is healthy:', statusBefore.isHealthy);
|
||||
console.log('');
|
||||
|
||||
// Run backup
|
||||
console.log('Running backup...');
|
||||
await runBackup();
|
||||
|
||||
// Get status after backup
|
||||
console.log('\nGetting status after backup...');
|
||||
const statusAfter = await getBackupStatus();
|
||||
console.log('Last run:', statusAfter.lastRun ? statusAfter.lastRun.started_at : 'Never');
|
||||
console.log('Status:', statusAfter.lastRun ? statusAfter.lastRun.status : 'Unknown');
|
||||
console.log('Files backed up:', statusAfter.lastRun ? statusAfter.lastRun.files_backed_up : 0);
|
||||
console.log('Total size:', statusAfter.lastRun ? `${(statusAfter.lastRun.total_size_bytes / 1024 / 1024).toFixed(2)} MB` : '0 MB');
|
||||
|
||||
if (statusAfter.lastRun && statusAfter.lastRun.error_message) {
|
||||
console.log('Error:', statusAfter.lastRun.error_message);
|
||||
}
|
||||
|
||||
console.log('\nBackup test completed!');
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('Test failed:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
testBackupService();
|
||||
@@ -0,0 +1,325 @@
|
||||
/**
|
||||
* Test script for the restore service
|
||||
*
|
||||
* This script demonstrates the restore service functionality with safety checks
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/test-restore-service.js [options]
|
||||
*
|
||||
* Options:
|
||||
* --dry-run Perform validation only without actual restore
|
||||
* --force Force restore even with warnings
|
||||
* --type Restore type: full, database, files, selective (default: full)
|
||||
* --source Backup source path or S3 URL
|
||||
* --manifest Path to backup manifest
|
||||
*/
|
||||
|
||||
require('dotenv').config();
|
||||
const { restoreService } = require('../src/services/restoreService');
|
||||
const { db } = require('../src/database/db');
|
||||
const logger = require('../src/utils/logger');
|
||||
const path = require('path');
|
||||
const fs = require('fs').promises;
|
||||
|
||||
// Parse command line arguments
|
||||
const args = process.argv.slice(2);
|
||||
const options = {
|
||||
dryRun: args.includes('--dry-run'),
|
||||
force: args.includes('--force'),
|
||||
restoreType: 'full',
|
||||
source: null,
|
||||
manifestPath: null
|
||||
};
|
||||
|
||||
// Parse restore type
|
||||
const typeIndex = args.indexOf('--type');
|
||||
if (typeIndex !== -1 && args[typeIndex + 1]) {
|
||||
options.restoreType = args[typeIndex + 1];
|
||||
}
|
||||
|
||||
// Parse source
|
||||
const sourceIndex = args.indexOf('--source');
|
||||
if (sourceIndex !== -1 && args[sourceIndex + 1]) {
|
||||
options.source = args[sourceIndex + 1];
|
||||
}
|
||||
|
||||
// Parse manifest
|
||||
const manifestIndex = args.indexOf('--manifest');
|
||||
if (manifestIndex !== -1 && args[manifestIndex + 1]) {
|
||||
options.manifestPath = args[manifestIndex + 1];
|
||||
}
|
||||
|
||||
async function testRestore() {
|
||||
console.log('=== PicPeak Restore Service Test ===\n');
|
||||
|
||||
try {
|
||||
// If no source/manifest provided, try to find a recent backup
|
||||
if (!options.source || !options.manifestPath) {
|
||||
console.log('No backup source specified. Looking for recent backups...\n');
|
||||
|
||||
const recentBackup = await db('backup_runs')
|
||||
.where('status', 'completed')
|
||||
.whereNotNull('manifest_path')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.first();
|
||||
|
||||
if (!recentBackup) {
|
||||
console.error('❌ No completed backups found in the database');
|
||||
console.log('\nPlease run a backup first or specify --source and --manifest');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`Found recent backup from ${recentBackup.completed_at}`);
|
||||
console.log(`Backup ID: ${recentBackup.manifest_id}`);
|
||||
console.log(`Files backed up: ${recentBackup.files_backed_up}`);
|
||||
console.log(`Total size: ${(recentBackup.total_size_bytes / 1024 / 1024).toFixed(2)} MB`);
|
||||
console.log(`Manifest: ${recentBackup.manifest_path}\n`);
|
||||
|
||||
// For this test, we'll create a mock scenario
|
||||
console.log('⚠️ This is a TEST MODE - using mock data for safety\n');
|
||||
|
||||
// Create test backup directory
|
||||
const testBackupDir = path.join(__dirname, '../temp/test-backup');
|
||||
await fs.mkdir(testBackupDir, { recursive: true });
|
||||
|
||||
// Create test manifest
|
||||
const testManifest = {
|
||||
manifest: {
|
||||
version: '2.0',
|
||||
created: new Date().toISOString(),
|
||||
generator: 'Test Script',
|
||||
format: 'json'
|
||||
},
|
||||
backup: {
|
||||
id: 'test-backup-' + Date.now(),
|
||||
type: 'full',
|
||||
timestamp: new Date().toISOString(),
|
||||
path: testBackupDir,
|
||||
parent_backup_id: null,
|
||||
retention_days: 30
|
||||
},
|
||||
system: {
|
||||
hostname: require('os').hostname(),
|
||||
platform: process.platform,
|
||||
os_release: require('os').release(),
|
||||
architecture: require('os').arch()
|
||||
},
|
||||
application: {
|
||||
name: 'PicPeak',
|
||||
version: require('../package.json').version,
|
||||
node_version: process.version,
|
||||
environment: 'test'
|
||||
},
|
||||
files: {
|
||||
count: 0,
|
||||
total_size: 0,
|
||||
checksums: {},
|
||||
manifest: []
|
||||
},
|
||||
database: {
|
||||
type: process.env.DB_TYPE === 'postgresql' ? 'postgresql' : 'sqlite',
|
||||
backup_file: null,
|
||||
size: 0,
|
||||
checksum: null,
|
||||
tables: {},
|
||||
row_counts: {}
|
||||
},
|
||||
verification: {
|
||||
total_checksum: null,
|
||||
file_count_check: 0,
|
||||
size_check: 0,
|
||||
integrity_timestamp: new Date().toISOString()
|
||||
},
|
||||
metadata: {
|
||||
test_mode: true
|
||||
}
|
||||
};
|
||||
|
||||
// Calculate checksum
|
||||
const crypto = require('crypto');
|
||||
const manifestCopy = JSON.parse(JSON.stringify(testManifest));
|
||||
delete manifestCopy.verification.total_checksum;
|
||||
testManifest.verification.total_checksum = crypto
|
||||
.createHash('sha256')
|
||||
.update(JSON.stringify(manifestCopy, Object.keys(manifestCopy).sort()))
|
||||
.digest('hex');
|
||||
|
||||
// Save test manifest
|
||||
const testManifestPath = path.join(testBackupDir, 'test-manifest.json');
|
||||
await fs.writeFile(testManifestPath, JSON.stringify(testManifest, null, 2));
|
||||
|
||||
options.source = testBackupDir;
|
||||
options.manifestPath = testManifestPath;
|
||||
}
|
||||
|
||||
// Display restore options
|
||||
console.log('Restore Options:');
|
||||
console.log(`- Type: ${options.restoreType}`);
|
||||
console.log(`- Source: ${options.source}`);
|
||||
console.log(`- Manifest: ${options.manifestPath}`);
|
||||
console.log(`- Dry Run: ${options.dryRun ? 'Yes' : 'No'}`);
|
||||
console.log(`- Force: ${options.force ? 'Yes' : 'No'}`);
|
||||
console.log('');
|
||||
|
||||
// Add S3 config if source is S3
|
||||
if (options.source.startsWith('s3://')) {
|
||||
options.s3Config = {
|
||||
accessKeyId: process.env.BACKUP_S3_ACCESS_KEY,
|
||||
secretAccessKey: process.env.BACKUP_S3_SECRET_KEY,
|
||||
region: process.env.BACKUP_S3_REGION || 'us-east-1',
|
||||
endpoint: process.env.BACKUP_S3_ENDPOINT
|
||||
};
|
||||
|
||||
if (!options.s3Config.accessKeyId || !options.s3Config.secretAccessKey) {
|
||||
console.error('❌ S3 credentials not configured in environment');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Confirm before proceeding (unless dry run)
|
||||
if (!options.dryRun) {
|
||||
console.log('⚠️ WARNING: This will restore data from the backup!');
|
||||
console.log('⚠️ Current data may be overwritten!');
|
||||
console.log('');
|
||||
console.log('Press Ctrl+C to cancel, or wait 5 seconds to continue...');
|
||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
||||
}
|
||||
|
||||
console.log('\nStarting restore operation...\n');
|
||||
|
||||
// Perform restore
|
||||
const result = await restoreService.restore(options);
|
||||
|
||||
if (options.dryRun) {
|
||||
console.log('\n=== DRY RUN RESULTS ===\n');
|
||||
|
||||
console.log('Validation:');
|
||||
console.log(`- Valid: ${result.validation.isValid ? '✅ Yes' : '❌ No'}`);
|
||||
|
||||
if (result.validation.errors.length > 0) {
|
||||
console.log('- Errors:');
|
||||
result.validation.errors.forEach(err => console.log(` ❌ ${err}`));
|
||||
}
|
||||
|
||||
if (result.validation.warnings.length > 0) {
|
||||
console.log('- Warnings:');
|
||||
result.validation.warnings.forEach(warn => console.log(` ⚠️ ${warn}`));
|
||||
}
|
||||
|
||||
console.log('\nDisk Space:');
|
||||
console.log(`- Required: ${result.spaceCheck.requiredFormatted}`);
|
||||
console.log(`- Available: ${result.spaceCheck.availableFormatted}`);
|
||||
console.log(`- Sufficient: ${result.spaceCheck.hasEnoughSpace ? '✅ Yes' : '❌ No'}`);
|
||||
|
||||
} else {
|
||||
console.log('\n=== RESTORE RESULTS ===\n');
|
||||
|
||||
console.log(`Status: ${result.success ? '✅ SUCCESS' : '❌ FAILED'}`);
|
||||
console.log(`Duration: ${result.duration}s`);
|
||||
|
||||
if (result.result) {
|
||||
console.log('\nItems Restored:');
|
||||
if (result.result.databaseRestored !== undefined) {
|
||||
console.log(`- Database: ${result.result.databaseRestored ? '✅' : '❌'}`);
|
||||
}
|
||||
if (result.result.filesRestored !== undefined) {
|
||||
console.log(`- Files: ${result.result.filesRestored}`);
|
||||
}
|
||||
if (result.result.errors && result.result.errors.length > 0) {
|
||||
console.log('- Errors:');
|
||||
result.result.errors.forEach(err => console.log(` ❌ ${err}`));
|
||||
}
|
||||
}
|
||||
|
||||
if (result.verification) {
|
||||
console.log('\nVerification:');
|
||||
console.log(`- Valid: ${result.verification.isValid ? '✅ Yes' : '❌ No'}`);
|
||||
if (result.verification.errors.length > 0) {
|
||||
console.log('- Errors:');
|
||||
result.verification.errors.forEach(err => console.log(` ❌ ${err}`));
|
||||
}
|
||||
}
|
||||
|
||||
if (result.preRestoreBackup) {
|
||||
console.log('\nSafety Backup:');
|
||||
console.log(`- Location: ${result.preRestoreBackup}`);
|
||||
console.log('- This backup can be used to rollback if needed');
|
||||
}
|
||||
}
|
||||
|
||||
// Show recent log entries
|
||||
console.log('\nRecent Log Entries:');
|
||||
result.logs.slice(-10).forEach(log => {
|
||||
const icon = log.level === 'error' ? '❌' : log.level === 'warn' ? '⚠️ ' : 'ℹ️ ';
|
||||
console.log(`${icon} [${log.timestamp}] ${log.message}`);
|
||||
});
|
||||
|
||||
// Clean up test files
|
||||
if (options.source && options.source.includes('test-backup')) {
|
||||
await fs.rmdir(path.dirname(options.source), { recursive: true }).catch(() => {});
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('\n❌ Restore operation failed:', error.message);
|
||||
|
||||
// Show logs if available
|
||||
if (restoreService.restoreLog && restoreService.restoreLog.length > 0) {
|
||||
console.log('\nError Log:');
|
||||
restoreService.restoreLog.slice(-10).forEach(log => {
|
||||
if (log.level === 'error' || log.level === 'warn') {
|
||||
console.log(`[${log.timestamp}] ${log.level.toUpperCase()}: ${log.message}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
await db.destroy();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Show help if requested
|
||||
if (args.includes('--help') || args.includes('-h')) {
|
||||
console.log(`
|
||||
PicPeak Restore Service Test
|
||||
|
||||
This script tests the restore service functionality with safety checks.
|
||||
|
||||
Usage:
|
||||
node scripts/test-restore-service.js [options]
|
||||
|
||||
Options:
|
||||
--dry-run Perform validation only without actual restore
|
||||
--force Force restore even with warnings
|
||||
--type Restore type: full, database, files, selective (default: full)
|
||||
--source Backup source path or S3 URL
|
||||
--manifest Path to backup manifest
|
||||
--help Show this help message
|
||||
|
||||
Examples:
|
||||
# Dry run with automatic backup selection
|
||||
node scripts/test-restore-service.js --dry-run
|
||||
|
||||
# Full restore from specific backup
|
||||
node scripts/test-restore-service.js --source /backup/2024-01-20 --manifest /backup/2024-01-20/manifest.json
|
||||
|
||||
# Database-only restore with force
|
||||
node scripts/test-restore-service.js --type database --force --source /backup/2024-01-20 --manifest /backup/2024-01-20/manifest.json
|
||||
|
||||
# Restore from S3
|
||||
node scripts/test-restore-service.js --source s3://my-bucket/backups/2024-01-20 --manifest s3://my-bucket/backups/2024-01-20/manifest.json
|
||||
|
||||
Safety Features:
|
||||
- Pre-restore validation checks compatibility and warns about potential issues
|
||||
- Automatic pre-restore backup is created (unless skipped)
|
||||
- Post-restore verification ensures data integrity
|
||||
- Rollback capability if restore fails
|
||||
- Detailed logging of all operations
|
||||
`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Run the test
|
||||
testRestore();
|
||||
Reference in New Issue
Block a user