feat: implement comprehensive backup and restore system with S3 support
- Add S3/MinIO storage adapter with multipart upload support - Implement database backup service for SQLite and PostgreSQL - Create backup manifest generator for tracking backup contents - Enhance backup service with S3 integration and incremental backups - Add restore service with safety measures and rollback capability - Create comprehensive test suite for all backup functionality - Add admin API endpoints for backup/restore management - Implement frontend UI with dashboard, configuration, and restore wizard - Add roadmap section to README with implemented backup feature This implementation provides: - Multiple backup destinations (local, rsync, S3/MinIO) - Intelligent change detection to minimize backup frequency - Full database backups with compression - Manifest-based restore with integrity validation - Pre-restore safety backups with rollback - Comprehensive error handling and monitoring - User-friendly admin interface 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,8 @@ const photosRoutes = require('./adminPhotos');
|
||||
const categoriesRoutes = require('./adminCategories');
|
||||
const cmsRoutes = require('./adminCMS');
|
||||
const notificationsRoutes = require('./adminNotifications');
|
||||
const backupRoutes = require('./adminBackup');
|
||||
const restoreRoutes = require('./adminRestore');
|
||||
|
||||
// Mount sub-routers
|
||||
router.use('/dashboard', dashboardRoutes);
|
||||
@@ -22,5 +24,7 @@ router.use('/events', photosRoutes);
|
||||
router.use('/categories', categoriesRoutes);
|
||||
router.use('/cms', cmsRoutes);
|
||||
router.use('/notifications', notificationsRoutes);
|
||||
router.use('/backup', backupRoutes);
|
||||
router.use('/restore', restoreRoutes);
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,956 @@
|
||||
const express = require('express');
|
||||
const { db } = require('../database/db');
|
||||
const { adminAuth } = require('../middleware/auth');
|
||||
const { triggerManualBackup, getBackupStatus, cleanupOldBackupRuns, getBackupManifest, validateBackupManifest } = require('../services/backupService');
|
||||
const logger = require('../utils/logger');
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const archiver = require('archiver');
|
||||
const S3StorageAdapter = require('../services/storage/s3Storage');
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// Get backup configuration
|
||||
router.get('/config', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
const config = {};
|
||||
settings.forEach(setting => {
|
||||
try {
|
||||
config[setting.setting_key] = JSON.parse(setting.setting_value);
|
||||
} catch (e) {
|
||||
config[setting.setting_key] = setting.setting_value;
|
||||
}
|
||||
});
|
||||
|
||||
res.json(config);
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup configuration:', error);
|
||||
res.status(500).json({ error: 'Failed to get backup configuration' });
|
||||
}
|
||||
});
|
||||
|
||||
// Update backup configuration
|
||||
router.put('/config', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const updates = req.body;
|
||||
|
||||
// Validate required fields based on destination type
|
||||
if (updates.backup_destination_type) {
|
||||
switch (updates.backup_destination_type) {
|
||||
case 'local':
|
||||
if (!updates.backup_destination_path) {
|
||||
return res.status(400).json({ error: 'Local backup requires destination path' });
|
||||
}
|
||||
break;
|
||||
case 'rsync':
|
||||
if (!updates.backup_rsync_host || !updates.backup_rsync_path) {
|
||||
return res.status(400).json({ error: 'Rsync backup requires host and path' });
|
||||
}
|
||||
break;
|
||||
case 's3':
|
||||
if (!updates.backup_s3_endpoint || !updates.backup_s3_bucket ||
|
||||
!updates.backup_s3_access_key || !updates.backup_s3_secret_key) {
|
||||
return res.status(400).json({ error: 'S3 backup requires endpoint, bucket, and credentials' });
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Update settings
|
||||
for (const [key, value] of Object.entries(updates)) {
|
||||
if (key.startsWith('backup_')) {
|
||||
await db('app_settings')
|
||||
.insert({
|
||||
setting_key: key,
|
||||
setting_value: JSON.stringify(value),
|
||||
setting_type: 'backup',
|
||||
updated_at: new Date()
|
||||
})
|
||||
.onConflict('setting_key')
|
||||
.merge({
|
||||
setting_value: JSON.stringify(value),
|
||||
updated_at: new Date()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Restart backup service if enabled status changed
|
||||
if ('backup_enabled' in updates) {
|
||||
const { startBackupService, stopBackupService } = require('../services/backupService');
|
||||
if (updates.backup_enabled) {
|
||||
await startBackupService();
|
||||
} else {
|
||||
stopBackupService();
|
||||
}
|
||||
}
|
||||
|
||||
res.json({ success: true, message: 'Backup configuration updated' });
|
||||
} catch (error) {
|
||||
logger.error('Failed to update backup configuration:', error);
|
||||
res.status(500).json({ error: 'Failed to update backup configuration' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get backup status and history
|
||||
router.get('/status', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const limit = parseInt(req.query.limit) || 10;
|
||||
const status = await getBackupStatus(limit);
|
||||
|
||||
res.json(status);
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup status:', error);
|
||||
res.status(500).json({ error: 'Failed to get backup status' });
|
||||
}
|
||||
});
|
||||
|
||||
// Trigger manual backup
|
||||
router.post('/run', adminAuth, async (req, res) => {
|
||||
try {
|
||||
// Check if backup is already running
|
||||
const status = await getBackupStatus();
|
||||
if (status.isRunning) {
|
||||
return res.status(409).json({ error: 'Backup is already running' });
|
||||
}
|
||||
|
||||
// Start backup in background
|
||||
triggerManualBackup().catch(error => {
|
||||
logger.error('Manual backup failed:', error);
|
||||
});
|
||||
|
||||
res.json({ success: true, message: 'Backup started' });
|
||||
} catch (error) {
|
||||
logger.error('Failed to trigger manual backup:', error);
|
||||
res.status(500).json({ error: 'Failed to trigger backup' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get backup run details
|
||||
router.get('/runs/:id', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { id } = req.params;
|
||||
|
||||
const run = await db('backup_runs')
|
||||
.where('id', id)
|
||||
.first();
|
||||
|
||||
if (!run) {
|
||||
return res.status(404).json({ error: 'Backup run not found' });
|
||||
}
|
||||
|
||||
// Parse JSON fields
|
||||
if (run.statistics) {
|
||||
try {
|
||||
run.statistics = JSON.parse(run.statistics);
|
||||
} catch (e) {
|
||||
// Keep as string if parsing fails
|
||||
}
|
||||
}
|
||||
|
||||
res.json(run);
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup run details:', error);
|
||||
res.status(500).json({ error: 'Failed to get backup run details' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get file states (for debugging/monitoring)
|
||||
router.get('/files', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { page = 1, limit = 50, search = '' } = req.query;
|
||||
const offset = (page - 1) * limit;
|
||||
|
||||
let query = db('backup_file_states');
|
||||
|
||||
if (search) {
|
||||
query = query.where('file_path', 'like', `%${search}%`);
|
||||
}
|
||||
|
||||
const [files, totalCount] = await Promise.all([
|
||||
query
|
||||
.orderBy('last_backed_up', 'desc')
|
||||
.limit(limit)
|
||||
.offset(offset),
|
||||
db('backup_file_states').count('* as count').first()
|
||||
]);
|
||||
|
||||
res.json({
|
||||
files,
|
||||
pagination: {
|
||||
page: parseInt(page),
|
||||
limit: parseInt(limit),
|
||||
total: totalCount.count,
|
||||
pages: Math.ceil(totalCount.count / limit)
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup file states:', error);
|
||||
res.status(500).json({ error: 'Failed to get file states' });
|
||||
}
|
||||
});
|
||||
|
||||
// Clean up old backup runs
|
||||
router.delete('/cleanup', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { days = 30 } = req.body;
|
||||
|
||||
await cleanupOldBackupRuns(days);
|
||||
|
||||
res.json({ success: true, message: `Cleaned up backup runs older than ${days} days` });
|
||||
} catch (error) {
|
||||
logger.error('Failed to cleanup old backup runs:', error);
|
||||
res.status(500).json({ error: 'Failed to cleanup backup runs' });
|
||||
}
|
||||
});
|
||||
|
||||
// Test backup destination connectivity
|
||||
router.post('/test-connection', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { destination_type, ...config } = req.body;
|
||||
|
||||
switch (destination_type) {
|
||||
case 'local':
|
||||
// Test local path access
|
||||
const fs = require('fs').promises;
|
||||
try {
|
||||
await fs.access(config.path, fs.constants.W_OK);
|
||||
res.json({ success: true, message: 'Local path is writable' });
|
||||
} catch (error) {
|
||||
res.json({ success: false, message: 'Cannot write to local path: ' + error.message });
|
||||
}
|
||||
break;
|
||||
|
||||
case 'rsync':
|
||||
// Test rsync connection
|
||||
const { exec } = require('child_process');
|
||||
const { promisify } = require('util');
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
const sshCommand = config.ssh_key
|
||||
? `ssh -i ${config.ssh_key} -o StrictHostKeyChecking=no -o ConnectTimeout=10`
|
||||
: 'ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10';
|
||||
|
||||
const testCommand = config.user
|
||||
? `${sshCommand} ${config.user}@${config.host} "echo 'Connection successful'"`
|
||||
: `${sshCommand} ${config.host} "echo 'Connection successful'"`;
|
||||
|
||||
try {
|
||||
const { stdout } = await execAsync(testCommand);
|
||||
res.json({ success: true, message: 'Rsync connection successful' });
|
||||
} catch (error) {
|
||||
res.json({ success: false, message: 'Rsync connection failed: ' + error.message });
|
||||
}
|
||||
break;
|
||||
|
||||
case 's3':
|
||||
// Test S3 connection (would need AWS SDK)
|
||||
res.json({ success: false, message: 'S3 testing not implemented yet' });
|
||||
break;
|
||||
|
||||
default:
|
||||
res.status(400).json({ error: 'Invalid destination type' });
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to test backup connection:', error);
|
||||
res.status(500).json({ error: 'Failed to test connection' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get backup manifest for a specific backup run
|
||||
router.get('/manifest/:backupRunId', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { backupRunId } = req.params;
|
||||
const result = await getBackupManifest(backupRunId);
|
||||
|
||||
res.json({
|
||||
backupRunId,
|
||||
manifest: result.manifest,
|
||||
summary: result.summary
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup manifest:', error);
|
||||
res.status(404).json({ error: error.message || 'Backup manifest not found' });
|
||||
}
|
||||
});
|
||||
|
||||
// Validate a backup manifest
|
||||
router.post('/manifest/validate', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { manifestPath } = req.body;
|
||||
|
||||
if (!manifestPath) {
|
||||
return res.status(400).json({ error: 'manifestPath is required' });
|
||||
}
|
||||
|
||||
const result = await validateBackupManifest(manifestPath);
|
||||
|
||||
res.json({
|
||||
valid: result.valid,
|
||||
error: result.error,
|
||||
manifestPath
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to validate manifest:', error);
|
||||
res.status(500).json({ error: 'Failed to validate manifest' });
|
||||
}
|
||||
});
|
||||
|
||||
// Download backup manifest
|
||||
router.get('/manifest/:backupRunId/download', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { backupRunId } = req.params;
|
||||
const { format = 'json' } = req.query;
|
||||
|
||||
const result = await getBackupManifest(backupRunId);
|
||||
|
||||
// Set appropriate headers
|
||||
const filename = `backup-manifest-${backupRunId}.${format}`;
|
||||
res.setHeader('Content-Type', format === 'yaml' ? 'text/yaml' : 'application/json');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
|
||||
|
||||
// Send the manifest in requested format
|
||||
if (format === 'yaml') {
|
||||
const yaml = require('js-yaml');
|
||||
res.send(yaml.dump(result.manifest, {
|
||||
indent: 2,
|
||||
lineWidth: -1,
|
||||
noRefs: true,
|
||||
sortKeys: true
|
||||
}));
|
||||
} else {
|
||||
res.json(result.manifest);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to download backup manifest:', error);
|
||||
res.status(404).json({ error: error.message || 'Backup manifest not found' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get manifest for specific backup
|
||||
router.get('/manifests/:backupId', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { backupId } = req.params;
|
||||
const result = await getBackupManifest(backupId);
|
||||
|
||||
res.json({
|
||||
backupId,
|
||||
manifest: result.manifest,
|
||||
summary: result.summary
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup manifest:', error);
|
||||
res.status(404).json({ error: error.message || 'Backup manifest not found' });
|
||||
}
|
||||
});
|
||||
|
||||
// Download manifest file
|
||||
router.get('/manifests/:backupId/download', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { backupId } = req.params;
|
||||
const { format = 'json' } = req.query;
|
||||
|
||||
const result = await getBackupManifest(backupId);
|
||||
|
||||
// Set appropriate headers
|
||||
const filename = `backup-manifest-${backupId}.${format}`;
|
||||
res.setHeader('Content-Type', format === 'yaml' ? 'text/yaml' : 'application/json');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${filename}"`);
|
||||
|
||||
// Send the manifest in requested format
|
||||
if (format === 'yaml') {
|
||||
const yaml = require('js-yaml');
|
||||
res.send(yaml.dump(result.manifest, {
|
||||
indent: 2,
|
||||
lineWidth: -1,
|
||||
noRefs: true,
|
||||
sortKeys: true
|
||||
}));
|
||||
} else {
|
||||
res.json(result.manifest);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to download backup manifest:', error);
|
||||
res.status(404).json({ error: error.message || 'Backup manifest not found' });
|
||||
}
|
||||
});
|
||||
|
||||
// Validate a manifest
|
||||
router.post('/manifests/validate', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { manifestPath, manifestData } = req.body;
|
||||
|
||||
if (!manifestPath && !manifestData) {
|
||||
return res.status(400).json({ error: 'Either manifestPath or manifestData is required' });
|
||||
}
|
||||
|
||||
if (manifestData) {
|
||||
// Validate provided manifest data directly
|
||||
const validationResult = await validateManifestData(manifestData);
|
||||
return res.json(validationResult);
|
||||
}
|
||||
|
||||
// Use existing validation function for path
|
||||
const result = await validateBackupManifest(manifestPath);
|
||||
|
||||
res.json({
|
||||
valid: result.valid,
|
||||
error: result.error,
|
||||
manifestPath
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to validate manifest:', error);
|
||||
res.status(500).json({ error: 'Failed to validate manifest' });
|
||||
}
|
||||
});
|
||||
|
||||
// List S3 buckets
|
||||
router.get('/s3/buckets', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const config = await getBackupConfig();
|
||||
|
||||
if (config.backup_destination_type !== 's3') {
|
||||
return res.status(400).json({ error: 'S3 backup not configured' });
|
||||
}
|
||||
|
||||
const s3Adapter = new S3StorageAdapter({
|
||||
endpoint: config.backup_s3_endpoint,
|
||||
bucket: config.backup_s3_bucket,
|
||||
accessKeyId: config.backup_s3_access_key,
|
||||
secretAccessKey: config.backup_s3_secret_key,
|
||||
region: config.backup_s3_region || 'us-east-1',
|
||||
forcePathStyle: config.backup_s3_force_path_style || false
|
||||
});
|
||||
|
||||
// List buckets using the S3 client
|
||||
const { ListBucketsCommand } = require('@aws-sdk/client-s3');
|
||||
const result = await s3Adapter.s3Client.send(new ListBucketsCommand({}));
|
||||
|
||||
res.json({
|
||||
buckets: result.Buckets || [],
|
||||
owner: result.Owner || null
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to list S3 buckets:', error);
|
||||
res.status(500).json({ error: 'Failed to list S3 buckets: ' + error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// List files in S3 backup location
|
||||
router.get('/s3/files', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { prefix = '', maxKeys = 100, continuationToken } = req.query;
|
||||
const config = await getBackupConfig();
|
||||
|
||||
if (config.backup_destination_type !== 's3') {
|
||||
return res.status(400).json({ error: 'S3 backup not configured' });
|
||||
}
|
||||
|
||||
const s3Adapter = new S3StorageAdapter({
|
||||
endpoint: config.backup_s3_endpoint,
|
||||
bucket: config.backup_s3_bucket,
|
||||
accessKeyId: config.backup_s3_access_key,
|
||||
secretAccessKey: config.backup_s3_secret_key,
|
||||
region: config.backup_s3_region || 'us-east-1',
|
||||
forcePathStyle: config.backup_s3_force_path_style || false
|
||||
});
|
||||
|
||||
const result = await s3Adapter.list(prefix, {
|
||||
maxKeys: parseInt(maxKeys),
|
||||
continuationToken
|
||||
});
|
||||
|
||||
res.json({
|
||||
files: result.objects || [],
|
||||
directories: result.directories || [],
|
||||
isTruncated: result.isTruncated,
|
||||
nextContinuationToken: result.nextContinuationToken,
|
||||
prefix: prefix
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to list S3 files:', error);
|
||||
res.status(500).json({ error: 'Failed to list S3 files: ' + error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Clean up old S3 backups
|
||||
router.delete('/s3/cleanup', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { retentionDays = 30, dryRun = false } = req.body;
|
||||
const config = await getBackupConfig();
|
||||
|
||||
if (config.backup_destination_type !== 's3') {
|
||||
return res.status(400).json({ error: 'S3 backup not configured' });
|
||||
}
|
||||
|
||||
const s3Adapter = new S3StorageAdapter({
|
||||
endpoint: config.backup_s3_endpoint,
|
||||
bucket: config.backup_s3_bucket,
|
||||
accessKeyId: config.backup_s3_access_key,
|
||||
secretAccessKey: config.backup_s3_secret_key,
|
||||
region: config.backup_s3_region || 'us-east-1',
|
||||
forcePathStyle: config.backup_s3_force_path_style || false
|
||||
});
|
||||
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
|
||||
// List all backup files
|
||||
const backupFiles = await s3Adapter.list('backups/', { maxKeys: 1000 });
|
||||
const filesToDelete = [];
|
||||
let totalSize = 0;
|
||||
|
||||
for (const file of backupFiles.objects || []) {
|
||||
if (file.lastModified && new Date(file.lastModified) < cutoffDate) {
|
||||
filesToDelete.push(file.key);
|
||||
totalSize += file.size || 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
return res.json({
|
||||
wouldDelete: filesToDelete.length,
|
||||
totalSize: totalSize,
|
||||
files: filesToDelete.slice(0, 100), // Limit preview
|
||||
message: 'Dry run completed - no files were deleted'
|
||||
});
|
||||
}
|
||||
|
||||
// Delete files in batches
|
||||
const deleteResult = await s3Adapter.deleteMany(filesToDelete);
|
||||
const deletedCount = deleteResult.Deleted ? deleteResult.Deleted.length : 0;
|
||||
|
||||
// Also clean up database records
|
||||
await cleanupOldBackupRuns(retentionDays);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
deletedCount: deletedCount,
|
||||
totalSize: totalSize,
|
||||
message: `Cleaned up ${deletedCount} S3 backup files older than ${retentionDays} days`
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to cleanup S3 backups:', error);
|
||||
res.status(500).json({ error: 'Failed to cleanup S3 backups: ' + error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Test S3 upload functionality
|
||||
router.post('/s3/test-upload', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const config = await getBackupConfig();
|
||||
|
||||
if (config.backup_destination_type !== 's3') {
|
||||
return res.status(400).json({ error: 'S3 backup not configured' });
|
||||
}
|
||||
|
||||
const s3Adapter = new S3StorageAdapter({
|
||||
endpoint: config.backup_s3_endpoint,
|
||||
bucket: config.backup_s3_bucket,
|
||||
accessKeyId: config.backup_s3_access_key,
|
||||
secretAccessKey: config.backup_s3_secret_key,
|
||||
region: config.backup_s3_region || 'us-east-1',
|
||||
forcePathStyle: config.backup_s3_force_path_style || false
|
||||
});
|
||||
|
||||
// Create test content
|
||||
const testKey = `test/backup-test-${Date.now()}.txt`;
|
||||
const testContent = `PicPeak S3 backup test\nTimestamp: ${new Date().toISOString()}\nEndpoint: ${config.backup_s3_endpoint || 'AWS'}\nBucket: ${config.backup_s3_bucket}`;
|
||||
|
||||
// Test upload
|
||||
const uploadStart = Date.now();
|
||||
await s3Adapter.upload(testKey, Buffer.from(testContent));
|
||||
const uploadTime = Date.now() - uploadStart;
|
||||
|
||||
// Test download
|
||||
const downloadStart = Date.now();
|
||||
const downloadedContent = await s3Adapter.download(testKey);
|
||||
const downloadTime = Date.now() - downloadStart;
|
||||
|
||||
// Verify content
|
||||
const contentMatch = downloadedContent.toString() === testContent;
|
||||
|
||||
// Test deletion
|
||||
await s3Adapter.delete(testKey);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
testKey: testKey,
|
||||
uploadTime: uploadTime,
|
||||
downloadTime: downloadTime,
|
||||
contentMatch: contentMatch,
|
||||
message: 'S3 upload test completed successfully'
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('S3 upload test failed:', error);
|
||||
res.status(500).json({ error: 'S3 upload test failed: ' + error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Download entire backup
|
||||
router.get('/download/:backupId', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { backupId } = req.params;
|
||||
|
||||
// Get backup run details
|
||||
const backupRun = await db('backup_runs')
|
||||
.where('id', backupId)
|
||||
.first();
|
||||
|
||||
if (!backupRun) {
|
||||
return res.status(404).json({ error: 'Backup not found' });
|
||||
}
|
||||
|
||||
if (backupRun.status !== 'completed') {
|
||||
return res.status(400).json({ error: 'Backup is not completed' });
|
||||
}
|
||||
|
||||
const config = await getBackupConfig();
|
||||
|
||||
// Handle different backup types
|
||||
switch (config.backup_destination_type) {
|
||||
case 'local':
|
||||
// Stream local backup as zip
|
||||
const backupPath = path.join(config.backup_destination_path, `backup-${backupRun.id}`);
|
||||
const archive = archiver('zip', { zlib: { level: 9 } });
|
||||
|
||||
res.attachment(`picpeak-backup-${backupRun.id}.zip`);
|
||||
archive.pipe(res);
|
||||
|
||||
// Add backup directory contents
|
||||
archive.directory(backupPath, false);
|
||||
|
||||
// Add manifest if exists
|
||||
if (backupRun.manifest_path && await fs.access(backupRun.manifest_path).then(() => true).catch(() => false)) {
|
||||
archive.file(backupRun.manifest_path, { name: 'manifest.json' });
|
||||
}
|
||||
|
||||
await archive.finalize();
|
||||
break;
|
||||
|
||||
case 's3':
|
||||
// For S3, provide pre-signed URLs or stream files
|
||||
const s3Adapter = new S3StorageAdapter({
|
||||
endpoint: config.backup_s3_endpoint,
|
||||
bucket: config.backup_s3_bucket,
|
||||
accessKeyId: config.backup_s3_access_key,
|
||||
secretAccessKey: config.backup_s3_secret_key,
|
||||
region: config.backup_s3_region || 'us-east-1',
|
||||
forcePathStyle: config.backup_s3_force_path_style || false
|
||||
});
|
||||
|
||||
// List all files for this backup
|
||||
const prefix = `backups/${backupRun.id}/`;
|
||||
const files = await s3Adapter.list(prefix, { maxKeys: 1000 });
|
||||
|
||||
// Generate pre-signed URLs
|
||||
const urls = [];
|
||||
for (const file of files.objects || []) {
|
||||
const url = await s3Adapter.getSignedUrl('getObject', file.key, { expiresIn: 3600 }); // 1 hour
|
||||
urls.push({
|
||||
key: file.key,
|
||||
size: file.size,
|
||||
url: url
|
||||
});
|
||||
}
|
||||
|
||||
res.json({
|
||||
backupId: backupRun.id,
|
||||
type: 's3',
|
||||
files: urls,
|
||||
expiresIn: 3600,
|
||||
message: 'Use the provided URLs to download individual files'
|
||||
});
|
||||
break;
|
||||
|
||||
case 'rsync':
|
||||
return res.status(400).json({ error: 'Direct download not available for rsync backups' });
|
||||
|
||||
default:
|
||||
return res.status(400).json({ error: 'Unknown backup type' });
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to download backup:', error);
|
||||
res.status(500).json({ error: 'Failed to download backup: ' + error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get current file checksums
|
||||
router.get('/checksums', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { path: targetPath = '', recursive = true } = req.query;
|
||||
const checksums = {};
|
||||
|
||||
// Get storage path
|
||||
const storagePath = process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
||||
const basePath = targetPath ? path.join(storagePath, targetPath) : storagePath;
|
||||
|
||||
// Calculate checksums for files
|
||||
async function calculateDirChecksums(dirPath, relative = '') {
|
||||
try {
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
const relativePath = path.join(relative, entry.name);
|
||||
|
||||
if (entry.isDirectory() && recursive) {
|
||||
await calculateDirChecksums(fullPath, relativePath);
|
||||
} else if (entry.isFile()) {
|
||||
const hash = crypto.createHash('sha256');
|
||||
const stream = require('fs').createReadStream(fullPath);
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
stream.on('data', data => hash.update(data));
|
||||
stream.on('end', () => {
|
||||
checksums[relativePath] = {
|
||||
checksum: hash.digest('hex'),
|
||||
size: entry.size,
|
||||
modified: entry.mtime
|
||||
};
|
||||
resolve();
|
||||
});
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(`Failed to calculate checksums for ${dirPath}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
await calculateDirChecksums(basePath);
|
||||
|
||||
// Also get database checksums from backup_file_states
|
||||
const dbChecksums = await db('backup_file_states')
|
||||
.select('file_path', 'checksum', 'size_bytes', 'last_modified');
|
||||
|
||||
res.json({
|
||||
currentChecksums: checksums,
|
||||
totalFiles: Object.keys(checksums).length,
|
||||
databaseChecksums: dbChecksums.reduce((acc, row) => {
|
||||
acc[row.file_path] = {
|
||||
checksum: row.checksum,
|
||||
size: row.size_bytes,
|
||||
modified: row.last_modified
|
||||
};
|
||||
return acc;
|
||||
}, {}),
|
||||
path: targetPath || '/'
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get file checksums:', error);
|
||||
res.status(500).json({ error: 'Failed to get file checksums: ' + error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Estimate backup size before running
|
||||
router.post('/estimate', adminAuth, async (req, res) => {
|
||||
try {
|
||||
const { includeArchived = true } = req.body;
|
||||
|
||||
// Get storage path
|
||||
const storagePath = process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
||||
let totalSize = 0;
|
||||
let fileCount = 0;
|
||||
const breakdown = {};
|
||||
|
||||
// Estimate size for each directory
|
||||
async function estimateDir(dirPath, category) {
|
||||
let dirSize = 0;
|
||||
let dirCount = 0;
|
||||
|
||||
try {
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
const subResult = await estimateDir(fullPath, category);
|
||||
dirSize += subResult.size;
|
||||
dirCount += subResult.count;
|
||||
} else if (entry.isFile()) {
|
||||
const stats = await fs.stat(fullPath);
|
||||
dirSize += stats.size;
|
||||
dirCount++;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
logger.error(`Failed to estimate ${dirPath}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return { size: dirSize, count: dirCount };
|
||||
}
|
||||
|
||||
// Estimate each category
|
||||
const categories = [
|
||||
{ path: 'events/active', name: 'Active Events' },
|
||||
{ path: 'thumbnails', name: 'Thumbnails' },
|
||||
{ path: 'uploads', name: 'Uploads' }
|
||||
];
|
||||
|
||||
if (includeArchived) {
|
||||
categories.push({ path: 'events/archived', name: 'Archived Events' });
|
||||
}
|
||||
|
||||
for (const category of categories) {
|
||||
const result = await estimateDir(path.join(storagePath, category.path), category.name);
|
||||
breakdown[category.name] = {
|
||||
size: result.size,
|
||||
sizeFormatted: formatBytes(result.size),
|
||||
fileCount: result.count
|
||||
};
|
||||
totalSize += result.size;
|
||||
fileCount += result.count;
|
||||
}
|
||||
|
||||
// Estimate database size
|
||||
const dbPath = process.env.DB_TYPE === 'postgresql'
|
||||
? null
|
||||
: path.join(__dirname, '../../database.sqlite');
|
||||
|
||||
if (dbPath) {
|
||||
try {
|
||||
const dbStats = await fs.stat(dbPath);
|
||||
breakdown['Database'] = {
|
||||
size: dbStats.size,
|
||||
sizeFormatted: formatBytes(dbStats.size),
|
||||
fileCount: 1
|
||||
};
|
||||
totalSize += dbStats.size;
|
||||
fileCount += 1;
|
||||
} catch (error) {
|
||||
logger.error('Failed to get database size:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Estimate compression ratio (typically 20-40% for mixed media)
|
||||
const estimatedCompressedSize = Math.round(totalSize * 0.7);
|
||||
|
||||
res.json({
|
||||
totalSize: totalSize,
|
||||
totalSizeFormatted: formatBytes(totalSize),
|
||||
estimatedCompressedSize: estimatedCompressedSize,
|
||||
estimatedCompressedSizeFormatted: formatBytes(estimatedCompressedSize),
|
||||
fileCount: fileCount,
|
||||
breakdown: breakdown,
|
||||
includeArchived: includeArchived,
|
||||
estimatedDuration: Math.max(60, Math.round(totalSize / (50 * 1024 * 1024))), // Estimate 50MB/s
|
||||
warnings: totalSize > 10 * 1024 * 1024 * 1024 ? ['Backup size exceeds 10GB, may take significant time'] : []
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to estimate backup size:', error);
|
||||
res.status(500).json({ error: 'Failed to estimate backup size: ' + error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Helper function to format bytes
|
||||
function formatBytes(bytes, decimals = 2) {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const k = 1024;
|
||||
const dm = decimals < 0 ? 0 : decimals;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
// Helper function to get backup configuration
|
||||
async function getBackupConfig() {
|
||||
try {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
const config = {};
|
||||
settings.forEach(setting => {
|
||||
try {
|
||||
config[setting.setting_key] = JSON.parse(setting.setting_value);
|
||||
} catch (e) {
|
||||
config[setting.setting_key] = setting.setting_value;
|
||||
}
|
||||
});
|
||||
|
||||
return config;
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup configuration:', error);
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to validate manifest data
|
||||
async function validateManifestData(manifestData) {
|
||||
try {
|
||||
// Check required fields
|
||||
const requiredFields = ['version', 'backupId', 'timestamp', 'files'];
|
||||
const missingFields = requiredFields.filter(field => !manifestData[field]);
|
||||
|
||||
if (missingFields.length > 0) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Missing required fields: ${missingFields.join(', ')}`,
|
||||
details: { missingFields }
|
||||
};
|
||||
}
|
||||
|
||||
// Validate version
|
||||
if (manifestData.version !== '1.0') {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Unsupported manifest version: ${manifestData.version}`,
|
||||
details: { version: manifestData.version }
|
||||
};
|
||||
}
|
||||
|
||||
// Validate files array
|
||||
if (!Array.isArray(manifestData.files)) {
|
||||
return {
|
||||
valid: false,
|
||||
error: 'Files must be an array',
|
||||
details: { filesType: typeof manifestData.files }
|
||||
};
|
||||
}
|
||||
|
||||
// Validate each file entry
|
||||
const invalidFiles = [];
|
||||
for (let i = 0; i < manifestData.files.length; i++) {
|
||||
const file = manifestData.files[i];
|
||||
if (!file.path || !file.checksum || typeof file.size !== 'number') {
|
||||
invalidFiles.push({ index: i, file });
|
||||
}
|
||||
}
|
||||
|
||||
if (invalidFiles.length > 0) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Invalid file entries: ${invalidFiles.length}`,
|
||||
details: { invalidFiles: invalidFiles.slice(0, 10) } // Limit to first 10
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
valid: true,
|
||||
details: {
|
||||
version: manifestData.version,
|
||||
backupId: manifestData.backupId,
|
||||
timestamp: manifestData.timestamp,
|
||||
fileCount: manifestData.files.length,
|
||||
totalSize: manifestData.files.reduce((sum, f) => sum + (f.size || 0), 0)
|
||||
}
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Validation error: ${error.message}`,
|
||||
details: { error: error.message }
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,273 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { adminAuth } = require('../middleware/auth');
|
||||
const { databaseBackupService } = require('../services/databaseBackup');
|
||||
const { db } = require('../database/db');
|
||||
const logger = require('../utils/logger');
|
||||
|
||||
// All routes require admin authentication
|
||||
router.use(adminAuth);
|
||||
|
||||
/**
|
||||
* Get database backup status and configuration
|
||||
*/
|
||||
router.get('/status', async (req, res) => {
|
||||
try {
|
||||
// Get configuration
|
||||
const config = await databaseBackupService.getBackupConfig();
|
||||
|
||||
// Get recent backup history
|
||||
const history = await databaseBackupService.getBackupHistory(10);
|
||||
|
||||
// Get current progress if running
|
||||
const progress = databaseBackupService.getProgress();
|
||||
|
||||
// Calculate health status
|
||||
const lastBackup = history[0];
|
||||
const isHealthy = lastBackup && lastBackup.status === 'completed' &&
|
||||
new Date(lastBackup.completed_at) > new Date(Date.now() - 48 * 60 * 60 * 1000); // Within 48 hours
|
||||
|
||||
res.json({
|
||||
config,
|
||||
isRunning: databaseBackupService.isRunning,
|
||||
isHealthy,
|
||||
currentProgress: progress,
|
||||
lastBackup,
|
||||
recentBackups: history,
|
||||
dbType: databaseBackupService.dbType
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get database backup status:', error);
|
||||
res.status(500).json({ error: 'Failed to get backup status' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Update database backup configuration
|
||||
*/
|
||||
router.put('/config', async (req, res) => {
|
||||
try {
|
||||
const allowedSettings = [
|
||||
'database_backup_enabled',
|
||||
'database_backup_schedule',
|
||||
'database_backup_destination_path',
|
||||
'database_backup_compress',
|
||||
'database_backup_validate_integrity',
|
||||
'database_backup_include_checksums',
|
||||
'database_backup_retention_days',
|
||||
'database_backup_email_on_failure',
|
||||
'database_backup_email_on_success'
|
||||
];
|
||||
|
||||
const updates = [];
|
||||
|
||||
for (const [key, value] of Object.entries(req.body)) {
|
||||
if (allowedSettings.includes(key)) {
|
||||
// Check if setting exists
|
||||
const existing = await db('app_settings')
|
||||
.where('setting_key', key)
|
||||
.first();
|
||||
|
||||
if (existing) {
|
||||
await db('app_settings')
|
||||
.where('setting_key', key)
|
||||
.update({
|
||||
setting_value: JSON.stringify(value),
|
||||
updated_at: new Date()
|
||||
});
|
||||
} else {
|
||||
await db('app_settings').insert({
|
||||
setting_key: key,
|
||||
setting_value: JSON.stringify(value),
|
||||
setting_type: 'database_backup'
|
||||
});
|
||||
}
|
||||
|
||||
updates.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
// Restart scheduled backups if enabled state changed
|
||||
if (updates.includes('database_backup_enabled') || updates.includes('database_backup_schedule')) {
|
||||
const { startScheduledBackups, stopScheduledBackups } = require('../services/databaseBackup');
|
||||
stopScheduledBackups();
|
||||
await startScheduledBackups();
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
updatedSettings: updates,
|
||||
message: 'Database backup configuration updated successfully'
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to update database backup config:', error);
|
||||
res.status(500).json({ error: 'Failed to update configuration' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Trigger manual database backup
|
||||
*/
|
||||
router.post('/backup', async (req, res) => {
|
||||
try {
|
||||
if (databaseBackupService.isRunning) {
|
||||
return res.status(409).json({ error: 'Backup already in progress' });
|
||||
}
|
||||
|
||||
// Start backup asynchronously
|
||||
res.json({
|
||||
success: true,
|
||||
message: 'Database backup started',
|
||||
trackingUrl: '/api/admin/database-backup/progress'
|
||||
});
|
||||
|
||||
// Run backup in background
|
||||
databaseBackupService.backup(req.body).catch(error => {
|
||||
logger.error('Manual database backup failed:', error);
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to start database backup:', error);
|
||||
res.status(500).json({ error: 'Failed to start backup' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Get current backup progress
|
||||
*/
|
||||
router.get('/progress', async (req, res) => {
|
||||
try {
|
||||
const progress = databaseBackupService.getProgress();
|
||||
|
||||
res.json({
|
||||
isRunning: databaseBackupService.isRunning,
|
||||
progress
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup progress:', error);
|
||||
res.status(500).json({ error: 'Failed to get progress' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Get backup history with pagination
|
||||
*/
|
||||
router.get('/history', async (req, res) => {
|
||||
try {
|
||||
const page = parseInt(req.query.page) || 1;
|
||||
const limit = parseInt(req.query.limit) || 20;
|
||||
const offset = (page - 1) * limit;
|
||||
|
||||
const [backups, totalCount] = await Promise.all([
|
||||
db('database_backup_runs')
|
||||
.orderBy('started_at', 'desc')
|
||||
.limit(limit)
|
||||
.offset(offset),
|
||||
db('database_backup_runs').count('* as count').first()
|
||||
]);
|
||||
|
||||
res.json({
|
||||
backups,
|
||||
pagination: {
|
||||
page,
|
||||
limit,
|
||||
total: totalCount.count,
|
||||
pages: Math.ceil(totalCount.count / limit)
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup history:', error);
|
||||
res.status(500).json({ error: 'Failed to get history' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Delete old backup files
|
||||
*/
|
||||
router.delete('/cleanup', async (req, res) => {
|
||||
try {
|
||||
const { retentionDays = 30 } = req.body;
|
||||
|
||||
await databaseBackupService.cleanupOldBackups(retentionDays);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
message: `Cleaned up backups older than ${retentionDays} days`
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to cleanup old backups:', error);
|
||||
res.status(500).json({ error: 'Failed to cleanup backups' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Test database backup configuration
|
||||
*/
|
||||
router.post('/test', async (req, res) => {
|
||||
try {
|
||||
const config = await databaseBackupService.getBackupConfig();
|
||||
|
||||
// Test database connection
|
||||
const testResults = {
|
||||
databaseConnection: false,
|
||||
destinationWritable: false,
|
||||
compressionAvailable: true,
|
||||
estimatedSize: null
|
||||
};
|
||||
|
||||
// Test database connection
|
||||
try {
|
||||
await db.raw('SELECT 1');
|
||||
testResults.databaseConnection = true;
|
||||
} catch (error) {
|
||||
testResults.databaseConnectionError = error.message;
|
||||
}
|
||||
|
||||
// Test destination path
|
||||
if (config.destinationPath) {
|
||||
try {
|
||||
const fs = require('fs').promises;
|
||||
const testFile = `${config.destinationPath}/.test-${Date.now()}`;
|
||||
await fs.writeFile(testFile, 'test');
|
||||
await fs.unlink(testFile);
|
||||
testResults.destinationWritable = true;
|
||||
} catch (error) {
|
||||
testResults.destinationError = error.message;
|
||||
}
|
||||
}
|
||||
|
||||
// Estimate database size
|
||||
try {
|
||||
testResults.estimatedSize = await databaseBackupService.getDatabaseSize();
|
||||
} catch (error) {
|
||||
testResults.sizeError = error.message;
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: testResults.databaseConnection && testResults.destinationWritable,
|
||||
results: testResults
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to test backup configuration:', error);
|
||||
res.status(500).json({ error: 'Failed to test configuration' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Get table checksums
|
||||
*/
|
||||
router.get('/checksums', async (req, res) => {
|
||||
try {
|
||||
const checksums = await databaseBackupService.getTableChecksums();
|
||||
|
||||
res.json({
|
||||
checksums,
|
||||
tableCount: Object.keys(checksums).length,
|
||||
totalRows: Object.values(checksums).reduce((sum, table) => sum + table.rowCount, 0)
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get table checksums:', error);
|
||||
res.status(500).json({ error: 'Failed to get checksums' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,461 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { restoreService } = require('../services/restoreService');
|
||||
const { adminAuth } = require('../middleware/auth');
|
||||
const { body, query, validationResult } = require('express-validator');
|
||||
const logger = require('../utils/logger');
|
||||
const { db } = require('../database/db');
|
||||
const path = require('path');
|
||||
const fs = require('fs').promises;
|
||||
|
||||
/**
|
||||
* Admin routes for restore operations
|
||||
* All routes require admin authentication
|
||||
*/
|
||||
|
||||
// Apply admin authentication to all routes
|
||||
router.use(adminAuth);
|
||||
|
||||
/**
|
||||
* Get restore service status and history
|
||||
*/
|
||||
router.get('/status', async (req, res) => {
|
||||
try {
|
||||
const limit = parseInt(req.query.limit) || 10;
|
||||
const history = await restoreService.getRestoreHistory(limit);
|
||||
|
||||
const status = {
|
||||
isRunning: restoreService.isRunning,
|
||||
currentProgress: restoreService.getProgress(),
|
||||
history: history,
|
||||
settings: await getRestoreSettings()
|
||||
};
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: status
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get restore status:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: 'Failed to get restore status'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Validate restore request
|
||||
*/
|
||||
router.post('/validate', [
|
||||
body('source').notEmpty().withMessage('Backup source is required'),
|
||||
body('manifestPath').notEmpty().withMessage('Manifest path is required'),
|
||||
body('restoreType').isIn(['full', 'database', 'files', 'selective']).withMessage('Invalid restore type'),
|
||||
body('selectedItems').optional().isArray(),
|
||||
body('s3Config').optional().isObject()
|
||||
], async (req, res) => {
|
||||
const errors = validationResult(req);
|
||||
if (!errors.isEmpty()) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
errors: errors.array()
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
// Perform dry run validation
|
||||
const result = await restoreService.restore({
|
||||
...req.body,
|
||||
dryRun: true,
|
||||
force: false
|
||||
});
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
validation: result.validation,
|
||||
spaceCheck: result.spaceCheck,
|
||||
logs: result.logs
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Restore validation failed:', error);
|
||||
res.status(400).json({
|
||||
success: false,
|
||||
error: error.message,
|
||||
logs: restoreService.restoreLog
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Start restore operation
|
||||
*/
|
||||
router.post('/start', [
|
||||
body('source').notEmpty().withMessage('Backup source is required'),
|
||||
body('manifestPath').notEmpty().withMessage('Manifest path is required'),
|
||||
body('restoreType').isIn(['full', 'database', 'files', 'selective']).withMessage('Invalid restore type'),
|
||||
body('selectedItems').optional().isArray(),
|
||||
body('skipPreBackup').optional().isBoolean(),
|
||||
body('force').optional().isBoolean(),
|
||||
body('s3Config').optional().isObject()
|
||||
], async (req, res) => {
|
||||
const errors = validationResult(req);
|
||||
if (!errors.isEmpty()) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
errors: errors.array()
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
// Check if restore is already running
|
||||
if (restoreService.isRunning) {
|
||||
return res.status(409).json({
|
||||
success: false,
|
||||
error: 'Restore operation already in progress'
|
||||
});
|
||||
}
|
||||
|
||||
// Check permissions for dangerous options
|
||||
const settings = await getRestoreSettings();
|
||||
if (req.body.force && !settings.restore_allow_force) {
|
||||
return res.status(403).json({
|
||||
success: false,
|
||||
error: 'Force restore is not allowed by system settings'
|
||||
});
|
||||
}
|
||||
|
||||
if (req.body.skipPreBackup && settings.restore_require_pre_backup) {
|
||||
return res.status(403).json({
|
||||
success: false,
|
||||
error: 'Skipping pre-restore backup is not allowed by system settings'
|
||||
});
|
||||
}
|
||||
|
||||
// Log restore attempt
|
||||
logger.warn('Restore operation started', {
|
||||
user: req.user.email,
|
||||
ip: req.ip,
|
||||
restoreType: req.body.restoreType,
|
||||
source: req.body.source
|
||||
});
|
||||
|
||||
// Start restore in background
|
||||
restoreService.restore({
|
||||
...req.body,
|
||||
dryRun: false,
|
||||
operator: {
|
||||
type: 'manual',
|
||||
userId: req.user.id,
|
||||
ip: req.ip
|
||||
}
|
||||
}).catch(error => {
|
||||
logger.error('Background restore failed:', error);
|
||||
});
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
message: 'Restore operation started'
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to start restore:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Get current restore progress
|
||||
*/
|
||||
router.get('/progress', async (req, res) => {
|
||||
try {
|
||||
const progress = restoreService.getProgress();
|
||||
const logs = restoreService.restoreLog.slice(-50); // Last 50 log entries
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
isRunning: restoreService.isRunning,
|
||||
progress: progress,
|
||||
logs: logs
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get restore progress:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: 'Failed to get restore progress'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Get restore run details
|
||||
*/
|
||||
router.get('/run/:id', async (req, res) => {
|
||||
try {
|
||||
const run = await db('restore_runs')
|
||||
.where('id', req.params.id)
|
||||
.first();
|
||||
|
||||
if (!run) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: 'Restore run not found'
|
||||
});
|
||||
}
|
||||
|
||||
// Parse JSON fields
|
||||
if (run.statistics) run.statistics = JSON.parse(run.statistics);
|
||||
if (run.restore_log) run.restore_log = JSON.parse(run.restore_log);
|
||||
if (run.metadata) run.metadata = JSON.parse(run.metadata);
|
||||
|
||||
// Get validation results
|
||||
const validations = await db('restore_validation_results')
|
||||
.where('restore_run_id', run.id)
|
||||
.select('*');
|
||||
|
||||
validations.forEach(v => {
|
||||
if (v.errors) v.errors = JSON.parse(v.errors);
|
||||
if (v.warnings) v.warnings = JSON.parse(v.warnings);
|
||||
if (v.checksums) v.checksums = JSON.parse(v.checksums);
|
||||
});
|
||||
|
||||
// Get file operations summary
|
||||
const fileOps = await db('restore_file_operations')
|
||||
.where('restore_run_id', run.id)
|
||||
.select('status', db.raw('COUNT(*) as count'))
|
||||
.groupBy('status');
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
run: run,
|
||||
validations: validations,
|
||||
fileOperations: fileOps
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get restore run details:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: 'Failed to get restore run details'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Get restore run report
|
||||
*/
|
||||
router.get('/run/:id/report', async (req, res) => {
|
||||
try {
|
||||
const run = await db('restore_runs')
|
||||
.where('id', req.params.id)
|
||||
.first();
|
||||
|
||||
if (!run) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: 'Restore run not found'
|
||||
});
|
||||
}
|
||||
|
||||
// Parse JSON fields
|
||||
if (run.statistics) run.statistics = JSON.parse(run.statistics);
|
||||
if (run.restore_log) run.restore_log = JSON.parse(run.restore_log);
|
||||
|
||||
// Generate report
|
||||
const report = restoreService.generateRestoreReport({
|
||||
success: run.status === 'completed',
|
||||
duration: run.duration_seconds,
|
||||
dryRun: run.is_dry_run,
|
||||
result: run.statistics,
|
||||
logs: run.restore_log || []
|
||||
});
|
||||
|
||||
res.type('text/plain').send(report);
|
||||
} catch (error) {
|
||||
logger.error('Failed to generate restore report:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: 'Failed to generate restore report'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* List available backups for restore
|
||||
*/
|
||||
router.get('/available-backups', async (req, res) => {
|
||||
try {
|
||||
const backups = [];
|
||||
|
||||
// Get local file backups
|
||||
const backupConfig = await getBackupConfig();
|
||||
if (backupConfig.backup_destination_type === 'local' && backupConfig.backup_destination_path) {
|
||||
try {
|
||||
const files = await fs.readdir(backupConfig.backup_destination_path);
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.json') || file.endsWith('.yaml')) {
|
||||
const filePath = path.join(backupConfig.backup_destination_path, file);
|
||||
const stats = await fs.stat(filePath);
|
||||
backups.push({
|
||||
type: 'local',
|
||||
name: file,
|
||||
path: filePath,
|
||||
size: stats.size,
|
||||
modified: stats.mtime
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.warn('Failed to list local backups:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Get database backups from backup_runs table
|
||||
const backupRuns = await db('backup_runs')
|
||||
.where('status', 'completed')
|
||||
.whereNotNull('manifest_path')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.limit(20);
|
||||
|
||||
for (const run of backupRuns) {
|
||||
backups.push({
|
||||
type: run.manifest_path.startsWith('s3://') ? 's3' : 'local',
|
||||
name: `Backup ${run.completed_at}`,
|
||||
path: run.manifest_path,
|
||||
manifestId: run.manifest_id,
|
||||
size: run.total_size_bytes,
|
||||
filesCount: run.files_backed_up,
|
||||
duration: run.duration_seconds,
|
||||
completed: run.completed_at
|
||||
});
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: backups
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to list available backups:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: 'Failed to list available backups'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Get restore settings
|
||||
*/
|
||||
router.get('/settings', async (req, res) => {
|
||||
try {
|
||||
const settings = await getRestoreSettings();
|
||||
res.json({
|
||||
success: true,
|
||||
data: settings
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to get restore settings:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: 'Failed to get restore settings'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Update restore settings
|
||||
*/
|
||||
router.put('/settings', [
|
||||
body('restore_allow_force').optional().isBoolean(),
|
||||
body('restore_require_pre_backup').optional().isBoolean(),
|
||||
body('restore_max_file_size_mb').optional().isInt({ min: 1 }),
|
||||
body('restore_verify_checksums').optional().isBoolean(),
|
||||
body('restore_email_on_completion').optional().isBoolean(),
|
||||
body('restore_retention_days').optional().isInt({ min: 1 })
|
||||
], async (req, res) => {
|
||||
const errors = validationResult(req);
|
||||
if (!errors.isEmpty()) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
errors: errors.array()
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
// Update settings
|
||||
for (const [key, value] of Object.entries(req.body)) {
|
||||
await db('app_settings')
|
||||
.where('setting_key', key)
|
||||
.where('setting_type', 'restore')
|
||||
.update({
|
||||
setting_value: typeof value === 'boolean' ? (value ? '1' : '0') : value.toString(),
|
||||
updated_at: db.fn.now()
|
||||
});
|
||||
}
|
||||
|
||||
logger.info('Restore settings updated', {
|
||||
user: req.user.email,
|
||||
settings: req.body
|
||||
});
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
message: 'Settings updated successfully'
|
||||
});
|
||||
} catch (error) {
|
||||
logger.error('Failed to update restore settings:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: 'Failed to update settings'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* Helper function to get restore settings
|
||||
*/
|
||||
async function getRestoreSettings() {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'restore')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
const result = {};
|
||||
settings.forEach(setting => {
|
||||
// Convert boolean strings to actual booleans
|
||||
if (setting.setting_value === '1' || setting.setting_value === '0') {
|
||||
result[setting.setting_key] = setting.setting_value === '1';
|
||||
} else {
|
||||
result[setting.setting_key] = setting.setting_value;
|
||||
}
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to get backup configuration
|
||||
*/
|
||||
async function getBackupConfig() {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
const config = {};
|
||||
settings.forEach(setting => {
|
||||
try {
|
||||
config[setting.setting_key] = JSON.parse(setting.setting_value);
|
||||
} catch (e) {
|
||||
config[setting.setting_key] = setting.setting_value;
|
||||
}
|
||||
});
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,251 @@
|
||||
const { DatabaseBackupService } = require('../databaseBackup');
|
||||
const { db } = require('../../database/db');
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
|
||||
// Mock dependencies
|
||||
jest.mock('../../database/db');
|
||||
jest.mock('../../utils/logger');
|
||||
jest.mock('../emailProcessor');
|
||||
jest.mock('child_process');
|
||||
|
||||
describe('DatabaseBackupService', () => {
|
||||
let service;
|
||||
let mockExecAsync;
|
||||
|
||||
beforeEach(() => {
|
||||
service = new DatabaseBackupService();
|
||||
mockExecAsync = jest.fn();
|
||||
|
||||
// Reset mocks
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Mock execAsync
|
||||
const childProcess = require('child_process');
|
||||
childProcess.exec = jest.fn((cmd, opts, callback) => {
|
||||
if (callback) {
|
||||
callback(null, { stdout: 'ok', stderr: '' });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// Cleanup test files
|
||||
try {
|
||||
await fs.rmdir('/tmp/test-backup', { recursive: true });
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
});
|
||||
|
||||
describe('calculateChecksum', () => {
|
||||
it('should calculate SHA256 checksum of a file', async () => {
|
||||
const testFile = '/tmp/test-checksum.txt';
|
||||
const testContent = 'Hello, World!';
|
||||
await fs.writeFile(testFile, testContent);
|
||||
|
||||
const checksum = await service.calculateChecksum(testFile);
|
||||
|
||||
// Expected checksum for "Hello, World!"
|
||||
const expectedChecksum = crypto
|
||||
.createHash('sha256')
|
||||
.update(testContent)
|
||||
.digest('hex');
|
||||
|
||||
expect(checksum).toBe(expectedChecksum);
|
||||
|
||||
await fs.unlink(testFile);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getTableChecksums', () => {
|
||||
it('should get checksums for all tables', async () => {
|
||||
// Mock getTables
|
||||
service.getTables = jest.fn().mockResolvedValue(['events', 'photos']);
|
||||
|
||||
// Mock SQLite response
|
||||
db.raw = jest.fn()
|
||||
.mockResolvedValueOnce([{ row_count: 10, data_sum: 1000 }])
|
||||
.mockResolvedValueOnce([{ row_count: 20, data_sum: 2000 }]);
|
||||
|
||||
const checksums = await service.getTableChecksums();
|
||||
|
||||
expect(checksums).toHaveProperty('events');
|
||||
expect(checksums).toHaveProperty('photos');
|
||||
expect(checksums.events.rowCount).toBe(10);
|
||||
expect(checksums.photos.rowCount).toBe(20);
|
||||
expect(checksums.events.checksum).toBeDefined();
|
||||
expect(checksums.photos.checksum).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getTables', () => {
|
||||
it('should get list of tables for SQLite', async () => {
|
||||
service.dbType = 'sqlite';
|
||||
|
||||
db.raw = jest.fn().mockResolvedValue([
|
||||
{ name: 'events' },
|
||||
{ name: 'photos' },
|
||||
{ name: 'admin_users' }
|
||||
]);
|
||||
|
||||
const tables = await service.getTables();
|
||||
|
||||
expect(tables).toEqual(['events', 'photos', 'admin_users']);
|
||||
expect(db.raw).toHaveBeenCalledWith(expect.stringContaining('sqlite_master'));
|
||||
});
|
||||
|
||||
it('should get list of tables for PostgreSQL', async () => {
|
||||
service.dbType = 'postgresql';
|
||||
|
||||
db.raw = jest.fn().mockResolvedValue({
|
||||
rows: [
|
||||
{ table_name: 'events' },
|
||||
{ table_name: 'photos' },
|
||||
{ table_name: 'admin_users' }
|
||||
]
|
||||
});
|
||||
|
||||
const tables = await service.getTables();
|
||||
|
||||
expect(tables).toEqual(['events', 'photos', 'admin_users']);
|
||||
expect(db.raw).toHaveBeenCalledWith(expect.stringContaining('information_schema.tables'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('getDatabaseSize', () => {
|
||||
it('should get database size for SQLite', async () => {
|
||||
service.dbType = 'sqlite';
|
||||
const mockSize = 1024 * 1024 * 10; // 10MB
|
||||
|
||||
// Mock fs.stat
|
||||
const originalStat = fs.stat;
|
||||
fs.stat = jest.fn().mockResolvedValue({ size: mockSize });
|
||||
|
||||
const size = await service.getDatabaseSize();
|
||||
|
||||
expect(size).toBe(mockSize);
|
||||
|
||||
fs.stat = originalStat;
|
||||
});
|
||||
|
||||
it('should get database size for PostgreSQL', async () => {
|
||||
service.dbType = 'postgresql';
|
||||
const mockSize = 1024 * 1024 * 100; // 100MB
|
||||
|
||||
db.raw = jest.fn().mockResolvedValue({
|
||||
rows: [{ size: mockSize.toString() }]
|
||||
});
|
||||
|
||||
const size = await service.getDatabaseSize();
|
||||
|
||||
expect(size).toBe(mockSize);
|
||||
expect(db.raw).toHaveBeenCalledWith(expect.stringContaining('pg_database_size'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('compressFile', () => {
|
||||
it('should compress file and return stats', async () => {
|
||||
const testFile = '/tmp/test-compress.txt';
|
||||
const compressedFile = '/tmp/test-compress.txt.gz';
|
||||
|
||||
// Create test file with repetitive content (compresses well)
|
||||
const testContent = 'Hello, World! '.repeat(1000);
|
||||
await fs.writeFile(testFile, testContent);
|
||||
|
||||
const stats = await service.compressFile(testFile, compressedFile);
|
||||
|
||||
expect(stats.originalSize).toBeGreaterThan(0);
|
||||
expect(stats.compressedSize).toBeGreaterThan(0);
|
||||
expect(stats.compressedSize).toBeLessThan(stats.originalSize);
|
||||
expect(parseFloat(stats.compressionRatio)).toBeGreaterThan(0);
|
||||
|
||||
// Cleanup
|
||||
await fs.unlink(testFile);
|
||||
await fs.unlink(compressedFile);
|
||||
});
|
||||
});
|
||||
|
||||
describe('backup configuration', () => {
|
||||
it('should get backup configuration from database', async () => {
|
||||
const mockConfig = [
|
||||
{ setting_key: 'database_backup_enabled', setting_value: 'true' },
|
||||
{ setting_key: 'database_backup_compress', setting_value: 'true' },
|
||||
{ setting_key: 'database_backup_retention_days', setting_value: '30' }
|
||||
];
|
||||
|
||||
db.mockReturnValue({
|
||||
where: jest.fn().mockReturnThis(),
|
||||
select: jest.fn().mockResolvedValue(mockConfig)
|
||||
});
|
||||
|
||||
const config = await service.getBackupConfig();
|
||||
|
||||
expect(config.database_backup_enabled).toBe(true);
|
||||
expect(config.database_backup_compress).toBe(true);
|
||||
expect(config.database_backup_retention_days).toBe(30);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanupOldBackups', () => {
|
||||
it('should delete old backup files and records', async () => {
|
||||
const oldBackups = [
|
||||
{ id: 1, file_path: '/backup/old1.sql.gz' },
|
||||
{ id: 2, file_path: '/backup/old2.sql.gz' }
|
||||
];
|
||||
|
||||
db.mockReturnValue({
|
||||
where: jest.fn().mockReturnThis(),
|
||||
select: jest.fn().mockResolvedValue(oldBackups),
|
||||
delete: jest.fn().mockResolvedValue(1)
|
||||
});
|
||||
|
||||
// Mock fs.unlink
|
||||
fs.unlink = jest.fn().mockResolvedValue(undefined);
|
||||
|
||||
await service.cleanupOldBackups(30);
|
||||
|
||||
expect(fs.unlink).toHaveBeenCalledTimes(2);
|
||||
expect(fs.unlink).toHaveBeenCalledWith('/backup/old1.sql.gz');
|
||||
expect(fs.unlink).toHaveBeenCalledWith('/backup/old2.sql.gz');
|
||||
});
|
||||
});
|
||||
|
||||
describe('progress tracking', () => {
|
||||
it('should update and retrieve progress', () => {
|
||||
expect(service.getProgress()).toBeNull();
|
||||
|
||||
service.updateProgress('Testing...', { step: 1 });
|
||||
|
||||
const progress = service.getProgress();
|
||||
expect(progress.message).toBe('Testing...');
|
||||
expect(progress.details.step).toBe(1);
|
||||
expect(progress.timestamp).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('backup history', () => {
|
||||
it('should retrieve backup history', async () => {
|
||||
const mockHistory = [
|
||||
{
|
||||
id: 1,
|
||||
started_at: new Date(),
|
||||
completed_at: new Date(),
|
||||
status: 'completed',
|
||||
file_size_bytes: 1024000
|
||||
}
|
||||
];
|
||||
|
||||
db.mockReturnValue({
|
||||
orderBy: jest.fn().mockReturnThis(),
|
||||
limit: jest.fn().mockResolvedValue(mockHistory)
|
||||
});
|
||||
|
||||
const history = await service.getBackupHistory(10);
|
||||
|
||||
expect(history).toEqual(mockHistory);
|
||||
expect(history.length).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,501 @@
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const yaml = require('js-yaml');
|
||||
const os = require('os');
|
||||
const { db } = require('../database/db');
|
||||
const logger = require('../utils/logger');
|
||||
|
||||
/**
|
||||
* Backup Manifest Generator
|
||||
*
|
||||
* Generates comprehensive manifests for backups including:
|
||||
* - Version information (app, node, OS)
|
||||
* - File listings with metadata and checksums
|
||||
* - Database information
|
||||
* - System state at backup time
|
||||
* - Support for both JSON and YAML formats
|
||||
* - Incremental backup support with parent references
|
||||
*/
|
||||
|
||||
class BackupManifestGenerator {
|
||||
constructor() {
|
||||
this.appVersion = require('../../package.json').version;
|
||||
this.nodeVersion = process.version;
|
||||
this.platform = process.platform;
|
||||
this.osRelease = os.release();
|
||||
this.hostname = os.hostname();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a comprehensive backup manifest
|
||||
* @param {Object} options - Manifest generation options
|
||||
* @param {string} options.backupType - 'full' or 'incremental'
|
||||
* @param {string} options.backupPath - Path to the backup directory
|
||||
* @param {Array} options.files - Array of backed up files with metadata
|
||||
* @param {Object} options.databaseInfo - Database backup information
|
||||
* @param {string} options.parentBackupId - For incremental backups, reference to parent
|
||||
* @param {string} options.format - 'json' or 'yaml' (default: 'json')
|
||||
* @param {Object} options.customMetadata - Additional metadata to include
|
||||
* @returns {Object} Generated manifest object
|
||||
*/
|
||||
async generateManifest(options) {
|
||||
const {
|
||||
backupType = 'full',
|
||||
backupPath,
|
||||
files = [],
|
||||
databaseInfo = {},
|
||||
parentBackupId = null,
|
||||
format = 'json',
|
||||
customMetadata = {}
|
||||
} = options;
|
||||
|
||||
const manifest = {
|
||||
// Manifest metadata
|
||||
manifest: {
|
||||
version: '2.0',
|
||||
created: new Date().toISOString(),
|
||||
generator: 'PicPeak Backup Manifest Generator',
|
||||
format: format
|
||||
},
|
||||
|
||||
// Backup information
|
||||
backup: {
|
||||
id: this.generateBackupId(),
|
||||
type: backupType,
|
||||
timestamp: new Date().toISOString(),
|
||||
path: backupPath,
|
||||
parent_backup_id: parentBackupId,
|
||||
retention_days: customMetadata.retentionDays || 30
|
||||
},
|
||||
|
||||
// System information
|
||||
system: {
|
||||
hostname: this.hostname,
|
||||
platform: this.platform,
|
||||
os_release: this.osRelease,
|
||||
architecture: os.arch(),
|
||||
cpu_count: os.cpus().length,
|
||||
total_memory: os.totalmem(),
|
||||
free_memory: os.freemem(),
|
||||
uptime: os.uptime()
|
||||
},
|
||||
|
||||
// Application information
|
||||
application: {
|
||||
name: 'PicPeak',
|
||||
version: this.appVersion,
|
||||
node_version: this.nodeVersion,
|
||||
environment: process.env.NODE_ENV || 'production',
|
||||
storage_path: process.env.STORAGE_PATH || path.join(__dirname, '../../../storage')
|
||||
},
|
||||
|
||||
// Files information
|
||||
files: {
|
||||
count: files.length,
|
||||
total_size: files.reduce((sum, file) => sum + (file.size || 0), 0),
|
||||
checksums: await this.generateFileChecksums(files),
|
||||
manifest: files.map(file => ({
|
||||
path: file.relativePath || file.path,
|
||||
size: file.size,
|
||||
modified: file.modified,
|
||||
checksum: file.checksum,
|
||||
type: this.getFileType(file.path),
|
||||
permissions: file.permissions
|
||||
}))
|
||||
},
|
||||
|
||||
// Database information
|
||||
database: {
|
||||
type: databaseInfo.type || this.getDatabaseType(),
|
||||
backup_file: databaseInfo.backupFile,
|
||||
size: databaseInfo.size,
|
||||
checksum: databaseInfo.checksum,
|
||||
tables: databaseInfo.tables || {},
|
||||
row_counts: databaseInfo.rowCounts || {},
|
||||
schema_version: await this.getSchemaVersion()
|
||||
},
|
||||
|
||||
// Verification information
|
||||
verification: {
|
||||
total_checksum: null, // Will be calculated after manifest is complete
|
||||
file_count_check: files.length,
|
||||
size_check: files.reduce((sum, file) => sum + (file.size || 0), 0),
|
||||
integrity_timestamp: new Date().toISOString()
|
||||
},
|
||||
|
||||
// Custom metadata
|
||||
metadata: {
|
||||
...customMetadata,
|
||||
backup_settings: await this.getBackupSettings(),
|
||||
active_events_count: await this.getActiveEventsCount(),
|
||||
archived_events_count: await this.getArchivedEventsCount(),
|
||||
total_photos_count: await this.getTotalPhotosCount()
|
||||
}
|
||||
};
|
||||
|
||||
// Calculate total checksum of the manifest
|
||||
manifest.verification.total_checksum = this.calculateManifestChecksum(manifest);
|
||||
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save manifest to file
|
||||
* @param {Object} manifest - Manifest object to save
|
||||
* @param {string} filePath - Path to save the manifest
|
||||
* @param {string} format - 'json' or 'yaml'
|
||||
*/
|
||||
async saveManifest(manifest, filePath, format = 'json') {
|
||||
try {
|
||||
let content;
|
||||
|
||||
if (format === 'yaml') {
|
||||
content = yaml.dump(manifest, {
|
||||
indent: 2,
|
||||
lineWidth: -1,
|
||||
noRefs: true,
|
||||
sortKeys: true
|
||||
});
|
||||
} else {
|
||||
content = JSON.stringify(manifest, null, 2);
|
||||
}
|
||||
|
||||
await fs.writeFile(filePath, content, 'utf8');
|
||||
logger.info(`Manifest saved to ${filePath} (format: ${format})`);
|
||||
|
||||
return filePath;
|
||||
} catch (error) {
|
||||
logger.error('Failed to save manifest:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load and validate an existing manifest
|
||||
* @param {string} filePath - Path to the manifest file
|
||||
* @returns {Object} Loaded and validated manifest
|
||||
*/
|
||||
async loadManifest(filePath) {
|
||||
try {
|
||||
const content = await fs.readFile(filePath, 'utf8');
|
||||
let manifest;
|
||||
|
||||
// Detect format and parse
|
||||
if (filePath.endsWith('.yaml') || filePath.endsWith('.yml')) {
|
||||
manifest = yaml.load(content);
|
||||
} else {
|
||||
manifest = JSON.parse(content);
|
||||
}
|
||||
|
||||
// Validate manifest structure
|
||||
this.validateManifest(manifest);
|
||||
|
||||
return manifest;
|
||||
} catch (error) {
|
||||
logger.error('Failed to load manifest:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate manifest structure and integrity
|
||||
* @param {Object} manifest - Manifest to validate
|
||||
* @throws {Error} If validation fails
|
||||
*/
|
||||
validateManifest(manifest) {
|
||||
// Check required sections
|
||||
const requiredSections = ['manifest', 'backup', 'system', 'application', 'files', 'database', 'verification'];
|
||||
for (const section of requiredSections) {
|
||||
if (!manifest[section]) {
|
||||
throw new Error(`Missing required section: ${section}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate manifest version
|
||||
if (!manifest.manifest.version) {
|
||||
throw new Error('Missing manifest version');
|
||||
}
|
||||
|
||||
// Validate file checksums
|
||||
if (manifest.files.count !== manifest.files.manifest.length) {
|
||||
throw new Error('File count mismatch');
|
||||
}
|
||||
|
||||
// Validate total checksum
|
||||
const calculatedChecksum = this.calculateManifestChecksum(manifest);
|
||||
if (manifest.verification.total_checksum !== calculatedChecksum) {
|
||||
throw new Error('Manifest checksum verification failed');
|
||||
}
|
||||
|
||||
logger.info('Manifest validation passed');
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two manifests for incremental backup
|
||||
* @param {Object} currentManifest - Current backup manifest
|
||||
* @param {Object} parentManifest - Parent backup manifest
|
||||
* @returns {Object} Comparison results
|
||||
*/
|
||||
compareManifests(currentManifest, parentManifest) {
|
||||
const comparison = {
|
||||
added_files: [],
|
||||
modified_files: [],
|
||||
deleted_files: [],
|
||||
unchanged_files: [],
|
||||
size_difference: 0,
|
||||
database_changes: {}
|
||||
};
|
||||
|
||||
// Create file maps for easy comparison
|
||||
const currentFiles = new Map(
|
||||
currentManifest.files.manifest.map(f => [f.path, f])
|
||||
);
|
||||
const parentFiles = new Map(
|
||||
parentManifest.files.manifest.map(f => [f.path, f])
|
||||
);
|
||||
|
||||
// Find added and modified files
|
||||
for (const [path, file] of currentFiles) {
|
||||
const parentFile = parentFiles.get(path);
|
||||
if (!parentFile) {
|
||||
comparison.added_files.push(file);
|
||||
comparison.size_difference += file.size;
|
||||
} else if (file.checksum !== parentFile.checksum) {
|
||||
comparison.modified_files.push(file);
|
||||
comparison.size_difference += file.size - parentFile.size;
|
||||
} else {
|
||||
comparison.unchanged_files.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
// Find deleted files
|
||||
for (const [path, file] of parentFiles) {
|
||||
if (!currentFiles.has(path)) {
|
||||
comparison.deleted_files.push(file);
|
||||
comparison.size_difference -= file.size;
|
||||
}
|
||||
}
|
||||
|
||||
// Compare database info
|
||||
comparison.database_changes = {
|
||||
size_difference: currentManifest.database.size - parentManifest.database.size,
|
||||
checksum_changed: currentManifest.database.checksum !== parentManifest.database.checksum,
|
||||
schema_version_changed: currentManifest.database.schema_version !== parentManifest.database.schema_version
|
||||
};
|
||||
|
||||
return comparison;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate incremental manifest based on parent
|
||||
* @param {Object} options - Manifest generation options
|
||||
* @param {Object} parentManifest - Parent backup manifest
|
||||
* @returns {Object} Incremental manifest
|
||||
*/
|
||||
async generateIncrementalManifest(options, parentManifest) {
|
||||
const fullManifest = await this.generateManifest({
|
||||
...options,
|
||||
backupType: 'incremental'
|
||||
});
|
||||
|
||||
const comparison = this.compareManifests(fullManifest, parentManifest);
|
||||
|
||||
// Add incremental-specific information
|
||||
fullManifest.incremental = {
|
||||
parent_backup_id: parentManifest.backup.id,
|
||||
parent_timestamp: parentManifest.backup.timestamp,
|
||||
changes: {
|
||||
added_files_count: comparison.added_files.length,
|
||||
modified_files_count: comparison.modified_files.length,
|
||||
deleted_files_count: comparison.deleted_files.length,
|
||||
unchanged_files_count: comparison.unchanged_files.length,
|
||||
size_difference: comparison.size_difference
|
||||
},
|
||||
added_files: comparison.added_files.map(f => f.path),
|
||||
modified_files: comparison.modified_files.map(f => f.path),
|
||||
deleted_files: comparison.deleted_files.map(f => f.path)
|
||||
};
|
||||
|
||||
return fullManifest;
|
||||
}
|
||||
|
||||
// Helper methods
|
||||
|
||||
generateBackupId() {
|
||||
const timestamp = new Date().toISOString().replace(/[:-]/g, '').replace('T', '-').split('.')[0];
|
||||
const random = crypto.randomBytes(4).toString('hex');
|
||||
return `backup-${timestamp}-${random}`;
|
||||
}
|
||||
|
||||
async generateFileChecksums(files) {
|
||||
const checksums = {};
|
||||
for (const file of files) {
|
||||
if (file.checksum) {
|
||||
checksums[file.relativePath || file.path] = file.checksum;
|
||||
}
|
||||
}
|
||||
return checksums;
|
||||
}
|
||||
|
||||
getFileType(filePath) {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
const typeMap = {
|
||||
'.jpg': 'image',
|
||||
'.jpeg': 'image',
|
||||
'.png': 'image',
|
||||
'.gif': 'image',
|
||||
'.webp': 'image',
|
||||
'.zip': 'archive',
|
||||
'.sql': 'database',
|
||||
'.db': 'database',
|
||||
'.json': 'config',
|
||||
'.yaml': 'config',
|
||||
'.yml': 'config'
|
||||
};
|
||||
return typeMap[ext] || 'other';
|
||||
}
|
||||
|
||||
getDatabaseType() {
|
||||
return process.env.DB_TYPE === 'postgresql' ? 'postgresql' : 'sqlite';
|
||||
}
|
||||
|
||||
async getSchemaVersion() {
|
||||
try {
|
||||
const result = await db('migrations')
|
||||
.orderBy('run_at', 'desc')
|
||||
.first();
|
||||
return result ? result.migration_name : 'unknown';
|
||||
} catch (error) {
|
||||
return 'unknown';
|
||||
}
|
||||
}
|
||||
|
||||
async getBackupSettings() {
|
||||
try {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
const config = {};
|
||||
settings.forEach(setting => {
|
||||
try {
|
||||
config[setting.setting_key] = JSON.parse(setting.setting_value);
|
||||
} catch (e) {
|
||||
config[setting.setting_key] = setting.setting_value;
|
||||
}
|
||||
});
|
||||
|
||||
return config;
|
||||
} catch (error) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
async getActiveEventsCount() {
|
||||
try {
|
||||
const result = await db('events')
|
||||
.where('status', 'active')
|
||||
.count('* as count')
|
||||
.first();
|
||||
return result ? parseInt(result.count) : 0;
|
||||
} catch (error) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async getArchivedEventsCount() {
|
||||
try {
|
||||
const result = await db('events')
|
||||
.where('status', 'archived')
|
||||
.count('* as count')
|
||||
.first();
|
||||
return result ? parseInt(result.count) : 0;
|
||||
} catch (error) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async getTotalPhotosCount() {
|
||||
try {
|
||||
const result = await db('photos')
|
||||
.count('* as count')
|
||||
.first();
|
||||
return result ? parseInt(result.count) : 0;
|
||||
} catch (error) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
calculateManifestChecksum(manifest) {
|
||||
// Create a copy without the checksum field
|
||||
const manifestCopy = JSON.parse(JSON.stringify(manifest));
|
||||
if (manifestCopy.verification) {
|
||||
delete manifestCopy.verification.total_checksum;
|
||||
}
|
||||
|
||||
// Calculate SHA256 of the sorted JSON
|
||||
const content = JSON.stringify(manifestCopy, Object.keys(manifestCopy).sort());
|
||||
return crypto.createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a summary report from a manifest
|
||||
* @param {Object} manifest - Manifest to summarize
|
||||
* @returns {string} Human-readable summary
|
||||
*/
|
||||
generateSummaryReport(manifest) {
|
||||
const report = [];
|
||||
|
||||
report.push('=== BACKUP MANIFEST SUMMARY ===');
|
||||
report.push(`Backup ID: ${manifest.backup.id}`);
|
||||
report.push(`Type: ${manifest.backup.type}`);
|
||||
report.push(`Created: ${manifest.backup.timestamp}`);
|
||||
|
||||
if (manifest.backup.parent_backup_id) {
|
||||
report.push(`Parent Backup: ${manifest.backup.parent_backup_id}`);
|
||||
}
|
||||
|
||||
report.push('\n--- System Information ---');
|
||||
report.push(`Host: ${manifest.system.hostname}`);
|
||||
report.push(`Platform: ${manifest.system.platform} ${manifest.system.os_release}`);
|
||||
report.push(`Architecture: ${manifest.system.architecture}`);
|
||||
|
||||
report.push('\n--- Application Information ---');
|
||||
report.push(`App Version: ${manifest.application.version}`);
|
||||
report.push(`Node Version: ${manifest.application.node_version}`);
|
||||
report.push(`Environment: ${manifest.application.environment}`);
|
||||
|
||||
report.push('\n--- Files Summary ---');
|
||||
report.push(`Total Files: ${manifest.files.count}`);
|
||||
report.push(`Total Size: ${(manifest.files.total_size / 1024 / 1024).toFixed(2)} MB`);
|
||||
|
||||
if (manifest.incremental) {
|
||||
report.push('\n--- Incremental Changes ---');
|
||||
report.push(`Added Files: ${manifest.incremental.changes.added_files_count}`);
|
||||
report.push(`Modified Files: ${manifest.incremental.changes.modified_files_count}`);
|
||||
report.push(`Deleted Files: ${manifest.incremental.changes.deleted_files_count}`);
|
||||
report.push(`Size Difference: ${(manifest.incremental.changes.size_difference / 1024 / 1024).toFixed(2)} MB`);
|
||||
}
|
||||
|
||||
report.push('\n--- Database Information ---');
|
||||
report.push(`Type: ${manifest.database.type}`);
|
||||
report.push(`Size: ${manifest.database.size ? (manifest.database.size / 1024 / 1024).toFixed(2) + ' MB' : 'N/A'}`);
|
||||
report.push(`Schema Version: ${manifest.database.schema_version}`);
|
||||
|
||||
report.push('\n--- Content Statistics ---');
|
||||
report.push(`Active Events: ${manifest.metadata.active_events_count}`);
|
||||
report.push(`Archived Events: ${manifest.metadata.archived_events_count}`);
|
||||
report.push(`Total Photos: ${manifest.metadata.total_photos_count}`);
|
||||
|
||||
report.push('\n--- Verification ---');
|
||||
report.push(`Manifest Checksum: ${manifest.verification.total_checksum}`);
|
||||
report.push(`Integrity Timestamp: ${manifest.verification.integrity_timestamp}`);
|
||||
|
||||
return report.join('\n');
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
module.exports = new BackupManifestGenerator();
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,720 @@
|
||||
const cron = require('node-cron');
|
||||
const path = require('path');
|
||||
const fs = require('fs').promises;
|
||||
const crypto = require('crypto');
|
||||
const { exec } = require('child_process');
|
||||
const { promisify } = require('util');
|
||||
const execAsync = promisify(exec);
|
||||
const { db } = require('../database/db');
|
||||
const { queueEmail } = require('./emailProcessor');
|
||||
const logger = require('../utils/logger');
|
||||
const { formatBoolean } = require('../utils/dbCompat');
|
||||
const backupManifest = require('./backupManifest');
|
||||
|
||||
// Backup job reference
|
||||
let backupJob = null;
|
||||
let backupConfig = null;
|
||||
let isRunning = false;
|
||||
|
||||
// Storage paths
|
||||
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
||||
|
||||
/**
|
||||
* Calculate file checksum using SHA256
|
||||
*/
|
||||
async function calculateChecksum(filePath) {
|
||||
const hash = crypto.createHash('sha256');
|
||||
const stream = require('fs').createReadStream(filePath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', data => hash.update(data));
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get database backup information
|
||||
*/
|
||||
async function getDatabaseBackupInfo() {
|
||||
try {
|
||||
// Check for recent database backup
|
||||
const recentDbBackup = await db('database_backup_runs')
|
||||
.where('status', 'completed')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.first();
|
||||
|
||||
if (recentDbBackup && recentDbBackup.file_path) {
|
||||
return {
|
||||
type: recentDbBackup.backup_type,
|
||||
backupFile: recentDbBackup.file_path,
|
||||
size: recentDbBackup.file_size_bytes,
|
||||
checksum: recentDbBackup.checksum,
|
||||
tables: recentDbBackup.statistics ? JSON.parse(recentDbBackup.statistics).tables : {},
|
||||
rowCounts: recentDbBackup.table_checksums ? JSON.parse(recentDbBackup.table_checksums) : {}
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
type: process.env.DB_TYPE === 'postgresql' ? 'postgresql' : 'sqlite',
|
||||
backupFile: null,
|
||||
size: 0,
|
||||
checksum: null,
|
||||
tables: {},
|
||||
rowCounts: {}
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get database backup info:', error);
|
||||
return {
|
||||
type: 'unknown',
|
||||
backupFile: null,
|
||||
size: 0,
|
||||
checksum: null,
|
||||
tables: {},
|
||||
rowCounts: {}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup configuration from database
|
||||
*/
|
||||
async function getBackupConfig() {
|
||||
try {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
const config = {};
|
||||
settings.forEach(setting => {
|
||||
try {
|
||||
config[setting.setting_key] = JSON.parse(setting.setting_value);
|
||||
} catch (e) {
|
||||
config[setting.setting_key] = setting.setting_value;
|
||||
}
|
||||
});
|
||||
|
||||
return config;
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup configuration:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of files to backup
|
||||
*/
|
||||
async function getFilesToBackup(includeArchived = true) {
|
||||
const files = [];
|
||||
const storagePath = getStoragePath();
|
||||
|
||||
try {
|
||||
// Active events
|
||||
const activePath = path.join(storagePath, 'events/active');
|
||||
await scanDirectory(activePath, files, storagePath);
|
||||
|
||||
// Archived events (if enabled)
|
||||
if (includeArchived) {
|
||||
const archivePath = path.join(storagePath, 'events/archived');
|
||||
await scanDirectory(archivePath, files, storagePath);
|
||||
}
|
||||
|
||||
// Thumbnails
|
||||
const thumbsPath = path.join(storagePath, 'thumbnails');
|
||||
await scanDirectory(thumbsPath, files, storagePath);
|
||||
|
||||
// Uploads (logos, favicons, etc.)
|
||||
const uploadsPath = path.join(storagePath, 'uploads');
|
||||
await scanDirectory(uploadsPath, files, storagePath);
|
||||
|
||||
return files;
|
||||
} catch (error) {
|
||||
logger.error('Failed to get files to backup:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively scan directory for files
|
||||
*/
|
||||
async function scanDirectory(dirPath, fileList, basePath, excludePatterns = []) {
|
||||
try {
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
const relativePath = path.relative(basePath, fullPath);
|
||||
|
||||
// Check exclude patterns
|
||||
if (excludePatterns.some(pattern => {
|
||||
if (pattern.includes('*')) {
|
||||
return new RegExp(pattern.replace(/\*/g, '.*')).test(entry.name);
|
||||
}
|
||||
return entry.name === pattern;
|
||||
})) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await scanDirectory(fullPath, fileList, basePath, excludePatterns);
|
||||
} else if (entry.isFile()) {
|
||||
const stats = await fs.stat(fullPath);
|
||||
fileList.push({
|
||||
path: fullPath,
|
||||
relativePath: relativePath,
|
||||
size: stats.size,
|
||||
modified: stats.mtime
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
logger.error(`Failed to scan directory ${dirPath}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file has changed since last backup
|
||||
*/
|
||||
async function hasFileChanged(filePath, checksum) {
|
||||
try {
|
||||
const fileState = await db('backup_file_states')
|
||||
.where('file_path', filePath)
|
||||
.first();
|
||||
|
||||
return !fileState || fileState.checksum !== checksum;
|
||||
} catch (error) {
|
||||
logger.error('Failed to check file state:', error);
|
||||
return true; // Assume changed if we can't check
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update file state in database
|
||||
*/
|
||||
async function updateFileState(filePath, checksum, size, modified) {
|
||||
try {
|
||||
const existing = await db('backup_file_states')
|
||||
.where('file_path', filePath)
|
||||
.first();
|
||||
|
||||
const data = {
|
||||
file_path: filePath,
|
||||
checksum: checksum,
|
||||
size_bytes: size,
|
||||
last_modified: modified,
|
||||
last_backed_up: new Date()
|
||||
};
|
||||
|
||||
if (existing) {
|
||||
await db('backup_file_states')
|
||||
.where('id', existing.id)
|
||||
.update(data);
|
||||
} else {
|
||||
await db('backup_file_states').insert(data);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to update file state:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform local directory backup
|
||||
*/
|
||||
async function performLocalBackup(config, files) {
|
||||
const destPath = config.backup_destination_path;
|
||||
const storagePath = getStoragePath();
|
||||
let backedUpCount = 0;
|
||||
let backedUpSize = 0;
|
||||
const backedUpFiles = [];
|
||||
|
||||
// Ensure destination exists
|
||||
await fs.mkdir(destPath, { recursive: true });
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
// Skip large files if configured
|
||||
const maxSizeMB = config.backup_max_file_size_mb || 5000;
|
||||
if (file.size > maxSizeMB * 1024 * 1024) {
|
||||
logger.warn(`Skipping large file: ${file.relativePath} (${(file.size / 1024 / 1024).toFixed(2)} MB)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Calculate checksum
|
||||
const checksum = await calculateChecksum(file.path);
|
||||
file.checksum = checksum; // Add checksum to file object
|
||||
|
||||
// Check if file has changed
|
||||
const changed = await hasFileChanged(file.relativePath, checksum);
|
||||
if (!changed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Copy file
|
||||
const destFilePath = path.join(destPath, file.relativePath);
|
||||
const destDir = path.dirname(destFilePath);
|
||||
await fs.mkdir(destDir, { recursive: true });
|
||||
await fs.copyFile(file.path, destFilePath);
|
||||
|
||||
// Update state
|
||||
await updateFileState(file.relativePath, checksum, file.size, file.modified);
|
||||
|
||||
backedUpCount++;
|
||||
backedUpSize += file.size;
|
||||
backedUpFiles.push(file.relativePath);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to backup file ${file.relativePath}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return { backedUpCount, backedUpSize, backedUpFiles };
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform rsync backup
|
||||
*/
|
||||
async function performRsyncBackup(config, files) {
|
||||
const storagePath = getStoragePath();
|
||||
const host = config.backup_rsync_host;
|
||||
const user = config.backup_rsync_user;
|
||||
const remotePath = config.backup_rsync_path;
|
||||
const sshKey = config.backup_rsync_ssh_key;
|
||||
|
||||
if (!host || !remotePath) {
|
||||
throw new Error('Rsync configuration incomplete');
|
||||
}
|
||||
|
||||
// Build rsync command
|
||||
const rsyncOptions = [
|
||||
'-avz', // archive, verbose, compress
|
||||
'--delete', // remove deleted files
|
||||
'--stats' // show statistics
|
||||
];
|
||||
|
||||
if (sshKey) {
|
||||
rsyncOptions.push(`-e "ssh -i ${sshKey} -o StrictHostKeyChecking=no"`);
|
||||
}
|
||||
|
||||
// Add exclude patterns
|
||||
const excludePatterns = config.backup_exclude_patterns || [];
|
||||
excludePatterns.forEach(pattern => {
|
||||
rsyncOptions.push(`--exclude="${pattern}"`);
|
||||
});
|
||||
|
||||
const source = `${storagePath}/`;
|
||||
const destination = user ? `${user}@${host}:${remotePath}` : `${host}:${remotePath}`;
|
||||
|
||||
const rsyncCommand = `rsync ${rsyncOptions.join(' ')} "${source}" "${destination}"`;
|
||||
|
||||
try {
|
||||
const { stdout, stderr } = await execAsync(rsyncCommand);
|
||||
|
||||
// Parse rsync stats
|
||||
const stats = parseRsyncStats(stdout);
|
||||
|
||||
// Update file states for successfully synced files
|
||||
for (const file of files) {
|
||||
try {
|
||||
const checksum = await calculateChecksum(file.path);
|
||||
await updateFileState(file.relativePath, checksum, file.size, file.modified);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to update state for ${file.relativePath}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
backedUpCount: stats.filesTransferred || files.length,
|
||||
backedUpSize: stats.totalSize || files.reduce((sum, f) => sum + f.size, 0),
|
||||
backedUpFiles: files.map(f => f.relativePath)
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Rsync backup failed:', error);
|
||||
throw new Error(`Rsync backup failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse rsync statistics from output
|
||||
*/
|
||||
function parseRsyncStats(output) {
|
||||
const stats = {};
|
||||
|
||||
// Extract files transferred
|
||||
const filesMatch = output.match(/Number of files transferred: (\d+)/);
|
||||
if (filesMatch) {
|
||||
stats.filesTransferred = parseInt(filesMatch[1]);
|
||||
}
|
||||
|
||||
// Extract total size
|
||||
const sizeMatch = output.match(/Total file size: ([\d,]+) bytes/);
|
||||
if (sizeMatch) {
|
||||
stats.totalSize = parseInt(sizeMatch[1].replace(/,/g, ''));
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform S3-compatible backup
|
||||
*/
|
||||
async function performS3Backup(config, files) {
|
||||
// This would require AWS SDK or similar
|
||||
// For now, return a placeholder
|
||||
throw new Error('S3 backup not implemented yet');
|
||||
}
|
||||
|
||||
/**
|
||||
* Run backup process
|
||||
*/
|
||||
async function runBackup() {
|
||||
if (isRunning) {
|
||||
logger.warn('Backup already running, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
isRunning = true;
|
||||
const startTime = new Date();
|
||||
let backupRun = null;
|
||||
|
||||
try {
|
||||
// Get current configuration
|
||||
const config = await getBackupConfig();
|
||||
if (!config.backup_enabled) {
|
||||
logger.info('Backup is disabled, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
// Create backup run record
|
||||
const [runId] = await db('backup_runs').insert({
|
||||
started_at: startTime,
|
||||
status: 'running',
|
||||
backup_type: 'scheduled'
|
||||
});
|
||||
|
||||
backupRun = { id: runId };
|
||||
|
||||
// Get files to backup
|
||||
const files = await getFilesToBackup(config.backup_include_archived);
|
||||
logger.info(`Found ${files.length} files to check for backup`);
|
||||
|
||||
// Perform backup based on destination type
|
||||
let result;
|
||||
switch (config.backup_destination_type) {
|
||||
case 'local':
|
||||
result = await performLocalBackup(config, files);
|
||||
break;
|
||||
case 'rsync':
|
||||
result = await performRsyncBackup(config, files);
|
||||
break;
|
||||
case 's3':
|
||||
result = await performS3Backup(config, files);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown backup destination type: ${config.backup_destination_type}`);
|
||||
}
|
||||
|
||||
// Calculate duration
|
||||
const endTime = new Date();
|
||||
const durationSeconds = Math.round((endTime - startTime) / 1000);
|
||||
|
||||
// Generate backup manifest
|
||||
let manifestPath = null;
|
||||
try {
|
||||
logger.info('Generating backup manifest...');
|
||||
|
||||
// Get database backup info if available
|
||||
const databaseInfo = await getDatabaseBackupInfo();
|
||||
|
||||
// Determine if this is an incremental backup
|
||||
const lastSuccessfulBackup = await db('backup_runs')
|
||||
.where('status', 'completed')
|
||||
.whereNot('id', runId)
|
||||
.orderBy('completed_at', 'desc')
|
||||
.first();
|
||||
|
||||
let manifest;
|
||||
const manifestOptions = {
|
||||
backupType: lastSuccessfulBackup ? 'incremental' : 'full',
|
||||
backupPath: config.backup_destination_path || config.backup_destination_type,
|
||||
files: files.filter(f => result.backedUpFiles && result.backedUpFiles.includes(f.relativePath)),
|
||||
databaseInfo: databaseInfo,
|
||||
parentBackupId: lastSuccessfulBackup ? lastSuccessfulBackup.manifest_id : null,
|
||||
format: config.backup_manifest_format || 'json',
|
||||
customMetadata: {
|
||||
backup_run_id: runId,
|
||||
destination_type: config.backup_destination_type,
|
||||
operator: 'system',
|
||||
reason: 'scheduled',
|
||||
retentionDays: config.backup_retention_days || 30
|
||||
}
|
||||
};
|
||||
|
||||
if (lastSuccessfulBackup && lastSuccessfulBackup.manifest_path) {
|
||||
try {
|
||||
const parentManifest = await backupManifest.loadManifest(lastSuccessfulBackup.manifest_path);
|
||||
manifest = await backupManifest.generateIncrementalManifest(manifestOptions, parentManifest);
|
||||
} catch (error) {
|
||||
logger.warn('Failed to load parent manifest, generating full manifest:', error);
|
||||
manifest = await backupManifest.generateManifest(manifestOptions);
|
||||
}
|
||||
} else {
|
||||
manifest = await backupManifest.generateManifest(manifestOptions);
|
||||
}
|
||||
|
||||
// Save manifest
|
||||
const manifestDir = config.backup_manifest_path || path.join(config.backup_destination_path || '/backup', 'manifests');
|
||||
await fs.mkdir(manifestDir, { recursive: true });
|
||||
|
||||
const manifestFileName = `backup-manifest-${manifest.backup.id}.${config.backup_manifest_format || 'json'}`;
|
||||
manifestPath = path.join(manifestDir, manifestFileName);
|
||||
await backupManifest.saveManifest(manifest, manifestPath, config.backup_manifest_format || 'json');
|
||||
|
||||
logger.info(`Backup manifest saved to ${manifestPath}`);
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to generate backup manifest:', error);
|
||||
// Don't fail the entire backup for manifest generation failure
|
||||
}
|
||||
|
||||
// Update backup run record
|
||||
await db('backup_runs')
|
||||
.where('id', runId)
|
||||
.update({
|
||||
completed_at: endTime,
|
||||
status: 'completed',
|
||||
files_backed_up: result.backedUpCount,
|
||||
total_size_bytes: result.backedUpSize,
|
||||
duration_seconds: durationSeconds,
|
||||
manifest_path: manifestPath,
|
||||
manifest_id: manifestPath ? path.basename(manifestPath, path.extname(manifestPath)) : null,
|
||||
statistics: JSON.stringify({
|
||||
totalFilesChecked: files.length,
|
||||
filesBackedUp: result.backedUpCount,
|
||||
totalSize: result.backedUpSize,
|
||||
averageFileSize: result.backedUpCount > 0 ? Math.round(result.backedUpSize / result.backedUpCount) : 0,
|
||||
manifestGenerated: !!manifestPath
|
||||
})
|
||||
});
|
||||
|
||||
logger.info(`Backup completed: ${result.backedUpCount} files, ${(result.backedUpSize / 1024 / 1024).toFixed(2)} MB in ${durationSeconds}s`);
|
||||
|
||||
// Send success email if configured
|
||||
if (config.backup_email_on_success) {
|
||||
// Get admin emails
|
||||
const admins = await db('admin_users').where('is_active', formatBoolean(true));
|
||||
for (const admin of admins) {
|
||||
await queueEmail(null, admin.email, 'backup_completed', {
|
||||
start_time: startTime.toISOString(),
|
||||
duration: `${durationSeconds} seconds`,
|
||||
files_count: result.backedUpCount.toString(),
|
||||
total_size: `${(result.backedUpSize / 1024 / 1024).toFixed(2)} MB`,
|
||||
backup_type: config.backup_destination_type
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Backup failed:', error);
|
||||
|
||||
// Update backup run record
|
||||
if (backupRun) {
|
||||
await db('backup_runs')
|
||||
.where('id', backupRun.id)
|
||||
.update({
|
||||
completed_at: new Date(),
|
||||
status: 'failed',
|
||||
error_message: error.message
|
||||
});
|
||||
}
|
||||
|
||||
// Send failure email
|
||||
const config = await getBackupConfig();
|
||||
if (config && config.backup_email_on_failure) {
|
||||
const admins = await db('admin_users').where('is_active', formatBoolean(true));
|
||||
for (const admin of admins) {
|
||||
await queueEmail(null, admin.email, 'backup_failed', {
|
||||
start_time: startTime.toISOString(),
|
||||
backup_type: config.backup_destination_type || 'unknown',
|
||||
error_message: error.message
|
||||
});
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
isRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start backup service
|
||||
*/
|
||||
async function startBackupService() {
|
||||
try {
|
||||
// Get configuration
|
||||
backupConfig = await getBackupConfig();
|
||||
|
||||
if (!backupConfig || !backupConfig.backup_enabled) {
|
||||
logger.info('Backup service is disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
// Cancel existing job if any
|
||||
if (backupJob) {
|
||||
backupJob.stop();
|
||||
}
|
||||
|
||||
// Schedule backup job
|
||||
const schedule = backupConfig.backup_schedule || '0 2 * * *'; // Default: 2 AM daily
|
||||
backupJob = cron.schedule(schedule, async () => {
|
||||
logger.info('Starting scheduled backup');
|
||||
await runBackup();
|
||||
});
|
||||
|
||||
logger.info(`Backup service started with schedule: ${schedule}`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to start backup service:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop backup service
|
||||
*/
|
||||
function stopBackupService() {
|
||||
if (backupJob) {
|
||||
backupJob.stop();
|
||||
backupJob = null;
|
||||
logger.info('Backup service stopped');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger manual backup
|
||||
*/
|
||||
async function triggerManualBackup() {
|
||||
logger.info('Starting manual backup');
|
||||
await runBackup();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup status and history
|
||||
*/
|
||||
async function getBackupStatus(limit = 10) {
|
||||
try {
|
||||
const runs = await db('backup_runs')
|
||||
.orderBy('started_at', 'desc')
|
||||
.limit(limit);
|
||||
|
||||
const lastRun = runs[0];
|
||||
const isHealthy = lastRun && lastRun.status === 'completed';
|
||||
|
||||
// Validate manifest if exists
|
||||
let manifestValid = false;
|
||||
if (lastRun && lastRun.manifest_path) {
|
||||
try {
|
||||
const manifest = await backupManifest.loadManifest(lastRun.manifest_path);
|
||||
backupManifest.validateManifest(manifest);
|
||||
manifestValid = true;
|
||||
} catch (error) {
|
||||
logger.warn('Manifest validation failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
isRunning,
|
||||
isHealthy,
|
||||
lastRun: lastRun ? {
|
||||
...lastRun,
|
||||
manifestValid
|
||||
} : null,
|
||||
recentRuns: runs,
|
||||
nextScheduledRun: backupJob ? getNextScheduledRun() : null
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup status:', error);
|
||||
return {
|
||||
isRunning,
|
||||
isHealthy: false,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get next scheduled run time
|
||||
*/
|
||||
function getNextScheduledRun() {
|
||||
// This is a simplified version - would need proper cron parsing
|
||||
const now = new Date();
|
||||
const tomorrow = new Date(now);
|
||||
tomorrow.setDate(tomorrow.getDate() + 1);
|
||||
tomorrow.setHours(2, 0, 0, 0); // Assuming default 2 AM schedule
|
||||
return tomorrow.toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old backup runs
|
||||
*/
|
||||
async function cleanupOldBackupRuns(retentionDays = 30) {
|
||||
try {
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
|
||||
const deleted = await db('backup_runs')
|
||||
.where('started_at', '<', cutoffDate)
|
||||
.delete();
|
||||
|
||||
if (deleted > 0) {
|
||||
logger.info(`Cleaned up ${deleted} old backup runs`);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to cleanup old backup runs:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup manifest for a specific backup run
|
||||
*/
|
||||
async function getBackupManifest(backupRunId) {
|
||||
try {
|
||||
const run = await db('backup_runs')
|
||||
.where('id', backupRunId)
|
||||
.first();
|
||||
|
||||
if (!run || !run.manifest_path) {
|
||||
throw new Error('Backup manifest not found');
|
||||
}
|
||||
|
||||
const manifest = await backupManifest.loadManifest(run.manifest_path);
|
||||
return {
|
||||
manifest,
|
||||
summary: backupManifest.generateSummaryReport(manifest)
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup manifest:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a backup manifest file
|
||||
*/
|
||||
async function validateBackupManifest(manifestPath) {
|
||||
try {
|
||||
const manifest = await backupManifest.loadManifest(manifestPath);
|
||||
backupManifest.validateManifest(manifest);
|
||||
return { valid: true, manifest };
|
||||
} catch (error) {
|
||||
return { valid: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
startBackupService,
|
||||
stopBackupService,
|
||||
triggerManualBackup,
|
||||
getBackupStatus,
|
||||
runBackup,
|
||||
cleanupOldBackupRuns,
|
||||
getBackupManifest,
|
||||
validateBackupManifest
|
||||
};
|
||||
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* Database Backup Service Usage Examples
|
||||
*
|
||||
* This service provides comprehensive database backup functionality
|
||||
* with support for both SQLite and PostgreSQL databases.
|
||||
*/
|
||||
|
||||
const { databaseBackupService } = require('./databaseBackup');
|
||||
|
||||
// Example 1: Manual backup with default settings
|
||||
async function manualBackup() {
|
||||
try {
|
||||
const result = await databaseBackupService.backup();
|
||||
console.log('Backup completed:', result);
|
||||
// Result includes: path, size, duration, checksum, compressionRatio
|
||||
} catch (error) {
|
||||
console.error('Backup failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Example 2: Backup with custom options
|
||||
async function customBackup() {
|
||||
try {
|
||||
const result = await databaseBackupService.backup({
|
||||
destinationPath: '/custom/backup/path',
|
||||
compress: true, // Enable gzip compression
|
||||
validateIntegrity: true, // Validate backup after creation
|
||||
includeChecksums: true, // Calculate table checksums
|
||||
noTransaction: false // Use transaction for consistency (PostgreSQL)
|
||||
});
|
||||
console.log('Custom backup completed:', result);
|
||||
} catch (error) {
|
||||
console.error('Backup failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Example 3: Check backup progress (useful for long-running backups)
|
||||
async function backupWithProgress() {
|
||||
// Start backup asynchronously
|
||||
const backupPromise = databaseBackupService.backup();
|
||||
|
||||
// Poll for progress
|
||||
const progressInterval = setInterval(() => {
|
||||
const progress = databaseBackupService.getProgress();
|
||||
if (progress) {
|
||||
console.log(`Progress: ${progress.message}`, progress.details);
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
try {
|
||||
const result = await backupPromise;
|
||||
clearInterval(progressInterval);
|
||||
console.log('Backup completed:', result);
|
||||
} catch (error) {
|
||||
clearInterval(progressInterval);
|
||||
console.error('Backup failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Example 4: Get backup history
|
||||
async function getBackupHistory() {
|
||||
const history = await databaseBackupService.getBackupHistory(10);
|
||||
|
||||
history.forEach(backup => {
|
||||
console.log(`Backup ${backup.id}:`);
|
||||
console.log(` Started: ${backup.started_at}`);
|
||||
console.log(` Status: ${backup.status}`);
|
||||
console.log(` Size: ${(backup.file_size_bytes / 1024 / 1024).toFixed(2)} MB`);
|
||||
console.log(` Duration: ${backup.duration_seconds}s`);
|
||||
});
|
||||
}
|
||||
|
||||
// Example 5: Clean up old backups
|
||||
async function cleanupBackups() {
|
||||
// Delete backups older than 30 days
|
||||
await databaseBackupService.cleanupOldBackups(30);
|
||||
console.log('Old backups cleaned up');
|
||||
}
|
||||
|
||||
// Example 6: Get table checksums (useful for monitoring changes)
|
||||
async function getTableChecksums() {
|
||||
const checksums = await databaseBackupService.getTableChecksums();
|
||||
|
||||
console.log('Table Checksums:');
|
||||
Object.entries(checksums).forEach(([table, info]) => {
|
||||
console.log(` ${table}: ${info.rowCount} rows, checksum: ${info.checksum}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Example 7: Using the scheduled backup service
|
||||
const { startScheduledBackups, stopScheduledBackups } = require('./databaseBackup');
|
||||
|
||||
async function setupScheduledBackups() {
|
||||
// Start scheduled backups (reads schedule from database config)
|
||||
await startScheduledBackups();
|
||||
console.log('Scheduled backups started');
|
||||
|
||||
// Later, if needed, stop scheduled backups
|
||||
// stopScheduledBackups();
|
||||
}
|
||||
|
||||
// Example 8: Admin API endpoints available
|
||||
/*
|
||||
GET /api/admin/database-backup/status - Get backup status and config
|
||||
PUT /api/admin/database-backup/config - Update backup configuration
|
||||
POST /api/admin/database-backup/backup - Trigger manual backup
|
||||
GET /api/admin/database-backup/progress - Get current backup progress
|
||||
GET /api/admin/database-backup/history - Get backup history with pagination
|
||||
DELETE /api/admin/database-backup/cleanup - Delete old backup files
|
||||
POST /api/admin/database-backup/test - Test backup configuration
|
||||
GET /api/admin/database-backup/checksums - Get current table checksums
|
||||
*/
|
||||
|
||||
// Example 9: Configuration options stored in database
|
||||
/*
|
||||
database_backup_enabled: boolean - Enable/disable scheduled backups
|
||||
database_backup_schedule: string - Cron schedule (default: '0 3 * * *')
|
||||
database_backup_destination_path: string - Where to store backups
|
||||
database_backup_compress: boolean - Enable gzip compression
|
||||
database_backup_validate_integrity: boolean - Validate after backup
|
||||
database_backup_include_checksums: boolean - Calculate table checksums
|
||||
database_backup_retention_days: number - Days to keep old backups
|
||||
database_backup_email_on_failure: boolean - Send email on failure
|
||||
database_backup_email_on_success: boolean - Send email on success
|
||||
*/
|
||||
|
||||
// Example 10: Production considerations
|
||||
/*
|
||||
1. Ensure destination path has sufficient space
|
||||
2. For large databases, backups may take significant time
|
||||
3. PostgreSQL backups use single-transaction mode by default
|
||||
4. Compression typically reduces size by 70-90%
|
||||
5. Schedule backups during low-traffic periods
|
||||
6. Monitor backup history for failures
|
||||
7. Test restore procedures regularly
|
||||
8. Consider replication for real-time redundancy
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
manualBackup,
|
||||
customBackup,
|
||||
backupWithProgress,
|
||||
getBackupHistory,
|
||||
cleanupBackups,
|
||||
getTableChecksums,
|
||||
setupScheduledBackups
|
||||
};
|
||||
@@ -0,0 +1,631 @@
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { exec } = require('child_process');
|
||||
const { promisify } = require('util');
|
||||
const execAsync = promisify(exec);
|
||||
const crypto = require('crypto');
|
||||
const zlib = require('zlib');
|
||||
const { pipeline } = require('stream/promises');
|
||||
const { createReadStream, createWriteStream } = require('fs');
|
||||
const { db } = require('../database/db');
|
||||
const knexConfig = require('../../knexfile');
|
||||
const logger = require('../utils/logger');
|
||||
const { queueEmail } = require('./emailProcessor');
|
||||
const { formatBoolean } = require('../utils/dbCompat');
|
||||
|
||||
// Constants
|
||||
const CHUNK_SIZE = 1024 * 1024; // 1MB chunks for streaming
|
||||
const PROGRESS_INTERVAL = 100; // Report progress every 100 rows
|
||||
|
||||
/**
|
||||
* Database Backup Service
|
||||
* Supports both SQLite and PostgreSQL with proper escaping,
|
||||
* compression, checksums, and validation
|
||||
*/
|
||||
class DatabaseBackupService {
|
||||
constructor() {
|
||||
this.isRunning = false;
|
||||
this.currentProgress = null;
|
||||
this.dbType = knexConfig.client === 'pg' ? 'postgresql' : 'sqlite';
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate checksum for a file
|
||||
*/
|
||||
async calculateChecksum(filePath) {
|
||||
const hash = crypto.createHash('sha256');
|
||||
const stream = createReadStream(filePath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', data => hash.update(data));
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress a file using gzip
|
||||
*/
|
||||
async compressFile(inputPath, outputPath) {
|
||||
const gzip = zlib.createGzip({ level: 6 }); // Balanced compression
|
||||
const source = createReadStream(inputPath);
|
||||
const destination = createWriteStream(outputPath);
|
||||
|
||||
await pipeline(source, gzip, destination);
|
||||
|
||||
// Get compression ratio
|
||||
const inputStats = await fs.stat(inputPath);
|
||||
const outputStats = await fs.stat(outputPath);
|
||||
const ratio = (1 - outputStats.size / inputStats.size) * 100;
|
||||
|
||||
return {
|
||||
originalSize: inputStats.size,
|
||||
compressedSize: outputStats.size,
|
||||
compressionRatio: ratio.toFixed(2)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get database size
|
||||
*/
|
||||
async getDatabaseSize() {
|
||||
if (this.dbType === 'sqlite') {
|
||||
const dbPath = knexConfig.connection.filename;
|
||||
const stats = await fs.stat(dbPath);
|
||||
return stats.size;
|
||||
} else {
|
||||
// PostgreSQL
|
||||
const result = await db.raw(`
|
||||
SELECT pg_database_size(current_database()) as size
|
||||
`);
|
||||
return parseInt(result.rows[0].size);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get table checksums for change detection
|
||||
*/
|
||||
async getTableChecksums() {
|
||||
const checksums = {};
|
||||
const tables = await this.getTables();
|
||||
|
||||
for (const table of tables) {
|
||||
if (this.dbType === 'sqlite') {
|
||||
// SQLite: Use aggregate of all row data
|
||||
const result = await db.raw(`
|
||||
SELECT
|
||||
COUNT(*) as row_count,
|
||||
COALESCE(SUM(LENGTH(CAST(t.* AS TEXT))), 0) as data_sum
|
||||
FROM "${table}" t
|
||||
`);
|
||||
|
||||
checksums[table] = {
|
||||
rowCount: result[0].row_count,
|
||||
checksum: crypto
|
||||
.createHash('md5')
|
||||
.update(`${result[0].row_count}-${result[0].data_sum}`)
|
||||
.digest('hex')
|
||||
};
|
||||
} else {
|
||||
// PostgreSQL: Use built-in functions
|
||||
const result = await db.raw(`
|
||||
SELECT
|
||||
COUNT(*) as row_count,
|
||||
MD5(COALESCE(STRING_AGG(MD5(t::text), ''), '')) as checksum
|
||||
FROM "${table}" t
|
||||
`);
|
||||
|
||||
checksums[table] = {
|
||||
rowCount: parseInt(result.rows[0].row_count),
|
||||
checksum: result.rows[0].checksum || 'empty'
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return checksums;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of tables
|
||||
*/
|
||||
async getTables() {
|
||||
if (this.dbType === 'sqlite') {
|
||||
const result = await db.raw(`
|
||||
SELECT name FROM sqlite_master
|
||||
WHERE type='table'
|
||||
AND name NOT LIKE 'sqlite_%'
|
||||
AND name != 'knex_migrations'
|
||||
AND name != 'knex_migrations_lock'
|
||||
ORDER BY name
|
||||
`);
|
||||
return result.map(row => row.name);
|
||||
} else {
|
||||
// PostgreSQL
|
||||
const result = await db.raw(`
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_type = 'BASE TABLE'
|
||||
AND table_name NOT IN ('knex_migrations', 'knex_migrations_lock')
|
||||
ORDER BY table_name
|
||||
`);
|
||||
return result.rows.map(row => row.table_name);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create SQLite backup
|
||||
*/
|
||||
async createSQLiteBackup(outputPath, options = {}) {
|
||||
const dbPath = knexConfig.connection.filename;
|
||||
const tempPath = `${outputPath}.tmp`;
|
||||
|
||||
try {
|
||||
// Use SQLite's backup API for consistency
|
||||
await execAsync(`sqlite3 "${dbPath}" ".backup '${tempPath}'"`);
|
||||
|
||||
// Verify the backup
|
||||
const verifyResult = await execAsync(`sqlite3 "${tempPath}" "PRAGMA integrity_check"`);
|
||||
if (!verifyResult.stdout.includes('ok')) {
|
||||
throw new Error('Backup integrity check failed');
|
||||
}
|
||||
|
||||
// Move temp file to final location
|
||||
await fs.rename(tempPath, outputPath);
|
||||
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
// Cleanup temp file if exists
|
||||
try {
|
||||
await fs.unlink(tempPath);
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create PostgreSQL backup with proper escaping
|
||||
*/
|
||||
async createPostgreSQLBackup(outputPath, options = {}) {
|
||||
const { host, port, user, password, database } = knexConfig.connection;
|
||||
|
||||
// Build connection string with proper escaping
|
||||
const connectionParts = [
|
||||
`host=${host}`,
|
||||
`port=${port}`,
|
||||
`dbname=${database}`,
|
||||
`user=${user}`
|
||||
];
|
||||
|
||||
// Set PGPASSWORD environment variable for security
|
||||
const env = { ...process.env };
|
||||
if (password) {
|
||||
env.PGPASSWORD = password;
|
||||
}
|
||||
|
||||
// Build pg_dump command with options
|
||||
const pgDumpOptions = [
|
||||
'--verbose',
|
||||
'--no-owner',
|
||||
'--no-privileges',
|
||||
'--clean',
|
||||
'--if-exists',
|
||||
'--format=plain',
|
||||
'--encoding=UTF8'
|
||||
];
|
||||
|
||||
// Add transaction support for consistency
|
||||
if (!options.noTransaction) {
|
||||
pgDumpOptions.push('--single-transaction');
|
||||
}
|
||||
|
||||
// Add compression if not doing it separately
|
||||
if (options.compress && !options.separateCompression) {
|
||||
pgDumpOptions.push('--compress=6');
|
||||
}
|
||||
|
||||
const command = `pg_dump "${connectionParts.join(' ')}" ${pgDumpOptions.join(' ')} > "${outputPath}"`;
|
||||
|
||||
try {
|
||||
const { stderr } = await execAsync(command, {
|
||||
env,
|
||||
maxBuffer: 1024 * 1024 * 100 // 100MB buffer
|
||||
});
|
||||
|
||||
// pg_dump writes progress to stderr, not an error
|
||||
if (stderr && !stderr.includes('dump complete')) {
|
||||
logger.warn('pg_dump warnings:', stderr);
|
||||
}
|
||||
|
||||
// Verify the dump file is not empty
|
||||
const stats = await fs.stat(outputPath);
|
||||
if (stats.size === 0) {
|
||||
throw new Error('Backup file is empty');
|
||||
}
|
||||
|
||||
return { success: true, warnings: stderr };
|
||||
} catch (error) {
|
||||
throw new Error(`PostgreSQL backup failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate backup integrity
|
||||
*/
|
||||
async validateBackup(backupPath, originalChecksums) {
|
||||
const tempDbPath = `${backupPath}.validate`;
|
||||
|
||||
try {
|
||||
if (this.dbType === 'sqlite') {
|
||||
// For SQLite, we can directly check integrity
|
||||
const result = await execAsync(`sqlite3 "${backupPath}" "PRAGMA integrity_check"`);
|
||||
if (!result.stdout.includes('ok')) {
|
||||
throw new Error('Backup integrity check failed');
|
||||
}
|
||||
} else {
|
||||
// For PostgreSQL, we'd need to restore to a temp database
|
||||
// This is more complex and might not be feasible in production
|
||||
logger.info('PostgreSQL backup validation would require restore test');
|
||||
}
|
||||
|
||||
return { valid: true };
|
||||
} finally {
|
||||
// Cleanup
|
||||
try {
|
||||
await fs.unlink(tempDbPath);
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main backup method
|
||||
*/
|
||||
async backup(options = {}) {
|
||||
if (this.isRunning) {
|
||||
throw new Error('Backup already in progress');
|
||||
}
|
||||
|
||||
this.isRunning = true;
|
||||
const startTime = new Date();
|
||||
let backupRun = null;
|
||||
|
||||
try {
|
||||
// Get configuration
|
||||
const config = await this.getBackupConfig();
|
||||
const {
|
||||
destinationPath = '/backup/database',
|
||||
compress = true,
|
||||
validateIntegrity = true,
|
||||
includeChecksums = true
|
||||
} = { ...config, ...options };
|
||||
|
||||
// Create backup directory
|
||||
await fs.mkdir(destinationPath, { recursive: true });
|
||||
|
||||
// Generate backup filename
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const baseName = `picpeak-db-${this.dbType}-${timestamp}`;
|
||||
const sqlFile = path.join(destinationPath, `${baseName}.sql`);
|
||||
const finalFile = compress ? path.join(destinationPath, `${baseName}.sql.gz`) : sqlFile;
|
||||
|
||||
// Create backup run record
|
||||
const [runId] = await db('database_backup_runs').insert({
|
||||
started_at: startTime,
|
||||
status: 'running',
|
||||
backup_type: this.dbType,
|
||||
destination_path: finalFile
|
||||
});
|
||||
|
||||
backupRun = { id: runId };
|
||||
|
||||
// Get initial checksums
|
||||
let tableChecksums = null;
|
||||
if (includeChecksums) {
|
||||
this.updateProgress('Calculating table checksums...');
|
||||
tableChecksums = await this.getTableChecksums();
|
||||
}
|
||||
|
||||
// Get database size
|
||||
const dbSize = await this.getDatabaseSize();
|
||||
|
||||
// Create the backup
|
||||
this.updateProgress('Creating database backup...');
|
||||
if (this.dbType === 'sqlite') {
|
||||
await this.createSQLiteBackup(sqlFile, options);
|
||||
} else {
|
||||
await this.createPostgreSQLBackup(sqlFile, options);
|
||||
}
|
||||
|
||||
// Compress if requested
|
||||
let compressionStats = null;
|
||||
if (compress) {
|
||||
this.updateProgress('Compressing backup...');
|
||||
compressionStats = await this.compressFile(sqlFile, finalFile);
|
||||
await fs.unlink(sqlFile); // Remove uncompressed file
|
||||
}
|
||||
|
||||
// Calculate checksum
|
||||
this.updateProgress('Calculating backup checksum...');
|
||||
const backupChecksum = await this.calculateChecksum(finalFile);
|
||||
|
||||
// Validate if requested
|
||||
if (validateIntegrity && !compress) {
|
||||
this.updateProgress('Validating backup integrity...');
|
||||
await this.validateBackup(finalFile, tableChecksums);
|
||||
}
|
||||
|
||||
// Get final file size
|
||||
const finalStats = await fs.stat(finalFile);
|
||||
|
||||
// Calculate duration
|
||||
const endTime = new Date();
|
||||
const durationSeconds = Math.round((endTime - startTime) / 1000);
|
||||
|
||||
// Update backup run record
|
||||
await db('database_backup_runs')
|
||||
.where('id', runId)
|
||||
.update({
|
||||
completed_at: endTime,
|
||||
status: 'completed',
|
||||
file_path: finalFile,
|
||||
file_size_bytes: finalStats.size,
|
||||
original_size_bytes: dbSize,
|
||||
duration_seconds: durationSeconds,
|
||||
checksum: backupChecksum,
|
||||
compression_ratio: compressionStats?.compressionRatio || null,
|
||||
table_checksums: tableChecksums ? JSON.stringify(tableChecksums) : null,
|
||||
statistics: JSON.stringify({
|
||||
dbType: this.dbType,
|
||||
compressed: compress,
|
||||
validated: validateIntegrity,
|
||||
compressionStats,
|
||||
tableCount: tableChecksums ? Object.keys(tableChecksums).length : null
|
||||
})
|
||||
});
|
||||
|
||||
logger.info(`Database backup completed: ${finalFile} (${(finalStats.size / 1024 / 1024).toFixed(2)} MB) in ${durationSeconds}s`);
|
||||
|
||||
// Send success notification if configured
|
||||
if (config.emailOnSuccess) {
|
||||
await this.sendBackupNotification('success', {
|
||||
duration: durationSeconds,
|
||||
size: finalStats.size,
|
||||
compressionRatio: compressionStats?.compressionRatio,
|
||||
path: finalFile
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
path: finalFile,
|
||||
size: finalStats.size,
|
||||
duration: durationSeconds,
|
||||
checksum: backupChecksum,
|
||||
compressionRatio: compressionStats?.compressionRatio
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Database backup failed:', error);
|
||||
|
||||
// Update backup run record
|
||||
if (backupRun) {
|
||||
await db('database_backup_runs')
|
||||
.where('id', backupRun.id)
|
||||
.update({
|
||||
completed_at: new Date(),
|
||||
status: 'failed',
|
||||
error_message: error.message
|
||||
});
|
||||
}
|
||||
|
||||
// Send failure notification
|
||||
const config = await this.getBackupConfig();
|
||||
if (config.emailOnFailure) {
|
||||
await this.sendBackupNotification('failure', {
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
|
||||
throw error;
|
||||
} finally {
|
||||
this.isRunning = false;
|
||||
this.currentProgress = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update progress
|
||||
*/
|
||||
updateProgress(message, details = {}) {
|
||||
this.currentProgress = {
|
||||
message,
|
||||
details,
|
||||
timestamp: new Date()
|
||||
};
|
||||
logger.info(`Backup progress: ${message}`, details);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current progress
|
||||
*/
|
||||
getProgress() {
|
||||
return this.currentProgress;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup configuration
|
||||
*/
|
||||
async getBackupConfig() {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'database_backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
const config = {};
|
||||
settings.forEach(setting => {
|
||||
try {
|
||||
config[setting.setting_key] = JSON.parse(setting.setting_value);
|
||||
} catch (e) {
|
||||
config[setting.setting_key] = setting.setting_value;
|
||||
}
|
||||
});
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send backup notification email
|
||||
*/
|
||||
async sendBackupNotification(type, details) {
|
||||
const admins = await db('admin_users').where('is_active', formatBoolean(true));
|
||||
|
||||
for (const admin of admins) {
|
||||
if (type === 'success') {
|
||||
await queueEmail(null, admin.email, 'database_backup_completed', {
|
||||
backup_type: this.dbType,
|
||||
duration: `${details.duration} seconds`,
|
||||
file_size: `${(details.size / 1024 / 1024).toFixed(2)} MB`,
|
||||
compression_ratio: details.compressionRatio ? `${details.compressionRatio}%` : 'N/A',
|
||||
file_path: details.path
|
||||
});
|
||||
} else {
|
||||
await queueEmail(null, admin.email, 'database_backup_failed', {
|
||||
backup_type: this.dbType,
|
||||
error_message: details.error,
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old backups
|
||||
*/
|
||||
async cleanupOldBackups(retentionDays = 30) {
|
||||
try {
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
|
||||
// Get old backup records
|
||||
const oldBackups = await db('database_backup_runs')
|
||||
.where('completed_at', '<', cutoffDate)
|
||||
.where('status', 'completed')
|
||||
.select('id', 'file_path');
|
||||
|
||||
let deletedCount = 0;
|
||||
|
||||
for (const backup of oldBackups) {
|
||||
try {
|
||||
// Delete the file
|
||||
if (backup.file_path) {
|
||||
await fs.unlink(backup.file_path);
|
||||
}
|
||||
|
||||
// Delete the record
|
||||
await db('database_backup_runs')
|
||||
.where('id', backup.id)
|
||||
.delete();
|
||||
|
||||
deletedCount++;
|
||||
} catch (error) {
|
||||
logger.error(`Failed to delete old backup ${backup.file_path}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
if (deletedCount > 0) {
|
||||
logger.info(`Cleaned up ${deletedCount} old database backups`);
|
||||
}
|
||||
|
||||
// Also clean up old failed runs
|
||||
await db('database_backup_runs')
|
||||
.where('started_at', '<', cutoffDate)
|
||||
.where('status', 'failed')
|
||||
.delete();
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to cleanup old database backups:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup history
|
||||
*/
|
||||
async getBackupHistory(limit = 10) {
|
||||
return await db('database_backup_runs')
|
||||
.orderBy('started_at', 'desc')
|
||||
.limit(limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore from backup (careful!)
|
||||
*/
|
||||
async restore(backupPath, options = {}) {
|
||||
// This is a dangerous operation and should be used with extreme caution
|
||||
throw new Error('Restore functionality not implemented for safety. Please restore manually.');
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
const databaseBackupService = new DatabaseBackupService();
|
||||
|
||||
// Scheduled backup runner
|
||||
let backupSchedule = null;
|
||||
|
||||
/**
|
||||
* Start scheduled database backups
|
||||
*/
|
||||
async function startScheduledBackups() {
|
||||
const cron = require('node-cron');
|
||||
|
||||
try {
|
||||
const config = await databaseBackupService.getBackupConfig();
|
||||
|
||||
if (!config.enabled) {
|
||||
logger.info('Database backup service is disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
// Stop existing schedule
|
||||
if (backupSchedule) {
|
||||
backupSchedule.stop();
|
||||
}
|
||||
|
||||
// Default schedule: 3 AM daily (offset from file backups at 2 AM)
|
||||
const schedule = config.schedule || '0 3 * * *';
|
||||
|
||||
backupSchedule = cron.schedule(schedule, async () => {
|
||||
logger.info('Starting scheduled database backup');
|
||||
try {
|
||||
await databaseBackupService.backup();
|
||||
await databaseBackupService.cleanupOldBackups(config.retentionDays || 30);
|
||||
} catch (error) {
|
||||
logger.error('Scheduled database backup failed:', error);
|
||||
}
|
||||
});
|
||||
|
||||
logger.info(`Database backup service started with schedule: ${schedule}`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to start database backup service:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop scheduled database backups
|
||||
*/
|
||||
function stopScheduledBackups() {
|
||||
if (backupSchedule) {
|
||||
backupSchedule.stop();
|
||||
backupSchedule = null;
|
||||
logger.info('Database backup service stopped');
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
databaseBackupService,
|
||||
startScheduledBackups,
|
||||
stopScheduledBackups,
|
||||
DatabaseBackupService // Export class for testing
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,311 @@
|
||||
const S3StorageAdapter = require('../s3Storage');
|
||||
const { S3Client } = require('@aws-sdk/client-s3');
|
||||
const { Upload } = require('@aws-sdk/lib-storage');
|
||||
const fs = require('fs');
|
||||
const stream = require('stream');
|
||||
|
||||
// Mock AWS SDK
|
||||
jest.mock('@aws-sdk/client-s3');
|
||||
jest.mock('@aws-sdk/lib-storage');
|
||||
jest.mock('@aws-sdk/s3-request-presigner');
|
||||
|
||||
describe('S3StorageAdapter', () => {
|
||||
let mockS3Client;
|
||||
let mockSend;
|
||||
let s3Storage;
|
||||
|
||||
beforeEach(() => {
|
||||
// Reset mocks
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Mock S3Client
|
||||
mockSend = jest.fn();
|
||||
mockS3Client = {
|
||||
send: mockSend
|
||||
};
|
||||
S3Client.mockImplementation(() => mockS3Client);
|
||||
|
||||
// Create adapter instance
|
||||
s3Storage = new S3StorageAdapter({
|
||||
bucket: 'test-bucket',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: 'test-key',
|
||||
secretAccessKey: 'test-secret'
|
||||
});
|
||||
});
|
||||
|
||||
describe('constructor', () => {
|
||||
it('should initialize with required config', () => {
|
||||
expect(s3Storage.bucket).toBe('test-bucket');
|
||||
expect(s3Storage.config.region).toBe('us-east-1');
|
||||
});
|
||||
|
||||
it('should throw error if bucket is not provided', () => {
|
||||
expect(() => {
|
||||
new S3StorageAdapter({ region: 'us-east-1' });
|
||||
}).toThrow('S3 bucket name is required');
|
||||
});
|
||||
|
||||
it('should configure for MinIO with path style', () => {
|
||||
const minioStorage = new S3StorageAdapter({
|
||||
bucket: 'test-bucket',
|
||||
endpoint: 'http://localhost:9000',
|
||||
forcePathStyle: true,
|
||||
sslEnabled: false
|
||||
});
|
||||
|
||||
expect(S3Client).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
endpoint: 'http://localhost:9000',
|
||||
forcePathStyle: true
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('testConnection', () => {
|
||||
it('should successfully test connection', async () => {
|
||||
mockSend.mockResolvedValueOnce({});
|
||||
|
||||
const result = await s3Storage.testConnection();
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(mockSend).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
input: { Bucket: 'test-bucket' }
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw error on connection failure', async () => {
|
||||
mockSend.mockRejectedValueOnce(new Error('Access Denied'));
|
||||
|
||||
await expect(s3Storage.testConnection()).rejects.toThrow('S3 connection test failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('upload', () => {
|
||||
let mockUpload;
|
||||
let mockDone;
|
||||
|
||||
beforeEach(() => {
|
||||
mockDone = jest.fn().mockResolvedValue({
|
||||
Location: 'https://test-bucket.s3.amazonaws.com/test-key',
|
||||
ETag: '"test-etag"'
|
||||
});
|
||||
|
||||
mockUpload = {
|
||||
on: jest.fn().mockReturnThis(),
|
||||
done: mockDone
|
||||
};
|
||||
|
||||
Upload.mockImplementation(() => mockUpload);
|
||||
|
||||
// Mock fs.stat
|
||||
jest.spyOn(fs.promises, 'stat').mockResolvedValue({
|
||||
size: 1024
|
||||
});
|
||||
|
||||
// Mock fs.createReadStream
|
||||
jest.spyOn(fs, 'createReadStream').mockReturnValue(new stream.Readable());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('should upload file successfully', async () => {
|
||||
const result = await s3Storage.upload('/path/to/file.jpg', 'test-key');
|
||||
|
||||
expect(result.Location).toBe('https://test-bucket.s3.amazonaws.com/test-key');
|
||||
expect(Upload).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
client: mockS3Client,
|
||||
params: expect.objectContaining({
|
||||
Bucket: 'test-bucket',
|
||||
Key: 'test-key',
|
||||
ContentType: 'application/octet-stream'
|
||||
})
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should track upload progress', async () => {
|
||||
const onProgress = jest.fn();
|
||||
let progressCallback;
|
||||
|
||||
mockUpload.on.mockImplementation((event, callback) => {
|
||||
if (event === 'httpUploadProgress') {
|
||||
progressCallback = callback;
|
||||
}
|
||||
return mockUpload;
|
||||
});
|
||||
|
||||
const uploadPromise = s3Storage.upload('/path/to/file.jpg', 'test-key', {
|
||||
onProgress
|
||||
});
|
||||
|
||||
// Simulate progress
|
||||
progressCallback({ loaded: 512, total: 1024 });
|
||||
|
||||
await uploadPromise;
|
||||
|
||||
expect(onProgress).toHaveBeenCalledWith(512, 1024);
|
||||
});
|
||||
|
||||
it('should emit upload events', async () => {
|
||||
const uploadStartSpy = jest.fn();
|
||||
const uploadCompleteSpy = jest.fn();
|
||||
|
||||
s3Storage.on('uploadStart', uploadStartSpy);
|
||||
s3Storage.on('uploadComplete', uploadCompleteSpy);
|
||||
|
||||
await s3Storage.upload('/path/to/file.jpg', 'test-key');
|
||||
|
||||
expect(uploadStartSpy).toHaveBeenCalledWith({ key: 'test-key', size: 1024 });
|
||||
expect(uploadCompleteSpy).toHaveBeenCalledWith({
|
||||
key: 'test-key',
|
||||
location: 'https://test-bucket.s3.amazonaws.com/test-key'
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('exists', () => {
|
||||
it('should return true if object exists', async () => {
|
||||
mockSend.mockResolvedValueOnce({});
|
||||
|
||||
const result = await s3Storage.exists('test-key');
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(mockSend).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
input: { Bucket: 'test-bucket', Key: 'test-key' }
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should return false if object does not exist', async () => {
|
||||
const error = new Error('Not Found');
|
||||
error.name = 'NotFound';
|
||||
error.$metadata = { httpStatusCode: 404 };
|
||||
mockSend.mockRejectedValueOnce(error);
|
||||
|
||||
const result = await s3Storage.exists('test-key');
|
||||
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateKey', () => {
|
||||
it('should generate unique key with timestamp and random string', () => {
|
||||
const key = s3Storage.generateKey('photo.jpg');
|
||||
|
||||
expect(key).toMatch(/^\d+_[a-f0-9]{16}_photo\.jpg$/);
|
||||
});
|
||||
|
||||
it('should add prefix if provided', () => {
|
||||
const key = s3Storage.generateKey('photo.jpg', 'events/wedding');
|
||||
|
||||
expect(key).toMatch(/^events\/wedding\/\d+_[a-f0-9]{16}_photo\.jpg$/);
|
||||
});
|
||||
|
||||
it('should sanitize filename', () => {
|
||||
const key = s3Storage.generateKey('my photo (1).jpg');
|
||||
|
||||
expect(key).toMatch(/^\d+_[a-f0-9]{16}_my_photo__1_\.jpg$/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('retry logic', () => {
|
||||
it('should retry on retryable errors', async () => {
|
||||
const retryableError = new Error('Connection reset');
|
||||
retryableError.code = 'ECONNRESET';
|
||||
|
||||
// First attempt fails, second succeeds
|
||||
mockSend
|
||||
.mockRejectedValueOnce(retryableError)
|
||||
.mockResolvedValueOnce({});
|
||||
|
||||
// Mock setTimeout to speed up test
|
||||
jest.useFakeTimers();
|
||||
|
||||
const promise = s3Storage.exists('test-key');
|
||||
|
||||
// Advance timers
|
||||
jest.runAllTimers();
|
||||
|
||||
const result = await promise;
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(mockSend).toHaveBeenCalledTimes(2);
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should not retry on non-retryable errors', async () => {
|
||||
const nonRetryableError = new Error('Invalid credentials');
|
||||
nonRetryableError.code = 'InvalidCredentials';
|
||||
|
||||
mockSend.mockRejectedValueOnce(nonRetryableError);
|
||||
|
||||
await expect(s3Storage.exists('test-key')).rejects.toThrow('Invalid credentials');
|
||||
expect(mockSend).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should stop retrying after max attempts', async () => {
|
||||
const retryableError = new Error('Service unavailable');
|
||||
retryableError.code = 'ServiceUnavailable';
|
||||
|
||||
mockSend.mockRejectedValue(retryableError);
|
||||
|
||||
// Mock setTimeout to speed up test
|
||||
jest.useFakeTimers();
|
||||
|
||||
const promise = s3Storage.exists('test-key');
|
||||
|
||||
// Advance timers for all retries
|
||||
for (let i = 0; i < 4; i++) {
|
||||
jest.runAllTimers();
|
||||
}
|
||||
|
||||
await expect(promise).rejects.toThrow('Service unavailable');
|
||||
expect(mockSend).toHaveBeenCalledTimes(4); // Initial + 3 retries
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getStats', () => {
|
||||
it('should calculate storage statistics', async () => {
|
||||
mockSend.mockResolvedValueOnce({
|
||||
Contents: [
|
||||
{ Key: 'file1.jpg', Size: 1024 },
|
||||
{ Key: 'file2.jpg', Size: 2048 }
|
||||
],
|
||||
NextContinuationToken: 'token123'
|
||||
}).mockResolvedValueOnce({
|
||||
Contents: [
|
||||
{ Key: 'file3.jpg', Size: 3072 }
|
||||
]
|
||||
});
|
||||
|
||||
const stats = await s3Storage.getStats('events/');
|
||||
|
||||
expect(stats).toEqual({
|
||||
totalSize: 6144,
|
||||
totalCount: 3,
|
||||
totalSizeFormatted: '6 KB'
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('_formatBytes', () => {
|
||||
it('should format bytes correctly', () => {
|
||||
expect(s3Storage._formatBytes(0)).toBe('0 Bytes');
|
||||
expect(s3Storage._formatBytes(1024)).toBe('1 KB');
|
||||
expect(s3Storage._formatBytes(1048576)).toBe('1 MB');
|
||||
expect(s3Storage._formatBytes(1073741824)).toBe('1 GB');
|
||||
expect(s3Storage._formatBytes(1536, 1)).toBe('1.5 KB');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,221 @@
|
||||
/**
|
||||
* Example usage of the S3StorageAdapter
|
||||
*
|
||||
* This file demonstrates how to use the S3 storage adapter for various operations
|
||||
*/
|
||||
|
||||
const S3StorageAdapter = require('./s3Storage');
|
||||
|
||||
// Example 1: Basic AWS S3 Configuration
|
||||
const s3Storage = new S3StorageAdapter({
|
||||
bucket: 'my-photo-bucket',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
});
|
||||
|
||||
// Example 2: MinIO Configuration (S3-compatible)
|
||||
const minioStorage = new S3StorageAdapter({
|
||||
bucket: 'photo-storage',
|
||||
endpoint: 'http://localhost:9000', // MinIO endpoint
|
||||
accessKeyId: 'minioadmin',
|
||||
secretAccessKey: 'minioadmin',
|
||||
forcePathStyle: true, // Required for MinIO
|
||||
sslEnabled: false // For local development
|
||||
});
|
||||
|
||||
// Example 3: DigitalOcean Spaces Configuration
|
||||
const spacesStorage = new S3StorageAdapter({
|
||||
bucket: 'my-space-name',
|
||||
endpoint: 'https://nyc3.digitaloceanspaces.com',
|
||||
region: 'nyc3',
|
||||
accessKeyId: process.env.DO_SPACES_KEY,
|
||||
secretAccessKey: process.env.DO_SPACES_SECRET
|
||||
});
|
||||
|
||||
// Usage Examples
|
||||
async function examples() {
|
||||
try {
|
||||
// Test connection
|
||||
await s3Storage.testConnection();
|
||||
console.log('Connection successful!');
|
||||
|
||||
// Upload a file with progress tracking
|
||||
const uploadResult = await s3Storage.upload(
|
||||
'/path/to/local/photo.jpg',
|
||||
'events/wedding-2024/photo.jpg',
|
||||
{
|
||||
contentType: 'image/jpeg',
|
||||
metadata: {
|
||||
event: 'wedding-2024',
|
||||
photographer: 'John Doe'
|
||||
},
|
||||
onProgress: (loaded, total) => {
|
||||
const percentage = Math.round((loaded / total) * 100);
|
||||
console.log(`Upload progress: ${percentage}%`);
|
||||
}
|
||||
}
|
||||
);
|
||||
console.log('Upload complete:', uploadResult.Location);
|
||||
|
||||
// Upload from stream
|
||||
const readStream = fs.createReadStream('/path/to/large-video.mp4');
|
||||
await s3Storage.uploadStream(readStream, 'events/wedding-2024/video.mp4', {
|
||||
contentType: 'video/mp4',
|
||||
onProgress: (loaded, total) => {
|
||||
console.log(`Streamed ${loaded} of ${total} bytes`);
|
||||
}
|
||||
});
|
||||
|
||||
// Download a file
|
||||
await s3Storage.download(
|
||||
'events/wedding-2024/photo.jpg',
|
||||
'/path/to/downloaded/photo.jpg',
|
||||
{
|
||||
onProgress: (loaded, total) => {
|
||||
const percentage = Math.round((loaded / total) * 100);
|
||||
console.log(`Download progress: ${percentage}%`);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// Get a download stream
|
||||
const downloadStream = await s3Storage.downloadStream('events/wedding-2024/photo.jpg');
|
||||
downloadStream.pipe(fs.createWriteStream('/path/to/output.jpg'));
|
||||
|
||||
// List files
|
||||
const listing = await s3Storage.list('events/wedding-2024/');
|
||||
console.log(`Found ${listing.Contents.length} files`);
|
||||
listing.Contents.forEach(file => {
|
||||
console.log(`- ${file.Key} (${file.Size} bytes)`);
|
||||
});
|
||||
|
||||
// Generate pre-signed URL for temporary access
|
||||
const downloadUrl = await s3Storage.getSignedUrl('getObject', 'events/wedding-2024/photo.jpg', {
|
||||
expiresIn: 3600 // 1 hour
|
||||
});
|
||||
console.log('Pre-signed download URL:', downloadUrl);
|
||||
|
||||
// Generate pre-signed upload URL
|
||||
const uploadUrl = await s3Storage.getSignedUrl('putObject', 'events/wedding-2024/new-photo.jpg', {
|
||||
expiresIn: 1800, // 30 minutes
|
||||
params: {
|
||||
ContentType: 'image/jpeg'
|
||||
}
|
||||
});
|
||||
console.log('Pre-signed upload URL:', uploadUrl);
|
||||
|
||||
// Check if file exists
|
||||
const exists = await s3Storage.exists('events/wedding-2024/photo.jpg');
|
||||
console.log('File exists:', exists);
|
||||
|
||||
// Get metadata
|
||||
const metadata = await s3Storage.getMetadata('events/wedding-2024/photo.jpg');
|
||||
console.log('File metadata:', metadata);
|
||||
|
||||
// Copy file
|
||||
await s3Storage.copy(
|
||||
'events/wedding-2024/photo.jpg',
|
||||
'events/wedding-2024/photo-copy.jpg'
|
||||
);
|
||||
|
||||
// Move file
|
||||
await s3Storage.move(
|
||||
'events/wedding-2024/photo-copy.jpg',
|
||||
'events/wedding-2024/archived/photo.jpg'
|
||||
);
|
||||
|
||||
// Delete file
|
||||
await s3Storage.delete('events/wedding-2024/temp-photo.jpg');
|
||||
|
||||
// Delete multiple files
|
||||
const deleteResult = await s3Storage.deleteMany([
|
||||
'events/wedding-2024/temp1.jpg',
|
||||
'events/wedding-2024/temp2.jpg',
|
||||
'events/wedding-2024/temp3.jpg'
|
||||
]);
|
||||
console.log(`Deleted ${deleteResult.Deleted.length} files`);
|
||||
|
||||
// Get storage statistics
|
||||
const stats = await s3Storage.getStats('events/');
|
||||
console.log(`Total files: ${stats.totalCount}`);
|
||||
console.log(`Total size: ${stats.totalSizeFormatted}`);
|
||||
|
||||
// Listen to events
|
||||
s3Storage.on('uploadProgress', (data) => {
|
||||
console.log(`Uploading ${data.key}: ${data.loaded}/${data.total}`);
|
||||
});
|
||||
|
||||
s3Storage.on('uploadComplete', (data) => {
|
||||
console.log(`Upload completed: ${data.key}`);
|
||||
});
|
||||
|
||||
s3Storage.on('uploadError', (data) => {
|
||||
console.error(`Upload failed for ${data.key}:`, data.error);
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Integration with existing photo upload workflow
|
||||
async function integrateWithPhotoUpload(eventId, files) {
|
||||
const storage = new S3StorageAdapter({
|
||||
bucket: process.env.S3_BUCKET,
|
||||
region: process.env.AWS_REGION,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
});
|
||||
|
||||
const uploadedPhotos = [];
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
// Generate unique S3 key
|
||||
const s3Key = storage.generateKey(file.originalname, `events/${eventId}`);
|
||||
|
||||
// Upload to S3
|
||||
const result = await storage.upload(file.path, s3Key, {
|
||||
contentType: file.mimetype,
|
||||
metadata: {
|
||||
eventId: eventId,
|
||||
originalName: file.originalname,
|
||||
uploadedAt: new Date().toISOString()
|
||||
}
|
||||
});
|
||||
|
||||
uploadedPhotos.push({
|
||||
filename: s3Key,
|
||||
originalName: file.originalname,
|
||||
size: file.size,
|
||||
mimeType: file.mimetype,
|
||||
s3Location: result.Location,
|
||||
s3Key: s3Key
|
||||
});
|
||||
|
||||
// Clean up local temp file
|
||||
await fs.promises.unlink(file.path);
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Failed to upload ${file.originalname}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return uploadedPhotos;
|
||||
}
|
||||
|
||||
// Environment variables needed:
|
||||
// AWS_ACCESS_KEY_ID=your-access-key
|
||||
// AWS_SECRET_ACCESS_KEY=your-secret-key
|
||||
// AWS_REGION=us-east-1
|
||||
// S3_BUCKET=your-bucket-name
|
||||
|
||||
// For MinIO:
|
||||
// MINIO_ENDPOINT=http://localhost:9000
|
||||
// MINIO_ACCESS_KEY=minioadmin
|
||||
// MINIO_SECRET_KEY=minioadmin
|
||||
// MINIO_BUCKET=photo-storage
|
||||
|
||||
module.exports = { examples, integrateWithPhotoUpload };
|
||||
@@ -0,0 +1,733 @@
|
||||
const { S3Client, HeadBucketCommand, HeadObjectCommand, GetObjectCommand, PutObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, ListObjectsV2Command, CopyObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand } = require('@aws-sdk/client-s3');
|
||||
const { Upload } = require('@aws-sdk/lib-storage');
|
||||
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
|
||||
const fs = require('fs');
|
||||
const fsPromises = require('fs').promises;
|
||||
const path = require('path');
|
||||
const stream = require('stream');
|
||||
const crypto = require('crypto');
|
||||
const logger = require('../../utils/logger');
|
||||
|
||||
/**
|
||||
* S3 Storage Adapter for handling file uploads to S3 and S3-compatible services
|
||||
*
|
||||
* Features:
|
||||
* - Support for S3 and S3-compatible services (MinIO, DigitalOcean Spaces, etc.)
|
||||
* - Multipart upload for large files (>100MB)
|
||||
* - Progress tracking with event emitter
|
||||
* - Retry logic with exponential backoff
|
||||
* - Stream support for memory efficiency
|
||||
* - Path-style URL support for MinIO
|
||||
* - Connection testing
|
||||
* - Comprehensive error handling
|
||||
*
|
||||
* @class S3StorageAdapter
|
||||
*/
|
||||
class S3StorageAdapter extends stream.EventEmitter {
|
||||
/**
|
||||
* Creates an instance of S3StorageAdapter
|
||||
*
|
||||
* @param {Object} config - Configuration object
|
||||
* @param {string} config.bucket - S3 bucket name
|
||||
* @param {string} [config.region='us-east-1'] - AWS region
|
||||
* @param {string} [config.endpoint] - Custom endpoint URL for S3-compatible services
|
||||
* @param {string} [config.accessKeyId] - AWS access key ID
|
||||
* @param {string} [config.secretAccessKey] - AWS secret access key
|
||||
* @param {boolean} [config.forcePathStyle=false] - Force path-style URLs (required for MinIO)
|
||||
* @param {boolean} [config.sslEnabled=true] - Enable SSL for connections
|
||||
* @param {number} [config.multipartThreshold=104857600] - Threshold for multipart upload (default 100MB)
|
||||
* @param {number} [config.partSize=10485760] - Part size for multipart upload (default 10MB)
|
||||
* @param {number} [config.maxRetries=3] - Maximum number of retry attempts
|
||||
* @param {number} [config.retryDelay=1000] - Initial retry delay in milliseconds
|
||||
*/
|
||||
constructor(config) {
|
||||
super();
|
||||
|
||||
// Validate required config
|
||||
if (!config.bucket) {
|
||||
throw new Error('S3 bucket name is required');
|
||||
}
|
||||
|
||||
// Set defaults
|
||||
this.config = {
|
||||
region: 'us-east-1',
|
||||
forcePathStyle: false,
|
||||
sslEnabled: true,
|
||||
multipartThreshold: 100 * 1024 * 1024, // 100MB
|
||||
partSize: 10 * 1024 * 1024, // 10MB
|
||||
maxRetries: 3,
|
||||
retryDelay: 1000,
|
||||
...config
|
||||
};
|
||||
|
||||
// Initialize S3 client
|
||||
const s3Config = {
|
||||
region: this.config.region,
|
||||
forcePathStyle: this.config.forcePathStyle
|
||||
};
|
||||
|
||||
// Add credentials if provided
|
||||
if (this.config.accessKeyId && this.config.secretAccessKey) {
|
||||
s3Config.credentials = {
|
||||
accessKeyId: this.config.accessKeyId,
|
||||
secretAccessKey: this.config.secretAccessKey
|
||||
};
|
||||
}
|
||||
|
||||
// Add custom endpoint if provided (for S3-compatible services)
|
||||
if (this.config.endpoint) {
|
||||
s3Config.endpoint = this.config.endpoint;
|
||||
// For MinIO and other S3-compatible services
|
||||
if (!this.config.endpoint.startsWith('https://') && this.config.sslEnabled) {
|
||||
s3Config.endpoint = `https://${this.config.endpoint}`;
|
||||
} else if (!this.config.endpoint.startsWith('http://') && !this.config.sslEnabled) {
|
||||
s3Config.endpoint = `http://${this.config.endpoint}`;
|
||||
}
|
||||
}
|
||||
|
||||
this.s3Client = new S3Client(s3Config);
|
||||
this.bucket = this.config.bucket;
|
||||
|
||||
// Bind methods to preserve context
|
||||
this.upload = this.upload.bind(this);
|
||||
this.uploadStream = this.uploadStream.bind(this);
|
||||
this.download = this.download.bind(this);
|
||||
this.downloadStream = this.downloadStream.bind(this);
|
||||
this.delete = this.delete.bind(this);
|
||||
this.exists = this.exists.bind(this);
|
||||
this.list = this.list.bind(this);
|
||||
this.copy = this.copy.bind(this);
|
||||
this.move = this.move.bind(this);
|
||||
this.getSignedUrl = this.getSignedUrl.bind(this);
|
||||
this.testConnection = this.testConnection.bind(this);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test connection to S3 bucket
|
||||
*
|
||||
* @returns {Promise<boolean>} - True if connection successful
|
||||
* @throws {Error} - If connection fails
|
||||
*/
|
||||
async testConnection() {
|
||||
try {
|
||||
await this.s3Client.send(new HeadBucketCommand({ Bucket: this.bucket }));
|
||||
logger.info(`Successfully connected to S3 bucket: ${this.bucket}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.error(`Failed to connect to S3 bucket ${this.bucket}:`, error);
|
||||
throw new Error(`S3 connection test failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a file to S3 with automatic multipart for large files
|
||||
*
|
||||
* @param {string} localPath - Local file path to upload
|
||||
* @param {string} s3Key - S3 object key (path in bucket)
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @param {Object} [options.metadata] - Object metadata
|
||||
* @param {string} [options.contentType] - Content type
|
||||
* @param {string} [options.cacheControl] - Cache control header
|
||||
* @param {Function} [options.onProgress] - Progress callback function(loaded, total)
|
||||
* @returns {Promise<Object>} - Upload result with Location, ETag, etc.
|
||||
*/
|
||||
async upload(localPath, s3Key, options = {}) {
|
||||
try {
|
||||
const stats = await fsPromises.stat(localPath);
|
||||
const fileSize = stats.size;
|
||||
|
||||
// Emit upload start event
|
||||
this.emit('uploadStart', { key: s3Key, size: fileSize });
|
||||
|
||||
// Create file stream
|
||||
const fileStream = fs.createReadStream(localPath);
|
||||
|
||||
// Prepare upload parameters
|
||||
const uploadParams = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
Body: fileStream,
|
||||
ContentType: options.contentType || 'application/octet-stream',
|
||||
Metadata: options.metadata || {},
|
||||
CacheControl: options.cacheControl
|
||||
};
|
||||
|
||||
// Remove undefined values
|
||||
Object.keys(uploadParams).forEach(key => uploadParams[key] === undefined && delete uploadParams[key]);
|
||||
|
||||
// Use AWS SDK v3 Upload for automatic multipart handling
|
||||
const parallelUploads3 = new Upload({
|
||||
client: this.s3Client,
|
||||
params: uploadParams,
|
||||
queueSize: 4, // Optional: concurrent uploads
|
||||
partSize: this.config.partSize,
|
||||
leavePartsOnError: false
|
||||
});
|
||||
|
||||
// Track upload progress
|
||||
parallelUploads3.on('httpUploadProgress', (progress) => {
|
||||
if (options.onProgress) {
|
||||
options.onProgress(progress.loaded, progress.total);
|
||||
}
|
||||
this.emit('uploadProgress', { key: s3Key, loaded: progress.loaded, total: progress.total });
|
||||
});
|
||||
|
||||
// Perform upload with retry
|
||||
const result = await this._retryOperation(() => parallelUploads3.done());
|
||||
|
||||
this.emit('uploadComplete', { key: s3Key, location: result.Location });
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error(`Failed to upload file ${localPath} to S3:`, error);
|
||||
this.emit('uploadError', { key: s3Key, error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a stream to S3
|
||||
*
|
||||
* @param {stream.Readable} readStream - Readable stream to upload
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @returns {Promise<Object>} - Upload result
|
||||
*/
|
||||
async uploadStream(readStream, s3Key, options = {}) {
|
||||
const uploadParams = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
Body: readStream,
|
||||
ContentType: options.contentType || 'application/octet-stream',
|
||||
Metadata: options.metadata || {},
|
||||
CacheControl: options.cacheControl
|
||||
};
|
||||
|
||||
// Remove undefined values
|
||||
Object.keys(uploadParams).forEach(key => uploadParams[key] === undefined && delete uploadParams[key]);
|
||||
|
||||
const parallelUploads3 = new Upload({
|
||||
client: this.s3Client,
|
||||
params: uploadParams,
|
||||
queueSize: 4,
|
||||
partSize: this.config.partSize,
|
||||
leavePartsOnError: false
|
||||
});
|
||||
|
||||
// Track progress if callback provided
|
||||
if (options.onProgress) {
|
||||
parallelUploads3.on('httpUploadProgress', (progress) => {
|
||||
options.onProgress(progress.loaded, progress.total);
|
||||
this.emit('uploadProgress', { key: s3Key, ...progress });
|
||||
});
|
||||
}
|
||||
|
||||
return await this._retryOperation(() => parallelUploads3.done());
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a file from S3
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {string} localPath - Local file path to save to
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @param {Function} [options.onProgress] - Progress callback
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async download(s3Key, localPath, options = {}) {
|
||||
try {
|
||||
// Ensure directory exists
|
||||
await fsPromises.mkdir(path.dirname(localPath), { recursive: true });
|
||||
|
||||
// Get object metadata first for progress tracking
|
||||
const headResult = await this._retryOperation(() =>
|
||||
this.s3Client.send(new HeadObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key
|
||||
}))
|
||||
);
|
||||
|
||||
const fileSize = headResult.ContentLength;
|
||||
this.emit('downloadStart', { key: s3Key, size: fileSize });
|
||||
|
||||
// Get object
|
||||
const getObjectResult = await this._retryOperation(() =>
|
||||
this.s3Client.send(new GetObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key
|
||||
}))
|
||||
);
|
||||
|
||||
// Create write stream
|
||||
const writeStream = fs.createWriteStream(localPath);
|
||||
|
||||
// Download with progress tracking
|
||||
await new Promise((resolve, reject) => {
|
||||
let downloaded = 0;
|
||||
|
||||
const bodyStream = getObjectResult.Body;
|
||||
|
||||
bodyStream.on('data', (chunk) => {
|
||||
downloaded += chunk.length;
|
||||
if (options.onProgress) {
|
||||
options.onProgress(downloaded, fileSize);
|
||||
}
|
||||
this.emit('downloadProgress', { key: s3Key, loaded: downloaded, total: fileSize });
|
||||
});
|
||||
|
||||
bodyStream.on('error', reject);
|
||||
writeStream.on('error', reject);
|
||||
writeStream.on('finish', resolve);
|
||||
|
||||
bodyStream.pipe(writeStream);
|
||||
});
|
||||
|
||||
this.emit('downloadComplete', { key: s3Key });
|
||||
} catch (error) {
|
||||
logger.error(`Failed to download file ${s3Key} from S3:`, error);
|
||||
this.emit('downloadError', { key: s3Key, error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a download stream from S3
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @param {string} [options.range] - Byte range to download (e.g., 'bytes=0-1023')
|
||||
* @returns {Promise<stream.Readable>} - Readable stream
|
||||
*/
|
||||
async downloadStream(s3Key, options = {}) {
|
||||
const params = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
Range: options.range
|
||||
};
|
||||
|
||||
// Remove undefined values
|
||||
Object.keys(params).forEach(key => params[key] === undefined && delete params[key]);
|
||||
|
||||
const result = await this.s3Client.send(new GetObjectCommand(params));
|
||||
return result.Body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file from S3
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async delete(s3Key) {
|
||||
return await this._retryOperation(() =>
|
||||
this.s3Client.send(new DeleteObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete multiple files from S3
|
||||
*
|
||||
* @param {string[]} s3Keys - Array of S3 object keys
|
||||
* @returns {Promise<Object>} - Delete result
|
||||
*/
|
||||
async deleteMany(s3Keys) {
|
||||
if (!s3Keys || s3Keys.length === 0) {
|
||||
return { Deleted: [], Errors: [] };
|
||||
}
|
||||
|
||||
// S3 deleteObjects has a limit of 1000 keys per request
|
||||
const chunks = [];
|
||||
for (let i = 0; i < s3Keys.length; i += 1000) {
|
||||
chunks.push(s3Keys.slice(i, i + 1000));
|
||||
}
|
||||
|
||||
const results = await Promise.all(
|
||||
chunks.map(chunk =>
|
||||
this._retryOperation(() =>
|
||||
this.s3Client.send(new DeleteObjectsCommand({
|
||||
Bucket: this.bucket,
|
||||
Delete: {
|
||||
Objects: chunk.map(key => ({ Key: key })),
|
||||
Quiet: false
|
||||
}
|
||||
}))
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// Combine results
|
||||
return {
|
||||
Deleted: results.flatMap(r => r.Deleted || []),
|
||||
Errors: results.flatMap(r => r.Errors || [])
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file exists in S3
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @returns {Promise<boolean>} - True if exists
|
||||
*/
|
||||
async exists(s3Key) {
|
||||
try {
|
||||
await this.s3Client.send(new HeadObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key
|
||||
}));
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List files in S3
|
||||
*
|
||||
* @param {string} prefix - S3 prefix to list
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @param {number} [options.maxKeys=1000] - Maximum number of keys to return
|
||||
* @param {string} [options.continuationToken] - Continuation token for pagination
|
||||
* @returns {Promise<Object>} - List result with Contents array and NextContinuationToken
|
||||
*/
|
||||
async list(prefix, options = {}) {
|
||||
const params = {
|
||||
Bucket: this.bucket,
|
||||
Prefix: prefix,
|
||||
MaxKeys: options.maxKeys || 1000,
|
||||
ContinuationToken: options.continuationToken
|
||||
};
|
||||
|
||||
return await this._retryOperation(() =>
|
||||
this.s3Client.send(new ListObjectsV2Command(params))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy a file within S3
|
||||
*
|
||||
* @param {string} sourceKey - Source S3 object key
|
||||
* @param {string} targetKey - Target S3 object key
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @returns {Promise<Object>} - Copy result
|
||||
*/
|
||||
async copy(sourceKey, targetKey, options = {}) {
|
||||
const copySource = `${this.bucket}/${sourceKey}`;
|
||||
|
||||
const params = {
|
||||
Bucket: this.bucket,
|
||||
CopySource: copySource,
|
||||
Key: targetKey,
|
||||
MetadataDirective: options.metadata ? 'REPLACE' : 'COPY',
|
||||
Metadata: options.metadata,
|
||||
ContentType: options.contentType,
|
||||
CacheControl: options.cacheControl
|
||||
};
|
||||
|
||||
// Remove undefined values
|
||||
Object.keys(params).forEach(key => params[key] === undefined && delete params[key]);
|
||||
|
||||
return await this._retryOperation(() =>
|
||||
this.s3Client.send(new CopyObjectCommand(params))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Move a file within S3 (copy then delete)
|
||||
*
|
||||
* @param {string} sourceKey - Source S3 object key
|
||||
* @param {string} targetKey - Target S3 object key
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @returns {Promise<Object>} - Move result
|
||||
*/
|
||||
async move(sourceKey, targetKey, options = {}) {
|
||||
// First copy the object
|
||||
const copyResult = await this.copy(sourceKey, targetKey, options);
|
||||
|
||||
// Then delete the original
|
||||
await this.delete(sourceKey);
|
||||
|
||||
return copyResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a pre-signed URL for downloading or uploading
|
||||
*
|
||||
* @param {string} operation - Operation type ('getObject' or 'putObject')
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @param {number} [options.expiresIn=3600] - URL expiration in seconds
|
||||
* @param {Object} [options.params] - Additional parameters for the operation
|
||||
* @returns {Promise<string>} - Pre-signed URL
|
||||
*/
|
||||
async getSignedUrl(operation, s3Key, options = {}) {
|
||||
const params = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
...options.params
|
||||
};
|
||||
|
||||
let command;
|
||||
switch (operation.toLowerCase()) {
|
||||
case 'getobject':
|
||||
command = new GetObjectCommand(params);
|
||||
break;
|
||||
case 'putobject':
|
||||
command = new PutObjectCommand(params);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unsupported operation: ${operation}`);
|
||||
}
|
||||
|
||||
return await getSignedUrl(this.s3Client, command, {
|
||||
expiresIn: options.expiresIn || 3600
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get object metadata
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @returns {Promise<Object>} - Object metadata
|
||||
*/
|
||||
async getMetadata(s3Key) {
|
||||
return await this._retryOperation(() =>
|
||||
this.s3Client.send(new HeadObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update object metadata
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {Object} metadata - New metadata
|
||||
* @returns {Promise<Object>} - Update result
|
||||
*/
|
||||
async updateMetadata(s3Key, metadata) {
|
||||
// S3 requires copying the object to itself to update metadata
|
||||
return await this.copy(s3Key, s3Key, { metadata });
|
||||
}
|
||||
|
||||
/**
|
||||
* Manual multipart upload for advanced use cases
|
||||
*
|
||||
* @param {string} localPath - Local file path
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {number} fileSize - File size in bytes
|
||||
* @param {Object} options - Upload options
|
||||
* @returns {Promise<Object>} - Upload result
|
||||
* @private
|
||||
*/
|
||||
async _manualMultipartUpload(localPath, s3Key, fileSize, options) {
|
||||
logger.info(`Starting manual multipart upload for ${s3Key} (${fileSize} bytes)`);
|
||||
|
||||
// Initiate multipart upload
|
||||
const multipartParams = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
ContentType: options.contentType || 'application/octet-stream',
|
||||
Metadata: options.metadata || {},
|
||||
CacheControl: options.cacheControl
|
||||
};
|
||||
|
||||
// Remove undefined values
|
||||
Object.keys(multipartParams).forEach(key => multipartParams[key] === undefined && delete multipartParams[key]);
|
||||
|
||||
const multipart = await this._retryOperation(() =>
|
||||
this.s3Client.send(new CreateMultipartUploadCommand(multipartParams))
|
||||
);
|
||||
|
||||
const uploadId = multipart.UploadId;
|
||||
const partSize = this.config.partSize;
|
||||
const numParts = Math.ceil(fileSize / partSize);
|
||||
|
||||
let uploaded = 0;
|
||||
const parts = [];
|
||||
|
||||
try {
|
||||
// Upload parts
|
||||
for (let partNum = 1; partNum <= numParts; partNum++) {
|
||||
const start = (partNum - 1) * partSize;
|
||||
const end = Math.min(start + partSize, fileSize);
|
||||
|
||||
const partStream = fs.createReadStream(localPath, {
|
||||
start,
|
||||
end: end - 1
|
||||
});
|
||||
|
||||
const partParams = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
PartNumber: partNum,
|
||||
UploadId: uploadId,
|
||||
Body: partStream
|
||||
};
|
||||
|
||||
// Upload part with retry
|
||||
const partResult = await this._retryOperation(() =>
|
||||
this.s3Client.send(new UploadPartCommand(partParams))
|
||||
);
|
||||
|
||||
parts.push({
|
||||
ETag: partResult.ETag,
|
||||
PartNumber: partNum
|
||||
});
|
||||
|
||||
uploaded += (end - start);
|
||||
|
||||
if (options.onProgress) {
|
||||
options.onProgress(uploaded, fileSize);
|
||||
}
|
||||
this.emit('uploadProgress', { key: s3Key, loaded: uploaded, total: fileSize });
|
||||
|
||||
logger.info(`Uploaded part ${partNum}/${numParts} for ${s3Key}`);
|
||||
}
|
||||
|
||||
// Complete multipart upload
|
||||
const completeParams = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: { Parts: parts }
|
||||
};
|
||||
|
||||
const result = await this._retryOperation(() =>
|
||||
this.s3Client.send(new CompleteMultipartUploadCommand(completeParams))
|
||||
);
|
||||
|
||||
this.emit('uploadComplete', { key: s3Key, location: result.Location });
|
||||
logger.info(`Completed multipart upload for ${s3Key}`);
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
// Abort multipart upload on error
|
||||
logger.error(`Multipart upload failed for ${s3Key}, aborting:`, error);
|
||||
|
||||
try {
|
||||
await this.s3Client.send(new AbortMultipartUploadCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
UploadId: uploadId
|
||||
}));
|
||||
} catch (abortError) {
|
||||
logger.error(`Failed to abort multipart upload:`, abortError);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry operation with exponential backoff
|
||||
* @private
|
||||
*/
|
||||
async _retryOperation(operation, retryCount = 0) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
if (retryCount >= this.config.maxRetries) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Check if error is retryable
|
||||
const retryableErrors = ['ECONNRESET', 'ETIMEDOUT', 'ENOTFOUND', 'ESOCKETTIMEDOUT', 'RequestTimeout', 'SlowDown', 'ServiceUnavailable', 'InternalError'];
|
||||
const isRetryable = retryableErrors.some(code =>
|
||||
error.code === code ||
|
||||
error.name === code ||
|
||||
error.$metadata?.httpStatusCode === 503 ||
|
||||
error.$metadata?.httpStatusCode === 500
|
||||
);
|
||||
|
||||
if (!isRetryable) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Calculate delay with exponential backoff and jitter
|
||||
const delay = Math.min(
|
||||
this.config.retryDelay * Math.pow(2, retryCount) + Math.random() * 1000,
|
||||
30000 // Max 30 seconds
|
||||
);
|
||||
|
||||
logger.warn(`Retrying operation after ${delay}ms (attempt ${retryCount + 1}/${this.config.maxRetries}):`, error.message);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
|
||||
return this._retryOperation(operation, retryCount + 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a unique S3 key for a file
|
||||
*
|
||||
* @param {string} originalName - Original filename
|
||||
* @param {string} [prefix=''] - Optional prefix for the key
|
||||
* @returns {string} - Generated S3 key
|
||||
*/
|
||||
generateKey(originalName, prefix = '') {
|
||||
const timestamp = Date.now();
|
||||
const randomStr = crypto.randomBytes(8).toString('hex');
|
||||
const ext = path.extname(originalName);
|
||||
const basename = path.basename(originalName, ext);
|
||||
|
||||
// Sanitize basename
|
||||
const safeName = basename.replace(/[^a-zA-Z0-9-_]/g, '_');
|
||||
|
||||
const key = `${timestamp}_${randomStr}_${safeName}${ext}`;
|
||||
|
||||
return prefix ? path.posix.join(prefix, key) : key;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get storage statistics
|
||||
*
|
||||
* @param {string} [prefix=''] - Optional prefix to filter
|
||||
* @returns {Promise<Object>} - Storage statistics
|
||||
*/
|
||||
async getStats(prefix = '') {
|
||||
let totalSize = 0;
|
||||
let totalCount = 0;
|
||||
let continuationToken;
|
||||
|
||||
do {
|
||||
const result = await this.list(prefix, { continuationToken });
|
||||
|
||||
if (result.Contents) {
|
||||
totalCount += result.Contents.length;
|
||||
totalSize += result.Contents.reduce((sum, obj) => sum + (obj.Size || 0), 0);
|
||||
}
|
||||
|
||||
continuationToken = result.NextContinuationToken;
|
||||
} while (continuationToken);
|
||||
|
||||
return {
|
||||
totalSize,
|
||||
totalCount,
|
||||
totalSizeFormatted: this._formatBytes(totalSize)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format bytes to human readable format
|
||||
* @private
|
||||
*/
|
||||
_formatBytes(bytes, decimals = 2) {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
|
||||
const k = 1024;
|
||||
const dm = decimals < 0 ? 0 : decimals;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
||||
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = S3StorageAdapter;
|
||||
Reference in New Issue
Block a user