feat: implement comprehensive backup and restore system with S3 support
- Add S3/MinIO storage adapter with multipart upload support - Implement database backup service for SQLite and PostgreSQL - Create backup manifest generator for tracking backup contents - Enhance backup service with S3 integration and incremental backups - Add restore service with safety measures and rollback capability - Create comprehensive test suite for all backup functionality - Add admin API endpoints for backup/restore management - Implement frontend UI with dashboard, configuration, and restore wizard - Add roadmap section to README with implemented backup feature This implementation provides: - Multiple backup destinations (local, rsync, S3/MinIO) - Intelligent change detection to minimize backup frequency - Full database backups with compression - Manifest-based restore with integrity validation - Pre-restore safety backups with rollback - Comprehensive error handling and monitoring - User-friendly admin interface 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,251 @@
|
||||
const { DatabaseBackupService } = require('../databaseBackup');
|
||||
const { db } = require('../../database/db');
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
|
||||
// Mock dependencies
|
||||
jest.mock('../../database/db');
|
||||
jest.mock('../../utils/logger');
|
||||
jest.mock('../emailProcessor');
|
||||
jest.mock('child_process');
|
||||
|
||||
describe('DatabaseBackupService', () => {
|
||||
let service;
|
||||
let mockExecAsync;
|
||||
|
||||
beforeEach(() => {
|
||||
service = new DatabaseBackupService();
|
||||
mockExecAsync = jest.fn();
|
||||
|
||||
// Reset mocks
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Mock execAsync
|
||||
const childProcess = require('child_process');
|
||||
childProcess.exec = jest.fn((cmd, opts, callback) => {
|
||||
if (callback) {
|
||||
callback(null, { stdout: 'ok', stderr: '' });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// Cleanup test files
|
||||
try {
|
||||
await fs.rmdir('/tmp/test-backup', { recursive: true });
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
});
|
||||
|
||||
describe('calculateChecksum', () => {
|
||||
it('should calculate SHA256 checksum of a file', async () => {
|
||||
const testFile = '/tmp/test-checksum.txt';
|
||||
const testContent = 'Hello, World!';
|
||||
await fs.writeFile(testFile, testContent);
|
||||
|
||||
const checksum = await service.calculateChecksum(testFile);
|
||||
|
||||
// Expected checksum for "Hello, World!"
|
||||
const expectedChecksum = crypto
|
||||
.createHash('sha256')
|
||||
.update(testContent)
|
||||
.digest('hex');
|
||||
|
||||
expect(checksum).toBe(expectedChecksum);
|
||||
|
||||
await fs.unlink(testFile);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getTableChecksums', () => {
|
||||
it('should get checksums for all tables', async () => {
|
||||
// Mock getTables
|
||||
service.getTables = jest.fn().mockResolvedValue(['events', 'photos']);
|
||||
|
||||
// Mock SQLite response
|
||||
db.raw = jest.fn()
|
||||
.mockResolvedValueOnce([{ row_count: 10, data_sum: 1000 }])
|
||||
.mockResolvedValueOnce([{ row_count: 20, data_sum: 2000 }]);
|
||||
|
||||
const checksums = await service.getTableChecksums();
|
||||
|
||||
expect(checksums).toHaveProperty('events');
|
||||
expect(checksums).toHaveProperty('photos');
|
||||
expect(checksums.events.rowCount).toBe(10);
|
||||
expect(checksums.photos.rowCount).toBe(20);
|
||||
expect(checksums.events.checksum).toBeDefined();
|
||||
expect(checksums.photos.checksum).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getTables', () => {
|
||||
it('should get list of tables for SQLite', async () => {
|
||||
service.dbType = 'sqlite';
|
||||
|
||||
db.raw = jest.fn().mockResolvedValue([
|
||||
{ name: 'events' },
|
||||
{ name: 'photos' },
|
||||
{ name: 'admin_users' }
|
||||
]);
|
||||
|
||||
const tables = await service.getTables();
|
||||
|
||||
expect(tables).toEqual(['events', 'photos', 'admin_users']);
|
||||
expect(db.raw).toHaveBeenCalledWith(expect.stringContaining('sqlite_master'));
|
||||
});
|
||||
|
||||
it('should get list of tables for PostgreSQL', async () => {
|
||||
service.dbType = 'postgresql';
|
||||
|
||||
db.raw = jest.fn().mockResolvedValue({
|
||||
rows: [
|
||||
{ table_name: 'events' },
|
||||
{ table_name: 'photos' },
|
||||
{ table_name: 'admin_users' }
|
||||
]
|
||||
});
|
||||
|
||||
const tables = await service.getTables();
|
||||
|
||||
expect(tables).toEqual(['events', 'photos', 'admin_users']);
|
||||
expect(db.raw).toHaveBeenCalledWith(expect.stringContaining('information_schema.tables'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('getDatabaseSize', () => {
|
||||
it('should get database size for SQLite', async () => {
|
||||
service.dbType = 'sqlite';
|
||||
const mockSize = 1024 * 1024 * 10; // 10MB
|
||||
|
||||
// Mock fs.stat
|
||||
const originalStat = fs.stat;
|
||||
fs.stat = jest.fn().mockResolvedValue({ size: mockSize });
|
||||
|
||||
const size = await service.getDatabaseSize();
|
||||
|
||||
expect(size).toBe(mockSize);
|
||||
|
||||
fs.stat = originalStat;
|
||||
});
|
||||
|
||||
it('should get database size for PostgreSQL', async () => {
|
||||
service.dbType = 'postgresql';
|
||||
const mockSize = 1024 * 1024 * 100; // 100MB
|
||||
|
||||
db.raw = jest.fn().mockResolvedValue({
|
||||
rows: [{ size: mockSize.toString() }]
|
||||
});
|
||||
|
||||
const size = await service.getDatabaseSize();
|
||||
|
||||
expect(size).toBe(mockSize);
|
||||
expect(db.raw).toHaveBeenCalledWith(expect.stringContaining('pg_database_size'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('compressFile', () => {
|
||||
it('should compress file and return stats', async () => {
|
||||
const testFile = '/tmp/test-compress.txt';
|
||||
const compressedFile = '/tmp/test-compress.txt.gz';
|
||||
|
||||
// Create test file with repetitive content (compresses well)
|
||||
const testContent = 'Hello, World! '.repeat(1000);
|
||||
await fs.writeFile(testFile, testContent);
|
||||
|
||||
const stats = await service.compressFile(testFile, compressedFile);
|
||||
|
||||
expect(stats.originalSize).toBeGreaterThan(0);
|
||||
expect(stats.compressedSize).toBeGreaterThan(0);
|
||||
expect(stats.compressedSize).toBeLessThan(stats.originalSize);
|
||||
expect(parseFloat(stats.compressionRatio)).toBeGreaterThan(0);
|
||||
|
||||
// Cleanup
|
||||
await fs.unlink(testFile);
|
||||
await fs.unlink(compressedFile);
|
||||
});
|
||||
});
|
||||
|
||||
describe('backup configuration', () => {
|
||||
it('should get backup configuration from database', async () => {
|
||||
const mockConfig = [
|
||||
{ setting_key: 'database_backup_enabled', setting_value: 'true' },
|
||||
{ setting_key: 'database_backup_compress', setting_value: 'true' },
|
||||
{ setting_key: 'database_backup_retention_days', setting_value: '30' }
|
||||
];
|
||||
|
||||
db.mockReturnValue({
|
||||
where: jest.fn().mockReturnThis(),
|
||||
select: jest.fn().mockResolvedValue(mockConfig)
|
||||
});
|
||||
|
||||
const config = await service.getBackupConfig();
|
||||
|
||||
expect(config.database_backup_enabled).toBe(true);
|
||||
expect(config.database_backup_compress).toBe(true);
|
||||
expect(config.database_backup_retention_days).toBe(30);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cleanupOldBackups', () => {
|
||||
it('should delete old backup files and records', async () => {
|
||||
const oldBackups = [
|
||||
{ id: 1, file_path: '/backup/old1.sql.gz' },
|
||||
{ id: 2, file_path: '/backup/old2.sql.gz' }
|
||||
];
|
||||
|
||||
db.mockReturnValue({
|
||||
where: jest.fn().mockReturnThis(),
|
||||
select: jest.fn().mockResolvedValue(oldBackups),
|
||||
delete: jest.fn().mockResolvedValue(1)
|
||||
});
|
||||
|
||||
// Mock fs.unlink
|
||||
fs.unlink = jest.fn().mockResolvedValue(undefined);
|
||||
|
||||
await service.cleanupOldBackups(30);
|
||||
|
||||
expect(fs.unlink).toHaveBeenCalledTimes(2);
|
||||
expect(fs.unlink).toHaveBeenCalledWith('/backup/old1.sql.gz');
|
||||
expect(fs.unlink).toHaveBeenCalledWith('/backup/old2.sql.gz');
|
||||
});
|
||||
});
|
||||
|
||||
describe('progress tracking', () => {
|
||||
it('should update and retrieve progress', () => {
|
||||
expect(service.getProgress()).toBeNull();
|
||||
|
||||
service.updateProgress('Testing...', { step: 1 });
|
||||
|
||||
const progress = service.getProgress();
|
||||
expect(progress.message).toBe('Testing...');
|
||||
expect(progress.details.step).toBe(1);
|
||||
expect(progress.timestamp).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('backup history', () => {
|
||||
it('should retrieve backup history', async () => {
|
||||
const mockHistory = [
|
||||
{
|
||||
id: 1,
|
||||
started_at: new Date(),
|
||||
completed_at: new Date(),
|
||||
status: 'completed',
|
||||
file_size_bytes: 1024000
|
||||
}
|
||||
];
|
||||
|
||||
db.mockReturnValue({
|
||||
orderBy: jest.fn().mockReturnThis(),
|
||||
limit: jest.fn().mockResolvedValue(mockHistory)
|
||||
});
|
||||
|
||||
const history = await service.getBackupHistory(10);
|
||||
|
||||
expect(history).toEqual(mockHistory);
|
||||
expect(history.length).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,501 @@
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const yaml = require('js-yaml');
|
||||
const os = require('os');
|
||||
const { db } = require('../database/db');
|
||||
const logger = require('../utils/logger');
|
||||
|
||||
/**
|
||||
* Backup Manifest Generator
|
||||
*
|
||||
* Generates comprehensive manifests for backups including:
|
||||
* - Version information (app, node, OS)
|
||||
* - File listings with metadata and checksums
|
||||
* - Database information
|
||||
* - System state at backup time
|
||||
* - Support for both JSON and YAML formats
|
||||
* - Incremental backup support with parent references
|
||||
*/
|
||||
|
||||
class BackupManifestGenerator {
|
||||
constructor() {
|
||||
this.appVersion = require('../../package.json').version;
|
||||
this.nodeVersion = process.version;
|
||||
this.platform = process.platform;
|
||||
this.osRelease = os.release();
|
||||
this.hostname = os.hostname();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a comprehensive backup manifest
|
||||
* @param {Object} options - Manifest generation options
|
||||
* @param {string} options.backupType - 'full' or 'incremental'
|
||||
* @param {string} options.backupPath - Path to the backup directory
|
||||
* @param {Array} options.files - Array of backed up files with metadata
|
||||
* @param {Object} options.databaseInfo - Database backup information
|
||||
* @param {string} options.parentBackupId - For incremental backups, reference to parent
|
||||
* @param {string} options.format - 'json' or 'yaml' (default: 'json')
|
||||
* @param {Object} options.customMetadata - Additional metadata to include
|
||||
* @returns {Object} Generated manifest object
|
||||
*/
|
||||
async generateManifest(options) {
|
||||
const {
|
||||
backupType = 'full',
|
||||
backupPath,
|
||||
files = [],
|
||||
databaseInfo = {},
|
||||
parentBackupId = null,
|
||||
format = 'json',
|
||||
customMetadata = {}
|
||||
} = options;
|
||||
|
||||
const manifest = {
|
||||
// Manifest metadata
|
||||
manifest: {
|
||||
version: '2.0',
|
||||
created: new Date().toISOString(),
|
||||
generator: 'PicPeak Backup Manifest Generator',
|
||||
format: format
|
||||
},
|
||||
|
||||
// Backup information
|
||||
backup: {
|
||||
id: this.generateBackupId(),
|
||||
type: backupType,
|
||||
timestamp: new Date().toISOString(),
|
||||
path: backupPath,
|
||||
parent_backup_id: parentBackupId,
|
||||
retention_days: customMetadata.retentionDays || 30
|
||||
},
|
||||
|
||||
// System information
|
||||
system: {
|
||||
hostname: this.hostname,
|
||||
platform: this.platform,
|
||||
os_release: this.osRelease,
|
||||
architecture: os.arch(),
|
||||
cpu_count: os.cpus().length,
|
||||
total_memory: os.totalmem(),
|
||||
free_memory: os.freemem(),
|
||||
uptime: os.uptime()
|
||||
},
|
||||
|
||||
// Application information
|
||||
application: {
|
||||
name: 'PicPeak',
|
||||
version: this.appVersion,
|
||||
node_version: this.nodeVersion,
|
||||
environment: process.env.NODE_ENV || 'production',
|
||||
storage_path: process.env.STORAGE_PATH || path.join(__dirname, '../../../storage')
|
||||
},
|
||||
|
||||
// Files information
|
||||
files: {
|
||||
count: files.length,
|
||||
total_size: files.reduce((sum, file) => sum + (file.size || 0), 0),
|
||||
checksums: await this.generateFileChecksums(files),
|
||||
manifest: files.map(file => ({
|
||||
path: file.relativePath || file.path,
|
||||
size: file.size,
|
||||
modified: file.modified,
|
||||
checksum: file.checksum,
|
||||
type: this.getFileType(file.path),
|
||||
permissions: file.permissions
|
||||
}))
|
||||
},
|
||||
|
||||
// Database information
|
||||
database: {
|
||||
type: databaseInfo.type || this.getDatabaseType(),
|
||||
backup_file: databaseInfo.backupFile,
|
||||
size: databaseInfo.size,
|
||||
checksum: databaseInfo.checksum,
|
||||
tables: databaseInfo.tables || {},
|
||||
row_counts: databaseInfo.rowCounts || {},
|
||||
schema_version: await this.getSchemaVersion()
|
||||
},
|
||||
|
||||
// Verification information
|
||||
verification: {
|
||||
total_checksum: null, // Will be calculated after manifest is complete
|
||||
file_count_check: files.length,
|
||||
size_check: files.reduce((sum, file) => sum + (file.size || 0), 0),
|
||||
integrity_timestamp: new Date().toISOString()
|
||||
},
|
||||
|
||||
// Custom metadata
|
||||
metadata: {
|
||||
...customMetadata,
|
||||
backup_settings: await this.getBackupSettings(),
|
||||
active_events_count: await this.getActiveEventsCount(),
|
||||
archived_events_count: await this.getArchivedEventsCount(),
|
||||
total_photos_count: await this.getTotalPhotosCount()
|
||||
}
|
||||
};
|
||||
|
||||
// Calculate total checksum of the manifest
|
||||
manifest.verification.total_checksum = this.calculateManifestChecksum(manifest);
|
||||
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save manifest to file
|
||||
* @param {Object} manifest - Manifest object to save
|
||||
* @param {string} filePath - Path to save the manifest
|
||||
* @param {string} format - 'json' or 'yaml'
|
||||
*/
|
||||
async saveManifest(manifest, filePath, format = 'json') {
|
||||
try {
|
||||
let content;
|
||||
|
||||
if (format === 'yaml') {
|
||||
content = yaml.dump(manifest, {
|
||||
indent: 2,
|
||||
lineWidth: -1,
|
||||
noRefs: true,
|
||||
sortKeys: true
|
||||
});
|
||||
} else {
|
||||
content = JSON.stringify(manifest, null, 2);
|
||||
}
|
||||
|
||||
await fs.writeFile(filePath, content, 'utf8');
|
||||
logger.info(`Manifest saved to ${filePath} (format: ${format})`);
|
||||
|
||||
return filePath;
|
||||
} catch (error) {
|
||||
logger.error('Failed to save manifest:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load and validate an existing manifest
|
||||
* @param {string} filePath - Path to the manifest file
|
||||
* @returns {Object} Loaded and validated manifest
|
||||
*/
|
||||
async loadManifest(filePath) {
|
||||
try {
|
||||
const content = await fs.readFile(filePath, 'utf8');
|
||||
let manifest;
|
||||
|
||||
// Detect format and parse
|
||||
if (filePath.endsWith('.yaml') || filePath.endsWith('.yml')) {
|
||||
manifest = yaml.load(content);
|
||||
} else {
|
||||
manifest = JSON.parse(content);
|
||||
}
|
||||
|
||||
// Validate manifest structure
|
||||
this.validateManifest(manifest);
|
||||
|
||||
return manifest;
|
||||
} catch (error) {
|
||||
logger.error('Failed to load manifest:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate manifest structure and integrity
|
||||
* @param {Object} manifest - Manifest to validate
|
||||
* @throws {Error} If validation fails
|
||||
*/
|
||||
validateManifest(manifest) {
|
||||
// Check required sections
|
||||
const requiredSections = ['manifest', 'backup', 'system', 'application', 'files', 'database', 'verification'];
|
||||
for (const section of requiredSections) {
|
||||
if (!manifest[section]) {
|
||||
throw new Error(`Missing required section: ${section}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate manifest version
|
||||
if (!manifest.manifest.version) {
|
||||
throw new Error('Missing manifest version');
|
||||
}
|
||||
|
||||
// Validate file checksums
|
||||
if (manifest.files.count !== manifest.files.manifest.length) {
|
||||
throw new Error('File count mismatch');
|
||||
}
|
||||
|
||||
// Validate total checksum
|
||||
const calculatedChecksum = this.calculateManifestChecksum(manifest);
|
||||
if (manifest.verification.total_checksum !== calculatedChecksum) {
|
||||
throw new Error('Manifest checksum verification failed');
|
||||
}
|
||||
|
||||
logger.info('Manifest validation passed');
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two manifests for incremental backup
|
||||
* @param {Object} currentManifest - Current backup manifest
|
||||
* @param {Object} parentManifest - Parent backup manifest
|
||||
* @returns {Object} Comparison results
|
||||
*/
|
||||
compareManifests(currentManifest, parentManifest) {
|
||||
const comparison = {
|
||||
added_files: [],
|
||||
modified_files: [],
|
||||
deleted_files: [],
|
||||
unchanged_files: [],
|
||||
size_difference: 0,
|
||||
database_changes: {}
|
||||
};
|
||||
|
||||
// Create file maps for easy comparison
|
||||
const currentFiles = new Map(
|
||||
currentManifest.files.manifest.map(f => [f.path, f])
|
||||
);
|
||||
const parentFiles = new Map(
|
||||
parentManifest.files.manifest.map(f => [f.path, f])
|
||||
);
|
||||
|
||||
// Find added and modified files
|
||||
for (const [path, file] of currentFiles) {
|
||||
const parentFile = parentFiles.get(path);
|
||||
if (!parentFile) {
|
||||
comparison.added_files.push(file);
|
||||
comparison.size_difference += file.size;
|
||||
} else if (file.checksum !== parentFile.checksum) {
|
||||
comparison.modified_files.push(file);
|
||||
comparison.size_difference += file.size - parentFile.size;
|
||||
} else {
|
||||
comparison.unchanged_files.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
// Find deleted files
|
||||
for (const [path, file] of parentFiles) {
|
||||
if (!currentFiles.has(path)) {
|
||||
comparison.deleted_files.push(file);
|
||||
comparison.size_difference -= file.size;
|
||||
}
|
||||
}
|
||||
|
||||
// Compare database info
|
||||
comparison.database_changes = {
|
||||
size_difference: currentManifest.database.size - parentManifest.database.size,
|
||||
checksum_changed: currentManifest.database.checksum !== parentManifest.database.checksum,
|
||||
schema_version_changed: currentManifest.database.schema_version !== parentManifest.database.schema_version
|
||||
};
|
||||
|
||||
return comparison;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate incremental manifest based on parent
|
||||
* @param {Object} options - Manifest generation options
|
||||
* @param {Object} parentManifest - Parent backup manifest
|
||||
* @returns {Object} Incremental manifest
|
||||
*/
|
||||
async generateIncrementalManifest(options, parentManifest) {
|
||||
const fullManifest = await this.generateManifest({
|
||||
...options,
|
||||
backupType: 'incremental'
|
||||
});
|
||||
|
||||
const comparison = this.compareManifests(fullManifest, parentManifest);
|
||||
|
||||
// Add incremental-specific information
|
||||
fullManifest.incremental = {
|
||||
parent_backup_id: parentManifest.backup.id,
|
||||
parent_timestamp: parentManifest.backup.timestamp,
|
||||
changes: {
|
||||
added_files_count: comparison.added_files.length,
|
||||
modified_files_count: comparison.modified_files.length,
|
||||
deleted_files_count: comparison.deleted_files.length,
|
||||
unchanged_files_count: comparison.unchanged_files.length,
|
||||
size_difference: comparison.size_difference
|
||||
},
|
||||
added_files: comparison.added_files.map(f => f.path),
|
||||
modified_files: comparison.modified_files.map(f => f.path),
|
||||
deleted_files: comparison.deleted_files.map(f => f.path)
|
||||
};
|
||||
|
||||
return fullManifest;
|
||||
}
|
||||
|
||||
// Helper methods
|
||||
|
||||
generateBackupId() {
|
||||
const timestamp = new Date().toISOString().replace(/[:-]/g, '').replace('T', '-').split('.')[0];
|
||||
const random = crypto.randomBytes(4).toString('hex');
|
||||
return `backup-${timestamp}-${random}`;
|
||||
}
|
||||
|
||||
async generateFileChecksums(files) {
|
||||
const checksums = {};
|
||||
for (const file of files) {
|
||||
if (file.checksum) {
|
||||
checksums[file.relativePath || file.path] = file.checksum;
|
||||
}
|
||||
}
|
||||
return checksums;
|
||||
}
|
||||
|
||||
getFileType(filePath) {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
const typeMap = {
|
||||
'.jpg': 'image',
|
||||
'.jpeg': 'image',
|
||||
'.png': 'image',
|
||||
'.gif': 'image',
|
||||
'.webp': 'image',
|
||||
'.zip': 'archive',
|
||||
'.sql': 'database',
|
||||
'.db': 'database',
|
||||
'.json': 'config',
|
||||
'.yaml': 'config',
|
||||
'.yml': 'config'
|
||||
};
|
||||
return typeMap[ext] || 'other';
|
||||
}
|
||||
|
||||
getDatabaseType() {
|
||||
return process.env.DB_TYPE === 'postgresql' ? 'postgresql' : 'sqlite';
|
||||
}
|
||||
|
||||
async getSchemaVersion() {
|
||||
try {
|
||||
const result = await db('migrations')
|
||||
.orderBy('run_at', 'desc')
|
||||
.first();
|
||||
return result ? result.migration_name : 'unknown';
|
||||
} catch (error) {
|
||||
return 'unknown';
|
||||
}
|
||||
}
|
||||
|
||||
async getBackupSettings() {
|
||||
try {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
const config = {};
|
||||
settings.forEach(setting => {
|
||||
try {
|
||||
config[setting.setting_key] = JSON.parse(setting.setting_value);
|
||||
} catch (e) {
|
||||
config[setting.setting_key] = setting.setting_value;
|
||||
}
|
||||
});
|
||||
|
||||
return config;
|
||||
} catch (error) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
async getActiveEventsCount() {
|
||||
try {
|
||||
const result = await db('events')
|
||||
.where('status', 'active')
|
||||
.count('* as count')
|
||||
.first();
|
||||
return result ? parseInt(result.count) : 0;
|
||||
} catch (error) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async getArchivedEventsCount() {
|
||||
try {
|
||||
const result = await db('events')
|
||||
.where('status', 'archived')
|
||||
.count('* as count')
|
||||
.first();
|
||||
return result ? parseInt(result.count) : 0;
|
||||
} catch (error) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async getTotalPhotosCount() {
|
||||
try {
|
||||
const result = await db('photos')
|
||||
.count('* as count')
|
||||
.first();
|
||||
return result ? parseInt(result.count) : 0;
|
||||
} catch (error) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
calculateManifestChecksum(manifest) {
|
||||
// Create a copy without the checksum field
|
||||
const manifestCopy = JSON.parse(JSON.stringify(manifest));
|
||||
if (manifestCopy.verification) {
|
||||
delete manifestCopy.verification.total_checksum;
|
||||
}
|
||||
|
||||
// Calculate SHA256 of the sorted JSON
|
||||
const content = JSON.stringify(manifestCopy, Object.keys(manifestCopy).sort());
|
||||
return crypto.createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a summary report from a manifest
|
||||
* @param {Object} manifest - Manifest to summarize
|
||||
* @returns {string} Human-readable summary
|
||||
*/
|
||||
generateSummaryReport(manifest) {
|
||||
const report = [];
|
||||
|
||||
report.push('=== BACKUP MANIFEST SUMMARY ===');
|
||||
report.push(`Backup ID: ${manifest.backup.id}`);
|
||||
report.push(`Type: ${manifest.backup.type}`);
|
||||
report.push(`Created: ${manifest.backup.timestamp}`);
|
||||
|
||||
if (manifest.backup.parent_backup_id) {
|
||||
report.push(`Parent Backup: ${manifest.backup.parent_backup_id}`);
|
||||
}
|
||||
|
||||
report.push('\n--- System Information ---');
|
||||
report.push(`Host: ${manifest.system.hostname}`);
|
||||
report.push(`Platform: ${manifest.system.platform} ${manifest.system.os_release}`);
|
||||
report.push(`Architecture: ${manifest.system.architecture}`);
|
||||
|
||||
report.push('\n--- Application Information ---');
|
||||
report.push(`App Version: ${manifest.application.version}`);
|
||||
report.push(`Node Version: ${manifest.application.node_version}`);
|
||||
report.push(`Environment: ${manifest.application.environment}`);
|
||||
|
||||
report.push('\n--- Files Summary ---');
|
||||
report.push(`Total Files: ${manifest.files.count}`);
|
||||
report.push(`Total Size: ${(manifest.files.total_size / 1024 / 1024).toFixed(2)} MB`);
|
||||
|
||||
if (manifest.incremental) {
|
||||
report.push('\n--- Incremental Changes ---');
|
||||
report.push(`Added Files: ${manifest.incremental.changes.added_files_count}`);
|
||||
report.push(`Modified Files: ${manifest.incremental.changes.modified_files_count}`);
|
||||
report.push(`Deleted Files: ${manifest.incremental.changes.deleted_files_count}`);
|
||||
report.push(`Size Difference: ${(manifest.incremental.changes.size_difference / 1024 / 1024).toFixed(2)} MB`);
|
||||
}
|
||||
|
||||
report.push('\n--- Database Information ---');
|
||||
report.push(`Type: ${manifest.database.type}`);
|
||||
report.push(`Size: ${manifest.database.size ? (manifest.database.size / 1024 / 1024).toFixed(2) + ' MB' : 'N/A'}`);
|
||||
report.push(`Schema Version: ${manifest.database.schema_version}`);
|
||||
|
||||
report.push('\n--- Content Statistics ---');
|
||||
report.push(`Active Events: ${manifest.metadata.active_events_count}`);
|
||||
report.push(`Archived Events: ${manifest.metadata.archived_events_count}`);
|
||||
report.push(`Total Photos: ${manifest.metadata.total_photos_count}`);
|
||||
|
||||
report.push('\n--- Verification ---');
|
||||
report.push(`Manifest Checksum: ${manifest.verification.total_checksum}`);
|
||||
report.push(`Integrity Timestamp: ${manifest.verification.integrity_timestamp}`);
|
||||
|
||||
return report.join('\n');
|
||||
}
|
||||
}
|
||||
|
||||
// Export singleton instance
|
||||
module.exports = new BackupManifestGenerator();
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,720 @@
|
||||
const cron = require('node-cron');
|
||||
const path = require('path');
|
||||
const fs = require('fs').promises;
|
||||
const crypto = require('crypto');
|
||||
const { exec } = require('child_process');
|
||||
const { promisify } = require('util');
|
||||
const execAsync = promisify(exec);
|
||||
const { db } = require('../database/db');
|
||||
const { queueEmail } = require('./emailProcessor');
|
||||
const logger = require('../utils/logger');
|
||||
const { formatBoolean } = require('../utils/dbCompat');
|
||||
const backupManifest = require('./backupManifest');
|
||||
|
||||
// Backup job reference
|
||||
let backupJob = null;
|
||||
let backupConfig = null;
|
||||
let isRunning = false;
|
||||
|
||||
// Storage paths
|
||||
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
||||
|
||||
/**
|
||||
* Calculate file checksum using SHA256
|
||||
*/
|
||||
async function calculateChecksum(filePath) {
|
||||
const hash = crypto.createHash('sha256');
|
||||
const stream = require('fs').createReadStream(filePath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', data => hash.update(data));
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get database backup information
|
||||
*/
|
||||
async function getDatabaseBackupInfo() {
|
||||
try {
|
||||
// Check for recent database backup
|
||||
const recentDbBackup = await db('database_backup_runs')
|
||||
.where('status', 'completed')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.first();
|
||||
|
||||
if (recentDbBackup && recentDbBackup.file_path) {
|
||||
return {
|
||||
type: recentDbBackup.backup_type,
|
||||
backupFile: recentDbBackup.file_path,
|
||||
size: recentDbBackup.file_size_bytes,
|
||||
checksum: recentDbBackup.checksum,
|
||||
tables: recentDbBackup.statistics ? JSON.parse(recentDbBackup.statistics).tables : {},
|
||||
rowCounts: recentDbBackup.table_checksums ? JSON.parse(recentDbBackup.table_checksums) : {}
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
type: process.env.DB_TYPE === 'postgresql' ? 'postgresql' : 'sqlite',
|
||||
backupFile: null,
|
||||
size: 0,
|
||||
checksum: null,
|
||||
tables: {},
|
||||
rowCounts: {}
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get database backup info:', error);
|
||||
return {
|
||||
type: 'unknown',
|
||||
backupFile: null,
|
||||
size: 0,
|
||||
checksum: null,
|
||||
tables: {},
|
||||
rowCounts: {}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup configuration from database
|
||||
*/
|
||||
async function getBackupConfig() {
|
||||
try {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
const config = {};
|
||||
settings.forEach(setting => {
|
||||
try {
|
||||
config[setting.setting_key] = JSON.parse(setting.setting_value);
|
||||
} catch (e) {
|
||||
config[setting.setting_key] = setting.setting_value;
|
||||
}
|
||||
});
|
||||
|
||||
return config;
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup configuration:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of files to backup
|
||||
*/
|
||||
async function getFilesToBackup(includeArchived = true) {
|
||||
const files = [];
|
||||
const storagePath = getStoragePath();
|
||||
|
||||
try {
|
||||
// Active events
|
||||
const activePath = path.join(storagePath, 'events/active');
|
||||
await scanDirectory(activePath, files, storagePath);
|
||||
|
||||
// Archived events (if enabled)
|
||||
if (includeArchived) {
|
||||
const archivePath = path.join(storagePath, 'events/archived');
|
||||
await scanDirectory(archivePath, files, storagePath);
|
||||
}
|
||||
|
||||
// Thumbnails
|
||||
const thumbsPath = path.join(storagePath, 'thumbnails');
|
||||
await scanDirectory(thumbsPath, files, storagePath);
|
||||
|
||||
// Uploads (logos, favicons, etc.)
|
||||
const uploadsPath = path.join(storagePath, 'uploads');
|
||||
await scanDirectory(uploadsPath, files, storagePath);
|
||||
|
||||
return files;
|
||||
} catch (error) {
|
||||
logger.error('Failed to get files to backup:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively scan directory for files
|
||||
*/
|
||||
async function scanDirectory(dirPath, fileList, basePath, excludePatterns = []) {
|
||||
try {
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
const relativePath = path.relative(basePath, fullPath);
|
||||
|
||||
// Check exclude patterns
|
||||
if (excludePatterns.some(pattern => {
|
||||
if (pattern.includes('*')) {
|
||||
return new RegExp(pattern.replace(/\*/g, '.*')).test(entry.name);
|
||||
}
|
||||
return entry.name === pattern;
|
||||
})) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await scanDirectory(fullPath, fileList, basePath, excludePatterns);
|
||||
} else if (entry.isFile()) {
|
||||
const stats = await fs.stat(fullPath);
|
||||
fileList.push({
|
||||
path: fullPath,
|
||||
relativePath: relativePath,
|
||||
size: stats.size,
|
||||
modified: stats.mtime
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
logger.error(`Failed to scan directory ${dirPath}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file has changed since last backup
|
||||
*/
|
||||
async function hasFileChanged(filePath, checksum) {
|
||||
try {
|
||||
const fileState = await db('backup_file_states')
|
||||
.where('file_path', filePath)
|
||||
.first();
|
||||
|
||||
return !fileState || fileState.checksum !== checksum;
|
||||
} catch (error) {
|
||||
logger.error('Failed to check file state:', error);
|
||||
return true; // Assume changed if we can't check
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update file state in database
|
||||
*/
|
||||
async function updateFileState(filePath, checksum, size, modified) {
|
||||
try {
|
||||
const existing = await db('backup_file_states')
|
||||
.where('file_path', filePath)
|
||||
.first();
|
||||
|
||||
const data = {
|
||||
file_path: filePath,
|
||||
checksum: checksum,
|
||||
size_bytes: size,
|
||||
last_modified: modified,
|
||||
last_backed_up: new Date()
|
||||
};
|
||||
|
||||
if (existing) {
|
||||
await db('backup_file_states')
|
||||
.where('id', existing.id)
|
||||
.update(data);
|
||||
} else {
|
||||
await db('backup_file_states').insert(data);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to update file state:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform local directory backup
|
||||
*/
|
||||
async function performLocalBackup(config, files) {
|
||||
const destPath = config.backup_destination_path;
|
||||
const storagePath = getStoragePath();
|
||||
let backedUpCount = 0;
|
||||
let backedUpSize = 0;
|
||||
const backedUpFiles = [];
|
||||
|
||||
// Ensure destination exists
|
||||
await fs.mkdir(destPath, { recursive: true });
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
// Skip large files if configured
|
||||
const maxSizeMB = config.backup_max_file_size_mb || 5000;
|
||||
if (file.size > maxSizeMB * 1024 * 1024) {
|
||||
logger.warn(`Skipping large file: ${file.relativePath} (${(file.size / 1024 / 1024).toFixed(2)} MB)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Calculate checksum
|
||||
const checksum = await calculateChecksum(file.path);
|
||||
file.checksum = checksum; // Add checksum to file object
|
||||
|
||||
// Check if file has changed
|
||||
const changed = await hasFileChanged(file.relativePath, checksum);
|
||||
if (!changed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Copy file
|
||||
const destFilePath = path.join(destPath, file.relativePath);
|
||||
const destDir = path.dirname(destFilePath);
|
||||
await fs.mkdir(destDir, { recursive: true });
|
||||
await fs.copyFile(file.path, destFilePath);
|
||||
|
||||
// Update state
|
||||
await updateFileState(file.relativePath, checksum, file.size, file.modified);
|
||||
|
||||
backedUpCount++;
|
||||
backedUpSize += file.size;
|
||||
backedUpFiles.push(file.relativePath);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to backup file ${file.relativePath}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return { backedUpCount, backedUpSize, backedUpFiles };
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform rsync backup
|
||||
*/
|
||||
async function performRsyncBackup(config, files) {
|
||||
const storagePath = getStoragePath();
|
||||
const host = config.backup_rsync_host;
|
||||
const user = config.backup_rsync_user;
|
||||
const remotePath = config.backup_rsync_path;
|
||||
const sshKey = config.backup_rsync_ssh_key;
|
||||
|
||||
if (!host || !remotePath) {
|
||||
throw new Error('Rsync configuration incomplete');
|
||||
}
|
||||
|
||||
// Build rsync command
|
||||
const rsyncOptions = [
|
||||
'-avz', // archive, verbose, compress
|
||||
'--delete', // remove deleted files
|
||||
'--stats' // show statistics
|
||||
];
|
||||
|
||||
if (sshKey) {
|
||||
rsyncOptions.push(`-e "ssh -i ${sshKey} -o StrictHostKeyChecking=no"`);
|
||||
}
|
||||
|
||||
// Add exclude patterns
|
||||
const excludePatterns = config.backup_exclude_patterns || [];
|
||||
excludePatterns.forEach(pattern => {
|
||||
rsyncOptions.push(`--exclude="${pattern}"`);
|
||||
});
|
||||
|
||||
const source = `${storagePath}/`;
|
||||
const destination = user ? `${user}@${host}:${remotePath}` : `${host}:${remotePath}`;
|
||||
|
||||
const rsyncCommand = `rsync ${rsyncOptions.join(' ')} "${source}" "${destination}"`;
|
||||
|
||||
try {
|
||||
const { stdout, stderr } = await execAsync(rsyncCommand);
|
||||
|
||||
// Parse rsync stats
|
||||
const stats = parseRsyncStats(stdout);
|
||||
|
||||
// Update file states for successfully synced files
|
||||
for (const file of files) {
|
||||
try {
|
||||
const checksum = await calculateChecksum(file.path);
|
||||
await updateFileState(file.relativePath, checksum, file.size, file.modified);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to update state for ${file.relativePath}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
backedUpCount: stats.filesTransferred || files.length,
|
||||
backedUpSize: stats.totalSize || files.reduce((sum, f) => sum + f.size, 0),
|
||||
backedUpFiles: files.map(f => f.relativePath)
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Rsync backup failed:', error);
|
||||
throw new Error(`Rsync backup failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse rsync statistics from output
|
||||
*/
|
||||
function parseRsyncStats(output) {
|
||||
const stats = {};
|
||||
|
||||
// Extract files transferred
|
||||
const filesMatch = output.match(/Number of files transferred: (\d+)/);
|
||||
if (filesMatch) {
|
||||
stats.filesTransferred = parseInt(filesMatch[1]);
|
||||
}
|
||||
|
||||
// Extract total size
|
||||
const sizeMatch = output.match(/Total file size: ([\d,]+) bytes/);
|
||||
if (sizeMatch) {
|
||||
stats.totalSize = parseInt(sizeMatch[1].replace(/,/g, ''));
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform S3-compatible backup
|
||||
*/
|
||||
async function performS3Backup(config, files) {
|
||||
// This would require AWS SDK or similar
|
||||
// For now, return a placeholder
|
||||
throw new Error('S3 backup not implemented yet');
|
||||
}
|
||||
|
||||
/**
|
||||
* Run backup process
|
||||
*/
|
||||
async function runBackup() {
|
||||
if (isRunning) {
|
||||
logger.warn('Backup already running, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
isRunning = true;
|
||||
const startTime = new Date();
|
||||
let backupRun = null;
|
||||
|
||||
try {
|
||||
// Get current configuration
|
||||
const config = await getBackupConfig();
|
||||
if (!config.backup_enabled) {
|
||||
logger.info('Backup is disabled, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
// Create backup run record
|
||||
const [runId] = await db('backup_runs').insert({
|
||||
started_at: startTime,
|
||||
status: 'running',
|
||||
backup_type: 'scheduled'
|
||||
});
|
||||
|
||||
backupRun = { id: runId };
|
||||
|
||||
// Get files to backup
|
||||
const files = await getFilesToBackup(config.backup_include_archived);
|
||||
logger.info(`Found ${files.length} files to check for backup`);
|
||||
|
||||
// Perform backup based on destination type
|
||||
let result;
|
||||
switch (config.backup_destination_type) {
|
||||
case 'local':
|
||||
result = await performLocalBackup(config, files);
|
||||
break;
|
||||
case 'rsync':
|
||||
result = await performRsyncBackup(config, files);
|
||||
break;
|
||||
case 's3':
|
||||
result = await performS3Backup(config, files);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown backup destination type: ${config.backup_destination_type}`);
|
||||
}
|
||||
|
||||
// Calculate duration
|
||||
const endTime = new Date();
|
||||
const durationSeconds = Math.round((endTime - startTime) / 1000);
|
||||
|
||||
// Generate backup manifest
|
||||
let manifestPath = null;
|
||||
try {
|
||||
logger.info('Generating backup manifest...');
|
||||
|
||||
// Get database backup info if available
|
||||
const databaseInfo = await getDatabaseBackupInfo();
|
||||
|
||||
// Determine if this is an incremental backup
|
||||
const lastSuccessfulBackup = await db('backup_runs')
|
||||
.where('status', 'completed')
|
||||
.whereNot('id', runId)
|
||||
.orderBy('completed_at', 'desc')
|
||||
.first();
|
||||
|
||||
let manifest;
|
||||
const manifestOptions = {
|
||||
backupType: lastSuccessfulBackup ? 'incremental' : 'full',
|
||||
backupPath: config.backup_destination_path || config.backup_destination_type,
|
||||
files: files.filter(f => result.backedUpFiles && result.backedUpFiles.includes(f.relativePath)),
|
||||
databaseInfo: databaseInfo,
|
||||
parentBackupId: lastSuccessfulBackup ? lastSuccessfulBackup.manifest_id : null,
|
||||
format: config.backup_manifest_format || 'json',
|
||||
customMetadata: {
|
||||
backup_run_id: runId,
|
||||
destination_type: config.backup_destination_type,
|
||||
operator: 'system',
|
||||
reason: 'scheduled',
|
||||
retentionDays: config.backup_retention_days || 30
|
||||
}
|
||||
};
|
||||
|
||||
if (lastSuccessfulBackup && lastSuccessfulBackup.manifest_path) {
|
||||
try {
|
||||
const parentManifest = await backupManifest.loadManifest(lastSuccessfulBackup.manifest_path);
|
||||
manifest = await backupManifest.generateIncrementalManifest(manifestOptions, parentManifest);
|
||||
} catch (error) {
|
||||
logger.warn('Failed to load parent manifest, generating full manifest:', error);
|
||||
manifest = await backupManifest.generateManifest(manifestOptions);
|
||||
}
|
||||
} else {
|
||||
manifest = await backupManifest.generateManifest(manifestOptions);
|
||||
}
|
||||
|
||||
// Save manifest
|
||||
const manifestDir = config.backup_manifest_path || path.join(config.backup_destination_path || '/backup', 'manifests');
|
||||
await fs.mkdir(manifestDir, { recursive: true });
|
||||
|
||||
const manifestFileName = `backup-manifest-${manifest.backup.id}.${config.backup_manifest_format || 'json'}`;
|
||||
manifestPath = path.join(manifestDir, manifestFileName);
|
||||
await backupManifest.saveManifest(manifest, manifestPath, config.backup_manifest_format || 'json');
|
||||
|
||||
logger.info(`Backup manifest saved to ${manifestPath}`);
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to generate backup manifest:', error);
|
||||
// Don't fail the entire backup for manifest generation failure
|
||||
}
|
||||
|
||||
// Update backup run record
|
||||
await db('backup_runs')
|
||||
.where('id', runId)
|
||||
.update({
|
||||
completed_at: endTime,
|
||||
status: 'completed',
|
||||
files_backed_up: result.backedUpCount,
|
||||
total_size_bytes: result.backedUpSize,
|
||||
duration_seconds: durationSeconds,
|
||||
manifest_path: manifestPath,
|
||||
manifest_id: manifestPath ? path.basename(manifestPath, path.extname(manifestPath)) : null,
|
||||
statistics: JSON.stringify({
|
||||
totalFilesChecked: files.length,
|
||||
filesBackedUp: result.backedUpCount,
|
||||
totalSize: result.backedUpSize,
|
||||
averageFileSize: result.backedUpCount > 0 ? Math.round(result.backedUpSize / result.backedUpCount) : 0,
|
||||
manifestGenerated: !!manifestPath
|
||||
})
|
||||
});
|
||||
|
||||
logger.info(`Backup completed: ${result.backedUpCount} files, ${(result.backedUpSize / 1024 / 1024).toFixed(2)} MB in ${durationSeconds}s`);
|
||||
|
||||
// Send success email if configured
|
||||
if (config.backup_email_on_success) {
|
||||
// Get admin emails
|
||||
const admins = await db('admin_users').where('is_active', formatBoolean(true));
|
||||
for (const admin of admins) {
|
||||
await queueEmail(null, admin.email, 'backup_completed', {
|
||||
start_time: startTime.toISOString(),
|
||||
duration: `${durationSeconds} seconds`,
|
||||
files_count: result.backedUpCount.toString(),
|
||||
total_size: `${(result.backedUpSize / 1024 / 1024).toFixed(2)} MB`,
|
||||
backup_type: config.backup_destination_type
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Backup failed:', error);
|
||||
|
||||
// Update backup run record
|
||||
if (backupRun) {
|
||||
await db('backup_runs')
|
||||
.where('id', backupRun.id)
|
||||
.update({
|
||||
completed_at: new Date(),
|
||||
status: 'failed',
|
||||
error_message: error.message
|
||||
});
|
||||
}
|
||||
|
||||
// Send failure email
|
||||
const config = await getBackupConfig();
|
||||
if (config && config.backup_email_on_failure) {
|
||||
const admins = await db('admin_users').where('is_active', formatBoolean(true));
|
||||
for (const admin of admins) {
|
||||
await queueEmail(null, admin.email, 'backup_failed', {
|
||||
start_time: startTime.toISOString(),
|
||||
backup_type: config.backup_destination_type || 'unknown',
|
||||
error_message: error.message
|
||||
});
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
isRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start backup service
|
||||
*/
|
||||
async function startBackupService() {
|
||||
try {
|
||||
// Get configuration
|
||||
backupConfig = await getBackupConfig();
|
||||
|
||||
if (!backupConfig || !backupConfig.backup_enabled) {
|
||||
logger.info('Backup service is disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
// Cancel existing job if any
|
||||
if (backupJob) {
|
||||
backupJob.stop();
|
||||
}
|
||||
|
||||
// Schedule backup job
|
||||
const schedule = backupConfig.backup_schedule || '0 2 * * *'; // Default: 2 AM daily
|
||||
backupJob = cron.schedule(schedule, async () => {
|
||||
logger.info('Starting scheduled backup');
|
||||
await runBackup();
|
||||
});
|
||||
|
||||
logger.info(`Backup service started with schedule: ${schedule}`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to start backup service:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop backup service
|
||||
*/
|
||||
function stopBackupService() {
|
||||
if (backupJob) {
|
||||
backupJob.stop();
|
||||
backupJob = null;
|
||||
logger.info('Backup service stopped');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger manual backup
|
||||
*/
|
||||
async function triggerManualBackup() {
|
||||
logger.info('Starting manual backup');
|
||||
await runBackup();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup status and history
|
||||
*/
|
||||
async function getBackupStatus(limit = 10) {
|
||||
try {
|
||||
const runs = await db('backup_runs')
|
||||
.orderBy('started_at', 'desc')
|
||||
.limit(limit);
|
||||
|
||||
const lastRun = runs[0];
|
||||
const isHealthy = lastRun && lastRun.status === 'completed';
|
||||
|
||||
// Validate manifest if exists
|
||||
let manifestValid = false;
|
||||
if (lastRun && lastRun.manifest_path) {
|
||||
try {
|
||||
const manifest = await backupManifest.loadManifest(lastRun.manifest_path);
|
||||
backupManifest.validateManifest(manifest);
|
||||
manifestValid = true;
|
||||
} catch (error) {
|
||||
logger.warn('Manifest validation failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
isRunning,
|
||||
isHealthy,
|
||||
lastRun: lastRun ? {
|
||||
...lastRun,
|
||||
manifestValid
|
||||
} : null,
|
||||
recentRuns: runs,
|
||||
nextScheduledRun: backupJob ? getNextScheduledRun() : null
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup status:', error);
|
||||
return {
|
||||
isRunning,
|
||||
isHealthy: false,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get next scheduled run time
|
||||
*/
|
||||
function getNextScheduledRun() {
|
||||
// This is a simplified version - would need proper cron parsing
|
||||
const now = new Date();
|
||||
const tomorrow = new Date(now);
|
||||
tomorrow.setDate(tomorrow.getDate() + 1);
|
||||
tomorrow.setHours(2, 0, 0, 0); // Assuming default 2 AM schedule
|
||||
return tomorrow.toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old backup runs
|
||||
*/
|
||||
async function cleanupOldBackupRuns(retentionDays = 30) {
|
||||
try {
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
|
||||
const deleted = await db('backup_runs')
|
||||
.where('started_at', '<', cutoffDate)
|
||||
.delete();
|
||||
|
||||
if (deleted > 0) {
|
||||
logger.info(`Cleaned up ${deleted} old backup runs`);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to cleanup old backup runs:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup manifest for a specific backup run
|
||||
*/
|
||||
async function getBackupManifest(backupRunId) {
|
||||
try {
|
||||
const run = await db('backup_runs')
|
||||
.where('id', backupRunId)
|
||||
.first();
|
||||
|
||||
if (!run || !run.manifest_path) {
|
||||
throw new Error('Backup manifest not found');
|
||||
}
|
||||
|
||||
const manifest = await backupManifest.loadManifest(run.manifest_path);
|
||||
return {
|
||||
manifest,
|
||||
summary: backupManifest.generateSummaryReport(manifest)
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup manifest:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a backup manifest file
|
||||
*/
|
||||
async function validateBackupManifest(manifestPath) {
|
||||
try {
|
||||
const manifest = await backupManifest.loadManifest(manifestPath);
|
||||
backupManifest.validateManifest(manifest);
|
||||
return { valid: true, manifest };
|
||||
} catch (error) {
|
||||
return { valid: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
startBackupService,
|
||||
stopBackupService,
|
||||
triggerManualBackup,
|
||||
getBackupStatus,
|
||||
runBackup,
|
||||
cleanupOldBackupRuns,
|
||||
getBackupManifest,
|
||||
validateBackupManifest
|
||||
};
|
||||
@@ -0,0 +1,147 @@
|
||||
/**
|
||||
* Database Backup Service Usage Examples
|
||||
*
|
||||
* This service provides comprehensive database backup functionality
|
||||
* with support for both SQLite and PostgreSQL databases.
|
||||
*/
|
||||
|
||||
const { databaseBackupService } = require('./databaseBackup');
|
||||
|
||||
// Example 1: Manual backup with default settings
|
||||
async function manualBackup() {
|
||||
try {
|
||||
const result = await databaseBackupService.backup();
|
||||
console.log('Backup completed:', result);
|
||||
// Result includes: path, size, duration, checksum, compressionRatio
|
||||
} catch (error) {
|
||||
console.error('Backup failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Example 2: Backup with custom options
|
||||
async function customBackup() {
|
||||
try {
|
||||
const result = await databaseBackupService.backup({
|
||||
destinationPath: '/custom/backup/path',
|
||||
compress: true, // Enable gzip compression
|
||||
validateIntegrity: true, // Validate backup after creation
|
||||
includeChecksums: true, // Calculate table checksums
|
||||
noTransaction: false // Use transaction for consistency (PostgreSQL)
|
||||
});
|
||||
console.log('Custom backup completed:', result);
|
||||
} catch (error) {
|
||||
console.error('Backup failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Example 3: Check backup progress (useful for long-running backups)
|
||||
async function backupWithProgress() {
|
||||
// Start backup asynchronously
|
||||
const backupPromise = databaseBackupService.backup();
|
||||
|
||||
// Poll for progress
|
||||
const progressInterval = setInterval(() => {
|
||||
const progress = databaseBackupService.getProgress();
|
||||
if (progress) {
|
||||
console.log(`Progress: ${progress.message}`, progress.details);
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
try {
|
||||
const result = await backupPromise;
|
||||
clearInterval(progressInterval);
|
||||
console.log('Backup completed:', result);
|
||||
} catch (error) {
|
||||
clearInterval(progressInterval);
|
||||
console.error('Backup failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Example 4: Get backup history
|
||||
async function getBackupHistory() {
|
||||
const history = await databaseBackupService.getBackupHistory(10);
|
||||
|
||||
history.forEach(backup => {
|
||||
console.log(`Backup ${backup.id}:`);
|
||||
console.log(` Started: ${backup.started_at}`);
|
||||
console.log(` Status: ${backup.status}`);
|
||||
console.log(` Size: ${(backup.file_size_bytes / 1024 / 1024).toFixed(2)} MB`);
|
||||
console.log(` Duration: ${backup.duration_seconds}s`);
|
||||
});
|
||||
}
|
||||
|
||||
// Example 5: Clean up old backups
|
||||
async function cleanupBackups() {
|
||||
// Delete backups older than 30 days
|
||||
await databaseBackupService.cleanupOldBackups(30);
|
||||
console.log('Old backups cleaned up');
|
||||
}
|
||||
|
||||
// Example 6: Get table checksums (useful for monitoring changes)
|
||||
async function getTableChecksums() {
|
||||
const checksums = await databaseBackupService.getTableChecksums();
|
||||
|
||||
console.log('Table Checksums:');
|
||||
Object.entries(checksums).forEach(([table, info]) => {
|
||||
console.log(` ${table}: ${info.rowCount} rows, checksum: ${info.checksum}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Example 7: Using the scheduled backup service
|
||||
const { startScheduledBackups, stopScheduledBackups } = require('./databaseBackup');
|
||||
|
||||
async function setupScheduledBackups() {
|
||||
// Start scheduled backups (reads schedule from database config)
|
||||
await startScheduledBackups();
|
||||
console.log('Scheduled backups started');
|
||||
|
||||
// Later, if needed, stop scheduled backups
|
||||
// stopScheduledBackups();
|
||||
}
|
||||
|
||||
// Example 8: Admin API endpoints available
|
||||
/*
|
||||
GET /api/admin/database-backup/status - Get backup status and config
|
||||
PUT /api/admin/database-backup/config - Update backup configuration
|
||||
POST /api/admin/database-backup/backup - Trigger manual backup
|
||||
GET /api/admin/database-backup/progress - Get current backup progress
|
||||
GET /api/admin/database-backup/history - Get backup history with pagination
|
||||
DELETE /api/admin/database-backup/cleanup - Delete old backup files
|
||||
POST /api/admin/database-backup/test - Test backup configuration
|
||||
GET /api/admin/database-backup/checksums - Get current table checksums
|
||||
*/
|
||||
|
||||
// Example 9: Configuration options stored in database
|
||||
/*
|
||||
database_backup_enabled: boolean - Enable/disable scheduled backups
|
||||
database_backup_schedule: string - Cron schedule (default: '0 3 * * *')
|
||||
database_backup_destination_path: string - Where to store backups
|
||||
database_backup_compress: boolean - Enable gzip compression
|
||||
database_backup_validate_integrity: boolean - Validate after backup
|
||||
database_backup_include_checksums: boolean - Calculate table checksums
|
||||
database_backup_retention_days: number - Days to keep old backups
|
||||
database_backup_email_on_failure: boolean - Send email on failure
|
||||
database_backup_email_on_success: boolean - Send email on success
|
||||
*/
|
||||
|
||||
// Example 10: Production considerations
|
||||
/*
|
||||
1. Ensure destination path has sufficient space
|
||||
2. For large databases, backups may take significant time
|
||||
3. PostgreSQL backups use single-transaction mode by default
|
||||
4. Compression typically reduces size by 70-90%
|
||||
5. Schedule backups during low-traffic periods
|
||||
6. Monitor backup history for failures
|
||||
7. Test restore procedures regularly
|
||||
8. Consider replication for real-time redundancy
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
manualBackup,
|
||||
customBackup,
|
||||
backupWithProgress,
|
||||
getBackupHistory,
|
||||
cleanupBackups,
|
||||
getTableChecksums,
|
||||
setupScheduledBackups
|
||||
};
|
||||
@@ -0,0 +1,631 @@
|
||||
const fs = require('fs').promises;
|
||||
const path = require('path');
|
||||
const { exec } = require('child_process');
|
||||
const { promisify } = require('util');
|
||||
const execAsync = promisify(exec);
|
||||
const crypto = require('crypto');
|
||||
const zlib = require('zlib');
|
||||
const { pipeline } = require('stream/promises');
|
||||
const { createReadStream, createWriteStream } = require('fs');
|
||||
const { db } = require('../database/db');
|
||||
const knexConfig = require('../../knexfile');
|
||||
const logger = require('../utils/logger');
|
||||
const { queueEmail } = require('./emailProcessor');
|
||||
const { formatBoolean } = require('../utils/dbCompat');
|
||||
|
||||
// Constants
|
||||
const CHUNK_SIZE = 1024 * 1024; // 1MB chunks for streaming
|
||||
const PROGRESS_INTERVAL = 100; // Report progress every 100 rows
|
||||
|
||||
/**
|
||||
* Database Backup Service
|
||||
* Supports both SQLite and PostgreSQL with proper escaping,
|
||||
* compression, checksums, and validation
|
||||
*/
|
||||
class DatabaseBackupService {
|
||||
constructor() {
|
||||
this.isRunning = false;
|
||||
this.currentProgress = null;
|
||||
this.dbType = knexConfig.client === 'pg' ? 'postgresql' : 'sqlite';
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate checksum for a file
|
||||
*/
|
||||
async calculateChecksum(filePath) {
|
||||
const hash = crypto.createHash('sha256');
|
||||
const stream = createReadStream(filePath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', data => hash.update(data));
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress a file using gzip
|
||||
*/
|
||||
async compressFile(inputPath, outputPath) {
|
||||
const gzip = zlib.createGzip({ level: 6 }); // Balanced compression
|
||||
const source = createReadStream(inputPath);
|
||||
const destination = createWriteStream(outputPath);
|
||||
|
||||
await pipeline(source, gzip, destination);
|
||||
|
||||
// Get compression ratio
|
||||
const inputStats = await fs.stat(inputPath);
|
||||
const outputStats = await fs.stat(outputPath);
|
||||
const ratio = (1 - outputStats.size / inputStats.size) * 100;
|
||||
|
||||
return {
|
||||
originalSize: inputStats.size,
|
||||
compressedSize: outputStats.size,
|
||||
compressionRatio: ratio.toFixed(2)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get database size
|
||||
*/
|
||||
async getDatabaseSize() {
|
||||
if (this.dbType === 'sqlite') {
|
||||
const dbPath = knexConfig.connection.filename;
|
||||
const stats = await fs.stat(dbPath);
|
||||
return stats.size;
|
||||
} else {
|
||||
// PostgreSQL
|
||||
const result = await db.raw(`
|
||||
SELECT pg_database_size(current_database()) as size
|
||||
`);
|
||||
return parseInt(result.rows[0].size);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get table checksums for change detection
|
||||
*/
|
||||
async getTableChecksums() {
|
||||
const checksums = {};
|
||||
const tables = await this.getTables();
|
||||
|
||||
for (const table of tables) {
|
||||
if (this.dbType === 'sqlite') {
|
||||
// SQLite: Use aggregate of all row data
|
||||
const result = await db.raw(`
|
||||
SELECT
|
||||
COUNT(*) as row_count,
|
||||
COALESCE(SUM(LENGTH(CAST(t.* AS TEXT))), 0) as data_sum
|
||||
FROM "${table}" t
|
||||
`);
|
||||
|
||||
checksums[table] = {
|
||||
rowCount: result[0].row_count,
|
||||
checksum: crypto
|
||||
.createHash('md5')
|
||||
.update(`${result[0].row_count}-${result[0].data_sum}`)
|
||||
.digest('hex')
|
||||
};
|
||||
} else {
|
||||
// PostgreSQL: Use built-in functions
|
||||
const result = await db.raw(`
|
||||
SELECT
|
||||
COUNT(*) as row_count,
|
||||
MD5(COALESCE(STRING_AGG(MD5(t::text), ''), '')) as checksum
|
||||
FROM "${table}" t
|
||||
`);
|
||||
|
||||
checksums[table] = {
|
||||
rowCount: parseInt(result.rows[0].row_count),
|
||||
checksum: result.rows[0].checksum || 'empty'
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return checksums;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of tables
|
||||
*/
|
||||
async getTables() {
|
||||
if (this.dbType === 'sqlite') {
|
||||
const result = await db.raw(`
|
||||
SELECT name FROM sqlite_master
|
||||
WHERE type='table'
|
||||
AND name NOT LIKE 'sqlite_%'
|
||||
AND name != 'knex_migrations'
|
||||
AND name != 'knex_migrations_lock'
|
||||
ORDER BY name
|
||||
`);
|
||||
return result.map(row => row.name);
|
||||
} else {
|
||||
// PostgreSQL
|
||||
const result = await db.raw(`
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_type = 'BASE TABLE'
|
||||
AND table_name NOT IN ('knex_migrations', 'knex_migrations_lock')
|
||||
ORDER BY table_name
|
||||
`);
|
||||
return result.rows.map(row => row.table_name);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create SQLite backup
|
||||
*/
|
||||
async createSQLiteBackup(outputPath, options = {}) {
|
||||
const dbPath = knexConfig.connection.filename;
|
||||
const tempPath = `${outputPath}.tmp`;
|
||||
|
||||
try {
|
||||
// Use SQLite's backup API for consistency
|
||||
await execAsync(`sqlite3 "${dbPath}" ".backup '${tempPath}'"`);
|
||||
|
||||
// Verify the backup
|
||||
const verifyResult = await execAsync(`sqlite3 "${tempPath}" "PRAGMA integrity_check"`);
|
||||
if (!verifyResult.stdout.includes('ok')) {
|
||||
throw new Error('Backup integrity check failed');
|
||||
}
|
||||
|
||||
// Move temp file to final location
|
||||
await fs.rename(tempPath, outputPath);
|
||||
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
// Cleanup temp file if exists
|
||||
try {
|
||||
await fs.unlink(tempPath);
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create PostgreSQL backup with proper escaping
|
||||
*/
|
||||
async createPostgreSQLBackup(outputPath, options = {}) {
|
||||
const { host, port, user, password, database } = knexConfig.connection;
|
||||
|
||||
// Build connection string with proper escaping
|
||||
const connectionParts = [
|
||||
`host=${host}`,
|
||||
`port=${port}`,
|
||||
`dbname=${database}`,
|
||||
`user=${user}`
|
||||
];
|
||||
|
||||
// Set PGPASSWORD environment variable for security
|
||||
const env = { ...process.env };
|
||||
if (password) {
|
||||
env.PGPASSWORD = password;
|
||||
}
|
||||
|
||||
// Build pg_dump command with options
|
||||
const pgDumpOptions = [
|
||||
'--verbose',
|
||||
'--no-owner',
|
||||
'--no-privileges',
|
||||
'--clean',
|
||||
'--if-exists',
|
||||
'--format=plain',
|
||||
'--encoding=UTF8'
|
||||
];
|
||||
|
||||
// Add transaction support for consistency
|
||||
if (!options.noTransaction) {
|
||||
pgDumpOptions.push('--single-transaction');
|
||||
}
|
||||
|
||||
// Add compression if not doing it separately
|
||||
if (options.compress && !options.separateCompression) {
|
||||
pgDumpOptions.push('--compress=6');
|
||||
}
|
||||
|
||||
const command = `pg_dump "${connectionParts.join(' ')}" ${pgDumpOptions.join(' ')} > "${outputPath}"`;
|
||||
|
||||
try {
|
||||
const { stderr } = await execAsync(command, {
|
||||
env,
|
||||
maxBuffer: 1024 * 1024 * 100 // 100MB buffer
|
||||
});
|
||||
|
||||
// pg_dump writes progress to stderr, not an error
|
||||
if (stderr && !stderr.includes('dump complete')) {
|
||||
logger.warn('pg_dump warnings:', stderr);
|
||||
}
|
||||
|
||||
// Verify the dump file is not empty
|
||||
const stats = await fs.stat(outputPath);
|
||||
if (stats.size === 0) {
|
||||
throw new Error('Backup file is empty');
|
||||
}
|
||||
|
||||
return { success: true, warnings: stderr };
|
||||
} catch (error) {
|
||||
throw new Error(`PostgreSQL backup failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate backup integrity
|
||||
*/
|
||||
async validateBackup(backupPath, originalChecksums) {
|
||||
const tempDbPath = `${backupPath}.validate`;
|
||||
|
||||
try {
|
||||
if (this.dbType === 'sqlite') {
|
||||
// For SQLite, we can directly check integrity
|
||||
const result = await execAsync(`sqlite3 "${backupPath}" "PRAGMA integrity_check"`);
|
||||
if (!result.stdout.includes('ok')) {
|
||||
throw new Error('Backup integrity check failed');
|
||||
}
|
||||
} else {
|
||||
// For PostgreSQL, we'd need to restore to a temp database
|
||||
// This is more complex and might not be feasible in production
|
||||
logger.info('PostgreSQL backup validation would require restore test');
|
||||
}
|
||||
|
||||
return { valid: true };
|
||||
} finally {
|
||||
// Cleanup
|
||||
try {
|
||||
await fs.unlink(tempDbPath);
|
||||
} catch (e) {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main backup method
|
||||
*/
|
||||
async backup(options = {}) {
|
||||
if (this.isRunning) {
|
||||
throw new Error('Backup already in progress');
|
||||
}
|
||||
|
||||
this.isRunning = true;
|
||||
const startTime = new Date();
|
||||
let backupRun = null;
|
||||
|
||||
try {
|
||||
// Get configuration
|
||||
const config = await this.getBackupConfig();
|
||||
const {
|
||||
destinationPath = '/backup/database',
|
||||
compress = true,
|
||||
validateIntegrity = true,
|
||||
includeChecksums = true
|
||||
} = { ...config, ...options };
|
||||
|
||||
// Create backup directory
|
||||
await fs.mkdir(destinationPath, { recursive: true });
|
||||
|
||||
// Generate backup filename
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
const baseName = `picpeak-db-${this.dbType}-${timestamp}`;
|
||||
const sqlFile = path.join(destinationPath, `${baseName}.sql`);
|
||||
const finalFile = compress ? path.join(destinationPath, `${baseName}.sql.gz`) : sqlFile;
|
||||
|
||||
// Create backup run record
|
||||
const [runId] = await db('database_backup_runs').insert({
|
||||
started_at: startTime,
|
||||
status: 'running',
|
||||
backup_type: this.dbType,
|
||||
destination_path: finalFile
|
||||
});
|
||||
|
||||
backupRun = { id: runId };
|
||||
|
||||
// Get initial checksums
|
||||
let tableChecksums = null;
|
||||
if (includeChecksums) {
|
||||
this.updateProgress('Calculating table checksums...');
|
||||
tableChecksums = await this.getTableChecksums();
|
||||
}
|
||||
|
||||
// Get database size
|
||||
const dbSize = await this.getDatabaseSize();
|
||||
|
||||
// Create the backup
|
||||
this.updateProgress('Creating database backup...');
|
||||
if (this.dbType === 'sqlite') {
|
||||
await this.createSQLiteBackup(sqlFile, options);
|
||||
} else {
|
||||
await this.createPostgreSQLBackup(sqlFile, options);
|
||||
}
|
||||
|
||||
// Compress if requested
|
||||
let compressionStats = null;
|
||||
if (compress) {
|
||||
this.updateProgress('Compressing backup...');
|
||||
compressionStats = await this.compressFile(sqlFile, finalFile);
|
||||
await fs.unlink(sqlFile); // Remove uncompressed file
|
||||
}
|
||||
|
||||
// Calculate checksum
|
||||
this.updateProgress('Calculating backup checksum...');
|
||||
const backupChecksum = await this.calculateChecksum(finalFile);
|
||||
|
||||
// Validate if requested
|
||||
if (validateIntegrity && !compress) {
|
||||
this.updateProgress('Validating backup integrity...');
|
||||
await this.validateBackup(finalFile, tableChecksums);
|
||||
}
|
||||
|
||||
// Get final file size
|
||||
const finalStats = await fs.stat(finalFile);
|
||||
|
||||
// Calculate duration
|
||||
const endTime = new Date();
|
||||
const durationSeconds = Math.round((endTime - startTime) / 1000);
|
||||
|
||||
// Update backup run record
|
||||
await db('database_backup_runs')
|
||||
.where('id', runId)
|
||||
.update({
|
||||
completed_at: endTime,
|
||||
status: 'completed',
|
||||
file_path: finalFile,
|
||||
file_size_bytes: finalStats.size,
|
||||
original_size_bytes: dbSize,
|
||||
duration_seconds: durationSeconds,
|
||||
checksum: backupChecksum,
|
||||
compression_ratio: compressionStats?.compressionRatio || null,
|
||||
table_checksums: tableChecksums ? JSON.stringify(tableChecksums) : null,
|
||||
statistics: JSON.stringify({
|
||||
dbType: this.dbType,
|
||||
compressed: compress,
|
||||
validated: validateIntegrity,
|
||||
compressionStats,
|
||||
tableCount: tableChecksums ? Object.keys(tableChecksums).length : null
|
||||
})
|
||||
});
|
||||
|
||||
logger.info(`Database backup completed: ${finalFile} (${(finalStats.size / 1024 / 1024).toFixed(2)} MB) in ${durationSeconds}s`);
|
||||
|
||||
// Send success notification if configured
|
||||
if (config.emailOnSuccess) {
|
||||
await this.sendBackupNotification('success', {
|
||||
duration: durationSeconds,
|
||||
size: finalStats.size,
|
||||
compressionRatio: compressionStats?.compressionRatio,
|
||||
path: finalFile
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
path: finalFile,
|
||||
size: finalStats.size,
|
||||
duration: durationSeconds,
|
||||
checksum: backupChecksum,
|
||||
compressionRatio: compressionStats?.compressionRatio
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Database backup failed:', error);
|
||||
|
||||
// Update backup run record
|
||||
if (backupRun) {
|
||||
await db('database_backup_runs')
|
||||
.where('id', backupRun.id)
|
||||
.update({
|
||||
completed_at: new Date(),
|
||||
status: 'failed',
|
||||
error_message: error.message
|
||||
});
|
||||
}
|
||||
|
||||
// Send failure notification
|
||||
const config = await this.getBackupConfig();
|
||||
if (config.emailOnFailure) {
|
||||
await this.sendBackupNotification('failure', {
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
|
||||
throw error;
|
||||
} finally {
|
||||
this.isRunning = false;
|
||||
this.currentProgress = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update progress
|
||||
*/
|
||||
updateProgress(message, details = {}) {
|
||||
this.currentProgress = {
|
||||
message,
|
||||
details,
|
||||
timestamp: new Date()
|
||||
};
|
||||
logger.info(`Backup progress: ${message}`, details);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current progress
|
||||
*/
|
||||
getProgress() {
|
||||
return this.currentProgress;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup configuration
|
||||
*/
|
||||
async getBackupConfig() {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'database_backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
const config = {};
|
||||
settings.forEach(setting => {
|
||||
try {
|
||||
config[setting.setting_key] = JSON.parse(setting.setting_value);
|
||||
} catch (e) {
|
||||
config[setting.setting_key] = setting.setting_value;
|
||||
}
|
||||
});
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Send backup notification email
|
||||
*/
|
||||
async sendBackupNotification(type, details) {
|
||||
const admins = await db('admin_users').where('is_active', formatBoolean(true));
|
||||
|
||||
for (const admin of admins) {
|
||||
if (type === 'success') {
|
||||
await queueEmail(null, admin.email, 'database_backup_completed', {
|
||||
backup_type: this.dbType,
|
||||
duration: `${details.duration} seconds`,
|
||||
file_size: `${(details.size / 1024 / 1024).toFixed(2)} MB`,
|
||||
compression_ratio: details.compressionRatio ? `${details.compressionRatio}%` : 'N/A',
|
||||
file_path: details.path
|
||||
});
|
||||
} else {
|
||||
await queueEmail(null, admin.email, 'database_backup_failed', {
|
||||
backup_type: this.dbType,
|
||||
error_message: details.error,
|
||||
timestamp: new Date().toISOString()
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old backups
|
||||
*/
|
||||
async cleanupOldBackups(retentionDays = 30) {
|
||||
try {
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
|
||||
// Get old backup records
|
||||
const oldBackups = await db('database_backup_runs')
|
||||
.where('completed_at', '<', cutoffDate)
|
||||
.where('status', 'completed')
|
||||
.select('id', 'file_path');
|
||||
|
||||
let deletedCount = 0;
|
||||
|
||||
for (const backup of oldBackups) {
|
||||
try {
|
||||
// Delete the file
|
||||
if (backup.file_path) {
|
||||
await fs.unlink(backup.file_path);
|
||||
}
|
||||
|
||||
// Delete the record
|
||||
await db('database_backup_runs')
|
||||
.where('id', backup.id)
|
||||
.delete();
|
||||
|
||||
deletedCount++;
|
||||
} catch (error) {
|
||||
logger.error(`Failed to delete old backup ${backup.file_path}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
if (deletedCount > 0) {
|
||||
logger.info(`Cleaned up ${deletedCount} old database backups`);
|
||||
}
|
||||
|
||||
// Also clean up old failed runs
|
||||
await db('database_backup_runs')
|
||||
.where('started_at', '<', cutoffDate)
|
||||
.where('status', 'failed')
|
||||
.delete();
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to cleanup old database backups:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup history
|
||||
*/
|
||||
async getBackupHistory(limit = 10) {
|
||||
return await db('database_backup_runs')
|
||||
.orderBy('started_at', 'desc')
|
||||
.limit(limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore from backup (careful!)
|
||||
*/
|
||||
async restore(backupPath, options = {}) {
|
||||
// This is a dangerous operation and should be used with extreme caution
|
||||
throw new Error('Restore functionality not implemented for safety. Please restore manually.');
|
||||
}
|
||||
}
|
||||
|
||||
// Create singleton instance
|
||||
const databaseBackupService = new DatabaseBackupService();
|
||||
|
||||
// Scheduled backup runner
|
||||
let backupSchedule = null;
|
||||
|
||||
/**
|
||||
* Start scheduled database backups
|
||||
*/
|
||||
async function startScheduledBackups() {
|
||||
const cron = require('node-cron');
|
||||
|
||||
try {
|
||||
const config = await databaseBackupService.getBackupConfig();
|
||||
|
||||
if (!config.enabled) {
|
||||
logger.info('Database backup service is disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
// Stop existing schedule
|
||||
if (backupSchedule) {
|
||||
backupSchedule.stop();
|
||||
}
|
||||
|
||||
// Default schedule: 3 AM daily (offset from file backups at 2 AM)
|
||||
const schedule = config.schedule || '0 3 * * *';
|
||||
|
||||
backupSchedule = cron.schedule(schedule, async () => {
|
||||
logger.info('Starting scheduled database backup');
|
||||
try {
|
||||
await databaseBackupService.backup();
|
||||
await databaseBackupService.cleanupOldBackups(config.retentionDays || 30);
|
||||
} catch (error) {
|
||||
logger.error('Scheduled database backup failed:', error);
|
||||
}
|
||||
});
|
||||
|
||||
logger.info(`Database backup service started with schedule: ${schedule}`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to start database backup service:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop scheduled database backups
|
||||
*/
|
||||
function stopScheduledBackups() {
|
||||
if (backupSchedule) {
|
||||
backupSchedule.stop();
|
||||
backupSchedule = null;
|
||||
logger.info('Database backup service stopped');
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
databaseBackupService,
|
||||
startScheduledBackups,
|
||||
stopScheduledBackups,
|
||||
DatabaseBackupService // Export class for testing
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,311 @@
|
||||
const S3StorageAdapter = require('../s3Storage');
|
||||
const { S3Client } = require('@aws-sdk/client-s3');
|
||||
const { Upload } = require('@aws-sdk/lib-storage');
|
||||
const fs = require('fs');
|
||||
const stream = require('stream');
|
||||
|
||||
// Mock AWS SDK
|
||||
jest.mock('@aws-sdk/client-s3');
|
||||
jest.mock('@aws-sdk/lib-storage');
|
||||
jest.mock('@aws-sdk/s3-request-presigner');
|
||||
|
||||
describe('S3StorageAdapter', () => {
|
||||
let mockS3Client;
|
||||
let mockSend;
|
||||
let s3Storage;
|
||||
|
||||
beforeEach(() => {
|
||||
// Reset mocks
|
||||
jest.clearAllMocks();
|
||||
|
||||
// Mock S3Client
|
||||
mockSend = jest.fn();
|
||||
mockS3Client = {
|
||||
send: mockSend
|
||||
};
|
||||
S3Client.mockImplementation(() => mockS3Client);
|
||||
|
||||
// Create adapter instance
|
||||
s3Storage = new S3StorageAdapter({
|
||||
bucket: 'test-bucket',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: 'test-key',
|
||||
secretAccessKey: 'test-secret'
|
||||
});
|
||||
});
|
||||
|
||||
describe('constructor', () => {
|
||||
it('should initialize with required config', () => {
|
||||
expect(s3Storage.bucket).toBe('test-bucket');
|
||||
expect(s3Storage.config.region).toBe('us-east-1');
|
||||
});
|
||||
|
||||
it('should throw error if bucket is not provided', () => {
|
||||
expect(() => {
|
||||
new S3StorageAdapter({ region: 'us-east-1' });
|
||||
}).toThrow('S3 bucket name is required');
|
||||
});
|
||||
|
||||
it('should configure for MinIO with path style', () => {
|
||||
const minioStorage = new S3StorageAdapter({
|
||||
bucket: 'test-bucket',
|
||||
endpoint: 'http://localhost:9000',
|
||||
forcePathStyle: true,
|
||||
sslEnabled: false
|
||||
});
|
||||
|
||||
expect(S3Client).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
endpoint: 'http://localhost:9000',
|
||||
forcePathStyle: true
|
||||
})
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('testConnection', () => {
|
||||
it('should successfully test connection', async () => {
|
||||
mockSend.mockResolvedValueOnce({});
|
||||
|
||||
const result = await s3Storage.testConnection();
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(mockSend).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
input: { Bucket: 'test-bucket' }
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should throw error on connection failure', async () => {
|
||||
mockSend.mockRejectedValueOnce(new Error('Access Denied'));
|
||||
|
||||
await expect(s3Storage.testConnection()).rejects.toThrow('S3 connection test failed');
|
||||
});
|
||||
});
|
||||
|
||||
describe('upload', () => {
|
||||
let mockUpload;
|
||||
let mockDone;
|
||||
|
||||
beforeEach(() => {
|
||||
mockDone = jest.fn().mockResolvedValue({
|
||||
Location: 'https://test-bucket.s3.amazonaws.com/test-key',
|
||||
ETag: '"test-etag"'
|
||||
});
|
||||
|
||||
mockUpload = {
|
||||
on: jest.fn().mockReturnThis(),
|
||||
done: mockDone
|
||||
};
|
||||
|
||||
Upload.mockImplementation(() => mockUpload);
|
||||
|
||||
// Mock fs.stat
|
||||
jest.spyOn(fs.promises, 'stat').mockResolvedValue({
|
||||
size: 1024
|
||||
});
|
||||
|
||||
// Mock fs.createReadStream
|
||||
jest.spyOn(fs, 'createReadStream').mockReturnValue(new stream.Readable());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('should upload file successfully', async () => {
|
||||
const result = await s3Storage.upload('/path/to/file.jpg', 'test-key');
|
||||
|
||||
expect(result.Location).toBe('https://test-bucket.s3.amazonaws.com/test-key');
|
||||
expect(Upload).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
client: mockS3Client,
|
||||
params: expect.objectContaining({
|
||||
Bucket: 'test-bucket',
|
||||
Key: 'test-key',
|
||||
ContentType: 'application/octet-stream'
|
||||
})
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should track upload progress', async () => {
|
||||
const onProgress = jest.fn();
|
||||
let progressCallback;
|
||||
|
||||
mockUpload.on.mockImplementation((event, callback) => {
|
||||
if (event === 'httpUploadProgress') {
|
||||
progressCallback = callback;
|
||||
}
|
||||
return mockUpload;
|
||||
});
|
||||
|
||||
const uploadPromise = s3Storage.upload('/path/to/file.jpg', 'test-key', {
|
||||
onProgress
|
||||
});
|
||||
|
||||
// Simulate progress
|
||||
progressCallback({ loaded: 512, total: 1024 });
|
||||
|
||||
await uploadPromise;
|
||||
|
||||
expect(onProgress).toHaveBeenCalledWith(512, 1024);
|
||||
});
|
||||
|
||||
it('should emit upload events', async () => {
|
||||
const uploadStartSpy = jest.fn();
|
||||
const uploadCompleteSpy = jest.fn();
|
||||
|
||||
s3Storage.on('uploadStart', uploadStartSpy);
|
||||
s3Storage.on('uploadComplete', uploadCompleteSpy);
|
||||
|
||||
await s3Storage.upload('/path/to/file.jpg', 'test-key');
|
||||
|
||||
expect(uploadStartSpy).toHaveBeenCalledWith({ key: 'test-key', size: 1024 });
|
||||
expect(uploadCompleteSpy).toHaveBeenCalledWith({
|
||||
key: 'test-key',
|
||||
location: 'https://test-bucket.s3.amazonaws.com/test-key'
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('exists', () => {
|
||||
it('should return true if object exists', async () => {
|
||||
mockSend.mockResolvedValueOnce({});
|
||||
|
||||
const result = await s3Storage.exists('test-key');
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(mockSend).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
input: { Bucket: 'test-bucket', Key: 'test-key' }
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
it('should return false if object does not exist', async () => {
|
||||
const error = new Error('Not Found');
|
||||
error.name = 'NotFound';
|
||||
error.$metadata = { httpStatusCode: 404 };
|
||||
mockSend.mockRejectedValueOnce(error);
|
||||
|
||||
const result = await s3Storage.exists('test-key');
|
||||
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateKey', () => {
|
||||
it('should generate unique key with timestamp and random string', () => {
|
||||
const key = s3Storage.generateKey('photo.jpg');
|
||||
|
||||
expect(key).toMatch(/^\d+_[a-f0-9]{16}_photo\.jpg$/);
|
||||
});
|
||||
|
||||
it('should add prefix if provided', () => {
|
||||
const key = s3Storage.generateKey('photo.jpg', 'events/wedding');
|
||||
|
||||
expect(key).toMatch(/^events\/wedding\/\d+_[a-f0-9]{16}_photo\.jpg$/);
|
||||
});
|
||||
|
||||
it('should sanitize filename', () => {
|
||||
const key = s3Storage.generateKey('my photo (1).jpg');
|
||||
|
||||
expect(key).toMatch(/^\d+_[a-f0-9]{16}_my_photo__1_\.jpg$/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('retry logic', () => {
|
||||
it('should retry on retryable errors', async () => {
|
||||
const retryableError = new Error('Connection reset');
|
||||
retryableError.code = 'ECONNRESET';
|
||||
|
||||
// First attempt fails, second succeeds
|
||||
mockSend
|
||||
.mockRejectedValueOnce(retryableError)
|
||||
.mockResolvedValueOnce({});
|
||||
|
||||
// Mock setTimeout to speed up test
|
||||
jest.useFakeTimers();
|
||||
|
||||
const promise = s3Storage.exists('test-key');
|
||||
|
||||
// Advance timers
|
||||
jest.runAllTimers();
|
||||
|
||||
const result = await promise;
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(mockSend).toHaveBeenCalledTimes(2);
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('should not retry on non-retryable errors', async () => {
|
||||
const nonRetryableError = new Error('Invalid credentials');
|
||||
nonRetryableError.code = 'InvalidCredentials';
|
||||
|
||||
mockSend.mockRejectedValueOnce(nonRetryableError);
|
||||
|
||||
await expect(s3Storage.exists('test-key')).rejects.toThrow('Invalid credentials');
|
||||
expect(mockSend).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should stop retrying after max attempts', async () => {
|
||||
const retryableError = new Error('Service unavailable');
|
||||
retryableError.code = 'ServiceUnavailable';
|
||||
|
||||
mockSend.mockRejectedValue(retryableError);
|
||||
|
||||
// Mock setTimeout to speed up test
|
||||
jest.useFakeTimers();
|
||||
|
||||
const promise = s3Storage.exists('test-key');
|
||||
|
||||
// Advance timers for all retries
|
||||
for (let i = 0; i < 4; i++) {
|
||||
jest.runAllTimers();
|
||||
}
|
||||
|
||||
await expect(promise).rejects.toThrow('Service unavailable');
|
||||
expect(mockSend).toHaveBeenCalledTimes(4); // Initial + 3 retries
|
||||
|
||||
jest.useRealTimers();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getStats', () => {
|
||||
it('should calculate storage statistics', async () => {
|
||||
mockSend.mockResolvedValueOnce({
|
||||
Contents: [
|
||||
{ Key: 'file1.jpg', Size: 1024 },
|
||||
{ Key: 'file2.jpg', Size: 2048 }
|
||||
],
|
||||
NextContinuationToken: 'token123'
|
||||
}).mockResolvedValueOnce({
|
||||
Contents: [
|
||||
{ Key: 'file3.jpg', Size: 3072 }
|
||||
]
|
||||
});
|
||||
|
||||
const stats = await s3Storage.getStats('events/');
|
||||
|
||||
expect(stats).toEqual({
|
||||
totalSize: 6144,
|
||||
totalCount: 3,
|
||||
totalSizeFormatted: '6 KB'
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('_formatBytes', () => {
|
||||
it('should format bytes correctly', () => {
|
||||
expect(s3Storage._formatBytes(0)).toBe('0 Bytes');
|
||||
expect(s3Storage._formatBytes(1024)).toBe('1 KB');
|
||||
expect(s3Storage._formatBytes(1048576)).toBe('1 MB');
|
||||
expect(s3Storage._formatBytes(1073741824)).toBe('1 GB');
|
||||
expect(s3Storage._formatBytes(1536, 1)).toBe('1.5 KB');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,221 @@
|
||||
/**
|
||||
* Example usage of the S3StorageAdapter
|
||||
*
|
||||
* This file demonstrates how to use the S3 storage adapter for various operations
|
||||
*/
|
||||
|
||||
const S3StorageAdapter = require('./s3Storage');
|
||||
|
||||
// Example 1: Basic AWS S3 Configuration
|
||||
const s3Storage = new S3StorageAdapter({
|
||||
bucket: 'my-photo-bucket',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
});
|
||||
|
||||
// Example 2: MinIO Configuration (S3-compatible)
|
||||
const minioStorage = new S3StorageAdapter({
|
||||
bucket: 'photo-storage',
|
||||
endpoint: 'http://localhost:9000', // MinIO endpoint
|
||||
accessKeyId: 'minioadmin',
|
||||
secretAccessKey: 'minioadmin',
|
||||
forcePathStyle: true, // Required for MinIO
|
||||
sslEnabled: false // For local development
|
||||
});
|
||||
|
||||
// Example 3: DigitalOcean Spaces Configuration
|
||||
const spacesStorage = new S3StorageAdapter({
|
||||
bucket: 'my-space-name',
|
||||
endpoint: 'https://nyc3.digitaloceanspaces.com',
|
||||
region: 'nyc3',
|
||||
accessKeyId: process.env.DO_SPACES_KEY,
|
||||
secretAccessKey: process.env.DO_SPACES_SECRET
|
||||
});
|
||||
|
||||
// Usage Examples
|
||||
async function examples() {
|
||||
try {
|
||||
// Test connection
|
||||
await s3Storage.testConnection();
|
||||
console.log('Connection successful!');
|
||||
|
||||
// Upload a file with progress tracking
|
||||
const uploadResult = await s3Storage.upload(
|
||||
'/path/to/local/photo.jpg',
|
||||
'events/wedding-2024/photo.jpg',
|
||||
{
|
||||
contentType: 'image/jpeg',
|
||||
metadata: {
|
||||
event: 'wedding-2024',
|
||||
photographer: 'John Doe'
|
||||
},
|
||||
onProgress: (loaded, total) => {
|
||||
const percentage = Math.round((loaded / total) * 100);
|
||||
console.log(`Upload progress: ${percentage}%`);
|
||||
}
|
||||
}
|
||||
);
|
||||
console.log('Upload complete:', uploadResult.Location);
|
||||
|
||||
// Upload from stream
|
||||
const readStream = fs.createReadStream('/path/to/large-video.mp4');
|
||||
await s3Storage.uploadStream(readStream, 'events/wedding-2024/video.mp4', {
|
||||
contentType: 'video/mp4',
|
||||
onProgress: (loaded, total) => {
|
||||
console.log(`Streamed ${loaded} of ${total} bytes`);
|
||||
}
|
||||
});
|
||||
|
||||
// Download a file
|
||||
await s3Storage.download(
|
||||
'events/wedding-2024/photo.jpg',
|
||||
'/path/to/downloaded/photo.jpg',
|
||||
{
|
||||
onProgress: (loaded, total) => {
|
||||
const percentage = Math.round((loaded / total) * 100);
|
||||
console.log(`Download progress: ${percentage}%`);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// Get a download stream
|
||||
const downloadStream = await s3Storage.downloadStream('events/wedding-2024/photo.jpg');
|
||||
downloadStream.pipe(fs.createWriteStream('/path/to/output.jpg'));
|
||||
|
||||
// List files
|
||||
const listing = await s3Storage.list('events/wedding-2024/');
|
||||
console.log(`Found ${listing.Contents.length} files`);
|
||||
listing.Contents.forEach(file => {
|
||||
console.log(`- ${file.Key} (${file.Size} bytes)`);
|
||||
});
|
||||
|
||||
// Generate pre-signed URL for temporary access
|
||||
const downloadUrl = await s3Storage.getSignedUrl('getObject', 'events/wedding-2024/photo.jpg', {
|
||||
expiresIn: 3600 // 1 hour
|
||||
});
|
||||
console.log('Pre-signed download URL:', downloadUrl);
|
||||
|
||||
// Generate pre-signed upload URL
|
||||
const uploadUrl = await s3Storage.getSignedUrl('putObject', 'events/wedding-2024/new-photo.jpg', {
|
||||
expiresIn: 1800, // 30 minutes
|
||||
params: {
|
||||
ContentType: 'image/jpeg'
|
||||
}
|
||||
});
|
||||
console.log('Pre-signed upload URL:', uploadUrl);
|
||||
|
||||
// Check if file exists
|
||||
const exists = await s3Storage.exists('events/wedding-2024/photo.jpg');
|
||||
console.log('File exists:', exists);
|
||||
|
||||
// Get metadata
|
||||
const metadata = await s3Storage.getMetadata('events/wedding-2024/photo.jpg');
|
||||
console.log('File metadata:', metadata);
|
||||
|
||||
// Copy file
|
||||
await s3Storage.copy(
|
||||
'events/wedding-2024/photo.jpg',
|
||||
'events/wedding-2024/photo-copy.jpg'
|
||||
);
|
||||
|
||||
// Move file
|
||||
await s3Storage.move(
|
||||
'events/wedding-2024/photo-copy.jpg',
|
||||
'events/wedding-2024/archived/photo.jpg'
|
||||
);
|
||||
|
||||
// Delete file
|
||||
await s3Storage.delete('events/wedding-2024/temp-photo.jpg');
|
||||
|
||||
// Delete multiple files
|
||||
const deleteResult = await s3Storage.deleteMany([
|
||||
'events/wedding-2024/temp1.jpg',
|
||||
'events/wedding-2024/temp2.jpg',
|
||||
'events/wedding-2024/temp3.jpg'
|
||||
]);
|
||||
console.log(`Deleted ${deleteResult.Deleted.length} files`);
|
||||
|
||||
// Get storage statistics
|
||||
const stats = await s3Storage.getStats('events/');
|
||||
console.log(`Total files: ${stats.totalCount}`);
|
||||
console.log(`Total size: ${stats.totalSizeFormatted}`);
|
||||
|
||||
// Listen to events
|
||||
s3Storage.on('uploadProgress', (data) => {
|
||||
console.log(`Uploading ${data.key}: ${data.loaded}/${data.total}`);
|
||||
});
|
||||
|
||||
s3Storage.on('uploadComplete', (data) => {
|
||||
console.log(`Upload completed: ${data.key}`);
|
||||
});
|
||||
|
||||
s3Storage.on('uploadError', (data) => {
|
||||
console.error(`Upload failed for ${data.key}:`, data.error);
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Integration with existing photo upload workflow
|
||||
async function integrateWithPhotoUpload(eventId, files) {
|
||||
const storage = new S3StorageAdapter({
|
||||
bucket: process.env.S3_BUCKET,
|
||||
region: process.env.AWS_REGION,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
});
|
||||
|
||||
const uploadedPhotos = [];
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
// Generate unique S3 key
|
||||
const s3Key = storage.generateKey(file.originalname, `events/${eventId}`);
|
||||
|
||||
// Upload to S3
|
||||
const result = await storage.upload(file.path, s3Key, {
|
||||
contentType: file.mimetype,
|
||||
metadata: {
|
||||
eventId: eventId,
|
||||
originalName: file.originalname,
|
||||
uploadedAt: new Date().toISOString()
|
||||
}
|
||||
});
|
||||
|
||||
uploadedPhotos.push({
|
||||
filename: s3Key,
|
||||
originalName: file.originalname,
|
||||
size: file.size,
|
||||
mimeType: file.mimetype,
|
||||
s3Location: result.Location,
|
||||
s3Key: s3Key
|
||||
});
|
||||
|
||||
// Clean up local temp file
|
||||
await fs.promises.unlink(file.path);
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Failed to upload ${file.originalname}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return uploadedPhotos;
|
||||
}
|
||||
|
||||
// Environment variables needed:
|
||||
// AWS_ACCESS_KEY_ID=your-access-key
|
||||
// AWS_SECRET_ACCESS_KEY=your-secret-key
|
||||
// AWS_REGION=us-east-1
|
||||
// S3_BUCKET=your-bucket-name
|
||||
|
||||
// For MinIO:
|
||||
// MINIO_ENDPOINT=http://localhost:9000
|
||||
// MINIO_ACCESS_KEY=minioadmin
|
||||
// MINIO_SECRET_KEY=minioadmin
|
||||
// MINIO_BUCKET=photo-storage
|
||||
|
||||
module.exports = { examples, integrateWithPhotoUpload };
|
||||
@@ -0,0 +1,733 @@
|
||||
const { S3Client, HeadBucketCommand, HeadObjectCommand, GetObjectCommand, PutObjectCommand, DeleteObjectCommand, DeleteObjectsCommand, ListObjectsV2Command, CopyObjectCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand } = require('@aws-sdk/client-s3');
|
||||
const { Upload } = require('@aws-sdk/lib-storage');
|
||||
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
|
||||
const fs = require('fs');
|
||||
const fsPromises = require('fs').promises;
|
||||
const path = require('path');
|
||||
const stream = require('stream');
|
||||
const crypto = require('crypto');
|
||||
const logger = require('../../utils/logger');
|
||||
|
||||
/**
|
||||
* S3 Storage Adapter for handling file uploads to S3 and S3-compatible services
|
||||
*
|
||||
* Features:
|
||||
* - Support for S3 and S3-compatible services (MinIO, DigitalOcean Spaces, etc.)
|
||||
* - Multipart upload for large files (>100MB)
|
||||
* - Progress tracking with event emitter
|
||||
* - Retry logic with exponential backoff
|
||||
* - Stream support for memory efficiency
|
||||
* - Path-style URL support for MinIO
|
||||
* - Connection testing
|
||||
* - Comprehensive error handling
|
||||
*
|
||||
* @class S3StorageAdapter
|
||||
*/
|
||||
class S3StorageAdapter extends stream.EventEmitter {
|
||||
/**
|
||||
* Creates an instance of S3StorageAdapter
|
||||
*
|
||||
* @param {Object} config - Configuration object
|
||||
* @param {string} config.bucket - S3 bucket name
|
||||
* @param {string} [config.region='us-east-1'] - AWS region
|
||||
* @param {string} [config.endpoint] - Custom endpoint URL for S3-compatible services
|
||||
* @param {string} [config.accessKeyId] - AWS access key ID
|
||||
* @param {string} [config.secretAccessKey] - AWS secret access key
|
||||
* @param {boolean} [config.forcePathStyle=false] - Force path-style URLs (required for MinIO)
|
||||
* @param {boolean} [config.sslEnabled=true] - Enable SSL for connections
|
||||
* @param {number} [config.multipartThreshold=104857600] - Threshold for multipart upload (default 100MB)
|
||||
* @param {number} [config.partSize=10485760] - Part size for multipart upload (default 10MB)
|
||||
* @param {number} [config.maxRetries=3] - Maximum number of retry attempts
|
||||
* @param {number} [config.retryDelay=1000] - Initial retry delay in milliseconds
|
||||
*/
|
||||
constructor(config) {
|
||||
super();
|
||||
|
||||
// Validate required config
|
||||
if (!config.bucket) {
|
||||
throw new Error('S3 bucket name is required');
|
||||
}
|
||||
|
||||
// Set defaults
|
||||
this.config = {
|
||||
region: 'us-east-1',
|
||||
forcePathStyle: false,
|
||||
sslEnabled: true,
|
||||
multipartThreshold: 100 * 1024 * 1024, // 100MB
|
||||
partSize: 10 * 1024 * 1024, // 10MB
|
||||
maxRetries: 3,
|
||||
retryDelay: 1000,
|
||||
...config
|
||||
};
|
||||
|
||||
// Initialize S3 client
|
||||
const s3Config = {
|
||||
region: this.config.region,
|
||||
forcePathStyle: this.config.forcePathStyle
|
||||
};
|
||||
|
||||
// Add credentials if provided
|
||||
if (this.config.accessKeyId && this.config.secretAccessKey) {
|
||||
s3Config.credentials = {
|
||||
accessKeyId: this.config.accessKeyId,
|
||||
secretAccessKey: this.config.secretAccessKey
|
||||
};
|
||||
}
|
||||
|
||||
// Add custom endpoint if provided (for S3-compatible services)
|
||||
if (this.config.endpoint) {
|
||||
s3Config.endpoint = this.config.endpoint;
|
||||
// For MinIO and other S3-compatible services
|
||||
if (!this.config.endpoint.startsWith('https://') && this.config.sslEnabled) {
|
||||
s3Config.endpoint = `https://${this.config.endpoint}`;
|
||||
} else if (!this.config.endpoint.startsWith('http://') && !this.config.sslEnabled) {
|
||||
s3Config.endpoint = `http://${this.config.endpoint}`;
|
||||
}
|
||||
}
|
||||
|
||||
this.s3Client = new S3Client(s3Config);
|
||||
this.bucket = this.config.bucket;
|
||||
|
||||
// Bind methods to preserve context
|
||||
this.upload = this.upload.bind(this);
|
||||
this.uploadStream = this.uploadStream.bind(this);
|
||||
this.download = this.download.bind(this);
|
||||
this.downloadStream = this.downloadStream.bind(this);
|
||||
this.delete = this.delete.bind(this);
|
||||
this.exists = this.exists.bind(this);
|
||||
this.list = this.list.bind(this);
|
||||
this.copy = this.copy.bind(this);
|
||||
this.move = this.move.bind(this);
|
||||
this.getSignedUrl = this.getSignedUrl.bind(this);
|
||||
this.testConnection = this.testConnection.bind(this);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test connection to S3 bucket
|
||||
*
|
||||
* @returns {Promise<boolean>} - True if connection successful
|
||||
* @throws {Error} - If connection fails
|
||||
*/
|
||||
async testConnection() {
|
||||
try {
|
||||
await this.s3Client.send(new HeadBucketCommand({ Bucket: this.bucket }));
|
||||
logger.info(`Successfully connected to S3 bucket: ${this.bucket}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.error(`Failed to connect to S3 bucket ${this.bucket}:`, error);
|
||||
throw new Error(`S3 connection test failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a file to S3 with automatic multipart for large files
|
||||
*
|
||||
* @param {string} localPath - Local file path to upload
|
||||
* @param {string} s3Key - S3 object key (path in bucket)
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @param {Object} [options.metadata] - Object metadata
|
||||
* @param {string} [options.contentType] - Content type
|
||||
* @param {string} [options.cacheControl] - Cache control header
|
||||
* @param {Function} [options.onProgress] - Progress callback function(loaded, total)
|
||||
* @returns {Promise<Object>} - Upload result with Location, ETag, etc.
|
||||
*/
|
||||
async upload(localPath, s3Key, options = {}) {
|
||||
try {
|
||||
const stats = await fsPromises.stat(localPath);
|
||||
const fileSize = stats.size;
|
||||
|
||||
// Emit upload start event
|
||||
this.emit('uploadStart', { key: s3Key, size: fileSize });
|
||||
|
||||
// Create file stream
|
||||
const fileStream = fs.createReadStream(localPath);
|
||||
|
||||
// Prepare upload parameters
|
||||
const uploadParams = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
Body: fileStream,
|
||||
ContentType: options.contentType || 'application/octet-stream',
|
||||
Metadata: options.metadata || {},
|
||||
CacheControl: options.cacheControl
|
||||
};
|
||||
|
||||
// Remove undefined values
|
||||
Object.keys(uploadParams).forEach(key => uploadParams[key] === undefined && delete uploadParams[key]);
|
||||
|
||||
// Use AWS SDK v3 Upload for automatic multipart handling
|
||||
const parallelUploads3 = new Upload({
|
||||
client: this.s3Client,
|
||||
params: uploadParams,
|
||||
queueSize: 4, // Optional: concurrent uploads
|
||||
partSize: this.config.partSize,
|
||||
leavePartsOnError: false
|
||||
});
|
||||
|
||||
// Track upload progress
|
||||
parallelUploads3.on('httpUploadProgress', (progress) => {
|
||||
if (options.onProgress) {
|
||||
options.onProgress(progress.loaded, progress.total);
|
||||
}
|
||||
this.emit('uploadProgress', { key: s3Key, loaded: progress.loaded, total: progress.total });
|
||||
});
|
||||
|
||||
// Perform upload with retry
|
||||
const result = await this._retryOperation(() => parallelUploads3.done());
|
||||
|
||||
this.emit('uploadComplete', { key: s3Key, location: result.Location });
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error(`Failed to upload file ${localPath} to S3:`, error);
|
||||
this.emit('uploadError', { key: s3Key, error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a stream to S3
|
||||
*
|
||||
* @param {stream.Readable} readStream - Readable stream to upload
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @returns {Promise<Object>} - Upload result
|
||||
*/
|
||||
async uploadStream(readStream, s3Key, options = {}) {
|
||||
const uploadParams = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
Body: readStream,
|
||||
ContentType: options.contentType || 'application/octet-stream',
|
||||
Metadata: options.metadata || {},
|
||||
CacheControl: options.cacheControl
|
||||
};
|
||||
|
||||
// Remove undefined values
|
||||
Object.keys(uploadParams).forEach(key => uploadParams[key] === undefined && delete uploadParams[key]);
|
||||
|
||||
const parallelUploads3 = new Upload({
|
||||
client: this.s3Client,
|
||||
params: uploadParams,
|
||||
queueSize: 4,
|
||||
partSize: this.config.partSize,
|
||||
leavePartsOnError: false
|
||||
});
|
||||
|
||||
// Track progress if callback provided
|
||||
if (options.onProgress) {
|
||||
parallelUploads3.on('httpUploadProgress', (progress) => {
|
||||
options.onProgress(progress.loaded, progress.total);
|
||||
this.emit('uploadProgress', { key: s3Key, ...progress });
|
||||
});
|
||||
}
|
||||
|
||||
return await this._retryOperation(() => parallelUploads3.done());
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a file from S3
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {string} localPath - Local file path to save to
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @param {Function} [options.onProgress] - Progress callback
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async download(s3Key, localPath, options = {}) {
|
||||
try {
|
||||
// Ensure directory exists
|
||||
await fsPromises.mkdir(path.dirname(localPath), { recursive: true });
|
||||
|
||||
// Get object metadata first for progress tracking
|
||||
const headResult = await this._retryOperation(() =>
|
||||
this.s3Client.send(new HeadObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key
|
||||
}))
|
||||
);
|
||||
|
||||
const fileSize = headResult.ContentLength;
|
||||
this.emit('downloadStart', { key: s3Key, size: fileSize });
|
||||
|
||||
// Get object
|
||||
const getObjectResult = await this._retryOperation(() =>
|
||||
this.s3Client.send(new GetObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key
|
||||
}))
|
||||
);
|
||||
|
||||
// Create write stream
|
||||
const writeStream = fs.createWriteStream(localPath);
|
||||
|
||||
// Download with progress tracking
|
||||
await new Promise((resolve, reject) => {
|
||||
let downloaded = 0;
|
||||
|
||||
const bodyStream = getObjectResult.Body;
|
||||
|
||||
bodyStream.on('data', (chunk) => {
|
||||
downloaded += chunk.length;
|
||||
if (options.onProgress) {
|
||||
options.onProgress(downloaded, fileSize);
|
||||
}
|
||||
this.emit('downloadProgress', { key: s3Key, loaded: downloaded, total: fileSize });
|
||||
});
|
||||
|
||||
bodyStream.on('error', reject);
|
||||
writeStream.on('error', reject);
|
||||
writeStream.on('finish', resolve);
|
||||
|
||||
bodyStream.pipe(writeStream);
|
||||
});
|
||||
|
||||
this.emit('downloadComplete', { key: s3Key });
|
||||
} catch (error) {
|
||||
logger.error(`Failed to download file ${s3Key} from S3:`, error);
|
||||
this.emit('downloadError', { key: s3Key, error });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a download stream from S3
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @param {string} [options.range] - Byte range to download (e.g., 'bytes=0-1023')
|
||||
* @returns {Promise<stream.Readable>} - Readable stream
|
||||
*/
|
||||
async downloadStream(s3Key, options = {}) {
|
||||
const params = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
Range: options.range
|
||||
};
|
||||
|
||||
// Remove undefined values
|
||||
Object.keys(params).forEach(key => params[key] === undefined && delete params[key]);
|
||||
|
||||
const result = await this.s3Client.send(new GetObjectCommand(params));
|
||||
return result.Body;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file from S3
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async delete(s3Key) {
|
||||
return await this._retryOperation(() =>
|
||||
this.s3Client.send(new DeleteObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete multiple files from S3
|
||||
*
|
||||
* @param {string[]} s3Keys - Array of S3 object keys
|
||||
* @returns {Promise<Object>} - Delete result
|
||||
*/
|
||||
async deleteMany(s3Keys) {
|
||||
if (!s3Keys || s3Keys.length === 0) {
|
||||
return { Deleted: [], Errors: [] };
|
||||
}
|
||||
|
||||
// S3 deleteObjects has a limit of 1000 keys per request
|
||||
const chunks = [];
|
||||
for (let i = 0; i < s3Keys.length; i += 1000) {
|
||||
chunks.push(s3Keys.slice(i, i + 1000));
|
||||
}
|
||||
|
||||
const results = await Promise.all(
|
||||
chunks.map(chunk =>
|
||||
this._retryOperation(() =>
|
||||
this.s3Client.send(new DeleteObjectsCommand({
|
||||
Bucket: this.bucket,
|
||||
Delete: {
|
||||
Objects: chunk.map(key => ({ Key: key })),
|
||||
Quiet: false
|
||||
}
|
||||
}))
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
// Combine results
|
||||
return {
|
||||
Deleted: results.flatMap(r => r.Deleted || []),
|
||||
Errors: results.flatMap(r => r.Errors || [])
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file exists in S3
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @returns {Promise<boolean>} - True if exists
|
||||
*/
|
||||
async exists(s3Key) {
|
||||
try {
|
||||
await this.s3Client.send(new HeadObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key
|
||||
}));
|
||||
return true;
|
||||
} catch (error) {
|
||||
if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List files in S3
|
||||
*
|
||||
* @param {string} prefix - S3 prefix to list
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @param {number} [options.maxKeys=1000] - Maximum number of keys to return
|
||||
* @param {string} [options.continuationToken] - Continuation token for pagination
|
||||
* @returns {Promise<Object>} - List result with Contents array and NextContinuationToken
|
||||
*/
|
||||
async list(prefix, options = {}) {
|
||||
const params = {
|
||||
Bucket: this.bucket,
|
||||
Prefix: prefix,
|
||||
MaxKeys: options.maxKeys || 1000,
|
||||
ContinuationToken: options.continuationToken
|
||||
};
|
||||
|
||||
return await this._retryOperation(() =>
|
||||
this.s3Client.send(new ListObjectsV2Command(params))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy a file within S3
|
||||
*
|
||||
* @param {string} sourceKey - Source S3 object key
|
||||
* @param {string} targetKey - Target S3 object key
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @returns {Promise<Object>} - Copy result
|
||||
*/
|
||||
async copy(sourceKey, targetKey, options = {}) {
|
||||
const copySource = `${this.bucket}/${sourceKey}`;
|
||||
|
||||
const params = {
|
||||
Bucket: this.bucket,
|
||||
CopySource: copySource,
|
||||
Key: targetKey,
|
||||
MetadataDirective: options.metadata ? 'REPLACE' : 'COPY',
|
||||
Metadata: options.metadata,
|
||||
ContentType: options.contentType,
|
||||
CacheControl: options.cacheControl
|
||||
};
|
||||
|
||||
// Remove undefined values
|
||||
Object.keys(params).forEach(key => params[key] === undefined && delete params[key]);
|
||||
|
||||
return await this._retryOperation(() =>
|
||||
this.s3Client.send(new CopyObjectCommand(params))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Move a file within S3 (copy then delete)
|
||||
*
|
||||
* @param {string} sourceKey - Source S3 object key
|
||||
* @param {string} targetKey - Target S3 object key
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @returns {Promise<Object>} - Move result
|
||||
*/
|
||||
async move(sourceKey, targetKey, options = {}) {
|
||||
// First copy the object
|
||||
const copyResult = await this.copy(sourceKey, targetKey, options);
|
||||
|
||||
// Then delete the original
|
||||
await this.delete(sourceKey);
|
||||
|
||||
return copyResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a pre-signed URL for downloading or uploading
|
||||
*
|
||||
* @param {string} operation - Operation type ('getObject' or 'putObject')
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {Object} [options={}] - Additional options
|
||||
* @param {number} [options.expiresIn=3600] - URL expiration in seconds
|
||||
* @param {Object} [options.params] - Additional parameters for the operation
|
||||
* @returns {Promise<string>} - Pre-signed URL
|
||||
*/
|
||||
async getSignedUrl(operation, s3Key, options = {}) {
|
||||
const params = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
...options.params
|
||||
};
|
||||
|
||||
let command;
|
||||
switch (operation.toLowerCase()) {
|
||||
case 'getobject':
|
||||
command = new GetObjectCommand(params);
|
||||
break;
|
||||
case 'putobject':
|
||||
command = new PutObjectCommand(params);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unsupported operation: ${operation}`);
|
||||
}
|
||||
|
||||
return await getSignedUrl(this.s3Client, command, {
|
||||
expiresIn: options.expiresIn || 3600
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get object metadata
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @returns {Promise<Object>} - Object metadata
|
||||
*/
|
||||
async getMetadata(s3Key) {
|
||||
return await this._retryOperation(() =>
|
||||
this.s3Client.send(new HeadObjectCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key
|
||||
}))
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update object metadata
|
||||
*
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {Object} metadata - New metadata
|
||||
* @returns {Promise<Object>} - Update result
|
||||
*/
|
||||
async updateMetadata(s3Key, metadata) {
|
||||
// S3 requires copying the object to itself to update metadata
|
||||
return await this.copy(s3Key, s3Key, { metadata });
|
||||
}
|
||||
|
||||
/**
|
||||
* Manual multipart upload for advanced use cases
|
||||
*
|
||||
* @param {string} localPath - Local file path
|
||||
* @param {string} s3Key - S3 object key
|
||||
* @param {number} fileSize - File size in bytes
|
||||
* @param {Object} options - Upload options
|
||||
* @returns {Promise<Object>} - Upload result
|
||||
* @private
|
||||
*/
|
||||
async _manualMultipartUpload(localPath, s3Key, fileSize, options) {
|
||||
logger.info(`Starting manual multipart upload for ${s3Key} (${fileSize} bytes)`);
|
||||
|
||||
// Initiate multipart upload
|
||||
const multipartParams = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
ContentType: options.contentType || 'application/octet-stream',
|
||||
Metadata: options.metadata || {},
|
||||
CacheControl: options.cacheControl
|
||||
};
|
||||
|
||||
// Remove undefined values
|
||||
Object.keys(multipartParams).forEach(key => multipartParams[key] === undefined && delete multipartParams[key]);
|
||||
|
||||
const multipart = await this._retryOperation(() =>
|
||||
this.s3Client.send(new CreateMultipartUploadCommand(multipartParams))
|
||||
);
|
||||
|
||||
const uploadId = multipart.UploadId;
|
||||
const partSize = this.config.partSize;
|
||||
const numParts = Math.ceil(fileSize / partSize);
|
||||
|
||||
let uploaded = 0;
|
||||
const parts = [];
|
||||
|
||||
try {
|
||||
// Upload parts
|
||||
for (let partNum = 1; partNum <= numParts; partNum++) {
|
||||
const start = (partNum - 1) * partSize;
|
||||
const end = Math.min(start + partSize, fileSize);
|
||||
|
||||
const partStream = fs.createReadStream(localPath, {
|
||||
start,
|
||||
end: end - 1
|
||||
});
|
||||
|
||||
const partParams = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
PartNumber: partNum,
|
||||
UploadId: uploadId,
|
||||
Body: partStream
|
||||
};
|
||||
|
||||
// Upload part with retry
|
||||
const partResult = await this._retryOperation(() =>
|
||||
this.s3Client.send(new UploadPartCommand(partParams))
|
||||
);
|
||||
|
||||
parts.push({
|
||||
ETag: partResult.ETag,
|
||||
PartNumber: partNum
|
||||
});
|
||||
|
||||
uploaded += (end - start);
|
||||
|
||||
if (options.onProgress) {
|
||||
options.onProgress(uploaded, fileSize);
|
||||
}
|
||||
this.emit('uploadProgress', { key: s3Key, loaded: uploaded, total: fileSize });
|
||||
|
||||
logger.info(`Uploaded part ${partNum}/${numParts} for ${s3Key}`);
|
||||
}
|
||||
|
||||
// Complete multipart upload
|
||||
const completeParams = {
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: { Parts: parts }
|
||||
};
|
||||
|
||||
const result = await this._retryOperation(() =>
|
||||
this.s3Client.send(new CompleteMultipartUploadCommand(completeParams))
|
||||
);
|
||||
|
||||
this.emit('uploadComplete', { key: s3Key, location: result.Location });
|
||||
logger.info(`Completed multipart upload for ${s3Key}`);
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
// Abort multipart upload on error
|
||||
logger.error(`Multipart upload failed for ${s3Key}, aborting:`, error);
|
||||
|
||||
try {
|
||||
await this.s3Client.send(new AbortMultipartUploadCommand({
|
||||
Bucket: this.bucket,
|
||||
Key: s3Key,
|
||||
UploadId: uploadId
|
||||
}));
|
||||
} catch (abortError) {
|
||||
logger.error(`Failed to abort multipart upload:`, abortError);
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry operation with exponential backoff
|
||||
* @private
|
||||
*/
|
||||
async _retryOperation(operation, retryCount = 0) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
if (retryCount >= this.config.maxRetries) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Check if error is retryable
|
||||
const retryableErrors = ['ECONNRESET', 'ETIMEDOUT', 'ENOTFOUND', 'ESOCKETTIMEDOUT', 'RequestTimeout', 'SlowDown', 'ServiceUnavailable', 'InternalError'];
|
||||
const isRetryable = retryableErrors.some(code =>
|
||||
error.code === code ||
|
||||
error.name === code ||
|
||||
error.$metadata?.httpStatusCode === 503 ||
|
||||
error.$metadata?.httpStatusCode === 500
|
||||
);
|
||||
|
||||
if (!isRetryable) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Calculate delay with exponential backoff and jitter
|
||||
const delay = Math.min(
|
||||
this.config.retryDelay * Math.pow(2, retryCount) + Math.random() * 1000,
|
||||
30000 // Max 30 seconds
|
||||
);
|
||||
|
||||
logger.warn(`Retrying operation after ${delay}ms (attempt ${retryCount + 1}/${this.config.maxRetries}):`, error.message);
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, delay));
|
||||
|
||||
return this._retryOperation(operation, retryCount + 1);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a unique S3 key for a file
|
||||
*
|
||||
* @param {string} originalName - Original filename
|
||||
* @param {string} [prefix=''] - Optional prefix for the key
|
||||
* @returns {string} - Generated S3 key
|
||||
*/
|
||||
generateKey(originalName, prefix = '') {
|
||||
const timestamp = Date.now();
|
||||
const randomStr = crypto.randomBytes(8).toString('hex');
|
||||
const ext = path.extname(originalName);
|
||||
const basename = path.basename(originalName, ext);
|
||||
|
||||
// Sanitize basename
|
||||
const safeName = basename.replace(/[^a-zA-Z0-9-_]/g, '_');
|
||||
|
||||
const key = `${timestamp}_${randomStr}_${safeName}${ext}`;
|
||||
|
||||
return prefix ? path.posix.join(prefix, key) : key;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get storage statistics
|
||||
*
|
||||
* @param {string} [prefix=''] - Optional prefix to filter
|
||||
* @returns {Promise<Object>} - Storage statistics
|
||||
*/
|
||||
async getStats(prefix = '') {
|
||||
let totalSize = 0;
|
||||
let totalCount = 0;
|
||||
let continuationToken;
|
||||
|
||||
do {
|
||||
const result = await this.list(prefix, { continuationToken });
|
||||
|
||||
if (result.Contents) {
|
||||
totalCount += result.Contents.length;
|
||||
totalSize += result.Contents.reduce((sum, obj) => sum + (obj.Size || 0), 0);
|
||||
}
|
||||
|
||||
continuationToken = result.NextContinuationToken;
|
||||
} while (continuationToken);
|
||||
|
||||
return {
|
||||
totalSize,
|
||||
totalCount,
|
||||
totalSizeFormatted: this._formatBytes(totalSize)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Format bytes to human readable format
|
||||
* @private
|
||||
*/
|
||||
_formatBytes(bytes, decimals = 2) {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
|
||||
const k = 1024;
|
||||
const dm = decimals < 0 ? 0 : decimals;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
||||
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = S3StorageAdapter;
|
||||
Reference in New Issue
Block a user