refactor: remove dead files, dedupe formatBytes and document numbering helpers
- Delete unused adminEvents-enhanced.js, backupService.original.js, databaseBackup.example.js, s3Storage.example.js, ThemeCustomizer.tsx - Extract shared formatBytes to utils/formatBytes.js (was copied 4x) - Centralize formatNumberInTemplate + next-document-number logic in utils/documentSequences.js (was copied in invoice/quote/contract services)
This commit is contained in:
Paul Nothaft
@@ -10,6 +10,7 @@ const cron = require('node-cron');
|
||||
const { db } = require('../database/db');
|
||||
const { queueEmail } = require('./emailProcessor');
|
||||
const logger = require('../utils/logger');
|
||||
const { formatBytes } = require('../utils/formatBytes');
|
||||
const { formatBoolean } = require('../utils/dbCompat');
|
||||
const backupManifest = require('./backupManifest');
|
||||
const S3StorageAdapter = require('./storage/s3Storage');
|
||||
@@ -778,19 +779,6 @@ async function performRsyncBackup(config, files) {
|
||||
};
|
||||
}
|
||||
|
||||
function formatBytes(bytes, decimals = 2) {
|
||||
if (!bytes) {
|
||||
return '0 Bytes';
|
||||
}
|
||||
|
||||
const k = 1024;
|
||||
const dm = decimals < 0 ? 0 : decimals;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
|
||||
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(dm))} ${sizes[i]}`;
|
||||
}
|
||||
|
||||
async function performS3Backup(config, files) {
|
||||
try {
|
||||
const bucket = config.backup_s3_bucket;
|
||||
|
||||
@@ -1,720 +0,0 @@
|
||||
const cron = require('node-cron');
|
||||
const path = require('path');
|
||||
const fs = require('fs').promises;
|
||||
const crypto = require('crypto');
|
||||
const { exec } = require('child_process');
|
||||
const { promisify } = require('util');
|
||||
const execAsync = promisify(exec);
|
||||
const { db } = require('../database/db');
|
||||
const { queueEmail } = require('./emailProcessor');
|
||||
const logger = require('../utils/logger');
|
||||
const { formatBoolean } = require('../utils/dbCompat');
|
||||
const backupManifest = require('./backupManifest');
|
||||
|
||||
// Backup job reference
|
||||
let backupJob = null;
|
||||
let backupConfig = null;
|
||||
let isRunning = false;
|
||||
|
||||
// Storage paths
|
||||
const getStoragePath = () => process.env.STORAGE_PATH || path.join(__dirname, '../../../storage');
|
||||
|
||||
/**
|
||||
* Calculate file checksum using SHA256
|
||||
*/
|
||||
async function calculateChecksum(filePath) {
|
||||
const hash = crypto.createHash('sha256');
|
||||
const stream = require('fs').createReadStream(filePath);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', data => hash.update(data));
|
||||
stream.on('end', () => resolve(hash.digest('hex')));
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get database backup information
|
||||
*/
|
||||
async function getDatabaseBackupInfo() {
|
||||
try {
|
||||
// Check for recent database backup
|
||||
const recentDbBackup = await db('database_backup_runs')
|
||||
.where('status', 'completed')
|
||||
.orderBy('completed_at', 'desc')
|
||||
.first();
|
||||
|
||||
if (recentDbBackup && recentDbBackup.file_path) {
|
||||
return {
|
||||
type: recentDbBackup.backup_type,
|
||||
backupFile: recentDbBackup.file_path,
|
||||
size: recentDbBackup.file_size_bytes,
|
||||
checksum: recentDbBackup.checksum,
|
||||
tables: recentDbBackup.statistics ? JSON.parse(recentDbBackup.statistics).tables : {},
|
||||
rowCounts: recentDbBackup.table_checksums ? JSON.parse(recentDbBackup.table_checksums) : {}
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
type: process.env.DB_TYPE === 'postgresql' ? 'postgresql' : 'sqlite',
|
||||
backupFile: null,
|
||||
size: 0,
|
||||
checksum: null,
|
||||
tables: {},
|
||||
rowCounts: {}
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get database backup info:', error);
|
||||
return {
|
||||
type: 'unknown',
|
||||
backupFile: null,
|
||||
size: 0,
|
||||
checksum: null,
|
||||
tables: {},
|
||||
rowCounts: {}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup configuration from database
|
||||
*/
|
||||
async function getBackupConfig() {
|
||||
try {
|
||||
const settings = await db('app_settings')
|
||||
.where('setting_type', 'backup')
|
||||
.select('setting_key', 'setting_value');
|
||||
|
||||
const config = {};
|
||||
settings.forEach(setting => {
|
||||
try {
|
||||
config[setting.setting_key] = JSON.parse(setting.setting_value);
|
||||
} catch (e) {
|
||||
config[setting.setting_key] = setting.setting_value;
|
||||
}
|
||||
});
|
||||
|
||||
return config;
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup configuration:', error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get list of files to backup
|
||||
*/
|
||||
async function getFilesToBackup(includeArchived = true) {
|
||||
const files = [];
|
||||
const storagePath = getStoragePath();
|
||||
|
||||
try {
|
||||
// Active events
|
||||
const activePath = path.join(storagePath, 'events/active');
|
||||
await scanDirectory(activePath, files, storagePath);
|
||||
|
||||
// Archived events (if enabled)
|
||||
if (includeArchived) {
|
||||
const archivePath = path.join(storagePath, 'events/archived');
|
||||
await scanDirectory(archivePath, files, storagePath);
|
||||
}
|
||||
|
||||
// Thumbnails
|
||||
const thumbsPath = path.join(storagePath, 'thumbnails');
|
||||
await scanDirectory(thumbsPath, files, storagePath);
|
||||
|
||||
// Uploads (logos, favicons, etc.)
|
||||
const uploadsPath = path.join(storagePath, 'uploads');
|
||||
await scanDirectory(uploadsPath, files, storagePath);
|
||||
|
||||
return files;
|
||||
} catch (error) {
|
||||
logger.error('Failed to get files to backup:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively scan directory for files
|
||||
*/
|
||||
async function scanDirectory(dirPath, fileList, basePath, excludePatterns = []) {
|
||||
try {
|
||||
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
const relativePath = path.relative(basePath, fullPath);
|
||||
|
||||
// Check exclude patterns
|
||||
if (excludePatterns.some(pattern => {
|
||||
if (pattern.includes('*')) {
|
||||
return new RegExp(pattern.replace(/\*/g, '.*')).test(entry.name);
|
||||
}
|
||||
return entry.name === pattern;
|
||||
})) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await scanDirectory(fullPath, fileList, basePath, excludePatterns);
|
||||
} else if (entry.isFile()) {
|
||||
const stats = await fs.stat(fullPath);
|
||||
fileList.push({
|
||||
path: fullPath,
|
||||
relativePath: relativePath,
|
||||
size: stats.size,
|
||||
modified: stats.mtime
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
logger.error(`Failed to scan directory ${dirPath}:`, error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file has changed since last backup
|
||||
*/
|
||||
async function hasFileChanged(filePath, checksum) {
|
||||
try {
|
||||
const fileState = await db('backup_file_states')
|
||||
.where('file_path', filePath)
|
||||
.first();
|
||||
|
||||
return !fileState || fileState.checksum !== checksum;
|
||||
} catch (error) {
|
||||
logger.error('Failed to check file state:', error);
|
||||
return true; // Assume changed if we can't check
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update file state in database
|
||||
*/
|
||||
async function updateFileState(filePath, checksum, size, modified) {
|
||||
try {
|
||||
const existing = await db('backup_file_states')
|
||||
.where('file_path', filePath)
|
||||
.first();
|
||||
|
||||
const data = {
|
||||
file_path: filePath,
|
||||
checksum: checksum,
|
||||
size_bytes: size,
|
||||
last_modified: modified,
|
||||
last_backed_up: new Date()
|
||||
};
|
||||
|
||||
if (existing) {
|
||||
await db('backup_file_states')
|
||||
.where('id', existing.id)
|
||||
.update(data);
|
||||
} else {
|
||||
await db('backup_file_states').insert(data);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to update file state:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform local directory backup
|
||||
*/
|
||||
async function performLocalBackup(config, files) {
|
||||
const destPath = config.backup_destination_path;
|
||||
const storagePath = getStoragePath();
|
||||
let backedUpCount = 0;
|
||||
let backedUpSize = 0;
|
||||
const backedUpFiles = [];
|
||||
|
||||
// Ensure destination exists
|
||||
await fs.mkdir(destPath, { recursive: true });
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
// Skip large files if configured
|
||||
const maxSizeMB = config.backup_max_file_size_mb || 5000;
|
||||
if (file.size > maxSizeMB * 1024 * 1024) {
|
||||
logger.warn(`Skipping large file: ${file.relativePath} (${(file.size / 1024 / 1024).toFixed(2)} MB)`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Calculate checksum
|
||||
const checksum = await calculateChecksum(file.path);
|
||||
file.checksum = checksum; // Add checksum to file object
|
||||
|
||||
// Check if file has changed
|
||||
const changed = await hasFileChanged(file.relativePath, checksum);
|
||||
if (!changed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Copy file
|
||||
const destFilePath = path.join(destPath, file.relativePath);
|
||||
const destDir = path.dirname(destFilePath);
|
||||
await fs.mkdir(destDir, { recursive: true });
|
||||
await fs.copyFile(file.path, destFilePath);
|
||||
|
||||
// Update state
|
||||
await updateFileState(file.relativePath, checksum, file.size, file.modified);
|
||||
|
||||
backedUpCount++;
|
||||
backedUpSize += file.size;
|
||||
backedUpFiles.push(file.relativePath);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to backup file ${file.relativePath}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return { backedUpCount, backedUpSize, backedUpFiles };
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform rsync backup
|
||||
*/
|
||||
async function performRsyncBackup(config, files) {
|
||||
const storagePath = getStoragePath();
|
||||
const host = config.backup_rsync_host;
|
||||
const user = config.backup_rsync_user;
|
||||
const remotePath = config.backup_rsync_path;
|
||||
const sshKey = config.backup_rsync_ssh_key;
|
||||
|
||||
if (!host || !remotePath) {
|
||||
throw new Error('Rsync configuration incomplete');
|
||||
}
|
||||
|
||||
// Build rsync command
|
||||
const rsyncOptions = [
|
||||
'-avz', // archive, verbose, compress
|
||||
'--delete', // remove deleted files
|
||||
'--stats' // show statistics
|
||||
];
|
||||
|
||||
if (sshKey) {
|
||||
rsyncOptions.push(`-e "ssh -i ${sshKey} -o StrictHostKeyChecking=no"`);
|
||||
}
|
||||
|
||||
// Add exclude patterns
|
||||
const excludePatterns = config.backup_exclude_patterns || [];
|
||||
excludePatterns.forEach(pattern => {
|
||||
rsyncOptions.push(`--exclude="${pattern}"`);
|
||||
});
|
||||
|
||||
const source = `${storagePath}/`;
|
||||
const destination = user ? `${user}@${host}:${remotePath}` : `${host}:${remotePath}`;
|
||||
|
||||
const rsyncCommand = `rsync ${rsyncOptions.join(' ')} "${source}" "${destination}"`;
|
||||
|
||||
try {
|
||||
const { stdout, stderr } = await execAsync(rsyncCommand);
|
||||
|
||||
// Parse rsync stats
|
||||
const stats = parseRsyncStats(stdout);
|
||||
|
||||
// Update file states for successfully synced files
|
||||
for (const file of files) {
|
||||
try {
|
||||
const checksum = await calculateChecksum(file.path);
|
||||
await updateFileState(file.relativePath, checksum, file.size, file.modified);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to update state for ${file.relativePath}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
backedUpCount: stats.filesTransferred || files.length,
|
||||
backedUpSize: stats.totalSize || files.reduce((sum, f) => sum + f.size, 0),
|
||||
backedUpFiles: files.map(f => f.relativePath)
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Rsync backup failed:', error);
|
||||
throw new Error(`Rsync backup failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse rsync statistics from output
|
||||
*/
|
||||
function parseRsyncStats(output) {
|
||||
const stats = {};
|
||||
|
||||
// Extract files transferred
|
||||
const filesMatch = output.match(/Number of files transferred: (\d+)/);
|
||||
if (filesMatch) {
|
||||
stats.filesTransferred = parseInt(filesMatch[1]);
|
||||
}
|
||||
|
||||
// Extract total size
|
||||
const sizeMatch = output.match(/Total file size: ([\d,]+) bytes/);
|
||||
if (sizeMatch) {
|
||||
stats.totalSize = parseInt(sizeMatch[1].replace(/,/g, ''));
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform S3-compatible backup
|
||||
*/
|
||||
async function performS3Backup(config, files) {
|
||||
// This would require AWS SDK or similar
|
||||
// For now, return a placeholder
|
||||
throw new Error('S3 backup not implemented yet');
|
||||
}
|
||||
|
||||
/**
|
||||
* Run backup process
|
||||
*/
|
||||
async function runBackup() {
|
||||
if (isRunning) {
|
||||
logger.warn('Backup already running, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
isRunning = true;
|
||||
const startTime = new Date();
|
||||
let backupRun = null;
|
||||
|
||||
try {
|
||||
// Get current configuration
|
||||
const config = await getBackupConfig();
|
||||
if (!config.backup_enabled) {
|
||||
logger.info('Backup is disabled, skipping');
|
||||
return;
|
||||
}
|
||||
|
||||
// Create backup run record
|
||||
const [runId] = await db('backup_runs').insert({
|
||||
started_at: startTime,
|
||||
status: 'running',
|
||||
backup_type: 'scheduled'
|
||||
});
|
||||
|
||||
backupRun = { id: runId };
|
||||
|
||||
// Get files to backup
|
||||
const files = await getFilesToBackup(config.backup_include_archived);
|
||||
logger.info(`Found ${files.length} files to check for backup`);
|
||||
|
||||
// Perform backup based on destination type
|
||||
let result;
|
||||
switch (config.backup_destination_type) {
|
||||
case 'local':
|
||||
result = await performLocalBackup(config, files);
|
||||
break;
|
||||
case 'rsync':
|
||||
result = await performRsyncBackup(config, files);
|
||||
break;
|
||||
case 's3':
|
||||
result = await performS3Backup(config, files);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown backup destination type: ${config.backup_destination_type}`);
|
||||
}
|
||||
|
||||
// Calculate duration
|
||||
const endTime = new Date();
|
||||
const durationSeconds = Math.round((endTime - startTime) / 1000);
|
||||
|
||||
// Generate backup manifest
|
||||
let manifestPath = null;
|
||||
try {
|
||||
logger.info('Generating backup manifest...');
|
||||
|
||||
// Get database backup info if available
|
||||
const databaseInfo = await getDatabaseBackupInfo();
|
||||
|
||||
// Determine if this is an incremental backup
|
||||
const lastSuccessfulBackup = await db('backup_runs')
|
||||
.where('status', 'completed')
|
||||
.whereNot('id', runId)
|
||||
.orderBy('completed_at', 'desc')
|
||||
.first();
|
||||
|
||||
let manifest;
|
||||
const manifestOptions = {
|
||||
backupType: lastSuccessfulBackup ? 'incremental' : 'full',
|
||||
backupPath: config.backup_destination_path || config.backup_destination_type,
|
||||
files: files.filter(f => result.backedUpFiles && result.backedUpFiles.includes(f.relativePath)),
|
||||
databaseInfo: databaseInfo,
|
||||
parentBackupId: lastSuccessfulBackup ? lastSuccessfulBackup.manifest_id : null,
|
||||
format: config.backup_manifest_format || 'json',
|
||||
customMetadata: {
|
||||
backup_run_id: runId,
|
||||
destination_type: config.backup_destination_type,
|
||||
operator: 'system',
|
||||
reason: 'scheduled',
|
||||
retentionDays: config.backup_retention_days || 30
|
||||
}
|
||||
};
|
||||
|
||||
if (lastSuccessfulBackup && lastSuccessfulBackup.manifest_path) {
|
||||
try {
|
||||
const parentManifest = await backupManifest.loadManifest(lastSuccessfulBackup.manifest_path);
|
||||
manifest = await backupManifest.generateIncrementalManifest(manifestOptions, parentManifest);
|
||||
} catch (error) {
|
||||
logger.warn('Failed to load parent manifest, generating full manifest:', error);
|
||||
manifest = await backupManifest.generateManifest(manifestOptions);
|
||||
}
|
||||
} else {
|
||||
manifest = await backupManifest.generateManifest(manifestOptions);
|
||||
}
|
||||
|
||||
// Save manifest
|
||||
const manifestDir = config.backup_manifest_path || path.join(config.backup_destination_path || '/backup', 'manifests');
|
||||
await fs.mkdir(manifestDir, { recursive: true });
|
||||
|
||||
const manifestFileName = `backup-manifest-${manifest.backup.id}.${config.backup_manifest_format || 'json'}`;
|
||||
manifestPath = path.join(manifestDir, manifestFileName);
|
||||
await backupManifest.saveManifest(manifest, manifestPath, config.backup_manifest_format || 'json');
|
||||
|
||||
logger.info(`Backup manifest saved to ${manifestPath}`);
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to generate backup manifest:', error);
|
||||
// Don't fail the entire backup for manifest generation failure
|
||||
}
|
||||
|
||||
// Update backup run record
|
||||
await db('backup_runs')
|
||||
.where('id', runId)
|
||||
.update({
|
||||
completed_at: endTime,
|
||||
status: 'completed',
|
||||
files_backed_up: result.backedUpCount,
|
||||
total_size_bytes: result.backedUpSize,
|
||||
duration_seconds: durationSeconds,
|
||||
manifest_path: manifestPath,
|
||||
manifest_id: manifestPath ? path.basename(manifestPath, path.extname(manifestPath)) : null,
|
||||
statistics: JSON.stringify({
|
||||
totalFilesChecked: files.length,
|
||||
filesBackedUp: result.backedUpCount,
|
||||
totalSize: result.backedUpSize,
|
||||
averageFileSize: result.backedUpCount > 0 ? Math.round(result.backedUpSize / result.backedUpCount) : 0,
|
||||
manifestGenerated: !!manifestPath
|
||||
})
|
||||
});
|
||||
|
||||
logger.info(`Backup completed: ${result.backedUpCount} files, ${(result.backedUpSize / 1024 / 1024).toFixed(2)} MB in ${durationSeconds}s`);
|
||||
|
||||
// Send success email if configured
|
||||
if (config.backup_email_on_success) {
|
||||
// Get admin emails
|
||||
const admins = await db('admin_users').where('is_active', formatBoolean(true));
|
||||
for (const admin of admins) {
|
||||
await queueEmail(null, admin.email, 'backup_completed', {
|
||||
start_time: startTime.toISOString(),
|
||||
duration: `${durationSeconds} seconds`,
|
||||
files_count: result.backedUpCount.toString(),
|
||||
total_size: `${(result.backedUpSize / 1024 / 1024).toFixed(2)} MB`,
|
||||
backup_type: config.backup_destination_type
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Backup failed:', error);
|
||||
|
||||
// Update backup run record
|
||||
if (backupRun) {
|
||||
await db('backup_runs')
|
||||
.where('id', backupRun.id)
|
||||
.update({
|
||||
completed_at: new Date(),
|
||||
status: 'failed',
|
||||
error_message: error.message
|
||||
});
|
||||
}
|
||||
|
||||
// Send failure email
|
||||
const config = await getBackupConfig();
|
||||
if (config && config.backup_email_on_failure) {
|
||||
const admins = await db('admin_users').where('is_active', formatBoolean(true));
|
||||
for (const admin of admins) {
|
||||
await queueEmail(null, admin.email, 'backup_failed', {
|
||||
start_time: startTime.toISOString(),
|
||||
backup_type: config.backup_destination_type || 'unknown',
|
||||
error_message: error.message
|
||||
});
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
isRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start backup service
|
||||
*/
|
||||
async function startBackupService() {
|
||||
try {
|
||||
// Get configuration
|
||||
backupConfig = await getBackupConfig();
|
||||
|
||||
if (!backupConfig || !backupConfig.backup_enabled) {
|
||||
logger.info('Backup service is disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
// Cancel existing job if any
|
||||
if (backupJob) {
|
||||
backupJob.stop();
|
||||
}
|
||||
|
||||
// Schedule backup job
|
||||
const schedule = backupConfig.backup_schedule || '0 2 * * *'; // Default: 2 AM daily
|
||||
backupJob = cron.schedule(schedule, async () => {
|
||||
logger.info('Starting scheduled backup');
|
||||
await runBackup();
|
||||
});
|
||||
|
||||
logger.info(`Backup service started with schedule: ${schedule}`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to start backup service:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop backup service
|
||||
*/
|
||||
function stopBackupService() {
|
||||
if (backupJob) {
|
||||
backupJob.stop();
|
||||
backupJob = null;
|
||||
logger.info('Backup service stopped');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger manual backup
|
||||
*/
|
||||
async function triggerManualBackup() {
|
||||
logger.info('Starting manual backup');
|
||||
await runBackup();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup status and history
|
||||
*/
|
||||
async function getBackupStatus(limit = 10) {
|
||||
try {
|
||||
const runs = await db('backup_runs')
|
||||
.orderBy('started_at', 'desc')
|
||||
.limit(limit);
|
||||
|
||||
const lastRun = runs[0];
|
||||
const isHealthy = lastRun && lastRun.status === 'completed';
|
||||
|
||||
// Validate manifest if exists
|
||||
let manifestValid = false;
|
||||
if (lastRun && lastRun.manifest_path) {
|
||||
try {
|
||||
const manifest = await backupManifest.loadManifest(lastRun.manifest_path);
|
||||
backupManifest.validateManifest(manifest);
|
||||
manifestValid = true;
|
||||
} catch (error) {
|
||||
logger.warn('Manifest validation failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
isRunning,
|
||||
isHealthy,
|
||||
lastRun: lastRun ? {
|
||||
...lastRun,
|
||||
manifestValid
|
||||
} : null,
|
||||
recentRuns: runs,
|
||||
nextScheduledRun: backupJob ? getNextScheduledRun() : null
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup status:', error);
|
||||
return {
|
||||
isRunning,
|
||||
isHealthy: false,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get next scheduled run time
|
||||
*/
|
||||
function getNextScheduledRun() {
|
||||
// This is a simplified version - would need proper cron parsing
|
||||
const now = new Date();
|
||||
const tomorrow = new Date(now);
|
||||
tomorrow.setDate(tomorrow.getDate() + 1);
|
||||
tomorrow.setHours(2, 0, 0, 0); // Assuming default 2 AM schedule
|
||||
return tomorrow.toISOString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up old backup runs
|
||||
*/
|
||||
async function cleanupOldBackupRuns(retentionDays = 30) {
|
||||
try {
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
|
||||
|
||||
const deleted = await db('backup_runs')
|
||||
.where('started_at', '<', cutoffDate)
|
||||
.delete();
|
||||
|
||||
if (deleted > 0) {
|
||||
logger.info(`Cleaned up ${deleted} old backup runs`);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Failed to cleanup old backup runs:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get backup manifest for a specific backup run
|
||||
*/
|
||||
async function getBackupManifest(backupRunId) {
|
||||
try {
|
||||
const run = await db('backup_runs')
|
||||
.where('id', backupRunId)
|
||||
.first();
|
||||
|
||||
if (!run || !run.manifest_path) {
|
||||
throw new Error('Backup manifest not found');
|
||||
}
|
||||
|
||||
const manifest = await backupManifest.loadManifest(run.manifest_path);
|
||||
return {
|
||||
manifest,
|
||||
summary: backupManifest.generateSummaryReport(manifest)
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('Failed to get backup manifest:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a backup manifest file
|
||||
*/
|
||||
async function validateBackupManifest(manifestPath) {
|
||||
try {
|
||||
const manifest = await backupManifest.loadManifest(manifestPath);
|
||||
backupManifest.validateManifest(manifest);
|
||||
return { valid: true, manifest };
|
||||
} catch (error) {
|
||||
return { valid: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
startBackupService,
|
||||
stopBackupService,
|
||||
triggerManualBackup,
|
||||
getBackupStatus,
|
||||
runBackup,
|
||||
cleanupOldBackupRuns,
|
||||
getBackupManifest,
|
||||
validateBackupManifest
|
||||
};
|
||||
@@ -39,7 +39,7 @@ const { db, withRetry, logActivity } = require('../database/db');
|
||||
const logger = require('../utils/logger');
|
||||
const { getAppSetting } = require('../utils/appSettings');
|
||||
const { AppError } = require('../utils/errors');
|
||||
const { claimNextSequence } = require('../utils/documentSequences');
|
||||
const { nextDocumentNumber } = require('../utils/documentSequences');
|
||||
const { hasColumnCached } = require('../utils/schemaCache');
|
||||
const { formatShortDate } = require('../utils/dateFormatter');
|
||||
const businessProfileService = require('./businessProfileService');
|
||||
@@ -151,14 +151,6 @@ async function maybeStoreIp(ip) {
|
||||
// D.2 — `ensureInt` consolidated into utils/numericHelpers.
|
||||
const { ensureInt } = require('../utils/numericHelpers');
|
||||
|
||||
function formatNumberInTemplate(format, year, seq) {
|
||||
return format
|
||||
.replace(/\{YEAR\}/g, String(year))
|
||||
.replace(/\{MONTH\}/g, String(new Date().getMonth() + 1).padStart(2, '0'))
|
||||
.replace(/\{SEQ:(\d+)d\}/g, (_, pad) => String(seq).padStart(parseInt(pad, 10), '0'))
|
||||
.replace(/\{SEQ\}/g, String(seq));
|
||||
}
|
||||
|
||||
/**
|
||||
* Gap-free per-year contract number sequence. See
|
||||
* utils/documentSequences.js for the locking story; migration 132
|
||||
@@ -167,13 +159,7 @@ function formatNumberInTemplate(format, year, seq) {
|
||||
* emit `C-2026-AB12C3` after 5 retries.
|
||||
*/
|
||||
async function nextContractNumber(trx) {
|
||||
// Read through `trx` when present — getAppSetting on the global db inside an
|
||||
// open transaction deadlocks the single-connection SQLite pool (the booking
|
||||
// flow's prepare_contract action runs createFromQuote unattended).
|
||||
const format = (await getAppSetting('crm_contracts_number_format', null, trx || db)) || 'C-{YEAR}-{SEQ:04d}';
|
||||
const year = new Date().getFullYear();
|
||||
const seq = await claimNextSequence('contract', year, trx);
|
||||
return formatNumberInTemplate(format, year, seq);
|
||||
return nextDocumentNumber('contract', 'crm_contracts_number_format', 'C-{YEAR}-{SEQ:04d}', trx);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,147 +0,0 @@
|
||||
/**
|
||||
* Database Backup Service Usage Examples
|
||||
*
|
||||
* This service provides comprehensive database backup functionality
|
||||
* with support for both SQLite and PostgreSQL databases.
|
||||
*/
|
||||
|
||||
const { databaseBackupService } = require('./databaseBackup');
|
||||
|
||||
// Example 1: Manual backup with default settings
|
||||
async function manualBackup() {
|
||||
try {
|
||||
const result = await databaseBackupService.backup();
|
||||
console.log('Backup completed:', result);
|
||||
// Result includes: path, size, duration, checksum, compressionRatio
|
||||
} catch (error) {
|
||||
console.error('Backup failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Example 2: Backup with custom options
|
||||
async function customBackup() {
|
||||
try {
|
||||
const result = await databaseBackupService.backup({
|
||||
destinationPath: '/custom/backup/path',
|
||||
compress: true, // Enable gzip compression
|
||||
validateIntegrity: true, // Validate backup after creation
|
||||
includeChecksums: true, // Calculate table checksums
|
||||
noTransaction: false // Use transaction for consistency (PostgreSQL)
|
||||
});
|
||||
console.log('Custom backup completed:', result);
|
||||
} catch (error) {
|
||||
console.error('Backup failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Example 3: Check backup progress (useful for long-running backups)
|
||||
async function backupWithProgress() {
|
||||
// Start backup asynchronously
|
||||
const backupPromise = databaseBackupService.backup();
|
||||
|
||||
// Poll for progress
|
||||
const progressInterval = setInterval(() => {
|
||||
const progress = databaseBackupService.getProgress();
|
||||
if (progress) {
|
||||
console.log(`Progress: ${progress.message}`, progress.details);
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
try {
|
||||
const result = await backupPromise;
|
||||
clearInterval(progressInterval);
|
||||
console.log('Backup completed:', result);
|
||||
} catch (error) {
|
||||
clearInterval(progressInterval);
|
||||
console.error('Backup failed:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Example 4: Get backup history
|
||||
async function getBackupHistory() {
|
||||
const history = await databaseBackupService.getBackupHistory(10);
|
||||
|
||||
history.forEach(backup => {
|
||||
console.log(`Backup ${backup.id}:`);
|
||||
console.log(` Started: ${backup.started_at}`);
|
||||
console.log(` Status: ${backup.status}`);
|
||||
console.log(` Size: ${(backup.file_size_bytes / 1024 / 1024).toFixed(2)} MB`);
|
||||
console.log(` Duration: ${backup.duration_seconds}s`);
|
||||
});
|
||||
}
|
||||
|
||||
// Example 5: Clean up old backups
|
||||
async function cleanupBackups() {
|
||||
// Delete backups older than 30 days
|
||||
await databaseBackupService.cleanupOldBackups(30);
|
||||
console.log('Old backups cleaned up');
|
||||
}
|
||||
|
||||
// Example 6: Get table checksums (useful for monitoring changes)
|
||||
async function getTableChecksums() {
|
||||
const checksums = await databaseBackupService.getTableChecksums();
|
||||
|
||||
console.log('Table Checksums:');
|
||||
Object.entries(checksums).forEach(([table, info]) => {
|
||||
console.log(` ${table}: ${info.rowCount} rows, checksum: ${info.checksum}`);
|
||||
});
|
||||
}
|
||||
|
||||
// Example 7: Using the scheduled backup service
|
||||
const { startScheduledBackups, stopScheduledBackups } = require('./databaseBackup');
|
||||
|
||||
async function setupScheduledBackups() {
|
||||
// Start scheduled backups (reads schedule from database config)
|
||||
await startScheduledBackups();
|
||||
console.log('Scheduled backups started');
|
||||
|
||||
// Later, if needed, stop scheduled backups
|
||||
// stopScheduledBackups();
|
||||
}
|
||||
|
||||
// Example 8: Admin API endpoints available
|
||||
/*
|
||||
GET /api/admin/database-backup/status - Get backup status and config
|
||||
PUT /api/admin/database-backup/config - Update backup configuration
|
||||
POST /api/admin/database-backup/backup - Trigger manual backup
|
||||
GET /api/admin/database-backup/progress - Get current backup progress
|
||||
GET /api/admin/database-backup/history - Get backup history with pagination
|
||||
DELETE /api/admin/database-backup/cleanup - Delete old backup files
|
||||
POST /api/admin/database-backup/test - Test backup configuration
|
||||
GET /api/admin/database-backup/checksums - Get current table checksums
|
||||
*/
|
||||
|
||||
// Example 9: Configuration options stored in database
|
||||
/*
|
||||
database_backup_enabled: boolean - Enable/disable scheduled backups
|
||||
database_backup_schedule: string - Cron schedule (default: '0 3 * * *')
|
||||
database_backup_destination_path: string - Where to store backups
|
||||
database_backup_compress: boolean - Enable gzip compression
|
||||
database_backup_validate_integrity: boolean - Validate after backup
|
||||
database_backup_include_checksums: boolean - Calculate table checksums
|
||||
database_backup_retention_days: number - Days to keep old backups
|
||||
database_backup_email_on_failure: boolean - Send email on failure
|
||||
database_backup_email_on_success: boolean - Send email on success
|
||||
*/
|
||||
|
||||
// Example 10: Production considerations
|
||||
/*
|
||||
1. Ensure destination path has sufficient space
|
||||
2. For large databases, backups may take significant time
|
||||
3. PostgreSQL backups use single-transaction mode by default
|
||||
4. Compression typically reduces size by 70-90%
|
||||
5. Schedule backups during low-traffic periods
|
||||
6. Monitor backup history for failures
|
||||
7. Test restore procedures regularly
|
||||
8. Consider replication for real-time redundancy
|
||||
*/
|
||||
|
||||
module.exports = {
|
||||
manualBackup,
|
||||
customBackup,
|
||||
backupWithProgress,
|
||||
getBackupHistory,
|
||||
cleanupBackups,
|
||||
getTableChecksums,
|
||||
setupScheduledBackups
|
||||
};
|
||||
@@ -26,7 +26,7 @@ const { getAppSetting } = require('../utils/appSettings');
|
||||
const { cleanNetMinor } = require('../utils/invoiceRounding');
|
||||
const { AppError } = require('../utils/errors');
|
||||
const { formatBoolean } = require('../utils/dbCompat');
|
||||
const { claimNextSequence } = require('../utils/documentSequences');
|
||||
const { nextDocumentNumber } = require('../utils/documentSequences');
|
||||
const { formatShortDate } = require('../utils/dateFormatter');
|
||||
const businessProfileService = require('./businessProfileService');
|
||||
const { buildIssuerBlock, buildRecipientBlock } = require('./_renderContext');
|
||||
@@ -46,26 +46,13 @@ function getHierarchyHelpers() {
|
||||
const { ensureInt, ensureNumber } = require('../utils/numericHelpers');
|
||||
const { hasColumnCached } = require('../utils/schemaCache');
|
||||
|
||||
function formatNumberInTemplate(format, year, seq) {
|
||||
return format
|
||||
.replace(/\{YEAR\}/g, String(year))
|
||||
.replace(/\{MONTH\}/g, String(new Date().getMonth() + 1).padStart(2, '0'))
|
||||
.replace(/\{SEQ:(\d+)d\}/g, (_, pad) => String(seq).padStart(parseInt(pad, 10), '0'))
|
||||
.replace(/\{SEQ\}/g, String(seq));
|
||||
}
|
||||
|
||||
// Atomic gap-free invoice number generator. See utils/documentSequences.js
|
||||
// for the locking story; migration 132 created the underlying table.
|
||||
// The previous SELECT-MAX-then-INSERT path raced under concurrent
|
||||
// admin creates and emitted a random `R-2026-AB12C3` after 5 retries,
|
||||
// breaking the §14 UStG single-sequence requirement.
|
||||
async function nextInvoiceNumber(trx) {
|
||||
// Read through `trx` when present — getAppSetting on the global db inside an
|
||||
// open transaction deadlocks the single-connection SQLite pool.
|
||||
const format = (await getAppSetting('crm_invoices_number_format', null, trx || db)) || 'R-{YEAR}-{SEQ:04d}';
|
||||
const year = new Date().getFullYear();
|
||||
const seq = await claimNextSequence('invoice', year, trx);
|
||||
return formatNumberInTemplate(format, year, seq);
|
||||
return nextDocumentNumber('invoice', 'crm_invoices_number_format', 'R-{YEAR}-{SEQ:04d}', trx);
|
||||
}
|
||||
|
||||
function ensureCustomerCanBill(customer) {
|
||||
|
||||
@@ -33,7 +33,7 @@ const { getAppSetting } = require('../utils/appSettings');
|
||||
const { cleanNetMinor } = require('../utils/invoiceRounding');
|
||||
const { AppError } = require('../utils/errors');
|
||||
const { formatBoolean } = require('../utils/dbCompat');
|
||||
const { claimNextSequence } = require('../utils/documentSequences');
|
||||
const { nextDocumentNumber } = require('../utils/documentSequences');
|
||||
const { formatShortDate } = require('../utils/dateFormatter');
|
||||
const businessProfileService = require('./businessProfileService');
|
||||
const { buildIssuerBlock, buildRecipientBlock } = require('./_renderContext');
|
||||
@@ -300,28 +300,12 @@ async function insertLineItemsHierarchical(trx, tableName, ownerColumn, ownerId,
|
||||
}
|
||||
}
|
||||
|
||||
function formatNumberInTemplate(format, year, seq) {
|
||||
// Tokens: {YEAR}, {MONTH}, {SEQ:04d}. Defaults handle padding via
|
||||
// a tiny formatter, kept inline to avoid a new dependency.
|
||||
return format
|
||||
.replace(/\{YEAR\}/g, String(year))
|
||||
.replace(/\{MONTH\}/g, String(new Date().getMonth() + 1).padStart(2, '0'))
|
||||
.replace(/\{SEQ:(\d+)d\}/g, (_, pad) => String(seq).padStart(parseInt(pad, 10), '0'))
|
||||
.replace(/\{SEQ\}/g, String(seq));
|
||||
}
|
||||
|
||||
// Atomic gap-free quote number generator. See utils/documentSequences.js
|
||||
// for the locking story; migration 132 created the underlying table.
|
||||
// The previous SELECT-MAX-then-INSERT path raced under concurrent
|
||||
// admin creates and could emit `Q-2026-AB12C3` after 5 retries.
|
||||
async function nextQuoteNumber(trx) {
|
||||
// Read through `trx` when present — getAppSetting on the global db inside an
|
||||
// open transaction deadlocks the single-connection SQLite pool (prepare_quote
|
||||
// runs createQuote unattended from a workflow).
|
||||
const format = (await getAppSetting('crm_quotes_number_format', null, trx || db)) || 'Q-{YEAR}-{SEQ:04d}';
|
||||
const year = new Date().getFullYear();
|
||||
const seq = await claimNextSequence('quote', year, trx);
|
||||
return formatNumberInTemplate(format, year, seq);
|
||||
return nextDocumentNumber('quote', 'crm_quotes_number_format', 'Q-{YEAR}-{SEQ:04d}', trx);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -12,6 +12,7 @@ const backupManifest = require('./backupManifest');
|
||||
const S3StorageAdapter = require('./storage/s3Storage');
|
||||
const { queueEmail } = require('./emailProcessor');
|
||||
const { formatBoolean } = require('../utils/dbCompat');
|
||||
const { formatBytes } = require('../utils/formatBytes');
|
||||
const os = require('os');
|
||||
|
||||
/**
|
||||
@@ -1528,15 +1529,7 @@ END $$;`
|
||||
* Format bytes to human readable
|
||||
*/
|
||||
formatBytes(bytes, decimals = 2) {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
|
||||
const k = 1024;
|
||||
const dm = decimals < 0 ? 0 : decimals;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
||||
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
|
||||
return formatBytes(bytes, decimals);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,221 +0,0 @@
|
||||
/**
|
||||
* Example usage of the S3StorageAdapter
|
||||
*
|
||||
* This file demonstrates how to use the S3 storage adapter for various operations
|
||||
*/
|
||||
|
||||
const S3StorageAdapter = require('./s3Storage');
|
||||
|
||||
// Example 1: Basic AWS S3 Configuration
|
||||
const s3Storage = new S3StorageAdapter({
|
||||
bucket: 'my-photo-bucket',
|
||||
region: 'us-east-1',
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
});
|
||||
|
||||
// Example 2: MinIO Configuration (S3-compatible)
|
||||
const minioStorage = new S3StorageAdapter({
|
||||
bucket: 'photo-storage',
|
||||
endpoint: 'http://localhost:9000', // MinIO endpoint
|
||||
accessKeyId: 'minioadmin',
|
||||
secretAccessKey: 'minioadmin',
|
||||
forcePathStyle: true, // Required for MinIO
|
||||
sslEnabled: false // For local development
|
||||
});
|
||||
|
||||
// Example 3: DigitalOcean Spaces Configuration
|
||||
const spacesStorage = new S3StorageAdapter({
|
||||
bucket: 'my-space-name',
|
||||
endpoint: 'https://nyc3.digitaloceanspaces.com',
|
||||
region: 'nyc3',
|
||||
accessKeyId: process.env.DO_SPACES_KEY,
|
||||
secretAccessKey: process.env.DO_SPACES_SECRET
|
||||
});
|
||||
|
||||
// Usage Examples
|
||||
async function examples() {
|
||||
try {
|
||||
// Test connection
|
||||
await s3Storage.testConnection();
|
||||
console.log('Connection successful!');
|
||||
|
||||
// Upload a file with progress tracking
|
||||
const uploadResult = await s3Storage.upload(
|
||||
'/path/to/local/photo.jpg',
|
||||
'events/wedding-2024/photo.jpg',
|
||||
{
|
||||
contentType: 'image/jpeg',
|
||||
metadata: {
|
||||
event: 'wedding-2024',
|
||||
photographer: 'John Doe'
|
||||
},
|
||||
onProgress: (loaded, total) => {
|
||||
const percentage = Math.round((loaded / total) * 100);
|
||||
console.log(`Upload progress: ${percentage}%`);
|
||||
}
|
||||
}
|
||||
);
|
||||
console.log('Upload complete:', uploadResult.Location);
|
||||
|
||||
// Upload from stream
|
||||
const readStream = fs.createReadStream('/path/to/large-video.mp4');
|
||||
await s3Storage.uploadStream(readStream, 'events/wedding-2024/video.mp4', {
|
||||
contentType: 'video/mp4',
|
||||
onProgress: (loaded, total) => {
|
||||
console.log(`Streamed ${loaded} of ${total} bytes`);
|
||||
}
|
||||
});
|
||||
|
||||
// Download a file
|
||||
await s3Storage.download(
|
||||
'events/wedding-2024/photo.jpg',
|
||||
'/path/to/downloaded/photo.jpg',
|
||||
{
|
||||
onProgress: (loaded, total) => {
|
||||
const percentage = Math.round((loaded / total) * 100);
|
||||
console.log(`Download progress: ${percentage}%`);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
// Get a download stream
|
||||
const downloadStream = await s3Storage.downloadStream('events/wedding-2024/photo.jpg');
|
||||
downloadStream.pipe(fs.createWriteStream('/path/to/output.jpg'));
|
||||
|
||||
// List files
|
||||
const listing = await s3Storage.list('events/wedding-2024/');
|
||||
console.log(`Found ${listing.Contents.length} files`);
|
||||
listing.Contents.forEach(file => {
|
||||
console.log(`- ${file.Key} (${file.Size} bytes)`);
|
||||
});
|
||||
|
||||
// Generate pre-signed URL for temporary access
|
||||
const downloadUrl = await s3Storage.getSignedUrl('getObject', 'events/wedding-2024/photo.jpg', {
|
||||
expiresIn: 3600 // 1 hour
|
||||
});
|
||||
console.log('Pre-signed download URL:', downloadUrl);
|
||||
|
||||
// Generate pre-signed upload URL
|
||||
const uploadUrl = await s3Storage.getSignedUrl('putObject', 'events/wedding-2024/new-photo.jpg', {
|
||||
expiresIn: 1800, // 30 minutes
|
||||
params: {
|
||||
ContentType: 'image/jpeg'
|
||||
}
|
||||
});
|
||||
console.log('Pre-signed upload URL:', uploadUrl);
|
||||
|
||||
// Check if file exists
|
||||
const exists = await s3Storage.exists('events/wedding-2024/photo.jpg');
|
||||
console.log('File exists:', exists);
|
||||
|
||||
// Get metadata
|
||||
const metadata = await s3Storage.getMetadata('events/wedding-2024/photo.jpg');
|
||||
console.log('File metadata:', metadata);
|
||||
|
||||
// Copy file
|
||||
await s3Storage.copy(
|
||||
'events/wedding-2024/photo.jpg',
|
||||
'events/wedding-2024/photo-copy.jpg'
|
||||
);
|
||||
|
||||
// Move file
|
||||
await s3Storage.move(
|
||||
'events/wedding-2024/photo-copy.jpg',
|
||||
'events/wedding-2024/archived/photo.jpg'
|
||||
);
|
||||
|
||||
// Delete file
|
||||
await s3Storage.delete('events/wedding-2024/temp-photo.jpg');
|
||||
|
||||
// Delete multiple files
|
||||
const deleteResult = await s3Storage.deleteMany([
|
||||
'events/wedding-2024/temp1.jpg',
|
||||
'events/wedding-2024/temp2.jpg',
|
||||
'events/wedding-2024/temp3.jpg'
|
||||
]);
|
||||
console.log(`Deleted ${deleteResult.Deleted.length} files`);
|
||||
|
||||
// Get storage statistics
|
||||
const stats = await s3Storage.getStats('events/');
|
||||
console.log(`Total files: ${stats.totalCount}`);
|
||||
console.log(`Total size: ${stats.totalSizeFormatted}`);
|
||||
|
||||
// Listen to events
|
||||
s3Storage.on('uploadProgress', (data) => {
|
||||
console.log(`Uploading ${data.key}: ${data.loaded}/${data.total}`);
|
||||
});
|
||||
|
||||
s3Storage.on('uploadComplete', (data) => {
|
||||
console.log(`Upload completed: ${data.key}`);
|
||||
});
|
||||
|
||||
s3Storage.on('uploadError', (data) => {
|
||||
console.error(`Upload failed for ${data.key}:`, data.error);
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Integration with existing photo upload workflow
|
||||
async function integrateWithPhotoUpload(eventId, files) {
|
||||
const storage = new S3StorageAdapter({
|
||||
bucket: process.env.S3_BUCKET,
|
||||
region: process.env.AWS_REGION,
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
|
||||
});
|
||||
|
||||
const uploadedPhotos = [];
|
||||
|
||||
for (const file of files) {
|
||||
try {
|
||||
// Generate unique S3 key
|
||||
const s3Key = storage.generateKey(file.originalname, `events/${eventId}`);
|
||||
|
||||
// Upload to S3
|
||||
const result = await storage.upload(file.path, s3Key, {
|
||||
contentType: file.mimetype,
|
||||
metadata: {
|
||||
eventId: eventId,
|
||||
originalName: file.originalname,
|
||||
uploadedAt: new Date().toISOString()
|
||||
}
|
||||
});
|
||||
|
||||
uploadedPhotos.push({
|
||||
filename: s3Key,
|
||||
originalName: file.originalname,
|
||||
size: file.size,
|
||||
mimeType: file.mimetype,
|
||||
s3Location: result.Location,
|
||||
s3Key: s3Key
|
||||
});
|
||||
|
||||
// Clean up local temp file
|
||||
await fs.promises.unlink(file.path);
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Failed to upload ${file.originalname}:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
return uploadedPhotos;
|
||||
}
|
||||
|
||||
// Environment variables needed:
|
||||
// AWS_ACCESS_KEY_ID=your-access-key
|
||||
// AWS_SECRET_ACCESS_KEY=your-secret-key
|
||||
// AWS_REGION=us-east-1
|
||||
// S3_BUCKET=your-bucket-name
|
||||
|
||||
// For MinIO:
|
||||
// MINIO_ENDPOINT=http://localhost:9000
|
||||
// MINIO_ACCESS_KEY=minioadmin
|
||||
// MINIO_SECRET_KEY=minioadmin
|
||||
// MINIO_BUCKET=photo-storage
|
||||
|
||||
module.exports = { examples, integrateWithPhotoUpload };
|
||||
@@ -7,6 +7,7 @@ const path = require('path');
|
||||
const stream = require('stream');
|
||||
const crypto = require('crypto');
|
||||
const logger = require('../../utils/logger');
|
||||
const { formatBytes } = require('../../utils/formatBytes');
|
||||
|
||||
/**
|
||||
* S3 Storage Adapter for handling file uploads to S3 and S3-compatible services
|
||||
@@ -738,15 +739,7 @@ class S3StorageAdapter extends stream.EventEmitter {
|
||||
* @private
|
||||
*/
|
||||
_formatBytes(bytes, decimals = 2) {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
|
||||
const k = 1024;
|
||||
const dm = decimals < 0 ? 0 : decimals;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
||||
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
|
||||
return formatBytes(bytes, decimals);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user