diff --git a/backend/knexfile.js b/backend/knexfile.js index aea520d..598d4fb 100644 --- a/backend/knexfile.js +++ b/backend/knexfile.js @@ -32,15 +32,27 @@ const config = { user: process.env.DB_USER || 'picpeak', password: process.env.DB_PASSWORD, database: process.env.DB_NAME || 'picpeak', - ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false + ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false, + // Connection stability settings + connectionTimeoutMillis: 30000, + idleTimeoutMillis: 30000, + keepAlive: true, + keepAliveInitialDelayMillis: 0 }, pool: { min: 2, - max: 10 + max: 10, + acquireTimeoutMillis: 30000, + createTimeoutMillis: 30000, + idleTimeoutMillis: 30000, + reapIntervalMillis: 1000, + createRetryIntervalMillis: 200, + propagateCreateError: false }, migrations: { directory: './migrations' - } + }, + acquireConnectionTimeout: 60000 } }; diff --git a/backend/src/database/connection-manager.js b/backend/src/database/connection-manager.js new file mode 100644 index 0000000..68dae7d --- /dev/null +++ b/backend/src/database/connection-manager.js @@ -0,0 +1,132 @@ +const knex = require('knex'); +const knexConfig = require('../../knexfile'); +const logger = require('../utils/logger'); + +class ConnectionManager { + constructor() { + this.db = null; + this.reconnectAttempts = 0; + this.maxReconnectAttempts = 10; + this.reconnectDelay = 5000; // 5 seconds + this.isReconnecting = false; + } + + async initialize() { + try { + this.db = knex(knexConfig); + + // Test the connection + await this.db.raw('SELECT 1'); + logger.info('Database connection established successfully'); + + // Set up connection error handling + this.setupErrorHandling(); + + this.reconnectAttempts = 0; + return this.db; + } catch (error) { + logger.error('Failed to initialize database connection:', error); + throw error; + } + } + + setupErrorHandling() { + if (!this.db) return; + + // Handle connection errors + this.db.on('error', async (error) => { + logger.error('Database connection error:', error); + + if (this.shouldReconnect(error)) { + await this.reconnect(); + } + }); + } + + shouldReconnect(error) { + const reconnectableErrors = [ + 'ECONNREFUSED', + 'ETIMEDOUT', + 'ECONNRESET', + 'Connection terminated unexpectedly', + 'Connection terminated' + ]; + + return reconnectableErrors.some(msg => + error.code === msg || error.message?.includes(msg) + ); + } + + async reconnect() { + if (this.isReconnecting) { + logger.info('Already attempting to reconnect...'); + return; + } + + this.isReconnecting = true; + + while (this.reconnectAttempts < this.maxReconnectAttempts) { + this.reconnectAttempts++; + + logger.info(`Attempting to reconnect to database (attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})...`); + + try { + // Destroy the old connection pool + if (this.db) { + await this.db.destroy(); + } + + // Create new connection + await this.initialize(); + + logger.info('Successfully reconnected to database'); + this.isReconnecting = false; + return; + } catch (error) { + logger.error(`Reconnection attempt ${this.reconnectAttempts} failed:`, error.message); + + if (this.reconnectAttempts < this.maxReconnectAttempts) { + await new Promise(resolve => setTimeout(resolve, this.reconnectDelay)); + } + } + } + + this.isReconnecting = false; + logger.error('Failed to reconnect to database after maximum attempts'); + + // In production, you might want to alert monitoring systems or restart the process + if (process.env.NODE_ENV === 'production') { + logger.error('Exiting process due to database connection failure'); + process.exit(1); + } + } + + getConnection() { + if (!this.db) { + throw new Error('Database connection not initialized'); + } + return this.db; + } + + async healthCheck() { + try { + await this.db.raw('SELECT 1'); + return { healthy: true }; + } catch (error) { + logger.error('Database health check failed:', error); + return { healthy: false, error: error.message }; + } + } + + async destroy() { + if (this.db) { + await this.db.destroy(); + this.db = null; + } + } +} + +// Create singleton instance +const connectionManager = new ConnectionManager(); + +module.exports = connectionManager; \ No newline at end of file diff --git a/backend/src/database/db.js b/backend/src/database/db.js index 773f6b8..1f50866 100644 --- a/backend/src/database/db.js +++ b/backend/src/database/db.js @@ -1,6 +1,7 @@ const knex = require('knex'); const knexConfig = require('../../knexfile'); +// Create database connection with built-in retry logic const db = knex(knexConfig); async function initializeDatabase() { diff --git a/backend/src/middleware/maintenance.js b/backend/src/middleware/maintenance.js index c0112ec..e3be0e0 100644 --- a/backend/src/middleware/maintenance.js +++ b/backend/src/middleware/maintenance.js @@ -5,6 +5,36 @@ let maintenanceMode = false; let lastCheck = 0; const CACHE_DURATION = 60000; // 1 minute +// Retry configuration for database queries +const MAX_RETRIES = 3; +const RETRY_DELAY = 1000; // 1 second + +async function queryWithRetry(queryFn, retries = MAX_RETRIES) { + for (let i = 0; i < retries; i++) { + try { + return await queryFn(); + } catch (error) { + if (i === retries - 1) { + throw error; + } + + // Check if it's a connection error that might benefit from retry + const isConnectionError = + error.message?.includes('Connection terminated') || + error.message?.includes('ECONNREFUSED') || + error.message?.includes('ETIMEDOUT') || + error.code === 'ECONNRESET'; + + if (isConnectionError) { + console.warn(`Database connection error, retrying in ${RETRY_DELAY}ms... (attempt ${i + 1}/${retries})`); + await new Promise(resolve => setTimeout(resolve, RETRY_DELAY)); + } else { + throw error; // Don't retry non-connection errors + } + } + } +} + async function checkMaintenanceMode() { const now = Date.now(); @@ -14,18 +44,21 @@ async function checkMaintenanceMode() { } try { - const setting = await db('app_settings') - .where('setting_key', 'general_maintenance_mode') - .where('setting_type', 'general') - .first(); + const setting = await queryWithRetry(async () => { + return await db('app_settings') + .where('setting_key', 'general_maintenance_mode') + .where('setting_type', 'general') + .first(); + }); maintenanceMode = setting ? (setting.setting_value === 'true' || setting.setting_value === true) : false; lastCheck = now; return maintenanceMode; } catch (error) { - console.error('Error checking maintenance mode:', error); - return false; + console.error('Error checking maintenance mode after retries:', error.message); + // Return cached value or false if no cache + return maintenanceMode; } } @@ -52,14 +85,19 @@ async function maintenanceMiddleware(req, res, next) { return next(); } - const inMaintenance = await checkMaintenanceMode(); - - if (inMaintenance && !isAdminRoute) { - return res.status(503).json({ - error: 'Service Unavailable', - message: 'The system is currently undergoing maintenance. Please try again later.', - maintenance: true - }); + try { + const inMaintenance = await checkMaintenanceMode(); + + if (inMaintenance && !isAdminRoute) { + return res.status(503).json({ + error: 'Service Unavailable', + message: 'The system is currently undergoing maintenance. Please try again later.', + maintenance: true + }); + } + } catch (error) { + // If we can't check maintenance mode, allow the request to proceed + console.error('Failed to check maintenance mode, allowing request:', error.message); } next();