fix: database connection stability issues in production

- Add robust connection pool configuration with timeouts and retry settings
- Implement retry logic in maintenance middleware for connection errors
- Increase connection stability with keepAlive and proper timeout values
- Handle "Connection terminated unexpectedly" errors gracefully

This prevents 503 errors when the database connection is temporarily interrupted
and ensures the application can recover from transient connection issues.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

backend/knexfile.js

@@ -32,15 +32,27 @@ const config = {
       user: process.env.DB_USER || 'picpeak',
       password: process.env.DB_PASSWORD,
       database: process.env.DB_NAME || 'picpeak',
-      ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false
+      ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false,
+      // Connection stability settings
+      connectionTimeoutMillis: 30000,
+      idleTimeoutMillis: 30000,
+      keepAlive: true,
+      keepAliveInitialDelayMillis: 0
     },
     pool: {
       min: 2,
-      max: 10
+      max: 10,
+      acquireTimeoutMillis: 30000,
+      createTimeoutMillis: 30000,
+      idleTimeoutMillis: 30000,
+      reapIntervalMillis: 1000,
+      createRetryIntervalMillis: 200,
+      propagateCreateError: false
     },
     migrations: {
       directory: './migrations'
-    }
+    },
+    acquireConnectionTimeout: 60000
   }
 };
 

backend/src/database/connection-manager.js

@@ -0,0 +1,132 @@
+const knex = require('knex');
+const knexConfig = require('../../knexfile');
+const logger = require('../utils/logger');
+
+class ConnectionManager {
+  constructor() {
+    this.db = null;
+    this.reconnectAttempts = 0;
+    this.maxReconnectAttempts = 10;
+    this.reconnectDelay = 5000; // 5 seconds
+    this.isReconnecting = false;
+  }
+
+  async initialize() {
+    try {
+      this.db = knex(knexConfig);
+      
+      // Test the connection
+      await this.db.raw('SELECT 1');
+      logger.info('Database connection established successfully');
+      
+      // Set up connection error handling
+      this.setupErrorHandling();
+      
+      this.reconnectAttempts = 0;
+      return this.db;
+    } catch (error) {
+      logger.error('Failed to initialize database connection:', error);
+      throw error;
+    }
+  }
+
+  setupErrorHandling() {
+    if (!this.db) return;
+
+    // Handle connection errors
+    this.db.on('error', async (error) => {
+      logger.error('Database connection error:', error);
+      
+      if (this.shouldReconnect(error)) {
+        await this.reconnect();
+      }
+    });
+  }
+
+  shouldReconnect(error) {
+    const reconnectableErrors = [
+      'ECONNREFUSED',
+      'ETIMEDOUT',
+      'ECONNRESET',
+      'Connection terminated unexpectedly',
+      'Connection terminated'
+    ];
+    
+    return reconnectableErrors.some(msg => 
+      error.code === msg || error.message?.includes(msg)
+    );
+  }
+
+  async reconnect() {
+    if (this.isReconnecting) {
+      logger.info('Already attempting to reconnect...');
+      return;
+    }
+
+    this.isReconnecting = true;
+    
+    while (this.reconnectAttempts < this.maxReconnectAttempts) {
+      this.reconnectAttempts++;
+      
+      logger.info(`Attempting to reconnect to database (attempt ${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
+      
+      try {
+        // Destroy the old connection pool
+        if (this.db) {
+          await this.db.destroy();
+        }
+        
+        // Create new connection
+        await this.initialize();
+        
+        logger.info('Successfully reconnected to database');
+        this.isReconnecting = false;
+        return;
+      } catch (error) {
+        logger.error(`Reconnection attempt ${this.reconnectAttempts} failed:`, error.message);
+        
+        if (this.reconnectAttempts < this.maxReconnectAttempts) {
+          await new Promise(resolve => setTimeout(resolve, this.reconnectDelay));
+        }
+      }
+    }
+    
+    this.isReconnecting = false;
+    logger.error('Failed to reconnect to database after maximum attempts');
+    
+    // In production, you might want to alert monitoring systems or restart the process
+    if (process.env.NODE_ENV === 'production') {
+      logger.error('Exiting process due to database connection failure');
+      process.exit(1);
+    }
+  }
+
+  getConnection() {
+    if (!this.db) {
+      throw new Error('Database connection not initialized');
+    }
+    return this.db;
+  }
+
+  async healthCheck() {
+    try {
+      await this.db.raw('SELECT 1');
+      return { healthy: true };
+    } catch (error) {
+      logger.error('Database health check failed:', error);
+      return { healthy: false, error: error.message };
+    }
+  }
+
+  async destroy() {
+    if (this.db) {
+      await this.db.destroy();
+      this.db = null;
+    }
+  }
+}
+
+// Create singleton instance
+const connectionManager = new ConnectionManager();
+
+module.exports = connectionManager;

backend/src/database/db.js

@@ -1,6 +1,7 @@
 const knex = require('knex');
 const knexConfig = require('../../knexfile');
 
+// Create database connection with built-in retry logic
 const db = knex(knexConfig);
 
 async function initializeDatabase() {

backend/src/middleware/maintenance.js

@@ -5,6 +5,36 @@ let maintenanceMode = false;
 let lastCheck = 0;
 const CACHE_DURATION = 60000; // 1 minute
 
+// Retry configuration for database queries
+const MAX_RETRIES = 3;
+const RETRY_DELAY = 1000; // 1 second
+
+async function queryWithRetry(queryFn, retries = MAX_RETRIES) {
+  for (let i = 0; i < retries; i++) {
+    try {
+      return await queryFn();
+    } catch (error) {
+      if (i === retries - 1) {
+        throw error;
+      }
+      
+      // Check if it's a connection error that might benefit from retry
+      const isConnectionError = 
+        error.message?.includes('Connection terminated') ||
+        error.message?.includes('ECONNREFUSED') ||
+        error.message?.includes('ETIMEDOUT') ||
+        error.code === 'ECONNRESET';
+      
+      if (isConnectionError) {
+        console.warn(`Database connection error, retrying in ${RETRY_DELAY}ms... (attempt ${i + 1}/${retries})`);
+        await new Promise(resolve => setTimeout(resolve, RETRY_DELAY));
+      } else {
+        throw error; // Don't retry non-connection errors
+      }
+    }
+  }
+}
+
 async function checkMaintenanceMode() {
   const now = Date.now();
   
@@ -14,18 +44,21 @@ async function checkMaintenanceMode() {
   }
   
   try {
-    const setting = await db('app_settings')
+    const setting = await queryWithRetry(async () => {
+      return await db('app_settings')
         .where('setting_key', 'general_maintenance_mode')
         .where('setting_type', 'general')
         .first();
+    });
     
     maintenanceMode = setting ? (setting.setting_value === 'true' || setting.setting_value === true) : false;
     lastCheck = now;
     
     return maintenanceMode;
   } catch (error) {
-    console.error('Error checking maintenance mode:', error);
+    console.error('Error checking maintenance mode after retries:', error.message);
-    return false;
+    // Return cached value or false if no cache
+    return maintenanceMode;
   }
 }
 
@@ -52,6 +85,7 @@ async function maintenanceMiddleware(req, res, next) {
     return next();
   }
   
+  try {
     const inMaintenance = await checkMaintenanceMode();
     
     if (inMaintenance && !isAdminRoute) {
@@ -61,6 +95,10 @@ async function maintenanceMiddleware(req, res, next) {
         maintenance: true
       });
     }
+  } catch (error) {
+    // If we can't check maintenance mode, allow the request to proceed
+    console.error('Failed to check maintenance mode, allowing request:', error.message);
+  }
   
   next();
 }