From 8588133a4e35774e46f7c605638758e5b2a4a9e2 Mon Sep 17 00:00:00 2001 From: paul Date: Sun, 20 Jul 2025 20:51:44 +0200 Subject: [PATCH] fix: critical database connection pool exhaustion issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Disabled duplicate email service (emailService.js) that was creating redundant connections - Increased connection pool size from 10 to 25 for production environment - Extended session timeout cache from 5 to 30 minutes to reduce DB queries - Added connection retry logic with exponential backoff for transient failures - Fixed password validation to use retry wrapper and correct setting key - Updated public settings and gallery middleware to handle connection failures gracefully These changes address the "Connection terminated unexpectedly" errors in production by: 1. Reducing unnecessary database connections 2. Increasing available connection pool capacity 3. Implementing automatic retry for transient connection failures 4. Caching frequently accessed data for longer periods 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- backend/knexfile.js | 8 +++---- backend/src/database/db.js | 29 +++++++++++++++++++++++- backend/src/middleware/gallery.js | 18 ++++++++------- backend/src/middleware/sessionTimeout.js | 2 +- backend/src/routes/publicSettings.js | 16 +++++++------ backend/src/services/emailService.js | 3 ++- backend/src/utils/passwordValidation.js | 15 +++++++----- 7 files changed, 63 insertions(+), 28 deletions(-) diff --git a/backend/knexfile.js b/backend/knexfile.js index 598d4fb..7264510 100644 --- a/backend/knexfile.js +++ b/backend/knexfile.js @@ -40,10 +40,10 @@ const config = { keepAliveInitialDelayMillis: 0 }, pool: { - min: 2, - max: 10, - acquireTimeoutMillis: 30000, - createTimeoutMillis: 30000, + min: 5, + max: 25, + acquireTimeoutMillis: 60000, + createTimeoutMillis: 60000, idleTimeoutMillis: 30000, reapIntervalMillis: 1000, createRetryIntervalMillis: 200, diff --git a/backend/src/database/db.js b/backend/src/database/db.js index 5eacda8..0e1f5bf 100644 --- a/backend/src/database/db.js +++ b/backend/src/database/db.js @@ -4,6 +4,33 @@ const knexConfig = require('../../knexfile'); // Create database connection with built-in retry logic const db = knex(knexConfig); +// Connection retry configuration +const MAX_RETRIES = 3; +const RETRY_DELAY = 1000; + +// Wrapper function to handle connection retries +async function withRetry(queryFn, retries = MAX_RETRIES) { + for (let i = 0; i < retries; i++) { + try { + return await queryFn(); + } catch (error) { + const isConnectionError = error.message && ( + error.message.includes('Connection terminated unexpectedly') || + error.message.includes('Connection ended unexpectedly') || + error.message.includes('ECONNREFUSED') || + error.message.includes('ETIMEDOUT') + ); + + if (isConnectionError && i < retries - 1) { + console.log(`Database connection error, retrying in ${RETRY_DELAY}ms... (attempt ${i + 1}/${retries})`); + await new Promise(resolve => setTimeout(resolve, RETRY_DELAY * (i + 1))); + continue; + } + throw error; + } + } +} + async function initializeDatabase() { // Events table const hasEventsTable = await db.schema.hasTable('events'); @@ -225,4 +252,4 @@ async function logActivity(activityType, metadata = {}, eventId = null, actor = } } -module.exports = { db, initializeDatabase, logActivity }; \ No newline at end of file +module.exports = { db, initializeDatabase, logActivity, withRetry }; \ No newline at end of file diff --git a/backend/src/middleware/gallery.js b/backend/src/middleware/gallery.js index 1852629..42cd7d4 100644 --- a/backend/src/middleware/gallery.js +++ b/backend/src/middleware/gallery.js @@ -1,5 +1,5 @@ const jwt = require('jsonwebtoken'); -const { db } = require('../database/db'); +const { db, withRetry } = require('../database/db'); const { formatBoolean } = require('../utils/dbCompat'); // Middleware to verify gallery access @@ -11,13 +11,15 @@ async function verifyGalleryAccess(req, res, next) { } const decoded = jwt.verify(token, process.env.JWT_SECRET); - const event = await db('events') - .where({ - id: decoded.eventId, - is_active: formatBoolean(true), - is_archived: formatBoolean(false) - }) - .first(); + const event = await withRetry(async () => { + return await db('events') + .where({ + id: decoded.eventId, + is_active: formatBoolean(true), + is_archived: formatBoolean(false) + }) + .first(); + }); if (!event) { return res.status(404).json({ error: 'Gallery not found or expired' }); diff --git a/backend/src/middleware/sessionTimeout.js b/backend/src/middleware/sessionTimeout.js index a3966ec..def5f03 100644 --- a/backend/src/middleware/sessionTimeout.js +++ b/backend/src/middleware/sessionTimeout.js @@ -10,7 +10,7 @@ const DEFAULT_SESSION_TIMEOUT = 60 * 60 * 1000; // Cache for session timeout setting let cachedTimeout = null; let cacheExpiry = 0; -const CACHE_DURATION = 5 * 60 * 1000; // 5 minutes +const CACHE_DURATION = 30 * 60 * 1000; // 30 minutes - reduced DB queries // Clean up expired sessions every 5 minutes setInterval(() => { diff --git a/backend/src/routes/publicSettings.js b/backend/src/routes/publicSettings.js index a002b2d..1a8c5ba 100644 --- a/backend/src/routes/publicSettings.js +++ b/backend/src/routes/publicSettings.js @@ -1,5 +1,5 @@ const express = require('express'); -const { db } = require('../database/db'); +const { db, withRetry } = require('../database/db'); const router = express.Router(); // Get public settings (branding and theme) @@ -7,12 +7,14 @@ router.get('/', async (req, res) => { try { // Fetch branding, theme, general, and security settings // Note: We include analytics in the query but it might not exist yet - const settings = await db('app_settings') - .where(function() { - this.whereIn('setting_type', ['branding', 'theme', 'general', 'security', 'analytics']) - .orWhere('setting_key', 'like', 'analytics_%'); - }) - .select('setting_key', 'setting_value'); + const settings = await withRetry(async () => { + return await db('app_settings') + .where(function() { + this.whereIn('setting_type', ['branding', 'theme', 'general', 'security', 'analytics']) + .orWhere('setting_key', 'like', 'analytics_%'); + }) + .select('setting_key', 'setting_value'); + }); // Convert to object format const settingsObject = {}; diff --git a/backend/src/services/emailService.js b/backend/src/services/emailService.js index 7ec83b5..2368622 100644 --- a/backend/src/services/emailService.js +++ b/backend/src/services/emailService.js @@ -60,6 +60,7 @@ async function processEmailQueue() { } // Start email queue processor -setInterval(processEmailQueue, 60000); // Process every minute +// DISABLED: Using emailProcessor.js instead to prevent duplicate connections +// setInterval(processEmailQueue, 60000); // Process every minute module.exports = { sendEmail, processEmailQueue }; diff --git a/backend/src/utils/passwordValidation.js b/backend/src/utils/passwordValidation.js index 1565477..64b1bab 100644 --- a/backend/src/utils/passwordValidation.js +++ b/backend/src/utils/passwordValidation.js @@ -119,11 +119,14 @@ function validatePassword(password, options = {}) { */ async function getPasswordComplexitySettings() { try { - const db = require('../db'); - const settings = await db('app_settings') - .where('setting_key', 'password_complexity') - .where('setting_type', 'security') - .first(); + const { db, withRetry } = require('../database/db'); + + // Use retry wrapper to handle connection failures + const settings = await withRetry(async () => { + return await db('app_settings') + .where('setting_key', 'security_password_complexity_level') + .first(); + }); if (!settings || !settings.setting_value) { return 'moderate'; // Default @@ -136,7 +139,7 @@ async function getPasswordComplexitySettings() { return value; } catch (error) { logger.error('Failed to get password complexity settings:', error); - return 'moderate'; // Default on error + return 'moderate'; // Default on error - ensures app continues working } }