From c9c0de46bff668b072ccbf60534cd7628a177b07 Mon Sep 17 00:00:00 2001 From: paul Date: Wed, 16 Jul 2025 14:29:33 +0200 Subject: [PATCH] fix: improve email processor reliability and add diagnostic tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Enhanced email processor with comprehensive logging - Added error handling for database queries in email processing - Fixed duplicate export in module.exports - Added testEmailConnection() function for connectivity testing - Created diagnostic scripts: - check-email-processor.js: Full diagnostic with manual processing - run-email-processor.js: Standalone processor (can run once or continuously) - check-email-environment.js: Environment and configuration checker - Fixed missing translation: added 'common.unknown' in EN/DE locales This should help identify why emails aren't being processed on production. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- backend/scripts/check-email-environment.js | 131 +++++++++++++++++ backend/scripts/check-email-processor.js | 157 +++++++++++++++++++++ backend/scripts/run-email-processor.js | 111 +++++++++++++++ backend/src/services/emailProcessor.js | 57 ++++++-- frontend/src/i18n/locales/de.json | 3 +- frontend/src/i18n/locales/en.json | 3 +- 6 files changed, 450 insertions(+), 12 deletions(-) create mode 100644 backend/scripts/check-email-environment.js create mode 100644 backend/scripts/check-email-processor.js create mode 100644 backend/scripts/run-email-processor.js diff --git a/backend/scripts/check-email-environment.js b/backend/scripts/check-email-environment.js new file mode 100644 index 0000000..bfac0f0 --- /dev/null +++ b/backend/scripts/check-email-environment.js @@ -0,0 +1,131 @@ +const { db } = require('../src/database/db'); + +async function checkEmailEnvironment() { + console.log('=== Email Environment Check ===\n'); + + // 1. Check environment variables + console.log('1. Environment Variables:'); + const envVars = [ + 'SMTP_HOST', + 'SMTP_PORT', + 'SMTP_USER', + 'SMTP_PASS', + 'SMTP_FROM', + 'SMTP_SECURE', + 'EMAIL_PROCESSOR_ENABLED', + 'NODE_ENV' + ]; + + envVars.forEach(varName => { + const value = process.env[varName]; + if (varName.includes('PASS')) { + console.log(` ${varName}: ${value ? '***' : 'NOT SET'}`); + } else { + console.log(` ${varName}: ${value || 'NOT SET'}`); + } + }); + + // 2. Check database configuration + console.log('\n2. Database Email Configuration:'); + try { + const emailConfig = await db('email_configs').first(); + if (emailConfig) { + console.log(' Email configuration found in database:'); + console.log(` - SMTP Host: ${emailConfig.smtp_host}`); + console.log(` - SMTP Port: ${emailConfig.smtp_port}`); + console.log(` - SMTP User: ${emailConfig.smtp_user || 'NOT SET'}`); + console.log(` - SMTP Secure: ${emailConfig.smtp_secure}`); + console.log(` - From Address: ${emailConfig.smtp_from}`); + } else { + console.log(' ⚠️ No email configuration found in database!'); + console.log(' This will prevent the email processor from initializing.'); + } + } catch (error) { + console.log(` ❌ Error reading email configuration: ${error.message}`); + } + + // 3. Check if the email processor should be disabled + console.log('\n3. Email Processor Status:'); + const isDisabled = process.env.EMAIL_PROCESSOR_ENABLED === 'false'; + if (isDisabled) { + console.log(' ⚠️ Email processor is DISABLED via EMAIL_PROCESSOR_ENABLED=false'); + } else { + console.log(' ✅ Email processor is enabled (default)'); + } + + // 4. Check pending emails + console.log('\n4. Email Queue Status:'); + try { + const pending = await db('email_queue') + .where('status', 'pending') + .count('* as count') + .first(); + + const failed = await db('email_queue') + .where('status', 'failed') + .where('retry_count', '>=', 3) + .count('* as count') + .first(); + + const sent = await db('email_queue') + .where('status', 'sent') + .count('* as count') + .first(); + + console.log(` - Pending emails: ${pending.count}`); + console.log(` - Failed emails (max retries): ${failed.count}`); + console.log(` - Sent emails: ${sent.count}`); + } catch (error) { + console.log(` ❌ Error querying email queue: ${error.message}`); + } + + // 5. Test database connection + console.log('\n5. Database Connection:'); + try { + await db.raw('SELECT 1'); + console.log(' ✅ Database connection successful'); + } catch (error) { + console.log(` ❌ Database connection failed: ${error.message}`); + } + + // 6. Check for any recent errors + console.log('\n6. Recent Email Errors:'); + try { + const recentErrors = await db('email_queue') + .whereNotNull('error_message') + .orderBy('id', 'desc') + .limit(3) + .select('id', 'email_type', 'error_message', 'retry_count'); + + if (recentErrors.length > 0) { + recentErrors.forEach((email, index) => { + console.log(` ${index + 1}. Email ID ${email.id} (${email.email_type}):`); + console.log(` Retries: ${email.retry_count}`); + console.log(` Error: ${email.error_message}`); + }); + } else { + console.log(' No recent errors found'); + } + } catch (error) { + console.log(` ❌ Error querying recent errors: ${error.message}`); + } + + console.log('\n=== Environment check complete ==='); + console.log('\nRecommendations:'); + + const emailConfig = await db('email_configs').first().catch(() => null); + if (!emailConfig) { + console.log('❗ Configure email settings in the admin panel or add email_configs record'); + } + + if (!process.env.SMTP_HOST && !emailConfig) { + console.log('❗ Set SMTP environment variables or configure in database'); + } + + await db.destroy(); +} + +checkEmailEnvironment().catch(error => { + console.error('Fatal error:', error); + process.exit(1); +}); \ No newline at end of file diff --git a/backend/scripts/check-email-processor.js b/backend/scripts/check-email-processor.js new file mode 100644 index 0000000..0fe47ed --- /dev/null +++ b/backend/scripts/check-email-processor.js @@ -0,0 +1,157 @@ +const { db } = require('../src/database/db'); +const winston = require('winston'); + +// Create a simple console logger +const logger = winston.createLogger({ + format: winston.format.simple(), + transports: [new winston.transports.Console()] +}); + +async function checkEmailProcessor() { + try { + logger.info('=== Email Processor Diagnostic Check ===\n'); + + // 1. Check pending emails + logger.info('1. Checking pending emails in queue...'); + const pendingEmails = await db('email_queue') + .where('status', 'pending') + .where('retry_count', '<', 3) + .orderBy('created_at', 'asc'); + + logger.info(`Found ${pendingEmails.length} pending emails\n`); + + if (pendingEmails.length > 0) { + logger.info('Pending email details:'); + pendingEmails.forEach((email, index) => { + logger.info(`\nEmail ${index + 1}:`); + logger.info(` ID: ${email.id}`); + logger.info(` Type: ${email.email_type}`); + logger.info(` Recipient: ${email.recipient_email}`); + logger.info(` Event ID: ${email.event_id}`); + logger.info(` Status: ${email.status}`); + logger.info(` Retry Count: ${email.retry_count}`); + logger.info(` Scheduled At: ${email.scheduled_at}`); + logger.info(` Created At: ${email.created_at}`); + logger.info(` Error: ${email.error_message || 'None'}`); + + // Check if email_data needs parsing + logger.info(` Email Data Type: ${typeof email.email_data}`); + if (email.email_data) { + try { + const data = typeof email.email_data === 'string' + ? JSON.parse(email.email_data) + : email.email_data; + logger.info(` Email Data Keys: ${Object.keys(data).join(', ')}`); + } catch (e) { + logger.error(` Failed to parse email_data: ${e.message}`); + } + } + }); + } + + // 2. Check failed emails + logger.info('\n\n2. Checking failed emails...'); + const failedEmails = await db('email_queue') + .where('status', 'failed') + .orderBy('created_at', 'desc') + .limit(5); + + logger.info(`Found ${failedEmails.length} failed emails (showing last 5)\n`); + + if (failedEmails.length > 0) { + failedEmails.forEach((email, index) => { + logger.info(`\nFailed Email ${index + 1}:`); + logger.info(` ID: ${email.id}`); + logger.info(` Type: ${email.email_type}`); + logger.info(` Retry Count: ${email.retry_count}`); + logger.info(` Error: ${email.error_message || 'No error message'}`); + logger.info(` Last Attempt: ${email.sent_at || 'Never'}`); + }); + } + + // 3. Check if email processor should be running + logger.info('\n\n3. Checking email processor configuration...'); + + // Check environment variables + const emailConfig = { + SMTP_HOST: process.env.SMTP_HOST, + SMTP_PORT: process.env.SMTP_PORT, + SMTP_USER: process.env.SMTP_USER, + SMTP_FROM: process.env.SMTP_FROM, + SMTP_SECURE: process.env.SMTP_SECURE, + EMAIL_PROCESSOR_ENABLED: process.env.EMAIL_PROCESSOR_ENABLED || 'true' + }; + + logger.info('Email configuration:'); + Object.entries(emailConfig).forEach(([key, value]) => { + if (key === 'SMTP_USER') { + logger.info(` ${key}: ${value ? '***' : 'NOT SET'}`); + } else { + logger.info(` ${key}: ${value || 'NOT SET'}`); + } + }); + + // 4. Test email processor functionality + logger.info('\n\n4. Testing email processor functionality...'); + + // Import the email processor + const { processEmailQueue, testEmailConnection } = require('../src/services/emailProcessor'); + + // Test email connection + logger.info('Testing email connection...'); + try { + const connectionTest = await testEmailConnection(); + logger.info(`Email connection test: ${connectionTest ? 'SUCCESS' : 'FAILED'}`); + } catch (error) { + logger.error(`Email connection test failed: ${error.message}`); + } + + // Try to process queue once manually + if (pendingEmails.length > 0) { + logger.info('\n\n5. Attempting to process email queue manually...'); + try { + await processEmailQueue(); + logger.info('Manual queue processing completed'); + + // Check status after processing + const stillPending = await db('email_queue') + .where('status', 'pending') + .where('retry_count', '<', 3) + .count('* as count') + .first(); + + logger.info(`Emails still pending after processing: ${stillPending.count}`); + } catch (error) { + logger.error(`Error processing queue: ${error.message}`); + logger.error(`Stack trace: ${error.stack}`); + } + } + + // 5. Check for any recent successful emails + logger.info('\n\n6. Checking recent successful emails...'); + const recentSuccess = await db('email_queue') + .where('status', 'sent') + .orderBy('sent_at', 'desc') + .limit(3); + + if (recentSuccess.length > 0) { + logger.info(`Last ${recentSuccess.length} successful emails:`); + recentSuccess.forEach((email, index) => { + logger.info(` ${index + 1}. Type: ${email.email_type}, Sent: ${email.sent_at}`); + }); + } else { + logger.info('No successfully sent emails found'); + } + + logger.info('\n\n=== Diagnostic check complete ==='); + + } catch (error) { + logger.error('Error running diagnostic check:', error); + } finally { + await db.destroy(); + process.exit(0); + } +} + +// Run the check +checkEmailProcessor(); \ No newline at end of file diff --git a/backend/scripts/run-email-processor.js b/backend/scripts/run-email-processor.js new file mode 100644 index 0000000..e229a4e --- /dev/null +++ b/backend/scripts/run-email-processor.js @@ -0,0 +1,111 @@ +#!/usr/bin/env node + +const { db } = require('../src/database/db'); +const { + initializeTransporter, + processEmailQueue, + testEmailConnection +} = require('../src/services/emailProcessor'); +const winston = require('winston'); + +// Create a simple console logger +const logger = winston.createLogger({ + format: winston.format.simple(), + transports: [new winston.transports.Console()] +}); + +async function runEmailProcessor(runOnce = false) { + try { + logger.info('=== Starting Email Processor ===\n'); + + // Initialize transporter + logger.info('Initializing email transporter...'); + await initializeTransporter(); + + // Test connection + logger.info('Testing email connection...'); + const connectionOk = await testEmailConnection(); + + if (!connectionOk) { + logger.error('Email connection test failed! Check your SMTP configuration.'); + logger.info('\nRequired environment variables:'); + logger.info('- SMTP_HOST'); + logger.info('- SMTP_PORT'); + logger.info('- SMTP_USER'); + logger.info('- SMTP_PASS'); + logger.info('- SMTP_FROM'); + process.exit(1); + } + + logger.info('Email connection test successful!\n'); + + if (runOnce) { + // Process queue once + logger.info('Processing email queue once...'); + await processEmailQueue(); + logger.info('Email processing complete'); + + // Show final status + const pendingCount = await db('email_queue') + .where('status', 'pending') + .where('retry_count', '<', 3) + .count('* as count') + .first(); + + logger.info(`\nEmails still pending: ${pendingCount.count}`); + + await db.destroy(); + process.exit(0); + } else { + // Run continuously + logger.info('Starting continuous email processor...'); + logger.info('Processing emails every 60 seconds. Press Ctrl+C to stop.\n'); + + // Process immediately + await processEmailQueue(); + + // Then every minute + setInterval(async () => { + try { + await processEmailQueue(); + } catch (error) { + logger.error('Error processing email queue:', error); + } + }, 60000); + } + + } catch (error) { + logger.error('Fatal error:', error); + await db.destroy(); + process.exit(1); + } +} + +// Handle graceful shutdown +process.on('SIGINT', async () => { + logger.info('\n\nShutting down email processor...'); + await db.destroy(); + process.exit(0); +}); + +// Check command line arguments +const args = process.argv.slice(2); +const runOnce = args.includes('--once') || args.includes('-o'); + +if (args.includes('--help') || args.includes('-h')) { + console.log(` +Email Processor Runner + +Usage: node run-email-processor.js [options] + +Options: + --once, -o Process the email queue once and exit + --help, -h Show this help message + +By default, the processor runs continuously, checking for emails every 60 seconds. + `); + process.exit(0); +} + +// Run the processor +runEmailProcessor(runOnce); \ No newline at end of file diff --git a/backend/src/services/emailProcessor.js b/backend/src/services/emailProcessor.js index d9efd04..0f5b6a8 100644 --- a/backend/src/services/emailProcessor.js +++ b/backend/src/services/emailProcessor.js @@ -325,6 +325,8 @@ async function sendTemplateEmail(to, templateKey, variables) { // Process email queue async function processEmailQueue() { + logger.info('Email queue processor: Checking for pending emails...'); + try { // Try to initialize transporter if it's null (in case it failed at startup) if (!transporter) { @@ -336,13 +338,20 @@ async function processEmailQueue() { } } - const pendingEmails = await db('email_queue') - .where('status', 'pending') - .where('retry_count', '<', 3) - .orderBy('created_at', 'asc') - .limit(10); + let pendingEmails = []; + try { + pendingEmails = await db('email_queue') + .where('status', 'pending') + .where('retry_count', '<', 3) + .orderBy('created_at', 'asc') + .limit(10); + } catch (dbError) { + logger.error('Failed to query email queue:', dbError); + return; + } if (pendingEmails.length === 0) { + logger.info('Email queue processor: No pending emails found'); return; } @@ -418,17 +427,45 @@ async function queueEmail(eventId, recipientEmail, emailType, emailData) { } } +// Test email connection +async function testEmailConnection() { + try { + if (!transporter) { + await initializeTransporter(); + } + if (!transporter) { + return false; + } + await transporter.verify(); + return true; + } catch (error) { + logger.error('Email connection test failed:', error); + return false; + } +} + // Start email queue processor let emailQueueInterval = null; function startEmailQueueProcessor() { + logger.info('Email queue processor: Attempting to start...'); + if (!emailQueueInterval) { // Process immediately on start - processEmailQueue(); + processEmailQueue().catch(err => { + logger.error('Email queue processor: Initial processing failed:', err); + }); // Then process every minute - emailQueueInterval = setInterval(processEmailQueue, 60000); - logger.info('Email queue processor started'); + emailQueueInterval = setInterval(() => { + processEmailQueue().catch(err => { + logger.error('Email queue processor: Periodic processing failed:', err); + }); + }, 60000); + + logger.info('Email queue processor started successfully'); + } else { + logger.info('Email queue processor: Already running'); } } @@ -452,6 +489,6 @@ module.exports = { sendTemplateEmail, processEmailQueue, queueEmail, - startEmailQueueProcessor, - stopEmailQueueProcessor + stopEmailQueueProcessor, + testEmailConnection }; \ No newline at end of file diff --git a/frontend/src/i18n/locales/de.json b/frontend/src/i18n/locales/de.json index 679976d..2097717 100644 --- a/frontend/src/i18n/locales/de.json +++ b/frontend/src/i18n/locales/de.json @@ -34,7 +34,8 @@ "upload": "Hochladen", "days": "Tage", "customize": "Anpassen", - "hide": "Ausblenden" + "hide": "Ausblenden", + "unknown": "Unbekannt" }, "upload": { "photoCategory": "Fotokategorie", diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index e7a3eca..b0443bc 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -34,7 +34,8 @@ "upload": "Upload", "days": "days", "customize": "Customize", - "hide": "Hide" + "hide": "Hide", + "unknown": "Unknown" }, "upload": { "photoCategory": "Photo Category",