From 83933baeecdc7e0272f2da0ff79fb1fb33bea114 Mon Sep 17 00:00:00 2001 From: Luca <102960244+Luca-Timo@users.noreply.github.com> Date: Wed, 27 May 2026 15:18:29 +0200 Subject: [PATCH] fix(crm): self-heal missing CRM email templates at boot + recover queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CRM template seeders (crmEmailTemplates / contractEmailTemplates / eventReminderTemplates) were idempotent and ready, but only contractEmailTemplates was actually called (lazily, by contractService sends). crmEmailTemplates had no caller anywhere — every install that didn't pre-exist its templates failed every quote_sent / invoice_sent / storno_issued / invoice_reminder_* send with "Email template '' not found". The queue processor retries 3 times then leaves the row in status='pending', retry_count=3, silently dead with no admin surface (see project_crm_backlog for the eventual System Health page). Fix: wire all three seeders into server.js startServer() right before startEmailQueueProcessor. The new _emailTemplateBoot.js orchestrates all three and then, for any template_key it just inserted, resets retry_count on stuck email_queue rows of that email_type so the queue processor's next tick picks them back up. Recovery is targeted: unrelated retry-exhausted rows (e.g. SMTP-timeout failures) are not touched. Integration test boots a fresh CRM DB, pre-seeds a stuck quote_sent row plus an unrelated stuck row, runs the boot helper, and asserts: templates landed, stuck quote_sent row was reset, unrelated row was left alone. Already-deployed installs heal automatically on the next backend restart after this lands. --- .../integration/emailTemplateBoot.test.js | 98 +++++++++++++++++++ backend/server.js | 10 ++ backend/src/services/_emailTemplateBoot.js | 97 ++++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 backend/__tests__/integration/emailTemplateBoot.test.js create mode 100644 backend/src/services/_emailTemplateBoot.js diff --git a/backend/__tests__/integration/emailTemplateBoot.test.js b/backend/__tests__/integration/emailTemplateBoot.test.js new file mode 100644 index 00000000..27265b6a --- /dev/null +++ b/backend/__tests__/integration/emailTemplateBoot.test.js @@ -0,0 +1,98 @@ +/** + * Boot-time email-template self-heal: + * 1. Seeds the CRM / contract / event-reminder templates on an + * install that's never had them before. + * 2. Recovers email_queue rows that previously exhausted their + * retries because their template was missing. + * + * The failure that triggered this fix (2026-05-27) had Ralf's beta + * box failing every `quote_sent` / `invoice_sent` send for ~14h + * because crmEmailTemplates.ensureCrmEmailTemplatesSeeded was + * defined but never called. After 3 retries the rows sat in + * status='pending' forever; nothing in the admin UI signalled the + * problem. Both halves of that regression are covered here. + */ + +const { bootCrmDb } = require('./helpers/crmDb'); + +describe('email template self-heal at boot', () => { + let db; + let cleanup; + + beforeAll(async () => { + ({ db, cleanup } = await bootCrmDb()); + }, 120000); + + afterAll(async () => { + if (cleanup) await cleanup(); + }); + + it('seeds crm/contract/event-reminder templates and recovers stuck queue rows', async () => { + // Sanity: a fresh CRM-migrated DB does NOT carry CRM templates — + // 107_crm_consolidated documents the deliberate split (templates + // are self-healed at runtime, not inserted by the migration). + const before = await db('email_templates') + .whereIn('template_key', ['quote_sent', 'invoice_sent', 'storno_issued']) + .pluck('template_key'); + expect(before).toEqual([]); + + // Seed a stuck queue row that mirrors what we found on Ralf's box: + // quote_sent send attempted 3 times, each time failed because the + // template didn't exist, queue processor gave up. + const queueRowIds = await db('email_queue').insert({ + recipient_email: 'customer@example.com', + email_type: 'quote_sent', + email_data: JSON.stringify({ quote_number: 'Q-2026-0001' }), + status: 'pending', + retry_count: 3, + error_message: "Email template 'quote_sent' not found", + created_at: new Date(), + }).returning('id'); + const queueRowId = typeof queueRowIds[0] === 'object' ? queueRowIds[0].id : queueRowIds[0]; + + // Also seed an UNRELATED stuck row (different template, NOT one + // we're going to insert) to confirm the recovery is targeted — + // it must not blanket-reset every retry-exhausted row. + const unrelatedIds = await db('email_queue').insert({ + recipient_email: 'someone@example.com', + email_type: 'some_other_template', + email_data: JSON.stringify({}), + status: 'pending', + retry_count: 3, + error_message: 'SMTP timeout', + created_at: new Date(), + }).returning('id'); + const unrelatedId = typeof unrelatedIds[0] === 'object' ? unrelatedIds[0].id : unrelatedIds[0]; + + // The seeders use module-level caches (`_seeded = true`). When + // jest runs this test in isolation that cache starts fresh; in + // the full suite no other test currently calls these seeders, so + // the first call here also runs the real work. Reset the cache + // defensively in case a future test changes that. + jest.resetModules(); + const { seedEmailTemplatesAndRecoverQueue } = require('../../src/services/_emailTemplateBoot'); + + const result = await seedEmailTemplatesAndRecoverQueue(db, null); + + // Templates landed. + expect(result.seeded).toEqual(expect.arrayContaining([ + 'quote_sent', 'invoice_sent', 'storno_issued', + ])); + const after = await db('email_templates') + .whereIn('template_key', ['quote_sent', 'invoice_sent', 'storno_issued']) + .pluck('template_key'); + expect(after.sort()).toEqual(['invoice_sent', 'quote_sent', 'storno_issued']); + + // Stuck quote_sent row was recovered. + expect(result.recovered).toBeGreaterThanOrEqual(1); + const recoveredRow = await db('email_queue').where({ id: queueRowId }).first(); + expect(recoveredRow.retry_count).toBe(0); + expect(recoveredRow.error_message).toBeNull(); + expect(recoveredRow.status).toBe('pending'); // ready for the next tick + + // Unrelated stuck row was NOT touched. + const unrelatedRow = await db('email_queue').where({ id: unrelatedId }).first(); + expect(unrelatedRow.retry_count).toBe(3); + expect(unrelatedRow.error_message).toBe('SMTP timeout'); + }); +}); diff --git a/backend/server.js b/backend/server.js index 8fae56a5..e7d57663 100644 --- a/backend/server.js +++ b/backend/server.js @@ -777,6 +777,16 @@ async function startServer() { // Initialize email transporter and start queue processor await initializeTransporter(); + // Seed CRM / contract / event-reminder email templates and recover + // any queue rows that exhausted retries because their template + // didn't exist yet. Runs once per boot via module-level caches in + // each seeder. See _emailTemplateBoot.js for the full rationale. + try { + const { seedEmailTemplatesAndRecoverQueue } = require('./src/services/_emailTemplateBoot'); + await seedEmailTemplatesAndRecoverQueue(db, logger); + } catch (err) { + logger.warn('Email template self-heal failed at boot:', err.message); + } startEmailQueueProcessor(); // Start webhook delivery worker (#327) diff --git a/backend/src/services/_emailTemplateBoot.js b/backend/src/services/_emailTemplateBoot.js new file mode 100644 index 00000000..a12b3f74 --- /dev/null +++ b/backend/src/services/_emailTemplateBoot.js @@ -0,0 +1,97 @@ +/** + * Boot-time wiring for the three self-heal seeders that own the + * CRM-era email templates (quotes / invoices / Storno / payment + * reminders / contract send + signed / event reminders). + * + * **Why this lives outside the individual services** + * + * The seeders themselves (`crmEmailTemplates.js`, + * `contractEmailTemplates.js`, `eventReminderTemplates.js`) are + * idempotent and module-cached, but they were never called at boot. + * contractEmailTemplates is called lazily by every contractService + * send; eventReminderTemplates by the admin email-templates list + * route. crmEmailTemplates was orphaned — no caller anywhere — so + * every install that didn't pre-exist its templates failed every + * quote_sent / invoice_sent / storno_issued send with + * `Email template '' not found`. The queue processor retries 3 + * times then leaves the row in `status='pending', retry_count=3`, + * silently dead with no admin surface — exactly the failure flagged + * in [[feedback_observable_failure_state]] and [[feedback_self_heal_pattern]]. + * + * Wiring all three into the boot path fixes new installs at first + * start AND retroactively fixes already-deployed installs whose + * queue is full of retry-exhausted rows: after we seed the missing + * template we reset retry_count on rows whose `email_type` matches + * a key we just inserted, so the queue processor's next tick picks + * them back up. + * + * Safe to call multiple times — each underlying seeder short- + * circuits after its first successful pass via a module-level flag. + */ + +const { ensureCrmEmailTemplatesSeeded } = require('./crmEmailTemplates'); +const { ensureContractEmailTemplatesSeeded } = require('./contractEmailTemplates'); +const { ensureEventReminderTemplatesSeeded } = require('./eventReminderTemplates'); + +/** + * Run all three template seeders, then recover any email_queue rows + * that exhausted their retries because the template they needed didn't + * exist yet. + * + * @param {object} db knex instance + * @param {object} logger app logger (must expose .info / .warn) + * @returns {Promise<{ seeded: string[], recovered: number }>} + * `seeded` — flat list of template_keys newly inserted across all + * three seeders. + * `recovered` — count of email_queue rows whose retry_count was + * reset to 0 because their template now exists. + */ +async function seedEmailTemplatesAndRecoverQueue(db, logger) { + const log = logger || { info: () => {}, warn: () => {} }; + const seeded = []; + + for (const seedFn of [ + ensureCrmEmailTemplatesSeeded, + ensureContractEmailTemplatesSeeded, + ensureEventReminderTemplatesSeeded, + ]) { + try { + const inserted = await seedFn(db, log); + if (Array.isArray(inserted) && inserted.length > 0) { + seeded.push(...inserted); + } + } catch (err) { + // Boot continues. A missing seed is annoying but not fatal — + // the lazy callers (where they exist) will retry; admin can + // re-trigger via the email-templates page. We just log loudly. + log.warn(`Email template self-heal failed for ${seedFn.name}: ${err.message}`); + } + } + + if (seeded.length === 0) return { seeded, recovered: 0 }; + + // Recover stuck queue rows. The queue processor caps retries at 3 + // (emailProcessor.processEmailQueue); rows past that are skipped + // forever. For every template we just inserted, find any pending + // rows of that email_type whose retries were exhausted and reset + // them so the processor picks them up on its next tick. + let recovered = 0; + if (await db.schema.hasTable('email_queue')) { + try { + recovered = await db('email_queue') + .where('status', 'pending') + .where('retry_count', '>=', 3) + .whereIn('email_type', seeded) + .update({ retry_count: 0, error_message: null }); + if (recovered > 0) { + log.info(`Recovered ${recovered} email_queue row(s) after self-healing templates: ${seeded.join(', ')}`); + } + } catch (err) { + log.warn(`email_queue recovery skipped after self-heal: ${err.message}`); + } + } + + return { seeded, recovered }; +} + +module.exports = { seedEmailTemplatesAndRecoverQueue };