feat(admin): System health page surfacing stuck/failed emails

New /admin/system-health page (sidebar entry, settings.view) that lists emails the queue gave up on (status='failed' or pending+retry>=3) with retry (re-queue) and dismiss (delete) actions. Backend adds /failures, /failures/email/:id/retry and DELETE on adminSystemHealth. First source is email failures (the original trigger — quote_sent template errors left invoices unsent for 14h with no signal); more sources can be added.
2026-06-03 13:56:08 +02:00
parent a6e6ef7b83
commit 5d7b545bf7
8 changed files with 275 additions and 0 deletions
@@ -23,6 +23,7 @@ const { requirePermission } = require('../middleware/permissions');
 const { handleAsync, validateRequest, successResponse } = require('../utils/routeHelpers');
 const { verifyDocumentArtefacts } = require('../services/backupIntegrityService');
 const { getCoverageReport } = require('../services/backupCoverageService');
+const { db } = require('../database/db');

 const router = express.Router();

@@ -84,4 +85,81 @@ router.get(
  }),
 );

+/**
+ * GET /api/admin/system-health/failures
+ *
+ * Surfaces background failures that would otherwise go unnoticed. v1
+ * covers stuck/failed outbound emails: rows the queue processor has
+ * given up on (status='failed') or exhausted its retries on
+ * (status='pending' AND retry_count >= 3 — the processor only picks up
+ * retry_count < 3). Trigger: a 14h window where 'quote_sent' template
+ * errors left invoices unsent with no admin-visible signal.
+ */
+router.get(
+  '/failures',
+  requirePermission('settings.view'),
+  handleAsync(async (req, res) => {
+    const stuckEmails = await db('email_queue')
+      .where(function () {
+        this.where('status', 'failed')
+          .orWhere(function () {
+            this.where('status', 'pending').andWhere('retry_count', '>=', 3);
+          });
+      })
+      .orderBy('created_at', 'desc')
+      .limit(200)
+      .select('id', 'recipient_email', 'email_type', 'status', 'retry_count', 'error_message', 'created_at');
+
+    return successResponse(res, {
+      stuckEmails: stuckEmails.map((r) => ({
+        id: r.id,
+        recipientEmail: r.recipient_email,
+        emailType: r.email_type,
+        status: r.status,
+        retryCount: r.retry_count,
+        errorMessage: r.error_message,
+        createdAt: r.created_at,
+      })),
+      counts: { stuckEmails: stuckEmails.length },
+    });
+  }),
+);
+
+/**
+ * POST /failures/email/:id/retry — re-queue a stuck email (status back to
+ * pending, retry_count reset, error cleared, scheduled_at cleared so the
+ * 60s processor picks it up on its next pass).
+ */
+router.post(
+  '/failures/email/:id/retry',
+  requirePermission('settings.edit'),
+  handleAsync(async (req, res) => {
+    const id = parseInt(req.params.id, 10);
+    if (!Number.isFinite(id) || id <= 0) return res.status(400).json({ error: 'Invalid id' });
+    const updated = await db('email_queue').where({ id }).update({
+      status: 'pending',
+      retry_count: 0,
+      error_message: null,
+      scheduled_at: null,
+    });
+    if (!updated) return res.status(404).json({ error: 'Email not found' });
+    return successResponse(res, { retried: true });
+  }),
+);
+
+/**
+ * DELETE /failures/email/:id — dismiss a stuck email (remove the row so
+ * it stops surfacing). Use when the failure is understood + won't be sent.
+ */
+router.delete(
+  '/failures/email/:id',
+  requirePermission('settings.edit'),
+  handleAsync(async (req, res) => {
+    const id = parseInt(req.params.id, 10);
+    if (!Number.isFinite(id) || id <= 0) return res.status(400).json({ error: 'Invalid id' });
+    await db('email_queue').where({ id }).del();
+    return successResponse(res, { dismissed: true });
+  }),
+);
+
 module.exports = router;
@@ -21,6 +21,7 @@ import {
  ArchivesPage,
  AnalyticsPage,
  SettingsPage,
+  SystemHealthPage,
  UserManagementPage,
  CustomerManagementPage,
  CustomerDetailPage,
@@ -267,6 +268,7 @@ function App() {
                      <Route path="customers/:id" element={<RedirectCustomerDetail />} />

                      <Route path="settings" element={<SettingsPage />} />
+                      <Route path="system-health" element={<SystemHealthPage />} />
                      <Route path="webhooks/:id/deliveries" element={<WebhookDeliveriesPage />} />

                      {/* Old top-level routes — these surfaces now live as
@@ -6,6 +6,7 @@ import {
  Archive,
  BarChart3,
  Settings,
+  Activity,
  X,
  Users,
  Briefcase,
@@ -62,6 +63,7 @@ const navigation: NavItem[] = [
  { nameKey: 'navigation.archives',  href: '/admin/archives',  icon: Archive,         permission: 'archives.view' },
  { nameKey: 'admin.analytics',      href: '/admin/analytics', icon: BarChart3,       permission: 'analytics.view', featureFlag: 'analytics' },
  { nameKey: 'navigation.settings',  href: '/admin/settings',  icon: Settings,        permission: 'settings.view' },
+  { nameKey: 'navigation.systemHealth', href: '/admin/system-health', icon: Activity,  permission: 'settings.view' },
  { nameKey: 'navigation.users',     href: '/admin/users',     icon: Users,           permission: 'users.view',     featureFlag: 'userManagement' },
  // Clients section (#354 follow-up) — admin-side surface for the
  // CRM-area sub-features. Today this entry leads to /admin/clients
@@ -73,6 +73,26 @@
      "message": "Sind Sie sicher, dass Sie die Einladung für {{email}} abbrechen möchten?"
    }
  },
+  "systemHealth": {
+    "title": "Systemzustand",
+    "subtitle": "Hintergrundfehler, die Aufmerksamkeit erfordern.",
+    "retry": "Erneut versuchen",
+    "dismiss": "Verwerfen",
+    "retriedToast": "E-Mail erneut eingereiht.",
+    "dismissedToast": "Verworfen.",
+    "stuckEmails": {
+      "title": "Hängende / fehlgeschlagene E-Mails",
+      "empty": "Keine hängenden oder fehlgeschlagenen E-Mails — alles in Ordnung.",
+      "noError": "Versuche aufgebraucht",
+      "col": {
+        "recipient": "Empfänger",
+        "type": "Typ",
+        "error": "Fehler",
+        "queued": "Eingereiht",
+        "actions": "Aktionen"
+      }
+    }
+  },
  "common": {
    "loading": "Wird geladen...",
    "error": "Fehler",
@@ -166,6 +186,7 @@
    "dashboard": "Dashboard",
    "events": "Veranstaltungen",
    "settings": "Einstellungen",
+    "systemHealth": "Systemzustand",
    "archives": "Archive",
    "emailSettings": "E-Mail-Einstellungen",
    "branding": "Markenidentität",
@@ -73,6 +73,26 @@
      "message": "Are you sure you want to cancel the invitation for {{email}}?"
    }
  },
+  "systemHealth": {
+    "title": "System health",
+    "subtitle": "Background failures that need attention.",
+    "retry": "Retry",
+    "dismiss": "Dismiss",
+    "retriedToast": "Email re-queued.",
+    "dismissedToast": "Dismissed.",
+    "stuckEmails": {
+      "title": "Stuck / failed emails",
+      "empty": "No stuck or failed emails — all clear.",
+      "noError": "retries exhausted",
+      "col": {
+        "recipient": "Recipient",
+        "type": "Type",
+        "error": "Error",
+        "queued": "Queued",
+        "actions": "Actions"
+      }
+    }
+  },
  "common": {
    "loading": "Loading...",
    "error": "Error",
@@ -167,6 +187,7 @@
    "events": "Events",
    "archives": "Archives",
    "settings": "Settings",
+    "systemHealth": "System health",
    "eventTypes": "Event Types",
    "branding": "Branding",
    "emailSettings": "Email Settings",
@@ -0,0 +1,115 @@
+/**
+ * Admin → System health. Aggregates background failures that would
+ * otherwise go unnoticed. v1: stuck/failed outbound emails (the queue
+ * processor gave up or exhausted retries), with retry + dismiss.
+ */
+import React from 'react';
+import { useTranslation } from 'react-i18next';
+import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
+import { toast } from 'react-toastify';
+import { AlertCircle, RefreshCw, Trash2, CheckCircle } from 'lucide-react';
+import { Button, Card, Loading } from '../../components/common';
+import { useLocalizedDate } from '../../hooks/useLocalizedDate';
+import { systemHealthService } from '../../services/systemHealth.service';
+
+export const SystemHealthPage: React.FC = () => {
+  const { t } = useTranslation();
+  const { formatDateTime: fmtDateTime } = useLocalizedDate();
+  const qc = useQueryClient();
+
+  const { data, isLoading } = useQuery({
+    queryKey: ['system-health-failures'],
+    queryFn: () => systemHealthService.getFailures(),
+  });
+
+  const invalidate = () => qc.invalidateQueries({ queryKey: ['system-health-failures'] });
+
+  const retryMutation = useMutation({
+    mutationFn: (id: number) => systemHealthService.retryEmail(id),
+    onSuccess: () => { toast.success(t('systemHealth.retriedToast', 'Email re-queued.')); invalidate(); },
+    onError: () => toast.error(t('toast.saveError')),
+  });
+  const dismissMutation = useMutation({
+    mutationFn: (id: number) => systemHealthService.dismissEmail(id),
+    onSuccess: () => { toast.success(t('systemHealth.dismissedToast', 'Dismissed.')); invalidate(); },
+    onError: () => toast.error(t('toast.saveError')),
+  });
+
+  const stuckEmails = data?.stuckEmails ?? [];
+
+  return (
+    <div className="container py-6">
+      <div className="mb-6">
+        <h1 className="text-2xl font-bold text-theme">{t('systemHealth.title', 'System health')}</h1>
+        <p className="text-sm text-muted-theme mt-1">
+          {t('systemHealth.subtitle', 'Background failures that need attention.')}
+        </p>
+      </div>
+
+      <Card padding="lg">
+        <div className="flex items-center gap-2 mb-3">
+          <AlertCircle className="w-5 h-5 text-amber-500" />
+          <h2 className="text-lg font-semibold text-neutral-900 dark:text-neutral-100">
+            {t('systemHealth.stuckEmails.title', 'Stuck / failed emails')}
+          </h2>
+          {!isLoading && (
+            <span className="ml-1 text-sm text-muted-theme">({stuckEmails.length})</span>
+          )}
+        </div>
+
+        {isLoading ? <Loading /> : stuckEmails.length === 0 ? (
+          <div className="flex items-center gap-2 text-sm text-green-700 dark:text-green-400 py-6">
+            <CheckCircle className="w-5 h-5" />
+            {t('systemHealth.stuckEmails.empty', 'No stuck or failed emails — all clear.')}
+          </div>
+        ) : (
+          <div className="rounded-lg border border-neutral-200 dark:border-neutral-700 overflow-hidden">
+            <div className="overflow-x-auto">
+              <table className="w-full text-sm">
+                <thead className="bg-neutral-50 dark:bg-neutral-800 text-neutral-700 dark:text-neutral-300">
+                  <tr>
+                    <th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.recipient', 'Recipient')}</th>
+                    <th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.type', 'Type')}</th>
+                    <th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.error', 'Error')}</th>
+                    <th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.queued', 'Queued')}</th>
+                    <th className="px-3 py-2 text-right">{t('systemHealth.stuckEmails.col.actions', 'Actions')}</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  {stuckEmails.map((m) => (
+                    <tr key={m.id} className="border-t border-neutral-200 dark:border-neutral-700 align-top">
+                      <td className="px-3 py-2 break-all">{m.recipientEmail}</td>
+                      <td className="px-3 py-2 font-mono text-xs">{m.emailType}</td>
+                      <td className="px-3 py-2 max-w-xs">
+                        <span className="text-xs text-red-700 dark:text-red-400 break-words">
+                          {m.errorMessage || t('systemHealth.stuckEmails.noError', 'retries exhausted')}
+                        </span>
+                      </td>
+                      <td className="px-3 py-2 whitespace-nowrap">{m.createdAt ? fmtDateTime(m.createdAt) : '—'}</td>
+                      <td className="px-3 py-2">
+                        <div className="flex items-center justify-end gap-1">
+                          <Button variant="outline" size="sm"
+                            isLoading={retryMutation.isPending && retryMutation.variables === m.id}
+                            onClick={() => retryMutation.mutate(m.id)}
+                            leftIcon={<RefreshCw className="w-3.5 h-3.5" />}>
+                            {t('systemHealth.retry', 'Retry')}
+                          </Button>
+                          <button type="button"
+                            aria-label={t('systemHealth.dismiss', 'Dismiss') as string}
+                            onClick={() => dismissMutation.mutate(m.id)}
+                            className="p-1.5 text-neutral-400 hover:text-red-600">
+                            <Trash2 className="w-4 h-4" />
+                          </button>
+                        </div>
+                      </td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            </div>
+          </div>
+        )}
+      </Card>
+    </div>
+  );
+};
@@ -8,6 +8,7 @@ export { ArchivesPage } from './ArchivesPage';
 export { AnalyticsPage } from './AnalyticsPage';
 export { BrandingPage } from './BrandingPage';
 export { SettingsPage } from './SettingsPage';
+export { SystemHealthPage } from './SystemHealthPage';
 export { CMSPage } from './CMSPage';
 export { BackupManagement } from './BackupManagement';
 export { EventFeedbackPage } from './EventFeedbackPage';
@@ -0,0 +1,35 @@
+/**
+ * Admin → System health. Surfaces background failures (v1: stuck/failed
+ * outbound emails) so they don't sit unnoticed, with retry/dismiss.
+ */
+import { api } from '../config/api';
+
+export interface StuckEmail {
+  id: number;
+  recipientEmail: string;
+  emailType: string;
+  status: 'pending' | 'failed';
+  retryCount: number;
+  errorMessage: string | null;
+  createdAt: string;
+}
+
+export interface SystemHealthFailures {
+  stuckEmails: StuckEmail[];
+  counts: { stuckEmails: number };
+}
+
+export const systemHealthService = {
+  async getFailures(): Promise<SystemHealthFailures> {
+    const { data } = await api.get('/admin/system-health/failures');
+    return data.data || data;
+  },
+
+  async retryEmail(id: number): Promise<void> {
+    await api.post(`/admin/system-health/failures/email/${id}/retry`);
+  },
+
+  async dismissEmail(id: number): Promise<void> {
+    await api.delete(`/admin/system-health/failures/email/${id}`);
+  },
+};