feat(admin): System health page surfacing stuck/failed emails
New /admin/system-health page (sidebar entry, settings.view) that lists emails the queue gave up on (status='failed' or pending+retry>=3) with retry (re-queue) and dismiss (delete) actions. Backend adds /failures, /failures/email/:id/retry and DELETE on adminSystemHealth. First source is email failures (the original trigger — quote_sent template errors left invoices unsent for 14h with no signal); more sources can be added.
This commit is contained in:
@@ -23,6 +23,7 @@ const { requirePermission } = require('../middleware/permissions');
|
|||||||
const { handleAsync, validateRequest, successResponse } = require('../utils/routeHelpers');
|
const { handleAsync, validateRequest, successResponse } = require('../utils/routeHelpers');
|
||||||
const { verifyDocumentArtefacts } = require('../services/backupIntegrityService');
|
const { verifyDocumentArtefacts } = require('../services/backupIntegrityService');
|
||||||
const { getCoverageReport } = require('../services/backupCoverageService');
|
const { getCoverageReport } = require('../services/backupCoverageService');
|
||||||
|
const { db } = require('../database/db');
|
||||||
|
|
||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
|
|
||||||
@@ -84,4 +85,81 @@ router.get(
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GET /api/admin/system-health/failures
|
||||||
|
*
|
||||||
|
* Surfaces background failures that would otherwise go unnoticed. v1
|
||||||
|
* covers stuck/failed outbound emails: rows the queue processor has
|
||||||
|
* given up on (status='failed') or exhausted its retries on
|
||||||
|
* (status='pending' AND retry_count >= 3 — the processor only picks up
|
||||||
|
* retry_count < 3). Trigger: a 14h window where 'quote_sent' template
|
||||||
|
* errors left invoices unsent with no admin-visible signal.
|
||||||
|
*/
|
||||||
|
router.get(
|
||||||
|
'/failures',
|
||||||
|
requirePermission('settings.view'),
|
||||||
|
handleAsync(async (req, res) => {
|
||||||
|
const stuckEmails = await db('email_queue')
|
||||||
|
.where(function () {
|
||||||
|
this.where('status', 'failed')
|
||||||
|
.orWhere(function () {
|
||||||
|
this.where('status', 'pending').andWhere('retry_count', '>=', 3);
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.orderBy('created_at', 'desc')
|
||||||
|
.limit(200)
|
||||||
|
.select('id', 'recipient_email', 'email_type', 'status', 'retry_count', 'error_message', 'created_at');
|
||||||
|
|
||||||
|
return successResponse(res, {
|
||||||
|
stuckEmails: stuckEmails.map((r) => ({
|
||||||
|
id: r.id,
|
||||||
|
recipientEmail: r.recipient_email,
|
||||||
|
emailType: r.email_type,
|
||||||
|
status: r.status,
|
||||||
|
retryCount: r.retry_count,
|
||||||
|
errorMessage: r.error_message,
|
||||||
|
createdAt: r.created_at,
|
||||||
|
})),
|
||||||
|
counts: { stuckEmails: stuckEmails.length },
|
||||||
|
});
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* POST /failures/email/:id/retry — re-queue a stuck email (status back to
|
||||||
|
* pending, retry_count reset, error cleared, scheduled_at cleared so the
|
||||||
|
* 60s processor picks it up on its next pass).
|
||||||
|
*/
|
||||||
|
router.post(
|
||||||
|
'/failures/email/:id/retry',
|
||||||
|
requirePermission('settings.edit'),
|
||||||
|
handleAsync(async (req, res) => {
|
||||||
|
const id = parseInt(req.params.id, 10);
|
||||||
|
if (!Number.isFinite(id) || id <= 0) return res.status(400).json({ error: 'Invalid id' });
|
||||||
|
const updated = await db('email_queue').where({ id }).update({
|
||||||
|
status: 'pending',
|
||||||
|
retry_count: 0,
|
||||||
|
error_message: null,
|
||||||
|
scheduled_at: null,
|
||||||
|
});
|
||||||
|
if (!updated) return res.status(404).json({ error: 'Email not found' });
|
||||||
|
return successResponse(res, { retried: true });
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DELETE /failures/email/:id — dismiss a stuck email (remove the row so
|
||||||
|
* it stops surfacing). Use when the failure is understood + won't be sent.
|
||||||
|
*/
|
||||||
|
router.delete(
|
||||||
|
'/failures/email/:id',
|
||||||
|
requirePermission('settings.edit'),
|
||||||
|
handleAsync(async (req, res) => {
|
||||||
|
const id = parseInt(req.params.id, 10);
|
||||||
|
if (!Number.isFinite(id) || id <= 0) return res.status(400).json({ error: 'Invalid id' });
|
||||||
|
await db('email_queue').where({ id }).del();
|
||||||
|
return successResponse(res, { dismissed: true });
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
module.exports = router;
|
module.exports = router;
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ import {
|
|||||||
ArchivesPage,
|
ArchivesPage,
|
||||||
AnalyticsPage,
|
AnalyticsPage,
|
||||||
SettingsPage,
|
SettingsPage,
|
||||||
|
SystemHealthPage,
|
||||||
UserManagementPage,
|
UserManagementPage,
|
||||||
CustomerManagementPage,
|
CustomerManagementPage,
|
||||||
CustomerDetailPage,
|
CustomerDetailPage,
|
||||||
@@ -267,6 +268,7 @@ function App() {
|
|||||||
<Route path="customers/:id" element={<RedirectCustomerDetail />} />
|
<Route path="customers/:id" element={<RedirectCustomerDetail />} />
|
||||||
|
|
||||||
<Route path="settings" element={<SettingsPage />} />
|
<Route path="settings" element={<SettingsPage />} />
|
||||||
|
<Route path="system-health" element={<SystemHealthPage />} />
|
||||||
<Route path="webhooks/:id/deliveries" element={<WebhookDeliveriesPage />} />
|
<Route path="webhooks/:id/deliveries" element={<WebhookDeliveriesPage />} />
|
||||||
|
|
||||||
{/* Old top-level routes — these surfaces now live as
|
{/* Old top-level routes — these surfaces now live as
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import {
|
|||||||
Archive,
|
Archive,
|
||||||
BarChart3,
|
BarChart3,
|
||||||
Settings,
|
Settings,
|
||||||
|
Activity,
|
||||||
X,
|
X,
|
||||||
Users,
|
Users,
|
||||||
Briefcase,
|
Briefcase,
|
||||||
@@ -62,6 +63,7 @@ const navigation: NavItem[] = [
|
|||||||
{ nameKey: 'navigation.archives', href: '/admin/archives', icon: Archive, permission: 'archives.view' },
|
{ nameKey: 'navigation.archives', href: '/admin/archives', icon: Archive, permission: 'archives.view' },
|
||||||
{ nameKey: 'admin.analytics', href: '/admin/analytics', icon: BarChart3, permission: 'analytics.view', featureFlag: 'analytics' },
|
{ nameKey: 'admin.analytics', href: '/admin/analytics', icon: BarChart3, permission: 'analytics.view', featureFlag: 'analytics' },
|
||||||
{ nameKey: 'navigation.settings', href: '/admin/settings', icon: Settings, permission: 'settings.view' },
|
{ nameKey: 'navigation.settings', href: '/admin/settings', icon: Settings, permission: 'settings.view' },
|
||||||
|
{ nameKey: 'navigation.systemHealth', href: '/admin/system-health', icon: Activity, permission: 'settings.view' },
|
||||||
{ nameKey: 'navigation.users', href: '/admin/users', icon: Users, permission: 'users.view', featureFlag: 'userManagement' },
|
{ nameKey: 'navigation.users', href: '/admin/users', icon: Users, permission: 'users.view', featureFlag: 'userManagement' },
|
||||||
// Clients section (#354 follow-up) — admin-side surface for the
|
// Clients section (#354 follow-up) — admin-side surface for the
|
||||||
// CRM-area sub-features. Today this entry leads to /admin/clients
|
// CRM-area sub-features. Today this entry leads to /admin/clients
|
||||||
|
|||||||
@@ -73,6 +73,26 @@
|
|||||||
"message": "Sind Sie sicher, dass Sie die Einladung für {{email}} abbrechen möchten?"
|
"message": "Sind Sie sicher, dass Sie die Einladung für {{email}} abbrechen möchten?"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"systemHealth": {
|
||||||
|
"title": "Systemzustand",
|
||||||
|
"subtitle": "Hintergrundfehler, die Aufmerksamkeit erfordern.",
|
||||||
|
"retry": "Erneut versuchen",
|
||||||
|
"dismiss": "Verwerfen",
|
||||||
|
"retriedToast": "E-Mail erneut eingereiht.",
|
||||||
|
"dismissedToast": "Verworfen.",
|
||||||
|
"stuckEmails": {
|
||||||
|
"title": "Hängende / fehlgeschlagene E-Mails",
|
||||||
|
"empty": "Keine hängenden oder fehlgeschlagenen E-Mails — alles in Ordnung.",
|
||||||
|
"noError": "Versuche aufgebraucht",
|
||||||
|
"col": {
|
||||||
|
"recipient": "Empfänger",
|
||||||
|
"type": "Typ",
|
||||||
|
"error": "Fehler",
|
||||||
|
"queued": "Eingereiht",
|
||||||
|
"actions": "Aktionen"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"common": {
|
"common": {
|
||||||
"loading": "Wird geladen...",
|
"loading": "Wird geladen...",
|
||||||
"error": "Fehler",
|
"error": "Fehler",
|
||||||
@@ -166,6 +186,7 @@
|
|||||||
"dashboard": "Dashboard",
|
"dashboard": "Dashboard",
|
||||||
"events": "Veranstaltungen",
|
"events": "Veranstaltungen",
|
||||||
"settings": "Einstellungen",
|
"settings": "Einstellungen",
|
||||||
|
"systemHealth": "Systemzustand",
|
||||||
"archives": "Archive",
|
"archives": "Archive",
|
||||||
"emailSettings": "E-Mail-Einstellungen",
|
"emailSettings": "E-Mail-Einstellungen",
|
||||||
"branding": "Markenidentität",
|
"branding": "Markenidentität",
|
||||||
|
|||||||
@@ -73,6 +73,26 @@
|
|||||||
"message": "Are you sure you want to cancel the invitation for {{email}}?"
|
"message": "Are you sure you want to cancel the invitation for {{email}}?"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"systemHealth": {
|
||||||
|
"title": "System health",
|
||||||
|
"subtitle": "Background failures that need attention.",
|
||||||
|
"retry": "Retry",
|
||||||
|
"dismiss": "Dismiss",
|
||||||
|
"retriedToast": "Email re-queued.",
|
||||||
|
"dismissedToast": "Dismissed.",
|
||||||
|
"stuckEmails": {
|
||||||
|
"title": "Stuck / failed emails",
|
||||||
|
"empty": "No stuck or failed emails — all clear.",
|
||||||
|
"noError": "retries exhausted",
|
||||||
|
"col": {
|
||||||
|
"recipient": "Recipient",
|
||||||
|
"type": "Type",
|
||||||
|
"error": "Error",
|
||||||
|
"queued": "Queued",
|
||||||
|
"actions": "Actions"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"common": {
|
"common": {
|
||||||
"loading": "Loading...",
|
"loading": "Loading...",
|
||||||
"error": "Error",
|
"error": "Error",
|
||||||
@@ -167,6 +187,7 @@
|
|||||||
"events": "Events",
|
"events": "Events",
|
||||||
"archives": "Archives",
|
"archives": "Archives",
|
||||||
"settings": "Settings",
|
"settings": "Settings",
|
||||||
|
"systemHealth": "System health",
|
||||||
"eventTypes": "Event Types",
|
"eventTypes": "Event Types",
|
||||||
"branding": "Branding",
|
"branding": "Branding",
|
||||||
"emailSettings": "Email Settings",
|
"emailSettings": "Email Settings",
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
/**
|
||||||
|
* Admin → System health. Aggregates background failures that would
|
||||||
|
* otherwise go unnoticed. v1: stuck/failed outbound emails (the queue
|
||||||
|
* processor gave up or exhausted retries), with retry + dismiss.
|
||||||
|
*/
|
||||||
|
import React from 'react';
|
||||||
|
import { useTranslation } from 'react-i18next';
|
||||||
|
import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
|
||||||
|
import { toast } from 'react-toastify';
|
||||||
|
import { AlertCircle, RefreshCw, Trash2, CheckCircle } from 'lucide-react';
|
||||||
|
import { Button, Card, Loading } from '../../components/common';
|
||||||
|
import { useLocalizedDate } from '../../hooks/useLocalizedDate';
|
||||||
|
import { systemHealthService } from '../../services/systemHealth.service';
|
||||||
|
|
||||||
|
export const SystemHealthPage: React.FC = () => {
|
||||||
|
const { t } = useTranslation();
|
||||||
|
const { formatDateTime: fmtDateTime } = useLocalizedDate();
|
||||||
|
const qc = useQueryClient();
|
||||||
|
|
||||||
|
const { data, isLoading } = useQuery({
|
||||||
|
queryKey: ['system-health-failures'],
|
||||||
|
queryFn: () => systemHealthService.getFailures(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const invalidate = () => qc.invalidateQueries({ queryKey: ['system-health-failures'] });
|
||||||
|
|
||||||
|
const retryMutation = useMutation({
|
||||||
|
mutationFn: (id: number) => systemHealthService.retryEmail(id),
|
||||||
|
onSuccess: () => { toast.success(t('systemHealth.retriedToast', 'Email re-queued.')); invalidate(); },
|
||||||
|
onError: () => toast.error(t('toast.saveError')),
|
||||||
|
});
|
||||||
|
const dismissMutation = useMutation({
|
||||||
|
mutationFn: (id: number) => systemHealthService.dismissEmail(id),
|
||||||
|
onSuccess: () => { toast.success(t('systemHealth.dismissedToast', 'Dismissed.')); invalidate(); },
|
||||||
|
onError: () => toast.error(t('toast.saveError')),
|
||||||
|
});
|
||||||
|
|
||||||
|
const stuckEmails = data?.stuckEmails ?? [];
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="container py-6">
|
||||||
|
<div className="mb-6">
|
||||||
|
<h1 className="text-2xl font-bold text-theme">{t('systemHealth.title', 'System health')}</h1>
|
||||||
|
<p className="text-sm text-muted-theme mt-1">
|
||||||
|
{t('systemHealth.subtitle', 'Background failures that need attention.')}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<Card padding="lg">
|
||||||
|
<div className="flex items-center gap-2 mb-3">
|
||||||
|
<AlertCircle className="w-5 h-5 text-amber-500" />
|
||||||
|
<h2 className="text-lg font-semibold text-neutral-900 dark:text-neutral-100">
|
||||||
|
{t('systemHealth.stuckEmails.title', 'Stuck / failed emails')}
|
||||||
|
</h2>
|
||||||
|
{!isLoading && (
|
||||||
|
<span className="ml-1 text-sm text-muted-theme">({stuckEmails.length})</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{isLoading ? <Loading /> : stuckEmails.length === 0 ? (
|
||||||
|
<div className="flex items-center gap-2 text-sm text-green-700 dark:text-green-400 py-6">
|
||||||
|
<CheckCircle className="w-5 h-5" />
|
||||||
|
{t('systemHealth.stuckEmails.empty', 'No stuck or failed emails — all clear.')}
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="rounded-lg border border-neutral-200 dark:border-neutral-700 overflow-hidden">
|
||||||
|
<div className="overflow-x-auto">
|
||||||
|
<table className="w-full text-sm">
|
||||||
|
<thead className="bg-neutral-50 dark:bg-neutral-800 text-neutral-700 dark:text-neutral-300">
|
||||||
|
<tr>
|
||||||
|
<th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.recipient', 'Recipient')}</th>
|
||||||
|
<th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.type', 'Type')}</th>
|
||||||
|
<th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.error', 'Error')}</th>
|
||||||
|
<th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.queued', 'Queued')}</th>
|
||||||
|
<th className="px-3 py-2 text-right">{t('systemHealth.stuckEmails.col.actions', 'Actions')}</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{stuckEmails.map((m) => (
|
||||||
|
<tr key={m.id} className="border-t border-neutral-200 dark:border-neutral-700 align-top">
|
||||||
|
<td className="px-3 py-2 break-all">{m.recipientEmail}</td>
|
||||||
|
<td className="px-3 py-2 font-mono text-xs">{m.emailType}</td>
|
||||||
|
<td className="px-3 py-2 max-w-xs">
|
||||||
|
<span className="text-xs text-red-700 dark:text-red-400 break-words">
|
||||||
|
{m.errorMessage || t('systemHealth.stuckEmails.noError', 'retries exhausted')}
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td className="px-3 py-2 whitespace-nowrap">{m.createdAt ? fmtDateTime(m.createdAt) : '—'}</td>
|
||||||
|
<td className="px-3 py-2">
|
||||||
|
<div className="flex items-center justify-end gap-1">
|
||||||
|
<Button variant="outline" size="sm"
|
||||||
|
isLoading={retryMutation.isPending && retryMutation.variables === m.id}
|
||||||
|
onClick={() => retryMutation.mutate(m.id)}
|
||||||
|
leftIcon={<RefreshCw className="w-3.5 h-3.5" />}>
|
||||||
|
{t('systemHealth.retry', 'Retry')}
|
||||||
|
</Button>
|
||||||
|
<button type="button"
|
||||||
|
aria-label={t('systemHealth.dismiss', 'Dismiss') as string}
|
||||||
|
onClick={() => dismissMutation.mutate(m.id)}
|
||||||
|
className="p-1.5 text-neutral-400 hover:text-red-600">
|
||||||
|
<Trash2 className="w-4 h-4" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</Card>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -8,6 +8,7 @@ export { ArchivesPage } from './ArchivesPage';
|
|||||||
export { AnalyticsPage } from './AnalyticsPage';
|
export { AnalyticsPage } from './AnalyticsPage';
|
||||||
export { BrandingPage } from './BrandingPage';
|
export { BrandingPage } from './BrandingPage';
|
||||||
export { SettingsPage } from './SettingsPage';
|
export { SettingsPage } from './SettingsPage';
|
||||||
|
export { SystemHealthPage } from './SystemHealthPage';
|
||||||
export { CMSPage } from './CMSPage';
|
export { CMSPage } from './CMSPage';
|
||||||
export { BackupManagement } from './BackupManagement';
|
export { BackupManagement } from './BackupManagement';
|
||||||
export { EventFeedbackPage } from './EventFeedbackPage';
|
export { EventFeedbackPage } from './EventFeedbackPage';
|
||||||
|
|||||||
@@ -0,0 +1,35 @@
|
|||||||
|
/**
|
||||||
|
* Admin → System health. Surfaces background failures (v1: stuck/failed
|
||||||
|
* outbound emails) so they don't sit unnoticed, with retry/dismiss.
|
||||||
|
*/
|
||||||
|
import { api } from '../config/api';
|
||||||
|
|
||||||
|
export interface StuckEmail {
|
||||||
|
id: number;
|
||||||
|
recipientEmail: string;
|
||||||
|
emailType: string;
|
||||||
|
status: 'pending' | 'failed';
|
||||||
|
retryCount: number;
|
||||||
|
errorMessage: string | null;
|
||||||
|
createdAt: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SystemHealthFailures {
|
||||||
|
stuckEmails: StuckEmail[];
|
||||||
|
counts: { stuckEmails: number };
|
||||||
|
}
|
||||||
|
|
||||||
|
export const systemHealthService = {
|
||||||
|
async getFailures(): Promise<SystemHealthFailures> {
|
||||||
|
const { data } = await api.get('/admin/system-health/failures');
|
||||||
|
return data.data || data;
|
||||||
|
},
|
||||||
|
|
||||||
|
async retryEmail(id: number): Promise<void> {
|
||||||
|
await api.post(`/admin/system-health/failures/email/${id}/retry`);
|
||||||
|
},
|
||||||
|
|
||||||
|
async dismissEmail(id: number): Promise<void> {
|
||||||
|
await api.delete(`/admin/system-health/failures/email/${id}`);
|
||||||
|
},
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user