fix(email): show a queue nobody is working instead of reporting all-clear
Closes #1262. "Gallery email queued" reads as a delivery confirmation, and System Health agreed with it: "No stuck or failed emails -- all clear", while not one email had gone out. Both statements were true and neither was the one the admin needed. Queueing writes an email_queue row at status='pending', retry_count 0 -- nothing more. /failures matched only status='failed' or pending-with-retry_count>=3, so it matched none of those rows, and there are two ordinary ways they never leave that state: - startEmailQueueProcessor() was never reached, so nothing polls the queue. - Every pass returns early. processEmailQueue bails when the transporter will not initialise, before it touches a single row, so retry_count stays 0 and no error_message is ever written. A working SMTP test button does not contradict this: that path builds its own transport. adminSystem.js made it worse by reporting `emailProcessor: { status: 'active' }` as a literal, so the one place that named the worker always said it was fine. - emailProcessor records what each pass did -- started, lastRunAt, lastResult, lastError -- and exports getQueueProcessorStatus(). The transporter bail and the queue-query failure, the two silent early returns, both write lastError. - /failures gains `waitingEmails`: pending, under the retry cap, past any scheduled_at, and queued more than 10 minutes ago. The predicate mirrors the processor's own pickup query, so a row listed there is one it should already have taken; rows over the cap stay in `stuckEmails` and are not counted twice. A future scheduled_at is left alone -- split-payment invoices and the business-hours floor park rows deliberately. - System Health leads with the processor's state (running / stopped / degraded) and lists waiting emails in their own table. The all-clear now needs both buckets empty. - adminSystem reports the real processor state instead of the literal. - The two "queued" toasts say the queue processor is what sends it and where to look if it doesn't arrive. 8 route tests, all 8 failing before the change.
This commit is contained in:
Paul Nothaft
@@ -108,6 +108,26 @@
|
||||
"queued": "Eingereiht",
|
||||
"actions": "Aktionen"
|
||||
}
|
||||
},
|
||||
"waitingEmails": {
|
||||
"title": "Wartet auf Versand",
|
||||
"empty": "Nichts in Wartestellung — die Warteschlange wird abgearbeitet.",
|
||||
"description": "Vor über 10 Minuten eingereiht, jetzt fällig und weiterhin nicht versendet. Diese E-Mails sind nicht fehlgeschlagen — es hat niemand versucht, sie zu senden.",
|
||||
"neverAttempted": "kein Versuch",
|
||||
"attempted": "{{count}} Versuch(e), letzter Fehler: {{error}}",
|
||||
"unknownError": "unbekannt",
|
||||
"col": {
|
||||
"attempts": "Versuche"
|
||||
}
|
||||
},
|
||||
"processor": {
|
||||
"title": "E-Mail-Warteschlangen-Prozessor",
|
||||
"running": "Läuft.",
|
||||
"stopped": "Läuft auf dieser Instanz nicht. Eingereihte E-Mails werden in die Datenbank geschrieben, aber niemand versendet sie.",
|
||||
"degraded": "Läuft, aber der letzte Durchlauf konnte nicht senden: {{error}}",
|
||||
"lastRun": "Letzter Durchlauf {{when}}",
|
||||
"neverRan": "Seit dem Start dieser Instanz nicht gelaufen.",
|
||||
"lastResult": "{{sent}} gesendet, {{failed}} fehlgeschlagen"
|
||||
}
|
||||
},
|
||||
"common": {
|
||||
@@ -1324,6 +1344,7 @@
|
||||
"resetGalleryPassword": "Galerie-Passwort zurücksetzen",
|
||||
"resendCreationEmail": "Erstellungs-E-Mail erneut senden",
|
||||
"creationEmailResent": "Die Erstellungs-E-Mail wurde zur Warteschlange hinzugefügt",
|
||||
"emailQueuedHint": "Der Warteschlangen-Prozessor versendet sie — prüfen Sie den Systemzustand, falls sie nicht ankommt.",
|
||||
"failedToResendEmail": "Fehler beim erneuten Senden der Erstellungs-E-Mail",
|
||||
"photoStatistics": "Fotostatistiken",
|
||||
"managePhotos": "Fotos verwalten",
|
||||
|
||||
@@ -108,6 +108,26 @@
|
||||
"queued": "Queued",
|
||||
"actions": "Actions"
|
||||
}
|
||||
},
|
||||
"waitingEmails": {
|
||||
"title": "Waiting to send",
|
||||
"empty": "Nothing waiting — the queue is being worked.",
|
||||
"description": "Queued more than 10 minutes ago, due now, and still unsent. These have not failed — nothing has tried to send them.",
|
||||
"neverAttempted": "never attempted",
|
||||
"attempted": "{{count}} attempt(s), last error: {{error}}",
|
||||
"unknownError": "unknown",
|
||||
"col": {
|
||||
"attempts": "Attempts"
|
||||
}
|
||||
},
|
||||
"processor": {
|
||||
"title": "Email queue processor",
|
||||
"running": "Running.",
|
||||
"stopped": "Not running on this instance. Queued emails are written to the database but nothing is sending them.",
|
||||
"degraded": "Running, but the last pass could not send: {{error}}",
|
||||
"lastRun": "Last pass {{when}}",
|
||||
"neverRan": "Has not run since this instance started.",
|
||||
"lastResult": "{{sent}} sent, {{failed}} failed"
|
||||
}
|
||||
},
|
||||
"common": {
|
||||
@@ -820,6 +840,7 @@
|
||||
"resetGalleryPassword": "Reset Gallery Password",
|
||||
"resendCreationEmail": "Resend Creation Email",
|
||||
"creationEmailResent": "Creation email has been queued for sending",
|
||||
"emailQueuedHint": "The queue processor sends it — check System health if it does not arrive.",
|
||||
"failedToResendEmail": "Failed to resend creation email",
|
||||
"photoStatistics": "Photo Statistics",
|
||||
"totalPhotos": "Total Photos",
|
||||
|
||||
@@ -316,11 +316,13 @@ export const EventDetailsPage: React.FC = () => {
|
||||
mutationFn: (password?: string) =>
|
||||
eventsService.sendGalleryEmail(parseInt(id!), password ? { password } : undefined),
|
||||
onSuccess: (result) => {
|
||||
// #1262 — queueing is not delivery, and a queue nobody is working
|
||||
// reports no failure at all. Point at where the queue is visible.
|
||||
toast.success(
|
||||
t('events.sendGalleryEmail.success', {
|
||||
`${t('events.sendGalleryEmail.success', {
|
||||
recipient: result.recipient,
|
||||
defaultValue: 'Gallery email queued to {{recipient}}.',
|
||||
}),
|
||||
})} ${t('events.emailQueuedHint', 'The queue processor sends it — check System health if it does not arrive.')}`,
|
||||
);
|
||||
setShowSendEmailDialog(false);
|
||||
},
|
||||
|
||||
@@ -2,15 +2,22 @@
|
||||
* Admin → System health. Aggregates background failures that would
|
||||
* otherwise go unnoticed. v1: stuck/failed outbound emails (the queue
|
||||
* processor gave up or exhausted retries), with retry + dismiss.
|
||||
*
|
||||
* #1262 — "no failures" was being read as "everything went out". It is not the
|
||||
* same claim: a queue nobody is working produces no failures at all, because
|
||||
* every row sits at status='pending' with retry_count 0. So the page now leads
|
||||
* with what the processor itself last did, and lists due-but-unsent emails
|
||||
* next to the failed ones. The all-clear only shows when both are empty and
|
||||
* the processor is running.
|
||||
*/
|
||||
import React from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { AlertCircle, RefreshCw, Trash2, CheckCircle } from 'lucide-react';
|
||||
import { AlertCircle, RefreshCw, Trash2, CheckCircle, Clock, Mail, MailX } from 'lucide-react';
|
||||
import { Button, Card, Loading } from '../../components/common';
|
||||
import { useMutationWithToast } from '../../hooks';
|
||||
import { useLocalizedDate } from '../../hooks/useLocalizedDate';
|
||||
import { systemHealthService } from '../../services/systemHealth.service';
|
||||
import { systemHealthService, type StuckEmail } from '../../services/systemHealth.service';
|
||||
|
||||
export const SystemHealthPage: React.FC = () => {
|
||||
const { t } = useTranslation();
|
||||
@@ -35,6 +42,82 @@ export const SystemHealthPage: React.FC = () => {
|
||||
});
|
||||
|
||||
const stuckEmails = data?.stuckEmails ?? [];
|
||||
const waitingEmails = data?.waitingEmails ?? [];
|
||||
const processor = data?.processor;
|
||||
|
||||
// The processor is only "fine" when it has been started AND its last pass
|
||||
// didn't bail. A started-but-erroring processor is the case that used to
|
||||
// read as healthy, so it gets its own state rather than folding into either.
|
||||
const processorState: 'ok' | 'degraded' | 'stopped' = !processor
|
||||
? 'ok'
|
||||
: !processor.started
|
||||
? 'stopped'
|
||||
: processor.lastError
|
||||
? 'degraded'
|
||||
: 'ok';
|
||||
|
||||
const emailTable = (rows: StuckEmail[], showError: boolean) => (
|
||||
<div className="rounded-lg border border-neutral-200 dark:border-neutral-700 overflow-hidden">
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-neutral-50 dark:bg-neutral-800 text-neutral-700 dark:text-neutral-300">
|
||||
<tr>
|
||||
<th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.recipient', 'Recipient')}</th>
|
||||
<th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.type', 'Type')}</th>
|
||||
<th className="px-3 py-2 text-left">
|
||||
{showError
|
||||
? t('systemHealth.stuckEmails.col.error', 'Error')
|
||||
: t('systemHealth.waitingEmails.col.attempts', 'Attempts')}
|
||||
</th>
|
||||
<th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.queued', 'Queued')}</th>
|
||||
<th className="px-3 py-2 text-right">{t('systemHealth.stuckEmails.col.actions', 'Actions')}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((m) => (
|
||||
<tr key={m.id} className="border-t border-neutral-200 dark:border-neutral-700 align-top">
|
||||
<td className="px-3 py-2 break-all">{m.recipientEmail}</td>
|
||||
<td className="px-3 py-2 font-mono text-xs">{m.emailType}</td>
|
||||
<td className="px-3 py-2 max-w-xs">
|
||||
{showError ? (
|
||||
<span className="text-xs text-red-700 dark:text-red-400 break-words">
|
||||
{m.errorMessage || t('systemHealth.stuckEmails.noError', 'retries exhausted')}
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-xs text-neutral-600 dark:text-neutral-400">
|
||||
{m.retryCount > 0
|
||||
? t('systemHealth.waitingEmails.attempted', '{{count}} attempt(s), last error: {{error}}', {
|
||||
count: m.retryCount,
|
||||
error: m.errorMessage || t('systemHealth.waitingEmails.unknownError', 'unknown'),
|
||||
})
|
||||
: t('systemHealth.waitingEmails.neverAttempted', 'never attempted')}
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
<td className="px-3 py-2 whitespace-nowrap">{m.createdAt ? fmtDateTime(m.createdAt) : '—'}</td>
|
||||
<td className="px-3 py-2">
|
||||
<div className="flex items-center justify-end gap-1">
|
||||
<Button variant="outline" size="sm"
|
||||
isLoading={retryMutation.isPending && retryMutation.variables === m.id}
|
||||
onClick={() => retryMutation.mutate(m.id)}
|
||||
leftIcon={<RefreshCw className="w-3.5 h-3.5" />}>
|
||||
{t('systemHealth.retry', 'Retry')}
|
||||
</Button>
|
||||
<button type="button"
|
||||
aria-label={t('systemHealth.dismiss', 'Dismiss') as string}
|
||||
onClick={() => dismissMutation.mutate(m.id)}
|
||||
className="p-1.5 text-neutral-400 hover:text-red-600">
|
||||
<Trash2 className="w-4 h-4" />
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="container py-6">
|
||||
@@ -45,6 +128,79 @@ export const SystemHealthPage: React.FC = () => {
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Queue processor. Listed first because when this is stopped, every
|
||||
other count on the page is explained by it — and a stopped processor
|
||||
shows no failures at all, which is what made it invisible. */}
|
||||
{!isLoading && processor && (
|
||||
<Card padding="lg" className="mb-4">
|
||||
<div className="flex items-start gap-3">
|
||||
{processorState === 'ok'
|
||||
? <Mail className="w-5 h-5 mt-0.5 text-green-600 dark:text-green-400 shrink-0" />
|
||||
: <MailX className="w-5 h-5 mt-0.5 text-red-600 dark:text-red-400 shrink-0" />}
|
||||
<div className="min-w-0">
|
||||
<h2 className="text-lg font-semibold text-neutral-900 dark:text-neutral-100">
|
||||
{t('systemHealth.processor.title', 'Email queue processor')}
|
||||
</h2>
|
||||
<p className={`text-sm mt-0.5 ${
|
||||
processorState === 'ok'
|
||||
? 'text-neutral-600 dark:text-neutral-400'
|
||||
: 'text-red-700 dark:text-red-400'
|
||||
}`}>
|
||||
{processorState === 'stopped'
|
||||
? t('systemHealth.processor.stopped',
|
||||
'Not running on this instance. Queued emails are written to the database but nothing is sending them.')
|
||||
: processorState === 'degraded'
|
||||
? t('systemHealth.processor.degraded',
|
||||
'Running, but the last pass could not send: {{error}}', { error: processor.lastError })
|
||||
: t('systemHealth.processor.running', 'Running.')}
|
||||
</p>
|
||||
<p className="text-xs text-neutral-500 dark:text-neutral-400 mt-1">
|
||||
{processor.lastRunAt
|
||||
? t('systemHealth.processor.lastRun', 'Last pass {{when}}', { when: fmtDateTime(processor.lastRunAt) })
|
||||
: t('systemHealth.processor.neverRan', 'Has not run since this instance started.')}
|
||||
{processor.lastResult && (
|
||||
<> {' · '}
|
||||
{t('systemHealth.processor.lastResult', '{{sent}} sent, {{failed}} failed', {
|
||||
sent: processor.lastResult.sent,
|
||||
failed: processor.lastResult.failed,
|
||||
})}
|
||||
</>
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{/* Due but unsent. Distinct from failed: nothing went wrong with these,
|
||||
they were simply never picked up. */}
|
||||
<Card padding="lg" className="mb-4">
|
||||
<div className="flex items-center gap-2 mb-3">
|
||||
<Clock className="w-5 h-5 text-amber-500" />
|
||||
<h2 className="text-lg font-semibold text-neutral-900 dark:text-neutral-100">
|
||||
{t('systemHealth.waitingEmails.title', 'Waiting to send')}
|
||||
</h2>
|
||||
{!isLoading && (
|
||||
<span className="ml-1 text-sm text-neutral-500 dark:text-neutral-400">({waitingEmails.length})</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{isLoading ? <Loading /> : waitingEmails.length === 0 ? (
|
||||
<div className="flex items-center gap-2 text-sm text-green-700 dark:text-green-400 py-6">
|
||||
<CheckCircle className="w-5 h-5" />
|
||||
{t('systemHealth.waitingEmails.empty', 'Nothing waiting — the queue is being worked.')}
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<p className="text-sm text-neutral-600 dark:text-neutral-400 mb-3">
|
||||
{t('systemHealth.waitingEmails.description',
|
||||
'Queued more than 10 minutes ago, due now, and still unsent. These have not failed — nothing has tried to send them.')}
|
||||
</p>
|
||||
{emailTable(waitingEmails, false)}
|
||||
</>
|
||||
)}
|
||||
</Card>
|
||||
|
||||
<Card padding="lg">
|
||||
<div className="flex items-center gap-2 mb-3">
|
||||
<AlertCircle className="w-5 h-5 text-amber-500" />
|
||||
@@ -61,53 +217,7 @@ export const SystemHealthPage: React.FC = () => {
|
||||
<CheckCircle className="w-5 h-5" />
|
||||
{t('systemHealth.stuckEmails.empty', 'No stuck or failed emails — all clear.')}
|
||||
</div>
|
||||
) : (
|
||||
<div className="rounded-lg border border-neutral-200 dark:border-neutral-700 overflow-hidden">
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-neutral-50 dark:bg-neutral-800 text-neutral-700 dark:text-neutral-300">
|
||||
<tr>
|
||||
<th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.recipient', 'Recipient')}</th>
|
||||
<th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.type', 'Type')}</th>
|
||||
<th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.error', 'Error')}</th>
|
||||
<th className="px-3 py-2 text-left">{t('systemHealth.stuckEmails.col.queued', 'Queued')}</th>
|
||||
<th className="px-3 py-2 text-right">{t('systemHealth.stuckEmails.col.actions', 'Actions')}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{stuckEmails.map((m) => (
|
||||
<tr key={m.id} className="border-t border-neutral-200 dark:border-neutral-700 align-top">
|
||||
<td className="px-3 py-2 break-all">{m.recipientEmail}</td>
|
||||
<td className="px-3 py-2 font-mono text-xs">{m.emailType}</td>
|
||||
<td className="px-3 py-2 max-w-xs">
|
||||
<span className="text-xs text-red-700 dark:text-red-400 break-words">
|
||||
{m.errorMessage || t('systemHealth.stuckEmails.noError', 'retries exhausted')}
|
||||
</span>
|
||||
</td>
|
||||
<td className="px-3 py-2 whitespace-nowrap">{m.createdAt ? fmtDateTime(m.createdAt) : '—'}</td>
|
||||
<td className="px-3 py-2">
|
||||
<div className="flex items-center justify-end gap-1">
|
||||
<Button variant="outline" size="sm"
|
||||
isLoading={retryMutation.isPending && retryMutation.variables === m.id}
|
||||
onClick={() => retryMutation.mutate(m.id)}
|
||||
leftIcon={<RefreshCw className="w-3.5 h-3.5" />}>
|
||||
{t('systemHealth.retry', 'Retry')}
|
||||
</Button>
|
||||
<button type="button"
|
||||
aria-label={t('systemHealth.dismiss', 'Dismiss') as string}
|
||||
onClick={() => dismissMutation.mutate(m.id)}
|
||||
className="p-1.5 text-neutral-400 hover:text-red-600">
|
||||
<Trash2 className="w-4 h-4" />
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
) : emailTable(stuckEmails, true)}
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -193,7 +193,10 @@ export const ShareLinkCard: React.FC<ShareLinkCardProps> = ({ event, setShowPass
|
||||
onClick={async () => {
|
||||
try {
|
||||
await eventsService.resendCreationEmail(event.id);
|
||||
toast.success(t('events.creationEmailResent'));
|
||||
// #1262 — "queued" was being read as "delivered". Queueing only
|
||||
// writes an email_queue row; say where to look when it doesn't
|
||||
// turn up, because a queue nobody is working raises no failure.
|
||||
toast.success(`${t('events.creationEmailResent')} ${t('events.emailQueuedHint', 'The queue processor sends it — check System health if it does not arrive.')}`);
|
||||
} catch {
|
||||
toast.error(t('events.failedToResendEmail'));
|
||||
}
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
/**
|
||||
* Admin → System health. Surfaces background failures (v1: stuck/failed
|
||||
* outbound emails) so they don't sit unnoticed, with retry/dismiss.
|
||||
*
|
||||
* #1262 added `waitingEmails` and `processor`: a queue nobody is working
|
||||
* produces no failures at all, so "no failures" was not the same claim as
|
||||
* "everything went out".
|
||||
*/
|
||||
import { api } from '../config/api';
|
||||
|
||||
@@ -14,9 +18,20 @@ export interface StuckEmail {
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** What the queue processor last did — the difference between "idle" and "dead". */
|
||||
export interface EmailProcessorStatus {
|
||||
started: boolean;
|
||||
lastRunAt: string | null;
|
||||
lastResult: { processed: number; sent: number; failed: number } | null;
|
||||
lastError: string | null;
|
||||
}
|
||||
|
||||
export interface SystemHealthFailures {
|
||||
stuckEmails: StuckEmail[];
|
||||
counts: { stuckEmails: number };
|
||||
/** Due, under the retry cap, and still unsent — nobody picked them up. */
|
||||
waitingEmails: StuckEmail[];
|
||||
processor: EmailProcessorStatus;
|
||||
counts: { stuckEmails: number; waitingEmails: number };
|
||||
}
|
||||
|
||||
export const systemHealthService = {
|
||||
|
||||
Reference in New Issue
Block a user