feat(messages): Phase 2 — customer (hello@) mailbox + inbound body capture

Second inbound mailbox and real message bodies for the Messages viewer.

Backend:
- Migration 154: mail_accounts table (additional inbound mailboxes beyond the
  primary accounting IMAP) + received_emails.{account_key,to_address,body_html,
  body_text}. Additive/guarded.
- emailIntakeService now polls the accounting mailbox AND every enabled
  mail_accounts row. Extracted pollAccountOnce(cfg, {accountKey, routeToExpenses});
  accounting keeps its exact attachment->expenses behavior, customer mail is
  logged with its body and NOT routed to accounting. Inbound HTML is sanitized
  server-side (sanitize-html) on ingest.
- adminEmail: /received gains an account filter + returns account_key/to_address
  (bodies excluded from the list); new GET /received/:id returns the body;
  GET/POST /accounts + /accounts/test manage the extra mailboxes.

Frontend:
- Customers inbox now pulls the hello@ mailbox; reading pane renders the
  sanitized body in a strict (script-less, no same-origin) sandboxed iframe.
  Accounting inbox shows bodies too. Toolbar context keys off the mailbox.
- CustomerMailboxCard in Settings -> Email (behind the messaging flag) to
  configure + test the hello@ IMAP box.

No behavior change to the existing accounting inbound flow. Frontend build +
migration boot verified.
This commit is contained in:
Luca
2026-07-07 10:11:58 +02:00
parent 26eeb76197
commit ee46cf2125
7 changed files with 514 additions and 77 deletions
@@ -0,0 +1,52 @@
/**
* Messages Phase 2 — additional inbound mailboxes + captured message bodies.
*
* `mail_accounts` holds inbound mailboxes BEYOND the primary accounting IMAP
* that already lives in `email_configs` (e.g. the customer `hello@` mailbox).
* The intake poller (emailIntakeService) polls the accounting mailbox AND every
* enabled row here; customer mail is logged with its body but not routed to the
* accounting inbox.
*
* The new `received_emails` columns capture the parsed message so the Messages
* reading pane can show it: `account_key` tags which mailbox it came from,
* `body_html`/`body_text` hold the (server-sanitized) body, `to_address` the
* envelope recipient. All additive + guarded.
*/
exports.up = async function up(knex) {
const hasAccounts = await knex.schema.hasTable('mail_accounts');
if (!hasAccounts) {
await knex.schema.createTable('mail_accounts', (t) => {
t.increments('id').primary();
t.string('account_key', 64).notNullable().unique(); // e.g. 'customers'
t.string('label', 120);
t.string('imap_host', 255);
t.integer('imap_port').defaultTo(993);
t.boolean('imap_secure').defaultTo(true);
t.string('imap_user', 255);
t.string('imap_pass', 512);
t.string('imap_folder', 255).defaultTo('INBOX');
t.boolean('enabled').defaultTo(false);
t.timestamp('created_at').defaultTo(knex.fn.now());
t.timestamp('updated_at').defaultTo(knex.fn.now());
});
}
const cols = [
['account_key', (t) => t.string('account_key', 64)],
['to_address', (t) => t.string('to_address', 512)],
['body_html', (t) => t.text('body_html')],
['body_text', (t) => t.text('body_text')],
];
for (const [name, add] of cols) {
// eslint-disable-next-line no-await-in-loop
const has = await knex.schema.hasColumn('received_emails', name);
// eslint-disable-next-line no-await-in-loop
if (!has) await knex.schema.alterTable('received_emails', add);
}
};
exports.down = async function down(knex) {
// Non-destructive on the audit log: leave the added columns in place (they're
// nullable and harmless). Only drop the new table.
await knex.schema.dropTableIfExists('mail_accounts');
};
+94 -3
View File
@@ -260,16 +260,107 @@ router.get('/received', adminAuth, requirePermission('email.view'), async (req,
try {
const page = Math.max(1, parseInt(req.query.page, 10) || 1);
const pageSize = Math.min(100, Math.max(1, parseInt(req.query.pageSize, 10) || 25));
const base = db('received_emails');
const countRow = await base.clone().count({ c: '*' }).first();
const account = req.query.account ? String(req.query.account) : null;
// 'accounting' matches legacy rows too (account_key was NULL before mig 154).
const applyAccount = (qb) => {
if (account === 'accounting') qb.where((b) => b.where('account_key', 'accounting').orWhereNull('account_key'));
else if (account) qb.where('account_key', account);
return qb;
};
const countRow = await applyAccount(db('received_emails')).count({ c: '*' }).first();
const total = parseInt(countRow?.c || 0, 10);
const items = await base.clone().orderBy('received_at', 'desc').limit(pageSize).offset((page - 1) * pageSize);
// Bodies are excluded from the list (can be large); fetched per-message.
const items = await applyAccount(db('received_emails'))
.select('id', 'message_id', 'account_key', 'from_address', 'to_address', 'subject',
'received_at', 'attachment_count', 'status', 'inbound_document_id', 'error')
.orderBy('received_at', 'desc').limit(pageSize).offset((page - 1) * pageSize);
res.json({ items, pagination: { page, pageSize, total, totalPages: Math.ceil(total / pageSize) } });
} catch (error) {
errorResponse(res, error, 500, 'Failed to fetch received emails');
}
});
// Single received email WITH its captured (server-sanitized) body — Messages
// reading pane. body_html was already sanitized on ingest; the viewer renders
// it in a script-less sandboxed iframe as well.
router.get('/received/:id', adminAuth, requirePermission('email.view'), async (req, res) => {
try {
const id = parseInt(req.params.id, 10);
if (!Number.isInteger(id)) return res.status(400).json({ error: 'Invalid id' });
const row = await db('received_emails').where({ id }).first();
if (!row) return res.status(404).json({ error: 'Email not found' });
res.json(row);
} catch (error) {
errorResponse(res, error, 500, 'Failed to fetch email');
}
});
// Additional inbound mailboxes (beyond the primary accounting IMAP in
// email_configs) — e.g. the customer hello@ box. Passwords are masked out.
router.get('/accounts', adminAuth, requirePermission('email.view'), async (req, res) => {
try {
const rows = await db('mail_accounts').orderBy('id');
res.json({ items: rows.map((a) => ({ ...a, imap_pass: a.imap_pass ? '********' : '' })) });
} catch (error) {
errorResponse(res, error, 500, 'Failed to load mail accounts');
}
});
// Upsert a mailbox by account_key. A masked password ('********') keeps the
// stored value so the admin never has to re-type it.
router.post('/accounts', adminAuth, requirePermission('email.edit'), async (req, res) => {
try {
const b = req.body || {};
if (!b.account_key) return res.status(400).json({ error: 'account_key is required' });
const patch = {
label: b.label || null,
imap_host: b.imap_host || null,
imap_port: b.imap_port ? parseInt(b.imap_port, 10) : 993,
imap_secure: b.imap_secure !== false,
imap_user: b.imap_user || null,
imap_folder: b.imap_folder || 'INBOX',
enabled: !!b.enabled,
updated_at: new Date(),
};
if (b.imap_pass && b.imap_pass !== '********') patch.imap_pass = b.imap_pass;
const existing = await db('mail_accounts').where({ account_key: b.account_key }).first();
if (existing) {
await db('mail_accounts').where({ account_key: b.account_key }).update(patch);
} else {
await db('mail_accounts').insert({
account_key: b.account_key,
imap_pass: (b.imap_pass && b.imap_pass !== '********') ? b.imap_pass : '',
created_at: new Date(),
...patch,
});
}
res.json({ ok: true });
} catch (error) {
errorResponse(res, error, 500, 'Failed to save mail account');
}
});
// Test an inbound mailbox's IMAP connection (before or after saving). Resolves
// a masked/blank password from the stored row for the given account_key.
router.post('/accounts/test', adminAuth, requirePermission('email.view'), async (req, res) => {
try {
const b = req.body || {};
let pass = b.imap_pass;
if ((!pass || pass === '********') && b.account_key) {
const stored = await db('mail_accounts').where({ account_key: b.account_key }).first();
pass = stored?.imap_pass || '';
}
const emailIntakeService = require('../services/emailIntakeService');
const result = await emailIntakeService.testConnection({
host: b.imap_host, port: b.imap_port, secure: b.imap_secure,
user: b.imap_user, pass, folder: b.imap_folder || 'INBOX',
});
res.json(result);
} catch (error) {
res.status(422).json({ ok: false, error: `Mailbox test failed (${error.message}).` });
}
});
// Test email configuration
router.post('/test', adminAuth, requirePermission('email.send'), async (req, res) => {
try {
+101 -23
View File
@@ -16,6 +16,7 @@ const { db } = require('../database/db');
const logger = require('../utils/logger');
const { getStoragePath } = require('../config/storage');
const expenseService = require('./expenseService');
const sanitizeHtml = require('sanitize-html');
const { isUniqueViolation } = require('../utils/dbErrors');
const ALLOWED_MIME = ['application/pdf', 'image/jpeg', 'image/png'];
@@ -230,14 +231,36 @@ async function roundTripTest({ timeoutMs = 30000, intervalMs = 3000 } = {}) {
}
}
/** Poll the mailbox once. Safe to call repeatedly; self-skips when busy/off. */
async function pollOnce() {
if (polling) return { skipped: 'busy' };
if (!(await isEnabled())) return { skipped: 'disabled' };
const cfg = await getImapConfig();
if (!cfg) return { skipped: 'unconfigured' };
// Sanitize an inbound HTML body before storing it. Inbound mail is untrusted,
// so this strips scripts/handlers/unknown schemes (the viewer ALSO renders it
// in a script-less sandboxed iframe — defense in depth). Remote images are kept
// (many legit emails embed them) but that is the only tracking-vector allowed.
function sanitizeBody(html) {
if (!html) return null;
try {
return sanitizeHtml(html, {
allowedTags: sanitizeHtml.defaults.allowedTags.concat(['img']),
allowedAttributes: {
...sanitizeHtml.defaults.allowedAttributes,
img: ['src', 'alt', 'width', 'height'],
'*': ['style'],
},
allowedSchemes: ['http', 'https', 'mailto', 'cid'],
});
} catch (_) {
return null;
}
}
polling = true;
/**
* Poll ONE mailbox once and return the count of newly-processed messages.
* `opts.accountKey` tags each received_emails row; `opts.routeToExpenses`
* controls whether PDF/image attachments are dropped into the accounting inbox
* (true for the primary rechnungen@ mailbox) or only logged with the body
* (customer mail, e.g. hello@). The claim/dedup/stale-recovery logic is
* identical for every mailbox.
*/
async function pollAccountOnce(cfg, { accountKey = 'accounting', routeToExpenses = true } = {}) {
const client = makeImapClient(cfg);
let processed = 0;
try {
@@ -304,6 +327,7 @@ async function pollOnce() {
try {
await db('received_emails').insert({
message_id: claimKey,
account_key: accountKey,
status: 'processing',
attachment_count: 0,
received_at: new Date(),
@@ -315,37 +339,47 @@ async function pollOnce() {
throw ce;
}
// Ingest attachments. Isolate each so one bad file can't prevent the
// audit row (the symptom: doc lands in Incoming invoices but the
// email never shows under Received).
const atts = (parsed.attachments || []).filter((a) => ALLOWED_MIME.includes(a.contentType));
// Attachment handling. The accounting mailbox drops PDF/image
// attachments into the incoming-invoices inbox (isolated so one bad
// file can't prevent the audit row). Customer mailboxes only COUNT
// attachments — they aren't supplier invoices.
let inboundId = null;
let count = 0;
const attErrors = [];
for (const att of atts) {
try {
const filePath = await saveAttachment(att);
const doc = await expenseService.recordInboundDocument({ source: 'email', filePath, originalFilename: att.filename || 'attachment', mimeType: att.contentType }, null);
inboundId = doc.id; count += 1;
} catch (ae) {
attErrors.push(ae.message);
logger.error?.(`emailIntake: attachment "${att.filename}" failed: ${ae.message}`);
if (routeToExpenses) {
const atts = (parsed.attachments || []).filter((a) => ALLOWED_MIME.includes(a.contentType));
for (const att of atts) {
try {
const filePath = await saveAttachment(att);
const doc = await expenseService.recordInboundDocument({ source: 'email', filePath, originalFilename: att.filename || 'attachment', mimeType: att.contentType }, null);
inboundId = doc.id; count += 1;
} catch (ae) {
attErrors.push(ae.message);
logger.error?.(`emailIntake: attachment "${att.filename}" failed: ${ae.message}`);
}
}
} else {
count = (parsed.attachments || []).length;
}
// A malformed Date: header yields an Invalid Date, which throws on a
// Postgres timestamp insert — coerce to now.
const receivedAt = (parsed.date instanceof Date && !Number.isNaN(parsed.date.getTime())) ? parsed.date : new Date();
const status = count > 0 ? 'ingested' : (attErrors.length ? 'error' : 'no_attachment');
const status = routeToExpenses
? (count > 0 ? 'ingested' : (attErrors.length ? 'error' : 'no_attachment'))
: 'received';
// Finalise the claimed row — every processed message ends up in the
// Received tab, even attachment-less ones.
// Received log with its (sanitized) body, even attachment-less ones.
await db('received_emails').where({ message_id: claimKey }).update({
from_address: ((parsed.from && parsed.from.text) || '').slice(0, 512) || null,
to_address: ((parsed.to && parsed.to.text) || '').slice(0, 512) || null,
subject: parsed.subject || null,
received_at: receivedAt,
attachment_count: count,
status,
inbound_document_id: inboundId,
body_html: sanitizeBody(parsed.html || null),
body_text: parsed.text || null,
error: attErrors.length ? attErrors.join('; ').slice(0, 2000) : null,
});
await client.messageFlagsAdd(cand.uid, ['\\Seen'], { uid: true });
@@ -360,7 +394,7 @@ async function pollOnce() {
await db('received_emails').where({ message_id: claimKey })
.update({ status: 'error', error: String(e.message).slice(0, 2000) });
} else {
await db('received_emails').insert({ message_id: `err-${cand.uid}-${Date.now()}`, status: 'error', error: e.message, attachment_count: 0, received_at: new Date(), created_at: new Date() });
await db('received_emails').insert({ message_id: `err-${cand.uid}-${Date.now()}`, account_key: accountKey, status: 'error', error: e.message, attachment_count: 0, received_at: new Date(), created_at: new Date() });
}
} catch (ie) {
logger.error?.(`emailIntake: could not even write the error row (received_emails insert failing): ${ie.message}`);
@@ -373,11 +407,55 @@ async function pollOnce() {
/* eslint-enable no-await-in-loop */
await client.logout();
} catch (e) {
logger.error?.(`emailIntake: poll failed: ${e.message}`);
logger.error?.(`emailIntake: poll failed (${accountKey}): ${e.message}`);
try { await client.close(); } catch (_e) { /* ignore */ }
}
return processed;
}
/**
* Poll ALL configured inbound mailboxes once: the primary accounting IMAP
* (email_configs) plus every enabled row in mail_accounts (e.g. hello@).
* Safe to call repeatedly; self-skips when busy/off.
*/
async function pollOnce() {
if (polling) return { skipped: 'busy' };
if (!(await isEnabled())) return { skipped: 'disabled' };
polling = true;
let processed = 0;
let anyConfigured = false;
try {
// 1) Primary accounting mailbox — routes attachments to the invoices inbox.
const acctCfg = await getImapConfig();
if (acctCfg) {
anyConfigured = true;
processed += await pollAccountOnce(acctCfg, { accountKey: 'accounting', routeToExpenses: true });
}
// 2) Additional mailboxes (customers/hello@) — body captured, no expense
// routing. Guarded so a pre-migration DB simply polls the accounting box.
let extras = [];
try {
if (await db.schema.hasTable('mail_accounts')) {
extras = await db('mail_accounts').where({ enabled: true });
}
} catch (_) { extras = []; }
for (const a of extras) {
if (!a.imap_host || !a.imap_user) continue;
anyConfigured = true;
const cfg = {
host: a.imap_host,
port: a.imap_port || 993,
secure: a.imap_secure !== false && a.imap_secure !== 0,
auth: { user: a.imap_user, pass: a.imap_pass || '' },
folder: a.imap_folder || 'INBOX',
};
// eslint-disable-next-line no-await-in-loop
processed += await pollAccountOnce(cfg, { accountKey: a.account_key, routeToExpenses: false });
}
} finally {
polling = false;
}
if (!anyConfigured) return { skipped: 'unconfigured' };
return { processed };
}