Luca
cd6d57839b
Blocker 1 — CSV/Banana formula injection. Neither csvEscape (ledgerService) nor the tax-report CSV escape nor the unquoted tab-separated Banana cell formatter prefixed risky leading chars, so an admin-/sender-controlled cell beginning with = + - @ TAB CR executes as a formula when the Treuhänder opens the export. New shared util neutralizeSpreadsheetFormula() prepends a single quote; wired into all three sinks (quoted CSV + unquoted Banana). Unit test pins one of each char. Blocker 2 — IMAP intake double-ingest race. received_emails.message_id was INDEX, not UNIQUE, and the poller ingested attachments BEFORE writing the audit row, so a second replica / rolling-deploy overlap double-ingested the same mail. Migration 128 makes message_id UNIQUE (nulls stay distinct); the intake now CLAIMS the message row (status='processing') BEFORE ingesting — a concurrent claim hits the unique constraint and skips cleanly (shared isUniqueViolation helper). Stale 'processing' rows (worker crashed mid-ingest) are reclaimed after 10 min so no attachment is orphaned. NOT done (deliberate): the suggested UNIQUE on inbound_documents.file_sha256 — that column is a SOFT dedup key by design (manual re-uploads are kept as flagged 'duplicate' rows + duplicate_of_id for the Duplikat disposition); a unique index would break that feature. The file race only yields an extra 'unsorted' row (a data-quality nit, caught by the existing manual Duplikat backstop), not a double-count. Rationale to be added to the PR reply.
71 lines
3.3 KiB
JavaScript
71 lines
3.3 KiB
JavaScript
/**
|
|
* Migration 128: incoming mail (IMAP) support.
|
|
*
|
|
* - email_configs gains imap_* columns (a second config block alongside the
|
|
* outgoing smtp_* one; single row, same field shape).
|
|
* - `incomingMail` feature flag (default OFF, standalone).
|
|
* - received_emails: an audit log of messages the IMAP poller processed
|
|
* (dedupe key = message_id), mirroring the outgoing email_queue / "Sent
|
|
* emails" surface with a "Received emails" one.
|
|
*/
|
|
async function addColumn(knex, table, column, builder) {
|
|
if (!(await knex.schema.hasColumn(table, column))) {
|
|
await knex.schema.alterTable(table, builder);
|
|
}
|
|
}
|
|
|
|
exports.up = async function (knex) {
|
|
if (await knex.schema.hasTable('email_configs')) {
|
|
await addColumn(knex, 'email_configs', 'imap_host', (t) => t.string('imap_host', 255));
|
|
await addColumn(knex, 'email_configs', 'imap_port', (t) => t.integer('imap_port'));
|
|
await addColumn(knex, 'email_configs', 'imap_secure', (t) => t.boolean('imap_secure').notNullable().defaultTo(true));
|
|
await addColumn(knex, 'email_configs', 'imap_user', (t) => t.string('imap_user', 255));
|
|
await addColumn(knex, 'email_configs', 'imap_pass', (t) => t.string('imap_pass', 512));
|
|
await addColumn(knex, 'email_configs', 'imap_folder', (t) => t.string('imap_folder', 128).defaultTo('INBOX'));
|
|
}
|
|
|
|
if (await knex.schema.hasTable('feature_flags')) {
|
|
const existing = await knex('feature_flags').where({ key: 'incomingMail' }).first();
|
|
if (!existing) await knex('feature_flags').insert({ key: 'incomingMail', value: false });
|
|
}
|
|
|
|
if (!(await knex.schema.hasTable('received_emails'))) {
|
|
await knex.schema.createTable('received_emails', (table) => {
|
|
table.increments('id').primary();
|
|
table.string('message_id', 512);
|
|
table.string('from_address', 512);
|
|
table.text('subject');
|
|
table.timestamp('received_at');
|
|
table.integer('attachment_count').notNullable().defaultTo(0);
|
|
// ingested | no_attachment | duplicate | error
|
|
table.string('status', 24).notNullable().defaultTo('ingested');
|
|
table.integer('inbound_document_id').unsigned();
|
|
table.text('error');
|
|
table.timestamp('created_at').defaultTo(knex.fn.now());
|
|
// UNIQUE (not just INDEX): message_id is the dedup/claim key for the IMAP
|
|
// poller. The in-process `polling` lock serialises within one backend, but a
|
|
// second replica / rolling-deploy overlap would otherwise let two workers
|
|
// both pass the check-then-insert and double-ingest the same mail. NULLs stay
|
|
// distinct (Postgres + SQLite) so no-Message-ID rows aren't blocked. The
|
|
// intake claims this row BEFORE ingesting.
|
|
table.unique(['message_id']);
|
|
table.index(['status']);
|
|
});
|
|
}
|
|
};
|
|
|
|
exports.down = async function (knex) {
|
|
await knex.schema.dropTableIfExists('received_emails');
|
|
if (await knex.schema.hasTable('feature_flags')) {
|
|
await knex('feature_flags').where({ key: 'incomingMail' }).del();
|
|
}
|
|
if (await knex.schema.hasTable('email_configs')) {
|
|
for (const col of ['imap_host', 'imap_port', 'imap_secure', 'imap_user', 'imap_pass', 'imap_folder']) {
|
|
if (await knex.schema.hasColumn('email_configs', col)) {
|
|
// eslint-disable-next-line no-await-in-loop
|
|
await knex.schema.alterTable('email_configs', (t) => t.dropColumn(col));
|
|
}
|
|
}
|
|
}
|
|
};
|