diff --git a/backend/migrations/core/128_incoming_mail.js b/backend/migrations/core/128_incoming_mail.js new file mode 100644 index 00000000..b9a0f4f3 --- /dev/null +++ b/backend/migrations/core/128_incoming_mail.js @@ -0,0 +1,64 @@ +/** + * Migration 128: incoming mail (IMAP) support. + * + * - email_configs gains imap_* columns (a second config block alongside the + * outgoing smtp_* one; single row, same field shape). + * - `incomingMail` feature flag (default OFF, standalone). + * - received_emails: an audit log of messages the IMAP poller processed + * (dedupe key = message_id), mirroring the outgoing email_queue / "Sent + * emails" surface with a "Received emails" one. + */ +async function addColumn(knex, table, column, builder) { + if (!(await knex.schema.hasColumn(table, column))) { + await knex.schema.alterTable(table, builder); + } +} + +exports.up = async function (knex) { + if (await knex.schema.hasTable('email_configs')) { + await addColumn(knex, 'email_configs', 'imap_host', (t) => t.string('imap_host', 255)); + await addColumn(knex, 'email_configs', 'imap_port', (t) => t.integer('imap_port')); + await addColumn(knex, 'email_configs', 'imap_secure', (t) => t.boolean('imap_secure').notNullable().defaultTo(true)); + await addColumn(knex, 'email_configs', 'imap_user', (t) => t.string('imap_user', 255)); + await addColumn(knex, 'email_configs', 'imap_pass', (t) => t.string('imap_pass', 512)); + await addColumn(knex, 'email_configs', 'imap_folder', (t) => t.string('imap_folder', 128).defaultTo('INBOX')); + } + + if (await knex.schema.hasTable('feature_flags')) { + const existing = await knex('feature_flags').where({ key: 'incomingMail' }).first(); + if (!existing) await knex('feature_flags').insert({ key: 'incomingMail', value: false }); + } + + if (!(await knex.schema.hasTable('received_emails'))) { + await knex.schema.createTable('received_emails', (table) => { + table.increments('id').primary(); + table.string('message_id', 512); + table.string('from_address', 512); + table.text('subject'); + table.timestamp('received_at'); + table.integer('attachment_count').notNullable().defaultTo(0); + // ingested | no_attachment | duplicate | error + table.string('status', 24).notNullable().defaultTo('ingested'); + table.integer('inbound_document_id').unsigned(); + table.text('error'); + table.timestamp('created_at').defaultTo(knex.fn.now()); + table.index(['message_id']); + table.index(['status']); + }); + } +}; + +exports.down = async function (knex) { + await knex.schema.dropTableIfExists('received_emails'); + if (await knex.schema.hasTable('feature_flags')) { + await knex('feature_flags').where({ key: 'incomingMail' }).del(); + } + if (await knex.schema.hasTable('email_configs')) { + for (const col of ['imap_host', 'imap_port', 'imap_secure', 'imap_user', 'imap_pass', 'imap_folder']) { + if (await knex.schema.hasColumn('email_configs', col)) { + // eslint-disable-next-line no-await-in-loop + await knex.schema.alterTable('email_configs', (t) => t.dropColumn(col)); + } + } + } +}; diff --git a/backend/package-lock.json b/backend/package-lock.json index 47878f76..04c4e230 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -1,12 +1,12 @@ { "name": "picpeak-backend", - "version": "3.47.2-beta.0", + "version": "3.60.6-beta.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "picpeak-backend", - "version": "3.47.2-beta.0", + "version": "3.60.6-beta.0", "dependencies": { "@aws-sdk/client-s3": "^3.850.0", "@aws-sdk/lib-storage": "^3.850.0", @@ -29,11 +29,13 @@ "i18next": "25.3.2", "i18next-browser-languagedetector": "^8.2.0", "i18next-http-backend": "^3.0.2", + "imapflow": "^1.4.0", "ipaddr.js": "^2.3.0", "joi": "^17.9.1", "js-yaml": "^4.1.1", "jsonwebtoken": "^9.0.0", "knex": "^2.4.2", + "mailparser": "^3.9.9", "mime-types": "^3.0.1", "multer": "^2.0.2", "node-cron": "^3.0.2", @@ -2725,6 +2727,12 @@ "pako": "^1.0.10" } }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", + "license": "MIT" + }, "node_modules/@scarf/scarf": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@scarf/scarf/-/scarf-1.4.0.tgz", @@ -2732,6 +2740,22 @@ "hasInstallScript": true, "license": "Apache-2.0" }, + "node_modules/@selderee/plugin-htmlparser2": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.12.0.tgz", + "integrity": "sha512-oELmoyA6ML9jDRMV3kgcMQFKxUfBU0yFVn6yTctVaLT5ygXnxH52I3TZEgV9EhXJC68/uFvE5Daj1/25c0Xa/A==", + "license": "MIT", + "dependencies": { + "domelementtype": "~2.3.0", + "domhandler": "~5.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/KillyMXI" + }, + "peerDependencies": { + "selderee": "~0.12.0" + } + }, "node_modules/@sideway/address": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz", @@ -3722,6 +3746,17 @@ "dev": true, "license": "ISC" }, + "node_modules/@zone-eu/mailsplit": { + "version": "5.4.12", + "resolved": "https://registry.npmjs.org/@zone-eu/mailsplit/-/mailsplit-5.4.12.tgz", + "integrity": "sha512-w7Gy+NvjZ0MiXm8F6zfjImAqcTONKDImgWVBjDKQVFUXWuz3VFM5levNArkL2M877ajql5+bkS2pDV56injlmg==", + "license": "(MIT OR EUPL-1.1+)", + "dependencies": { + "libbase64": "1.3.0", + "libmime": "5.3.8", + "libqp": "2.1.1" + } + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -4047,6 +4082,15 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", "license": "MIT" }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/axios": { "version": "1.15.2", "resolved": "https://registry.npmjs.org/axios/-/axios-1.15.2.tgz", @@ -5087,6 +5131,15 @@ "node": ">=0.10.0" } }, + "node_modules/deepmerge-ts": { + "version": "7.1.5", + "resolved": "https://registry.npmjs.org/deepmerge-ts/-/deepmerge-ts-7.1.5.tgz", + "integrity": "sha512-HOJkrhaYsweh+W+e74Yn7YStZOilkoPb6fycpwNLKzSPtruFs48nYis0zy5yJz1+ktUhHxoRDJ27RQAWLIJVJw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -5329,6 +5382,15 @@ "node": ">= 0.8" } }, + "node_modules/encoding-japanese": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/encoding-japanese/-/encoding-japanese-2.2.0.tgz", + "integrity": "sha512-EuJWwlHPZ1LbADuKTClvHtwbaFn4rOD+dRAbWysqEOXRc2Uui0hJInNJrsdH0c+OhJA4nrCBdSkW4DD5YxAo6A==", + "license": "MIT", + "engines": { + "node": ">=8.10.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", @@ -6531,6 +6593,15 @@ "node": ">= 0.4" } }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, "node_modules/helmet": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/helmet/-/helmet-7.2.0.tgz", @@ -6556,6 +6627,56 @@ "void-elements": "3.1.0" } }, + "node_modules/html-to-text": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/html-to-text/-/html-to-text-10.0.0.tgz", + "integrity": "sha512-2OH59Gtprdczel+7Rxgpz9hGVJREaf8Lt1H4kZwWHpEn70VQKRuMNGsb2eDbwaTzrYzb0hheiOG1P7Dim0B4dQ==", + "license": "MIT", + "dependencies": { + "@selderee/plugin-htmlparser2": "~0.12.0", + "deepmerge-ts": "^7.1.5", + "dom-serializer": "^2.0.0", + "htmlparser2": "^10.1.0", + "selderee": "~0.12.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/sponsors/KillyMXI" + } + }, + "node_modules/html-to-text/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/html-to-text/node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, "node_modules/htmlparser2": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", @@ -6700,6 +6821,22 @@ "cross-fetch": "4.1.0" } }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -6737,6 +6874,23 @@ "dev": true, "license": "ISC" }, + "node_modules/imapflow": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/imapflow/-/imapflow-1.4.0.tgz", + "integrity": "sha512-bpNWv3AwzZryMMYoKiqPebcxmldCQwWxqhBQ5b/nTlJYAgexCzDIpN0LdhVJpJp6H25lAmwvShX+Fu/9AU5Spg==", + "license": "MIT", + "dependencies": { + "@zone-eu/mailsplit": "5.4.12", + "encoding-japanese": "2.2.0", + "iconv-lite": "0.7.2", + "libbase64": "1.3.0", + "libmime": "5.3.8", + "libqp": "2.1.1", + "nodemailer": "8.0.10", + "pino": "10.3.1", + "socks": "2.8.9" + } + }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", @@ -6827,7 +6981,6 @@ "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz", "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==", "license": "MIT", - "optional": true, "engines": { "node": ">= 12" } @@ -7972,6 +8125,15 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/leac": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/leac/-/leac-0.7.0.tgz", + "integrity": "sha512-qMrZeyEekgdRQ9o6a4NAB2EQZrv827GJdn1vnapwSJ90hWRB4TzUSunvacPkxQ2TnNqHNI1/zSt0hlo0crG8Jw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/KillyMXI" + } + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -7996,6 +8158,30 @@ "node": ">= 0.8.0" } }, + "node_modules/libbase64": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-1.3.0.tgz", + "integrity": "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg==", + "license": "MIT" + }, + "node_modules/libmime": { + "version": "5.3.8", + "resolved": "https://registry.npmjs.org/libmime/-/libmime-5.3.8.tgz", + "integrity": "sha512-ZrCY+Q66mPvasAfjsQ/IgahzoBvfE1VdtGRpo1hwRB1oK3wJKxhKA3GOcd2a6j7AH5eMFccxK9fBoCpRZTf8ng==", + "license": "MIT", + "dependencies": { + "encoding-japanese": "2.2.0", + "iconv-lite": "0.7.2", + "libbase64": "1.3.0", + "libqp": "2.1.1" + } + }, + "node_modules/libqp": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/libqp/-/libqp-2.1.1.tgz", + "integrity": "sha512-0Wd+GPz1O134cP62YU2GTOPNA7Qgl09XwCqM5zpBv87ERCXdfDtyKXvV7c9U22yWJh44QZqBocFnXN11K96qow==", + "license": "MIT" + }, "node_modules/linebreak": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/linebreak/-/linebreak-1.1.0.tgz", @@ -8022,6 +8208,25 @@ "dev": true, "license": "MIT" }, + "node_modules/linkify-it": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.1.tgz", + "integrity": "sha512-wVoTjP4Q6R0NW5hiZkVJaFZPWgtXfoGF+6LucL3/FtiNjmcHhYjEr5f1Kqjirc1nBW07J/ZuRFumqr2oqccEWg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/markdown-it" + } + ], + "license": "MIT", + "dependencies": { + "uc.micro": "^2.0.0" + } + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -8164,6 +8369,24 @@ "yallist": "^3.0.2" } }, + "node_modules/mailparser": { + "version": "3.9.9", + "resolved": "https://registry.npmjs.org/mailparser/-/mailparser-3.9.9.tgz", + "integrity": "sha512-ulZi7h1eKm8WQmXibIgj8dmMQGDQCUS/g+XHkxxjcLDq4Dwn2ppo+0hz5Fi+ltvu4eN7mh3ykIp5RcpiWWav1w==", + "license": "MIT", + "dependencies": { + "@zone-eu/mailsplit": "5.4.12", + "encoding-japanese": "2.2.0", + "he": "1.2.0", + "html-to-text": "10.0.0", + "iconv-lite": "0.7.2", + "libmime": "5.3.8", + "linkify-it": "5.0.1", + "nodemailer": "8.0.10", + "punycode.js": "2.3.1", + "tlds": "1.261.0" + } + }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", @@ -8836,9 +9059,9 @@ "license": "MIT" }, "node_modules/nodemailer": { - "version": "8.0.7", - "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-8.0.7.tgz", - "integrity": "sha512-pkjE4mkBzQjdJT4/UmlKl3pX0rC9fZmjh7c6C9o7lv66Ac6w9WCnzPzhbPNxwZAzlF4mdq4CSWB5+FbK6FWCow==", + "version": "8.0.10", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-8.0.10.tgz", + "integrity": "sha512-BLFuSth7QtHOkBzyqTehWWyub0NTRDuK2Q2SQfnGLsrJnzyU+Yeh4WpV1eZGuARFj1xQJHIdnTuJZLP+b9R1GQ==", "license": "MIT-0", "engines": { "node": ">=6.0.0" @@ -9036,6 +9259,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -9214,6 +9446,19 @@ "integrity": "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==", "license": "MIT" }, + "node_modules/parseley": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.13.1.tgz", + "integrity": "sha512-uNBJZzmb60l6p6VWLTmevizNAGnE0xoSf1n0B4q3ntegDNzcS68NRCcBDZTcyXHxt2XhBChsCuqj4M+nChvE/A==", + "license": "MIT", + "dependencies": { + "leac": "^0.7.0", + "peberminta": "^0.10.0" + }, + "funding": { + "url": "https://github.com/sponsors/KillyMXI" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -9318,6 +9563,15 @@ "png-js": "^1.0.0" } }, + "node_modules/peberminta": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/peberminta/-/peberminta-0.10.0.tgz", + "integrity": "sha512-80B2AsU+I4Qdb0ZAPSfe9UwvGzwkM37IKIFEvdS3D/3Ndgv2bsuJ0bfG1+iEYO+l7Gfd4EUJmuRyq7efLgRMzQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/KillyMXI" + } + }, "node_modules/pg": { "version": "8.16.3", "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.3.tgz", @@ -9432,6 +9686,43 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pino": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", + "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz", + "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", + "license": "MIT" + }, "node_modules/pirates": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", @@ -9665,6 +9956,22 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, "node_modules/promise-inflight": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", @@ -9758,6 +10065,15 @@ "node": ">=6" } }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/pure-rand": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", @@ -9966,6 +10282,12 @@ ], "license": "MIT" }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -10078,6 +10400,15 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, "node_modules/rechoir": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", @@ -10285,6 +10616,18 @@ "postcss": "^8.3.11" } }, + "node_modules/selderee": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.12.0.tgz", + "integrity": "sha512-b1YMh3+DHZp59DLna3qVwQ5iOla/nrI6mLBNW02XxU77M3046Df6VLkoaJyFz20VsGIG5kkp+FK0kg4K4HnUFw==", + "license": "MIT", + "dependencies": { + "parseley": "~0.13.1" + }, + "funding": { + "url": "https://github.com/sponsors/KillyMXI" + } + }, "node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -10628,20 +10971,18 @@ "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", "license": "MIT", - "optional": true, "engines": { "node": ">= 6.0.0", "npm": ">= 3.0.0" } }, "node_modules/socks": { - "version": "2.8.7", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", - "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "version": "2.8.9", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz", + "integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==", "license": "MIT", - "optional": true, "dependencies": { - "ip-address": "^10.0.1", + "ip-address": "^10.1.1", "smart-buffer": "^4.2.0" }, "engines": { @@ -10664,6 +11005,15 @@ "node": ">= 10" } }, + "node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -11284,6 +11634,24 @@ "dev": true, "license": "MIT" }, + "node_modules/thread-stream": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", + "integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==", + "license": "MIT", + "dependencies": { + "real-require": "^1.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/thread-stream/node_modules/real-require": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz", + "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==", + "license": "MIT" + }, "node_modules/tildify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", @@ -11299,6 +11667,15 @@ "integrity": "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==", "license": "MIT" }, + "node_modules/tlds": { + "version": "1.261.0", + "resolved": "https://registry.npmjs.org/tlds/-/tlds-1.261.0.tgz", + "integrity": "sha512-QXqwfEl9ddlGBaRFXIvNKK6OhipSiLXuRuLJX5DErz0o0Q0rYxulWLdFryTkV5PkdZct5iMInwYEGe/eR++1AA==", + "license": "MIT", + "bin": { + "tlds": "bin.js" + } + }, "node_modules/tmpl": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", @@ -11447,6 +11824,12 @@ "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", "license": "MIT" }, + "node_modules/uc.micro": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", + "license": "MIT" + }, "node_modules/undefsafe": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", diff --git a/backend/package.json b/backend/package.json index 82401815..a97036ed 100644 --- a/backend/package.json +++ b/backend/package.json @@ -35,11 +35,13 @@ "i18next": "25.3.2", "i18next-browser-languagedetector": "^8.2.0", "i18next-http-backend": "^3.0.2", + "imapflow": "^1.4.0", "ipaddr.js": "^2.3.0", "joi": "^17.9.1", "js-yaml": "^4.1.1", "jsonwebtoken": "^9.0.0", "knex": "^2.4.2", + "mailparser": "^3.9.9", "mime-types": "^3.0.1", "multer": "^2.0.2", "node-cron": "^3.0.2", diff --git a/backend/server.js b/backend/server.js index 5c823e74..1327d18d 100644 --- a/backend/server.js +++ b/backend/server.js @@ -836,7 +836,16 @@ async function startServer() { logger.warn('Email template self-heal failed at boot:', err.message); } startEmailQueueProcessor(); - + + // Start incoming-mail (IMAP) poller — no-ops each minute unless the + // `incomingMail` flag is on and a mailbox is configured (migration 128). + try { + const { startIncomingMailPoller } = require('./src/services/emailIntakeService'); + startIncomingMailPoller(); + } catch (err) { + logger.warn('Incoming-mail poller failed to start:', err.message); + } + // Start webhook delivery worker (#327) const { startWebhookDeliveryWorker } = require('./src/services/webhookDeliveryWorker'); startWebhookDeliveryWorker(); diff --git a/backend/src/routes/adminEmail.js b/backend/src/routes/adminEmail.js index d880b884..2a7281b0 100644 --- a/backend/src/routes/adminEmail.js +++ b/backend/src/routes/adminEmail.js @@ -118,6 +118,74 @@ router.post('/config', [ } }); +// ── Incoming mail (IMAP) config — a second block alongside outgoing SMTP ── +router.get('/incoming-config', adminAuth, requirePermission('email.view'), async (req, res) => { + try { + const c = await db('email_configs').first(); + res.json({ + imap_host: c?.imap_host || '', + imap_port: c?.imap_port || 993, + imap_secure: c?.imap_secure !== false, + imap_user: c?.imap_user || '', + imap_pass: c?.imap_pass ? '********' : '', // never send the real password + imap_folder: c?.imap_folder || 'INBOX', + }); + } catch (error) { + console.error('Incoming mail config fetch error:', error); + res.status(500).json({ error: 'Failed to fetch incoming mail configuration' }); + } +}); + +router.post('/incoming-config', [ + adminAuth, + requirePermission('email.edit'), + body('imap_host').notEmpty().withMessage('IMAP host is required'), + body('imap_port').isInt({ min: 1, max: 65535 }).withMessage('Invalid port number'), +], async (req, res) => { + try { + const errors = validationResult(req); + if (!errors.isEmpty()) return res.status(400).json({ errors: errors.array() }); + const { imap_host, imap_port, imap_secure, imap_user, imap_pass, imap_folder } = req.body; + const { isPrivateIP } = require('../utils/networkValidation'); + if (isPrivateIP(imap_host)) { + return res.status(400).json({ error: 'IMAP host cannot point to a private or internal network address' }); + } + const existing = await db('email_configs').first(); + const data = { + imap_host, + imap_port: parseInt(imap_port), + imap_secure: imap_secure || false, + imap_user: imap_user || '', + imap_folder: imap_folder || 'INBOX', + updated_at: new Date(), + }; + if (imap_pass && imap_pass !== '********') data.imap_pass = imap_pass; + if (existing) await db('email_configs').where('id', existing.id).update(data); + else await db('email_configs').insert(data); + await logActivity('incoming_mail_config_updated', { imap_host }, null, { type: 'admin', id: req.admin.id, name: req.admin.username }); + res.json({ message: 'Incoming mail configuration updated successfully' }); + } catch (error) { + console.error('Incoming mail config update error:', error); + res.status(500).json({ error: 'Failed to update incoming mail configuration' }); + } +}); + +// Received-emails log (the IMAP poller's audit trail) — "Received emails" tab. +router.get('/received', adminAuth, requirePermission('email.view'), async (req, res) => { + try { + const page = Math.max(1, parseInt(req.query.page, 10) || 1); + const pageSize = Math.min(100, Math.max(1, parseInt(req.query.pageSize, 10) || 25)); + const base = db('received_emails'); + const countRow = await base.clone().count({ c: '*' }).first(); + const total = parseInt(countRow?.c || 0, 10); + const items = await base.clone().orderBy('received_at', 'desc').limit(pageSize).offset((page - 1) * pageSize); + res.json({ items, pagination: { page, pageSize, total, totalPages: Math.ceil(total / pageSize) } }); + } catch (error) { + console.error('Received emails fetch error:', error); + res.status(500).json({ error: 'Failed to fetch received emails' }); + } +}); + // Test email configuration router.post('/test', adminAuth, requirePermission('email.send'), async (req, res) => { try { diff --git a/backend/src/routes/adminFeatureFlags.js b/backend/src/routes/adminFeatureFlags.js index 03508a18..3f655b26 100644 --- a/backend/src/routes/adminFeatureFlags.js +++ b/backend/src/routes/adminFeatureFlags.js @@ -25,6 +25,9 @@ const logger = require('../utils/logger'); const KNOWN_FLAGS = [ 'galleries', 'reminderEmails', + // Incoming mail (migration 128) — IMAP polling of a dedicated mailbox into + // the incoming-invoices inbox. Standalone toggle. + 'incomingMail', 'calendar', 'calendarBooking', 'quotes', @@ -79,6 +82,7 @@ const KNOWN_FLAGS = [ // new release that hasn't run its migration yet on this instance). const DEFAULT_FLAGS = { galleries: true, + incomingMail: false, // F.3 — reminderEmails is a placeholder card in the Features tab // (lockedReason: NOT_YET_AVAILABLE). Default FALSE so it matches // the locked-but-off visual state of messaging / calendarBooking diff --git a/backend/src/services/emailIntakeService.js b/backend/src/services/emailIntakeService.js new file mode 100644 index 00000000..1c76b62c --- /dev/null +++ b/backend/src/services/emailIntakeService.js @@ -0,0 +1,123 @@ +/** + * Incoming-mail intake (migration 128). Polls the configured IMAP mailbox + * every minute, parses each unseen message, and drops PDF/image attachments + * into the incoming-invoices inbox (inbound_documents, source='email'). + * + * Gated by the `incomingMail` feature flag. Idempotent: each message is logged + * in received_emails keyed by message-id (skip if seen); duplicate attachments + * are caught downstream by the inbound_documents SHA-256 dedup. Handles + * forwarded messages because mailparser flattens nested attachments. + */ +const fsp = require('fs').promises; +const path = require('path'); +const { ImapFlow } = require('imapflow'); +const { simpleParser } = require('mailparser'); +const { db } = require('../database/db'); +const logger = require('../utils/logger'); +const { getStoragePath } = require('../config/storage'); +const expenseService = require('./expenseService'); + +const ALLOWED_MIME = ['application/pdf', 'image/jpeg', 'image/png']; +let polling = false; + +async function isEnabled() { + const flag = await db('feature_flags').where({ key: 'incomingMail' }).first(); + return !!(flag && (flag.value === true || flag.value === 1 || flag.value === '1')); +} + +async function getImapConfig() { + const c = await db('email_configs').first(); + if (!c || !c.imap_host || !c.imap_user) return null; + return { + host: c.imap_host, + port: c.imap_port || 993, + secure: c.imap_secure !== false && c.imap_secure !== 0, + auth: { user: c.imap_user, pass: c.imap_pass || '' }, + folder: c.imap_folder || 'INBOX', + }; +} + +async function saveAttachment(att) { + const year = new Date().getFullYear(); + const dir = path.join(getStoragePath(), 'business-docs', 'inbound', String(year)); + await fsp.mkdir(dir, { recursive: true }); + const ext = path.extname(att.filename || '') + || (att.contentType === 'application/pdf' ? '.pdf' : att.contentType === 'image/png' ? '.png' : '.jpg'); + const filePath = path.join(dir, `email-${Date.now()}-${Math.floor(Math.random() * 1e6)}${ext}`); + await fsp.writeFile(filePath, att.content); + return filePath; +} + +/** Poll the mailbox once. Safe to call repeatedly; self-skips when busy/off. */ +async function pollOnce() { + if (polling) return { skipped: 'busy' }; + if (!(await isEnabled())) return { skipped: 'disabled' }; + const cfg = await getImapConfig(); + if (!cfg) return { skipped: 'unconfigured' }; + + polling = true; + const client = new ImapFlow({ host: cfg.host, port: cfg.port, secure: cfg.secure, auth: cfg.auth, logger: false }); + let processed = 0; + try { + await client.connect(); + const lock = await client.getMailboxLock(cfg.folder); + try { + // eslint-disable-next-line no-restricted-syntax + for await (const msg of client.fetch({ seen: false }, { source: true, uid: true })) { + try { + const parsed = await simpleParser(msg.source); + const messageId = parsed.messageId || `uid-${cfg.folder}-${msg.uid}`; + const seen = await db('received_emails').where({ message_id: messageId }).first(); + if (seen) { await client.messageFlagsAdd(msg.uid, ['\\Seen'], { uid: true }); continue; } + + const atts = (parsed.attachments || []).filter((a) => ALLOWED_MIME.includes(a.contentType)); + let inboundId = null; + let count = 0; + for (const att of atts) { + // eslint-disable-next-line no-await-in-loop + const filePath = await saveAttachment(att); + // eslint-disable-next-line no-await-in-loop + const doc = await expenseService.recordInboundDocument({ source: 'email', filePath, originalFilename: att.filename || 'attachment', mimeType: att.contentType }, null); + inboundId = doc.id; count += 1; + } + await db('received_emails').insert({ + message_id: messageId, + from_address: (parsed.from && parsed.from.text) || null, + subject: parsed.subject || null, + received_at: parsed.date || new Date(), + attachment_count: count, + status: count > 0 ? 'ingested' : 'no_attachment', + inbound_document_id: inboundId, + created_at: new Date(), + }); + await client.messageFlagsAdd(msg.uid, ['\\Seen'], { uid: true }); + processed += 1; + } catch (e) { + logger.error?.(`emailIntake: message uid ${msg.uid} failed: ${e.message}`); + try { + await db('received_emails').insert({ message_id: `err-${msg.uid}-${Date.now()}`, status: 'error', error: e.message, attachment_count: 0, received_at: new Date(), created_at: new Date() }); + } catch (_e) { /* ignore */ } + } + } + } finally { + lock.release(); + } + await client.logout(); + } catch (e) { + logger.error?.(`emailIntake: poll failed: ${e.message}`); + try { await client.close(); } catch (_e) { /* ignore */ } + } finally { + polling = false; + } + return { processed }; +} + +/** Start the 1-minute poll loop (mirrors the outgoing queue cadence). */ +function startIncomingMailPoller() { + const run = () => pollOnce().catch((e) => logger.error?.(`emailIntake: ${e.message}`)); + setTimeout(run, 15000); // first run shortly after boot + setInterval(run, 60 * 1000); + logger.info?.('Incoming-mail poller started (every 60s when enabled)'); +} + +module.exports = { pollOnce, startIncomingMailPoller, _internal: { getImapConfig, isEnabled, saveAttachment } };