feat(upload): async photo processing — backend (PR-B part 1)
Move thumbnail / EXIF / dimensions / watermark / webhook work off the
upload request thread and into a background worker pool. Upload
requests now return 202 in seconds even on NFS-backed storage; the
worker(s) drain the pending queue independently and update each
photo's processing_status to 'complete' or 'failed' on its own.
Schema (migration 085_async_photo_processing.js):
- photos.processing_status enum default 'complete' (existing
rows are already done)
- photos.processing_error populated on 'failed'
- photos.processing_started_at timestamp for janitor recovery
- photos.upload_id groups all photos from one upload
request so the frontend can poll
status by group
- indexes on processing_status and upload_id for queue lookups
services/photoProcessor.js
- queueFilesForProcessing(files, options) — shared helper used by
the admin and gallery upload routes. Moves files to final storage
+ inserts pending rows; returns { uploadId, photos, errors }.
- processPhoto(photoId) — worker-mode: reads original from storage
via withLocalCopy (transparent local/S3), generates thumbnail and
EXIF/dimensions or video metadata, queues watermark, fires
photo.uploaded webhook, marks 'complete'. Throws => caller marks
'failed' with the error message.
- processUploadedPhotos kept untouched — chunkedUploadService still
uses the synchronous path.
services/backgroundProcessor.js (new)
- N independent worker loops per backend instance (default 2,
UPLOAD_PROCESSOR_CONCURRENCY env override).
- Multi-pod safe: postgres SELECT FOR UPDATE SKIP LOCKED, sqlite
UPDATE-with-status-guard. Pods race on rows, exactly one wins.
- Janitor every minute resets photos stuck in 'processing' for >10
minutes (worker died, pod restarted) back to 'pending'.
- UPLOAD_PROCESSOR_DISABLED=true opt-out for CI/test.
- Started from server.js after the other long-running workers.
routes/adminPhotos.js — POST /:eventId/upload
- Replaced batch-of-25 sync processing loop with per-file
move-to-storage + insert-pending. Response is now 202 with
upload_id, count, photo_ids in addition to the legacy
successCount / replacedCount fields the existing frontend reads.
- Per-request temp directory cleanup is now a single idempotent
handler on res.finish/res.close (was three inline blocks for
error paths only, leaking dirs on success — original bug from
contributor analysis).
- GET /uploads/:upload_id/status — JSON snapshot of pending /
processing / complete / failed counts plus per-photo state.
- GET /uploads/:upload_id/stream — SSE upgrade. Polls internally
every 1.5s, emits on snapshot change, ends when all photos
reach a terminal state.
- POST /photos/:photoId/retry — flips a 'failed' photo back to
'pending' so the worker picks it up again.
- GET /:eventId/thumbnail/:photoId now returns 503 with Retry-After
while the photo is still pending/processing, and 422 on 'failed'.
The admin grid renders placeholders accordingly.
routes/gallery.js — POST /:eventId/upload (guest)
- Refactored to use queueFilesForProcessing instead of the synchronous
processUploadedPhotos. Same 202 + upload_id shape.
- GET /:slug/photos now filters processing_status to 'complete' (or
NULL for pre-migration rows) so guests never see in-flight photos.
Side-effect timing change:
- photo.uploaded webhook now fires from the worker after the photo
is actually processed (thumbnail + dimensions populated) instead
of from inside the upload request. Same payload fields. Worth a
one-line note in the changelog.
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Async photo-processing infrastructure.
|
||||
*
|
||||
* Adds:
|
||||
* - photos.processing_status — enum: pending | processing | complete | failed
|
||||
* - photos.processing_error — text, populated on 'failed'
|
||||
* - photos.processing_started_at — claim timestamp for janitor recovery
|
||||
* - photos.upload_id — groups all photos from one upload request
|
||||
* so the frontend can poll/stream by group
|
||||
*
|
||||
* All existing rows default to 'complete' (they were processed synchronously
|
||||
* before this migration and there's nothing pending). New uploads insert
|
||||
* with 'pending' and a background worker (services/backgroundProcessor.js)
|
||||
* picks them up.
|
||||
*
|
||||
* Partial-style indexes keep lookups fast as the queue drains. We use plain
|
||||
* indexes here instead of postgres-specific WHERE clauses so the migration
|
||||
* works on SQLite too; the workload (only-pending rows) keeps the index small.
|
||||
*/
|
||||
|
||||
exports.up = async function up(knex) {
|
||||
if (!(await knex.schema.hasTable('photos'))) return;
|
||||
|
||||
const hasStatus = await knex.schema.hasColumn('photos', 'processing_status');
|
||||
if (!hasStatus) {
|
||||
await knex.schema.alterTable('photos', (table) => {
|
||||
table.string('processing_status', 16).notNullable().defaultTo('complete');
|
||||
table.text('processing_error').nullable();
|
||||
table.timestamp('processing_started_at').nullable();
|
||||
table.string('upload_id', 64).nullable();
|
||||
});
|
||||
}
|
||||
|
||||
// Indexes — wrap in try/catch so re-running the migration on a partially
|
||||
// applied schema is a no-op rather than an error.
|
||||
try {
|
||||
await knex.schema.alterTable('photos', (table) => {
|
||||
table.index(['processing_status'], 'idx_photos_processing_status');
|
||||
});
|
||||
} catch (_) { /* already exists */ }
|
||||
|
||||
try {
|
||||
await knex.schema.alterTable('photos', (table) => {
|
||||
table.index(['upload_id'], 'idx_photos_upload_id');
|
||||
});
|
||||
} catch (_) { /* already exists */ }
|
||||
};
|
||||
|
||||
exports.down = async function down(knex) {
|
||||
if (!(await knex.schema.hasTable('photos'))) return;
|
||||
|
||||
// Drop indexes first (best-effort)
|
||||
try {
|
||||
await knex.schema.alterTable('photos', (t) => t.dropIndex([], 'idx_photos_upload_id'));
|
||||
} catch (_) { /* not present */ }
|
||||
try {
|
||||
await knex.schema.alterTable('photos', (t) => t.dropIndex([], 'idx_photos_processing_status'));
|
||||
} catch (_) { /* not present */ }
|
||||
|
||||
if (await knex.schema.hasColumn('photos', 'upload_id')) {
|
||||
await knex.schema.alterTable('photos', (t) => t.dropColumn('upload_id'));
|
||||
}
|
||||
if (await knex.schema.hasColumn('photos', 'processing_started_at')) {
|
||||
await knex.schema.alterTable('photos', (t) => t.dropColumn('processing_started_at'));
|
||||
}
|
||||
if (await knex.schema.hasColumn('photos', 'processing_error')) {
|
||||
await knex.schema.alterTable('photos', (t) => t.dropColumn('processing_error'));
|
||||
}
|
||||
if (await knex.schema.hasColumn('photos', 'processing_status')) {
|
||||
await knex.schema.alterTable('photos', (t) => t.dropColumn('processing_status'));
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user