* fix(external-media): one row per external file per event (#1162) Stable twin of #1167. Two overlapping import-external runs against the same event inserted every file twice. The route checked for an existing external_relpath and then inserted, with an fs.stat and a sharp().metadata() read sitting in between — a window wide enough for both runs to see "not there". A reporter's event held 8004 rows for 6012 distinct paths. Nothing at the storage layer stopped it: migration 041 created only a NON-unique (event_id, source_origin) index. - migration 176 removes the existing duplicates and adds a partial unique index on (event_id, external_relpath), verified against the catalog afterwards — a failed CREATE INDEX raises 23505 on Postgres, which run-migrations-safe treats as "schema already exists" and would record as applied on an install that never got the index. - dependent rows are removed explicitly rather than by cascade: PicPeak never sets `PRAGMA foreign_keys = ON`, so on SQLite the declared CASCADE is inert and a bare delete strands feedback and access-log rows. Guest feedback moves to the survivor instead of being discarded, keyed on guest identity the way feedbackService defines it, and the survivor's denormalized counters are recomputed. - the route treats a unique violation as a skip, so a writer this process cannot see converges instead of duplicating, and a second import while one is running gets a 409. - a .picpeak taken before migration 176 carries exactly these duplicates, and suspending FK enforcement does not suspend a unique index — so the restore drops the index for the load and rebuilds it after running the same dedupe. Divergences from the main twin, both because the feature is absent here: faces (no faceProcessor, so no purgePhotoFaces reconciliation — the rows are still deleted so nothing dangles), admin marks, transfer membership, and photos.view_count/download_count. The service guards each on hasTable / hasColumn, so those branches simply do not fire. Verified on this branch: 36 new tests pass; full suite leaves the same 5 pre-existing failures as origin/stable, unchanged. * fix(external-media): invalidate the download zip when duplicates are removed (#1162) External review. Same fix as the main twin. The pre-built "download everything" archive still contained the duplicate rows the dedupe had just deleted, so guests kept receiving them. Every ordinary photo-deletion path calls downloadZipService.invalidate for exactly this reason. The columns are cleared rather than the service being called: that service carries debounce timers and a regeneration queue, which a migration should not start. getZipInfo already treats a cleared record as a cache miss and rebuilds on the next request. The stale object is left in storage, as elsewhere. --------- Co-authored-by: Paul Nothaft <[email protected]>
This commit is contained in:
Paul Nothaft
co-authored by
Paul Nothaft
Paul Nothaft
parent
f83d144f28
commit
e9fcf4960e
@@ -9,9 +9,24 @@ const { db, logActivity } = require('../database/db');
|
||||
const sharp = require('sharp');
|
||||
const logger = require('../utils/logger');
|
||||
const { generateThumbnail } = require('../services/imageProcessor');
|
||||
const { isUniqueViolation } = require('../utils/dbErrors');
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// Events with an import running in THIS process (#1162).
|
||||
//
|
||||
// The second line of defence, not the first: migration 176 puts a unique index
|
||||
// on (event_id, external_relpath), and that is what actually makes a duplicate
|
||||
// impossible — it holds across replicas, across restarts, and against anything
|
||||
// that inserts external rows without going through this route.
|
||||
//
|
||||
// This set exists for the reason the duplicates got filed in the first place:
|
||||
// a large tree takes long enough that the run LOOKS hung, so admins click
|
||||
// again. Letting that second run walk the whole tree only to have every insert
|
||||
// bounce off the index wastes minutes of CPU and reports a nonsense
|
||||
// `skipped: 6012` back. Failing it immediately with 409 says what happened.
|
||||
const importsInFlight = new Set();
|
||||
|
||||
// GET /api/admin/external-media/list?path=relative/dir
|
||||
router.get('/list', adminAuth, requirePermission('photos.view'), async (req, res) => {
|
||||
try {
|
||||
@@ -50,8 +65,14 @@ async function walkDir(dir, baseDir) {
|
||||
// POST /api/admin/events/:id/import-external
|
||||
// Body: { external_path: string, recursive?: boolean, map?: { individual?: string, collages?: string } }
|
||||
router.post('/events/:id/import-external', adminAuth, requirePermission('photos.upload'), requireEventOwnership, async (req, res) => {
|
||||
const eventId = parseInt(req.params.id);
|
||||
if (importsInFlight.has(eventId)) {
|
||||
return res.status(409).json({
|
||||
error: 'An import is already running for this event. Wait for it to finish before starting another.'
|
||||
});
|
||||
}
|
||||
importsInFlight.add(eventId);
|
||||
try {
|
||||
const eventId = parseInt(req.params.id);
|
||||
const { external_path, recursive = true, map = { individual: 'individual', collages: 'collages' } } = req.body || {};
|
||||
if (!external_path) return res.status(400).json({ error: 'external_path is required' });
|
||||
|
||||
@@ -107,7 +128,13 @@ router.post('/events/:id/import-external', adminAuth, requirePermission('photos.
|
||||
if (segs[0] === map.individual) type = 'individual';
|
||||
|
||||
try {
|
||||
// Check if already exists (by external_relpath)
|
||||
// Fast path only. This SELECT settles the common case — a re-import of
|
||||
// a folder already in the event — without paying for a stat and a
|
||||
// Sharp metadata read per file. It is NOT the guard: those two calls
|
||||
// sit between here and the INSERT below, which is exactly the window
|
||||
// two overlapping imports both walked through (#1162). The unique
|
||||
// index from migration 176 is the guard, and the catch below is how
|
||||
// this loop converges when it fires.
|
||||
const exists = await db('photos')
|
||||
.where({ event_id: eventId, external_relpath: f.rel })
|
||||
.first();
|
||||
@@ -125,21 +152,33 @@ router.post('/events/:id/import-external', adminAuth, requirePermission('photos.
|
||||
logger.warn(`Could not extract dimensions for ${f.rel}: ${dimErr.message}`);
|
||||
}
|
||||
|
||||
const inserted = await db('photos')
|
||||
.insert({
|
||||
event_id: eventId,
|
||||
filename: f.name,
|
||||
// Keep path as a hint for legacy code but not used for resolution in external mode
|
||||
path: path.join(event.slug, f.name),
|
||||
thumbnail_path: null,
|
||||
type,
|
||||
size_bytes: stats.size,
|
||||
width,
|
||||
height,
|
||||
source_origin: 'external',
|
||||
external_relpath: f.rel
|
||||
})
|
||||
.returning('id');
|
||||
let inserted;
|
||||
try {
|
||||
inserted = await db('photos')
|
||||
.insert({
|
||||
event_id: eventId,
|
||||
filename: f.name,
|
||||
// Keep path as a hint for legacy code but not used for resolution in external mode
|
||||
path: path.join(event.slug, f.name),
|
||||
thumbnail_path: null,
|
||||
type,
|
||||
size_bytes: stats.size,
|
||||
width,
|
||||
height,
|
||||
source_origin: 'external',
|
||||
external_relpath: f.rel
|
||||
})
|
||||
.returning('id');
|
||||
} catch (insertErr) {
|
||||
// Another writer inserted this exact path while we were reading
|
||||
// metadata. That is the outcome the index exists to produce, and it
|
||||
// is a skip rather than a failure — the row is there, it just isn't
|
||||
// ours. Counting it as `skipped` keeps the reported totals honest;
|
||||
// before the index this landed in the outer catch as a nameless
|
||||
// failure, or (more often) never fired at all and duplicated the row.
|
||||
if (isUniqueViolation(insertErr)) { skipped++; continue; }
|
||||
throw insertErr;
|
||||
}
|
||||
|
||||
const photoId = Array.isArray(inserted) && inserted.length
|
||||
? (typeof inserted[0] === 'object' ? inserted[0].id : inserted[0])
|
||||
@@ -193,6 +232,11 @@ router.post('/events/:id/import-external', adminAuth, requirePermission('photos.
|
||||
error: error.message
|
||||
});
|
||||
res.status(500).json({ error: 'Failed to import external media' });
|
||||
} finally {
|
||||
// In `finally` and not at the end of `try`: an import that throws must
|
||||
// still release the event, or a single failure locks out every retry
|
||||
// until the process restarts.
|
||||
importsInFlight.delete(eventId);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
/**
|
||||
* One row per external file per event (#1162).
|
||||
*
|
||||
* The import route used to check for an existing external_relpath and then
|
||||
* insert, with an fs.stat and a Sharp decode in between — wide enough that two
|
||||
* overlapping imports both walked through it. A reporter's event held 8004 rows
|
||||
* for 6012 distinct paths.
|
||||
*
|
||||
* This lives in a service rather than inside migration 176 because it has two
|
||||
* callers. The migration is one. The other is a .picpeak restore: the archive
|
||||
* carries the photos table verbatim, so a backup taken before this fix lands
|
||||
* duplicate rows into a schema that now has a unique index on them — and
|
||||
* neither Postgres' `session_replication_role = replica` nor SQLite's
|
||||
* `defer_foreign_keys` disables a UNIQUE index, so batchInsert would abort the
|
||||
* whole restore after every table had already been emptied.
|
||||
*
|
||||
* DELETING dependent rows explicitly, rather than trusting ON DELETE CASCADE,
|
||||
* is the load-bearing part. Every FK into photos declares CASCADE, but PicPeak
|
||||
* does not set `PRAGMA foreign_keys = ON` — the codebase says so in as many
|
||||
* words where it deletes an event (adminEvents/helpers.js:245-249) — so on
|
||||
* every SQLite install the cascade is inert and a bare delete would leave
|
||||
* dangling face embeddings, feedback and marks behind. The same reason
|
||||
* `hero_photo_id` is repointed by hand: its SET NULL is inert there too, so
|
||||
* without it a SQLite install keeps a hero pointing at a row that is gone.
|
||||
*
|
||||
* Guest and admin state is MOVED to the survivor where it can be, not
|
||||
* discarded. The duplicates were separate tiles in the grid, so a guest's
|
||||
* comment or an admin's rating could legitimately be attached to either, and
|
||||
* silently deleting it inside a fix for silent data loss would be its own bug.
|
||||
* Where the target already holds an equivalent row — the same guest's like on
|
||||
* the same photo, the same admin's mark, the same transfer's entry — the loser
|
||||
* is dropped instead, because those tables mean "one per (photo, actor)" and
|
||||
* moving would either violate a unique constraint or double-count.
|
||||
*
|
||||
* photo_faces is the deliberate exception: both rows were scanned
|
||||
* independently, so the survivor already has its own embeddings and moving the
|
||||
* duplicate's would fabricate a second copy of every face and split the
|
||||
* person clusters built from them.
|
||||
*/
|
||||
|
||||
const { isUniqueViolation } = require('../utils/dbErrors');
|
||||
|
||||
const CHUNK = 400; // SQLite caps a statement at 999 bound parameters.
|
||||
// Joins the parts of an equivalence key. Escaped, not a literal: a raw NUL in
|
||||
// the source makes git classify this whole file as binary and hide its diffs.
|
||||
const KEY_SEP = '\u0000';
|
||||
const INDEX_NAME = 'photos_event_external_relpath_uniq';
|
||||
|
||||
const chunked = (arr) => {
|
||||
const out = [];
|
||||
for (let i = 0; i < arr.length; i += CHUNK) out.push(arr.slice(i, i + CHUNK));
|
||||
return out;
|
||||
};
|
||||
|
||||
/** Pure log rows — nothing is lost by dropping them with the duplicate. */
|
||||
const LOG_TABLES = [
|
||||
['image_access_logs', 'photo_id'],
|
||||
['transfer_downloads', 'photo_id'],
|
||||
];
|
||||
|
||||
/**
|
||||
* Tables holding one row per (photo, actor). `keys` is what makes two rows
|
||||
* equivalent, so a move that would collide becomes a delete instead.
|
||||
*/
|
||||
const MOVE_TABLES = [
|
||||
{
|
||||
table: 'photo_feedback',
|
||||
// Guest identity, the way feedbackService defines it: guest_id when the
|
||||
// gallery uses per-person guests (migration 078), guest_identifier
|
||||
// otherwise — the same COALESCE its own duplicate-check and stats
|
||||
// aggregate use (feedbackService.js:208, :559). Keying on
|
||||
// guest_identifier alone would treat two DIFFERENT people sharing a
|
||||
// device as one and delete one of their ratings.
|
||||
identity: (row) => (row.guest_id != null ? `id:${row.guest_id}` : `anon:${row.guest_identifier}`),
|
||||
// is_hidden is part of the identity, not noise: feedbackService lets a
|
||||
// moderator-hidden row coexist with the guest's visible replacement and
|
||||
// excludes hidden rows from the counts. Without it the visible row is
|
||||
// dropped as redundant against the hidden one.
|
||||
keys: ['feedback_type', 'is_hidden'],
|
||||
// A comment is distinct content, never a per-guest toggle: two comments
|
||||
// from one guest are two comments, so they always move.
|
||||
alwaysMove: (row) => row.feedback_type === 'comment',
|
||||
},
|
||||
{
|
||||
table: 'photo_admin_marks',
|
||||
keys: ['admin_id'],
|
||||
// rating and color_label are independently writable, so the same admin can
|
||||
// have rated one tile and colour-labelled the other. Dropping the loser
|
||||
// outright would lose a half the survivor's row has no value for.
|
||||
mergeFields: ['rating', 'color_label'],
|
||||
},
|
||||
{ table: 'transfer_files', keys: ['transfer_id'] },
|
||||
];
|
||||
|
||||
const equivalenceKey = (spec, row) => [
|
||||
spec.identity ? spec.identity(row) : '',
|
||||
...spec.keys.map((k) => row[k]),
|
||||
].join(KEY_SEP);
|
||||
|
||||
async function repointColumn(knex, table, column, doomedToSurvivor) {
|
||||
if (!(await knex.schema.hasTable(table))) return;
|
||||
if (!(await knex.schema.hasColumn(table, column))) return;
|
||||
for (const [doomed, survivor] of doomedToSurvivor) {
|
||||
await knex(table).where(column, doomed).update({ [column]: survivor });
|
||||
}
|
||||
}
|
||||
|
||||
async function moveOrDrop(knex, spec, doomedToSurvivor, touched) {
|
||||
if (!(await knex.schema.hasTable(spec.table))) return;
|
||||
|
||||
for (const [doomed, survivor] of doomedToSurvivor) {
|
||||
const rows = await knex(spec.table).where('photo_id', doomed);
|
||||
if (!rows.length) continue;
|
||||
if (touched) touched.add(survivor);
|
||||
|
||||
const existing = await knex(spec.table).where('photo_id', survivor);
|
||||
const taken = new Set(existing.map((r) => equivalenceKey(spec, r)));
|
||||
|
||||
for (const row of rows) {
|
||||
const key = equivalenceKey(spec, row);
|
||||
const move = (spec.alwaysMove && spec.alwaysMove(row)) || !taken.has(key);
|
||||
if (!move) {
|
||||
// Before dropping the loser, hand over any field the winner has no
|
||||
// value for — otherwise an independently-set half goes with it.
|
||||
if (spec.mergeFields) {
|
||||
const winner = existing.find((r) => equivalenceKey(spec, r) === key);
|
||||
const fill = {};
|
||||
for (const field of spec.mergeFields) {
|
||||
if (winner && winner[field] == null && row[field] != null) fill[field] = row[field];
|
||||
}
|
||||
if (winner && Object.keys(fill).length) {
|
||||
await knex(spec.table).where('id', winner.id).update(fill);
|
||||
Object.assign(winner, fill);
|
||||
}
|
||||
}
|
||||
await knex(spec.table).where('id', row.id).del();
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await knex(spec.table).where('id', row.id).update({ photo_id: survivor });
|
||||
taken.add(key);
|
||||
} catch (err) {
|
||||
// A unique constraint we did not model. The row is redundant with one
|
||||
// the survivor already has, so dropping it is correct — but anything
|
||||
// else must surface rather than leave a dangling photo_id behind.
|
||||
if (!isUniqueViolation(err)) throw err;
|
||||
await knex(spec.table).where('id', row.id).del();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove every photo row in `doomedToSurvivor`, moving or dropping the state
|
||||
* that hangs off it first. Safe on both engines and on schemas that predate
|
||||
* any of the dependent tables.
|
||||
*/
|
||||
async function deleteDuplicatePhotos(knex, doomedToSurvivor) {
|
||||
const doomed = [...doomedToSurvivor.keys()];
|
||||
if (!doomed.length) return 0;
|
||||
|
||||
// SET NULL is as inert as CASCADE on SQLite, so an event whose hero happened
|
||||
// to be the duplicate would silently lose its hero image.
|
||||
await repointColumn(knex, 'events', 'hero_photo_id', doomedToSurvivor);
|
||||
await repointColumn(knex, 'photo_categories', 'hero_photo_id', doomedToSurvivor);
|
||||
|
||||
const feedbackTouched = new Set();
|
||||
for (const spec of MOVE_TABLES) {
|
||||
await moveOrDrop(knex, spec, doomedToSurvivor, spec.table === 'photo_feedback' ? feedbackTouched : null);
|
||||
}
|
||||
|
||||
// photos carries denormalized feedback totals (migration 033:
|
||||
// feedback_count, like_count, average_rating, favorite_count, and later
|
||||
// reaction/colour counts). Reparenting rows without recomputing leaves a
|
||||
// survivor that now OWNS feedback still rendering zero.
|
||||
if (feedbackTouched.size && await knex.schema.hasColumn('photos', 'feedback_count')) {
|
||||
const feedbackService = require('./feedbackService');
|
||||
for (const survivor of feedbackTouched) {
|
||||
await feedbackService.updatePhotoFeedbackStats(survivor, knex);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [table, column] of LOG_TABLES) {
|
||||
if (!(await knex.schema.hasTable(table))) continue;
|
||||
for (const ids of chunked(doomed)) await knex(table).whereIn(column, ids).del();
|
||||
}
|
||||
|
||||
// Both rows were scanned, so the survivor has its own faces; moving the
|
||||
// duplicate's would double every embedding and split the person clusters.
|
||||
//
|
||||
// Through purgePhotoFaces, not a raw delete: deleting the rows is only half
|
||||
// of it. event_people counts and centroids are derived from the faces being
|
||||
// removed, and #1132's separation snapshots hold a COPY of each side's
|
||||
// centroid — so a bare delete leaves ghost or inflated people and vectors
|
||||
// built from photos that no longer exist. faceProcessor says as much: it is
|
||||
// "called from every photo-deletion path".
|
||||
if (await knex.schema.hasTable('photo_faces')) {
|
||||
// If the ONLY completed scan of this file belonged to the duplicate, the
|
||||
// purge below takes the sole embeddings with it and nothing re-queues the
|
||||
// survivor — it just silently stops having a face. Mark those for a
|
||||
// rescan; the worker picks up 'pending' on its own.
|
||||
const needsRescan = [];
|
||||
for (const [doomedId, survivorId] of doomedToSurvivor) {
|
||||
if (!(await knex('photo_faces').where('photo_id', doomedId).first())) continue;
|
||||
if (!(await knex('photo_faces').where('photo_id', survivorId).first())) needsRescan.push(survivorId);
|
||||
}
|
||||
|
||||
let purgePhotoFaces = null;
|
||||
try {
|
||||
({ purgePhotoFaces } = require('./faceProcessor'));
|
||||
} catch (err) {
|
||||
// Face detection is optional; an install without it still needs the rows
|
||||
// gone so nothing dangles on SQLite.
|
||||
purgePhotoFaces = null;
|
||||
}
|
||||
if (purgePhotoFaces) {
|
||||
for (const id of doomed) await purgePhotoFaces(id, knex);
|
||||
} else {
|
||||
for (const ids of chunked(doomed)) await knex('photo_faces').whereIn('photo_id', ids).del();
|
||||
}
|
||||
|
||||
if (needsRescan.length && await knex.schema.hasColumn('photos', 'face_status')) {
|
||||
for (const ids of chunked(needsRescan)) {
|
||||
await knex('photos').whereIn('id', ids).update({ face_status: 'pending' });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Real interactions, recorded per row. Deleting the duplicate would quietly
|
||||
// lower the engagement the admin grid shows for a photo people did view and
|
||||
// download.
|
||||
if (await knex.schema.hasColumn('photos', 'view_count')) {
|
||||
for (const [doomedId, survivorId] of doomedToSurvivor) {
|
||||
const from = await knex('photos').where('id', doomedId)
|
||||
.select('view_count', 'download_count').first();
|
||||
if (!from) continue;
|
||||
const add = {};
|
||||
if (from.view_count) add.view_count = knex.raw('COALESCE(view_count, 0) + ?', [from.view_count]);
|
||||
if (from.download_count) add.download_count = knex.raw('COALESCE(download_count, 0) + ?', [from.download_count]);
|
||||
if (Object.keys(add).length) await knex('photos').where('id', survivorId).update(add);
|
||||
}
|
||||
}
|
||||
|
||||
for (const ids of chunked(doomed)) await knex('photos').whereIn('id', ids).del();
|
||||
|
||||
// The pre-built "download everything" zip still contains the rows just
|
||||
// removed. Every ordinary photo-deletion path calls
|
||||
// downloadZipService.invalidate for this reason (adminPhotos.js:450 and
|
||||
// friends) — but that service carries debounce timers and a regeneration
|
||||
// queue, which is not something a migration should be starting. Clearing the
|
||||
// columns is the durable half of what invalidate does: getZipInfo already
|
||||
// treats a missing or absent record as a cache miss and rebuilds on the next
|
||||
// request, so guests stop receiving an archive containing deleted duplicates.
|
||||
//
|
||||
// The stale object itself is left for the same reason the duplicates'
|
||||
// thumbnails are — a migration is the wrong place to reach into storage,
|
||||
// which may be S3.
|
||||
if (await knex.schema.hasColumn('events', 'download_zip_path')) {
|
||||
const affected = [...new Set([...doomedToSurvivor.values()])];
|
||||
const eventIds = affected.length
|
||||
? (await knex('photos').whereIn('id', affected).distinct('event_id')).map((r) => r.event_id)
|
||||
: [];
|
||||
for (const ids of chunked(eventIds)) {
|
||||
await knex('events').whereIn('id', ids).update({
|
||||
download_zip_path: null,
|
||||
download_zip_generated_at: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return doomed.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Which rows are duplicates, and which one survives.
|
||||
*
|
||||
* Survivor: the lowest id that has a thumbnail_path, else the lowest id.
|
||||
* Thumbnails are generated per row during import, so on a duplicated pair both
|
||||
* usually have one and the tie-break never fires — but an import killed
|
||||
* mid-flight leaves rows without, and dropping the one that HAS the thumbnail
|
||||
* would blank a grid tile for no reason.
|
||||
*/
|
||||
async function planDedupe(knex) {
|
||||
const dupKeys = await knex('photos')
|
||||
.whereNotNull('external_relpath')
|
||||
.select('event_id')
|
||||
.count('* as c')
|
||||
.groupBy('event_id', 'external_relpath')
|
||||
.havingRaw('count(*) > 1');
|
||||
|
||||
const doomedToSurvivor = new Map();
|
||||
|
||||
for (const eventId of new Set(dupKeys.map((r) => r.event_id))) {
|
||||
const rows = await knex('photos')
|
||||
.where('event_id', eventId)
|
||||
.whereNotNull('external_relpath')
|
||||
.select('id', 'external_relpath', 'thumbnail_path')
|
||||
.orderBy('id', 'asc');
|
||||
|
||||
const byPath = new Map();
|
||||
for (const row of rows) {
|
||||
const group = byPath.get(row.external_relpath);
|
||||
if (group) group.push(row);
|
||||
else byPath.set(row.external_relpath, [row]);
|
||||
}
|
||||
|
||||
for (const group of byPath.values()) {
|
||||
if (group.length < 2) continue;
|
||||
const survivor = group.find((r) => r.thumbnail_path) || group[0];
|
||||
for (const row of group) {
|
||||
if (row.id !== survivor.id) doomedToSurvivor.set(row.id, survivor.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return doomedToSurvivor;
|
||||
}
|
||||
|
||||
/** @returns {Promise<number>} how many duplicate rows were removed. */
|
||||
async function dedupeExternalPhotos(knex) {
|
||||
if (!(await knex.schema.hasTable('photos'))) return 0;
|
||||
if (!(await knex.schema.hasColumn('photos', 'external_relpath'))) return 0;
|
||||
return deleteDuplicatePhotos(knex, await planDedupe(knex));
|
||||
}
|
||||
|
||||
/** Is the index actually there? Asked of the catalog, not inferred. */
|
||||
async function externalRelpathIndexExists(knex) {
|
||||
const isPg = knex.client && knex.client.config && knex.client.config.client === 'pg';
|
||||
const row = isPg
|
||||
? await knex('pg_indexes').where('indexname', INDEX_NAME).first()
|
||||
: await knex('sqlite_master').where({ type: 'index', name: INDEX_NAME }).first();
|
||||
return !!row;
|
||||
}
|
||||
|
||||
/**
|
||||
* The error a failed index MUST raise.
|
||||
*
|
||||
* Deliberately carries no `code`. run-migrations-safe.js treats 23505, 42P07,
|
||||
* 42701 and 42710 as "schema already exists" and marks the migration applied
|
||||
* (run-migrations-safe.js:138) — and a CREATE UNIQUE INDEX that finds
|
||||
* duplicate rows raises exactly 23505 on Postgres. Letting the driver's error
|
||||
* through would therefore record 176 as done on an install that never got the
|
||||
* index, with nothing to trigger a retry: the precise outcome the throw
|
||||
* exists to prevent.
|
||||
*/
|
||||
function indexFailure(detail) {
|
||||
return new Error(
|
||||
`Could not create ${INDEX_NAME}: ${detail}. The photos table still holds `
|
||||
+ 'duplicate (event_id, external_relpath) rows — most likely inserted by a '
|
||||
+ 'concurrent import while this migration ran. Stop other writers and re-run.'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Partial, so the managed rows — which all carry NULL — are not indexed at
|
||||
* all. Both engines treat NULLs as distinct in a unique index, so a plain one
|
||||
* would also be correct, but it would carry every managed photo for no query
|
||||
* that ever uses it.
|
||||
*/
|
||||
async function createExternalRelpathIndex(knex) {
|
||||
try {
|
||||
await knex.raw(
|
||||
`CREATE UNIQUE INDEX IF NOT EXISTS ${INDEX_NAME} `
|
||||
+ 'ON photos (event_id, external_relpath) WHERE external_relpath IS NOT NULL'
|
||||
);
|
||||
} catch (err) {
|
||||
throw indexFailure(err.message);
|
||||
}
|
||||
// IF NOT EXISTS makes the statement itself a poor witness, and a replica
|
||||
// inserting a duplicate between the dedupe and this lock is a real rolling-
|
||||
// deploy shape. Ask the catalog.
|
||||
if (!(await externalRelpathIndexExists(knex))) {
|
||||
throw indexFailure('the index is absent afterwards');
|
||||
}
|
||||
}
|
||||
|
||||
async function dropExternalRelpathIndex(knex) {
|
||||
await knex.raw(`DROP INDEX IF EXISTS ${INDEX_NAME}`);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dedupeExternalPhotos,
|
||||
externalRelpathIndexExists,
|
||||
deleteDuplicatePhotos,
|
||||
planDedupe,
|
||||
createExternalRelpathIndex,
|
||||
dropExternalRelpathIndex,
|
||||
INDEX_NAME,
|
||||
};
|
||||
@@ -334,25 +334,30 @@ class FeedbackService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Update photo feedback statistics
|
||||
* Update photo feedback statistics.
|
||||
*
|
||||
* `trx` so a caller running outside the request path — the duplicate-photo
|
||||
* dedupe (#1162), which reparents feedback rows and must leave the
|
||||
* survivor's denormalized totals correct — can recompute on its own
|
||||
* connection.
|
||||
*/
|
||||
async updatePhotoFeedbackStats(photoId) {
|
||||
async updatePhotoFeedbackStats(photoId, trx = db) {
|
||||
try {
|
||||
// Get aggregated stats
|
||||
const stats = await db('photo_feedback')
|
||||
const stats = await trx('photo_feedback')
|
||||
.where('photo_id', photoId)
|
||||
.where('is_hidden', false)
|
||||
.select(
|
||||
db.raw('COUNT(CASE WHEN feedback_type = ? AND is_approved = ? THEN 1 END) as comment_count', ['comment', formatBoolean(true)]),
|
||||
db.raw('COUNT(CASE WHEN feedback_type = ? THEN 1 END) as like_count', ['like']),
|
||||
db.raw('COUNT(CASE WHEN feedback_type = ? THEN 1 END) as favorite_count', ['favorite']),
|
||||
db.raw('AVG(CASE WHEN feedback_type = ? THEN rating END) as average_rating', ['rating']),
|
||||
db.raw('COUNT(DISTINCT COALESCE(CAST(guest_id AS VARCHAR), guest_identifier)) as feedback_count')
|
||||
trx.raw('COUNT(CASE WHEN feedback_type = ? AND is_approved = ? THEN 1 END) as comment_count', ['comment', formatBoolean(true)]),
|
||||
trx.raw('COUNT(CASE WHEN feedback_type = ? THEN 1 END) as like_count', ['like']),
|
||||
trx.raw('COUNT(CASE WHEN feedback_type = ? THEN 1 END) as favorite_count', ['favorite']),
|
||||
trx.raw('AVG(CASE WHEN feedback_type = ? THEN rating END) as average_rating', ['rating']),
|
||||
trx.raw('COUNT(DISTINCT COALESCE(CAST(guest_id AS VARCHAR), guest_identifier)) as feedback_count')
|
||||
)
|
||||
.first();
|
||||
|
||||
// Update photo table
|
||||
await db('photos')
|
||||
await trx('photos')
|
||||
.where('id', photoId)
|
||||
.update({
|
||||
feedback_count: stats.feedback_count || 0,
|
||||
|
||||
@@ -26,6 +26,11 @@ const { getStoragePath } = require('../config/storage');
|
||||
const { hasColumnCached } = require('../utils/schemaCache');
|
||||
const logger = require('../utils/logger');
|
||||
const { PICPEAK_FORMAT_VERSION, EXCLUDED_TABLES, listDataTables } = require('./picpeakExportService');
|
||||
const {
|
||||
dedupeExternalPhotos,
|
||||
createExternalRelpathIndex,
|
||||
dropExternalRelpathIndex,
|
||||
} = require('./externalPhotoDedupe');
|
||||
|
||||
const isPostgres = () => knexConfig.client === 'pg';
|
||||
|
||||
@@ -288,6 +293,19 @@ async function replaceAllTables(tables, dataDir, currentAdmin, { crossEngine = f
|
||||
await trx.raw('PRAGMA defer_foreign_keys = ON');
|
||||
}
|
||||
|
||||
// Suspending FK enforcement does not suspend UNIQUE indexes on either
|
||||
// engine (#1162). A backup taken before migration 176 carries the
|
||||
// duplicate photo rows that migration exists to remove, so batchInsert
|
||||
// below would hit photos_event_external_relpath_uniq and roll the whole
|
||||
// restore back — after every table had already been emptied. Drop it for
|
||||
// the load and rebuild it once the rows are deduped, which is the same
|
||||
// repair the migration performs.
|
||||
let hadRelpathIndex = false;
|
||||
if (await trx.schema.hasColumn('photos', 'external_relpath')) {
|
||||
hadRelpathIndex = true;
|
||||
await dropExternalRelpathIndex(trx);
|
||||
}
|
||||
|
||||
for (const table of tables) {
|
||||
await trx(table).del();
|
||||
}
|
||||
@@ -309,6 +327,18 @@ async function replaceAllTables(tables, dataDir, currentAdmin, { crossEngine = f
|
||||
await trx.batchInsert(table, prepared, 100);
|
||||
}
|
||||
|
||||
// Restore the constraint the load ran without. Deduping first because the
|
||||
// incoming rows may be exactly the duplicates migration 176 removes; the
|
||||
// index creation then also proves the repair worked, inside the same
|
||||
// transaction that would otherwise leave the target unprotected.
|
||||
if (hadRelpathIndex) {
|
||||
const removed = await dedupeExternalPhotos(trx);
|
||||
if (removed) {
|
||||
logger.info(`picpeakImport: removed ${removed} duplicate external photo row(s) from the archive (#1162)`);
|
||||
}
|
||||
await createExternalRelpathIndex(trx);
|
||||
}
|
||||
|
||||
await reinjectCurrentAdmin(trx, currentAdmin);
|
||||
|
||||
// Reset the pg session flag BEFORE the connection returns to the pool.
|
||||
|
||||
Reference in New Issue
Block a user