Stable twin of #1184. Both photo sweeps tracked whether they were running in a module-level variable, which is invisible to every other replica: a status poll routed to an idle replica reports isRunning false while another is mid-run, and the next POST starts a second pass over the whole library. Migration 179 adds one row per job, claimed with a conditional UPDATE whose affected-row count is the answer. The lease is fenced on a per-claim token so a runner superseded by a stale takeover cannot renew a claim it has lost or release one it no longer owns; renewal runs on a timer spanning the claim through release, since one hung NAS read can outlast the stale window inside a single iteration. maintenance_jobs is excluded from .picpeak archives. Gated on settings.edit / settings.view rather than main's system.manage, which does not exist on this branch — they are what settings.edit was later split into, so both branches let the same people through. Co-authored-by: Paul Nothaft <[email protected]>
This commit is contained in:
Paul Nothaft
co-authored by
Paul Nothaft
Paul Nothaft
parent
7f0ed23ea4
commit
58ccecc304
@@ -0,0 +1,233 @@
|
||||
/**
|
||||
* Shared run state for the maintenance sweeps (#1181).
|
||||
*
|
||||
* The behaviour that matters here cannot be observed from one process holding
|
||||
* a module-level flag, which is exactly why the flag moved into the database.
|
||||
* A second replica is simulated the only way that is honest in a single-process
|
||||
* test: by asserting on the shared row itself, and by driving claim() twice —
|
||||
* a second caller getting null is precisely what a second replica gets.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
const express = require('express');
|
||||
const request = require('supertest');
|
||||
|
||||
describe('maintenance job state (#1181)', () => {
|
||||
let tmpDir; let db; let app; let jobs;
|
||||
|
||||
const dimStatus = () => request(app).get('/api/admin/photos/repair-dimensions/status');
|
||||
const capStatus = () => request(app).get('/api/admin/photos/repair-capture-dates/status');
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'picpeak-mjs-'));
|
||||
process.env.NODE_ENV = 'test';
|
||||
process.env.TEST_DATABASE_PATH = path.join(tmpDir, 'data', 'db.sqlite');
|
||||
await fs.promises.mkdir(path.dirname(process.env.TEST_DATABASE_PATH), { recursive: true });
|
||||
process.env.STORAGE_PATH = path.join(tmpDir, 'storage');
|
||||
process.env.JWT_SECRET = process.env.JWT_SECRET || 'mjs-secret';
|
||||
|
||||
jest.resetModules();
|
||||
jest.doMock('../../src/middleware/auth', () => ({
|
||||
adminAuth: (req, _res, next) => { req.admin = { id: 1, username: 'tester', roleName: 'admin' }; next(); },
|
||||
}));
|
||||
jest.doMock('../../src/middleware/permissions', () => ({
|
||||
requirePermission: () => (_req, _res, next) => next(),
|
||||
}));
|
||||
jest.doMock('../../src/utils/logger', () => ({
|
||||
debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn(),
|
||||
}));
|
||||
|
||||
({ db } = await require('./helpers/crmDb').bootCrmDb());
|
||||
jobs = require('../../src/services/maintenanceJobState');
|
||||
|
||||
app = express();
|
||||
app.use(express.json());
|
||||
app.use('/api/admin/photos', require('../../src/routes/adminPhotoDimensions'));
|
||||
}, 180000);
|
||||
|
||||
afterAll(async () => {
|
||||
if (db) await db.destroy?.();
|
||||
await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await db('maintenance_jobs').update({
|
||||
is_running: false, started_at: null, heartbeat_at: null, finished_at: null, last_result: null, owner: null, claim_token: null,
|
||||
});
|
||||
});
|
||||
|
||||
test('the lease table is kept out of .picpeak archives', () => {
|
||||
// It is live state, not data. An archive taken mid-sweep would otherwise
|
||||
// carry is_running = true and a claim token owned by a process on the
|
||||
// SOURCE install; restored inside the staleness window, the target reports
|
||||
// the job as running and refuses new POSTs with no runner to release it.
|
||||
// The importer filters on this same set, so archives written before the
|
||||
// exclusion are skipped on restore too.
|
||||
const { EXCLUDED_TABLES } = require('../../src/services/picpeakExportService');
|
||||
expect(EXCLUDED_TABLES.has('maintenance_jobs')).toBe(true);
|
||||
});
|
||||
|
||||
test('the migration seeds a row for each job', async () => {
|
||||
const names = await db('maintenance_jobs').pluck('job_name');
|
||||
expect(names.sort()).toEqual(['photo_capture_date_backfill', 'photo_dimension_repair']);
|
||||
});
|
||||
|
||||
test('a second claim is refused while the first is alive', async () => {
|
||||
expect(await jobs.claim(jobs.JOB_DIMENSION_REPAIR)).toEqual(expect.any(String));
|
||||
// What a second replica's POST does. Nothing about the first claim lives in
|
||||
// this process, so this is the same question the other replica asks.
|
||||
expect(await jobs.claim(jobs.JOB_DIMENSION_REPAIR)).toBeNull();
|
||||
});
|
||||
|
||||
test('the two jobs claim independently', async () => {
|
||||
expect(await jobs.claim(jobs.JOB_DIMENSION_REPAIR)).toEqual(expect.any(String));
|
||||
expect(await jobs.claim(jobs.JOB_CAPTURE_DATE_BACKFILL)).toEqual(expect.any(String));
|
||||
});
|
||||
|
||||
test('each claim gets a distinct token', async () => {
|
||||
const first = await jobs.claim(jobs.JOB_DIMENSION_REPAIR);
|
||||
await jobs.release(jobs.JOB_DIMENSION_REPAIR, first);
|
||||
const second = await jobs.claim(jobs.JOB_DIMENSION_REPAIR);
|
||||
// Same process, same pid — so an owner string would have collided here and
|
||||
// the fencing below would be worthless.
|
||||
expect(second).not.toBe(first);
|
||||
});
|
||||
|
||||
test('a claim whose heartbeat has gone quiet can be taken over', async () => {
|
||||
expect(await jobs.claim(jobs.JOB_DIMENSION_REPAIR)).toEqual(expect.any(String));
|
||||
expect(await jobs.claim(jobs.JOB_DIMENSION_REPAIR)).toBeNull();
|
||||
|
||||
// The replica holding it was killed: no release, no further heartbeats.
|
||||
const longAgo = new Date(Date.now() - jobs.DEFAULT_STALE_MS - 60000).toISOString();
|
||||
await db('maintenance_jobs').where({ job_name: jobs.JOB_DIMENSION_REPAIR }).update({ heartbeat_at: longAgo });
|
||||
|
||||
expect(await jobs.claim(jobs.JOB_DIMENSION_REPAIR)).toEqual(expect.any(String));
|
||||
});
|
||||
|
||||
test('a superseded runner cannot renew its lease', async () => {
|
||||
const oldToken = await jobs.claim(jobs.JOB_DIMENSION_REPAIR);
|
||||
const longAgo = new Date(Date.now() - jobs.DEFAULT_STALE_MS - 60000).toISOString();
|
||||
await db('maintenance_jobs').where({ job_name: jobs.JOB_DIMENSION_REPAIR }).update({ heartbeat_at: longAgo });
|
||||
const newToken = await jobs.claim(jobs.JOB_DIMENSION_REPAIR);
|
||||
expect(newToken).toEqual(expect.any(String));
|
||||
|
||||
// The old runner is still alive and mid-loop. Its renewal must tell it so,
|
||||
// which is what makes the route loop stop instead of running alongside the
|
||||
// new owner.
|
||||
expect(await jobs.heartbeat(jobs.JOB_DIMENSION_REPAIR, oldToken)).toBe(false);
|
||||
expect(await jobs.heartbeat(jobs.JOB_DIMENSION_REPAIR, newToken)).toBe(true);
|
||||
});
|
||||
|
||||
test('a superseded runner cannot release the new owner\'s claim', async () => {
|
||||
const oldToken = await jobs.claim(jobs.JOB_CAPTURE_DATE_BACKFILL);
|
||||
const longAgo = new Date(Date.now() - jobs.DEFAULT_STALE_MS - 60000).toISOString();
|
||||
await db('maintenance_jobs').where({ job_name: jobs.JOB_CAPTURE_DATE_BACKFILL }).update({ heartbeat_at: longAgo });
|
||||
const newToken = await jobs.claim(jobs.JOB_CAPTURE_DATE_BACKFILL);
|
||||
|
||||
// The old runner finishes late and tries to write its result. Unfenced,
|
||||
// this cleared is_running under the new owner and let a THIRD sweep start.
|
||||
expect(await jobs.release(jobs.JOB_CAPTURE_DATE_BACKFILL, oldToken, { success: 999, noExif: 0, failed: 0 })).toBe(false);
|
||||
|
||||
const state = await jobs.read(jobs.JOB_CAPTURE_DATE_BACKFILL);
|
||||
expect(state.isRunning).toBe(true);
|
||||
expect(state.lastResult).toBeNull();
|
||||
// And the row is still the new owner's to release.
|
||||
expect(await jobs.release(jobs.JOB_CAPTURE_DATE_BACKFILL, newToken, { success: 1, noExif: 0, failed: 0 })).toBe(true);
|
||||
});
|
||||
|
||||
test('a stale run reads as not running, so the button comes back', async () => {
|
||||
await jobs.claim(jobs.JOB_DIMENSION_REPAIR);
|
||||
expect((await jobs.read(jobs.JOB_DIMENSION_REPAIR)).isRunning).toBe(true);
|
||||
|
||||
const longAgo = new Date(Date.now() - jobs.DEFAULT_STALE_MS - 60000).toISOString();
|
||||
await db('maintenance_jobs').where({ job_name: jobs.JOB_DIMENSION_REPAIR }).update({ heartbeat_at: longAgo });
|
||||
|
||||
// is_running is still true in the row — nothing released it — but a status
|
||||
// poll must not leave the operator staring at a job that cannot finish.
|
||||
expect((await db('maintenance_jobs').where({ job_name: jobs.JOB_DIMENSION_REPAIR }).first()).is_running).toBeTruthy();
|
||||
expect((await jobs.read(jobs.JOB_DIMENSION_REPAIR)).isRunning).toBe(false);
|
||||
});
|
||||
|
||||
test('a heartbeat keeps a long run claimed', async () => {
|
||||
const token = await jobs.claim(jobs.JOB_DIMENSION_REPAIR);
|
||||
const longAgo = new Date(Date.now() - jobs.DEFAULT_STALE_MS - 60000).toISOString();
|
||||
await db('maintenance_jobs').where({ job_name: jobs.JOB_DIMENSION_REPAIR }).update({ heartbeat_at: longAgo });
|
||||
|
||||
expect(await jobs.heartbeat(jobs.JOB_DIMENSION_REPAIR, token)).toBe(true);
|
||||
|
||||
expect(await jobs.claim(jobs.JOB_DIMENSION_REPAIR)).toBeNull();
|
||||
expect((await jobs.read(jobs.JOB_DIMENSION_REPAIR)).isRunning).toBe(true);
|
||||
});
|
||||
|
||||
test('release stores the result and read gives it back parsed', async () => {
|
||||
const token = await jobs.claim(jobs.JOB_CAPTURE_DATE_BACKFILL);
|
||||
await jobs.release(jobs.JOB_CAPTURE_DATE_BACKFILL, token, { success: 3, noExif: 2, failed: 1 });
|
||||
|
||||
const state = await jobs.read(jobs.JOB_CAPTURE_DATE_BACKFILL);
|
||||
expect(state.isRunning).toBe(false);
|
||||
expect(state.lastResult).toEqual({ success: 3, noExif: 2, failed: 1 });
|
||||
});
|
||||
|
||||
test('releasing without a result keeps the previous run visible', async () => {
|
||||
const first = await jobs.claim(jobs.JOB_CAPTURE_DATE_BACKFILL);
|
||||
await jobs.release(jobs.JOB_CAPTURE_DATE_BACKFILL, first, { success: 7, noExif: 0, failed: 0 });
|
||||
|
||||
// The "nothing to do" path: claimed, found no candidates, released. It must
|
||||
// not blank the numbers the last real run reported.
|
||||
const second = await jobs.claim(jobs.JOB_CAPTURE_DATE_BACKFILL);
|
||||
await jobs.release(jobs.JOB_CAPTURE_DATE_BACKFILL, second);
|
||||
|
||||
expect((await jobs.read(jobs.JOB_CAPTURE_DATE_BACKFILL)).lastResult).toEqual({ success: 7, noExif: 0, failed: 0 });
|
||||
});
|
||||
|
||||
test('a malformed result does not take the status endpoint down', async () => {
|
||||
await db('maintenance_jobs').where({ job_name: jobs.JOB_DIMENSION_REPAIR }).update({ last_result: 'not json' });
|
||||
const state = await jobs.read(jobs.JOB_DIMENSION_REPAIR);
|
||||
expect(state.lastResult).toBeNull();
|
||||
expect(state.isRunning).toBe(false);
|
||||
});
|
||||
|
||||
test('both status endpoints report the shared row, not process memory', async () => {
|
||||
await jobs.claim(jobs.JOB_DIMENSION_REPAIR);
|
||||
const capToken = await jobs.claim(jobs.JOB_CAPTURE_DATE_BACKFILL);
|
||||
await jobs.release(jobs.JOB_CAPTURE_DATE_BACKFILL, capToken, { success: 1, noExif: 0, failed: 0 });
|
||||
|
||||
// Written straight to the row, exactly as another replica would have.
|
||||
const dim = await dimStatus();
|
||||
expect(dim.status).toBe(200);
|
||||
expect(dim.body.isRunning).toBe(true);
|
||||
|
||||
const cap = await capStatus();
|
||||
expect(cap.status).toBe(200);
|
||||
expect(cap.body.isRunning).toBe(false);
|
||||
expect(cap.body.lastResult).toEqual({ success: 1, noExif: 0, failed: 0 });
|
||||
});
|
||||
|
||||
test('a POST is refused while another replica holds the claim', async () => {
|
||||
// The claim was taken by "another replica" — this process knows nothing
|
||||
// about it beyond the row.
|
||||
await jobs.claim(jobs.JOB_CAPTURE_DATE_BACKFILL);
|
||||
|
||||
const res = await request(app).post('/api/admin/photos/repair-capture-dates');
|
||||
expect(res.status).toBe(409);
|
||||
|
||||
const dimRes = await request(app).post('/api/admin/photos/repair-dimensions');
|
||||
// The other job is untouched by that claim, so it is free to start.
|
||||
expect(dimRes.status).toBe(200);
|
||||
});
|
||||
|
||||
test('the no-op path releases the claim it took', async () => {
|
||||
// No photos at all, so both endpoints take their "nothing to do" exit.
|
||||
await db('photos').del();
|
||||
|
||||
const res = await request(app).post('/api/admin/photos/repair-capture-dates');
|
||||
expect(res.body.count).toBe(0);
|
||||
|
||||
const row = await db('maintenance_jobs').where({ job_name: jobs.JOB_CAPTURE_DATE_BACKFILL }).first();
|
||||
expect(row.is_running).toBeFalsy();
|
||||
// ...and a second POST is therefore accepted rather than 409ing forever.
|
||||
expect((await request(app).post('/api/admin/photos/repair-capture-dates')).status).toBe(200);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,102 @@
|
||||
/**
|
||||
* PostgreSQL checks for the shared maintenance-job state (#1181).
|
||||
* Gated: runs only when PICPEAK_PG_TEST_URL points at a throwaway database, e.g.
|
||||
* PICPEAK_PG_TEST_URL="postgres://picpeak:[email protected]:7102/picpeak_mjs_test" \
|
||||
* npx jest __tests__/integration/maintenanceJobStatePg.test.js
|
||||
*
|
||||
* What SQLite cannot answer: the claim leans on comparing a `timestamp` column
|
||||
* against an ISO-8601 string, and on an UPDATE ... WHERE guard being atomic
|
||||
* under real concurrent connections. SQLite compares those strings
|
||||
* lexicographically and serialises writes anyway, so it would pass either way —
|
||||
* exactly the shape of divergence that has bitten this repo before.
|
||||
*/
|
||||
|
||||
const knex = require('knex');
|
||||
|
||||
const PG_URL = process.env.PICPEAK_PG_TEST_URL;
|
||||
const maybe = PG_URL ? describe : describe.skip;
|
||||
|
||||
maybe('maintenance job state on Postgres', () => {
|
||||
let pgDb;
|
||||
let jobs;
|
||||
const JOB = 'photo_dimension_repair';
|
||||
|
||||
beforeAll(async () => {
|
||||
pgDb = knex({ client: 'pg', connection: PG_URL, pool: { min: 0, max: 10 } });
|
||||
await pgDb.raw('DROP TABLE IF EXISTS maintenance_jobs');
|
||||
await require('../../migrations/core/179_maintenance_job_state').up(pgDb);
|
||||
|
||||
jest.resetModules();
|
||||
jest.doMock('../../src/database/db', () => ({ db: pgDb }));
|
||||
jest.doMock('../../src/utils/logger', () => ({
|
||||
debug: jest.fn(), info: jest.fn(), warn: jest.fn(), error: jest.fn(),
|
||||
}));
|
||||
jobs = require('../../src/services/maintenanceJobState');
|
||||
}, 60000);
|
||||
|
||||
afterAll(async () => {
|
||||
jest.dontMock('../../src/database/db');
|
||||
if (pgDb) await pgDb.destroy();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await pgDb('maintenance_jobs').update({
|
||||
is_running: false, started_at: null, heartbeat_at: null, finished_at: null, last_result: null, owner: null, claim_token: null,
|
||||
});
|
||||
});
|
||||
|
||||
test('the ISO-string cutoff really compares as a timestamp, not as text', async () => {
|
||||
expect(await jobs.claim(JOB)).toEqual(expect.any(String));
|
||||
expect(await jobs.claim(JOB)).toBeNull();
|
||||
|
||||
await pgDb('maintenance_jobs').where({ job_name: JOB })
|
||||
.update({ heartbeat_at: new Date(Date.now() - jobs.DEFAULT_STALE_MS - 60000).toISOString() });
|
||||
|
||||
// If Postgres had rejected or mis-cast the ISO string this would either
|
||||
// throw or never match.
|
||||
expect(await jobs.claim(JOB)).toEqual(expect.any(String));
|
||||
|
||||
const row = await pgDb('maintenance_jobs').where({ job_name: JOB }).first();
|
||||
expect(row.heartbeat_at).toBeInstanceOf(Date);
|
||||
});
|
||||
|
||||
test('concurrent claims on real connections produce exactly one winner', async () => {
|
||||
// The whole point of the conditional UPDATE. Ten connections race; nine
|
||||
// must lose. SQLite cannot demonstrate this — it serialises writers.
|
||||
const results = await Promise.all(Array.from({ length: 10 }, () => jobs.claim(JOB)));
|
||||
expect(results.filter(Boolean)).toHaveLength(1);
|
||||
// ...and the winner holds a token nobody else can forge.
|
||||
expect(results.find(Boolean)).toEqual(expect.any(String));
|
||||
});
|
||||
|
||||
test('a released job can be re-claimed exactly once again', async () => {
|
||||
const token = await jobs.claim(JOB);
|
||||
await jobs.release(JOB, token, { success: 2, failed: 0 });
|
||||
|
||||
const results = await Promise.all(Array.from({ length: 5 }, () => jobs.claim(JOB)));
|
||||
expect(results.filter(Boolean)).toHaveLength(1);
|
||||
expect((await jobs.read(JOB)).lastResult).toEqual({ success: 2, failed: 0 });
|
||||
});
|
||||
|
||||
test('a superseded runner is fenced out on real Postgres', async () => {
|
||||
const oldToken = await jobs.claim(JOB);
|
||||
await pgDb('maintenance_jobs').where({ job_name: JOB })
|
||||
.update({ heartbeat_at: new Date(Date.now() - jobs.DEFAULT_STALE_MS - 60000).toISOString() });
|
||||
const newToken = await jobs.claim(JOB);
|
||||
|
||||
expect(await jobs.heartbeat(JOB, oldToken)).toBe(false);
|
||||
expect(await jobs.release(JOB, oldToken, { success: 999, failed: 0 })).toBe(false);
|
||||
// The new owner still holds it, with its result unwritten.
|
||||
expect((await jobs.read(JOB)).isRunning).toBe(true);
|
||||
expect(await jobs.release(JOB, newToken, { success: 4, failed: 0 })).toBe(true);
|
||||
});
|
||||
|
||||
test('read() reports a live claim as running and a stale one as not', async () => {
|
||||
await jobs.claim(JOB);
|
||||
expect((await jobs.read(JOB)).isRunning).toBe(true);
|
||||
|
||||
await pgDb('maintenance_jobs').where({ job_name: JOB })
|
||||
.update({ heartbeat_at: new Date(Date.now() - jobs.DEFAULT_STALE_MS - 1000).toISOString() });
|
||||
expect((await jobs.read(JOB)).isRunning).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user