From c6ec93eef9f18e8867e86691800a60379bb16591 Mon Sep 17 00:00:00 2001 From: Paul Nothaft <53005142+the-luap@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:07:32 +0200 Subject: [PATCH] fix(dates): normalize SQLite epoch timestamps at remaining API surfaces (#485 follow-up) (#857) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit #485 called for: on SQLite (native installs), timestamp columns written with a raw `new Date()` through knex store epoch-ms numbers; Postgres returns ISO strings. Frontend code written against Postgres calls parseISO() on them — parseISO(number) throws and crashes the page. #485 fixed admin Users and listed api tokens / photos / activity as out-of-scope follow-ups. Verified crash on main: Timeline gallery layout parseISO(uploaded_at) against photos written by the archive-RESTORE path (raw Date). Other raw-write surfaces (api_tokens last_used_at/revoked_at, email_queue) degrade rather than crash but violate the ISO contract. - extract toIso() from adminUsers.js into utils/dateNormalize.js (contract unchanged — the 10 existing #485 tests still pin it) - write-side: archive-restore uploaded_at, api-token last_used_at / revoked_at, email_queue created_at/sent_at now write ISO strings - read-side (heals existing corrupted rows): gallery /photos normalizes uploaded_at/captured_at; api-tokens list normalizes all four timestamp fields - frontend defence-in-depth: Timeline layout parses uploaded_at tolerantly (typeof guard) for stale caches / old backends - 2 regression tests seed literal epoch numbers and assert the API serves ISO strings activity_logs turned out safe (created_at comes from the DB default, not a raw Date) — left untouched. Co-authored-by: Paul Nothaft --- .../integration/sqliteEpochTimestamps.test.js | 136 ++++++++++++++++++ backend/src/middleware/apiTokenAuth.js | 2 +- backend/src/routes/adminApiTokens.js | 13 +- backend/src/routes/adminArchives.js | 2 +- backend/src/routes/adminEmail.js | 10 +- backend/src/routes/adminUsers.js | 32 +---- backend/src/routes/gallery.js | 8 +- backend/src/services/emailProcessor.js | 2 +- backend/src/utils/dateNormalize.js | 36 +++++ .../gallery/layouts/TimelineGalleryLayout.tsx | 11 +- 10 files changed, 207 insertions(+), 45 deletions(-) create mode 100644 backend/__tests__/integration/sqliteEpochTimestamps.test.js create mode 100644 backend/src/utils/dateNormalize.js diff --git a/backend/__tests__/integration/sqliteEpochTimestamps.test.js b/backend/__tests__/integration/sqliteEpochTimestamps.test.js new file mode 100644 index 00000000..955e331f --- /dev/null +++ b/backend/__tests__/integration/sqliteEpochTimestamps.test.js @@ -0,0 +1,136 @@ +/** + * SQLite epoch-timestamp normalization (#485 follow-up). + * + * On SQLite, timestamp columns written with a raw `new Date()` through knex + * hold epoch-millisecond numbers. Postgres returns ISO strings, so frontend + * code written against Postgres calls parseISO() and crashes on native + * (SQLite) installs — the exact class fixed for admin Users in #485, which + * listed api tokens / photos / activity as an out-of-scope follow-up. + * + * Pins: + * - gallery /photos serializes uploaded_at / captured_at as ISO strings + * even when the row holds an epoch number (pre-fix archive restores) + * - the api-tokens list serializes created_at / expires_at / last_used_at / + * revoked_at as ISO strings for epoch-stored rows + */ + +const request = require('supertest'); +const express = require('express'); +const cookieParser = require('cookie-parser'); +const bcrypt = require('bcrypt'); +const jwt = require('jsonwebtoken'); + +const { bootCrmDb, seedMinimal } = require('./helpers/crmDb'); + +process.env.JWT_SECRET = process.env.JWT_SECRET || 'epoch-test-secret'; + +const SLUG = 'epoch-test-event'; +const ISO_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/; + +describe('SQLite epoch timestamp normalization', () => { + let db; + let cleanup; + let app; + let eventId; + let adminToken; + + beforeAll(async () => { + ({ db, cleanup } = await bootCrmDb()); + await seedMinimal(db); + + const inserted = await db('events').insert({ + slug: SLUG, + event_type: 'wedding', + event_name: 'Epoch Test', + event_date: '2026-08-01', + host_email: 'host@example.com', + admin_email: 'admin@example.com', + password_hash: 'x', + share_link: `/gallery/${SLUG}/share`, + share_token: 'epoch-test-share', + expires_at: new Date(Date.now() + 7 * 24 * 3600 * 1000).toISOString(), + is_active: 1, + is_archived: 0, + is_draft: 0, + created_at: new Date().toISOString(), + }).returning('id'); + eventId = inserted[0]?.id ?? inserted[0]; + + // The pre-fix corruption shape: epoch numbers in timestamp columns. + await db('photos').insert({ + event_id: eventId, + filename: 'restored.jpg', + path: 'events/epoch/restored.jpg', + type: 'individual', + uploaded_at: Date.now() - 3600_000, + captured_at: Date.now() - 7200_000, + }); + + const superRole = await db('roles').where({ name: 'super_admin' }).first(); + const [rootId] = await db('admin_users').insert({ + username: 'epoch-admin', + email: 'epoch-admin@example.com', + password_hash: await bcrypt.hash('EpochAdmin123', 4), + role_id: superRole.id, + is_active: 1, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + }).returning('id').then((r) => [r[0]?.id || r[0]]); + adminToken = jwt.sign( + { id: rootId, username: 'epoch-admin', type: 'admin', role: 'super_admin', loginTime: Date.now() }, + process.env.JWT_SECRET, + { expiresIn: '1h', issuer: 'picpeak-auth' } + ); + + await db('api_tokens').insert({ + name: 'epoch-token', + hashed_token: 'x'.repeat(64), + preview: 'pk_test…abcd', + scopes: JSON.stringify(['events:read']), + created_by: rootId, + created_at: Date.now() - 86400_000, + last_used_at: Date.now() - 3600_000, + revoked_at: Date.now() - 60_000, + }); + + app = express(); + app.use(express.json()); + app.use(cookieParser()); + app.use('/api/gallery', require('../../src/routes/gallery')); + app.use('/api/admin/api-tokens', require('../../src/routes/adminApiTokens')); + }, 120000); + + afterAll(async () => { + if (cleanup) await cleanup(); + }); + + it('gallery /photos serializes epoch-stored uploaded_at/captured_at as ISO strings', async () => { + const galleryToken = jwt.sign( + { eventId, eventSlug: SLUG, type: 'gallery' }, + process.env.JWT_SECRET, + { expiresIn: '1h', issuer: 'picpeak-auth' } + ); + const res = await request(app) + .get(`/api/gallery/${SLUG}/photos`) + .set('Authorization', `Bearer ${galleryToken}`); + expect(res.status).toBe(200); + expect(res.body.photos).toHaveLength(1); + const photo = res.body.photos[0]; + expect(typeof photo.uploaded_at).toBe('string'); + expect(photo.uploaded_at).toMatch(ISO_RE); + expect(photo.captured_at).toMatch(ISO_RE); + }); + + it('api-tokens list serializes epoch-stored timestamps as ISO strings', async () => { + const res = await request(app) + .get('/api/admin/api-tokens') + .set('Authorization', `Bearer ${adminToken}`); + expect(res.status).toBe(200); + const token = res.body.find((t) => t.name === 'epoch-token'); + expect(token).toBeTruthy(); + for (const field of ['created_at', 'last_used_at', 'revoked_at']) { + expect(`${field}:${typeof token[field]}`).toBe(`${field}:string`); + expect(token[field]).toMatch(ISO_RE); + } + }); +}); diff --git a/backend/src/middleware/apiTokenAuth.js b/backend/src/middleware/apiTokenAuth.js index c3d826af..c42f5c44 100644 --- a/backend/src/middleware/apiTokenAuth.js +++ b/backend/src/middleware/apiTokenAuth.js @@ -72,7 +72,7 @@ async function apiTokenAuth(req, res, next) { } // Touch last_used_at — async, don't block the request. - db('api_tokens').where({ id: row.id }).update({ last_used_at: new Date() }) + db('api_tokens').where({ id: row.id }).update({ last_used_at: new Date().toISOString() }) .catch((err) => logger.debug('api_tokens last_used update failed', { err: err.message })); req.admin = admin; diff --git a/backend/src/routes/adminApiTokens.js b/backend/src/routes/adminApiTokens.js index b2a74d2a..95bcc81d 100644 --- a/backend/src/routes/adminApiTokens.js +++ b/backend/src/routes/adminApiTokens.js @@ -11,6 +11,7 @@ const { db, logActivity } = require('../database/db'); const { adminAuth } = require('./../middleware/auth'); const { requirePermission } = require('./../middleware/permissions'); const { generateApiToken, VALID_SCOPES } = require('./../middleware/apiTokenAuth'); +const { toIso } = require('../utils/dateNormalize'); const logger = require('../utils/logger'); const router = express.Router(); @@ -33,7 +34,15 @@ router.get('/', adminAuth, requirePermission('settings.view'), async (req, res) 'admin_users.username as owner_username' ) .orderBy('api_tokens.created_at', 'desc'); - res.json(tokens); + // toIso: last_used_at / revoked_at were written as raw Dates before + // this fix — SQLite installs hold epoch numbers in existing rows. + res.json(tokens.map((t) => ({ + ...t, + created_at: toIso(t.created_at), + expires_at: toIso(t.expires_at), + last_used_at: toIso(t.last_used_at), + revoked_at: toIso(t.revoked_at), + }))); } catch (error) { logger.error('Failed to list API tokens', { error: error.message }); res.status(500).json({ error: 'Failed to list tokens' }); @@ -103,7 +112,7 @@ router.delete('/:id', adminAuth, requirePermission('settings.edit'), async (req, if (!row) return res.status(404).json({ error: 'Token not found' }); if (row.revoked_at) return res.status(400).json({ error: 'Token already revoked' }); - await db('api_tokens').where({ id }).update({ revoked_at: new Date() }); + await db('api_tokens').where({ id }).update({ revoked_at: new Date().toISOString() }); await logActivity('api_token_revoked', { name: row.name }, null, { type: 'admin', id: req.admin.id, name: req.admin.username }); diff --git a/backend/src/routes/adminArchives.js b/backend/src/routes/adminArchives.js index fcebccfa..8734917e 100644 --- a/backend/src/routes/adminArchives.js +++ b/backend/src/routes/adminArchives.js @@ -293,7 +293,7 @@ router.post('/:id/restore', adminAuth, requirePermission('archives.restore'), re type: path.extname(filename).substring(1).toLowerCase(), size_bytes: stats.size, category_id: categoryId, - uploaded_at: new Date() + uploaded_at: new Date().toISOString() }); } } catch (statError) { diff --git a/backend/src/routes/adminEmail.js b/backend/src/routes/adminEmail.js index d5fe0e20..169c6061 100644 --- a/backend/src/routes/adminEmail.js +++ b/backend/src/routes/adminEmail.js @@ -419,7 +419,7 @@ router.post('/accounts', adminAuth, messagingGate, requirePermission('email.edit account_key: b.account_key, imap_pass: (b.imap_pass && b.imap_pass !== '********') ? b.imap_pass : '', smtp_pass: (b.smtp_pass && b.smtp_pass !== '********') ? b.smtp_pass : '', - created_at: new Date(), + created_at: new Date().toISOString(), ...patch, }); } @@ -808,8 +808,8 @@ router.post('/send', adminAuth, messagingGate, requirePermission('email.send'), status: 'sent', origin: 'manual', rendered_html: html, - created_at: new Date(), - sent_at: new Date(), + created_at: new Date().toISOString(), + sent_at: new Date().toISOString(), }); res.json({ ok: true }); } catch (error) { @@ -985,7 +985,7 @@ router.put('/templates/:key', [ template_id: template.id, language, ...row, - created_at: new Date(), + created_at: new Date().toISOString(), }); } } @@ -1099,7 +1099,7 @@ router.post('/templates', [ subject: content.subject || '', body_html: content.body_html || '', body_text: content.body_text || '', - created_at: new Date(), + created_at: new Date().toISOString(), updated_at: new Date(), }); } diff --git a/backend/src/routes/adminUsers.js b/backend/src/routes/adminUsers.js index 2ea6dca4..2d2279e4 100644 --- a/backend/src/routes/adminUsers.js +++ b/backend/src/routes/adminUsers.js @@ -10,39 +10,9 @@ const { requirePermission, requireSuperAdmin, getUserPermissions } = require('.. const { handleAsync, validateRequest, successResponse } = require('../utils/routeHelpers'); const userManagementService = require('../services/userManagementService'); const { IDENTITY_PRESERVING_NORMALIZE_EMAIL } = require('../utils/emailNormalization'); +const { toIso } = require('../utils/dateNormalize'); const router = express.Router(); -/** - * Coerce any of the shapes a TIMESTAMP column produces across our - * supported drivers into a single ISO 8601 string the frontend (and - * any external API consumer) can safely pass to date-fns / new Date. - * - * Postgres → Date object (becomes ISO via JSON.stringify anyway, but - * pinning the format defends against driver-side surprises). - * SQLite → integer milliseconds since epoch (the surface that crashed - * the admin Users page in #485 — `parseISO(123456789)` blows up - * with "e.split is not a function"). Native installs default to - * SQLite, so this path matters every release. - * Already a string → assume it's a parseable ISO/RFC3339 (Postgres - * driver may stringify under JSON serialization mid-pipeline). - * - * Returns null/undefined unchanged so an unset last_login surfaces as - * "Never" in the UI rather than 1970-01-01T00:00:00Z. - */ -function toIso(value) { - if (value === null || value === undefined || value === '') return value; - if (value instanceof Date) return value.toISOString(); - if (typeof value === 'number') return new Date(value).toISOString(); - if (typeof value === 'string') { - // Numeric-as-string ("1778752458666") happens when the SQLite - // driver stringifies large integers — re-coerce so the frontend - // doesn't try to parseISO('1778752458666'). - if (/^\d{10,}$/.test(value)) return new Date(Number(value)).toISOString(); - return value; - } - return value; -} - /** * Transform user object from snake_case (DB) to camelCase (API) */ diff --git a/backend/src/routes/gallery.js b/backend/src/routes/gallery.js index 61cd4d29..b05344cf 100644 --- a/backend/src/routes/gallery.js +++ b/backend/src/routes/gallery.js @@ -29,6 +29,7 @@ const { getEventCategoriesOrdered } = require('../utils/categoryOrder'); const { getEventShareToken, resolveShareIdentifier, buildShareLinkVariants } = require('../services/shareLinkService'); const { handleAsync, errorResponse } = require('../utils/routeHelpers'); const { isGalleryHidden, guestBlockedByReveal, blockHiddenGallery } = require('../utils/revealMode'); +const { toIso } = require('../utils/dateNormalize'); const { NotFoundError } = require('../utils/errors'); const { ensureThumbnail, ensureHeroImage, ensurePreviewImage, withLocalCopy } = require('../services/imageProcessor'); const downloadZipService = require('../services/downloadZipService'); @@ -936,14 +937,17 @@ router.get('/:slug/photos', verifyGalleryAccess, resolveGuest, async (req, res) : true, category_slug: photo.category_id && categoryMap[photo.category_id] ? categoryMap[photo.category_id].slug : null, size: photo.size_bytes, - uploaded_at: photo.uploaded_at, + // toIso: on SQLite installs rows written with a raw Date (e.g. + // the pre-fix archive-restore path) hold epoch numbers — the + // Timeline layout's parseISO() crashes on those (#485 class). + uploaded_at: toIso(photo.uploaded_at), // Image dimensions for layout calculations width: photo.width || null, height: photo.height || null, // Fixed: Use the calculated useJwtUrl variable instead of recalculating requires_token: !useJwtUrl, // EXIF capture date - captured_at: photo.captured_at || null, + captured_at: toIso(photo.captured_at) || null, // Media type media_type: photo.media_type || null, mime_type: photo.mime_type || null, diff --git a/backend/src/services/emailProcessor.js b/backend/src/services/emailProcessor.js index 6d8e7bb8..b91bcdec 100644 --- a/backend/src/services/emailProcessor.js +++ b/backend/src/services/emailProcessor.js @@ -941,7 +941,7 @@ async function processEmailQueue({ ignoreSchedule = false, limit = 10, onlyId = // Mark as sent, persisting the actual rendered HTML for the Project // Overview email preview (guarded — older installs without migration // 119 just skip it). - const sentUpdate = { status: 'sent', sent_at: new Date() }; + const sentUpdate = { status: 'sent', sent_at: new Date().toISOString() }; try { if (sendResult && sendResult.html && await hasColumnCached('email_queue', 'rendered_html')) { sentUpdate.rendered_html = sendResult.html; diff --git a/backend/src/utils/dateNormalize.js b/backend/src/utils/dateNormalize.js new file mode 100644 index 00000000..fb67dbbe --- /dev/null +++ b/backend/src/utils/dateNormalize.js @@ -0,0 +1,36 @@ +/** + * Coerce any of the shapes a TIMESTAMP column produces across our + * supported drivers into a single ISO 8601 string the frontend (and + * any external API consumer) can safely pass to date-fns / new Date. + * + * Postgres → Date object (becomes ISO via JSON.stringify anyway, but + * pinning the format defends against driver-side surprises). + * SQLite → integer milliseconds since epoch when a raw `new Date()` was + * written through knex (the surface that crashed the admin Users page + * in #485 — `parseISO(123456789)` blows up with "e.split is not a + * function"). Native installs default to SQLite, so this path matters + * every release. + * Already a string → assume it's a parseable ISO/RFC3339 (Postgres + * driver may stringify under JSON serialization mid-pipeline). + * + * Returns null/undefined unchanged so an unset value surfaces as + * "Never" in the UI rather than 1970-01-01T00:00:00Z. + * + * Extracted from routes/adminUsers.js (#485) so every route that + * serializes timestamps can share one contract. + */ +function toIso(value) { + if (value === null || value === undefined || value === '') return value; + if (value instanceof Date) return value.toISOString(); + if (typeof value === 'number') return new Date(value).toISOString(); + if (typeof value === 'string') { + // Numeric-as-string ("1778752458666") happens when the SQLite + // driver stringifies large integers — re-coerce so the frontend + // doesn't try to parseISO('1778752458666'). + if (/^\d{10,}$/.test(value)) return new Date(Number(value)).toISOString(); + return value; + } + return value; +} + +module.exports = { toIso }; diff --git a/frontend/src/components/gallery/layouts/TimelineGalleryLayout.tsx b/frontend/src/components/gallery/layouts/TimelineGalleryLayout.tsx index 63253138..8e4807af 100644 --- a/frontend/src/components/gallery/layouts/TimelineGalleryLayout.tsx +++ b/frontend/src/components/gallery/layouts/TimelineGalleryLayout.tsx @@ -41,12 +41,19 @@ export const TimelineGalleryLayout: React.FC = ({ const showDates = gallerySettings.timelineShowDates !== false; const canQuickComment = Boolean(feedbackEnabled && feedbackOptions?.allowComments && onOpenPhotoWithFeedback); + // Tolerant timestamp parse: SQLite installs can hold epoch numbers in + // uploaded_at (pre-fix archive restores) — parseISO throws on numbers + // and crashed the whole Timeline view (#485 class). The API normalizes + // to ISO now; this guard covers stale caches and old backends. + const parseUploadedAt = (value: string | number) => + typeof value === 'string' ? parseISO(value) : new Date(value); + // Group photos by date const groupedPhotos = useMemo(() => { const groups = new Map(); photos.forEach(photo => { - const date = parseISO(photo.uploaded_at); + const date = parseUploadedAt(photo.uploaded_at); let groupKey: string; switch (grouping) { @@ -76,7 +83,7 @@ export const TimelineGalleryLayout: React.FC = ({ return Array.from(groups.entries()) .map(([date, photos]) => ({ date, - label: photos[0] ? format(parseISO(photos[0].uploaded_at), grouping === 'month' ? 'MMMM yyyy' : grouping === 'week' ? "'Week of' MMM d, yyyy" : 'EEEE, MMMM d, yyyy') : date, + label: photos[0] ? format(parseUploadedAt(photos[0].uploaded_at), grouping === 'month' ? 'MMMM yyyy' : grouping === 'week' ? "'Week of' MMM d, yyyy" : 'EEEE, MMMM d, yyyy') : date, photos: photos.sort((a, b) => new Date(b.uploaded_at).getTime() - new Date(a.uploaded_at).getTime()) })) .sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime());