From ac7ef266dcd0d2b146c9739710c7631e672ef40f Mon Sep 17 00:00:00 2001 From: Paul Nothaft <53005142+the-luap@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:14:48 +0200 Subject: [PATCH] fix(admin): make "Storage used" report storage used (#1164) (#1177) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(admin): make "Storage used" report storage used (#1164) Stable twin of #1170. The tile summed photos.size_bytes — the catalogued size of the ORIGINALS, which in reference mode live on external storage and have no relationship to the disk PicPeak runs on. The reporter's tile read ~80 GB against 21 GB of real usage. Worse than the label: the same number drove the soft-limit warning bar and, via /storage/info, the recommended soft limit — so a reference-mode install got a disk-capacity recommendation computed from bytes that are not on the disk. - new localStorageUsage service walks the storage root and reports the total plus a breakdown. Walking rather than summing DB columns is the point: thumbnail/preview/hero rows record a key and never a byte count, and orphans from a deleted event or an interrupted import are real bytes. - the external media root is excluded when it sits inside the storage root. Its compose default is /external-media, where the NAS is bind-mounted — a plain directory, not a symlink — so walking it would put every referenced original back into a figure whose purpose is to leave them out. Symlinks are not followed either. - .download-cache gets its own line: it lives inside the event directory, so the naive rule files a multi-GB zip as photography. - concurrent cold-cache callers share one walk; the dashboard, /storage/info and the sidebar are routinely requested together. - S3 installs keep the catalogued figure and the walk is skipped before it runs, since the objects are in the bucket and STORAGE_PATH holds only incidental local files. - an absent measurement reads as "unavailable" and a partial one is marked `+` across the dashboard, analytics, sidebar and status tab — a floor silently compared against a soft limit reads as "safely under". Verified on this branch: 11 new service tests, dashboardScope updated for the changed contract, full suite leaves the same 5 pre-existing failures as origin/stable. Frontend 20 files / 104 tests, tsc clean. * fix(admin): tell "no disk to measure" apart from "the measurement failed" (#1164) External review found both of these on this branch. Both were reported as `storage_measurement: 'catalog'`, so a failed local walk made the dashboard claim the objects live in S3. They are different things — one is a fact about the install, the other is a fault — and there is now an `unavailable` state for the second. The analytics percentage could reach the billions. `safeSoftLimit` fell back to `storageUsed || 1`, and on S3 that is null → 1, while the figure beside it came from `catalogedBytes`. An editor or viewer holds `analytics.view` but not `settings.view`, so `/storage/info` 403s for them and `storageInfo` is undefined — which is exactly when that fallback fires. It now falls back to the measured figure, and suppresses the percentage entirely when there is no real limit rather than dividing usage by itself and always reading 100%. --------- Co-authored-by: Paul Nothaft --- .../__tests__/routes/dashboardScope.test.js | 39 +++- .../services/localStorageUsage.test.js | 201 ++++++++++++++++ backend/src/routes/adminDashboard.js | 41 +++- backend/src/routes/adminSettings.js | 30 ++- backend/src/services/localStorageUsage.js | 216 ++++++++++++++++++ .../src/components/admin/AdminSidebar.tsx | 4 +- .../src/features/settings/tabs/StatusTab.tsx | 5 +- frontend/src/i18n/locales/de.json | 10 +- frontend/src/i18n/locales/en.json | 10 +- frontend/src/i18n/locales/fr.json | 10 +- frontend/src/i18n/locales/sl.json | 10 +- frontend/src/pages/admin/AdminDashboard.tsx | 15 +- frontend/src/pages/admin/AnalyticsPage.tsx | 41 +++- frontend/src/services/admin.service.ts | 19 +- frontend/src/services/settings.service.ts | 8 + 15 files changed, 634 insertions(+), 25 deletions(-) create mode 100644 backend/__tests__/services/localStorageUsage.test.js create mode 100644 backend/src/services/localStorageUsage.js diff --git a/backend/__tests__/routes/dashboardScope.test.js b/backend/__tests__/routes/dashboardScope.test.js index ece77993..54e505aa 100644 --- a/backend/__tests__/routes/dashboardScope.test.js +++ b/backend/__tests__/routes/dashboardScope.test.js @@ -127,7 +127,44 @@ describe('dashboard scoping (GHSA-c2jj / gqx7 / jhcf)', () => { expect(res.status).toBe(200); expect(Number(res.body.totalEvents)).toBe(1); expect(Number(res.body.totalPhotos)).toBe(1); - expect(Number(res.body.storageUsed)).toBe(1000); + // The catalogued original bytes — this is what carries the per-event + // scoping, and what `storageUsed` reported before #1164. + expect(Number(res.body.catalogedBytes)).toBe(1000); + }); + + it('/stats reports disk usage unscoped, because disk is not per-event', async () => { + // storageUsed is a measurement of the storage root (#1164), so it is the + // same number for every admin by design. Pinned so a future reviewer + // reading "everything on this endpoint is scoped" does not turn it into a + // sum of this editor's photos again — which is the bug that was fixed. + const res = await request(app) + .get('/api/admin/dashboard/stats') + .set('Authorization', `Bearer ${editorToken}`); + + expect(res.status).toBe(200); + expect(res.body.storageUsed).not.toBe(1000); + expect(res.body).toHaveProperty('storageBreakdown'); + }); + + it('/stats reports the catalogued figure on an S3 backend, not a near-zero disk walk', async () => { + // STORAGE_PATH holds only incidental local files when objects live in a + // bucket, so walking it would report near-zero and drag the soft-limit + // recommendation with it. + const prev = process.env.STORAGE_BACKEND; + process.env.STORAGE_BACKEND = 's3'; + try { + const res = await request(app) + .get('/api/admin/dashboard/stats') + .set('Authorization', `Bearer ${editorToken}`); + + expect(res.status).toBe(200); + expect(res.body.storageUsed).toBeNull(); + expect(res.body.storageMeasurement).toBe('catalog'); + expect(Number(res.body.catalogedBytes)).toBe(1000); + } finally { + if (prev === undefined) delete process.env.STORAGE_BACKEND; + else process.env.STORAGE_BACKEND = prev; + } }); it('/analytics does not expose a foreign gallery name or slug', async () => { diff --git a/backend/__tests__/services/localStorageUsage.test.js b/backend/__tests__/services/localStorageUsage.test.js new file mode 100644 index 00000000..2b98cfb5 --- /dev/null +++ b/backend/__tests__/services/localStorageUsage.test.js @@ -0,0 +1,201 @@ +/** + * "Storage used" has to mean storage used (#1164). + * + * The tile summed photos.size_bytes, so on a reference-mode install it + * reported the size of files sitting on a NAS — the reporter's read ~80 GB + * against 21 GB of real local usage — while omitting everything PicPeak does + * write locally, including an 11.8 GB download-cache zip. + * + * These pin the measurement against a real directory tree, since the whole + * point is counting bytes that are actually there. + */ + +const path = require('path'); +const fs = require('fs'); +const os = require('os'); + +const { + measureLocalStorageUsage, + resetLocalStorageUsageCache, +} = require('../../src/services/localStorageUsage'); + +describe('localStorageUsage (#1164)', () => { + let root; + + const write = async (rel, bytes) => { + const full = path.join(root, rel); + await fs.promises.mkdir(path.dirname(full), { recursive: true }); + await fs.promises.writeFile(full, Buffer.alloc(bytes)); + }; + + beforeEach(async () => { + root = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'picpeak-usage-')); + process.env.STORAGE_PATH = root; + delete process.env.EXTERNAL_MEDIA_ROOT; + jest.resetModules(); + resetLocalStorageUsageCache(); + }); + + afterEach(async () => { + await fs.promises.rm(root, { recursive: true, force: true }).catch(() => {}); + delete process.env.STORAGE_PATH; + delete process.env.EXTERNAL_MEDIA_ROOT; + resetLocalStorageUsageCache(); + }); + + it('counts every byte under the storage root', async () => { + await write(path.join('events', 'active', 'wed', 'individual', 'a.jpg'), 1000); + await write(path.join('thumbnails', 'a.jpg'), 100); + await write(path.join('previews', 'a.jpg'), 300); + + const usage = await measureLocalStorageUsage(); + + expect(usage.total).toBe(1400); + expect(usage.files).toBe(3); + }); + + it('breaks the total down by what the bytes are', async () => { + // The specific complaint: the derived artefacts PicPeak writes were + // invisible, so "what is filling my disk" had no answer in the UI. + await write(path.join('events', 'active', 'wed', 'individual', 'a.jpg'), 1000); + await write(path.join('events', 'archived', 'old.zip'), 5000); + await write(path.join('thumbnails', 'a.jpg'), 100); + await write(path.join('previews', 'a.jpg'), 300); + await write(path.join('heroes', 'a.jpg'), 200); + await write(path.join('watermarks', 'a.jpg'), 700); + await write(path.join('uploads', 'logo.png'), 50); + + const { breakdown } = await measureLocalStorageUsage(); + + expect(breakdown).toMatchObject({ + originals: 1000, + archives: 5000, + thumbnails: 100, + previews: 300, + heroes: 200, + watermarks: 700, + uploads: 50, + }); + }); + + it('files the download cache separately from the originals it sits among', async () => { + // `.download-cache` lives INSIDE the event directory, so the naive rule + // files an 11.8 GB zip as photography. It is the one bucket that is pure + // disposable cache and the one an admin most needs to see. + await write(path.join('events', 'active', 'wed', 'individual', 'a.jpg'), 1000); + await write(path.join('events', 'active', 'wed', '.download-cache', 'all.zip'), 9000); + + const { breakdown, total } = await measureLocalStorageUsage(); + + expect(breakdown.downloadCache).toBe(9000); + expect(breakdown.originals).toBe(1000); + expect(total).toBe(10000); + }); + + it('counts orphans no database row knows about', async () => { + // A deleted event's leftovers and an interrupted import's thumbnails are + // real bytes on a real disk. Summing DB columns would miss them, which is + // half of why this walks instead. + await write(path.join('thumbnails', 'ext999_gone.jpg'), 777); + + expect((await measureLocalStorageUsage()).total).toBe(777); + }); + + it('reports zero on a fresh install rather than failing', async () => { + const usage = await measureLocalStorageUsage(); + + expect(usage.total).toBe(0); + expect(usage.partial).toBe(false); + }); + + it('survives a storage root that does not exist', async () => { + process.env.STORAGE_PATH = path.join(root, 'nope'); + resetLocalStorageUsageCache(); + + const usage = await measureLocalStorageUsage(); + + // ENOENT on the root is a fresh/misconfigured install, not a partial read. + expect(usage.total).toBe(0); + expect(usage.partial).toBe(false); + }); + + it('does not walk the media share bind-mounted under the storage root', async () => { + // The compose default puts EXTERNAL_MEDIA_ROOT at /external-media, + // where the NAS is bind-mounted — a plain directory, not a symlink. Walking + // it would put every referenced original back into a figure that exists to + // leave them out, which is the over-count this measurement replaces. + await write(path.join('thumbnails', 'a.jpg'), 100); + await write(path.join('external-media', 'nas', 'huge.jpg'), 50000); + process.env.EXTERNAL_MEDIA_ROOT = path.join(root, 'external-media'); + jest.resetModules(); + const svc = require('../../src/services/localStorageUsage'); + svc.resetLocalStorageUsageCache(); + + const usage = await svc.measureLocalStorageUsage(); + + expect(usage.total).toBe(100); + expect(usage.excludedExternalRoot).toBe(path.join(root, 'external-media')); + }); + + it('still counts a directory that merely looks like the media share', async () => { + // Only the CONFIGURED root is skipped. An install whose media lives + // elsewhere keeps whatever is in this directory in the total, because + // those really are local bytes. + await write(path.join('external-media', 'leftover.jpg'), 700); + // The production shape: the share is mounted well outside the storage root. + const elsewhere = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'picpeak-nas-elsewhere-')); + process.env.EXTERNAL_MEDIA_ROOT = elsewhere; + jest.resetModules(); + const svc = require('../../src/services/localStorageUsage'); + svc.resetLocalStorageUsageCache(); + + const usage = await svc.measureLocalStorageUsage(); + + expect(usage.total).toBe(700); + expect(usage.excludedExternalRoot).toBeNull(); + await fs.promises.rm(elsewhere, { recursive: true, force: true }); + }); + + it('shares one walk between concurrent cold-cache callers', async () => { + // /dashboard/stats and /storage/info are routinely requested together, and + // the sidebar adds a third. Each starting its own full stat-per-file walk + // multiplies the cost on exactly the large libraries where it hurts. + await write(path.join('thumbnails', 'a.jpg'), 100); + const readdir = jest.spyOn(fs.promises, 'readdir'); + + const [a, b, c] = await Promise.all([ + measureLocalStorageUsage(), + measureLocalStorageUsage(), + measureLocalStorageUsage(), + ]); + + expect([a.total, b.total, c.total]).toEqual([100, 100, 100]); + // One walk: the storage root plus its one subdirectory. + expect(readdir).toHaveBeenCalledTimes(2); + readdir.mockRestore(); + }); + + it('caches, and honours force', async () => { + await write(path.join('thumbnails', 'a.jpg'), 100); + expect((await measureLocalStorageUsage()).total).toBe(100); + + await write(path.join('thumbnails', 'b.jpg'), 400); + // One stat per file is not free; the dashboard polls. + expect((await measureLocalStorageUsage()).total).toBe(100); + expect((await measureLocalStorageUsage({ force: true })).total).toBe(500); + }); + + it('does not follow a symlink out of the storage root', async () => { + // A link into EXTERNAL_MEDIA_ROOT would add the NAS back into the local + // total — reinstating the exact confusion this replaces. + const outside = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'picpeak-nas-')); + await fs.promises.writeFile(path.join(outside, 'huge.jpg'), Buffer.alloc(50000)); + await fs.promises.mkdir(path.join(root, 'events'), { recursive: true }); + await fs.promises.symlink(outside, path.join(root, 'events', 'nas'), 'dir'); + + const usage = await measureLocalStorageUsage(); + + expect(usage.total).toBe(0); + await fs.promises.rm(outside, { recursive: true, force: true }); + }); +}); diff --git a/backend/src/routes/adminDashboard.js b/backend/src/routes/adminDashboard.js index 9c04840e..dd0c9be7 100644 --- a/backend/src/routes/adminDashboard.js +++ b/backend/src/routes/adminDashboard.js @@ -7,6 +7,7 @@ const { formatBoolean } = require('../utils/dbCompat'); const { resolveAdapter } = require('../services/trackers'); const logger = require('../utils/logger'); const { errorResponse, getPagination } = require('../utils/routeHelpers'); +const { measureLocalStorageUsage } = require('../services/localStorageUsage'); const router = express.Router(); /** @@ -88,11 +89,33 @@ router.get('/stats', adminAuth, requirePermission('analytics.view'), async (req, .count('id as count') .first(); - // Get storage usage (sum of all photo sizes) - const storageUsed = await applyEventScope(db('photos'), req.admin, 'event_id') + // The catalogued size of the ORIGINALS. Kept, and still worth showing — + // it answers "how much photography is in here" — but it is emphatically + // NOT storage used, which is what it was labelled for years (#1164). + const catalogedBytes = await applyEventScope(db('photos'), req.admin, 'event_id') .sum('size_bytes as total') .first(); + // Storage used: what is actually on this machine. In reference mode the + // originals above live on a NAS and contribute nothing here; conversely + // this counts what the sum never did — thumbnails, previews, hero + // renditions, watermarks and the per-event download cache. + // + // Deliberately NOT event-scoped, unlike everything else on this endpoint: + // it is a disk measurement, and disk is not divisible by which admin owns + // which event. + // + // Skipped entirely on an S3 backend: the objects are in the bucket and a + // walk of STORAGE_PATH would report near-zero, which is worse than the + // catalogued figure those installs had before #1164. + const usesLocalBackend = (process.env.STORAGE_BACKEND || 'local').toLowerCase() !== 's3'; + let localStorage = null; + try { + if (usesLocalBackend) localStorage = await measureLocalStorageUsage(); + } catch (err) { + logger.warn(`Dashboard storage measurement failed: ${err.message}`); + } + // Get total views (last 30 days) const thirtyDaysAgo = new Date(); thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30); @@ -154,7 +177,19 @@ router.get('/stats', adminAuth, requirePermission('analytics.view'), async (req, activeEvents: activeEvents.count || 0, expiringEvents: expiringEvents.count || 0, totalPhotos: totalPhotos.count || 0, - storageUsed: storageUsed.total || 0, + // Real bytes on this disk. Null when the measurement failed or was + // skipped, which the UI shows as "unavailable" rather than substituting + // a number that means something else. + storageUsed: localStorage ? localStorage.total : null, + // Three states, not two: 'catalog' means the backend is S3 and the + // objects are in the bucket, which is a fact about the install; + // 'unavailable' means the walk failed, which is a fault. Collapsing them + // made a failed local measurement claim the objects live in S3. + storageMeasurement: localStorage ? 'disk' : (usesLocalBackend ? 'unavailable' : 'catalog'), + storageBreakdown: localStorage ? localStorage.breakdown : null, + storagePartial: localStorage ? localStorage.partial : false, + // Catalogued original bytes — what `storageUsed` used to report (#1164). + catalogedBytes: Number(catalogedBytes.total) || 0, totalViews: totalViews.count || 0, totalDownloads: totalDownloads.count || 0, viewsTrend: Math.round(viewsTrend * 10) / 10, diff --git a/backend/src/routes/adminSettings.js b/backend/src/routes/adminSettings.js index d2e25168..739c1c61 100644 --- a/backend/src/routes/adminSettings.js +++ b/backend/src/routes/adminSettings.js @@ -24,6 +24,7 @@ const { clearShareLinkSettingsCache } = require('../services/shareLinkService'); const { resetSecurityConfigCache } = require('../utils/authSecurity'); const { errorResponse } = require('../utils/routeHelpers'); const logger = require('../utils/logger'); +const { measureLocalStorageUsage } = require('../services/localStorageUsage'); const router = express.Router(); const { clearMaxFilesPerUploadCache, MAX_ALLOWED_FILES_PER_UPLOAD } = require('../services/uploadSettings'); const watermarkService = require('../services/watermarkService'); @@ -1174,6 +1175,8 @@ router.put('/seo', adminAuth, requirePermission('settings.edit'), async (req, re router.get('/storage/info', adminAuth, requirePermission('settings.view'), async (req, res) => { try { // Get total storage used + // Catalogued original bytes. Reported, but no longer as "used" (#1164) — + // in reference mode those files are on a NAS and none of them are here. const totalStorage = await db('photos') .sum('size_bytes as total') .first(); @@ -1254,7 +1257,25 @@ router.get('/storage/info', adminAuth, requirePermission('settings.view'), async } } - const totalUsed = totalStorage?.total || 0; + // What is actually on this disk. This is what the soft limit is compared + // against and what the recommendation below is derived from, so getting it + // from the catalogued originals was the load-bearing half of #1164: a + // reference-mode install got a disk-capacity recommendation computed from + // bytes that are not on the disk. + // + // Gated BEFORE the walk: this endpoint is polled by the sidebar, and an S3 + // install with a large local tree would otherwise pay a full traversal on + // every cold cache only to discard the result. + const catalogedBytes = Number(totalStorage?.total) || 0; + const usesLocalBackend = (process.env.STORAGE_BACKEND || 'local').toLowerCase() !== 's3'; + let localUsage = null; + try { + if (usesLocalBackend) localUsage = await measureLocalStorageUsage(); + } catch (err) { + logger.warn(`Storage measurement failed, falling back to catalogued bytes: ${err.message}`); + } + const measuredFromDisk = usesLocalBackend && !!localUsage; + const totalUsed = measuredFromDisk ? localUsage.total : catalogedBytes; const parseBytesValue = (value) => { const numeric = Number(value); @@ -1376,6 +1397,13 @@ router.get('/storage/info', adminAuth, requirePermission('settings.view'), async res.json({ total_used: totalUsed, + // What total_used used to be, kept so the UI can show both and the + // difference stops being invisible. + cataloged_bytes: catalogedBytes, + storage_measurement: measuredFromDisk ? 'disk' : (usesLocalBackend ? 'unavailable' : 'catalog'), + storage_breakdown: measuredFromDisk ? localUsage.breakdown : null, + storage_partial: measuredFromDisk ? localUsage.partial : false, + excluded_external_root: measuredFromDisk ? localUsage.excludedExternalRoot : null, archive_storage: archiveStorage, storage_by_event: storageByEvent, storage_limit: effectiveSoftLimit, diff --git a/backend/src/services/localStorageUsage.js b/backend/src/services/localStorageUsage.js new file mode 100644 index 00000000..7c529019 --- /dev/null +++ b/backend/src/services/localStorageUsage.js @@ -0,0 +1,216 @@ +/** + * What PicPeak actually occupies on this machine (#1164). + * + * The dashboard's "Storage used" tile summed photos.size_bytes, which is the + * catalogued size of the ORIGINALS — a number with no relationship to the disk + * PicPeak runs on: + * + * - in reference mode the originals are never copied. Those bytes are on the + * NAS. The reporter's tile read ~80 GB against 21 GB of real local usage. + * - duplicate rows counted the same file twice (#1162). + * - it ignored everything PicPeak genuinely does write: thumbnails, previews, + * hero renditions, and the per-event download cache — an 11.8 GB + * `.download-cache/all.zip` sat outside the figure entirely. + * + * So the one number an admin reaches for when asking "am I running out of + * disk" pointed away from the answer and omitted exactly the things filling + * the disk. This walks the storage root instead and reports what is there. + * + * Walking rather than summing DB columns is deliberate: thumbnail/preview/hero + * rows record a key, never a byte count, and orphans (a deleted event's + * leftovers, an interrupted import's thumbnails) are real bytes on a real + * disk. A `du` is the only honest answer, and the only one that notices what + * PicPeak has forgotten about. + * + * The external media root is EXCLUDED, and that is the whole point rather than + * a detail. Its compose default is `/external-media`, where the NAS is + * bind-mounted — a plain directory, not a symlink — so walking it would add + * every referenced original back into a figure that exists to leave them out, + * and compare NAS bytes against statfs() of the local disk. That is the + * over-count this replaces, reintroduced by the fix for it. + * + * Cached, because it is one stat per file. On a large install that is seconds, + * and the dashboard is polled — and concurrent misses share one walk rather + * than each starting their own. + */ + +const path = require('path'); +const fsp = require('fs').promises; +const logger = require('../utils/logger'); +const { getStoragePath } = require('../config/storage'); + +const TTL_MS = 5 * 60 * 1000; + +let cache = null; +// The walk in flight, if any. Two admins loading the dashboard, or the sidebar +// and the storage tab on one page, hit the same cold cache and would otherwise +// each stat every file on the disk. +let inFlight = null; + +/** + * The external media root, resolved, when it lies inside the storage root. + * Returns null when it is elsewhere (the usual production case) or cannot be + * resolved — nothing to exclude then. + */ +function nestedExternalRoot(storageRoot) { + let externalRoot; + try { + externalRoot = require('./externalMediaService').getExternalMediaRoot(); + } catch (err) { + return null; + } + if (!externalRoot) return null; + const resolvedExternal = path.resolve(externalRoot); + const resolvedStorage = path.resolve(storageRoot); + if (resolvedExternal === resolvedStorage) return null; + return resolvedExternal.startsWith(resolvedStorage + path.sep) ? resolvedExternal : null; +} + +/** + * Which line of the breakdown a path belongs to. + * + * The names are the ones the writers actually use — `heroes` from + * imageProcessor, `watermarks` from watermarkService, and so on — rather than + * the ones the layout docs imply. Getting one wrong is not a crash, it is a + * silent 30 MB in "other", which is the least useful place for it to land. + * + * `.download-cache` is the case that needs the explicit check: it lives INSIDE + * an event directory, so the naive rule files an 11.8 GB zip as photography — + * and it is the one bucket that is pure disposable cache, which makes it the + * one an admin most wants to see on its own. + * + * There is no external-media bucket: that subtree is not walked at all (see + * nestedExternalRoot). Those bytes live on the media share, and counting them + * is the exact over-count this measurement exists to end. + */ +function categorize(relPath) { + const segments = relPath.split(path.sep); + if (segments.includes('.download-cache')) return 'downloadCache'; + switch (segments[0]) { + case 'thumbnails': return 'thumbnails'; + case 'previews': return 'previews'; + case 'heroes': return 'heroes'; + case 'watermarks': return 'watermarks'; + case 'uploads': return 'uploads'; + case 'temp': return 'temp'; + case 'business-docs': return 'businessDocs'; + case 'events': + return segments[1] === 'archived' ? 'archives' : 'originals'; + default: + return 'other'; + } +} + +const EMPTY_BREAKDOWN = () => ({ + originals: 0, + archives: 0, + thumbnails: 0, + previews: 0, + heroes: 0, + watermarks: 0, + uploads: 0, + businessDocs: 0, + downloadCache: 0, + temp: 0, + other: 0, +}); + +async function walk(absDir, relDir, acc) { + // The media share, bind-mounted under the storage root. Walking it would put + // every referenced original back into a local-usage figure, and on a real + // NAS the traversal alone would take far longer than the measurement is + // worth. + if (acc.excludeRoot && path.resolve(absDir) === acc.excludeRoot) { + acc.excludedExternalRoot = acc.excludeRoot; + return; + } + + let entries; + try { + entries = await fsp.readdir(absDir, { withFileTypes: true }); + } catch (err) { + // A directory that is not there yet (a fresh install has no /previews) is + // not an error. Anything else is worth knowing about but must not abort + // the measurement — a partial number beats no number, and `partial` says + // so to the caller. + if (err.code !== 'ENOENT') { + acc.partial = true; + logger.debug?.(`localStorageUsage: skipped ${absDir}: ${err.message}`); + } + return; + } + + for (const entry of entries) { + const abs = path.join(absDir, entry.name); + const rel = relDir ? path.join(relDir, entry.name) : entry.name; + // Symlinks are not followed: a link into the external media mount would + // otherwise add the NAS to the local total, which is the exact confusion + // this replaces. + if (entry.isDirectory()) { + await walk(abs, rel, acc); + } else if (entry.isFile()) { + try { + const stats = await fsp.stat(abs); + acc.total += stats.size; + acc.files += 1; + acc.breakdown[categorize(rel)] += stats.size; + } catch (err) { + // Raced with a delete, most likely. Nothing to add. + if (err.code !== 'ENOENT') acc.partial = true; + } + } + } +} + +/** + * @param {{ force?: boolean }} [opts] force skips the TTL cache. + * @returns {Promise<{total:number, files:number, breakdown:object, partial:boolean, measuredAt:string, root:string}>} + */ +async function measureLocalStorageUsage(opts = {}) { + const now = Date.now(); + if (!opts.force && cache && now - cache.at < TTL_MS) return cache.value; + // Share a walk already underway rather than starting a second one. + if (!opts.force && inFlight) return inFlight; + + inFlight = runMeasurement() + .then((value) => { cache = { at: Date.now(), value }; return value; }) + .finally(() => { inFlight = null; }); + return inFlight; +} + +async function runMeasurement() { + + const root = getStoragePath(); + const acc = { + total: 0, + files: 0, + breakdown: EMPTY_BREAKDOWN(), + partial: false, + excludeRoot: nestedExternalRoot(root), + excludedExternalRoot: null, + }; + await walk(root, '', acc); + + return { + total: acc.total, + files: acc.files, + breakdown: acc.breakdown, + partial: acc.partial, + // Set when the media share sits inside the storage root and was skipped, + // so the UI can say why the figure is smaller than `du` would report. + excludedExternalRoot: acc.excludedExternalRoot, + measuredAt: new Date().toISOString(), + root, + }; +} + +/** Test seam — the TTL cache would otherwise outlive a temp storage root. */ +function resetLocalStorageUsageCache() { + cache = null; + inFlight = null; +} + +module.exports = { + measureLocalStorageUsage, + resetLocalStorageUsageCache, +}; diff --git a/frontend/src/components/admin/AdminSidebar.tsx b/frontend/src/components/admin/AdminSidebar.tsx index 3d154f9a..4d106834 100644 --- a/frontend/src/components/admin/AdminSidebar.tsx +++ b/frontend/src/components/admin/AdminSidebar.tsx @@ -361,7 +361,9 @@ const StorageInfo: React.FC = () => {
{t('admin.storageUsed')} - {settingsService.formatBytes(storageInfo.total_used)} + {/* `+` marks a floor: part of the storage root was unreadable, so + the real figure — and the percentage below — is higher (#1164). */} + {settingsService.formatBytes(storageInfo.total_used)}{storageInfo.storage_partial ? '+' : ''}
diff --git a/frontend/src/features/settings/tabs/StatusTab.tsx b/frontend/src/features/settings/tabs/StatusTab.tsx index 58580556..6159d980 100644 --- a/frontend/src/features/settings/tabs/StatusTab.tsx +++ b/frontend/src/features/settings/tabs/StatusTab.tsx @@ -229,7 +229,10 @@ export const StatusTab: React.FC = ({

{t('settings.storage.totalUsed')}

- {settingsService.formatBytes(storageInfo.total_used)} + {/* `+` marks a floor: part of the storage root was + unreadable, so the real figure — and the limit + percentage derived from it — is higher (#1164). */} + {settingsService.formatBytes(storageInfo.total_used)}{storageInfo.storage_partial ? '+' : ''}

diff --git a/frontend/src/i18n/locales/de.json b/frontend/src/i18n/locales/de.json index 03dde375..85e059e9 100644 --- a/frontend/src/i18n/locales/de.json +++ b/frontend/src/i18n/locales/de.json @@ -2834,7 +2834,10 @@ "email_template_created": "E-Mail-Vorlage erstellt: {{template_key}}", "event_duplicated": "Event dupliziert aus {{source_event_name}}", "whatsapp_config_updated": "WhatsApp-Konfiguration aktualisiert" - } + }, + "catalogedMedia": "{{size}} katalogisiert", + "catalogedMediaOnly": "katalogisiert — Objekte liegen in S3", + "storageUnavailable": "nicht verfügbar" }, "acceptInvitation": { "title": "Einladung annehmen", @@ -2955,7 +2958,10 @@ "totalPhotos": "Gesamte Fotos", "activeEvents": "Aktive Veranstaltungen", "notConfigured": "Umami Analytics nicht konfiguriert", - "configureInstructions": "Um echte Analysedaten zu sehen, konfigurieren Sie Umami in Ihren Umgebungsvariablen und Admin-Panel-Einstellungen." + "configureInstructions": "Um echte Analysedaten zu sehen, konfigurieren Sie Umami in Ihren Umgebungsvariablen und Admin-Panel-Einstellungen.", + "catalogedMedia": "Katalogisierte Medien", + "storageUnavailable": "nicht verfügbar", + "storageNoMeasurement": "keine Messung verfügbar" }, "email": { "title": "E-Mail-Konfiguration", diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index 1ad0051d..09521472 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -1650,7 +1650,10 @@ "totalPhotos": "Total Photos", "activeEvents": "Active Events", "notConfigured": "Umami Analytics Not Configured", - "configureInstructions": "To see real analytics data, configure Umami in your environment variables and admin panel settings." + "configureInstructions": "To see real analytics data, configure Umami in your environment variables and admin panel settings.", + "catalogedMedia": "Catalogued media", + "storageUnavailable": "unavailable", + "storageNoMeasurement": "no measurement available" }, "branding": { "title": "Branding & Themes", @@ -2406,7 +2409,10 @@ "email_template_created": "Email template created: {{template_key}}", "event_duplicated": "Event duplicated from {{source_event_name}}", "whatsapp_config_updated": "WhatsApp configuration updated" - } + }, + "catalogedMedia": "{{size}} catalogued", + "catalogedMediaOnly": "catalogued — objects live in S3", + "storageUnavailable": "unavailable" }, "acceptInvitation": { "title": "Accept Invitation", diff --git a/frontend/src/i18n/locales/fr.json b/frontend/src/i18n/locales/fr.json index 2783b102..e51c7b00 100644 --- a/frontend/src/i18n/locales/fr.json +++ b/frontend/src/i18n/locales/fr.json @@ -1213,7 +1213,10 @@ "totalPhotos": "Photos totales", "activeEvents": "Événements actifs", "notConfigured": "Umami Analytics non configuré", - "configureInstructions": "Pour voir les données analytiques réelles, configurez Umami dans vos variables d'environnement et les paramètres du panneau d'administration." + "configureInstructions": "Pour voir les données analytiques réelles, configurez Umami dans vos variables d'environnement et les paramètres du panneau d'administration.", + "catalogedMedia": "Médias catalogués", + "storageUnavailable": "indisponible", + "storageNoMeasurement": "aucune mesure disponible" }, "branding": { "title": "Marque & Thèmes", @@ -1817,7 +1820,10 @@ "category_hero_updated": "Photo de couverture de catégorie mise à jour", "public_site_reset_to_default": "Site public réinitialisé aux valeurs par défaut", "cms_page_logo_uploaded": "Logo de la page CMS téléversé : {{slug}}" - } + }, + "catalogedMedia": "{{size}} catalogués", + "catalogedMediaOnly": "catalogué — les objets sont dans S3", + "storageUnavailable": "indisponible" }, "acceptInvitation": { "title": "Accepter l'invitation", diff --git a/frontend/src/i18n/locales/sl.json b/frontend/src/i18n/locales/sl.json index 0c67a701..d1521f0e 100644 --- a/frontend/src/i18n/locales/sl.json +++ b/frontend/src/i18n/locales/sl.json @@ -1213,7 +1213,10 @@ "totalPhotos": "Skupaj fotografij", "activeEvents": "Aktivni dogodki", "notConfigured": "Umami Analytics ni nastavljen", - "configureInstructions": "Za prikaz resničnih analitičnih podatkov nastavite Umami v okoljskih spremenljivkah in nastavitvah administratorske plošče." + "configureInstructions": "Za prikaz resničnih analitičnih podatkov nastavite Umami v okoljskih spremenljivkah in nastavitvah administratorske plošče.", + "catalogedMedia": "Katalogizirani mediji", + "storageUnavailable": "ni na voljo", + "storageNoMeasurement": "meritev ni na voljo" }, "branding": { "title": "Blagovna znamka in teme", @@ -1806,7 +1809,10 @@ "category_hero_updated": "Hero fotografija kategorije posodobljena", "public_site_reset_to_default": "Javna stran ponastavljena na privzeto", "cms_page_logo_uploaded": "Logotip CMS strani naložen: {{slug}}" - } + }, + "catalogedMedia": "{{size}} katalogizirano", + "catalogedMediaOnly": "katalogizirano — objekti so v S3", + "storageUnavailable": "ni na voljo" }, "acceptInvitation": { "title": "Sprejmi povabilo", diff --git a/frontend/src/pages/admin/AdminDashboard.tsx b/frontend/src/pages/admin/AdminDashboard.tsx index d4ea94fa..5c6b8425 100644 --- a/frontend/src/pages/admin/AdminDashboard.tsx +++ b/frontend/src/pages/admin/AdminDashboard.tsx @@ -169,8 +169,21 @@ export const AdminDashboard: React.FC = () => { color: 'text-blue-600', }, { + // Real bytes under the storage root (#1164). This used to be the summed + // size of the catalogued originals, which on a reference-mode install is + // the size of a NAS. `== null` rather than `||`: an absent measurement + // must read as unavailable, not as 0 Bytes. title: t('admin.storageUsed'), - value: adminService.formatBytes(dashboardStats?.storageUsed || 0), + value: dashboardStats?.storageUsed == null + ? (dashboardStats?.storageMeasurement === 'catalog' + ? adminService.formatBytes(dashboardStats.catalogedBytes) + : t('admin.storageUnavailable', 'unavailable')) + : `${adminService.formatBytes(dashboardStats.storageUsed)}${dashboardStats.storagePartial ? '+' : ''}`, + change: dashboardStats + ? (dashboardStats.storageMeasurement === 'catalog' + ? t('admin.catalogedMediaOnly', 'catalogued — objects live in S3') + : t('admin.catalogedMedia', { size: adminService.formatBytes(dashboardStats.catalogedBytes) })) + : undefined, icon: HardDrive, color: 'text-purple-600', }, diff --git a/frontend/src/pages/admin/AnalyticsPage.tsx b/frontend/src/pages/admin/AnalyticsPage.tsx index f1253a54..68a2d2d9 100644 --- a/frontend/src/pages/admin/AnalyticsPage.tsx +++ b/frontend/src/pages/admin/AnalyticsPage.tsx @@ -438,15 +438,30 @@ export const AnalyticsPage: React.FC = () => { {/* Storage Information */} {dashboardStats && (() => { + // On S3 there is no disk to walk and the catalogued figure IS the + // available answer; a failed local walk has none at all. + const measured = dashboardStats.storageUsed ?? ( + dashboardStats.storageMeasurement === 'catalog' ? dashboardStats.catalogedBytes : null); const softLimitBytes = storageInfo?.storage_soft_limit ?? storageInfo?.storage_limit ?? storageInfo?.recommended_soft_limit ?? null; + // `measured`, not storageUsed. An editor or viewer holds + // analytics.view but not settings.view, so /storage/info 403s and + // storageInfo is undefined — and on S3 storageUsed is null, which + // made this denominator 1 and rendered percentages in the billions. const safeSoftLimit = Math.max( - softLimitBytes ?? storageInfo?.recommended_soft_limit ?? (dashboardStats.storageUsed || 1), + softLimitBytes ?? storageInfo?.recommended_soft_limit ?? (measured || 1), 1 ); - const usageRatio = dashboardStats.storageUsed / safeSoftLimit; - const usagePercent = Math.round(usageRatio * 100); - const usageWidth = Math.min(usageRatio * 100, 100); - const overSoftLimit = softLimitBytes != null && dashboardStats.storageUsed >= softLimitBytes; + // Without a real limit there is no percentage worth showing: the + // denominator would be the usage itself, which always reads 100%. + const hasLimit = softLimitBytes != null || storageInfo?.recommended_soft_limit != null; + // No measurement, or no limit, means no percentage. Coercing null + // to 0 drew an empty bar at "0% of limit" and suppressed the + // over-limit state — reading as plenty of room precisely when + // nothing is known (#1164). + const usageRatio = (measured == null || !hasLimit) ? null : measured / safeSoftLimit; + const usagePercent = usageRatio == null ? null : Math.round(usageRatio * 100); + const usageWidth = usageRatio == null ? 0 : Math.min(usageRatio * 100, 100); + const overSoftLimit = softLimitBytes != null && measured != null && measured >= softLimitBytes; const limitDisplay = softLimitBytes != null ? adminService.formatBytes(softLimitBytes) : storageInfo?.recommended_soft_limit != null @@ -454,7 +469,7 @@ export const AnalyticsPage: React.FC = () => { : t('settings.storage.unlimited'); const progressColor = overSoftLimit ? 'bg-red-600' - : usagePercent >= 90 + : (usagePercent != null && usagePercent >= 90) ? 'bg-amber-500' : 'bg-accent-dark'; const limitDescriptor = storageInfo @@ -470,7 +485,11 @@ export const AnalyticsPage: React.FC = () => {
{t('analytics.used')} - {adminService.formatBytes(dashboardStats.storageUsed)} + + {measured == null + ? t('analytics.storageUnavailable', 'unavailable') + : `${adminService.formatBytes(measured)}${dashboardStats.storagePartial ? '+' : ''}`} +
{ />

- {usagePercent}% {t('analytics.of')} {limitDisplay} + {usagePercent == null + ? t('analytics.storageNoMeasurement', 'no measurement available') + : `${usagePercent}% ${t('analytics.of')} ${limitDisplay}`}

{limitDescriptor} @@ -487,6 +508,10 @@ export const AnalyticsPage: React.FC = () => {

+ {t('analytics.catalogedMedia', 'Catalogued media')} + {adminService.formatBytes(dashboardStats.catalogedBytes)} +
+
{t('analytics.totalPhotos')} {dashboardStats.totalPhotos.toLocaleString()}
diff --git a/frontend/src/services/admin.service.ts b/frontend/src/services/admin.service.ts index 46591991..53c53185 100644 --- a/frontend/src/services/admin.service.ts +++ b/frontend/src/services/admin.service.ts @@ -61,7 +61,24 @@ export interface DashboardStats { activeEvents: number; expiringEvents: number; totalPhotos: number; - storageUsed: number; + // Real bytes under the storage root — thumbnails, previews, hero + // renditions, watermarks, download caches and any managed originals + // (#1164). Null when the measurement failed, or when the backend is S3 and + // the objects live in the bucket. The UI must show that as unavailable + // rather than substituting `catalogedBytes`: they are different quantities + // and on a reference-mode install they are wildly different. + storageUsed: number | null; + // 'catalog' when the backend is S3 — the objects are in the bucket, so no + // disk walk was made. 'unavailable' when a local walk was attempted and + // failed. Distinct because the first is a fact about the install and the + // second is a fault, and the UI must not claim S3 for a broken measurement. + storageMeasurement?: 'disk' | 'catalog' | 'unavailable'; + storageBreakdown: Record | null; + // The total is a floor: part of the storage root could not be read. + storagePartial?: boolean; + // Summed photos.size_bytes — what this endpoint used to label "storage + // used". In reference mode those files are on external storage. + catalogedBytes: number; totalViews: number; totalDownloads: number; viewsTrend: number; diff --git a/frontend/src/services/settings.service.ts b/frontend/src/services/settings.service.ts index d278f687..078bf69e 100644 --- a/frontend/src/services/settings.service.ts +++ b/frontend/src/services/settings.service.ts @@ -81,7 +81,15 @@ export interface PasswordComplexitySettings { } export interface StorageInfo { + // Real bytes under the storage root (#1164), excluding the external media + // share. Was the summed size of the catalogued originals, which on a + // reference-mode install is the size of a NAS. total_used: number; + cataloged_bytes?: number; + // True when part of the storage root could not be read, so total_used is a + // floor. Anything comparing it against a limit has to say so. + storage_partial?: boolean; + storage_measurement?: 'disk' | 'catalog' | 'unavailable'; archive_storage: number; storage_by_event: Array<{ event_name: string;