diff --git a/backend/__tests__/routes/dashboardScope.test.js b/backend/__tests__/routes/dashboardScope.test.js index ece77993..1b1d640a 100644 --- a/backend/__tests__/routes/dashboardScope.test.js +++ b/backend/__tests__/routes/dashboardScope.test.js @@ -127,7 +127,44 @@ describe('dashboard scoping (GHSA-c2jj / gqx7 / jhcf)', () => { expect(res.status).toBe(200); expect(Number(res.body.totalEvents)).toBe(1); expect(Number(res.body.totalPhotos)).toBe(1); - expect(Number(res.body.storageUsed)).toBe(1000); + // The catalogued original bytes — this is what carries the per-event + // scoping, and what `storageUsed` reported before #1164. + expect(Number(res.body.catalogedBytes)).toBe(1000); + }); + + it('/stats reports disk usage unscoped, because disk is not per-event', async () => { + // storageUsed is a measurement of the storage root (#1164), so it is the + // same number for every admin by design. Pinned so a future reviewer + // reading "everything on this endpoint is scoped" does not turn it into a + // sum of this editor's photos again — which is the bug that was fixed. + const res = await request(app) + .get('/api/admin/dashboard/stats') + .set('Authorization', `Bearer ${editorToken}`); + + expect(res.status).toBe(200); + expect(res.body.storageUsed).not.toBe(1000); + expect(res.body).toHaveProperty('storageBreakdown'); + }); + + it('/stats reports the catalogued figure on an S3 backend, not a near-zero disk walk', async () => { + // STORAGE_PATH holds only incidental local files when objects live in a + // bucket, so walking it would report near-zero and drag the soft-limit + // recommendation with it (#1164 review). + const prev = process.env.STORAGE_BACKEND; + process.env.STORAGE_BACKEND = 's3'; + try { + const res = await request(app) + .get('/api/admin/dashboard/stats') + .set('Authorization', `Bearer ${editorToken}`); + + expect(res.status).toBe(200); + expect(res.body.storageUsed).toBeNull(); + expect(res.body.storageMeasurement).toBe('catalog'); + expect(Number(res.body.catalogedBytes)).toBe(1000); + } finally { + if (prev === undefined) delete process.env.STORAGE_BACKEND; + else process.env.STORAGE_BACKEND = prev; + } }); it('/analytics does not expose a foreign gallery name or slug', async () => { diff --git a/backend/__tests__/services/localStorageUsage.test.js b/backend/__tests__/services/localStorageUsage.test.js new file mode 100644 index 00000000..2b98cfb5 --- /dev/null +++ b/backend/__tests__/services/localStorageUsage.test.js @@ -0,0 +1,201 @@ +/** + * "Storage used" has to mean storage used (#1164). + * + * The tile summed photos.size_bytes, so on a reference-mode install it + * reported the size of files sitting on a NAS — the reporter's read ~80 GB + * against 21 GB of real local usage — while omitting everything PicPeak does + * write locally, including an 11.8 GB download-cache zip. + * + * These pin the measurement against a real directory tree, since the whole + * point is counting bytes that are actually there. + */ + +const path = require('path'); +const fs = require('fs'); +const os = require('os'); + +const { + measureLocalStorageUsage, + resetLocalStorageUsageCache, +} = require('../../src/services/localStorageUsage'); + +describe('localStorageUsage (#1164)', () => { + let root; + + const write = async (rel, bytes) => { + const full = path.join(root, rel); + await fs.promises.mkdir(path.dirname(full), { recursive: true }); + await fs.promises.writeFile(full, Buffer.alloc(bytes)); + }; + + beforeEach(async () => { + root = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'picpeak-usage-')); + process.env.STORAGE_PATH = root; + delete process.env.EXTERNAL_MEDIA_ROOT; + jest.resetModules(); + resetLocalStorageUsageCache(); + }); + + afterEach(async () => { + await fs.promises.rm(root, { recursive: true, force: true }).catch(() => {}); + delete process.env.STORAGE_PATH; + delete process.env.EXTERNAL_MEDIA_ROOT; + resetLocalStorageUsageCache(); + }); + + it('counts every byte under the storage root', async () => { + await write(path.join('events', 'active', 'wed', 'individual', 'a.jpg'), 1000); + await write(path.join('thumbnails', 'a.jpg'), 100); + await write(path.join('previews', 'a.jpg'), 300); + + const usage = await measureLocalStorageUsage(); + + expect(usage.total).toBe(1400); + expect(usage.files).toBe(3); + }); + + it('breaks the total down by what the bytes are', async () => { + // The specific complaint: the derived artefacts PicPeak writes were + // invisible, so "what is filling my disk" had no answer in the UI. + await write(path.join('events', 'active', 'wed', 'individual', 'a.jpg'), 1000); + await write(path.join('events', 'archived', 'old.zip'), 5000); + await write(path.join('thumbnails', 'a.jpg'), 100); + await write(path.join('previews', 'a.jpg'), 300); + await write(path.join('heroes', 'a.jpg'), 200); + await write(path.join('watermarks', 'a.jpg'), 700); + await write(path.join('uploads', 'logo.png'), 50); + + const { breakdown } = await measureLocalStorageUsage(); + + expect(breakdown).toMatchObject({ + originals: 1000, + archives: 5000, + thumbnails: 100, + previews: 300, + heroes: 200, + watermarks: 700, + uploads: 50, + }); + }); + + it('files the download cache separately from the originals it sits among', async () => { + // `.download-cache` lives INSIDE the event directory, so the naive rule + // files an 11.8 GB zip as photography. It is the one bucket that is pure + // disposable cache and the one an admin most needs to see. + await write(path.join('events', 'active', 'wed', 'individual', 'a.jpg'), 1000); + await write(path.join('events', 'active', 'wed', '.download-cache', 'all.zip'), 9000); + + const { breakdown, total } = await measureLocalStorageUsage(); + + expect(breakdown.downloadCache).toBe(9000); + expect(breakdown.originals).toBe(1000); + expect(total).toBe(10000); + }); + + it('counts orphans no database row knows about', async () => { + // A deleted event's leftovers and an interrupted import's thumbnails are + // real bytes on a real disk. Summing DB columns would miss them, which is + // half of why this walks instead. + await write(path.join('thumbnails', 'ext999_gone.jpg'), 777); + + expect((await measureLocalStorageUsage()).total).toBe(777); + }); + + it('reports zero on a fresh install rather than failing', async () => { + const usage = await measureLocalStorageUsage(); + + expect(usage.total).toBe(0); + expect(usage.partial).toBe(false); + }); + + it('survives a storage root that does not exist', async () => { + process.env.STORAGE_PATH = path.join(root, 'nope'); + resetLocalStorageUsageCache(); + + const usage = await measureLocalStorageUsage(); + + // ENOENT on the root is a fresh/misconfigured install, not a partial read. + expect(usage.total).toBe(0); + expect(usage.partial).toBe(false); + }); + + it('does not walk the media share bind-mounted under the storage root', async () => { + // The compose default puts EXTERNAL_MEDIA_ROOT at /external-media, + // where the NAS is bind-mounted — a plain directory, not a symlink. Walking + // it would put every referenced original back into a figure that exists to + // leave them out, which is the over-count this measurement replaces. + await write(path.join('thumbnails', 'a.jpg'), 100); + await write(path.join('external-media', 'nas', 'huge.jpg'), 50000); + process.env.EXTERNAL_MEDIA_ROOT = path.join(root, 'external-media'); + jest.resetModules(); + const svc = require('../../src/services/localStorageUsage'); + svc.resetLocalStorageUsageCache(); + + const usage = await svc.measureLocalStorageUsage(); + + expect(usage.total).toBe(100); + expect(usage.excludedExternalRoot).toBe(path.join(root, 'external-media')); + }); + + it('still counts a directory that merely looks like the media share', async () => { + // Only the CONFIGURED root is skipped. An install whose media lives + // elsewhere keeps whatever is in this directory in the total, because + // those really are local bytes. + await write(path.join('external-media', 'leftover.jpg'), 700); + // The production shape: the share is mounted well outside the storage root. + const elsewhere = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'picpeak-nas-elsewhere-')); + process.env.EXTERNAL_MEDIA_ROOT = elsewhere; + jest.resetModules(); + const svc = require('../../src/services/localStorageUsage'); + svc.resetLocalStorageUsageCache(); + + const usage = await svc.measureLocalStorageUsage(); + + expect(usage.total).toBe(700); + expect(usage.excludedExternalRoot).toBeNull(); + await fs.promises.rm(elsewhere, { recursive: true, force: true }); + }); + + it('shares one walk between concurrent cold-cache callers', async () => { + // /dashboard/stats and /storage/info are routinely requested together, and + // the sidebar adds a third. Each starting its own full stat-per-file walk + // multiplies the cost on exactly the large libraries where it hurts. + await write(path.join('thumbnails', 'a.jpg'), 100); + const readdir = jest.spyOn(fs.promises, 'readdir'); + + const [a, b, c] = await Promise.all([ + measureLocalStorageUsage(), + measureLocalStorageUsage(), + measureLocalStorageUsage(), + ]); + + expect([a.total, b.total, c.total]).toEqual([100, 100, 100]); + // One walk: the storage root plus its one subdirectory. + expect(readdir).toHaveBeenCalledTimes(2); + readdir.mockRestore(); + }); + + it('caches, and honours force', async () => { + await write(path.join('thumbnails', 'a.jpg'), 100); + expect((await measureLocalStorageUsage()).total).toBe(100); + + await write(path.join('thumbnails', 'b.jpg'), 400); + // One stat per file is not free; the dashboard polls. + expect((await measureLocalStorageUsage()).total).toBe(100); + expect((await measureLocalStorageUsage({ force: true })).total).toBe(500); + }); + + it('does not follow a symlink out of the storage root', async () => { + // A link into EXTERNAL_MEDIA_ROOT would add the NAS back into the local + // total — reinstating the exact confusion this replaces. + const outside = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'picpeak-nas-')); + await fs.promises.writeFile(path.join(outside, 'huge.jpg'), Buffer.alloc(50000)); + await fs.promises.mkdir(path.join(root, 'events'), { recursive: true }); + await fs.promises.symlink(outside, path.join(root, 'events', 'nas'), 'dir'); + + const usage = await measureLocalStorageUsage(); + + expect(usage.total).toBe(0); + await fs.promises.rm(outside, { recursive: true, force: true }); + }); +}); diff --git a/backend/src/routes/adminDashboard.js b/backend/src/routes/adminDashboard.js index 9c04840e..9202134b 100644 --- a/backend/src/routes/adminDashboard.js +++ b/backend/src/routes/adminDashboard.js @@ -7,6 +7,7 @@ const { formatBoolean } = require('../utils/dbCompat'); const { resolveAdapter } = require('../services/trackers'); const logger = require('../utils/logger'); const { errorResponse, getPagination } = require('../utils/routeHelpers'); +const { measureLocalStorageUsage } = require('../services/localStorageUsage'); const router = express.Router(); /** @@ -88,11 +89,38 @@ router.get('/stats', adminAuth, requirePermission('analytics.view'), async (req, .count('id as count') .first(); - // Get storage usage (sum of all photo sizes) - const storageUsed = await applyEventScope(db('photos'), req.admin, 'event_id') + // The catalogued size of the ORIGINALS. Kept, and still worth showing — + // it answers "how much photography is in here" — but it is emphatically + // NOT storage used, which is what it was labelled for years (#1164). + const catalogedBytes = await applyEventScope(db('photos'), req.admin, 'event_id') .sum('size_bytes as total') .first(); + // Storage used: what is actually on this machine. In reference mode the + // originals above live on a NAS and contribute nothing here; conversely + // this counts what the sum never did — thumbnails, previews, hero + // renditions and the per-event download cache. + // + // Deliberately NOT event-scoped, unlike everything else on this endpoint: + // it is a disk measurement, and disk is not divisible by which admin owns + // which event. A scoped admin gets the same number a super_admin does, + // which is the honest answer to "is this box running out of space" and + // leaks nothing beyond the aggregate they can already infer from the + // system-health page. + // Skipped entirely on an S3 backend: the objects are in the bucket and a + // walk of STORAGE_PATH would report near-zero, which is worse than the + // catalogued figure those installs had before #1164. `storageUsed` is null + // there and the tile shows the catalogued number it already carries. + const usesLocalBackend = (process.env.STORAGE_BACKEND || 'local').toLowerCase() !== 's3'; + let localStorage = null; + try { + if (usesLocalBackend) localStorage = await measureLocalStorageUsage(); + } catch (err) { + // The dashboard must render without it; the tile falls back to showing + // the catalogued figure, labelled as such. + logger.warn(`Dashboard storage measurement failed: ${err.message}`); + } + // Get total views (last 30 days) const thirtyDaysAgo = new Date(); thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30); @@ -154,7 +182,21 @@ router.get('/stats', adminAuth, requirePermission('analytics.view'), async (req, activeEvents: activeEvents.count || 0, expiringEvents: expiringEvents.count || 0, totalPhotos: totalPhotos.count || 0, - storageUsed: storageUsed.total || 0, + // Real bytes on this disk. Null when the measurement failed, which the + // UI shows as "unavailable" rather than substituting a number that + // means something else. + storageUsed: localStorage ? localStorage.total : null, + // Three states, not two: 'catalog' means the backend is S3 and the + // objects are in the bucket, which is a fact about the install; + // 'unavailable' means the walk failed, which is a fault. Collapsing them + // made a failed local measurement claim the objects live in S3. + storageMeasurement: localStorage ? 'disk' : (usesLocalBackend ? 'unavailable' : 'catalog'), + storageBreakdown: localStorage ? localStorage.breakdown : null, + // True when part of the storage root could not be read, so the total is + // a floor rather than the answer. + storagePartial: localStorage ? localStorage.partial : false, + // Catalogued original bytes — what `storageUsed` used to report (#1164). + catalogedBytes: Number(catalogedBytes.total) || 0, totalViews: totalViews.count || 0, totalDownloads: totalDownloads.count || 0, viewsTrend: Math.round(viewsTrend * 10) / 10, diff --git a/backend/src/routes/adminSettings.js b/backend/src/routes/adminSettings.js index d3f0383f..65b3c606 100644 --- a/backend/src/routes/adminSettings.js +++ b/backend/src/routes/adminSettings.js @@ -25,6 +25,7 @@ const { clearShareLinkSettingsCache } = require('../services/shareLinkService'); const { invalidateSiteUrlCache, isEnvPinned, envPinnedBase } = require('../utils/frontendUrl'); const { resetSecurityConfigCache } = require('../utils/authSecurity'); const { errorResponse } = require('../utils/routeHelpers'); +const { measureLocalStorageUsage } = require('../services/localStorageUsage'); const logger = require('../utils/logger'); const router = express.Router(); const { clearMaxFilesPerUploadCache, MAX_ALLOWED_FILES_PER_UPLOAD, clearMaxFileSizeCache, MAX_ALLOWED_FILE_SIZE_MB } = require('../services/uploadSettings'); @@ -1717,7 +1718,8 @@ router.put('/seo', adminAuth, requirePermission('settings.edit'), async (req, re // Get storage info router.get('/storage/info', adminAuth, requirePermission('settings.view'), async (req, res) => { try { - // Get total storage used + // Catalogued original bytes. Reported, but no longer as "used" (#1164) — + // in reference mode those files are on a NAS and none of them are here. const totalStorage = await db('photos') .sum('size_bytes as total') .first(); @@ -1798,7 +1800,32 @@ router.get('/storage/info', adminAuth, requirePermission('settings.view'), async } } - const totalUsed = totalStorage?.total || 0; + // What is actually on this disk. This is what the soft limit is compared + // against and what the recommendation below is derived from, so getting it + // from the catalogued originals was the load-bearing half of #1164: a + // reference-mode install got a disk-capacity recommendation computed from + // bytes that are not on the disk. + const catalogedBytes = Number(totalStorage?.total) || 0; + // Gated BEFORE the walk, not after. This endpoint is polled by the sidebar, + // and an S3 install that still has a large local tree from before the + // migration would otherwise pay a full stat-per-file traversal on every + // cold cache only to discard the result. + const usesLocalBackend = (process.env.STORAGE_BACKEND || 'local').toLowerCase() !== 's3'; + let localUsage = null; + try { + if (usesLocalBackend) localUsage = await measureLocalStorageUsage(); + } catch (err) { + logger.warn(`Storage measurement failed, falling back to catalogued bytes: ${err.message}`); + } + // On an S3 backend the originals, renditions, archives and download caches + // are all objects in the bucket, and STORAGE_PATH holds only incidental + // local files — so the walk would report near-zero and drag the soft-limit + // recommendation down with it. Those installs keep the catalogued figure, + // which is the approximation they had before #1164, and the response says + // which one this is so the UI can label it rather than implying a disk + // measurement it never made. + const measuredFromDisk = usesLocalBackend && !!localUsage; + const totalUsed = measuredFromDisk ? localUsage.total : catalogedBytes; const parseBytesValue = (value) => { const numeric = Number(value); @@ -1920,6 +1947,13 @@ router.get('/storage/info', adminAuth, requirePermission('settings.view'), async res.json({ total_used: totalUsed, + // What total_used used to be, kept so the UI can show both and the + // difference stops being invisible. + cataloged_bytes: catalogedBytes, + storage_measurement: measuredFromDisk ? 'disk' : (usesLocalBackend ? 'unavailable' : 'catalog'), + storage_breakdown: measuredFromDisk ? localUsage.breakdown : null, + storage_partial: measuredFromDisk ? localUsage.partial : false, + excluded_external_root: measuredFromDisk ? localUsage.excludedExternalRoot : null, archive_storage: archiveStorage, storage_by_event: storageByEvent, storage_limit: effectiveSoftLimit, diff --git a/backend/src/services/localStorageUsage.js b/backend/src/services/localStorageUsage.js new file mode 100644 index 00000000..7c529019 --- /dev/null +++ b/backend/src/services/localStorageUsage.js @@ -0,0 +1,216 @@ +/** + * What PicPeak actually occupies on this machine (#1164). + * + * The dashboard's "Storage used" tile summed photos.size_bytes, which is the + * catalogued size of the ORIGINALS — a number with no relationship to the disk + * PicPeak runs on: + * + * - in reference mode the originals are never copied. Those bytes are on the + * NAS. The reporter's tile read ~80 GB against 21 GB of real local usage. + * - duplicate rows counted the same file twice (#1162). + * - it ignored everything PicPeak genuinely does write: thumbnails, previews, + * hero renditions, and the per-event download cache — an 11.8 GB + * `.download-cache/all.zip` sat outside the figure entirely. + * + * So the one number an admin reaches for when asking "am I running out of + * disk" pointed away from the answer and omitted exactly the things filling + * the disk. This walks the storage root instead and reports what is there. + * + * Walking rather than summing DB columns is deliberate: thumbnail/preview/hero + * rows record a key, never a byte count, and orphans (a deleted event's + * leftovers, an interrupted import's thumbnails) are real bytes on a real + * disk. A `du` is the only honest answer, and the only one that notices what + * PicPeak has forgotten about. + * + * The external media root is EXCLUDED, and that is the whole point rather than + * a detail. Its compose default is `/external-media`, where the NAS is + * bind-mounted — a plain directory, not a symlink — so walking it would add + * every referenced original back into a figure that exists to leave them out, + * and compare NAS bytes against statfs() of the local disk. That is the + * over-count this replaces, reintroduced by the fix for it. + * + * Cached, because it is one stat per file. On a large install that is seconds, + * and the dashboard is polled — and concurrent misses share one walk rather + * than each starting their own. + */ + +const path = require('path'); +const fsp = require('fs').promises; +const logger = require('../utils/logger'); +const { getStoragePath } = require('../config/storage'); + +const TTL_MS = 5 * 60 * 1000; + +let cache = null; +// The walk in flight, if any. Two admins loading the dashboard, or the sidebar +// and the storage tab on one page, hit the same cold cache and would otherwise +// each stat every file on the disk. +let inFlight = null; + +/** + * The external media root, resolved, when it lies inside the storage root. + * Returns null when it is elsewhere (the usual production case) or cannot be + * resolved — nothing to exclude then. + */ +function nestedExternalRoot(storageRoot) { + let externalRoot; + try { + externalRoot = require('./externalMediaService').getExternalMediaRoot(); + } catch (err) { + return null; + } + if (!externalRoot) return null; + const resolvedExternal = path.resolve(externalRoot); + const resolvedStorage = path.resolve(storageRoot); + if (resolvedExternal === resolvedStorage) return null; + return resolvedExternal.startsWith(resolvedStorage + path.sep) ? resolvedExternal : null; +} + +/** + * Which line of the breakdown a path belongs to. + * + * The names are the ones the writers actually use — `heroes` from + * imageProcessor, `watermarks` from watermarkService, and so on — rather than + * the ones the layout docs imply. Getting one wrong is not a crash, it is a + * silent 30 MB in "other", which is the least useful place for it to land. + * + * `.download-cache` is the case that needs the explicit check: it lives INSIDE + * an event directory, so the naive rule files an 11.8 GB zip as photography — + * and it is the one bucket that is pure disposable cache, which makes it the + * one an admin most wants to see on its own. + * + * There is no external-media bucket: that subtree is not walked at all (see + * nestedExternalRoot). Those bytes live on the media share, and counting them + * is the exact over-count this measurement exists to end. + */ +function categorize(relPath) { + const segments = relPath.split(path.sep); + if (segments.includes('.download-cache')) return 'downloadCache'; + switch (segments[0]) { + case 'thumbnails': return 'thumbnails'; + case 'previews': return 'previews'; + case 'heroes': return 'heroes'; + case 'watermarks': return 'watermarks'; + case 'uploads': return 'uploads'; + case 'temp': return 'temp'; + case 'business-docs': return 'businessDocs'; + case 'events': + return segments[1] === 'archived' ? 'archives' : 'originals'; + default: + return 'other'; + } +} + +const EMPTY_BREAKDOWN = () => ({ + originals: 0, + archives: 0, + thumbnails: 0, + previews: 0, + heroes: 0, + watermarks: 0, + uploads: 0, + businessDocs: 0, + downloadCache: 0, + temp: 0, + other: 0, +}); + +async function walk(absDir, relDir, acc) { + // The media share, bind-mounted under the storage root. Walking it would put + // every referenced original back into a local-usage figure, and on a real + // NAS the traversal alone would take far longer than the measurement is + // worth. + if (acc.excludeRoot && path.resolve(absDir) === acc.excludeRoot) { + acc.excludedExternalRoot = acc.excludeRoot; + return; + } + + let entries; + try { + entries = await fsp.readdir(absDir, { withFileTypes: true }); + } catch (err) { + // A directory that is not there yet (a fresh install has no /previews) is + // not an error. Anything else is worth knowing about but must not abort + // the measurement — a partial number beats no number, and `partial` says + // so to the caller. + if (err.code !== 'ENOENT') { + acc.partial = true; + logger.debug?.(`localStorageUsage: skipped ${absDir}: ${err.message}`); + } + return; + } + + for (const entry of entries) { + const abs = path.join(absDir, entry.name); + const rel = relDir ? path.join(relDir, entry.name) : entry.name; + // Symlinks are not followed: a link into the external media mount would + // otherwise add the NAS to the local total, which is the exact confusion + // this replaces. + if (entry.isDirectory()) { + await walk(abs, rel, acc); + } else if (entry.isFile()) { + try { + const stats = await fsp.stat(abs); + acc.total += stats.size; + acc.files += 1; + acc.breakdown[categorize(rel)] += stats.size; + } catch (err) { + // Raced with a delete, most likely. Nothing to add. + if (err.code !== 'ENOENT') acc.partial = true; + } + } + } +} + +/** + * @param {{ force?: boolean }} [opts] force skips the TTL cache. + * @returns {Promise<{total:number, files:number, breakdown:object, partial:boolean, measuredAt:string, root:string}>} + */ +async function measureLocalStorageUsage(opts = {}) { + const now = Date.now(); + if (!opts.force && cache && now - cache.at < TTL_MS) return cache.value; + // Share a walk already underway rather than starting a second one. + if (!opts.force && inFlight) return inFlight; + + inFlight = runMeasurement() + .then((value) => { cache = { at: Date.now(), value }; return value; }) + .finally(() => { inFlight = null; }); + return inFlight; +} + +async function runMeasurement() { + + const root = getStoragePath(); + const acc = { + total: 0, + files: 0, + breakdown: EMPTY_BREAKDOWN(), + partial: false, + excludeRoot: nestedExternalRoot(root), + excludedExternalRoot: null, + }; + await walk(root, '', acc); + + return { + total: acc.total, + files: acc.files, + breakdown: acc.breakdown, + partial: acc.partial, + // Set when the media share sits inside the storage root and was skipped, + // so the UI can say why the figure is smaller than `du` would report. + excludedExternalRoot: acc.excludedExternalRoot, + measuredAt: new Date().toISOString(), + root, + }; +} + +/** Test seam — the TTL cache would otherwise outlive a temp storage root. */ +function resetLocalStorageUsageCache() { + cache = null; + inFlight = null; +} + +module.exports = { + measureLocalStorageUsage, + resetLocalStorageUsageCache, +}; diff --git a/frontend/src/components/admin/AdminSidebar.tsx b/frontend/src/components/admin/AdminSidebar.tsx index fb35a537..41659f58 100644 --- a/frontend/src/components/admin/AdminSidebar.tsx +++ b/frontend/src/components/admin/AdminSidebar.tsx @@ -379,7 +379,11 @@ const StorageInfo: React.FC = () => {
{t('admin.storageUsed')} - {settingsService.formatBytes(storageInfo.total_used)} + {/* The `+` marks a floor: part of the storage root was unreadable, + so the real figure — and the percentage below — is higher than + this. Without it an EACCES subtree reads as "safely under the + limit" (#1164). */} + {settingsService.formatBytes(storageInfo.total_used)}{storageInfo.storage_partial ? '+' : ''}
diff --git a/frontend/src/features/settings/tabs/StatusTab.tsx b/frontend/src/features/settings/tabs/StatusTab.tsx index 6f2d630a..6a6395af 100644 --- a/frontend/src/features/settings/tabs/StatusTab.tsx +++ b/frontend/src/features/settings/tabs/StatusTab.tsx @@ -228,8 +228,16 @@ export const StatusTab: React.FC = ({

{t('settings.storage.totalUsed')}

- {settingsService.formatBytes(storageInfo.total_used)} + {/* `+` marks a floor: part of the storage root was + unreadable, so the real figure — and the limit + percentage derived from it — is higher (#1164). */} + {settingsService.formatBytes(storageInfo.total_used)}{storageInfo.storage_partial ? '+' : ''}

+ {storageInfo.storage_measurement === 'catalog' && ( +

+ {t('settings.storage.catalogMeasurement', 'Catalogued size — objects live in the configured S3 bucket, not on this disk')} +

+ )}

{t('settings.storage.archiveStorage')}

diff --git a/frontend/src/i18n/locales/de.json b/frontend/src/i18n/locales/de.json index dd320767..9952acc0 100644 --- a/frontend/src/i18n/locales/de.json +++ b/frontend/src/i18n/locales/de.json @@ -3223,7 +3223,10 @@ "sidecarStateNow": "Aktueller Zustand des Dienstes — einige der Fehler oben können eine andere Ursache haben, aber ein erneuter Scan wird erst nach der Behebung erfolgreich sein:", "consolidated_one": "Beim letzten Scan wurde {{count}} ähnliches Paar automatisch zusammengeführt. Prüfen Sie es unter „Personen verwalten“ — falsch Zusammengeführtes lässt sich mit „Trennen“ wieder aufteilen.", "consolidated_other": "Beim letzten Scan wurden {{count}} ähnliche Paare automatisch zusammengeführt. Prüfen Sie sie unter „Personen verwalten“ — falsch Zusammengeführtes lässt sich mit „Trennen“ wieder aufteilen." - } + }, + "catalogedMedia": "{{size}} katalogisiert", + "storageUnavailable": "nicht verfügbar", + "catalogedMediaOnly": "katalogisiert — Objekte liegen in S3" }, "acceptInvitation": { "title": "Einladung annehmen", @@ -3345,7 +3348,10 @@ "totalPhotos": "Gesamte Fotos", "activeEvents": "Aktive Veranstaltungen", "notConfigured": "Umami Analytics nicht konfiguriert", - "configureInstructions": "Um echte Analysedaten zu sehen, konfigurieren Sie Umami in Ihren Umgebungsvariablen und Admin-Panel-Einstellungen." + "configureInstructions": "Um echte Analysedaten zu sehen, konfigurieren Sie Umami in Ihren Umgebungsvariablen und Admin-Panel-Einstellungen.", + "catalogedMedia": "Katalogisierte Medien", + "storageUnavailable": "nicht verfügbar", + "storageNoMeasurement": "keine Messung verfügbar" }, "email": { "title": "E-Mail-Konfiguration", diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index f41874b0..44d9e1eb 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -1935,7 +1935,10 @@ "totalPhotos": "Total Photos", "activeEvents": "Active Events", "notConfigured": "Umami Analytics Not Configured", - "configureInstructions": "To see real analytics data, configure Umami in your environment variables and admin panel settings." + "configureInstructions": "To see real analytics data, configure Umami in your environment variables and admin panel settings.", + "catalogedMedia": "Catalogued media", + "storageUnavailable": "unavailable", + "storageNoMeasurement": "no measurement available" }, "branding": { "title": "Branding & Themes", @@ -2795,7 +2798,10 @@ "sidecarStateNow": "Service state right now — some of the failures above may have a different cause, but a re-scan will not succeed until this is fixed:", "consolidated_one": "Grouping merged {{count}} look-alike pair automatically after the last scan. Open Manage people to check it — anything merged wrongly can be separated again with Split.", "consolidated_other": "Grouping merged {{count}} look-alike pairs automatically after the last scan. Open Manage people to check them — anything merged wrongly can be separated again with Split." - } + }, + "catalogedMedia": "{{size}} catalogued", + "storageUnavailable": "unavailable", + "catalogedMediaOnly": "catalogued — objects live in S3" }, "acceptInvitation": { "title": "Accept Invitation", diff --git a/frontend/src/i18n/locales/es.json b/frontend/src/i18n/locales/es.json index 3e467f4c..0b1f36ae 100644 --- a/frontend/src/i18n/locales/es.json +++ b/frontend/src/i18n/locales/es.json @@ -1109,7 +1109,10 @@ "notConfigured": "Umami no está configurado", "configureInstructions": "Para ver datos reales, configura Umami en tus variables de entorno y en los ajustes.", "noData": "No hay datos disponibles", - "percentChange": "{{percent}}% respecto al periodo anterior" + "percentChange": "{{percent}}% respecto al periodo anterior", + "catalogedMedia": "Medios catalogados", + "storageUnavailable": "no disponible", + "storageNoMeasurement": "sin medición disponible" }, "branding": { "title": "Marca y temas", @@ -1598,7 +1601,10 @@ "detail": { "labeled": "Etiquetas de color" } - } + }, + "catalogedMedia": "{{size}} catalogados", + "storageUnavailable": "no disponible", + "catalogedMediaOnly": "catalogado — los objetos están en S3" }, "permissions": { "insufficient": "No tienes permiso para realizar esta acción", diff --git a/frontend/src/i18n/locales/fr.json b/frontend/src/i18n/locales/fr.json index 9af88b0f..64a17d5a 100644 --- a/frontend/src/i18n/locales/fr.json +++ b/frontend/src/i18n/locales/fr.json @@ -1243,7 +1243,10 @@ "totalPhotos": "Photos totales", "activeEvents": "Événements actifs", "notConfigured": "Umami Analytics non configuré", - "configureInstructions": "Pour voir les données analytiques réelles, configurez Umami dans vos variables d'environnement et les paramètres du panneau d'administration." + "configureInstructions": "Pour voir les données analytiques réelles, configurez Umami dans vos variables d'environnement et les paramètres du panneau d'administration.", + "catalogedMedia": "Médias catalogués", + "storageUnavailable": "indisponible", + "storageNoMeasurement": "aucune mesure disponible" }, "branding": { "title": "Marque & Thèmes", @@ -1867,7 +1870,10 @@ "category_hero_updated": "Photo de couverture de catégorie mise à jour", "public_site_reset_to_default": "Site public réinitialisé aux valeurs par défaut", "cms_page_logo_uploaded": "Logo de la page CMS téléversé : {{slug}}" - } + }, + "catalogedMedia": "{{size}} catalogués", + "storageUnavailable": "indisponible", + "catalogedMediaOnly": "catalogué — les objets sont dans S3" }, "acceptInvitation": { "title": "Accepter l'invitation", diff --git a/frontend/src/i18n/locales/nl.json b/frontend/src/i18n/locales/nl.json index 6c08f5c8..9a645024 100644 --- a/frontend/src/i18n/locales/nl.json +++ b/frontend/src/i18n/locales/nl.json @@ -1221,7 +1221,10 @@ "totalPhotos": "Totaal foto's", "activeEvents": "Actieve evenementen", "notConfigured": "Umami Analytics niet geconfigureerd", - "configureInstructions": "Configureer Umami in uw omgevingsvariabelen en beheerdersinstellingen om echte statistieken te zien." + "configureInstructions": "Configureer Umami in uw omgevingsvariabelen en beheerdersinstellingen om echte statistieken te zien.", + "catalogedMedia": "Gecatalogiseerde media", + "storageUnavailable": "niet beschikbaar", + "storageNoMeasurement": "geen meting beschikbaar" }, "branding": { "title": "Huisstijl & Thema's", @@ -1845,7 +1848,10 @@ "category_hero_updated": "Categorie hero-foto bijgewerkt", "public_site_reset_to_default": "Publieke site teruggezet naar standaard", "cms_page_logo_uploaded": "CMS-pagina-logo geüpload: {{slug}}" - } + }, + "catalogedMedia": "{{size}} gecatalogiseerd", + "storageUnavailable": "niet beschikbaar", + "catalogedMediaOnly": "gecatalogiseerd — objecten staan in S3" }, "acceptInvitation": { "title": "Uitnodiging accepteren", diff --git a/frontend/src/i18n/locales/pt.json b/frontend/src/i18n/locales/pt.json index de176fa7..ec9a4094 100644 --- a/frontend/src/i18n/locales/pt.json +++ b/frontend/src/i18n/locales/pt.json @@ -1238,7 +1238,10 @@ "totalPhotos": "Total de Fotos", "activeEvents": "Eventos Ativos", "notConfigured": "Umami Analytics não configurado", - "configureInstructions": "Para ver dados reais, configure o Umami nas variáveis de ambiente e no painel admin." + "configureInstructions": "Para ver dados reais, configure o Umami nas variáveis de ambiente e no painel admin.", + "catalogedMedia": "Mídia catalogada", + "storageUnavailable": "indisponível", + "storageNoMeasurement": "sem medição disponível" }, "branding": { "title": "Marca e Temas", @@ -1870,7 +1873,10 @@ "category_hero_updated": "Foto principal da categoria atualizada", "public_site_reset_to_default": "Site público redefinido ao padrão", "cms_page_logo_uploaded": "Logotipo da página CMS carregado: {{slug}}" - } + }, + "catalogedMedia": "{{size}} catalogados", + "storageUnavailable": "indisponível", + "catalogedMediaOnly": "catalogado — os objetos estão no S3" }, "acceptInvitation": { "title": "Aceitar Convite", diff --git a/frontend/src/i18n/locales/ru.json b/frontend/src/i18n/locales/ru.json index 1a7042ef..b3f65566 100644 --- a/frontend/src/i18n/locales/ru.json +++ b/frontend/src/i18n/locales/ru.json @@ -1255,7 +1255,10 @@ "totalPhotos": "Всего фото", "activeEvents": "Активных событий", "notConfigured": "Umami Analytics не настроен", - "configureInstructions": "Для отображения данных аналитики настройте Umami в переменных среды и настройках панели администратора." + "configureInstructions": "Для отображения данных аналитики настройте Umami в переменных среды и настройках панели администратора.", + "catalogedMedia": "Каталогизированные медиа", + "storageUnavailable": "недоступно", + "storageNoMeasurement": "измерение недоступно" }, "branding": { "title": "Брендинг и темы", @@ -1895,7 +1898,10 @@ "category_hero_updated": "Главное фото категории обновлено", "public_site_reset_to_default": "Публичный сайт сброшен к значениям по умолчанию", "cms_page_logo_uploaded": "Логотип CMS-страницы загружен: {{slug}}" - } + }, + "catalogedMedia": "{{size}} в каталоге", + "storageUnavailable": "недоступно", + "catalogedMediaOnly": "в каталоге — объекты хранятся в S3" }, "acceptInvitation": { "title": "Принять приглашение", diff --git a/frontend/src/i18n/locales/sl.json b/frontend/src/i18n/locales/sl.json index e15e2d7a..a4fce31f 100644 --- a/frontend/src/i18n/locales/sl.json +++ b/frontend/src/i18n/locales/sl.json @@ -1243,7 +1243,10 @@ "totalPhotos": "Skupaj fotografij", "activeEvents": "Aktivni dogodki", "notConfigured": "Umami Analytics ni nastavljen", - "configureInstructions": "Za prikaz resničnih analitičnih podatkov nastavite Umami v okoljskih spremenljivkah in nastavitvah administratorske plošče." + "configureInstructions": "Za prikaz resničnih analitičnih podatkov nastavite Umami v okoljskih spremenljivkah in nastavitvah administratorske plošče.", + "catalogedMedia": "Katalogizirani mediji", + "storageUnavailable": "ni na voljo", + "storageNoMeasurement": "meritev ni na voljo" }, "branding": { "title": "Blagovna znamka in teme", @@ -1856,7 +1859,10 @@ "category_hero_updated": "Hero fotografija kategorije posodobljena", "public_site_reset_to_default": "Javna stran ponastavljena na privzeto", "cms_page_logo_uploaded": "Logotip CMS strani naložen: {{slug}}" - } + }, + "catalogedMedia": "{{size}} katalogizirano", + "storageUnavailable": "ni na voljo", + "catalogedMediaOnly": "katalogizirano — objekti so v S3" }, "acceptInvitation": { "title": "Sprejmi povabilo", diff --git a/frontend/src/pages/admin/AdminDashboard.tsx b/frontend/src/pages/admin/AdminDashboard.tsx index d4ea94fa..db4b3ab2 100644 --- a/frontend/src/pages/admin/AdminDashboard.tsx +++ b/frontend/src/pages/admin/AdminDashboard.tsx @@ -169,8 +169,29 @@ export const AdminDashboard: React.FC = () => { color: 'text-blue-600', }, { + // Real bytes under the storage root (#1164). This used to be the summed + // size of the catalogued originals, which on a reference-mode install is + // the size of a NAS — the one number an admin reaches for when asking + // "am I running out of disk" pointed away from the answer. `?? ` rather + // than `|| `: null means the measurement failed and must read as + // unavailable, not as 0 Bytes. title: t('admin.storageUsed'), - value: adminService.formatBytes(dashboardStats?.storageUsed || 0), + // On an S3 backend there is no disk to measure, so the catalogued figure + // IS the answer available and stands in — labelled by the subtitle below + // rather than pretending a walk happened. + value: dashboardStats?.storageUsed == null + ? (dashboardStats?.storageMeasurement === 'catalog' + ? adminService.formatBytes(dashboardStats.catalogedBytes) + : t('admin.storageUnavailable', 'unavailable')) + : `${adminService.formatBytes(dashboardStats.storageUsed)}${dashboardStats.storagePartial ? '+' : ''}`, + // The catalogued figure alongside, so the difference is visible rather + // than conflated. On a managed install they track each other; on a + // reference one they are supposed to diverge. + change: dashboardStats + ? (dashboardStats.storageMeasurement === 'catalog' + ? t('admin.catalogedMediaOnly', 'catalogued — objects live in S3') + : t('admin.catalogedMedia', { size: adminService.formatBytes(dashboardStats.catalogedBytes) })) + : undefined, icon: HardDrive, color: 'text-purple-600', }, diff --git a/frontend/src/pages/admin/AnalyticsPage.tsx b/frontend/src/pages/admin/AnalyticsPage.tsx index f1253a54..cbdf4a0c 100644 --- a/frontend/src/pages/admin/AnalyticsPage.tsx +++ b/frontend/src/pages/admin/AnalyticsPage.tsx @@ -438,15 +438,36 @@ export const AnalyticsPage: React.FC = () => { {/* Storage Information */} {dashboardStats && (() => { + // Real local bytes (#1164). This used to be the summed size of the + // catalogued originals, so a reference-mode install — where those + // files are on a NAS — compared a number from the NAS against a + // limit meant for this disk. + const localUsed = dashboardStats.storageUsed; + // On S3 there is no disk to walk and the catalogued figure IS the + // available answer; a failed local walk has none at all. + const measured = localUsed ?? (dashboardStats.storageMeasurement === 'catalog' + ? dashboardStats.catalogedBytes + : null); const softLimitBytes = storageInfo?.storage_soft_limit ?? storageInfo?.storage_limit ?? storageInfo?.recommended_soft_limit ?? null; + // `measured`, not `localUsed`. An editor or viewer holds + // analytics.view but not settings.view, so /storage/info 403s and + // storageInfo is undefined — and on S3 localUsed is null, which + // made this denominator 1 and rendered percentages in the billions. const safeSoftLimit = Math.max( - softLimitBytes ?? storageInfo?.recommended_soft_limit ?? (dashboardStats.storageUsed || 1), + softLimitBytes ?? storageInfo?.recommended_soft_limit ?? (measured || 1), 1 ); - const usageRatio = dashboardStats.storageUsed / safeSoftLimit; - const usagePercent = Math.round(usageRatio * 100); - const usageWidth = Math.min(usageRatio * 100, 100); - const overSoftLimit = softLimitBytes != null && dashboardStats.storageUsed >= softLimitBytes; + // Without a real limit there is no percentage worth showing: the + // denominator would be the usage itself, which always reads 100%. + const hasLimit = softLimitBytes != null || storageInfo?.recommended_soft_limit != null; + // No measurement, or no limit, means no percentage. Coercing null + // to 0 drew an empty bar at "0% of limit" and suppressed the + // over-limit state — reading as plenty of room precisely when + // nothing is known. + const usageRatio = (measured == null || !hasLimit) ? null : measured / safeSoftLimit; + const usagePercent = usageRatio == null ? null : Math.round(usageRatio * 100); + const usageWidth = usageRatio == null ? 0 : Math.min(usageRatio * 100, 100); + const overSoftLimit = softLimitBytes != null && measured != null && measured >= softLimitBytes; const limitDisplay = softLimitBytes != null ? adminService.formatBytes(softLimitBytes) : storageInfo?.recommended_soft_limit != null @@ -454,7 +475,7 @@ export const AnalyticsPage: React.FC = () => { : t('settings.storage.unlimited'); const progressColor = overSoftLimit ? 'bg-red-600' - : usagePercent >= 90 + : (usagePercent != null && usagePercent >= 90) ? 'bg-amber-500' : 'bg-accent-dark'; const limitDescriptor = storageInfo @@ -470,7 +491,11 @@ export const AnalyticsPage: React.FC = () => {
{t('analytics.used')} - {adminService.formatBytes(dashboardStats.storageUsed)} + + {measured == null + ? t('analytics.storageUnavailable', 'unavailable') + : `${adminService.formatBytes(measured)}${dashboardStats.storagePartial ? '+' : ''}`} +
{ />

- {usagePercent}% {t('analytics.of')} {limitDisplay} + {usagePercent == null + ? t('analytics.storageNoMeasurement', 'no measurement available') + : `${usagePercent}% ${t('analytics.of')} ${limitDisplay}`}

{limitDescriptor}

+ {/* The catalogued size of the originals, shown separately + rather than as "used" (#1164). On a reference-mode + install this is large and none of it is on this disk, + which is the distinction the old single figure hid. */}
+ {t('analytics.catalogedMedia', 'Catalogued media')} + {adminService.formatBytes(dashboardStats.catalogedBytes)} +
+
{t('analytics.totalPhotos')} {dashboardStats.totalPhotos.toLocaleString()}
diff --git a/frontend/src/services/admin.service.ts b/frontend/src/services/admin.service.ts index ad73bbcc..cb969e20 100644 --- a/frontend/src/services/admin.service.ts +++ b/frontend/src/services/admin.service.ts @@ -61,7 +61,37 @@ export interface DashboardStats { activeEvents: number; expiringEvents: number; totalPhotos: number; - storageUsed: number; + // Real bytes under the storage root — thumbnails, previews, hero + // renditions, download caches and any managed originals (#1164). Null when + // the measurement failed, which the UI must show as unavailable rather than + // substituting `catalogedBytes`: they are different quantities and on a + // reference-mode install they are wildly different. + storageUsed: number | null; + // 'catalog' when the backend is S3 — the objects are in the bucket, so no + // disk walk was made. 'unavailable' when a local walk was attempted and + // failed. Distinct because the first is a fact about the install and the + // second is a fault, and the UI must not claim S3 for a broken measurement. + storageMeasurement?: 'disk' | 'catalog' | 'unavailable'; + storageBreakdown: { + originals: number; + archives: number; + thumbnails: number; + previews: number; + heroes: number; + watermarks: number; + uploads: number; + businessDocs: number; + downloadCache: number; + externalMedia: number; + temp: number; + other: number; + } | null; + // The total is a floor: part of the storage root could not be read. + storagePartial?: boolean; + // Summed photos.size_bytes — the catalogued size of the originals, which is + // what this endpoint used to label "storage used". In reference mode those + // files are on external storage and none of those bytes are local. + catalogedBytes: number; totalViews: number; totalDownloads: number; viewsTrend: number; diff --git a/frontend/src/services/settings.service.ts b/frontend/src/services/settings.service.ts index 1e51334b..afa5d961 100644 --- a/frontend/src/services/settings.service.ts +++ b/frontend/src/services/settings.service.ts @@ -86,7 +86,20 @@ export interface PasswordComplexitySettings { } export interface StorageInfo { + // Real bytes under the storage root (#1164), excluding the external media + // share. Was the summed size of the catalogued originals, which on a + // reference-mode install is the size of a NAS. total_used: number; + // Summed photos.size_bytes — what total_used used to be. + cataloged_bytes?: number; + // True when part of the storage root could not be read, so total_used is a + // floor rather than the answer. Anything comparing it against a limit has to + // say so, or an unreadable subtree reads as "safely under". + storage_partial?: boolean; + // Where total_used came from. 'disk' is the filesystem walk; 'catalog' means + // the backend is S3, where the objects are in the bucket and a walk of the + // local storage root would report near-zero. + storage_measurement?: 'disk' | 'catalog' | 'unavailable'; archive_storage: number; storage_by_event: Array<{ event_name: string;