paul/picpeak
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(usage): introduce consented v4 without changing historical reports

Browse Source
This commit is contained in:
Paul Nothaft
2026-09-06 21:34:04 +02:00
parent 3cc893126d
commit ef8a52f02c
21 changed files with 5046 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/__tests__/routes/adminUsage.test.js
+4 -4
View File
@@ -201,13 +201,13 @@ test('public/gallery paths and failed/unauthenticated admin operations never set
expect(JSON.stringify(service.markUsed.mock.calls)).not.toContain('42');
});
test('consent upgrade accepts exactly the explicit v2 choice, never extra fields', async () => {
for (const data of [{}, { consent_version: 'usage-consent.v1' }, { consent_version: 'usage-consent.v2', user: 'PRIVATE' }])
test.each(['usage-consent.v2', 'usage-consent.v3', 'usage-consent.v4'])('consent upgrade accepts exactly the explicit %s choice, never extra fields', async (consent_version) => {
for (const data of [{}, { consent_version: 'usage-consent.v1' }, { consent_version: 'usage-consent.v5' }, { consent_version, user: 'PRIVATE' }])
await request(app).post('/api/admin/usage/consent').set('Authorization', `Bearer ${token('admin')}`).send(data).expect(400);
expect(service.command).not.toHaveBeenCalled();
await request(app).post('/api/admin/usage/consent').set('Authorization', `Bearer ${token('admin')}`)
.send({ consent_version: 'usage-consent.v2' }).expect(200);
expect(service.command).toHaveBeenCalledWith('consent', { consent_version: 'usage-consent.v2' });
.send({ consent_version }).expect(200);
expect(service.command).toHaveBeenCalledWith('consent', { consent_version });
});
test('only a backup that writes to the configured destination flags S3', () => {
backend/__tests__/services/usageCoverageInventory.test.js
+10 -4
View File
@@ -1,8 +1,8 @@
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const catalog = require('../../src/usage/features.v3.json');
const inventory = require('../../../docs/usage-coverage.v3.json');
const catalog = require('../../src/usage/features.v4.json');
const inventory = require('../../../docs/usage-coverage.v4.json');
const protocol = require('../../src/usage/schema.cjs');
const { RULES_V2, capabilityKeys } = require('../../src/usage/capabilityRules');
const { acceptedUpload, capabilityEvidence } = require('../../src/usage/capabilityEvidence');
@@ -52,16 +52,22 @@ test('all current settings tabs have an explicit scope decision', () => {
}
});
test('v1 wire validation is immutable; catalog, UI and translated descriptions agree', () => {
test('v1/v2/v3 wire validation is immutable; v4 catalog, UI and translated descriptions agree', () => {
expect(crypto.createHash('sha256').update(JSON.stringify(protocol.envelopeSchemas['usage.v1'].properties)).digest('hex'))
.toBe('cc8d0a865d21e36d2b24d23ca6aa8dd8d48000cb17aef83996786f70755bc922');
expect(crypto.createHash('sha256').update(JSON.stringify(protocol.envelopeSchemas['usage.v2'].properties)).digest('hex'))
.toBe('159821cf45c1951016d33a4ed9ca55a0a7ee1b60dd715b803fcfed33e5c8a846');
expect(protocol.FEATURE_KEYS).toHaveLength(86);
expect(crypto.createHash('sha256').update(JSON.stringify(protocol.envelopeSchemas['usage.v3'].properties)).digest('hex'))
.toBe('93214702c79f47823f154544ebad6612dd313604f69e60b86de4c0e4c904571a');
expect(protocol.FEATURE_KEYS).toContain('gallery_downloads_restricted');
expect(protocol.FEATURE_KEYS).not.toContain('gallery_downloads');
expect(protocol.ALL_FEATURE_KEYS).toHaveLength(87);
expect(protocol.ALL_FEATURE_KEYS).toContain('gallery_downloads');
expect(protocol.LEGACY_FEATURE_KEYS).toHaveLength(19);
expect(inventory.configuration_only).toHaveLength(23);
const frontend = path.resolve(__dirname, '../../../frontend');
expect(JSON.parse(fs.readFileSync(path.join(frontend, 'src/features/settings/usageFeatures.v3.json')))).toEqual(catalog);
expect(JSON.parse(fs.readFileSync(path.join(frontend, 'src/features/settings/usageFeatures.v4.json')))).toEqual(catalog);
for (const lang of ['en', 'de']) {
const translated = JSON.parse(fs.readFileSync(path.join(frontend, `src/i18n/locales/${lang}.json`))).productUsage.catalog;
for (const [key, value] of Object.entries(catalog.features)) {
backend/__tests__/services/usageIngressCompatibility.test.js
+2 -2
View File
@@ -7,13 +7,13 @@ const signHistorical = (packet, id) => {
return { ...signed, signature: crypto.sign(null, Buffer.from(p.canonical(signed)), id.private_key).toString('base64url') };
};
describe.each(['usage.v1', 'usage.v2', 'usage.v3'])('%s receiver compatibility never loosens the PicPeak sender', version => {
describe.each(['usage.v1', 'usage.v2', 'usage.v3', 'usage.v4'])('%s receiver compatibility never loosens the PicPeak sender', version => {
test('complete original reports still sign and verify', () => {
const id = p.generateIdentity();
const packet = p.makePacket(id, 'report', 1, {
picpeak_version: '1.0.0', report_date: '2026-09-06', generated_at: new Date(now).toISOString(),
features: p.emptyFeatures(version), gallery_layouts: [],
...(version === 'usage.v3' ? { inventory: { galleries: 0, photos: 0 } } : {}),
...(['usage.v3', 'usage.v4'].includes(version) ? { inventory: { galleries: 0, photos: 0 } } : {}),
}, version);
const envelope = p.signPacket(packet, id, new Date(now));
expect(p.verifyEnvelope(envelope, now)).toEqual(packet);
backend/__tests__/services/usageV3.test.js
+35 -64
View File
@@ -62,7 +62,7 @@ for (const engine of ['sqlite3', ...(process.env.PICPEAK_PG_TEST_URL ? ['pg'] :
expect(queries.filter(sql => /from ["`]photos["`]/.test(sql))).toEqual([expect.stringMatching(/select count\(\*\)/)]);
expect(JSON.stringify(report)).not.toContain('PRIVATE');
const identity = p.generateIdentity();
const envelope = p.signPacket(p.makePacket(identity, 'report', 1, report), identity, new Date(now));
const envelope = p.signPacket(p.makePacket(identity, 'report', 1, report, 'usage.v3'), identity, new Date(now));
expect(p.verifyEnvelope(envelope, now).payload).toEqual(report);
await db('photos').where({ id: 1 }).delete();
await db('events').where({ id: 1 }).delete();
@@ -126,69 +126,6 @@ for (const engine of ['sqlite3', ...(process.env.PICPEAK_PG_TEST_URL ? ['pg'] :
expect((await snap({})).gallery_folders.configured).toBe(false);
});
test('gallery_downloads_restricted counts galleries with downloads switched off, and v2 keeps its old key', async () => {
// allow_downloads ships true, so the v2 key was true on every install
// with a gallery. Only switching downloads off is a decision.
const snap = version => expandSnapshot(db, { features: p.emptyFeatures('usage.v1'), flags: {}, used: new Set(), now, version });
expect((await snap('usage.v3')).gallery_downloads_restricted).toEqual({ configured: false });
expect(await snap('usage.v3')).not.toHaveProperty('gallery_downloads');
await db('events').insert([{ allow_downloads: true }, { allow_downloads: true }]);
expect((await snap('usage.v3')).gallery_downloads_restricted.configured).toBe(false);
expect((await snap('usage.v2')).gallery_downloads).toEqual({ configured: true });
expect(await snap('usage.v2')).not.toHaveProperty('gallery_downloads_restricted');
await db('events').insert({ allow_downloads: false });
expect((await snap('usage.v3')).gallery_downloads_restricted.configured).toBe(true);
expect((await snap('usage.v2')).gallery_downloads.configured).toBe(true);
});
test('a report queued under the replaced catalog is rebuilt in place, keeping its packet id', async () => {
const identity = p.generateIdentity();
const posted = [];
const service = new UsageService(db, {
now: () => now, secret: 'v3-test-only-secret'.repeat(3), endpoint: 'http://127.0.0.1:9/',
fetch: async (_url, init) => { posted.push(JSON.parse(init.body).packet); throw new Error('collector unreachable'); },
});
service.binding = async () => 'b'.repeat(64);
const report = (features) => ({
picpeak_version: '1.0.0', report_date: '2026-09-05', generated_at: new Date(now).toISOString(),
features, gallery_layouts: [], inventory: { galleries: 0, photos: 0 },
});
// The v3 catalog as it stood before gallery_downloads_restricted replaced gallery_downloads.
const { gallery_downloads_restricted, ...rest } = p.emptyFeatures('usage.v3');
const stale = { ...rest, gallery_downloads: gallery_downloads_restricted };
const seed = (payload) => db('product_usage_state').where({ id: 1 }).update({
status: 'active', installation_id: identity.installation_id, public_key: identity.public_key,
private_key_encrypted: service.encrypt(identity.private_key), instance_binding: 'b'.repeat(64),
sequence: 1, last_error: null, attempts: 3, next_attempt_at: now + 60_000,
pending_packet: JSON.stringify(p.makePacket(identity, 'report', 2, payload, 'usage.v3')),
});
await seed(report(stale));
const queued = JSON.parse((await db('product_usage_state').where({ id: 1 }).first()).pending_packet);
await service.deliver(await db('product_usage_state').where({ id: 1 }).first());
// Sent once, under the current catalog, as the same packet.
expect(posted).toHaveLength(1);
expect(posted[0].packet_id).toBe(queued.packet_id);
expect(posted[0].sequence).toBe(2);
expect(posted[0].payload.features).toHaveProperty('gallery_downloads_restricted');
expect(posted[0].payload.features).not.toHaveProperty('gallery_downloads');
// The rebuilt packet is what stays queued for the ordinary retry path.
let row = await db('product_usage_state').where({ id: 1 }).first();
const retained = JSON.parse(row.pending_packet);
expect(retained.packet_id).toBe(queued.packet_id);
expect(retained.payload.features).toHaveProperty('gallery_downloads_restricted');
expect(row.status).toBe('active');
expect(row.last_error).toBe('DELIVERY_FAILED');
// Narrow: a report that still validates is sent as queued, payload untouched.
await seed(report(p.emptyFeatures('usage.v3')));
await service.deliver(await db('product_usage_state').where({ id: 1 }).first());
expect(posted).toHaveLength(2);
expect(posted[1].payload.report_date).toBe('2026-09-05');
row = await db('product_usage_state').where({ id: 1 }).first();
expect(JSON.parse(row.pending_packet).payload.report_date).toBe('2026-09-05');
});
test('ML recognition is already represented without querying faces or results', async () => {
await db('feature_flags').insert({ key: 'faces', value: true });
await client.markUsed(['face_recognition']);
@@ -197,5 +134,39 @@ for (const engine of ['sqlite3', ...(process.env.PICPEAK_PG_TEST_URL ? ['pg'] :
expect((await client.snapshot()).features.face_recognition).toEqual({ configured: false, used: true });
// No faces, people, embeddings or recognition-result tables exist in this fixture.
});
test.each([
[[], false, false], [[true], true, false], [[false], false, true],
[[true, false], true, true], [[null], false, false],
])('v4 measures explicit restrictions independently from legacy allowed downloads: %p', async (values, allowed, restricted) => {
if (values.length) await db('events').insert(values.map(allow_downloads => ({ allow_downloads })));
const queries = [];
db.on('query', q => queries.push(q));
for (const version of ['usage.v1', 'usage.v2', 'usage.v3', 'usage.v4']) {
await db('product_usage_state').where({ id: 1 }).update({ consent_version: p.CONSENT_VERSIONS[version] });
queries.length = 0;
const report = await client.preview();
const downloadQueries = queries.filter(q => /where ["`]allow_downloads["`] =/.test(q.sql));
if (version === 'usage.v4') {
expect(report.features.gallery_downloads_restricted).toEqual({ configured: restricted });
expect(report.features).not.toHaveProperty('gallery_downloads');
expect(report.inventory).toEqual({ galleries: values.length, photos: 0 });
expect(downloadQueries).toHaveLength(1);
expect(downloadQueries[0].sql).toMatch(/select 1 as present/);
expect(Number(downloadQueries[0].bindings[0])).toBe(0);
} else {
expect(report.features).not.toHaveProperty('gallery_downloads_restricted');
if (version === 'usage.v1') expect(downloadQueries).toHaveLength(0);
else {
expect(report.features.gallery_downloads).toEqual({ configured: allowed });
expect(downloadQueries).toHaveLength(1);
expect(Number(downloadQueries[0].bindings[0])).toBe(1);
}
}
const identity = p.generateIdentity();
const envelope = p.signPacket(p.makePacket(identity, 'report', 1, report, version), identity, new Date(now));
expect(p.verifyEnvelope(envelope, now).payload).toEqual(report);
}
});
});
}
backend/src/routes/adminUsage.js
+1 -1
View File
@@ -82,7 +82,7 @@ router.post(
router.post(
'/consent',
wrap(async (req, res) => {
if (!req.body || Object.keys(req.body).length !== 1 || !['usage.v2', 'usage.v3'].includes(schemaForConsent(req.body.consent_version)))
if (!req.body || Object.keys(req.body).length !== 1 || !['usage.v2', 'usage.v3', 'usage.v4'].includes(schemaForConsent(req.body.consent_version)))
throw new ValidationError('Explicit usage consent is required');
res.json(await service.command('consent', { consent_version: req.body.consent_version }));
})
backend/src/usage/UsageService.js
+12 -26
View File
@@ -539,6 +539,9 @@ class UsageService {
return value;
}
async deliver(state) {
// A lost receipt may mean this packet is already stored by the collector.
// Preserve its original schema, date and payload across binary upgrades;
// only re-sign transport metadata, never rebuild under the same packet ID.
const packet = JSON.parse(state.pending_packet);
if (
packet.action !== 'delete' &&
@@ -564,31 +567,14 @@ class UsageService {
});
return null;
}
const identity = {
public_key: state.public_key,
private_key: this.decrypt(state.private_key_encrypted)
};
let envelope;
try {
envelope = signPacket(packet, identity, new Date(this.now()));
} catch (error) {
// A report queued under a catalog this build no longer ships — the
// upgrade replaced a key in the same wire version — fails local
// validation before anything is sent, and retrying cannot repair it.
// Left as it was it blocked every operation behind it for good. A
// report's payload is derived state, so rebuild it from the current
// snapshot in place. The packet ID and sequence are kept: a re-signed
// retry must reuse them so a lost acknowledgement does not duplicate
// data. Reports only — a stale registration, deletion or command is a
// genuine conflict and keeps the handling below.
if (error.code !== 'INVALID_PACKET' || packet.action !== 'report') throw error;
packet.payload = await this.snapshot(packet.schema_version);
state.pending_packet = JSON.stringify(packet);
await this.db('product_usage_state')
.where({ id: 1, status: 'active' })
.update({ pending_packet: state.pending_packet });
envelope = signPacket(packet, identity, new Date(this.now()));
}
const envelope = signPacket(
packet,
{
public_key: state.public_key,
private_key: this.decrypt(state.private_key_encrypted)
},
new Date(this.now())
);
// Last check before anything leaves. The guard at the top of this
// method runs before the binding lookup above, which is asynchronous —
// so a withdrawal that COMPLETED during it would previously still have
@@ -974,7 +960,7 @@ class UsageService {
generated_at: now,
features: expanded,
gallery_layouts: [...layouts].sort(),
...(version === 'usage.v3' ? { inventory: await require('./inventorySnapshot').inventorySnapshot(this.db) } : {})
...(['usage.v3', 'usage.v4'].includes(version) ? { inventory: await require('./inventorySnapshot').inventorySnapshot(this.db) } : {})
};
}
backend/src/usage/expandedSnapshot.js
+2 -7
View File
@@ -85,12 +85,7 @@ async function expandSnapshot(db, { features, flags, used, now, version = 'usage
gallery_guest_uploads: 'allow_user_uploads',
gallery_client_access: 'client_access_enabled', gallery_watermarks: 'watermark_downloads'
})) result[key].configured = await enabled('events', column);
// allow_downloads ships true — column default and the create route both set
// it — so "at least one gallery allows downloads" is true on every install
// with a gallery and says nothing. v2 consented to that key under that
// description, so v2 keeps sending it unchanged. v3 asks the question that
// is actually a decision: has anyone switched downloads off.
if (version === 'usage.v3') {
if (version === 'usage.v4') {
result.gallery_downloads_restricted.configured = await exists('events', ['allow_downloads'], (query) =>
query.where('allow_downloads', formatBoolean(false)));
} else {
@@ -127,7 +122,7 @@ async function expandSnapshot(db, { features, flags, used, now, version = 'usage
query.where({ feedback_enabled: formatBoolean(true), [column]: formatBoolean(true) }));
result.gallery_guest_accounts.configured = await exists('event_feedback_settings', ['feedback_enabled', 'identity_mode'], (query) =>
query.where('feedback_enabled', formatBoolean(true)).whereIn('identity_mode', ['guest', 'shared']));
if (version === 'usage.v3') {
if (['usage.v3', 'usage.v4'].includes(version)) {
result.gallery_folders.configured = await exists('photo_categories', ['is_folder', 'event_id'], (query) =>
query.where('is_folder', formatBoolean(true)).where((q) => q.whereNull('event_id').orWhereIn('event_id', db('events').select('id'))));
result.transfer_upload_links.configured = Boolean(effective.transfers) && await exists('transfers',
backend/src/usage/features.v3.json
+6 -6
View File
@@ -1182,18 +1182,18 @@
},
"used": null
},
"gallery_downloads_restricted": {
"gallery_downloads": {
"category": "gallery_configuration",
"since": "usage.v3",
"since": "usage.v2",
"measurement": "configuration",
"configuration": "configuration",
"name": {
"en": "Gallery downloads restricted",
"de": "Galerie-Downloads eingeschränkt"
"en": "Gallery downloads allowed",
"de": "Galerie-Downloads erlaubt"
},
"configured": {
"en": "At least one gallery has downloads switched off; only existence across the installation, never gallery IDs or counts.",
"de": "Mindestens eine Galerie hat Downloads abgeschaltet; nur installationsweite Existenz, niemals Galeriekennungen oder Anzahlen."
"en": "Enabled in applicable gallery/global configuration; only existence across the installation, never gallery IDs or counts.",
"de": "In der betreffenden Galerie-/Globalkonfiguration aktiviert; nur installationsweite Existenz, niemals Galeriekennungen oder Anzahlen."
},
"used": null
},
backend/src/usage/features.v4.json
+1531
View File
File diff suppressed because it is too large Load Diff
backend/src/usage/schema.cjs
+10 -6
View File
@@ -2,10 +2,10 @@
// Vendored byte-identical in PicPeak. Existing wire versions stay immutable;
// every expansion requires explicit consent to its own version.
const CATALOG = require("./features.v3.json");
const CATALOGS = { "usage.v2": require("./features.v2.json"), "usage.v3": CATALOG };
const CONSENT_VERSIONS = { "usage.v1": "usage-consent.v1", "usage.v2": "usage-consent.v2", "usage.v3": "usage-consent.v3" };
const CURRENT_SCHEMA_VERSION = "usage.v3";
const CATALOG = require("./features.v4.json");
const CATALOGS = { "usage.v2": require("./features.v2.json"), "usage.v3": require("./features.v3.json"), "usage.v4": CATALOG };
const CONSENT_VERSIONS = { "usage.v1": "usage-consent.v1", "usage.v2": "usage-consent.v2", "usage.v3": "usage-consent.v3", "usage.v4": "usage-consent.v4" };
const CURRENT_SCHEMA_VERSION = "usage.v4";
const CURRENT_CONSENT_VERSION = CONSENT_VERSIONS[CURRENT_SCHEMA_VERSION];
const schemaForConsent = (consent) => Object.keys(CONSENT_VERSIONS).find((version) => CONSENT_VERSIONS[version] === consent);
const schemaRank = (version) => Object.keys(CONSENT_VERSIONS).indexOf(version);
@@ -19,6 +19,10 @@ const LEGACY_FEATURE_KEYS = [
"whatsapp", "backup", "s3_storage", "share_mounts",
];
const FEATURE_KEYS = Object.keys(CATALOG.features);
// Historical questions remain independently visible after a newer schema
// stops asking them. Never rename or invert a retained report's values.
const ALL_FEATURES = Object.assign({}, ...Object.values(CATALOGS).map(c => c.features));
const ALL_FEATURE_KEYS = Object.keys(ALL_FEATURES);
const LAYOUTS = ["grid", "masonry", "carousel", "timeline", "mosaic", "gallery-premium", "gallery-story", "other"];
const object = (properties, required = Object.keys(properties)) => ({
type: "object", additionalProperties: false, properties, required,
@@ -45,7 +49,7 @@ const report = (version) => object({
key, object({ configured: boolean, ...(observesUse(key, version) ? { used: boolean } : {}) })
]))),
gallery_layouts: { type: "array", uniqueItems: true, maxItems: LAYOUTS.length, items: { enum: LAYOUTS } },
...(version === "usage.v3" ? { inventory: object(Object.fromEntries(INVENTORY_KEYS.map(key => [key,
...(["usage.v3", "usage.v4"].includes(version) ? { inventory: object(Object.fromEntries(INVENTORY_KEYS.map(key => [key,
{ type: "integer", minimum: 0, maximum: MAX_INVENTORY_COUNT }
]))) } : {}),
});
@@ -109,7 +113,7 @@ const ingressEnvelopeSchemas = Object.fromEntries(Object.entries(envelopeSchemas
return [version, schema];
}));
module.exports = {
FEATURE_KEYS, LEGACY_FEATURE_KEYS, LAYOUTS, CATALOG, CATALOGS, CONSENT_VERSIONS,
FEATURE_KEYS, ALL_FEATURES, ALL_FEATURE_KEYS, LEGACY_FEATURE_KEYS, LAYOUTS, CATALOG, CATALOGS, CONSENT_VERSIONS,
schemaForConsent, schemaRank, INVENTORY_KEYS, MAX_INVENTORY_COUNT, CURRENT_SCHEMA_VERSION,
CURRENT_CONSENT_VERSION, featureKeysFor, observesUse, emptyFeatures,
envelopeSchema, envelopeSchemas, ingressEnvelopeSchemas, payloads, payloadsByVersion,