From 7ca783f89b8ed735ec3e69306a837c0f40e0670b Mon Sep 17 00:00:00 2001 From: Paul Nothaft Date: Sun, 6 Sep 2026 19:45:41 +0200 Subject: [PATCH] fix(usage): preserve report contracts with compatible receiver validation --- .../usageIngressCompatibility.test.js | 41 +++++++++++++++++++ backend/src/usage/protocol.cjs | 15 ++++++- backend/src/usage/schema.cjs | 28 ++++++++++++- docs/PRODUCT_USAGE.md | 10 +++++ 4 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 backend/__tests__/services/usageIngressCompatibility.test.js diff --git a/backend/__tests__/services/usageIngressCompatibility.test.js b/backend/__tests__/services/usageIngressCompatibility.test.js new file mode 100644 index 00000000..03e9531f --- /dev/null +++ b/backend/__tests__/services/usageIngressCompatibility.test.js @@ -0,0 +1,41 @@ +const crypto = require('crypto'); +const p = require('../../src/usage/protocol.cjs'); +const now = Date.parse('2026-09-06T12:00:00.000Z'); + +const signHistorical = (packet, id) => { + const signed = { packet, public_key: id.public_key, issued_at: new Date(now).toISOString(), nonce: crypto.randomUUID() }; + return { ...signed, signature: crypto.sign(null, Buffer.from(p.canonical(signed)), id.private_key).toString('base64url') }; +}; + +describe.each(['usage.v1', 'usage.v2', 'usage.v3'])('%s receiver compatibility never loosens the PicPeak sender', version => { + test('complete original reports still sign and verify', () => { + const id = p.generateIdentity(); + const packet = p.makePacket(id, 'report', 1, { + picpeak_version: '1.0.0', report_date: '2026-09-06', generated_at: new Date(now).toISOString(), + features: p.emptyFeatures(version), gallery_layouts: [], + ...(version === 'usage.v3' ? { inventory: { galleries: 0, photos: 0 } } : {}), + }, version); + const envelope = p.signPacket(packet, id, new Date(now)); + expect(p.verifyEnvelope(envelope, now)).toEqual(packet); + expect(p.verifyReceivedEnvelope(envelope, now)).toEqual(packet); + }); + + test.each([undefined, null, {}, { crm: { used: true, configured: null } }])('accepts partial incoming measurements %p without rewriting them', features => { + const id = p.generateIdentity(); + const packet = p.makePacket(id, 'report', 1, { + report_date: '2020-01-02', generated_at: '2020-01-02T12:00:00.000Z', + ...(features === undefined ? {} : { features }), + }, version); + expect(() => p.signPacket(packet, id, new Date(now))).toThrow('INVALID_PACKET'); + const envelope = signHistorical(packet, id), original = JSON.stringify(envelope); + expect(() => p.verifyEnvelope(envelope, now)).toThrow('INVALID_PACKET'); + expect(p.verifyReceivedEnvelope(envelope, now)).toEqual(packet); + expect(JSON.stringify(envelope)).toBe(original); + }); + + test('registration still requires exact explicit consent', () => { + const id = p.generateIdentity(); + const packet = p.makePacket(id, 'register', 0, {}, version); + expect(() => p.verifyReceivedEnvelope(signHistorical(packet, id), now)).toThrow('INVALID_PACKET'); + }); +}); diff --git a/backend/src/usage/protocol.cjs b/backend/src/usage/protocol.cjs index 8ea362a6..54efa897 100644 --- a/backend/src/usage/protocol.cjs +++ b/backend/src/usage/protocol.cjs @@ -4,6 +4,7 @@ const Ajv = require("ajv"); const { envelopeSchema, envelopeSchemas, + ingressEnvelopeSchemas, CURRENT_SCHEMA_VERSION, FEATURE_KEYS, LAYOUTS, @@ -15,6 +16,11 @@ const validators = new Map(Object.entries(envelopeSchemas).map( )); const validate = (envelope) => Boolean(validators.get(envelope?.packet?.schema_version)?.(envelope)); +const ingressValidators = new Map(Object.entries(ingressEnvelopeSchemas).map( + ([version, schema]) => [version, ajv.compile(schema)], +)); +const validateIngress = (envelope) => + Boolean(ingressValidators.get(envelope?.packet?.schema_version)?.(envelope)); const MAX_BYTES = 16384; const MAX_AGE_MS = 5 * 60 * 1000; @@ -90,9 +96,15 @@ function signPacket(packet, identity, now = new Date()) { return envelope; } function verifyEnvelope(envelope, now = Date.now()) { + return verify(envelope, now, validate); +} +function verifyReceivedEnvelope(envelope, now = Date.now()) { + return verify(envelope, now, validateIngress); +} +function verify(envelope, now, validateEnvelope) { if ( Buffer.byteLength(JSON.stringify(envelope) || "") > MAX_BYTES || - !validate(envelope) + !validateEnvelope(envelope) ) throw new ProtocolError("INVALID_PACKET"); const issued = Date.parse(envelope.issued_at); @@ -151,6 +163,7 @@ module.exports = { makePacket, signPacket, verifyEnvelope, + verifyReceivedEnvelope, ProtocolError, MAX_BYTES, MAX_AGE_MS, diff --git a/backend/src/usage/schema.cjs b/backend/src/usage/schema.cjs index 0c254f10..0a8074ce 100644 --- a/backend/src/usage/schema.cjs +++ b/backend/src/usage/schema.cjs @@ -82,9 +82,35 @@ const envelopeSchemas = Object.fromEntries(Object.entries(payloadsByVersion).map }])); const envelopeSchema = envelopeSchemas[CURRENT_SCHEMA_VERSION]; const payloads = payloadsByVersion[CURRENT_SCHEMA_VERSION]; +// Writers retain their immutable, complete contracts. The receiver accepts +// missing measurements from older/partial reporters, without expanding any +// version's allowlist or changing the signed object. Null also means unknown. +const nullable = (schema) => ({ anyOf: [schema, { type: "null" }] }); +const ingressEnvelopeSchemas = Object.fromEntries(Object.entries(envelopeSchemas).map(([version, source]) => { + const schema = structuredClone(source); + schema.$id = `https://usage.picpeak.app/schema/ingress/${version}.json`; + schema.title = `PicPeak ${version} compatible report reception`; + const payload = schema.properties.packet.oneOf.find(p => p.properties.action.const === "report").properties.payload; + payload.required = ["report_date", "generated_at"]; + const features = payload.properties.features; + features.required = []; + for (const signal of Object.values(features.properties)) { + signal.required = []; + for (const key of Object.keys(signal.properties)) signal.properties[key] = nullable(signal.properties[key]); + } + for (const key of Object.keys(features.properties)) features.properties[key] = nullable(features.properties[key]); + if (payload.properties.inventory) { + const inventory = payload.properties.inventory; + inventory.required = []; + for (const key of Object.keys(inventory.properties)) inventory.properties[key] = nullable(inventory.properties[key]); + } + for (const key of ["picpeak_version", "features", "gallery_layouts", "inventory"]) + if (payload.properties[key]) payload.properties[key] = nullable(payload.properties[key]); + return [version, schema]; +})); module.exports = { FEATURE_KEYS, LEGACY_FEATURE_KEYS, LAYOUTS, CATALOG, CATALOGS, CONSENT_VERSIONS, schemaForConsent, schemaRank, INVENTORY_KEYS, MAX_INVENTORY_COUNT, CURRENT_SCHEMA_VERSION, CURRENT_CONSENT_VERSION, featureKeysFor, observesUse, emptyFeatures, - envelopeSchema, envelopeSchemas, payloads, payloadsByVersion, + envelopeSchema, envelopeSchemas, ingressEnvelopeSchemas, payloads, payloadsByVersion, }; diff --git a/docs/PRODUCT_USAGE.md b/docs/PRODUCT_USAGE.md index 9b8f1981..ac008881 100644 --- a/docs/PRODUCT_USAGE.md +++ b/docs/PRODUCT_USAGE.md @@ -16,6 +16,16 @@ The sections below also document the historical v1/v2 implementation. Any statements excluding all gallery/photo counts describe those earlier versions; v3 adds only the two installation totals above. +Backward compatibility is required for future changes. The collector continues +to accept v1/v2/v3 reports, including omitted or null measurements, using their +declared schema and original reporting day. Missing values remain unknown in +aggregates and histories. PicPeak still emits complete reports through the +unchanged sender schemas; only reception is more tolerant. Consent, field +allowlists, signatures and retry/deletion rules remain mandatory. Deploy the +compatible collector first; old PicPeak clients require no update or renewed +consent to keep delivering their existing report scope. New fields still +require a new explicitly consented schema; never redefine an old field. + Tracking is disabled by default. After updating, settings editors see a dismissible invitation in the admin shell. Only explicit consent in Settings →