Merge pull request #1313 from PicPeak/codex/usage-report-compatibility

fix(usage): preserve compatibility with old and partial reports

backend/__tests__/services/usageIngressCompatibility.test.js

@@ -0,0 +1,41 @@
+const crypto = require('crypto');
+const p = require('../../src/usage/protocol.cjs');
+const now = Date.parse('2026-09-06T12:00:00.000Z');
+
+const signHistorical = (packet, id) => {
+  const signed = { packet, public_key: id.public_key, issued_at: new Date(now).toISOString(), nonce: crypto.randomUUID() };
+  return { ...signed, signature: crypto.sign(null, Buffer.from(p.canonical(signed)), id.private_key).toString('base64url') };
+};
+
+describe.each(['usage.v1', 'usage.v2', 'usage.v3'])('%s receiver compatibility never loosens the PicPeak sender', version => {
+  test('complete original reports still sign and verify', () => {
+    const id = p.generateIdentity();
+    const packet = p.makePacket(id, 'report', 1, {
+      picpeak_version: '1.0.0', report_date: '2026-09-06', generated_at: new Date(now).toISOString(),
+      features: p.emptyFeatures(version), gallery_layouts: [],
+      ...(version === 'usage.v3' ? { inventory: { galleries: 0, photos: 0 } } : {}),
+    }, version);
+    const envelope = p.signPacket(packet, id, new Date(now));
+    expect(p.verifyEnvelope(envelope, now)).toEqual(packet);
+    expect(p.verifyReceivedEnvelope(envelope, now)).toEqual(packet);
+  });
+
+  test.each([undefined, null, {}, { crm: { used: true, configured: null } }])('accepts partial incoming measurements %p without rewriting them', features => {
+    const id = p.generateIdentity();
+    const packet = p.makePacket(id, 'report', 1, {
+      report_date: '2020-01-02', generated_at: '2020-01-02T12:00:00.000Z',
+      ...(features === undefined ? {} : { features }),
+    }, version);
+    expect(() => p.signPacket(packet, id, new Date(now))).toThrow('INVALID_PACKET');
+    const envelope = signHistorical(packet, id), original = JSON.stringify(envelope);
+    expect(() => p.verifyEnvelope(envelope, now)).toThrow('INVALID_PACKET');
+    expect(p.verifyReceivedEnvelope(envelope, now)).toEqual(packet);
+    expect(JSON.stringify(envelope)).toBe(original);
+  });
+
+  test('registration still requires exact explicit consent', () => {
+    const id = p.generateIdentity();
+    const packet = p.makePacket(id, 'register', 0, {}, version);
+    expect(() => p.verifyReceivedEnvelope(signHistorical(packet, id), now)).toThrow('INVALID_PACKET');
+  });
+});

backend/src/usage/protocol.cjs

@@ -4,6 +4,7 @@ const Ajv = require("ajv");
 const {
   envelopeSchema,
   envelopeSchemas,
+  ingressEnvelopeSchemas,
   CURRENT_SCHEMA_VERSION,
   FEATURE_KEYS,
   LAYOUTS,
@@ -15,6 +16,11 @@ const validators = new Map(Object.entries(envelopeSchemas).map(
 ));
 const validate = (envelope) =>
   Boolean(validators.get(envelope?.packet?.schema_version)?.(envelope));
+const ingressValidators = new Map(Object.entries(ingressEnvelopeSchemas).map(
+  ([version, schema]) => [version, ajv.compile(schema)],
+));
+const validateIngress = (envelope) =>
+  Boolean(ingressValidators.get(envelope?.packet?.schema_version)?.(envelope));
 const MAX_BYTES = 16384;
 const MAX_AGE_MS = 5 * 60 * 1000;
 
@@ -90,9 +96,15 @@ function signPacket(packet, identity, now = new Date()) {
   return envelope;
 }
 function verifyEnvelope(envelope, now = Date.now()) {
+  return verify(envelope, now, validate);
+}
+function verifyReceivedEnvelope(envelope, now = Date.now()) {
+  return verify(envelope, now, validateIngress);
+}
+function verify(envelope, now, validateEnvelope) {
   if (
     Buffer.byteLength(JSON.stringify(envelope) || "") > MAX_BYTES ||
-    !validate(envelope)
+    !validateEnvelope(envelope)
   )
     throw new ProtocolError("INVALID_PACKET");
   const issued = Date.parse(envelope.issued_at);
@@ -151,6 +163,7 @@ module.exports = {
   makePacket,
   signPacket,
   verifyEnvelope,
+  verifyReceivedEnvelope,
   ProtocolError,
   MAX_BYTES,
   MAX_AGE_MS,

backend/src/usage/schema.cjs

@@ -82,9 +82,35 @@ const envelopeSchemas = Object.fromEntries(Object.entries(payloadsByVersion).map
 }]));
 const envelopeSchema = envelopeSchemas[CURRENT_SCHEMA_VERSION];
 const payloads = payloadsByVersion[CURRENT_SCHEMA_VERSION];
+// Writers retain their immutable, complete contracts. The receiver accepts
+// missing measurements from older/partial reporters, without expanding any
+// version's allowlist or changing the signed object. Null also means unknown.
+const nullable = (schema) => ({ anyOf: [schema, { type: "null" }] });
+const ingressEnvelopeSchemas = Object.fromEntries(Object.entries(envelopeSchemas).map(([version, source]) => {
+  const schema = structuredClone(source);
+  schema.$id = `https://usage.picpeak.app/schema/ingress/${version}.json`;
+  schema.title = `PicPeak ${version} compatible report reception`;
+  const payload = schema.properties.packet.oneOf.find(p => p.properties.action.const === "report").properties.payload;
+  payload.required = ["report_date", "generated_at"];
+  const features = payload.properties.features;
+  features.required = [];
+  for (const signal of Object.values(features.properties)) {
+    signal.required = [];
+    for (const key of Object.keys(signal.properties)) signal.properties[key] = nullable(signal.properties[key]);
+  }
+  for (const key of Object.keys(features.properties)) features.properties[key] = nullable(features.properties[key]);
+  if (payload.properties.inventory) {
+    const inventory = payload.properties.inventory;
+    inventory.required = [];
+    for (const key of Object.keys(inventory.properties)) inventory.properties[key] = nullable(inventory.properties[key]);
+  }
+  for (const key of ["picpeak_version", "features", "gallery_layouts", "inventory"])
+    if (payload.properties[key]) payload.properties[key] = nullable(payload.properties[key]);
+  return [version, schema];
+}));
 module.exports = {
   FEATURE_KEYS, LEGACY_FEATURE_KEYS, LAYOUTS, CATALOG, CATALOGS, CONSENT_VERSIONS,
   schemaForConsent, schemaRank, INVENTORY_KEYS, MAX_INVENTORY_COUNT, CURRENT_SCHEMA_VERSION,
   CURRENT_CONSENT_VERSION, featureKeysFor, observesUse, emptyFeatures,
-  envelopeSchema, envelopeSchemas, payloads, payloadsByVersion,
+  envelopeSchema, envelopeSchemas, ingressEnvelopeSchemas, payloads, payloadsByVersion,
 };

docs/PRODUCT_USAGE.md

@@ -16,6 +16,16 @@ The sections below also document the historical v1/v2 implementation. Any
 statements excluding all gallery/photo counts describe those earlier versions;
 v3 adds only the two installation totals above.
 
+Backward compatibility is required for future changes. The collector continues
+to accept v1/v2/v3 reports, including omitted or null measurements, using their
+declared schema and original reporting day. Missing values remain unknown in
+aggregates and histories. PicPeak still emits complete reports through the
+unchanged sender schemas; only reception is more tolerant. Consent, field
+allowlists, signatures and retry/deletion rules remain mandatory. Deploy the
+compatible collector first; old PicPeak clients require no update or renewed
+consent to keep delivering their existing report scope. New fields still
+require a new explicitly consented schema; never redefine an old field.
+
 
 Tracking is disabled by default. After updating, settings editors see a
 dismissible invitation in the admin shell. Only explicit consent in Settings →