fix(backup): make .picpeak roundtrip work on Postgres

Two Postgres-only bugs found by a live docker-pg roundtrip (SQLite tests
passed because neither reproduces on SQLite):

- Export: knex `.stream()` pulls in the optional `pg-query-stream` module
  (not bundled) and throws on pg. Switched to a plain per-table `select`
  — works on both engines, no new dependency. Rows are DB metadata
  (blobs live under files/), so holding a table in memory is fine.
- Import: the pg driver returns json/jsonb columns as parsed JS values,
  so re-inserting a scalar like the string "PicPeak" sent it unquoted and
  pg rejected it ("invalid input syntax for type json"). Now introspects
  each table's json/jsonb columns and re-serialises those values before
  insert (pg only; SQLite stores json as TEXT and round-trips as-is).

Verified end-to-end on docker Postgres: export 85 tables, full-override
import, current account preserved, post-backup data removed.
This commit is contained in:
Luca
2026-07-02 20:13:18 +02:00
parent 66d61c87ca
commit f57462f798
2 changed files with 41 additions and 21 deletions
+14 -20
View File
@@ -76,32 +76,26 @@ async function getLatestMigration() {
} }
} }
// Stream one table to <dataDir>/<table>.ndjson (one JSON object per line). // Write one table to <dataDir>/<table>.ndjson (one JSON object per line).
// Returns { rowCount, checksum } for the manifest. JSON.stringify serialises // Returns { rowCount, checksum } for the manifest. JSON.stringify serialises
// Dates to ISO strings, which re-import cleanly on both engines. // Dates to ISO strings, which re-import cleanly on both engines.
//
// Uses a plain select rather than knex `.stream()`: streaming on Postgres pulls
// in the optional `pg-query-stream` dependency (not bundled), so it throws on
// pg. A select works on both engines with no extra dependency. Rows are DB
// metadata (blobs live on disk under files/), so holding a table in memory is
// fine for the instance sizes PicPeak targets.
async function writeTableNdjson(table, dataDir) { async function writeTableNdjson(table, dataDir) {
const outPath = path.join(dataDir, `${table}.ndjson`); const outPath = path.join(dataDir, `${table}.ndjson`);
const out = fs.createWriteStream(outPath, { encoding: 'utf8' });
const hash = crypto.createHash('sha256'); const hash = crypto.createHash('sha256');
let rowCount = 0; const rows = await db(table).select('*');
const lines = rows.map((row) => {
await new Promise((resolve, reject) => { const line = JSON.stringify(row);
out.on('error', reject); hash.update(`${line}\n`);
const stream = db(table).stream(); return line;
stream.on('error', reject);
stream.on('data', (row) => {
const line = `${JSON.stringify(row)}\n`;
hash.update(line);
rowCount += 1;
if (!out.write(line)) {
stream.pause();
out.once('drain', () => stream.resume());
}
}); });
stream.on('end', () => out.end(resolve)); await fsp.writeFile(outPath, lines.length ? `${lines.join('\n')}\n` : '', 'utf8');
}); return { rowCount: rows.length, checksum: hash.digest('hex') };
return { rowCount, checksum: hash.digest('hex') };
} }
// Recursively collect files under a storage subdir as { abs, rel } where rel is // Recursively collect files under a storage subdir as { abs, rel } where rel is
+27 -1
View File
@@ -99,6 +99,30 @@ async function reinjectCurrentAdmin(trx, currentAdmin) {
} }
} }
// The json/jsonb columns of a table (Postgres only). The pg driver returns
// jsonb as parsed JS values, so on re-insert they must be serialised back to
// valid JSON text — otherwise a scalar like the string "PicPeak" is sent
// unquoted and pg rejects it ("invalid input syntax for type json").
async function jsonColumnsFor(trx, table) {
if (!isPostgres()) return new Set();
const res = await trx.raw(
"SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = ? AND data_type IN ('json', 'jsonb')",
[table]
);
return new Set(res.rows.map((r) => r.column_name));
}
function serialiseJsonColumns(rows, jsonCols) {
if (!jsonCols.size) return rows;
return rows.map((row) => {
const out = { ...row };
for (const col of jsonCols) {
if (out[col] !== undefined && out[col] !== null) out[col] = JSON.stringify(out[col]);
}
return out;
});
}
// Whole-DB replace in one transaction with FK enforcement suspended (pg: // Whole-DB replace in one transaction with FK enforcement suspended (pg:
// session_replication_role=replica on the trx connection, reset before commit; // session_replication_role=replica on the trx connection, reset before commit;
// sqlite: defer_foreign_keys so checks run at commit). knex_migrations is never // sqlite: defer_foreign_keys so checks run at commit). knex_migrations is never
@@ -113,7 +137,9 @@ async function replaceAllTables(tables, dataDir, currentAdmin) {
} }
for (const table of tables) { for (const table of tables) {
const rows = parseNdjson(path.join(dataDir, `${table}.ndjson`)); const rows = parseNdjson(path.join(dataDir, `${table}.ndjson`));
if (rows.length) await trx.batchInsert(table, rows, 100); if (!rows.length) continue;
const jsonCols = await jsonColumnsFor(trx, table);
await trx.batchInsert(table, serialiseJsonColumns(rows, jsonCols), 100);
} }
await reinjectCurrentAdmin(trx, currentAdmin); await reinjectCurrentAdmin(trx, currentAdmin);