fix(backup): make .picpeak roundtrip work on Postgres
Two Postgres-only bugs found by a live docker-pg roundtrip (SQLite tests
passed because neither reproduces on SQLite):
- Export: knex `.stream()` pulls in the optional `pg-query-stream` module
(not bundled) and throws on pg. Switched to a plain per-table `select`
— works on both engines, no new dependency. Rows are DB metadata
(blobs live under files/), so holding a table in memory is fine.
- Import: the pg driver returns json/jsonb columns as parsed JS values,
so re-inserting a scalar like the string "PicPeak" sent it unquoted and
pg rejected it ("invalid input syntax for type json"). Now introspects
each table's json/jsonb columns and re-serialises those values before
insert (pg only; SQLite stores json as TEXT and round-trips as-is).
Verified end-to-end on docker Postgres: export 85 tables, full-override
import, current account preserved, post-backup data removed.
This commit is contained in:
@@ -76,32 +76,26 @@ async function getLatestMigration() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stream one table to <dataDir>/<table>.ndjson (one JSON object per line).
|
// Write one table to <dataDir>/<table>.ndjson (one JSON object per line).
|
||||||
// Returns { rowCount, checksum } for the manifest. JSON.stringify serialises
|
// Returns { rowCount, checksum } for the manifest. JSON.stringify serialises
|
||||||
// Dates to ISO strings, which re-import cleanly on both engines.
|
// Dates to ISO strings, which re-import cleanly on both engines.
|
||||||
|
//
|
||||||
|
// Uses a plain select rather than knex `.stream()`: streaming on Postgres pulls
|
||||||
|
// in the optional `pg-query-stream` dependency (not bundled), so it throws on
|
||||||
|
// pg. A select works on both engines with no extra dependency. Rows are DB
|
||||||
|
// metadata (blobs live on disk under files/), so holding a table in memory is
|
||||||
|
// fine for the instance sizes PicPeak targets.
|
||||||
async function writeTableNdjson(table, dataDir) {
|
async function writeTableNdjson(table, dataDir) {
|
||||||
const outPath = path.join(dataDir, `${table}.ndjson`);
|
const outPath = path.join(dataDir, `${table}.ndjson`);
|
||||||
const out = fs.createWriteStream(outPath, { encoding: 'utf8' });
|
|
||||||
const hash = crypto.createHash('sha256');
|
const hash = crypto.createHash('sha256');
|
||||||
let rowCount = 0;
|
const rows = await db(table).select('*');
|
||||||
|
const lines = rows.map((row) => {
|
||||||
await new Promise((resolve, reject) => {
|
const line = JSON.stringify(row);
|
||||||
out.on('error', reject);
|
hash.update(`${line}\n`);
|
||||||
const stream = db(table).stream();
|
return line;
|
||||||
stream.on('error', reject);
|
|
||||||
stream.on('data', (row) => {
|
|
||||||
const line = `${JSON.stringify(row)}\n`;
|
|
||||||
hash.update(line);
|
|
||||||
rowCount += 1;
|
|
||||||
if (!out.write(line)) {
|
|
||||||
stream.pause();
|
|
||||||
out.once('drain', () => stream.resume());
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
stream.on('end', () => out.end(resolve));
|
await fsp.writeFile(outPath, lines.length ? `${lines.join('\n')}\n` : '', 'utf8');
|
||||||
});
|
return { rowCount: rows.length, checksum: hash.digest('hex') };
|
||||||
|
|
||||||
return { rowCount, checksum: hash.digest('hex') };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Recursively collect files under a storage subdir as { abs, rel } where rel is
|
// Recursively collect files under a storage subdir as { abs, rel } where rel is
|
||||||
|
|||||||
@@ -99,6 +99,30 @@ async function reinjectCurrentAdmin(trx, currentAdmin) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The json/jsonb columns of a table (Postgres only). The pg driver returns
|
||||||
|
// jsonb as parsed JS values, so on re-insert they must be serialised back to
|
||||||
|
// valid JSON text — otherwise a scalar like the string "PicPeak" is sent
|
||||||
|
// unquoted and pg rejects it ("invalid input syntax for type json").
|
||||||
|
async function jsonColumnsFor(trx, table) {
|
||||||
|
if (!isPostgres()) return new Set();
|
||||||
|
const res = await trx.raw(
|
||||||
|
"SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = ? AND data_type IN ('json', 'jsonb')",
|
||||||
|
[table]
|
||||||
|
);
|
||||||
|
return new Set(res.rows.map((r) => r.column_name));
|
||||||
|
}
|
||||||
|
|
||||||
|
function serialiseJsonColumns(rows, jsonCols) {
|
||||||
|
if (!jsonCols.size) return rows;
|
||||||
|
return rows.map((row) => {
|
||||||
|
const out = { ...row };
|
||||||
|
for (const col of jsonCols) {
|
||||||
|
if (out[col] !== undefined && out[col] !== null) out[col] = JSON.stringify(out[col]);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// Whole-DB replace in one transaction with FK enforcement suspended (pg:
|
// Whole-DB replace in one transaction with FK enforcement suspended (pg:
|
||||||
// session_replication_role=replica on the trx connection, reset before commit;
|
// session_replication_role=replica on the trx connection, reset before commit;
|
||||||
// sqlite: defer_foreign_keys so checks run at commit). knex_migrations is never
|
// sqlite: defer_foreign_keys so checks run at commit). knex_migrations is never
|
||||||
@@ -113,7 +137,9 @@ async function replaceAllTables(tables, dataDir, currentAdmin) {
|
|||||||
}
|
}
|
||||||
for (const table of tables) {
|
for (const table of tables) {
|
||||||
const rows = parseNdjson(path.join(dataDir, `${table}.ndjson`));
|
const rows = parseNdjson(path.join(dataDir, `${table}.ndjson`));
|
||||||
if (rows.length) await trx.batchInsert(table, rows, 100);
|
if (!rows.length) continue;
|
||||||
|
const jsonCols = await jsonColumnsFor(trx, table);
|
||||||
|
await trx.batchInsert(table, serialiseJsonColumns(rows, jsonCols), 100);
|
||||||
}
|
}
|
||||||
|
|
||||||
await reinjectCurrentAdmin(trx, currentAdmin);
|
await reinjectCurrentAdmin(trx, currentAdmin);
|
||||||
|
|||||||
Reference in New Issue
Block a user