diff --git a/backend/src/services/picpeakExportService.js b/backend/src/services/picpeakExportService.js index 663ca252..0fb8f469 100644 --- a/backend/src/services/picpeakExportService.js +++ b/backend/src/services/picpeakExportService.js @@ -76,32 +76,26 @@ async function getLatestMigration() { } } -// Stream one table to /.ndjson (one JSON object per line). +// Write one table to /
.ndjson (one JSON object per line). // Returns { rowCount, checksum } for the manifest. JSON.stringify serialises // Dates to ISO strings, which re-import cleanly on both engines. +// +// Uses a plain select rather than knex `.stream()`: streaming on Postgres pulls +// in the optional `pg-query-stream` dependency (not bundled), so it throws on +// pg. A select works on both engines with no extra dependency. Rows are DB +// metadata (blobs live on disk under files/), so holding a table in memory is +// fine for the instance sizes PicPeak targets. async function writeTableNdjson(table, dataDir) { const outPath = path.join(dataDir, `${table}.ndjson`); - const out = fs.createWriteStream(outPath, { encoding: 'utf8' }); const hash = crypto.createHash('sha256'); - let rowCount = 0; - - await new Promise((resolve, reject) => { - out.on('error', reject); - const stream = db(table).stream(); - stream.on('error', reject); - stream.on('data', (row) => { - const line = `${JSON.stringify(row)}\n`; - hash.update(line); - rowCount += 1; - if (!out.write(line)) { - stream.pause(); - out.once('drain', () => stream.resume()); - } - }); - stream.on('end', () => out.end(resolve)); + const rows = await db(table).select('*'); + const lines = rows.map((row) => { + const line = JSON.stringify(row); + hash.update(`${line}\n`); + return line; }); - - return { rowCount, checksum: hash.digest('hex') }; + await fsp.writeFile(outPath, lines.length ? `${lines.join('\n')}\n` : '', 'utf8'); + return { rowCount: rows.length, checksum: hash.digest('hex') }; } // Recursively collect files under a storage subdir as { abs, rel } where rel is diff --git a/backend/src/services/picpeakImportService.js b/backend/src/services/picpeakImportService.js index 4b61391f..ae37455d 100644 --- a/backend/src/services/picpeakImportService.js +++ b/backend/src/services/picpeakImportService.js @@ -99,6 +99,30 @@ async function reinjectCurrentAdmin(trx, currentAdmin) { } } +// The json/jsonb columns of a table (Postgres only). The pg driver returns +// jsonb as parsed JS values, so on re-insert they must be serialised back to +// valid JSON text — otherwise a scalar like the string "PicPeak" is sent +// unquoted and pg rejects it ("invalid input syntax for type json"). +async function jsonColumnsFor(trx, table) { + if (!isPostgres()) return new Set(); + const res = await trx.raw( + "SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = ? AND data_type IN ('json', 'jsonb')", + [table] + ); + return new Set(res.rows.map((r) => r.column_name)); +} + +function serialiseJsonColumns(rows, jsonCols) { + if (!jsonCols.size) return rows; + return rows.map((row) => { + const out = { ...row }; + for (const col of jsonCols) { + if (out[col] !== undefined && out[col] !== null) out[col] = JSON.stringify(out[col]); + } + return out; + }); +} + // Whole-DB replace in one transaction with FK enforcement suspended (pg: // session_replication_role=replica on the trx connection, reset before commit; // sqlite: defer_foreign_keys so checks run at commit). knex_migrations is never @@ -113,7 +137,9 @@ async function replaceAllTables(tables, dataDir, currentAdmin) { } for (const table of tables) { const rows = parseNdjson(path.join(dataDir, `${table}.ndjson`)); - if (rows.length) await trx.batchInsert(table, rows, 100); + if (!rows.length) continue; + const jsonCols = await jsonColumnsFor(trx, table); + await trx.batchInsert(table, serialiseJsonColumns(rows, jsonCols), 100); } await reinjectCurrentAdmin(trx, currentAdmin);