From cfee69c9a6b07fa3a1085a1b382a618920363e24 Mon Sep 17 00:00:00 2001 From: Danila Khodjaef Date: Wed, 5 Aug 2026 19:26:52 +0300 Subject: [PATCH] Keep prod users local and fix post-restore id sequences. Prod restore skips users/session/audit, syncs serial sequences after load, and user create re-aligns users_id_seq so new accounts are not misreported as duplicates. Co-authored-by: Cursor --- Documentation/DB_structure.md | 2 +- Documentation/FAC.md | 4 +-- Documentation/deploy-dev-to-prod.md | 7 +++-- Documentation/environments.md | 2 +- Documentation/setup.md | 2 +- scripts/restore-db-data.js | 48 ++++++++++++++++++++++++++++- server/routes/users.js | 27 ++++++++++++++++ 7 files changed, 83 insertions(+), 9 deletions(-) diff --git a/Documentation/DB_structure.md b/Documentation/DB_structure.md index a241f5f..6335d2c 100644 --- a/Documentation/DB_structure.md +++ b/Documentation/DB_structure.md @@ -251,7 +251,7 @@ Append-only log of staff mutations (fix/clear/upload/delete, checkup flags, tran **Logged `action` values:** `painting.fix_image`, `painting.clear_image`, `painting.upload_image`, `painting.delete`, `painting.checkup_flags`, `painting.update_curator_notes`, `artist.fix_portrait`, `artist.clear_portrait`, `artist.upload_portrait`, `artist.checkup_flags`, `translation.upsert`, `translation.publish`, `influence.create`, `influence.update`, `influence.delete`, `influence.import`, `tour.create`, `tour.update`, `tour.delete`, `tour.stops`, `user.create`, `user.update`, `user.reset_password`. -Admins browse this table in the app (**Activity** / `GET /api/audit*`). Each environment’s API uses its own DB (`gallery_dev` vs `gallery_prod`); audit history is not synced by harmonize/devtoprod. +Admins browse this table in the app (**Activity** / `GET /api/audit*`). Each environment’s API uses its own DB (`gallery_dev` vs `gallery_prod`); **`users`**, **`session`**, and audit history are not synced by harmonize/`devtoprod:db:restore`. Example query in pgAdmin: diff --git a/Documentation/FAC.md b/Documentation/FAC.md index 22420a0..1ce4f1b 100644 --- a/Documentation/FAC.md +++ b/Documentation/FAC.md @@ -107,7 +107,7 @@ If login fails after changing `CURATOR_PASSWORD` in `.env`, run `npm run dev:res | Curator | Public browse + assigned permission flags (`images`, `checkup`, `curator_notes`, `translations`, `influences`, `tours`, `users`) | | Admin | All curator tools + **Users** + **Activity** audit reports | -**Users page:** after admin login, header → **Users** — create/edit staff, reset passwords, disable accounts. +**Users page:** after admin login, header → **Users** — create/edit staff, reset passwords, disable accounts. Prod and dev keep **separate** `users` tables: `devtoprod:db:restore` and harmonize never copy staff accounts. If create fails with a confusing “already exists” after a restore, serial sequences may be lagging — current restore syncs them to `MAX(id)`, and user create re-syncs `users_id_seq` before insert. **Activity page:** after admin login, header → **Activity** — filterable curator action log (date/time, curator, action, resource, details, IP) plus summary charts. Reads the DB for that environment (`gallery_dev` on devgallery / `npm run dev:web`, `gallery_prod` on prod). @@ -145,7 +145,7 @@ Run `npm run dev:migrate` against prod DB after first deploy with auth vars set | `npm run dev:db:backup` | Dev data-only backup → `db/DataBackup/*.txt` + `.zip` | | `npm run prod:db:backup` | Prod backup (reads `infra/docker/.env.prod`) | | `npm run dev:db:restore -- --file ` | Restore backup into **dev** (truncates tables first; prompts `yes`) | -| `npm run devtoprod:db:restore -- --file ` | Restore into **prod** (requires confirmation) | +| `npm run devtoprod:db:restore -- --file ` | Restore catalog into **prod** (skips `users` / `session` / `curator_audit_log`; syncs serial sequences; requires confirmation) | | `npm run harmonize` | Bidirectional catalog DB + image merge by `updated_at` / file mtime — [harmonize-dev-prod.md](harmonize-dev-prod.md) | | `npm run harmonize:schema` | Apply dev migrations to prod schema only (dev → prod) | | `npm run harmonize:db` / `harmonize:images` | DB or image merge only (`harmonize:images` also merges artists/paintings checkup flags + image paths, then regenerates thumbs on both sides) | diff --git a/Documentation/deploy-dev-to-prod.md b/Documentation/deploy-dev-to-prod.md index 0b0b6e4..83097c1 100644 --- a/Documentation/deploy-dev-to-prod.md +++ b/Documentation/deploy-dev-to-prod.md @@ -244,10 +244,11 @@ The restore loads rows in two ways automatically: Multi-line values (e.g. artist bios with embedded newlines) are parsed as whole statements, so long text restores correctly. -**Caveats — prod tables are replaced by dev's contents:** +**Caveats — prod catalog tables are replaced by dev's contents:** -- `users` is overwritten. The **dev curator account and password become the prod login**. `curator_audit_log` is **not** synced — prod keeps its existing audit history. -- The `session` table is truncated, so any active prod curator sessions are logged out. +- **`users`**, **`session`**, and **`curator_audit_log`** are **not** truncated or loaded from the backup. Prod staff accounts, passwords, active sessions, and audit history stay as they are on `gallery_prod`. +- Catalog / content tables (`artists`, `paintings`, tours, translations, etc.) are fully replaced by the dev dump. +- After load, serial sequences are reset to `MAX(id)` so new rows (including staff users) do not collide with restored ids. - The target is guarded: the restore refuses to run unless the database name ends with `_prod` and only reads `infra/docker/.env.prod`. > **Optional** — to use the faster single-pass load, have the postgres superuser run this once in pgAdmin (role-global, covers dev and prod): `GRANT SET ON PARAMETER session_replication_role TO gallery;` diff --git a/Documentation/environments.md b/Documentation/environments.md index 964dc7b..78bdce0 100644 --- a/Documentation/environments.md +++ b/Documentation/environments.md @@ -117,7 +117,7 @@ npm run infra:db:split-dev-prod CURATOR_PASSWORD=your-secure-password ``` - Omit `SESSION_COOKIE_SECURE` so cookies follow the request scheme (`TRUST_PROXY` + HTTPS → Secure). Set `true`/`false` to force. `npm run dev:migrate` creates auth tables/roles and bootstraps the first **admin** when `users` is empty. Reset that account later with `npm run dev:reset-curator`. Create additional staff via the in-app **Users** page. Admins browse curator actions on **Activity** (`/api/audit`), which always reads the DB named by `DB_NAME` for that environment (`gallery_dev` here; `gallery_prod` on prod). Audit history is not synced by harmonize/devtoprod. + Omit `SESSION_COOKIE_SECURE` so cookies follow the request scheme (`TRUST_PROXY` + HTTPS → Secure). Set `true`/`false` to force. `npm run dev:migrate` creates auth tables/roles and bootstraps the first **admin** when `users` is empty. Reset that account later with `npm run dev:reset-curator`. Create additional staff via the in-app **Users** page. Admins browse curator actions on **Activity** (`/api/audit`), which always reads the DB named by `DB_NAME` for that environment (`gallery_dev` here; `gallery_prod` on prod). **`users`**, **`session`**, and **`curator_audit_log`** are never copied by harmonize or `devtoprod:db:restore` — each env keeps its own staff accounts and audit history. 2. Run: diff --git a/Documentation/setup.md b/Documentation/setup.md index 3720956..7198956 100644 --- a/Documentation/setup.md +++ b/Documentation/setup.md @@ -112,7 +112,7 @@ Image fetch can take hours if you run it for the entire catalog. The first line | `npm run devtoprod:images` | Copy `data/images/` → TrueNAS via SMB `Gallery` share | | `npm run prodto:dev:images` | Copy prod images → dev repo | | `npm run prodto:dev:db` | Clone `gallery_prod` → `gallery_dev` | -| `npm run dev:db:backup` / `devtoprod:db:restore` | Dev backup / promote DB to prod | +| `npm run dev:db:backup` / `devtoprod:db:restore` | Dev backup / promote catalog DB to prod (prod restore skips staff/session/audit; syncs sequences) | | `npm run harmonize` | Bidirectional catalog + image merge (last-write-wins) — [harmonize-dev-prod.md](harmonize-dev-prod.md) | | `npm run prod:build` | Build production SPA into `client/dist` | | `npm run dev:start` | API + static SPA on `HOST`:`PORT` (uses root `.env`) | diff --git a/scripts/restore-db-data.js b/scripts/restore-db-data.js index ec1d437..fa90c1d 100644 --- a/scripts/restore-db-data.js +++ b/scripts/restore-db-data.js @@ -19,7 +19,47 @@ const { printCliResult } = require('./lib/cli-result'); const { Client } = pg; // Prod restore keeps these tables untouched (no TRUNCATE, no INSERT from dev backup). -const PROD_RESTORE_SKIP_TABLES = new Set(['curator_audit_log']); +// Staff accounts, sessions, and audit history stay env-local — never overwrite from gallery_dev. +const PROD_RESTORE_SKIP_TABLES = new Set(['users', 'session', 'curator_audit_log']); + +function quoteIdent(name) { + return `"${String(name).replace(/"/g, '""')}"`; +} + +/** Align serial/identity sequences with MAX(column) after explicit-id INSERTs. */ +async function syncSerialSequences(client) { + const { rows } = await client.query( + `SELECT + c.relname AS table_name, + a.attname AS column_name, + pg_get_serial_sequence(format('%I.%I', n.nspname, c.relname), a.attname) AS seq_name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped + JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum + WHERE c.relkind = 'r' + AND n.nspname = 'public' + AND pg_get_expr(d.adbin, d.adrelid) LIKE 'nextval(%'` + ); + + let synced = 0; + for (const row of rows) { + if (!row.seq_name) continue; + await client.query( + `SELECT setval( + $1::regclass, + GREATEST( + 1, + COALESCE((SELECT MAX(${quoteIdent(row.column_name)}) FROM ${quoteIdent(row.table_name)}), 1) + ), + true + )`, + [row.seq_name] + ); + synced += 1; + } + return synced; +} function getInsertTableName(statement) { const match = statement.match(/^INSERT INTO "([^"]+)"/i) @@ -239,6 +279,12 @@ async function main() { } } + // Backups insert explicit primary keys; without this, serial nextval() can + // collide with existing ids (e.g. creating a user fails as "already exists"). + console.log('Syncing serial sequences to MAX(id)...'); + const synced = await syncSerialSequences(client); + console.log(` synced ${synced} sequence(s)`); + await client.end(); console.log(`Restore complete: ${restored} statements into "${dbName}".`); if (skippedInserts > 0) { diff --git a/server/routes/users.js b/server/routes/users.js index c6a897a..5ac7720 100644 --- a/server/routes/users.js +++ b/server/routes/users.js @@ -38,6 +38,17 @@ async function clearUserSessions(userId) { await pool.query(`DELETE FROM session WHERE (sess->>'userId')::int = $1`, [userId]); } +/** Keep users_id_seq ahead of existing rows (restore inserts explicit ids). */ +async function syncUsersIdSequence() { + await pool.query( + `SELECT setval( + pg_get_serial_sequence('users', 'id'), + GREATEST(1, COALESCE((SELECT MAX(id) FROM users), 1)), + true + )` + ); +} + router.use(requirePermission('users')); router.get('/', async (_req, res) => { @@ -76,6 +87,16 @@ router.post('/', async (req, res) => { const passwordHash = await bcrypt.hash(password, 10); const storedPermissions = role === 'admin' ? ALL_PERMISSIONS : permissions; + const { rows: taken } = await pool.query( + `SELECT username FROM users WHERE LOWER(username) = LOWER($1) LIMIT 1`, + [username] + ); + if (taken[0]) { + return res.status(409).json({ error: 'Username already exists' }); + } + + await syncUsersIdSequence(); + const { rows } = await pool.query( `INSERT INTO users (username, password_hash, role, permissions, is_active) VALUES ($1, $2, $3, $4::text[], true) @@ -95,6 +116,12 @@ router.post('/', async (req, res) => { res.status(201).json({ user: mapUser(rows[0]) }); } catch (err) { if (err.code === '23505') { + if (err.constraint === 'users_pkey') { + console.error('Users create PK conflict (sequence lag):', err.detail || err.message); + return res.status(409).json({ + error: 'Could not allocate user id — retry create (sequence was out of sync)', + }); + } return res.status(409).json({ error: 'Username already exists' }); } console.error('Users create error:', err.message);