Make dev->prod deploy work for non-superuser role and fix image sync.
restore-db-data.js: fall back to multi-pass insert when the DB role cannot set session_replication_role, and parse multi-line INSERT values (bios) as whole statements. sync-images-to-prod.ps1: fix SMB root variable typo and fail fast with guidance when the share is not mapped. Document SMB mapping, non-superuser restore, and a troubleshooting table in the release runbook. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Danila Khodjaef
co-authored by
Cursor
Cursor
parent
ac1ae5794f
commit
313666a4ab
@@ -132,12 +132,21 @@ npm run devtoprod:db:restore -- --file db/DataBackup/gallery_dev_data_YYYYMMDD_H
|
||||
|
||||
Type `yes` at the confirmation prompt (or set `CONFIRM_PROD=1` to skip it in automation).
|
||||
|
||||
The restore loads rows in two ways automatically:
|
||||
|
||||
- **Fast path** — if the DB role may set `session_replication_role` (superuser, or a PG 15+ `GRANT SET ON PARAMETER`), foreign-key checks are disabled for a single-pass load.
|
||||
- **Multi-pass fallback** — the `gallery` role is **not** a superuser, so you will see `note: ... using multi-pass insert`. Rows whose foreign keys are not yet satisfied are retried on later passes until everything loads. No action needed.
|
||||
|
||||
Multi-line values (e.g. artist bios with embedded newlines) are parsed as whole statements, so long text restores correctly.
|
||||
|
||||
**Caveats — prod tables are replaced by dev's contents:**
|
||||
|
||||
- `users` and `curator_audit_log` are overwritten. The **dev curator account and password become the prod login**, and prod audit history is replaced. Make sure the dev curator credentials are the ones you want in prod.
|
||||
- The `session` table is truncated, so any active prod curator sessions are logged out.
|
||||
- The target is guarded: the restore refuses to run unless the database name ends with `_prod` and only reads `infra/docker/.env.prod`.
|
||||
|
||||
> **Optional** — to use the faster single-pass load, have the postgres superuser run this once in pgAdmin (role-global, covers dev and prod): `GRANT SET ON PARAMETER session_replication_role TO gallery;`
|
||||
|
||||
---
|
||||
|
||||
## Step 5 — Sync images to prod (only if image files changed)
|
||||
@@ -146,12 +155,23 @@ Type `yes` at the confirmation prompt (or set `CONFIRM_PROD=1` to skip it in aut
|
||||
|
||||
Copies `data/images/` (paintings, portraits, and `thumbs/`) from the repo to the TrueNAS volume.
|
||||
|
||||
**Map the share in this Windows session first** (a fresh terminal has no mapping — robocopy will fail with `ERROR 5 Access is denied` otherwise):
|
||||
|
||||
```powershell
|
||||
net use \\192.168.10.122\Gallery /user:YOUR_TRUENAS_USER # prompts for password
|
||||
Test-Path "\\192.168.10.122\Gallery" # must print True
|
||||
```
|
||||
|
||||
Then sync:
|
||||
|
||||
```powershell
|
||||
npm run devtoprod:images
|
||||
```
|
||||
|
||||
Type `yes` when prompted. Robocopy exit codes **0–7** = success. Destination: `\\192.168.10.122\Gallery\data\images`.
|
||||
|
||||
Alternatively, let the script map the share by exporting credentials first: `$env:SMB_USER="…"; $env:SMB_PASSWORD="…"` before running.
|
||||
|
||||
> Thumbnails are files, generated on dev (`npm run dev:regenerate-thumbnails` / `dev:regenerate-portrait-thumbs`) and shipped here — prod does **not** regenerate them. Regenerate on dev **before** this step if needed.
|
||||
|
||||
---
|
||||
@@ -240,6 +260,18 @@ Keep at least the most recent `gallery_prod_data_*.txt` from Step 2 so a data ro
|
||||
- Prod restore/backup require typing `yes` (or `CONFIRM_PROD=1`).
|
||||
- `npm run dev:migrate` targets whatever `DB_NAME` is set — always `Remove-Item Env:\DB_NAME` after Step 3 so later commands stay on dev.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Symptom | Cause | Fix |
|
||||
|---------|-------|-----|
|
||||
| `/api/bounds` returns `{"min_year":null,"max_year":null}` on prod | `gallery_prod` has schema but no data | Run Step 4 (restore); confirm rows with a count query on `art_movements` / `paintings` |
|
||||
| `robocopy ... ERROR 5 (0x00000005) Access is denied` on Step 5 | SMB share not mapped/authenticated in this Windows session | `net use \\192.168.10.122\Gallery /user:YOUR_TRUENAS_USER`, verify `Test-Path`, then re-run. If still denied after auth, fix TrueNAS dataset ACL (`chown -R 1001:1001`, grant the SMB user write) |
|
||||
| Restore: `permission denied to set parameter "session_replication_role"` | DB role is not superuser | Handled automatically (multi-pass insert). For the fast path, superuser runs `GRANT SET ON PARAMETER session_replication_role TO gallery;` |
|
||||
| Restore: `unterminated quoted string` | Old parser split multi-line values (bios) | Fixed — statements are accumulated until quotes balance; update to latest `scripts/restore-db-data.js` |
|
||||
| `git push` → `Failed to authenticate user` (Gitea) | Git Credential Manager cached an expired token | Clear it: `"protocol=https`nhost=gitea.mysuperlab.netcraze.pro`n" \| git credential reject`, then push again to re-prompt |
|
||||
| Prod tables empty after a failed restore | Restore truncates **before** inserting; a mid-run error leaves tables empty | Dev is untouched — just re-run Step 4 |
|
||||
| `502 / 504` on the public prod URL | Keenetic upstream wrong (IP/port/protocol) | `192.168.10.122:5173`, protocol **http**; verify LAN `curl.exe -s http://192.168.10.122:5173/api/bounds` first |
|
||||
|
||||
## Related docs
|
||||
|
||||
| Document | Contents |
|
||||
|
||||
@@ -27,8 +27,15 @@ if (-not $SkipConfirm) {
|
||||
|
||||
$smbRoot = "\\192.168.10.122\Gallery"
|
||||
if ($env:SMB_USER -and $env:SMB_PASSWORD) {
|
||||
Write-Host "Mapping $sbmRoot ..."
|
||||
net use $sbmRoot /user:$env:SMB_USER $env:SMB_PASSWORD 2>&1 | Out-Host
|
||||
Write-Host "Mapping $smbRoot ..."
|
||||
net use $smbRoot /user:$env:SMB_USER $env:SMB_PASSWORD 2>&1 | Out-Host
|
||||
} elseif (-not (Test-Path $smbRoot)) {
|
||||
Write-Warning "Cannot access $smbRoot (not authenticated)."
|
||||
Write-Warning "Map the share first, e.g.:"
|
||||
Write-Warning (' net use ' + $smbRoot + ' /user:YOUR_TRUENAS_USER')
|
||||
Write-Warning 'or set $env:SMB_USER and $env:SMB_PASSWORD before running this script.'
|
||||
Write-Error 'SMB share not reachable - aborting before robocopy.'
|
||||
exit 1
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
+100
-11
@@ -28,11 +28,46 @@ function parseArgs(argv) {
|
||||
};
|
||||
}
|
||||
|
||||
// A single-quoted SQL string is closed only when the running count of quote
|
||||
// characters is even (doubled '' escapes count as two, so they stay even).
|
||||
function quotesBalanced(text) {
|
||||
let count = 0;
|
||||
for (let i = 0; i < text.length; i += 1) {
|
||||
if (text[i] === "'") count += 1;
|
||||
}
|
||||
return count % 2 === 0;
|
||||
}
|
||||
|
||||
// Values (e.g. artist bios) can contain embedded newlines, so a statement may
|
||||
// span multiple physical lines. Accumulate lines until the quotes are balanced
|
||||
// and the statement ends with ';'.
|
||||
function extractInsertStatements(content) {
|
||||
return content
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.startsWith('INSERT INTO '));
|
||||
const lines = content.split(/\r?\n/);
|
||||
const statements = [];
|
||||
let current = null;
|
||||
|
||||
for (const line of lines) {
|
||||
if (current === null) {
|
||||
if (line.startsWith('INSERT INTO ')) {
|
||||
current = line;
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
current += `\n${line}`;
|
||||
}
|
||||
|
||||
if (quotesBalanced(current) && current.trimEnd().endsWith(';')) {
|
||||
statements.push(current);
|
||||
current = null;
|
||||
}
|
||||
}
|
||||
|
||||
if (current !== null) {
|
||||
statements.push(current);
|
||||
}
|
||||
|
||||
return statements;
|
||||
}
|
||||
|
||||
async function truncatePublicTables(client) {
|
||||
@@ -79,19 +114,73 @@ async function main() {
|
||||
console.log(`Truncating public tables in "${dbName}"...`);
|
||||
await truncatePublicTables(client);
|
||||
|
||||
// Fast path: disable FK/trigger checks for a single-pass load. Requires
|
||||
// permission to set session_replication_role (superuser, or on PG 15+ a
|
||||
// "GRANT SET ON PARAMETER session_replication_role TO <role>"). If the role
|
||||
// lacks that privilege, fall back to a multi-pass insert that retries rows
|
||||
// whose foreign keys are not yet satisfied, so no superuser is needed.
|
||||
let replicaMode = false;
|
||||
try {
|
||||
await client.query('SET session_replication_role = replica');
|
||||
replicaMode = true;
|
||||
} catch (err) {
|
||||
if (err.code === '42501') {
|
||||
console.warn(
|
||||
' note: cannot set session_replication_role (role is not superuser); using multi-pass insert',
|
||||
);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Restoring ${inserts.length} INSERT statements...`);
|
||||
await client.query('SET session_replication_role = replica');
|
||||
let restored = 0;
|
||||
try {
|
||||
for (const statement of inserts) {
|
||||
await client.query(statement);
|
||||
restored += 1;
|
||||
if (restored % 500 === 0) {
|
||||
console.log(` ${restored}/${inserts.length}`);
|
||||
if (replicaMode) {
|
||||
for (const statement of inserts) {
|
||||
await client.query(statement);
|
||||
restored += 1;
|
||||
if (restored % 500 === 0) {
|
||||
console.log(` ${restored}/${inserts.length}`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Each statement auto-commits on its own, so a foreign-key failure only
|
||||
// rejects that one row; retry it on a later pass once its parent exists.
|
||||
let remaining = inserts;
|
||||
let pass = 0;
|
||||
while (remaining.length > 0) {
|
||||
pass += 1;
|
||||
const retry = [];
|
||||
let progressed = 0;
|
||||
for (const statement of remaining) {
|
||||
try {
|
||||
await client.query(statement);
|
||||
restored += 1;
|
||||
progressed += 1;
|
||||
if (restored % 500 === 0) {
|
||||
console.log(` ${restored}/${inserts.length}`);
|
||||
}
|
||||
} catch (err) {
|
||||
if (err.code === '23503') {
|
||||
retry.push(statement);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (progressed === 0) {
|
||||
throw new Error(
|
||||
`Restore stalled on pass ${pass}: ${retry.length} rows have unresolved foreign keys`,
|
||||
);
|
||||
}
|
||||
remaining = retry;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await client.query('SET session_replication_role = DEFAULT');
|
||||
if (replicaMode) {
|
||||
await client.query('SET session_replication_role = DEFAULT');
|
||||
}
|
||||
}
|
||||
|
||||
await client.end();
|
||||
|
||||
Reference in New Issue
Block a user