Add dev-prod harmonize tool for bidirectional catalog and image sync.
Schema migrates dev to prod only; catalog rows merge by updated_at and images by mtime with union merge and conflict reporting. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
81ebad2120
commit
f247b418d8
@@ -6,6 +6,8 @@ client/node_modules/
|
|||||||
.env.*.local
|
.env.*.local
|
||||||
infra/docker/.env.prod
|
infra/docker/.env.prod
|
||||||
infra/deploy/devtoprod.config.json
|
infra/deploy/devtoprod.config.json
|
||||||
|
infra/deploy/harmonize.config.json
|
||||||
|
db/SyncReports/
|
||||||
infra/deploy/last-docker-release.json
|
infra/deploy/last-docker-release.json
|
||||||
|
|
||||||
# DB backups (may contain data)
|
# DB backups (may contain data)
|
||||||
|
|||||||
@@ -138,6 +138,10 @@ Run `npm run dev:migrate` against prod DB after first deploy with auth vars set
|
|||||||
| `npm run prod:db:backup` | Prod backup (reads `infra/docker/.env.prod`) |
|
| `npm run prod:db:backup` | Prod backup (reads `infra/docker/.env.prod`) |
|
||||||
| `npm run dev:db:restore -- --file <path>` | Restore backup into **dev** (truncates tables first; prompts `yes`) |
|
| `npm run dev:db:restore -- --file <path>` | Restore backup into **dev** (truncates tables first; prompts `yes`) |
|
||||||
| `npm run devtoprod:db:restore -- --file <path>` | Restore into **prod** (requires confirmation) |
|
| `npm run devtoprod:db:restore -- --file <path>` | Restore into **prod** (requires confirmation) |
|
||||||
|
| `npm run harmonize` | Bidirectional catalog DB + image merge by `updated_at` / file mtime — [harmonize-dev-prod.md](harmonize-dev-prod.md) |
|
||||||
|
| `npm run harmonize:schema` | Apply dev migrations to prod schema only (dev → prod) |
|
||||||
|
| `npm run harmonize:db` / `harmonize:images` | DB or image merge only |
|
||||||
|
| `npm run dev:backfill-updated-at` | Backfill catalog `updated_at` from image mtimes (dev) |
|
||||||
|
|
||||||
**One-time split (recommended):** pgAdmin on dev PC → open [`db/split-dev-prod-pgadmin.sql`](../db/split-dev-prod-pgadmin.sql) → run each STEP on database `postgres`, then verify on `gallery_dev`.
|
**One-time split (recommended):** pgAdmin on dev PC → open [`db/split-dev-prod-pgadmin.sql`](../db/split-dev-prod-pgadmin.sql) → run each STEP on database `postgres`, then verify on `gallery_dev`.
|
||||||
|
|
||||||
@@ -237,6 +241,8 @@ npm run devtoprod:release # full promote from infra/deploy/devtoprod.conf
|
|||||||
npm run devtoprod:thumbnails # rebuild thumb files + DB paths on dev (before backup)
|
npm run devtoprod:thumbnails # rebuild thumb files + DB paths on dev (before backup)
|
||||||
npm run devtoprod:images # dev repo → TrueNAS (promote / first deploy)
|
npm run devtoprod:images # dev repo → TrueNAS (promote / first deploy)
|
||||||
npm run prodto:dev:images # TrueNAS → dev repo
|
npm run prodto:dev:images # TrueNAS → dev repo
|
||||||
|
npm run harmonize # bidirectional merge (newer wins) — see harmonize-dev-prod.md
|
||||||
|
npm run harmonize:dry-run # preview DB + image changes only
|
||||||
```
|
```
|
||||||
|
|
||||||
Type `yes` when prompted (or set `autoConfirm: true` in release config). Robocopy exit codes **0–7** = success. Deploy scripts print a final **`===== SUCCESS =====`** or **`===== FAILED =====`** banner.
|
Type `yes` when prompted (or set `autoConfirm: true` in release config). Robocopy exit codes **0–7** = success. Deploy scripts print a final **`===== SUCCESS =====`** or **`===== FAILED =====`** banner.
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ this file contains draft for future releases and features
|
|||||||
1. Multy language support, russian version at least, search for best implementation, preferably story in db and easily expandable. tool to check and correct translation
|
1. Multy language support, russian version at least, search for best implementation, preferably story in db and easily expandable. tool to check and correct translation
|
||||||
2. tool to manage links (influence/influenced by ) import csv's ( define format), edit ,add, delete, visualize, map to pictures/ entities
|
2. tool to manage links (influence/influenced by ) import csv's ( define format), edit ,add, delete, visualize, map to pictures/ entities
|
||||||
3. tool to monitor/manage (plan actions) of curator actions, markers to check painting/text ?
|
3. tool to monitor/manage (plan actions) of curator actions, markers to check painting/text ?
|
||||||
4. tool to sync prod /env resources (both ways), db structure, db data, images, users etc
|
4. ~~tool to sync prod /env resources (both ways), db structure, db data, images, users etc~~ — done for catalog DB + images: `npm run harmonize` (schema dev→prod only; users/audit excluded) — [harmonize-dev-prod.md](harmonize-dev-prod.md)
|
||||||
5. curator_audit_log should contain log of actions like fixit, checked, upload etc with details for which entity it was made and details what was the action and outcome
|
5. curator_audit_log should contain log of actions like fixit, checked, upload etc with details for which entity it was made and details what was the action and outcome
|
||||||
6. ~~create search by entity (painting, artist, movement)~~ — done: timeline header + `GET /api/search`
|
6. ~~create search by entity (painting, artist, movement)~~ — done: timeline header + `GET /api/search`
|
||||||
7. create guided tours (with text/extra infor, set of entities)
|
7. create guided tours (with text/extra infor, set of entities)
|
||||||
|
|||||||
@@ -471,6 +471,7 @@ See [API.md](API.md#authentication) and [data-and-images.md](data-and-images.md#
|
|||||||
|----------|----------|
|
|----------|----------|
|
||||||
| [setup.md](setup.md) | Install, database, npm scripts |
|
| [setup.md](setup.md) | Install, database, npm scripts |
|
||||||
| [deploy-dev-to-prod.md](deploy-dev-to-prod.md) | Release runbook + one-command `devtoprod:release` |
|
| [deploy-dev-to-prod.md](deploy-dev-to-prod.md) | Release runbook + one-command `devtoprod:release` |
|
||||||
|
| [harmonize-dev-prod.md](harmonize-dev-prod.md) | Incremental dev ↔ prod merge (catalog DB + images) |
|
||||||
| [DB_structure.md](DB_structure.md) | Tables and relationships |
|
| [DB_structure.md](DB_structure.md) | Tables and relationships |
|
||||||
| [API.md](API.md) | REST endpoints |
|
| [API.md](API.md) | REST endpoints |
|
||||||
| [data-and-images.md](data-and-images.md) | Image pipeline and seeding |
|
| [data-and-images.md](data-and-images.md) | Image pipeline and seeding |
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ File names are sanitised `{Artist}_{Title}.{ext}`. The image service can redisco
|
|||||||
| **Development** | `./data/images/` in repo | Working copy on dev PC |
|
| **Development** | `./data/images/` in repo | Working copy on dev PC |
|
||||||
| **Production** | `/mnt/BasePool/Applications/Gallery/data/images` on TrueNAS | SMB `\\192.168.10.122\Gallery\data\images` |
|
| **Production** | `/mnt/BasePool/Applications/Gallery/data/images` on TrueNAS | SMB `\\192.168.10.122\Gallery\data\images` |
|
||||||
|
|
||||||
Promote dev → prod files: `npm run devtoprod:images` (after `net use \\192.168.10.122\Gallery`). Refresh dev from prod: `npm run prodto:dev:images`. See [environments.md](environments.md).
|
**One-direction promote:** `npm run devtoprod:images` (dev → prod, skip older). **Refresh dev from prod:** `npm run prodto:dev:images`. **Bidirectional merge** (newer file wins): `npm run harmonize:images` or full `npm run harmonize` — see [harmonize-dev-prod.md](harmonize-dev-prod.md). General sync reference: [environments.md](environments.md).
|
||||||
|
|
||||||
## Scripts overview
|
## Scripts overview
|
||||||
|
|
||||||
@@ -49,6 +49,8 @@ Promote dev → prod files: `npm run devtoprod:images` (after `net use \\192.168
|
|||||||
| `fetch-missing-images.js` | `npm run dev:fetch-images` | Downloads files for paintings missing on disk |
|
| `fetch-missing-images.js` | `npm run dev:fetch-images` | Downloads files for paintings missing on disk |
|
||||||
| `image-fetcher.js` | *(library)* | Wikimedia / museum resolution used by fetch scripts and API |
|
| `image-fetcher.js` | *(library)* | Wikimedia / museum resolution used by fetch scripts and API |
|
||||||
| `sync-images-to-prod.ps1` / `sync-images-from-prod.ps1` | `npm run devtoprod:images` / `npm run prodto:dev:images` | Robocopy via SMB `\\192.168.10.122\Gallery` |
|
| `sync-images-to-prod.ps1` / `sync-images-from-prod.ps1` | `npm run devtoprod:images` / `npm run prodto:dev:images` | Robocopy via SMB `\\192.168.10.122\Gallery` |
|
||||||
|
| `harmonize-db.js` / `harmonize-images.js` | `npm run harmonize:db` / `harmonize:images` | Bidirectional merge by `updated_at` / file mtime |
|
||||||
|
| `harmonize.ps1` | `npm run harmonize` | Orchestrator: backups, optional schema, DB + image merge |
|
||||||
| `regenerate-thumbnails.js` | `npm run dev:regenerate-thumbnails` | Rebuild painting thumbs from full images via `sharp` |
|
| `regenerate-thumbnails.js` | `npm run dev:regenerate-thumbnails` | Rebuild painting thumbs from full images via `sharp` |
|
||||||
| `regenerate-portrait-thumbs.js` | `npm run dev:regenerate-portrait-thumbs` | Rebuild timeline portrait thumbs (~256px) and set `portrait_thumb_path` |
|
| `regenerate-portrait-thumbs.js` | `npm run dev:regenerate-portrait-thumbs` | Rebuild timeline portrait thumbs (~256px) and set `portrait_thumb_path` |
|
||||||
| `audit-painting-images.js` | `npm run dev:audit-painting-images` | Detect thumb/full aspect-ratio mismatches |
|
| `audit-painting-images.js` | `npm run dev:audit-painting-images` | Detect thumb/full aspect-ratio mismatches |
|
||||||
|
|||||||
@@ -2,7 +2,11 @@
|
|||||||
|
|
||||||
Step-by-step guide for promoting the **development** version of Gallery to **production**. Covers code, database schema, database data, thumbnail generation, and image files.
|
Step-by-step guide for promoting the **development** version of Gallery to **production**. Covers code, database schema, database data, thumbnail generation, and image files.
|
||||||
|
|
||||||
> **Default is dev.** This runbook is for a **scheduled release** (~weekly, or when you explicitly decide to ship). Day-to-day work stays on dev — see [environments.md](environments.md#development-first-workflow-default). If the one-time prod install is not done yet, follow [environments.md → One-time setup](environments.md#one-time-setup-full-walkthrough) and [infra/docker/DEPLOY-truenas.md](../infra/docker/DEPLOY-truenas.md) first.
|
> **Default is dev.** This runbook is for a **scheduled release** (~weekly, or when you explicitly decide to ship). Day-to-day work stays on dev — see [environments.md](environments.md#development-first-workflow-default).
|
||||||
|
>
|
||||||
|
> **Mid-week merge:** If both dev and prod have catalog or image edits and you need **last-write-wins** sync instead of a full prod overwrite, use [harmonize-dev-prod.md](harmonize-dev-prod.md) (`npm run harmonize`). Keep this runbook for releases where prod should exactly match dev.
|
||||||
|
>
|
||||||
|
> If the one-time prod install is not done yet, follow [environments.md → One-time setup](environments.md#one-time-setup-full-walkthrough) and [infra/docker/DEPLOY-truenas.md](../infra/docker/DEPLOY-truenas.md) first.
|
||||||
|
|
||||||
| | Dev (source) | Prod (target) |
|
| | Dev (source) | Prod (target) |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
|
|||||||
@@ -305,6 +305,7 @@ See also [Drunkmeyou gitea-https-keenetic-npm-setup.md](../../Drunkmeyou/Documen
|
|||||||
| Fast local HMR (no Keenetic) | `npm run dev:server` + `npm run dev:client` |
|
| Fast local HMR (no Keenetic) | `npm run dev:server` + `npm run dev:client` |
|
||||||
| Refresh dev DB from prod | `npm run prodto:dev:db` |
|
| Refresh dev DB from prod | `npm run prodto:dev:db` |
|
||||||
| Pull prod images to dev | `npm run prodto:dev:images` |
|
| Pull prod images to dev | `npm run prodto:dev:images` |
|
||||||
|
| Merge dev ↔ prod catalog + images (incremental) | `npm run harmonize` — see [harmonize-dev-prod.md](harmonize-dev-prod.md) |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -312,6 +313,8 @@ See also [Drunkmeyou gitea-https-keenetic-npm-setup.md](../../Drunkmeyou/Documen
|
|||||||
|
|
||||||
Run this when you are ready to ship dev to production — **not** after every small change. Typical cadence: **about once a week**.
|
Run this when you are ready to ship dev to production — **not** after every small change. Typical cadence: **about once a week**.
|
||||||
|
|
||||||
|
For **incremental** dev ↔ prod merge (both sides edited), use [harmonize-dev-prod.md](harmonize-dev-prod.md) instead of full restore.
|
||||||
|
|
||||||
**One command (recommended):** copy [`infra/deploy/devtoprod.config.example.json`](../infra/deploy/devtoprod.config.example.json) to `infra/deploy/devtoprod.config.json`, edit it, then `npm run devtoprod:release` or `deploy-dev-to-prod.cmd`. See [deploy-dev-to-prod.md → One-command release](deploy-dev-to-prod.md#one-command-release-automated).
|
**One command (recommended):** copy [`infra/deploy/devtoprod.config.example.json`](../infra/deploy/devtoprod.config.example.json) to `infra/deploy/devtoprod.config.json`, edit it, then `npm run devtoprod:release` or `deploy-dev-to-prod.cmd`. See [deploy-dev-to-prod.md → One-command release](deploy-dev-to-prod.md#one-command-release-automated).
|
||||||
|
|
||||||
**Manual steps:** detailed runbook (per-change decision matrix, schema migration, rollback): [deploy-dev-to-prod.md](deploy-dev-to-prod.md).
|
**Manual steps:** detailed runbook (per-change decision matrix, schema migration, rollback): [deploy-dev-to-prod.md](deploy-dev-to-prod.md).
|
||||||
@@ -338,11 +341,15 @@ Run this when you are ready to ship dev to production — **not** after every sm
|
|||||||
| `npm run prodto:dev:db` | Dev PC PowerShell | Clone prod → dev |
|
| `npm run prodto:dev:db` | Dev PC PowerShell | Clone prod → dev |
|
||||||
| `npm run dev:db:backup` | Dev PC PowerShell | Dev backup |
|
| `npm run dev:db:backup` | Dev PC PowerShell | Dev backup |
|
||||||
| `npm run devtoprod:db:restore` | Dev PC PowerShell | Restore into prod |
|
| `npm run devtoprod:db:restore` | Dev PC PowerShell | Restore into prod |
|
||||||
|
| `npm run harmonize` | Dev PC PowerShell | Bidirectional catalog + image merge — [harmonize-dev-prod.md](harmonize-dev-prod.md) |
|
||||||
|
| `npm run harmonize:db` | Dev PC PowerShell | DB merge only |
|
||||||
|
| `npm run harmonize:images` | Dev PC PowerShell | Image merge only |
|
||||||
|
|
||||||
## Image sync
|
## Image sync
|
||||||
|
|
||||||
| Command | Where | Direction |
|
| Command | Where | Direction |
|
||||||
|---------|-------|-----------|
|
|---------|-------|-----------|
|
||||||
|
| `npm run harmonize` | Dev PC PowerShell | Bidirectional merge (mtime newer wins) — [harmonize-dev-prod.md](harmonize-dev-prod.md) |
|
||||||
| `npm run devtoprod:release` | Dev PC PowerShell | Full config-driven promote (see [deploy-dev-to-prod.md](deploy-dev-to-prod.md#one-command-release-automated)) |
|
| `npm run devtoprod:release` | Dev PC PowerShell | Full config-driven promote (see [deploy-dev-to-prod.md](deploy-dev-to-prod.md#one-command-release-automated)) |
|
||||||
| `npm run devtoprod:thumbnails` | Dev PC PowerShell | Rebuild painting + portrait thumbs on dev before promote |
|
| `npm run devtoprod:thumbnails` | Dev PC PowerShell | Rebuild painting + portrait thumbs on dev before promote |
|
||||||
| `npm run devtoprod:images` | Dev PC PowerShell | Dev → TrueNAS volume |
|
| `npm run devtoprod:images` | Dev PC PowerShell | Dev → TrueNAS volume |
|
||||||
|
|||||||
@@ -0,0 +1,173 @@
|
|||||||
|
# Harmonize dev and prod (incremental merge)
|
||||||
|
|
||||||
|
Bidirectional **catalog data** and **image** sync between `gallery_dev` and `gallery_prod`, with **last-write-wins** by timestamp. Use this when **both** environments may have curator edits since the last release — not when prod should become an exact copy of dev.
|
||||||
|
|
||||||
|
For a full prod replace (weekly release), use [deploy-dev-to-prod.md](deploy-dev-to-prod.md) (`npm run devtoprod:release`).
|
||||||
|
|
||||||
|
For refreshing dev from prod entirely, use `npm run prodto:dev:db` (destructive to dev).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Rules
|
||||||
|
|
||||||
|
| Layer | Direction | Conflict resolution |
|
||||||
|
|-------|-----------|---------------------|
|
||||||
|
| **Schema** | dev → prod only | Run `npm run harmonize:schema` (same as `dev:migrate` on `gallery_prod`) |
|
||||||
|
| **Catalog DB** | dev ↔ prod | Newer `updated_at` wins; missing rows copied to the other side (union merge) |
|
||||||
|
| **Images** | dev ↔ prod | Newer file mtime wins; missing files copied to the other side |
|
||||||
|
| **Users / sessions / audit** | not synced | `users`, `session`, `curator_audit_log` stay env-local |
|
||||||
|
|
||||||
|
**Never auto-deletes** rows or files that exist on only one side.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
1. Both databases on Postgres `192.168.10.122` with `updated_at` columns applied:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
npm run dev:migrate
|
||||||
|
$env:DB_NAME = "gallery_prod"; npm run dev:migrate; Remove-Item Env:\DB_NAME
|
||||||
|
```
|
||||||
|
|
||||||
|
2. Backfill `updated_at` from image file mtimes (recommended once after migration):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
npm run dev:backfill-updated-at
|
||||||
|
npm run harmonize:backfill-updated-at
|
||||||
|
```
|
||||||
|
|
||||||
|
3. [`infra/docker/.env.prod`](../infra/docker/.env.prod) present with `DB_NAME=gallery_prod`.
|
||||||
|
|
||||||
|
4. SMB share reachable for prod images:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
net use \\192.168.10.122\Gallery /user:YOUR_TRUENAS_USER
|
||||||
|
```
|
||||||
|
|
||||||
|
5. Copy harmonize config:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
Copy-Item infra/deploy/harmonize.config.example.json infra/deploy/harmonize.config.json
|
||||||
|
```
|
||||||
|
|
||||||
|
Edit `harmonize.config.json` (gitignored) — optional `smb.user` / `smb.password`, `prefer` for tie-breaks (`dev` | `prod`), `schemaChanged: true` when new migrations shipped.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## One-command harmonize
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
npm run harmonize
|
||||||
|
```
|
||||||
|
|
||||||
|
Dry-run (report only, no writes):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
npm run harmonize -- -DryRun
|
||||||
|
```
|
||||||
|
|
||||||
|
Or:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
npm run harmonize:dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
### Orchestrator steps
|
||||||
|
|
||||||
|
| Step | npm script | Purpose |
|
||||||
|
|------|------------|---------|
|
||||||
|
| `backupDev` | `dev:db:backup` | Safety snapshot |
|
||||||
|
| `backupProd` | `prod:db:backup` | Safety snapshot |
|
||||||
|
| `schema` | `harmonize:schema` | Apply dev migrations to prod (when `schemaChanged: true`) |
|
||||||
|
| `db` | `harmonize:db` | Row-level catalog merge |
|
||||||
|
| `images` | `harmonize:images` | Bidirectional file merge |
|
||||||
|
| `verify` | curl `/api/bounds` | Optional smoke check |
|
||||||
|
|
||||||
|
Reports are written to `db/SyncReports/harmonize_db_*.json` and `harmonize_images_*.json` (gitignored).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Individual commands
|
||||||
|
|
||||||
|
| Command | Purpose |
|
||||||
|
|---------|---------|
|
||||||
|
| `npm run harmonize:schema` | Migrate prod schema from dev migration files |
|
||||||
|
| `npm run harmonize:db` | Merge catalog rows by `updated_at` |
|
||||||
|
| `npm run harmonize:db -- --dry-run` | Preview DB changes |
|
||||||
|
| `npm run harmonize:db -- --prefer=dev` | On equal `updated_at`, dev wins |
|
||||||
|
| `npm run harmonize:images` | Merge image files by mtime |
|
||||||
|
| `npm run dev:migrate:sync-timestamps` | Apply `updated_at` migration on dev only |
|
||||||
|
| `npm run dev:backfill-updated-at` | Backfill dev `updated_at` from image mtimes |
|
||||||
|
| `npm run harmonize:backfill-updated-at` | Same backfill on prod |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Catalog tables synced
|
||||||
|
|
||||||
|
Processed in FK order:
|
||||||
|
|
||||||
|
`historical_eras` → `art_movements` → `artists` → `artist_periods` → `paintings` → `painting_influences` → `painting_influence_sources` → `painting_annotations`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Conflict handling
|
||||||
|
|
||||||
|
Harmonize reports conflicts in the JSON report and skips those rows:
|
||||||
|
|
||||||
|
| Conflict | Cause | Resolution |
|
||||||
|
|----------|-------|------------|
|
||||||
|
| `id_collision` | Same `id` but different natural key (e.g. artist name) | Manual fix in pgAdmin; environments diverged too far |
|
||||||
|
| `equal_updated_at` | Same timestamp, different row content | Re-run with `--prefer=dev` or `--prefer=prod`, or edit one side and re-run |
|
||||||
|
|
||||||
|
**Tip:** Harmonize regularly from a shared baseline (e.g. after each weekly release) to avoid ID/natural-key collisions from independent inserts.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## When to use what
|
||||||
|
|
||||||
|
| Situation | Tool |
|
||||||
|
|-----------|------|
|
||||||
|
| Weekly release — prod should match dev exactly | `npm run devtoprod:release` |
|
||||||
|
| Mid-week prod curator fix + dev also changed | `npm run harmonize` |
|
||||||
|
| Dev workspace stale — full prod copy | `npm run prodto:dev:db` |
|
||||||
|
| New migration in repo | `harmonize:schema` or deploy step 4 |
|
||||||
|
| Only images changed on one side | `npm run harmonize:images` |
|
||||||
|
| Only DB metadata changed | `npm run harmonize:db` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Example flows
|
||||||
|
|
||||||
|
### Prod curator uploaded a painting; dev also edited metadata
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
npm run harmonize
|
||||||
|
```
|
||||||
|
|
||||||
|
DB rows merge by `updated_at`; image files merge by mtime. Both sides receive the latest version of each entity.
|
||||||
|
|
||||||
|
### Schema change + mixed edits
|
||||||
|
|
||||||
|
1. Finish and test on dev: `npm run dev:migrate`
|
||||||
|
2. Set `schemaChanged: true` in `harmonize.config.json` (or enable `steps.schema`)
|
||||||
|
3. `npm run harmonize`
|
||||||
|
|
||||||
|
### Preview before writing
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
npm run harmonize:dry-run
|
||||||
|
# Review db/SyncReports/harmonize_*.json
|
||||||
|
npm run harmonize
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Safety
|
||||||
|
|
||||||
|
- Pre-flight backups of dev and prod DB (configurable; on by default)
|
||||||
|
- Prod writes require `yes` or `CONFIRM_PROD=1` (orchestrator sets `CONFIRM_PROD=1` when `autoConfirm`-style run)
|
||||||
|
- No TRUNCATE — harmonize only inserts/updates changed rows
|
||||||
|
- Rollback: restore from `db/DataBackup/gallery_*_data_*.txt` using `dev:db:restore` or `devtoprod:db:restore`
|
||||||
|
|
||||||
|
See also [environments.md](environments.md) and [deploy-dev-to-prod.md](deploy-dev-to-prod.md).
|
||||||
@@ -110,13 +110,14 @@ Image fetch can take hours if you run it for the entire catalog. The first line
|
|||||||
| `npm run prodto:dev:images` | Copy prod images → dev repo |
|
| `npm run prodto:dev:images` | Copy prod images → dev repo |
|
||||||
| `npm run prodto:dev:db` | Clone `gallery_prod` → `gallery_dev` |
|
| `npm run prodto:dev:db` | Clone `gallery_prod` → `gallery_dev` |
|
||||||
| `npm run dev:db:backup` / `devtoprod:db:restore` | Dev backup / promote DB to prod |
|
| `npm run dev:db:backup` / `devtoprod:db:restore` | Dev backup / promote DB to prod |
|
||||||
|
| `npm run harmonize` | Bidirectional catalog + image merge (last-write-wins) — [harmonize-dev-prod.md](harmonize-dev-prod.md) |
|
||||||
| `npm run prod:build` | Build production SPA into `client/dist` |
|
| `npm run prod:build` | Build production SPA into `client/dist` |
|
||||||
| `npm run dev:start` | API + static SPA on `HOST`:`PORT` (uses root `.env`) |
|
| `npm run dev:start` | API + static SPA on `HOST`:`PORT` (uses root `.env`) |
|
||||||
| `npm run prod:start` | Build client, then start server |
|
| `npm run prod:start` | Build client, then start server |
|
||||||
| `npm run dev:server` | API with nodemon reload (local HMR workflow) |
|
| `npm run dev:server` | API with nodemon reload (local HMR workflow) |
|
||||||
| `npm run dev:client` | Vite dev server on :5173 |
|
| `npm run dev:client` | Vite dev server on :5173 |
|
||||||
|
|
||||||
See [environments.md](environments.md) for dev/prod URLs, database split, Docker deploy, and sync commands. Quick reference: [FAC.md](FAC.md).
|
See [environments.md](environments.md) for dev/prod URLs, database split, Docker deploy, and sync commands. For incremental dev ↔ prod merge (both sides edited), see [harmonize-dev-prod.md](harmonize-dev-prod.md). Quick reference: [FAC.md](FAC.md).
|
||||||
|
|
||||||
**Production frontend:** build the client, then start the server:
|
**Production frontend:** build the client, then start the server:
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
-- Catalog row timestamps for dev/prod harmonize (last-write-wins merge).
|
||||||
|
-- Idempotent: safe to re-run on existing databases.
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION sync_touch_updated_at()
|
||||||
|
RETURNS TRIGGER AS $$
|
||||||
|
BEGIN
|
||||||
|
NEW.updated_at = now();
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
tbl TEXT;
|
||||||
|
BEGIN
|
||||||
|
FOREACH tbl IN ARRAY ARRAY[
|
||||||
|
'historical_eras',
|
||||||
|
'art_movements',
|
||||||
|
'artists',
|
||||||
|
'artist_periods',
|
||||||
|
'paintings',
|
||||||
|
'painting_influences',
|
||||||
|
'painting_influence_sources',
|
||||||
|
'painting_annotations'
|
||||||
|
]
|
||||||
|
LOOP
|
||||||
|
EXECUTE format(
|
||||||
|
'ALTER TABLE %I ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now()',
|
||||||
|
tbl
|
||||||
|
);
|
||||||
|
|
||||||
|
EXECUTE format('DROP TRIGGER IF EXISTS %I ON %I', tbl || '_updated_at', tbl);
|
||||||
|
EXECUTE format(
|
||||||
|
'CREATE TRIGGER %I BEFORE UPDATE ON %I FOR EACH ROW EXECUTE PROCEDURE sync_touch_updated_at()',
|
||||||
|
tbl || '_updated_at',
|
||||||
|
tbl
|
||||||
|
);
|
||||||
|
END LOOP;
|
||||||
|
END $$;
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
{
|
||||||
|
"_comment": "Copy to harmonize.config.json (gitignored) before running npm run harmonize",
|
||||||
|
"dryRun": false,
|
||||||
|
"prefer": null,
|
||||||
|
"schemaChanged": false,
|
||||||
|
"steps": {
|
||||||
|
"backupDev": true,
|
||||||
|
"backupProd": true,
|
||||||
|
"schema": false,
|
||||||
|
"db": true,
|
||||||
|
"images": true,
|
||||||
|
"verify": false
|
||||||
|
},
|
||||||
|
"smb": {
|
||||||
|
"host": "192.168.10.122",
|
||||||
|
"share": "Gallery",
|
||||||
|
"user": "",
|
||||||
|
"password": ""
|
||||||
|
},
|
||||||
|
"paths": {
|
||||||
|
"prodImages": "\\\\192.168.10.122\\Gallery\\data\\images"
|
||||||
|
},
|
||||||
|
"verify": {
|
||||||
|
"devUrl": "https://devgallery.mysuperlab.netcraze.pro/api/bounds",
|
||||||
|
"publicUrl": "https://gallery.mysuperlab.netcraze.pro/api/bounds"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Map SMB share for harmonize image step (optional wrapper).
|
||||||
|
param(
|
||||||
|
[switch]$SkipConfirm
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
. (Join-Path $PSScriptRoot "lib\Deploy-CliResult.ps1")
|
||||||
|
|
||||||
|
$smbRoot = "\\192.168.10.122\Gallery"
|
||||||
|
if ($env:SMB_USER -and $env:SMB_PASSWORD) {
|
||||||
|
Write-Host "Mapping $smbRoot ..."
|
||||||
|
net use $smbRoot /user:$env:SMB_USER $env:SMB_PASSWORD 2>&1 | Out-Host
|
||||||
|
} elseif (-not (Test-Path $smbRoot)) {
|
||||||
|
Write-Warning "Cannot access $smbRoot (not authenticated)."
|
||||||
|
Write-Warning "Map the share first or set `$env:SMB_USER and `$env:SMB_PASSWORD."
|
||||||
|
}
|
||||||
|
|
||||||
|
$npmArgs = @('run', 'harmonize:images')
|
||||||
|
if ($args.Count -gt 0) {
|
||||||
|
$npmArgs += '--'
|
||||||
|
$npmArgs += $args
|
||||||
|
}
|
||||||
|
|
||||||
|
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
|
||||||
|
Set-Location $RepoRoot
|
||||||
|
& npm @npmArgs
|
||||||
|
exit $LASTEXITCODE
|
||||||
@@ -0,0 +1,234 @@
|
|||||||
|
# Dev <-> prod harmonize orchestrator (incremental merge; not full replace).
|
||||||
|
#
|
||||||
|
# Usage (from repo root):
|
||||||
|
# npm run harmonize
|
||||||
|
# npm run harmonize -- -DryRun
|
||||||
|
# npm run harmonize -- -Config infra/deploy/harmonize.config.json
|
||||||
|
#
|
||||||
|
# Requires infra/deploy/harmonize.config.json (copy from harmonize.config.example.json)
|
||||||
|
# or infra/deploy/devtoprod.config.json with optional "harmonize" section.
|
||||||
|
|
||||||
|
param(
|
||||||
|
[string]$Config = (Join-Path $PSScriptRoot "..\deploy\harmonize.config.json"),
|
||||||
|
[switch]$DryRun
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
. (Join-Path $PSScriptRoot "lib\Deploy-CliResult.ps1")
|
||||||
|
|
||||||
|
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
|
||||||
|
Set-Location $RepoRoot
|
||||||
|
|
||||||
|
$BannerWidth = 64
|
||||||
|
$CompletedSteps = [System.Collections.Generic.List[string]]::new()
|
||||||
|
$FailedStep = $null
|
||||||
|
|
||||||
|
function Write-HarmonizeBanner {
|
||||||
|
param([bool]$Success, [string]$FailedAt = '')
|
||||||
|
|
||||||
|
Write-Host ''
|
||||||
|
Write-Host ('=' * $BannerWidth)
|
||||||
|
if ($Success) {
|
||||||
|
Write-Host ' HARMONIZE SUCCEEDED - dev and prod converged'
|
||||||
|
Write-Host (' Steps: ' + ($CompletedSteps -join ', '))
|
||||||
|
} else {
|
||||||
|
Write-Host " HARMONIZE FAILED at step: $FailedAt"
|
||||||
|
if ($CompletedSteps.Count -gt 0) {
|
||||||
|
Write-Host (' Completed: ' + ($CompletedSteps -join ', '))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Write-Host ('=' * $BannerWidth)
|
||||||
|
}
|
||||||
|
|
||||||
|
function Invoke-Npm {
|
||||||
|
param([string]$Script, [string[]]$ExtraArgs = @())
|
||||||
|
$npmArgs = @('run', $Script)
|
||||||
|
if ($ExtraArgs.Count -gt 0) {
|
||||||
|
$npmArgs += '--'
|
||||||
|
$npmArgs += $ExtraArgs
|
||||||
|
}
|
||||||
|
Write-Host ("Running npm: npm " + ($npmArgs -join ' ')) -ForegroundColor DarkGray
|
||||||
|
$prevEap = $ErrorActionPreference
|
||||||
|
$ErrorActionPreference = 'Continue'
|
||||||
|
$output = & npm @npmArgs 2>&1
|
||||||
|
$exit = $LASTEXITCODE
|
||||||
|
$ErrorActionPreference = $prevEap
|
||||||
|
foreach ($line in $output) { Write-Host $line }
|
||||||
|
return $exit
|
||||||
|
}
|
||||||
|
|
||||||
|
function Invoke-Step {
|
||||||
|
param([string]$Name, [scriptblock]$Action)
|
||||||
|
|
||||||
|
if ($DryRun) {
|
||||||
|
Write-Host "[dry-run] Would run: $Name"
|
||||||
|
$script:CompletedSteps.Add($Name)
|
||||||
|
return $true
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Host ''
|
||||||
|
Write-Host "--- Step: $Name ---" -ForegroundColor Cyan
|
||||||
|
try {
|
||||||
|
$code = & $Action
|
||||||
|
if ($null -eq $code) { $code = if ($null -ne $LASTEXITCODE) { $LASTEXITCODE } else { 0 } }
|
||||||
|
if ($code -ne 0) {
|
||||||
|
$script:FailedStep = $Name
|
||||||
|
return $false
|
||||||
|
}
|
||||||
|
$script:CompletedSteps.Add($Name)
|
||||||
|
return $true
|
||||||
|
} catch {
|
||||||
|
Write-Host $_.Exception.Message -ForegroundColor Red
|
||||||
|
$script:FailedStep = $Name
|
||||||
|
return $false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Resolve-ConfigPath {
|
||||||
|
if (Test-Path $Config) { return (Resolve-Path $Config).Path }
|
||||||
|
$fallback = Join-Path $PSScriptRoot "..\deploy\devtoprod.config.json"
|
||||||
|
if (Test-Path $fallback) {
|
||||||
|
Write-Host "Using fallback config: $fallback" -ForegroundColor DarkYellow
|
||||||
|
return (Resolve-Path $fallback).Path
|
||||||
|
}
|
||||||
|
throw "Missing harmonize config. Copy infra/deploy/harmonize.config.example.json to infra/deploy/harmonize.config.json"
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-HarmonizeSteps {
|
||||||
|
param([object]$Raw)
|
||||||
|
|
||||||
|
$harmonize = $Raw.harmonize
|
||||||
|
if ($null -eq $harmonize) { $harmonize = $Raw }
|
||||||
|
|
||||||
|
$schemaChanged = $false
|
||||||
|
if ($null -ne $harmonize.schemaChanged) { $schemaChanged = [bool]$harmonize.schemaChanged }
|
||||||
|
elseif ($null -ne $Raw.schemaChanged) { $schemaChanged = [bool]$Raw.schemaChanged }
|
||||||
|
|
||||||
|
$steps = @{
|
||||||
|
backupDev = $true
|
||||||
|
backupProd = $true
|
||||||
|
schema = $schemaChanged
|
||||||
|
db = $true
|
||||||
|
images = $true
|
||||||
|
verify = $false
|
||||||
|
}
|
||||||
|
|
||||||
|
$cfgSteps = $harmonize.steps
|
||||||
|
if ($null -ne $cfgSteps) {
|
||||||
|
foreach ($key in $cfgSteps.PSObject.Properties.Name) {
|
||||||
|
$steps[$key] = [bool]$cfgSteps.$key
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return $steps
|
||||||
|
}
|
||||||
|
|
||||||
|
function Set-SmbCredentials {
|
||||||
|
param([object]$Smb)
|
||||||
|
|
||||||
|
if ($Smb.user -and $Smb.password) {
|
||||||
|
$env:SMB_USER = [string]$Smb.user
|
||||||
|
$env:SMB_PASSWORD = [string]$Smb.password
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Test-ApiUrl {
|
||||||
|
param([string]$Url)
|
||||||
|
if (-not $Url) { return $true }
|
||||||
|
try {
|
||||||
|
$resp = Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec 20
|
||||||
|
Write-Host " OK $Url ($($resp.StatusCode))" -ForegroundColor DarkGreen
|
||||||
|
return $true
|
||||||
|
} catch {
|
||||||
|
Write-Host " FAIL $Url : $($_.Exception.Message)" -ForegroundColor Red
|
||||||
|
return $false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$configPath = Resolve-ConfigPath
|
||||||
|
$raw = Get-Content -LiteralPath $configPath -Raw | ConvertFrom-Json
|
||||||
|
$steps = Get-HarmonizeSteps -Raw $raw
|
||||||
|
$harmonize = if ($raw.harmonize) { $raw.harmonize } else { $raw }
|
||||||
|
|
||||||
|
if ($DryRun -or ($harmonize.dryRun -eq $true)) {
|
||||||
|
$DryRun = $true
|
||||||
|
$env:CONFIRM_PROD = '1'
|
||||||
|
}
|
||||||
|
|
||||||
|
Set-SmbCredentials -Smb $raw.smb
|
||||||
|
if ($harmonize.smb) { Set-SmbCredentials -Smb $harmonize.smb }
|
||||||
|
|
||||||
|
$preferArg = @()
|
||||||
|
if ($harmonize.prefer) {
|
||||||
|
$preferArg = @("--prefer=$($harmonize.prefer)")
|
||||||
|
}
|
||||||
|
|
||||||
|
$dryRunArgs = @()
|
||||||
|
if ($DryRun) { $dryRunArgs = @('--dry-run') }
|
||||||
|
|
||||||
|
Write-Host "Harmonize config: $configPath"
|
||||||
|
Write-Host ("Steps enabled: " + (($steps.GetEnumerator() | Where-Object { $_.Value } | ForEach-Object { $_.Key }) -join ', '))
|
||||||
|
|
||||||
|
if (-not $DryRun) {
|
||||||
|
$env:CONFIRM_PROD = '1'
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($steps.backupDev) {
|
||||||
|
if (-not (Invoke-Step 'backupDev' { Invoke-Npm 'dev:db:backup' })) {
|
||||||
|
Write-HarmonizeBanner -Success $false -FailedAt 'backupDev'
|
||||||
|
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at backupDev.'
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($steps.backupProd) {
|
||||||
|
if (-not (Invoke-Step 'backupProd' { Invoke-Npm 'prod:db:backup' })) {
|
||||||
|
Write-HarmonizeBanner -Success $false -FailedAt 'backupProd'
|
||||||
|
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at backupProd.'
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($steps.schema) {
|
||||||
|
if (-not (Invoke-Step 'schema' { Invoke-Npm 'harmonize:schema' })) {
|
||||||
|
Write-HarmonizeBanner -Success $false -FailedAt 'schema'
|
||||||
|
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at schema migrate.'
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($steps.db) {
|
||||||
|
if (-not (Invoke-Step 'db' { Invoke-Npm 'harmonize:db' @dryRunArgs @preferArg })) {
|
||||||
|
Write-HarmonizeBanner -Success $false -FailedAt 'db'
|
||||||
|
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at DB harmonize.'
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($steps.images) {
|
||||||
|
if (-not (Invoke-Step 'images' { Invoke-Npm 'harmonize:images' @dryRunArgs })) {
|
||||||
|
Write-HarmonizeBanner -Success $false -FailedAt 'images'
|
||||||
|
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at image harmonize.'
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($steps.verify -and -not $DryRun) {
|
||||||
|
$verify = $harmonize.verify
|
||||||
|
if (-not $verify) { $verify = $raw.verify }
|
||||||
|
$ok = $true
|
||||||
|
if (-not (Invoke-Step 'verify' {
|
||||||
|
if ($verify.devUrl -and -not (Test-ApiUrl $verify.devUrl)) { return 1 }
|
||||||
|
if ($verify.publicUrl -and -not (Test-ApiUrl $verify.publicUrl)) { return 1 }
|
||||||
|
return 0
|
||||||
|
})) {
|
||||||
|
Write-HarmonizeBanner -Success $false -FailedAt 'verify'
|
||||||
|
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Verify step failed.'
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-HarmonizeBanner -Success $true
|
||||||
|
$summary = if ($DryRun) { 'Harmonize dry-run complete (no changes written).' } else { 'Harmonize complete.' }
|
||||||
|
Write-DeployCliResult -Script 'harmonize' -Success $true -Summary $summary -Details @("Config: $configPath")
|
||||||
|
exit 0
|
||||||
@@ -29,6 +29,8 @@
|
|||||||
"dev:migrate:painting-annotations": "node scripts/migrate-painting-annotations.js",
|
"dev:migrate:painting-annotations": "node scripts/migrate-painting-annotations.js",
|
||||||
"dev:migrate:artist-palette": "node scripts/migrate-artist-palette.js",
|
"dev:migrate:artist-palette": "node scripts/migrate-artist-palette.js",
|
||||||
"dev:migrate:search": "node scripts/migrate-search.js",
|
"dev:migrate:search": "node scripts/migrate-search.js",
|
||||||
|
"dev:migrate:sync-timestamps": "node scripts/migrate-sync-timestamps.js",
|
||||||
|
"dev:backfill-updated-at": "node scripts/backfill-updated-at.js",
|
||||||
"dev:import-painter-palette": "node scripts/import-painter-palette.js",
|
"dev:import-painter-palette": "node scripts/import-painter-palette.js",
|
||||||
"dev:analyze-painter-palette": "node scripts/analyze-painter-palette.js",
|
"dev:analyze-painter-palette": "node scripts/analyze-painter-palette.js",
|
||||||
"dev:export-paintings": "node scripts/export-paintings-csv.js",
|
"dev:export-paintings": "node scripts/export-paintings-csv.js",
|
||||||
@@ -48,6 +50,12 @@
|
|||||||
"devtoprod:release": "powershell -NoProfile -ExecutionPolicy Bypass -File infra/scripts/deploy-dev-to-prod.ps1",
|
"devtoprod:release": "powershell -NoProfile -ExecutionPolicy Bypass -File infra/scripts/deploy-dev-to-prod.ps1",
|
||||||
"prodto:dev:db": "node scripts/sync-prod-to-dev.js",
|
"prodto:dev:db": "node scripts/sync-prod-to-dev.js",
|
||||||
"prodto:dev:images": "powershell -NoProfile -ExecutionPolicy Bypass -File infra/scripts/sync-images-from-prod.ps1",
|
"prodto:dev:images": "powershell -NoProfile -ExecutionPolicy Bypass -File infra/scripts/sync-images-from-prod.ps1",
|
||||||
|
"harmonize:schema": "node scripts/harmonize-schema.js",
|
||||||
|
"harmonize:db": "node scripts/harmonize-db.js",
|
||||||
|
"harmonize:images": "node scripts/harmonize-images.js",
|
||||||
|
"harmonize:backfill-updated-at": "node scripts/backfill-updated-at.js --prod",
|
||||||
|
"harmonize:dry-run": "node scripts/harmonize-db.js --dry-run && node scripts/harmonize-images.js --dry-run",
|
||||||
|
"harmonize": "powershell -NoProfile -ExecutionPolicy Bypass -File infra/scripts/harmonize.ps1",
|
||||||
"infra:db:split-dev-prod": "node scripts/split-dev-prod-databases.js"
|
"infra:db:split-dev-prod": "node scripts/split-dev-prod-databases.js"
|
||||||
},
|
},
|
||||||
"keywords": [],
|
"keywords": [],
|
||||||
|
|||||||
@@ -0,0 +1,151 @@
|
|||||||
|
/**
|
||||||
|
* Backfill catalog updated_at from linked image file mtimes where possible.
|
||||||
|
*
|
||||||
|
* Dev: npm run dev:backfill-updated-at
|
||||||
|
* Prod: npm run harmonize:backfill-updated-at
|
||||||
|
*/
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const pg = require('pg');
|
||||||
|
const {
|
||||||
|
assertDevDatabase,
|
||||||
|
assertProdDatabase,
|
||||||
|
loadDevPgConfig,
|
||||||
|
loadProdPgConfig,
|
||||||
|
rootDir,
|
||||||
|
} = require('./db-env');
|
||||||
|
const { printCliResult } = require('./lib/cli-result');
|
||||||
|
|
||||||
|
const { Client } = pg;
|
||||||
|
|
||||||
|
function parseArgs(argv) {
|
||||||
|
return { prod: argv.includes('--prod') };
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveImageDir(prod) {
|
||||||
|
if (prod) {
|
||||||
|
const envPath = path.join(rootDir, 'infra', 'docker', '.env.prod');
|
||||||
|
if (fs.existsSync(envPath)) {
|
||||||
|
for (const line of fs.readFileSync(envPath, 'utf8').split(/\r?\n/)) {
|
||||||
|
const trimmed = line.trim();
|
||||||
|
if (trimmed.startsWith('IMAGE_DIR=')) {
|
||||||
|
const val = trimmed.slice('IMAGE_DIR='.length).trim().replace(/^["']|["']$/g, '');
|
||||||
|
if (val) return path.resolve(val);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return path.resolve('\\\\192.168.10.122\\Gallery\\data\\images');
|
||||||
|
}
|
||||||
|
require('dotenv').config({ path: path.join(rootDir, '.env') });
|
||||||
|
return path.resolve(process.env.IMAGE_DIR || path.join(rootDir, 'data', 'images'));
|
||||||
|
}
|
||||||
|
|
||||||
|
function fileMtimeMs(imageDir, relPath) {
|
||||||
|
if (!relPath || typeof relPath !== 'string') return null;
|
||||||
|
const abs = path.join(imageDir, relPath.replace(/^\//, ''));
|
||||||
|
try {
|
||||||
|
if (!fs.existsSync(abs)) return null;
|
||||||
|
return fs.statSync(abs).mtimeMs;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function maxMtime(imageDir, paths) {
|
||||||
|
let max = null;
|
||||||
|
for (const rel of paths) {
|
||||||
|
const ms = fileMtimeMs(imageDir, rel);
|
||||||
|
if (ms != null && (max == null || ms > max)) max = ms;
|
||||||
|
}
|
||||||
|
return max;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const { prod } = parseArgs(process.argv.slice(2));
|
||||||
|
const config = prod ? loadProdPgConfig() : loadDevPgConfig();
|
||||||
|
if (prod) assertProdDatabase(config.database);
|
||||||
|
else assertDevDatabase(config.database);
|
||||||
|
|
||||||
|
const imageDir = resolveImageDir(prod);
|
||||||
|
const client = new Client(config);
|
||||||
|
await client.connect();
|
||||||
|
|
||||||
|
const fallback = new Date();
|
||||||
|
let updated = 0;
|
||||||
|
|
||||||
|
const { rows: artists } = await client.query(
|
||||||
|
'SELECT id, portrait_path, portrait_thumb_path FROM artists',
|
||||||
|
);
|
||||||
|
for (const row of artists) {
|
||||||
|
const ms = maxMtime(imageDir, [row.portrait_path, row.portrait_thumb_path]);
|
||||||
|
const ts = ms != null ? new Date(ms) : fallback;
|
||||||
|
await client.query('UPDATE artists SET updated_at = $1 WHERE id = $2', [ts, row.id]);
|
||||||
|
updated += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { rows: paintings } = await client.query(
|
||||||
|
'SELECT id, image_path, thumbnail_path FROM paintings',
|
||||||
|
);
|
||||||
|
for (const row of paintings) {
|
||||||
|
const ms = maxMtime(imageDir, [row.image_path, row.thumbnail_path]);
|
||||||
|
const ts = ms != null ? new Date(ms) : fallback;
|
||||||
|
await client.query('UPDATE paintings SET updated_at = $1 WHERE id = $2', [ts, row.id]);
|
||||||
|
updated += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const childUpdates = [
|
||||||
|
['artist_periods', 'artist_id', 'artists'],
|
||||||
|
['painting_influences', 'painting_id', 'paintings'],
|
||||||
|
['painting_influence_sources', 'painting_id', 'paintings'],
|
||||||
|
['painting_annotations', 'painting_id', 'paintings'],
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const [child, fk, parent] of childUpdates) {
|
||||||
|
const res = await client.query(
|
||||||
|
`UPDATE ${child} c
|
||||||
|
SET updated_at = GREATEST(c.updated_at, p.updated_at)
|
||||||
|
FROM ${parent} p
|
||||||
|
WHERE c.${fk} = p.id`,
|
||||||
|
);
|
||||||
|
updated += res.rowCount || 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const table of ['historical_eras', 'art_movements']) {
|
||||||
|
const res = await client.query(
|
||||||
|
`UPDATE ${table} SET updated_at = $1 WHERE updated_at IS NOT NULL`,
|
||||||
|
[fallback],
|
||||||
|
);
|
||||||
|
updated += res.rowCount || 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
await client.query(
|
||||||
|
`UPDATE art_movements m
|
||||||
|
SET updated_at = GREATEST(m.updated_at, e.updated_at)
|
||||||
|
FROM historical_eras e
|
||||||
|
WHERE m.era_id = e.id`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await client.query(
|
||||||
|
`UPDATE artists a
|
||||||
|
SET updated_at = GREATEST(a.updated_at, m.updated_at)
|
||||||
|
FROM art_movements m
|
||||||
|
WHERE a.movement_id = m.id`,
|
||||||
|
);
|
||||||
|
|
||||||
|
await client.end();
|
||||||
|
|
||||||
|
printCliResult({
|
||||||
|
script: 'backfill-updated-at',
|
||||||
|
ok: true,
|
||||||
|
summary: `Backfilled updated_at on ${config.database} (${updated} row touches).`,
|
||||||
|
details: [`Image dir: ${imageDir}`],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
printCliResult({
|
||||||
|
script: 'backfill-updated-at',
|
||||||
|
ok: false,
|
||||||
|
summary: err.message,
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,304 @@
|
|||||||
|
/**
|
||||||
|
* Bidirectional catalog DB harmonize (dev <-> prod) by updated_at.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* npm run harmonize:db
|
||||||
|
* npm run harmonize:db -- --dry-run
|
||||||
|
* npm run harmonize:db -- --prefer=dev
|
||||||
|
*/
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const crypto = require('crypto');
|
||||||
|
const pg = require('pg');
|
||||||
|
const {
|
||||||
|
loadDevPgConfig,
|
||||||
|
loadProdPgConfig,
|
||||||
|
confirmProdAction,
|
||||||
|
rootDir,
|
||||||
|
} = require('./db-env');
|
||||||
|
const { printCliResult } = require('./lib/cli-result');
|
||||||
|
|
||||||
|
const { Client } = pg;
|
||||||
|
|
||||||
|
const CATALOG_TABLES = [
|
||||||
|
'historical_eras',
|
||||||
|
'art_movements',
|
||||||
|
'artists',
|
||||||
|
'artist_periods',
|
||||||
|
'paintings',
|
||||||
|
'painting_influences',
|
||||||
|
'painting_influence_sources',
|
||||||
|
'painting_annotations',
|
||||||
|
];
|
||||||
|
|
||||||
|
const NATURAL_KEY_FN = {
|
||||||
|
historical_eras: (r) => String(r.name || '').trim().toLowerCase(),
|
||||||
|
art_movements: (r) => String(r.name || '').trim().toLowerCase(),
|
||||||
|
artists: (r) => String(r.name || '').trim().toLowerCase(),
|
||||||
|
artist_periods: (r) => `${r.artist_id}:${String(r.name || '').trim().toLowerCase()}`,
|
||||||
|
paintings: (r) => `${r.artist_id}:${String(r.title || '').trim().toLowerCase()}`,
|
||||||
|
painting_influences: (r) => `${r.painting_id}:${r.influenced_by_painting_id}`,
|
||||||
|
painting_influence_sources: (r) => `${r.painting_id}:${r.source_type}:${r.source_painting_id || 0}:${r.source_artist_id || 0}:${r.source_movement_id || 0}`,
|
||||||
|
painting_annotations: (r) => `${r.painting_id}:${String(r.label || '').trim().toLowerCase()}:${r.sort_order}`,
|
||||||
|
};
|
||||||
|
|
||||||
|
function parseArgs(argv) {
|
||||||
|
const tablesArg = argv.find((a) => a.startsWith('--tables='));
|
||||||
|
const preferArg = argv.find((a) => a.startsWith('--prefer='));
|
||||||
|
return {
|
||||||
|
dryRun: argv.includes('--dry-run'),
|
||||||
|
verbose: argv.includes('--verbose'),
|
||||||
|
tables: tablesArg ? tablesArg.slice('--tables='.length).split(',').map((t) => t.trim()).filter(Boolean) : null,
|
||||||
|
prefer: preferArg ? preferArg.slice('--prefer='.length).trim().toLowerCase() : null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function stableRowHash(row, columns) {
|
||||||
|
const payload = {};
|
||||||
|
for (const col of columns) {
|
||||||
|
const val = row[col];
|
||||||
|
if (val instanceof Date) payload[col] = val.toISOString();
|
||||||
|
else if (val != null && typeof val === 'object') payload[col] = JSON.stringify(val);
|
||||||
|
else payload[col] = val;
|
||||||
|
}
|
||||||
|
return crypto.createHash('sha256').update(JSON.stringify(payload)).digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
function compareUpdatedAt(a, b) {
|
||||||
|
const ta = a?.updated_at ? new Date(a.updated_at).getTime() : 0;
|
||||||
|
const tb = b?.updated_at ? new Date(b.updated_at).getTime() : 0;
|
||||||
|
if (ta === tb) return 0;
|
||||||
|
return ta > tb ? 1 : -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getTableColumns(client, tableName) {
|
||||||
|
const { rows } = await client.query(
|
||||||
|
`SELECT column_name
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = 'public' AND table_name = $1
|
||||||
|
ORDER BY ordinal_position`,
|
||||||
|
[tableName],
|
||||||
|
);
|
||||||
|
return rows.map((r) => r.column_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchTableRows(client, tableName) {
|
||||||
|
const { rows } = await client.query(`SELECT * FROM ${tableName} ORDER BY id`);
|
||||||
|
const map = new Map();
|
||||||
|
for (const row of rows) map.set(row.id, row);
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildInsertSql(tableName, columns) {
|
||||||
|
const cols = columns.map((c) => `"${c}"`).join(', ');
|
||||||
|
const placeholders = columns.map((_, i) => `$${i + 1}`).join(', ');
|
||||||
|
return `INSERT INTO ${tableName} (${cols}) OVERRIDING SYSTEM VALUE VALUES (${placeholders})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildUpsertSql(tableName, columns) {
|
||||||
|
const insertCols = columns.map((c) => `"${c}"`).join(', ');
|
||||||
|
const placeholders = columns.map((_, i) => `$${i + 1}`).join(', ');
|
||||||
|
const updates = columns
|
||||||
|
.filter((c) => c !== 'id')
|
||||||
|
.map((c) => `"${c}" = EXCLUDED."${c}"`)
|
||||||
|
.join(', ');
|
||||||
|
return `INSERT INTO ${tableName} (${insertCols}) OVERRIDING SYSTEM VALUE VALUES (${placeholders})
|
||||||
|
ON CONFLICT (id) DO UPDATE SET ${updates}
|
||||||
|
WHERE ${tableName}.updated_at < EXCLUDED.updated_at`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rowValues(row, columns) {
|
||||||
|
return columns.map((col) => row[col]);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function insertRow(client, tableName, columns, row, dryRun) {
|
||||||
|
if (dryRun) return;
|
||||||
|
const sql = buildInsertSql(tableName, columns);
|
||||||
|
await client.query(sql, rowValues(row, columns));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function upsertRow(client, tableName, columns, row, dryRun) {
|
||||||
|
if (dryRun) return;
|
||||||
|
const sql = buildUpsertSql(tableName, columns);
|
||||||
|
await client.query(sql, rowValues(row, columns));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fixSequence(client, tableName) {
|
||||||
|
await client.query(
|
||||||
|
`SELECT setval(pg_get_serial_sequence($1, 'id'), COALESCE((SELECT MAX(id) FROM ${tableName}), 1), true)`,
|
||||||
|
[tableName],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function harmonizeTable(tableName, devClient, prodClient, options, stats) {
|
||||||
|
const naturalKeyFn = NATURAL_KEY_FN[tableName];
|
||||||
|
const columns = await getTableColumns(devClient, tableName);
|
||||||
|
if (!columns.includes('updated_at')) {
|
||||||
|
throw new Error(`Table ${tableName} missing updated_at — run migrate-sync-timestamps first`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const devRows = await fetchTableRows(devClient, tableName);
|
||||||
|
const prodRows = await fetchTableRows(prodClient, tableName);
|
||||||
|
const allIds = new Set([...devRows.keys(), ...prodRows.keys()]);
|
||||||
|
|
||||||
|
for (const id of allIds) {
|
||||||
|
const devRow = devRows.get(id);
|
||||||
|
const prodRow = prodRows.get(id);
|
||||||
|
|
||||||
|
if (devRow && !prodRow) {
|
||||||
|
stats.devToProd += 1;
|
||||||
|
stats.actions.push({ table: tableName, id, action: 'dev→prod insert' });
|
||||||
|
await insertRow(prodClient, tableName, columns, devRow, options.dryRun);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (prodRow && !devRow) {
|
||||||
|
stats.prodToDev += 1;
|
||||||
|
stats.actions.push({ table: tableName, id, action: 'prod→dev insert' });
|
||||||
|
await insertRow(devClient, tableName, columns, prodRow, options.dryRun);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const devKey = naturalKeyFn(devRow);
|
||||||
|
const prodKey = naturalKeyFn(prodRow);
|
||||||
|
if (devKey !== prodKey) {
|
||||||
|
stats.conflicts += 1;
|
||||||
|
stats.conflictDetails.push({
|
||||||
|
table: tableName,
|
||||||
|
id,
|
||||||
|
reason: 'id_collision',
|
||||||
|
devKey,
|
||||||
|
prodKey,
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const devHash = stableRowHash(devRow, columns);
|
||||||
|
const prodHash = stableRowHash(prodRow, columns);
|
||||||
|
if (devHash === prodHash) {
|
||||||
|
stats.skipped += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cmp = compareUpdatedAt(devRow, prodRow);
|
||||||
|
if (cmp > 0) {
|
||||||
|
stats.devToProd += 1;
|
||||||
|
stats.actions.push({ table: tableName, id, action: 'dev→prod update' });
|
||||||
|
await upsertRow(prodClient, tableName, columns, devRow, options.dryRun);
|
||||||
|
} else if (cmp < 0) {
|
||||||
|
stats.prodToDev += 1;
|
||||||
|
stats.actions.push({ table: tableName, id, action: 'prod→dev update' });
|
||||||
|
await upsertRow(devClient, tableName, columns, prodRow, options.dryRun);
|
||||||
|
} else if (options.prefer === 'dev') {
|
||||||
|
stats.devToProd += 1;
|
||||||
|
stats.actions.push({ table: tableName, id, action: 'dev→prod update (prefer tie)' });
|
||||||
|
await upsertRow(prodClient, tableName, columns, devRow, options.dryRun);
|
||||||
|
} else if (options.prefer === 'prod') {
|
||||||
|
stats.prodToDev += 1;
|
||||||
|
stats.actions.push({ table: tableName, id, action: 'prod→dev update (prefer tie)' });
|
||||||
|
await upsertRow(devClient, tableName, columns, prodRow, options.dryRun);
|
||||||
|
} else {
|
||||||
|
stats.conflicts += 1;
|
||||||
|
stats.conflictDetails.push({
|
||||||
|
table: tableName,
|
||||||
|
id,
|
||||||
|
reason: 'equal_updated_at',
|
||||||
|
devUpdatedAt: devRow.updated_at,
|
||||||
|
prodUpdatedAt: prodRow.updated_at,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!options.dryRun) {
|
||||||
|
await fixSequence(devClient, tableName);
|
||||||
|
await fixSequence(prodClient, tableName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function timestampSlug(date = new Date()) {
|
||||||
|
const pad = (n) => String(n).padStart(2, '0');
|
||||||
|
return [
|
||||||
|
date.getFullYear(),
|
||||||
|
pad(date.getMonth() + 1),
|
||||||
|
pad(date.getDate()),
|
||||||
|
'_',
|
||||||
|
pad(date.getHours()),
|
||||||
|
pad(date.getMinutes()),
|
||||||
|
pad(date.getSeconds()),
|
||||||
|
].join('');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const options = parseArgs(process.argv.slice(2));
|
||||||
|
const tables = options.tables || CATALOG_TABLES;
|
||||||
|
|
||||||
|
for (const table of tables) {
|
||||||
|
if (!CATALOG_TABLES.includes(table)) {
|
||||||
|
throw new Error(`Unknown catalog table: ${table}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!options.dryRun && process.env.CONFIRM_PROD !== '1') {
|
||||||
|
await confirmProdAction(
|
||||||
|
'Harmonize will merge catalog rows between gallery_dev and gallery_prod (prod may be modified).',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const devClient = new Client(loadDevPgConfig());
|
||||||
|
const prodClient = new Client(loadProdPgConfig());
|
||||||
|
await devClient.connect();
|
||||||
|
await prodClient.connect();
|
||||||
|
|
||||||
|
const stats = {
|
||||||
|
devToProd: 0,
|
||||||
|
prodToDev: 0,
|
||||||
|
skipped: 0,
|
||||||
|
conflicts: 0,
|
||||||
|
actions: [],
|
||||||
|
conflictDetails: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (const table of tables) {
|
||||||
|
if (options.verbose) console.log(`Harmonizing ${table}…`);
|
||||||
|
await harmonizeTable(table, devClient, prodClient, options, stats);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await devClient.end();
|
||||||
|
await prodClient.end();
|
||||||
|
}
|
||||||
|
|
||||||
|
const reportDir = path.join(rootDir, 'db', 'SyncReports');
|
||||||
|
fs.mkdirSync(reportDir, { recursive: true });
|
||||||
|
const reportPath = path.join(reportDir, `harmonize_db_${timestampSlug()}.json`);
|
||||||
|
fs.writeFileSync(reportPath, JSON.stringify({
|
||||||
|
generatedAt: new Date().toISOString(),
|
||||||
|
dryRun: options.dryRun,
|
||||||
|
prefer: options.prefer,
|
||||||
|
tables,
|
||||||
|
stats: {
|
||||||
|
devToProd: stats.devToProd,
|
||||||
|
prodToDev: stats.prodToDev,
|
||||||
|
skipped: stats.skipped,
|
||||||
|
conflicts: stats.conflicts,
|
||||||
|
},
|
||||||
|
conflicts: stats.conflictDetails,
|
||||||
|
actions: options.verbose ? stats.actions : stats.actions.slice(0, 200),
|
||||||
|
}, null, 2));
|
||||||
|
|
||||||
|
const mode = options.dryRun ? ' (dry-run)' : '';
|
||||||
|
printCliResult({
|
||||||
|
script: 'harmonize-db',
|
||||||
|
ok: true,
|
||||||
|
summary: `DB harmonize complete${mode}: dev→prod ${stats.devToProd}, prod→dev ${stats.prodToDev}, skipped ${stats.skipped}, conflicts ${stats.conflicts}.`,
|
||||||
|
details: [`Report: ${reportPath}`],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
printCliResult({
|
||||||
|
script: 'harmonize-db',
|
||||||
|
ok: false,
|
||||||
|
summary: err.message,
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
/**
|
||||||
|
* Bidirectional image harmonize (dev <-> prod) by file mtime.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* npm run harmonize:images
|
||||||
|
* npm run harmonize:images -- --dry-run
|
||||||
|
*/
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const { rootDir, confirmProdAction } = require('./db-env');
|
||||||
|
const { loadHarmonizeConfig, prodImagesPath } = require('./lib/harmonize-config');
|
||||||
|
const { printCliResult } = require('./lib/cli-result');
|
||||||
|
|
||||||
|
const IMAGE_SUBDIRS = ['portraits', 'paintings'];
|
||||||
|
const MTIME_TOLERANCE_MS = 1000;
|
||||||
|
|
||||||
|
function parseArgs(argv) {
|
||||||
|
const configArg = argv.find((a) => a.startsWith('--config='));
|
||||||
|
return {
|
||||||
|
dryRun: argv.includes('--dry-run'),
|
||||||
|
verbose: argv.includes('--verbose'),
|
||||||
|
configPath: configArg ? configArg.slice('--config='.length) : null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveDevImageDir() {
|
||||||
|
require('dotenv').config({ path: path.join(rootDir, '.env') });
|
||||||
|
return path.resolve(process.env.IMAGE_DIR || path.join(rootDir, 'data', 'images'));
|
||||||
|
}
|
||||||
|
|
||||||
|
function walkImageFiles(rootDir, baseRel = '') {
|
||||||
|
const files = [];
|
||||||
|
const abs = path.join(rootDir, baseRel);
|
||||||
|
if (!fs.existsSync(abs)) return files;
|
||||||
|
|
||||||
|
for (const entry of fs.readdirSync(abs, { withFileTypes: true })) {
|
||||||
|
const rel = baseRel ? path.join(baseRel, entry.name).replace(/\\/g, '/') : entry.name;
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
files.push(...walkImageFiles(rootDir, rel));
|
||||||
|
} else if (entry.isFile()) {
|
||||||
|
files.push(rel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return files;
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectRelativePaths(imageRoot) {
|
||||||
|
const relPaths = new Set();
|
||||||
|
for (const sub of IMAGE_SUBDIRS) {
|
||||||
|
const subRoot = path.join(imageRoot, sub);
|
||||||
|
if (!fs.existsSync(subRoot)) continue;
|
||||||
|
for (const rel of walkImageFiles(subRoot, sub)) {
|
||||||
|
relPaths.add(rel.replace(/\\/g, '/'));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return relPaths;
|
||||||
|
}
|
||||||
|
|
||||||
|
function statFile(imageRoot, relPath) {
|
||||||
|
const abs = path.join(imageRoot, relPath);
|
||||||
|
try {
|
||||||
|
if (!fs.existsSync(abs)) return null;
|
||||||
|
const st = fs.statSync(abs);
|
||||||
|
if (!st.isFile()) return null;
|
||||||
|
return { size: st.size, mtimeMs: st.mtimeMs };
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensureDirForFile(filePath) {
|
||||||
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
function copyFile(src, dest, dryRun) {
|
||||||
|
if (dryRun) return;
|
||||||
|
ensureDirForFile(dest);
|
||||||
|
fs.copyFileSync(src, dest);
|
||||||
|
}
|
||||||
|
|
||||||
|
function timestampSlug(date = new Date()) {
|
||||||
|
const pad = (n) => String(n).padStart(2, '0');
|
||||||
|
return [
|
||||||
|
date.getFullYear(),
|
||||||
|
pad(date.getMonth() + 1),
|
||||||
|
pad(date.getDate()),
|
||||||
|
'_',
|
||||||
|
pad(date.getHours()),
|
||||||
|
pad(date.getMinutes()),
|
||||||
|
pad(date.getSeconds()),
|
||||||
|
].join('');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const options = parseArgs(process.argv.slice(2));
|
||||||
|
const config = loadHarmonizeConfig(options.configPath);
|
||||||
|
const dryRun = options.dryRun || config.dryRun;
|
||||||
|
|
||||||
|
const devDir = resolveDevImageDir();
|
||||||
|
const prodDir = prodImagesPath(config);
|
||||||
|
|
||||||
|
if (!fs.existsSync(prodDir)) {
|
||||||
|
throw new Error(
|
||||||
|
`Prod image path not reachable: ${prodDir}. Map SMB share or set paths.prodImages in harmonize config.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!dryRun && process.env.CONFIRM_PROD !== '1') {
|
||||||
|
await confirmProdAction(
|
||||||
|
'Harmonize will copy image files between dev and prod (prod files may be overwritten).',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const devPaths = collectRelativePaths(devDir);
|
||||||
|
const prodPaths = collectRelativePaths(prodDir);
|
||||||
|
const allPaths = new Set([...devPaths, ...prodPaths]);
|
||||||
|
|
||||||
|
const stats = {
|
||||||
|
devToProd: 0,
|
||||||
|
prodToDev: 0,
|
||||||
|
skipped: 0,
|
||||||
|
actions: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
for (const rel of allPaths) {
|
||||||
|
const devStat = statFile(devDir, rel);
|
||||||
|
const prodStat = statFile(prodDir, rel);
|
||||||
|
|
||||||
|
if (devStat && !prodStat) {
|
||||||
|
stats.devToProd += 1;
|
||||||
|
stats.actions.push({ path: rel, action: 'dev→prod copy' });
|
||||||
|
copyFile(path.join(devDir, rel), path.join(prodDir, rel), dryRun);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (prodStat && !devStat) {
|
||||||
|
stats.prodToDev += 1;
|
||||||
|
stats.actions.push({ path: rel, action: 'prod→dev copy' });
|
||||||
|
copyFile(path.join(prodDir, rel), path.join(devDir, rel), dryRun);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!devStat || !prodStat) continue;
|
||||||
|
|
||||||
|
const sameSize = devStat.size === prodStat.size;
|
||||||
|
const mtimeClose = Math.abs(devStat.mtimeMs - prodStat.mtimeMs) <= MTIME_TOLERANCE_MS;
|
||||||
|
if (sameSize && mtimeClose) {
|
||||||
|
stats.skipped += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (devStat.mtimeMs > prodStat.mtimeMs) {
|
||||||
|
stats.devToProd += 1;
|
||||||
|
stats.actions.push({ path: rel, action: 'dev→prod update' });
|
||||||
|
copyFile(path.join(devDir, rel), path.join(prodDir, rel), dryRun);
|
||||||
|
} else if (prodStat.mtimeMs > devStat.mtimeMs) {
|
||||||
|
stats.prodToDev += 1;
|
||||||
|
stats.actions.push({ path: rel, action: 'prod→dev update' });
|
||||||
|
copyFile(path.join(prodDir, rel), path.join(devDir, rel), dryRun);
|
||||||
|
} else {
|
||||||
|
stats.skipped += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const reportDir = path.join(rootDir, 'db', 'SyncReports');
|
||||||
|
fs.mkdirSync(reportDir, { recursive: true });
|
||||||
|
const reportPath = path.join(reportDir, `harmonize_images_${timestampSlug()}.json`);
|
||||||
|
fs.writeFileSync(reportPath, JSON.stringify({
|
||||||
|
generatedAt: new Date().toISOString(),
|
||||||
|
dryRun,
|
||||||
|
devDir,
|
||||||
|
prodDir,
|
||||||
|
stats: {
|
||||||
|
devToProd: stats.devToProd,
|
||||||
|
prodToDev: stats.prodToDev,
|
||||||
|
skipped: stats.skipped,
|
||||||
|
},
|
||||||
|
actions: options.verbose ? stats.actions : stats.actions.slice(0, 500),
|
||||||
|
}, null, 2));
|
||||||
|
|
||||||
|
const mode = dryRun ? ' (dry-run)' : '';
|
||||||
|
printCliResult({
|
||||||
|
script: 'harmonize-images',
|
||||||
|
ok: true,
|
||||||
|
summary: `Image harmonize complete${mode}: dev→prod ${stats.devToProd}, prod→dev ${stats.prodToDev}, skipped ${stats.skipped}.`,
|
||||||
|
details: [`Dev: ${devDir}`, `Prod: ${prodDir}`, `Report: ${reportPath}`],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
printCliResult({
|
||||||
|
script: 'harmonize-images',
|
||||||
|
ok: false,
|
||||||
|
summary: err.message,
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
/**
|
||||||
|
* Migrate prod schema from dev migrations (schema dev → prod only).
|
||||||
|
*
|
||||||
|
* Usage: npm run harmonize:schema
|
||||||
|
*/
|
||||||
|
const { spawnSync } = require('child_process');
|
||||||
|
const path = require('path');
|
||||||
|
const { rootDir, loadProdPgConfig, confirmProdAction } = require('./db-env');
|
||||||
|
const { printCliResult } = require('./lib/cli-result');
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
if (process.env.CONFIRM_PROD !== '1') {
|
||||||
|
await confirmProdAction('Apply dev migrations to gallery_prod schema?');
|
||||||
|
}
|
||||||
|
|
||||||
|
const prodConfig = loadProdPgConfig();
|
||||||
|
const env = {
|
||||||
|
...process.env,
|
||||||
|
DB_HOST: prodConfig.host,
|
||||||
|
DB_PORT: String(prodConfig.port),
|
||||||
|
DB_USER: prodConfig.user,
|
||||||
|
DB_PASSWORD: prodConfig.password,
|
||||||
|
DB_NAME: prodConfig.database,
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = spawnSync(process.execPath, [path.join(rootDir, 'server', 'migrate.js')], {
|
||||||
|
cwd: rootDir,
|
||||||
|
env,
|
||||||
|
stdio: 'inherit',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (result.status !== 0) {
|
||||||
|
printCliResult({
|
||||||
|
script: 'harmonize-schema',
|
||||||
|
ok: false,
|
||||||
|
summary: 'Prod schema migration failed.',
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
printCliResult({
|
||||||
|
script: 'harmonize-schema',
|
||||||
|
ok: true,
|
||||||
|
summary: 'Prod schema migrated from dev migration files (gallery_prod).',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
printCliResult({
|
||||||
|
script: 'harmonize-schema',
|
||||||
|
ok: false,
|
||||||
|
summary: err.message,
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const { rootDir } = require('./db-env');
|
||||||
|
|
||||||
|
const DEFAULT_CONFIG_PATH = path.join(rootDir, 'infra', 'deploy', 'harmonize.config.json');
|
||||||
|
const FALLBACK_CONFIG_PATH = path.join(rootDir, 'infra', 'deploy', 'devtoprod.config.json');
|
||||||
|
|
||||||
|
function loadHarmonizeConfig(configPath) {
|
||||||
|
const resolved = configPath
|
||||||
|
? path.resolve(configPath)
|
||||||
|
: (fs.existsSync(DEFAULT_CONFIG_PATH) ? DEFAULT_CONFIG_PATH : FALLBACK_CONFIG_PATH);
|
||||||
|
|
||||||
|
if (!fs.existsSync(resolved)) {
|
||||||
|
return {
|
||||||
|
configPath: resolved,
|
||||||
|
dryRun: false,
|
||||||
|
prefer: null,
|
||||||
|
schemaChanged: false,
|
||||||
|
steps: {
|
||||||
|
backupDev: true,
|
||||||
|
backupProd: true,
|
||||||
|
schema: false,
|
||||||
|
db: true,
|
||||||
|
images: true,
|
||||||
|
verify: false,
|
||||||
|
},
|
||||||
|
smb: {
|
||||||
|
host: '192.168.10.122',
|
||||||
|
share: 'Gallery',
|
||||||
|
user: '',
|
||||||
|
password: '',
|
||||||
|
},
|
||||||
|
paths: {
|
||||||
|
prodImages: '\\\\192.168.10.122\\Gallery\\data\\images',
|
||||||
|
},
|
||||||
|
verify: {},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const raw = JSON.parse(fs.readFileSync(resolved, 'utf8'));
|
||||||
|
const harmonize = raw.harmonize || raw;
|
||||||
|
return {
|
||||||
|
configPath: resolved,
|
||||||
|
dryRun: Boolean(harmonize.dryRun),
|
||||||
|
prefer: harmonize.prefer || null,
|
||||||
|
schemaChanged: Boolean(harmonize.schemaChanged ?? raw.schemaChanged),
|
||||||
|
steps: {
|
||||||
|
backupDev: harmonize.steps?.backupDev ?? true,
|
||||||
|
backupProd: harmonize.steps?.backupProd ?? true,
|
||||||
|
schema: harmonize.steps?.schema ?? Boolean(harmonize.schemaChanged ?? raw.schemaChanged),
|
||||||
|
db: harmonize.steps?.db ?? true,
|
||||||
|
images: harmonize.steps?.images ?? true,
|
||||||
|
verify: harmonize.steps?.verify ?? false,
|
||||||
|
},
|
||||||
|
smb: {
|
||||||
|
host: harmonize.smb?.host ?? raw.smb?.host ?? '192.168.10.122',
|
||||||
|
share: harmonize.smb?.share ?? raw.smb?.share ?? 'Gallery',
|
||||||
|
user: harmonize.smb?.user ?? raw.smb?.user ?? '',
|
||||||
|
password: harmonize.smb?.password ?? raw.smb?.password ?? '',
|
||||||
|
},
|
||||||
|
paths: {
|
||||||
|
prodImages: harmonize.paths?.prodImages
|
||||||
|
?? `\\\\${harmonize.smb?.host ?? raw.smb?.host ?? '192.168.10.122'}\\${harmonize.smb?.share ?? raw.smb?.share ?? 'Gallery'}\\data\\images`,
|
||||||
|
},
|
||||||
|
verify: harmonize.verify ?? raw.verify ?? {},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function prodImagesPath(config) {
|
||||||
|
return config.paths?.prodImages
|
||||||
|
|| `\\\\${config.smb.host}\\${config.smb.share}\\data\\images`;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
DEFAULT_CONFIG_PATH,
|
||||||
|
loadHarmonizeConfig,
|
||||||
|
prodImagesPath,
|
||||||
|
};
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
require('dotenv').config();
|
||||||
|
const fs = require('fs');
|
||||||
|
const path = require('path');
|
||||||
|
const pool = require('../server/db');
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const sqlPath = path.join(__dirname, '../db/migrate-sync-timestamps.sql');
|
||||||
|
const sql = fs.readFileSync(sqlPath, 'utf8');
|
||||||
|
await pool.query(sql);
|
||||||
|
console.log('Catalog sync timestamps ready (updated_at + triggers)');
|
||||||
|
await pool.end();
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
console.error(err);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
@@ -13,6 +13,7 @@ const INCREMENTAL_MIGRATIONS = [
|
|||||||
'migrate-portrait-thumbs.sql',
|
'migrate-portrait-thumbs.sql',
|
||||||
'migrate-perf-indexes.sql',
|
'migrate-perf-indexes.sql',
|
||||||
'migrate-search.sql',
|
'migrate-search.sql',
|
||||||
|
'migrate-sync-timestamps.sql',
|
||||||
];
|
];
|
||||||
|
|
||||||
async function bootstrapCurator() {
|
async function bootstrapCurator() {
|
||||||
|
|||||||
Reference in New Issue
Block a user