Add dev-prod harmonize tool for bidirectional catalog and image sync.

Schema migrates dev to prod only; catalog rows merge by updated_at and images by mtime with union merge and conflict reporting.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Danila Khodjaef
2026-07-15 11:02:17 +03:00
co-authored by Cursor
parent 81ebad2120
commit f247b418d8
21 changed files with 1336 additions and 4 deletions
+2
View File
@@ -6,6 +6,8 @@ client/node_modules/
.env.*.local
infra/docker/.env.prod
infra/deploy/devtoprod.config.json
infra/deploy/harmonize.config.json
db/SyncReports/
infra/deploy/last-docker-release.json
# DB backups (may contain data)
+6
View File
@@ -138,6 +138,10 @@ Run `npm run dev:migrate` against prod DB after first deploy with auth vars set
| `npm run prod:db:backup` | Prod backup (reads `infra/docker/.env.prod`) |
| `npm run dev:db:restore -- --file <path>` | Restore backup into **dev** (truncates tables first; prompts `yes`) |
| `npm run devtoprod:db:restore -- --file <path>` | Restore into **prod** (requires confirmation) |
| `npm run harmonize` | Bidirectional catalog DB + image merge by `updated_at` / file mtime — [harmonize-dev-prod.md](harmonize-dev-prod.md) |
| `npm run harmonize:schema` | Apply dev migrations to prod schema only (dev → prod) |
| `npm run harmonize:db` / `harmonize:images` | DB or image merge only |
| `npm run dev:backfill-updated-at` | Backfill catalog `updated_at` from image mtimes (dev) |
**One-time split (recommended):** pgAdmin on dev PC → open [`db/split-dev-prod-pgadmin.sql`](../db/split-dev-prod-pgadmin.sql) → run each STEP on database `postgres`, then verify on `gallery_dev`.
@@ -237,6 +241,8 @@ npm run devtoprod:release # full promote from infra/deploy/devtoprod.conf
npm run devtoprod:thumbnails # rebuild thumb files + DB paths on dev (before backup)
npm run devtoprod:images # dev repo → TrueNAS (promote / first deploy)
npm run prodto:dev:images # TrueNAS → dev repo
npm run harmonize # bidirectional merge (newer wins) — see harmonize-dev-prod.md
npm run harmonize:dry-run # preview DB + image changes only
```
Type `yes` when prompted (or set `autoConfirm: true` in release config). Robocopy exit codes **07** = success. Deploy scripts print a final **`===== SUCCESS =====`** or **`===== FAILED =====`** banner.
+1 -1
View File
@@ -3,7 +3,7 @@ this file contains draft for future releases and features
1. Multy language support, russian version at least, search for best implementation, preferably story in db and easily expandable. tool to check and correct translation
2. tool to manage links (influence/influenced by ) import csv's ( define format), edit ,add, delete, visualize, map to pictures/ entities
3. tool to monitor/manage (plan actions) of curator actions, markers to check painting/text ?
4. tool to sync prod /env resources (both ways), db structure, db data, images, users etc
4. ~~tool to sync prod /env resources (both ways), db structure, db data, images, users etc~~ — done for catalog DB + images: `npm run harmonize` (schema dev→prod only; users/audit excluded) — [harmonize-dev-prod.md](harmonize-dev-prod.md)
5. curator_audit_log should contain log of actions like fixit, checked, upload etc with details for which entity it was made and details what was the action and outcome
6. ~~create search by entity (painting, artist, movement)~~ — done: timeline header + `GET /api/search`
7. create guided tours (with text/extra infor, set of entities)
+1
View File
@@ -471,6 +471,7 @@ See [API.md](API.md#authentication) and [data-and-images.md](data-and-images.md#
|----------|----------|
| [setup.md](setup.md) | Install, database, npm scripts |
| [deploy-dev-to-prod.md](deploy-dev-to-prod.md) | Release runbook + one-command `devtoprod:release` |
| [harmonize-dev-prod.md](harmonize-dev-prod.md) | Incremental dev ↔ prod merge (catalog DB + images) |
| [DB_structure.md](DB_structure.md) | Tables and relationships |
| [API.md](API.md) | REST endpoints |
| [data-and-images.md](data-and-images.md) | Image pipeline and seeding |
+3 -1
View File
@@ -31,7 +31,7 @@ File names are sanitised `{Artist}_{Title}.{ext}`. The image service can redisco
| **Development** | `./data/images/` in repo | Working copy on dev PC |
| **Production** | `/mnt/BasePool/Applications/Gallery/data/images` on TrueNAS | SMB `\\192.168.10.122\Gallery\data\images` |
Promote dev → prod files: `npm run devtoprod:images` (after `net use \\192.168.10.122\Gallery`). Refresh dev from prod: `npm run prodto:dev:images`. See [environments.md](environments.md).
**One-direction promote:** `npm run devtoprod:images` (dev → prod, skip older). **Refresh dev from prod:** `npm run prodto:dev:images`. **Bidirectional merge** (newer file wins): `npm run harmonize:images` or full `npm run harmonize` — see [harmonize-dev-prod.md](harmonize-dev-prod.md). General sync reference: [environments.md](environments.md).
## Scripts overview
@@ -49,6 +49,8 @@ Promote dev → prod files: `npm run devtoprod:images` (after `net use \\192.168
| `fetch-missing-images.js` | `npm run dev:fetch-images` | Downloads files for paintings missing on disk |
| `image-fetcher.js` | *(library)* | Wikimedia / museum resolution used by fetch scripts and API |
| `sync-images-to-prod.ps1` / `sync-images-from-prod.ps1` | `npm run devtoprod:images` / `npm run prodto:dev:images` | Robocopy via SMB `\\192.168.10.122\Gallery` |
| `harmonize-db.js` / `harmonize-images.js` | `npm run harmonize:db` / `harmonize:images` | Bidirectional merge by `updated_at` / file mtime |
| `harmonize.ps1` | `npm run harmonize` | Orchestrator: backups, optional schema, DB + image merge |
| `regenerate-thumbnails.js` | `npm run dev:regenerate-thumbnails` | Rebuild painting thumbs from full images via `sharp` |
| `regenerate-portrait-thumbs.js` | `npm run dev:regenerate-portrait-thumbs` | Rebuild timeline portrait thumbs (~256px) and set `portrait_thumb_path` |
| `audit-painting-images.js` | `npm run dev:audit-painting-images` | Detect thumb/full aspect-ratio mismatches |
+5 -1
View File
@@ -2,7 +2,11 @@
Step-by-step guide for promoting the **development** version of Gallery to **production**. Covers code, database schema, database data, thumbnail generation, and image files.
> **Default is dev.** This runbook is for a **scheduled release** (~weekly, or when you explicitly decide to ship). Day-to-day work stays on dev — see [environments.md](environments.md#development-first-workflow-default). If the one-time prod install is not done yet, follow [environments.md → One-time setup](environments.md#one-time-setup-full-walkthrough) and [infra/docker/DEPLOY-truenas.md](../infra/docker/DEPLOY-truenas.md) first.
> **Default is dev.** This runbook is for a **scheduled release** (~weekly, or when you explicitly decide to ship). Day-to-day work stays on dev — see [environments.md](environments.md#development-first-workflow-default).
>
> **Mid-week merge:** If both dev and prod have catalog or image edits and you need **last-write-wins** sync instead of a full prod overwrite, use [harmonize-dev-prod.md](harmonize-dev-prod.md) (`npm run harmonize`). Keep this runbook for releases where prod should exactly match dev.
>
> If the one-time prod install is not done yet, follow [environments.md → One-time setup](environments.md#one-time-setup-full-walkthrough) and [infra/docker/DEPLOY-truenas.md](../infra/docker/DEPLOY-truenas.md) first.
| | Dev (source) | Prod (target) |
|---|---|---|
+7
View File
@@ -305,6 +305,7 @@ See also [Drunkmeyou gitea-https-keenetic-npm-setup.md](../../Drunkmeyou/Documen
| Fast local HMR (no Keenetic) | `npm run dev:server` + `npm run dev:client` |
| Refresh dev DB from prod | `npm run prodto:dev:db` |
| Pull prod images to dev | `npm run prodto:dev:images` |
| Merge dev ↔ prod catalog + images (incremental) | `npm run harmonize` — see [harmonize-dev-prod.md](harmonize-dev-prod.md) |
---
@@ -312,6 +313,8 @@ See also [Drunkmeyou gitea-https-keenetic-npm-setup.md](../../Drunkmeyou/Documen
Run this when you are ready to ship dev to production — **not** after every small change. Typical cadence: **about once a week**.
For **incremental** dev ↔ prod merge (both sides edited), use [harmonize-dev-prod.md](harmonize-dev-prod.md) instead of full restore.
**One command (recommended):** copy [`infra/deploy/devtoprod.config.example.json`](../infra/deploy/devtoprod.config.example.json) to `infra/deploy/devtoprod.config.json`, edit it, then `npm run devtoprod:release` or `deploy-dev-to-prod.cmd`. See [deploy-dev-to-prod.md → One-command release](deploy-dev-to-prod.md#one-command-release-automated).
**Manual steps:** detailed runbook (per-change decision matrix, schema migration, rollback): [deploy-dev-to-prod.md](deploy-dev-to-prod.md).
@@ -338,11 +341,15 @@ Run this when you are ready to ship dev to production — **not** after every sm
| `npm run prodto:dev:db` | Dev PC PowerShell | Clone prod → dev |
| `npm run dev:db:backup` | Dev PC PowerShell | Dev backup |
| `npm run devtoprod:db:restore` | Dev PC PowerShell | Restore into prod |
| `npm run harmonize` | Dev PC PowerShell | Bidirectional catalog + image merge — [harmonize-dev-prod.md](harmonize-dev-prod.md) |
| `npm run harmonize:db` | Dev PC PowerShell | DB merge only |
| `npm run harmonize:images` | Dev PC PowerShell | Image merge only |
## Image sync
| Command | Where | Direction |
|---------|-------|-----------|
| `npm run harmonize` | Dev PC PowerShell | Bidirectional merge (mtime newer wins) — [harmonize-dev-prod.md](harmonize-dev-prod.md) |
| `npm run devtoprod:release` | Dev PC PowerShell | Full config-driven promote (see [deploy-dev-to-prod.md](deploy-dev-to-prod.md#one-command-release-automated)) |
| `npm run devtoprod:thumbnails` | Dev PC PowerShell | Rebuild painting + portrait thumbs on dev before promote |
| `npm run devtoprod:images` | Dev PC PowerShell | Dev → TrueNAS volume |
+173
View File
@@ -0,0 +1,173 @@
# Harmonize dev and prod (incremental merge)
Bidirectional **catalog data** and **image** sync between `gallery_dev` and `gallery_prod`, with **last-write-wins** by timestamp. Use this when **both** environments may have curator edits since the last release — not when prod should become an exact copy of dev.
For a full prod replace (weekly release), use [deploy-dev-to-prod.md](deploy-dev-to-prod.md) (`npm run devtoprod:release`).
For refreshing dev from prod entirely, use `npm run prodto:dev:db` (destructive to dev).
---
## Rules
| Layer | Direction | Conflict resolution |
|-------|-----------|---------------------|
| **Schema** | dev → prod only | Run `npm run harmonize:schema` (same as `dev:migrate` on `gallery_prod`) |
| **Catalog DB** | dev ↔ prod | Newer `updated_at` wins; missing rows copied to the other side (union merge) |
| **Images** | dev ↔ prod | Newer file mtime wins; missing files copied to the other side |
| **Users / sessions / audit** | not synced | `users`, `session`, `curator_audit_log` stay env-local |
**Never auto-deletes** rows or files that exist on only one side.
---
## Prerequisites
1. Both databases on Postgres `192.168.10.122` with `updated_at` columns applied:
```powershell
npm run dev:migrate
$env:DB_NAME = "gallery_prod"; npm run dev:migrate; Remove-Item Env:\DB_NAME
```
2. Backfill `updated_at` from image file mtimes (recommended once after migration):
```powershell
npm run dev:backfill-updated-at
npm run harmonize:backfill-updated-at
```
3. [`infra/docker/.env.prod`](../infra/docker/.env.prod) present with `DB_NAME=gallery_prod`.
4. SMB share reachable for prod images:
```powershell
net use \\192.168.10.122\Gallery /user:YOUR_TRUENAS_USER
```
5. Copy harmonize config:
```powershell
Copy-Item infra/deploy/harmonize.config.example.json infra/deploy/harmonize.config.json
```
Edit `harmonize.config.json` (gitignored) — optional `smb.user` / `smb.password`, `prefer` for tie-breaks (`dev` | `prod`), `schemaChanged: true` when new migrations shipped.
---
## One-command harmonize
```powershell
npm run harmonize
```
Dry-run (report only, no writes):
```powershell
npm run harmonize -- -DryRun
```
Or:
```powershell
npm run harmonize:dry-run
```
### Orchestrator steps
| Step | npm script | Purpose |
|------|------------|---------|
| `backupDev` | `dev:db:backup` | Safety snapshot |
| `backupProd` | `prod:db:backup` | Safety snapshot |
| `schema` | `harmonize:schema` | Apply dev migrations to prod (when `schemaChanged: true`) |
| `db` | `harmonize:db` | Row-level catalog merge |
| `images` | `harmonize:images` | Bidirectional file merge |
| `verify` | curl `/api/bounds` | Optional smoke check |
Reports are written to `db/SyncReports/harmonize_db_*.json` and `harmonize_images_*.json` (gitignored).
---
## Individual commands
| Command | Purpose |
|---------|---------|
| `npm run harmonize:schema` | Migrate prod schema from dev migration files |
| `npm run harmonize:db` | Merge catalog rows by `updated_at` |
| `npm run harmonize:db -- --dry-run` | Preview DB changes |
| `npm run harmonize:db -- --prefer=dev` | On equal `updated_at`, dev wins |
| `npm run harmonize:images` | Merge image files by mtime |
| `npm run dev:migrate:sync-timestamps` | Apply `updated_at` migration on dev only |
| `npm run dev:backfill-updated-at` | Backfill dev `updated_at` from image mtimes |
| `npm run harmonize:backfill-updated-at` | Same backfill on prod |
---
## Catalog tables synced
Processed in FK order:
`historical_eras` → `art_movements` → `artists` → `artist_periods` → `paintings` → `painting_influences` → `painting_influence_sources` → `painting_annotations`
---
## Conflict handling
Harmonize reports conflicts in the JSON report and skips those rows:
| Conflict | Cause | Resolution |
|----------|-------|------------|
| `id_collision` | Same `id` but different natural key (e.g. artist name) | Manual fix in pgAdmin; environments diverged too far |
| `equal_updated_at` | Same timestamp, different row content | Re-run with `--prefer=dev` or `--prefer=prod`, or edit one side and re-run |
**Tip:** Harmonize regularly from a shared baseline (e.g. after each weekly release) to avoid ID/natural-key collisions from independent inserts.
---
## When to use what
| Situation | Tool |
|-----------|------|
| Weekly release — prod should match dev exactly | `npm run devtoprod:release` |
| Mid-week prod curator fix + dev also changed | `npm run harmonize` |
| Dev workspace stale — full prod copy | `npm run prodto:dev:db` |
| New migration in repo | `harmonize:schema` or deploy step 4 |
| Only images changed on one side | `npm run harmonize:images` |
| Only DB metadata changed | `npm run harmonize:db` |
---
## Example flows
### Prod curator uploaded a painting; dev also edited metadata
```powershell
npm run harmonize
```
DB rows merge by `updated_at`; image files merge by mtime. Both sides receive the latest version of each entity.
### Schema change + mixed edits
1. Finish and test on dev: `npm run dev:migrate`
2. Set `schemaChanged: true` in `harmonize.config.json` (or enable `steps.schema`)
3. `npm run harmonize`
### Preview before writing
```powershell
npm run harmonize:dry-run
# Review db/SyncReports/harmonize_*.json
npm run harmonize
```
---
## Safety
- Pre-flight backups of dev and prod DB (configurable; on by default)
- Prod writes require `yes` or `CONFIRM_PROD=1` (orchestrator sets `CONFIRM_PROD=1` when `autoConfirm`-style run)
- No TRUNCATE — harmonize only inserts/updates changed rows
- Rollback: restore from `db/DataBackup/gallery_*_data_*.txt` using `dev:db:restore` or `devtoprod:db:restore`
See also [environments.md](environments.md) and [deploy-dev-to-prod.md](deploy-dev-to-prod.md).
+2 -1
View File
@@ -110,13 +110,14 @@ Image fetch can take hours if you run it for the entire catalog. The first line
| `npm run prodto:dev:images` | Copy prod images → dev repo |
| `npm run prodto:dev:db` | Clone `gallery_prod``gallery_dev` |
| `npm run dev:db:backup` / `devtoprod:db:restore` | Dev backup / promote DB to prod |
| `npm run harmonize` | Bidirectional catalog + image merge (last-write-wins) — [harmonize-dev-prod.md](harmonize-dev-prod.md) |
| `npm run prod:build` | Build production SPA into `client/dist` |
| `npm run dev:start` | API + static SPA on `HOST`:`PORT` (uses root `.env`) |
| `npm run prod:start` | Build client, then start server |
| `npm run dev:server` | API with nodemon reload (local HMR workflow) |
| `npm run dev:client` | Vite dev server on :5173 |
See [environments.md](environments.md) for dev/prod URLs, database split, Docker deploy, and sync commands. Quick reference: [FAC.md](FAC.md).
See [environments.md](environments.md) for dev/prod URLs, database split, Docker deploy, and sync commands. For incremental dev ↔ prod merge (both sides edited), see [harmonize-dev-prod.md](harmonize-dev-prod.md). Quick reference: [FAC.md](FAC.md).
**Production frontend:** build the client, then start the server:
+39
View File
@@ -0,0 +1,39 @@
-- Catalog row timestamps for dev/prod harmonize (last-write-wins merge).
-- Idempotent: safe to re-run on existing databases.
CREATE OR REPLACE FUNCTION sync_touch_updated_at()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = now();
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DO $$
DECLARE
tbl TEXT;
BEGIN
FOREACH tbl IN ARRAY ARRAY[
'historical_eras',
'art_movements',
'artists',
'artist_periods',
'paintings',
'painting_influences',
'painting_influence_sources',
'painting_annotations'
]
LOOP
EXECUTE format(
'ALTER TABLE %I ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now()',
tbl
);
EXECUTE format('DROP TRIGGER IF EXISTS %I ON %I', tbl || '_updated_at', tbl);
EXECUTE format(
'CREATE TRIGGER %I BEFORE UPDATE ON %I FOR EACH ROW EXECUTE PROCEDURE sync_touch_updated_at()',
tbl || '_updated_at',
tbl
);
END LOOP;
END $$;
@@ -0,0 +1,27 @@
{
"_comment": "Copy to harmonize.config.json (gitignored) before running npm run harmonize",
"dryRun": false,
"prefer": null,
"schemaChanged": false,
"steps": {
"backupDev": true,
"backupProd": true,
"schema": false,
"db": true,
"images": true,
"verify": false
},
"smb": {
"host": "192.168.10.122",
"share": "Gallery",
"user": "",
"password": ""
},
"paths": {
"prodImages": "\\\\192.168.10.122\\Gallery\\data\\images"
},
"verify": {
"devUrl": "https://devgallery.mysuperlab.netcraze.pro/api/bounds",
"publicUrl": "https://gallery.mysuperlab.netcraze.pro/api/bounds"
}
}
+27
View File
@@ -0,0 +1,27 @@
# Map SMB share for harmonize image step (optional wrapper).
param(
[switch]$SkipConfirm
)
$ErrorActionPreference = "Stop"
. (Join-Path $PSScriptRoot "lib\Deploy-CliResult.ps1")
$smbRoot = "\\192.168.10.122\Gallery"
if ($env:SMB_USER -and $env:SMB_PASSWORD) {
Write-Host "Mapping $smbRoot ..."
net use $smbRoot /user:$env:SMB_USER $env:SMB_PASSWORD 2>&1 | Out-Host
} elseif (-not (Test-Path $smbRoot)) {
Write-Warning "Cannot access $smbRoot (not authenticated)."
Write-Warning "Map the share first or set `$env:SMB_USER and `$env:SMB_PASSWORD."
}
$npmArgs = @('run', 'harmonize:images')
if ($args.Count -gt 0) {
$npmArgs += '--'
$npmArgs += $args
}
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
Set-Location $RepoRoot
& npm @npmArgs
exit $LASTEXITCODE
+234
View File
@@ -0,0 +1,234 @@
# Dev <-> prod harmonize orchestrator (incremental merge; not full replace).
#
# Usage (from repo root):
# npm run harmonize
# npm run harmonize -- -DryRun
# npm run harmonize -- -Config infra/deploy/harmonize.config.json
#
# Requires infra/deploy/harmonize.config.json (copy from harmonize.config.example.json)
# or infra/deploy/devtoprod.config.json with optional "harmonize" section.
param(
[string]$Config = (Join-Path $PSScriptRoot "..\deploy\harmonize.config.json"),
[switch]$DryRun
)
$ErrorActionPreference = "Stop"
. (Join-Path $PSScriptRoot "lib\Deploy-CliResult.ps1")
$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path
Set-Location $RepoRoot
$BannerWidth = 64
$CompletedSteps = [System.Collections.Generic.List[string]]::new()
$FailedStep = $null
function Write-HarmonizeBanner {
param([bool]$Success, [string]$FailedAt = '')
Write-Host ''
Write-Host ('=' * $BannerWidth)
if ($Success) {
Write-Host ' HARMONIZE SUCCEEDED - dev and prod converged'
Write-Host (' Steps: ' + ($CompletedSteps -join ', '))
} else {
Write-Host " HARMONIZE FAILED at step: $FailedAt"
if ($CompletedSteps.Count -gt 0) {
Write-Host (' Completed: ' + ($CompletedSteps -join ', '))
}
}
Write-Host ('=' * $BannerWidth)
}
function Invoke-Npm {
param([string]$Script, [string[]]$ExtraArgs = @())
$npmArgs = @('run', $Script)
if ($ExtraArgs.Count -gt 0) {
$npmArgs += '--'
$npmArgs += $ExtraArgs
}
Write-Host ("Running npm: npm " + ($npmArgs -join ' ')) -ForegroundColor DarkGray
$prevEap = $ErrorActionPreference
$ErrorActionPreference = 'Continue'
$output = & npm @npmArgs 2>&1
$exit = $LASTEXITCODE
$ErrorActionPreference = $prevEap
foreach ($line in $output) { Write-Host $line }
return $exit
}
function Invoke-Step {
param([string]$Name, [scriptblock]$Action)
if ($DryRun) {
Write-Host "[dry-run] Would run: $Name"
$script:CompletedSteps.Add($Name)
return $true
}
Write-Host ''
Write-Host "--- Step: $Name ---" -ForegroundColor Cyan
try {
$code = & $Action
if ($null -eq $code) { $code = if ($null -ne $LASTEXITCODE) { $LASTEXITCODE } else { 0 } }
if ($code -ne 0) {
$script:FailedStep = $Name
return $false
}
$script:CompletedSteps.Add($Name)
return $true
} catch {
Write-Host $_.Exception.Message -ForegroundColor Red
$script:FailedStep = $Name
return $false
}
}
function Resolve-ConfigPath {
if (Test-Path $Config) { return (Resolve-Path $Config).Path }
$fallback = Join-Path $PSScriptRoot "..\deploy\devtoprod.config.json"
if (Test-Path $fallback) {
Write-Host "Using fallback config: $fallback" -ForegroundColor DarkYellow
return (Resolve-Path $fallback).Path
}
throw "Missing harmonize config. Copy infra/deploy/harmonize.config.example.json to infra/deploy/harmonize.config.json"
}
function Get-HarmonizeSteps {
param([object]$Raw)
$harmonize = $Raw.harmonize
if ($null -eq $harmonize) { $harmonize = $Raw }
$schemaChanged = $false
if ($null -ne $harmonize.schemaChanged) { $schemaChanged = [bool]$harmonize.schemaChanged }
elseif ($null -ne $Raw.schemaChanged) { $schemaChanged = [bool]$Raw.schemaChanged }
$steps = @{
backupDev = $true
backupProd = $true
schema = $schemaChanged
db = $true
images = $true
verify = $false
}
$cfgSteps = $harmonize.steps
if ($null -ne $cfgSteps) {
foreach ($key in $cfgSteps.PSObject.Properties.Name) {
$steps[$key] = [bool]$cfgSteps.$key
}
}
return $steps
}
function Set-SmbCredentials {
param([object]$Smb)
if ($Smb.user -and $Smb.password) {
$env:SMB_USER = [string]$Smb.user
$env:SMB_PASSWORD = [string]$Smb.password
}
}
function Test-ApiUrl {
param([string]$Url)
if (-not $Url) { return $true }
try {
$resp = Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec 20
Write-Host " OK $Url ($($resp.StatusCode))" -ForegroundColor DarkGreen
return $true
} catch {
Write-Host " FAIL $Url : $($_.Exception.Message)" -ForegroundColor Red
return $false
}
}
$configPath = Resolve-ConfigPath
$raw = Get-Content -LiteralPath $configPath -Raw | ConvertFrom-Json
$steps = Get-HarmonizeSteps -Raw $raw
$harmonize = if ($raw.harmonize) { $raw.harmonize } else { $raw }
if ($DryRun -or ($harmonize.dryRun -eq $true)) {
$DryRun = $true
$env:CONFIRM_PROD = '1'
}
Set-SmbCredentials -Smb $raw.smb
if ($harmonize.smb) { Set-SmbCredentials -Smb $harmonize.smb }
$preferArg = @()
if ($harmonize.prefer) {
$preferArg = @("--prefer=$($harmonize.prefer)")
}
$dryRunArgs = @()
if ($DryRun) { $dryRunArgs = @('--dry-run') }
Write-Host "Harmonize config: $configPath"
Write-Host ("Steps enabled: " + (($steps.GetEnumerator() | Where-Object { $_.Value } | ForEach-Object { $_.Key }) -join ', '))
if (-not $DryRun) {
$env:CONFIRM_PROD = '1'
}
if ($steps.backupDev) {
if (-not (Invoke-Step 'backupDev' { Invoke-Npm 'dev:db:backup' })) {
Write-HarmonizeBanner -Success $false -FailedAt 'backupDev'
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at backupDev.'
exit 1
}
}
if ($steps.backupProd) {
if (-not (Invoke-Step 'backupProd' { Invoke-Npm 'prod:db:backup' })) {
Write-HarmonizeBanner -Success $false -FailedAt 'backupProd'
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at backupProd.'
exit 1
}
}
if ($steps.schema) {
if (-not (Invoke-Step 'schema' { Invoke-Npm 'harmonize:schema' })) {
Write-HarmonizeBanner -Success $false -FailedAt 'schema'
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at schema migrate.'
exit 1
}
}
if ($steps.db) {
if (-not (Invoke-Step 'db' { Invoke-Npm 'harmonize:db' @dryRunArgs @preferArg })) {
Write-HarmonizeBanner -Success $false -FailedAt 'db'
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at DB harmonize.'
exit 1
}
}
if ($steps.images) {
if (-not (Invoke-Step 'images' { Invoke-Npm 'harmonize:images' @dryRunArgs })) {
Write-HarmonizeBanner -Success $false -FailedAt 'images'
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at image harmonize.'
exit 1
}
}
if ($steps.verify -and -not $DryRun) {
$verify = $harmonize.verify
if (-not $verify) { $verify = $raw.verify }
$ok = $true
if (-not (Invoke-Step 'verify' {
if ($verify.devUrl -and -not (Test-ApiUrl $verify.devUrl)) { return 1 }
if ($verify.publicUrl -and -not (Test-ApiUrl $verify.publicUrl)) { return 1 }
return 0
})) {
Write-HarmonizeBanner -Success $false -FailedAt 'verify'
Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Verify step failed.'
exit 1
}
}
Write-HarmonizeBanner -Success $true
$summary = if ($DryRun) { 'Harmonize dry-run complete (no changes written).' } else { 'Harmonize complete.' }
Write-DeployCliResult -Script 'harmonize' -Success $true -Summary $summary -Details @("Config: $configPath")
exit 0
+8
View File
@@ -29,6 +29,8 @@
"dev:migrate:painting-annotations": "node scripts/migrate-painting-annotations.js",
"dev:migrate:artist-palette": "node scripts/migrate-artist-palette.js",
"dev:migrate:search": "node scripts/migrate-search.js",
"dev:migrate:sync-timestamps": "node scripts/migrate-sync-timestamps.js",
"dev:backfill-updated-at": "node scripts/backfill-updated-at.js",
"dev:import-painter-palette": "node scripts/import-painter-palette.js",
"dev:analyze-painter-palette": "node scripts/analyze-painter-palette.js",
"dev:export-paintings": "node scripts/export-paintings-csv.js",
@@ -48,6 +50,12 @@
"devtoprod:release": "powershell -NoProfile -ExecutionPolicy Bypass -File infra/scripts/deploy-dev-to-prod.ps1",
"prodto:dev:db": "node scripts/sync-prod-to-dev.js",
"prodto:dev:images": "powershell -NoProfile -ExecutionPolicy Bypass -File infra/scripts/sync-images-from-prod.ps1",
"harmonize:schema": "node scripts/harmonize-schema.js",
"harmonize:db": "node scripts/harmonize-db.js",
"harmonize:images": "node scripts/harmonize-images.js",
"harmonize:backfill-updated-at": "node scripts/backfill-updated-at.js --prod",
"harmonize:dry-run": "node scripts/harmonize-db.js --dry-run && node scripts/harmonize-images.js --dry-run",
"harmonize": "powershell -NoProfile -ExecutionPolicy Bypass -File infra/scripts/harmonize.ps1",
"infra:db:split-dev-prod": "node scripts/split-dev-prod-databases.js"
},
"keywords": [],
+151
View File
@@ -0,0 +1,151 @@
/**
* Backfill catalog updated_at from linked image file mtimes where possible.
*
* Dev: npm run dev:backfill-updated-at
* Prod: npm run harmonize:backfill-updated-at
*/
const fs = require('fs');
const path = require('path');
const pg = require('pg');
const {
assertDevDatabase,
assertProdDatabase,
loadDevPgConfig,
loadProdPgConfig,
rootDir,
} = require('./db-env');
const { printCliResult } = require('./lib/cli-result');
const { Client } = pg;
function parseArgs(argv) {
return { prod: argv.includes('--prod') };
}
function resolveImageDir(prod) {
if (prod) {
const envPath = path.join(rootDir, 'infra', 'docker', '.env.prod');
if (fs.existsSync(envPath)) {
for (const line of fs.readFileSync(envPath, 'utf8').split(/\r?\n/)) {
const trimmed = line.trim();
if (trimmed.startsWith('IMAGE_DIR=')) {
const val = trimmed.slice('IMAGE_DIR='.length).trim().replace(/^["']|["']$/g, '');
if (val) return path.resolve(val);
}
}
}
return path.resolve('\\\\192.168.10.122\\Gallery\\data\\images');
}
require('dotenv').config({ path: path.join(rootDir, '.env') });
return path.resolve(process.env.IMAGE_DIR || path.join(rootDir, 'data', 'images'));
}
function fileMtimeMs(imageDir, relPath) {
if (!relPath || typeof relPath !== 'string') return null;
const abs = path.join(imageDir, relPath.replace(/^\//, ''));
try {
if (!fs.existsSync(abs)) return null;
return fs.statSync(abs).mtimeMs;
} catch {
return null;
}
}
function maxMtime(imageDir, paths) {
let max = null;
for (const rel of paths) {
const ms = fileMtimeMs(imageDir, rel);
if (ms != null && (max == null || ms > max)) max = ms;
}
return max;
}
async function main() {
const { prod } = parseArgs(process.argv.slice(2));
const config = prod ? loadProdPgConfig() : loadDevPgConfig();
if (prod) assertProdDatabase(config.database);
else assertDevDatabase(config.database);
const imageDir = resolveImageDir(prod);
const client = new Client(config);
await client.connect();
const fallback = new Date();
let updated = 0;
const { rows: artists } = await client.query(
'SELECT id, portrait_path, portrait_thumb_path FROM artists',
);
for (const row of artists) {
const ms = maxMtime(imageDir, [row.portrait_path, row.portrait_thumb_path]);
const ts = ms != null ? new Date(ms) : fallback;
await client.query('UPDATE artists SET updated_at = $1 WHERE id = $2', [ts, row.id]);
updated += 1;
}
const { rows: paintings } = await client.query(
'SELECT id, image_path, thumbnail_path FROM paintings',
);
for (const row of paintings) {
const ms = maxMtime(imageDir, [row.image_path, row.thumbnail_path]);
const ts = ms != null ? new Date(ms) : fallback;
await client.query('UPDATE paintings SET updated_at = $1 WHERE id = $2', [ts, row.id]);
updated += 1;
}
const childUpdates = [
['artist_periods', 'artist_id', 'artists'],
['painting_influences', 'painting_id', 'paintings'],
['painting_influence_sources', 'painting_id', 'paintings'],
['painting_annotations', 'painting_id', 'paintings'],
];
for (const [child, fk, parent] of childUpdates) {
const res = await client.query(
`UPDATE ${child} c
SET updated_at = GREATEST(c.updated_at, p.updated_at)
FROM ${parent} p
WHERE c.${fk} = p.id`,
);
updated += res.rowCount || 0;
}
for (const table of ['historical_eras', 'art_movements']) {
const res = await client.query(
`UPDATE ${table} SET updated_at = $1 WHERE updated_at IS NOT NULL`,
[fallback],
);
updated += res.rowCount || 0;
}
await client.query(
`UPDATE art_movements m
SET updated_at = GREATEST(m.updated_at, e.updated_at)
FROM historical_eras e
WHERE m.era_id = e.id`,
);
await client.query(
`UPDATE artists a
SET updated_at = GREATEST(a.updated_at, m.updated_at)
FROM art_movements m
WHERE a.movement_id = m.id`,
);
await client.end();
printCliResult({
script: 'backfill-updated-at',
ok: true,
summary: `Backfilled updated_at on ${config.database} (${updated} row touches).`,
details: [`Image dir: ${imageDir}`],
});
}
main().catch((err) => {
printCliResult({
script: 'backfill-updated-at',
ok: false,
summary: err.message,
});
});
+304
View File
@@ -0,0 +1,304 @@
/**
* Bidirectional catalog DB harmonize (dev <-> prod) by updated_at.
*
* Usage:
* npm run harmonize:db
* npm run harmonize:db -- --dry-run
* npm run harmonize:db -- --prefer=dev
*/
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const pg = require('pg');
const {
loadDevPgConfig,
loadProdPgConfig,
confirmProdAction,
rootDir,
} = require('./db-env');
const { printCliResult } = require('./lib/cli-result');
const { Client } = pg;
const CATALOG_TABLES = [
'historical_eras',
'art_movements',
'artists',
'artist_periods',
'paintings',
'painting_influences',
'painting_influence_sources',
'painting_annotations',
];
const NATURAL_KEY_FN = {
historical_eras: (r) => String(r.name || '').trim().toLowerCase(),
art_movements: (r) => String(r.name || '').trim().toLowerCase(),
artists: (r) => String(r.name || '').trim().toLowerCase(),
artist_periods: (r) => `${r.artist_id}:${String(r.name || '').trim().toLowerCase()}`,
paintings: (r) => `${r.artist_id}:${String(r.title || '').trim().toLowerCase()}`,
painting_influences: (r) => `${r.painting_id}:${r.influenced_by_painting_id}`,
painting_influence_sources: (r) => `${r.painting_id}:${r.source_type}:${r.source_painting_id || 0}:${r.source_artist_id || 0}:${r.source_movement_id || 0}`,
painting_annotations: (r) => `${r.painting_id}:${String(r.label || '').trim().toLowerCase()}:${r.sort_order}`,
};
function parseArgs(argv) {
const tablesArg = argv.find((a) => a.startsWith('--tables='));
const preferArg = argv.find((a) => a.startsWith('--prefer='));
return {
dryRun: argv.includes('--dry-run'),
verbose: argv.includes('--verbose'),
tables: tablesArg ? tablesArg.slice('--tables='.length).split(',').map((t) => t.trim()).filter(Boolean) : null,
prefer: preferArg ? preferArg.slice('--prefer='.length).trim().toLowerCase() : null,
};
}
function stableRowHash(row, columns) {
const payload = {};
for (const col of columns) {
const val = row[col];
if (val instanceof Date) payload[col] = val.toISOString();
else if (val != null && typeof val === 'object') payload[col] = JSON.stringify(val);
else payload[col] = val;
}
return crypto.createHash('sha256').update(JSON.stringify(payload)).digest('hex');
}
function compareUpdatedAt(a, b) {
const ta = a?.updated_at ? new Date(a.updated_at).getTime() : 0;
const tb = b?.updated_at ? new Date(b.updated_at).getTime() : 0;
if (ta === tb) return 0;
return ta > tb ? 1 : -1;
}
async function getTableColumns(client, tableName) {
const { rows } = await client.query(
`SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = $1
ORDER BY ordinal_position`,
[tableName],
);
return rows.map((r) => r.column_name);
}
async function fetchTableRows(client, tableName) {
const { rows } = await client.query(`SELECT * FROM ${tableName} ORDER BY id`);
const map = new Map();
for (const row of rows) map.set(row.id, row);
return map;
}
function buildInsertSql(tableName, columns) {
const cols = columns.map((c) => `"${c}"`).join(', ');
const placeholders = columns.map((_, i) => `$${i + 1}`).join(', ');
return `INSERT INTO ${tableName} (${cols}) OVERRIDING SYSTEM VALUE VALUES (${placeholders})`;
}
function buildUpsertSql(tableName, columns) {
const insertCols = columns.map((c) => `"${c}"`).join(', ');
const placeholders = columns.map((_, i) => `$${i + 1}`).join(', ');
const updates = columns
.filter((c) => c !== 'id')
.map((c) => `"${c}" = EXCLUDED."${c}"`)
.join(', ');
return `INSERT INTO ${tableName} (${insertCols}) OVERRIDING SYSTEM VALUE VALUES (${placeholders})
ON CONFLICT (id) DO UPDATE SET ${updates}
WHERE ${tableName}.updated_at < EXCLUDED.updated_at`;
}
function rowValues(row, columns) {
return columns.map((col) => row[col]);
}
async function insertRow(client, tableName, columns, row, dryRun) {
if (dryRun) return;
const sql = buildInsertSql(tableName, columns);
await client.query(sql, rowValues(row, columns));
}
async function upsertRow(client, tableName, columns, row, dryRun) {
if (dryRun) return;
const sql = buildUpsertSql(tableName, columns);
await client.query(sql, rowValues(row, columns));
}
async function fixSequence(client, tableName) {
await client.query(
`SELECT setval(pg_get_serial_sequence($1, 'id'), COALESCE((SELECT MAX(id) FROM ${tableName}), 1), true)`,
[tableName],
);
}
async function harmonizeTable(tableName, devClient, prodClient, options, stats) {
const naturalKeyFn = NATURAL_KEY_FN[tableName];
const columns = await getTableColumns(devClient, tableName);
if (!columns.includes('updated_at')) {
throw new Error(`Table ${tableName} missing updated_at — run migrate-sync-timestamps first`);
}
const devRows = await fetchTableRows(devClient, tableName);
const prodRows = await fetchTableRows(prodClient, tableName);
const allIds = new Set([...devRows.keys(), ...prodRows.keys()]);
for (const id of allIds) {
const devRow = devRows.get(id);
const prodRow = prodRows.get(id);
if (devRow && !prodRow) {
stats.devToProd += 1;
stats.actions.push({ table: tableName, id, action: 'dev→prod insert' });
await insertRow(prodClient, tableName, columns, devRow, options.dryRun);
continue;
}
if (prodRow && !devRow) {
stats.prodToDev += 1;
stats.actions.push({ table: tableName, id, action: 'prod→dev insert' });
await insertRow(devClient, tableName, columns, prodRow, options.dryRun);
continue;
}
const devKey = naturalKeyFn(devRow);
const prodKey = naturalKeyFn(prodRow);
if (devKey !== prodKey) {
stats.conflicts += 1;
stats.conflictDetails.push({
table: tableName,
id,
reason: 'id_collision',
devKey,
prodKey,
});
continue;
}
const devHash = stableRowHash(devRow, columns);
const prodHash = stableRowHash(prodRow, columns);
if (devHash === prodHash) {
stats.skipped += 1;
continue;
}
const cmp = compareUpdatedAt(devRow, prodRow);
if (cmp > 0) {
stats.devToProd += 1;
stats.actions.push({ table: tableName, id, action: 'dev→prod update' });
await upsertRow(prodClient, tableName, columns, devRow, options.dryRun);
} else if (cmp < 0) {
stats.prodToDev += 1;
stats.actions.push({ table: tableName, id, action: 'prod→dev update' });
await upsertRow(devClient, tableName, columns, prodRow, options.dryRun);
} else if (options.prefer === 'dev') {
stats.devToProd += 1;
stats.actions.push({ table: tableName, id, action: 'dev→prod update (prefer tie)' });
await upsertRow(prodClient, tableName, columns, devRow, options.dryRun);
} else if (options.prefer === 'prod') {
stats.prodToDev += 1;
stats.actions.push({ table: tableName, id, action: 'prod→dev update (prefer tie)' });
await upsertRow(devClient, tableName, columns, prodRow, options.dryRun);
} else {
stats.conflicts += 1;
stats.conflictDetails.push({
table: tableName,
id,
reason: 'equal_updated_at',
devUpdatedAt: devRow.updated_at,
prodUpdatedAt: prodRow.updated_at,
});
}
}
if (!options.dryRun) {
await fixSequence(devClient, tableName);
await fixSequence(prodClient, tableName);
}
}
function timestampSlug(date = new Date()) {
const pad = (n) => String(n).padStart(2, '0');
return [
date.getFullYear(),
pad(date.getMonth() + 1),
pad(date.getDate()),
'_',
pad(date.getHours()),
pad(date.getMinutes()),
pad(date.getSeconds()),
].join('');
}
async function main() {
const options = parseArgs(process.argv.slice(2));
const tables = options.tables || CATALOG_TABLES;
for (const table of tables) {
if (!CATALOG_TABLES.includes(table)) {
throw new Error(`Unknown catalog table: ${table}`);
}
}
if (!options.dryRun && process.env.CONFIRM_PROD !== '1') {
await confirmProdAction(
'Harmonize will merge catalog rows between gallery_dev and gallery_prod (prod may be modified).',
);
}
const devClient = new Client(loadDevPgConfig());
const prodClient = new Client(loadProdPgConfig());
await devClient.connect();
await prodClient.connect();
const stats = {
devToProd: 0,
prodToDev: 0,
skipped: 0,
conflicts: 0,
actions: [],
conflictDetails: [],
};
try {
for (const table of tables) {
if (options.verbose) console.log(`Harmonizing ${table}`);
await harmonizeTable(table, devClient, prodClient, options, stats);
}
} finally {
await devClient.end();
await prodClient.end();
}
const reportDir = path.join(rootDir, 'db', 'SyncReports');
fs.mkdirSync(reportDir, { recursive: true });
const reportPath = path.join(reportDir, `harmonize_db_${timestampSlug()}.json`);
fs.writeFileSync(reportPath, JSON.stringify({
generatedAt: new Date().toISOString(),
dryRun: options.dryRun,
prefer: options.prefer,
tables,
stats: {
devToProd: stats.devToProd,
prodToDev: stats.prodToDev,
skipped: stats.skipped,
conflicts: stats.conflicts,
},
conflicts: stats.conflictDetails,
actions: options.verbose ? stats.actions : stats.actions.slice(0, 200),
}, null, 2));
const mode = options.dryRun ? ' (dry-run)' : '';
printCliResult({
script: 'harmonize-db',
ok: true,
summary: `DB harmonize complete${mode}: dev→prod ${stats.devToProd}, prod→dev ${stats.prodToDev}, skipped ${stats.skipped}, conflicts ${stats.conflicts}.`,
details: [`Report: ${reportPath}`],
});
}
main().catch((err) => {
printCliResult({
script: 'harmonize-db',
ok: false,
summary: err.message,
});
});
+196
View File
@@ -0,0 +1,196 @@
/**
* Bidirectional image harmonize (dev <-> prod) by file mtime.
*
* Usage:
* npm run harmonize:images
* npm run harmonize:images -- --dry-run
*/
const fs = require('fs');
const path = require('path');
const { rootDir, confirmProdAction } = require('./db-env');
const { loadHarmonizeConfig, prodImagesPath } = require('./lib/harmonize-config');
const { printCliResult } = require('./lib/cli-result');
const IMAGE_SUBDIRS = ['portraits', 'paintings'];
const MTIME_TOLERANCE_MS = 1000;
function parseArgs(argv) {
const configArg = argv.find((a) => a.startsWith('--config='));
return {
dryRun: argv.includes('--dry-run'),
verbose: argv.includes('--verbose'),
configPath: configArg ? configArg.slice('--config='.length) : null,
};
}
function resolveDevImageDir() {
require('dotenv').config({ path: path.join(rootDir, '.env') });
return path.resolve(process.env.IMAGE_DIR || path.join(rootDir, 'data', 'images'));
}
function walkImageFiles(rootDir, baseRel = '') {
const files = [];
const abs = path.join(rootDir, baseRel);
if (!fs.existsSync(abs)) return files;
for (const entry of fs.readdirSync(abs, { withFileTypes: true })) {
const rel = baseRel ? path.join(baseRel, entry.name).replace(/\\/g, '/') : entry.name;
if (entry.isDirectory()) {
files.push(...walkImageFiles(rootDir, rel));
} else if (entry.isFile()) {
files.push(rel);
}
}
return files;
}
function collectRelativePaths(imageRoot) {
const relPaths = new Set();
for (const sub of IMAGE_SUBDIRS) {
const subRoot = path.join(imageRoot, sub);
if (!fs.existsSync(subRoot)) continue;
for (const rel of walkImageFiles(subRoot, sub)) {
relPaths.add(rel.replace(/\\/g, '/'));
}
}
return relPaths;
}
function statFile(imageRoot, relPath) {
const abs = path.join(imageRoot, relPath);
try {
if (!fs.existsSync(abs)) return null;
const st = fs.statSync(abs);
if (!st.isFile()) return null;
return { size: st.size, mtimeMs: st.mtimeMs };
} catch {
return null;
}
}
function ensureDirForFile(filePath) {
fs.mkdirSync(path.dirname(filePath), { recursive: true });
}
function copyFile(src, dest, dryRun) {
if (dryRun) return;
ensureDirForFile(dest);
fs.copyFileSync(src, dest);
}
function timestampSlug(date = new Date()) {
const pad = (n) => String(n).padStart(2, '0');
return [
date.getFullYear(),
pad(date.getMonth() + 1),
pad(date.getDate()),
'_',
pad(date.getHours()),
pad(date.getMinutes()),
pad(date.getSeconds()),
].join('');
}
async function main() {
const options = parseArgs(process.argv.slice(2));
const config = loadHarmonizeConfig(options.configPath);
const dryRun = options.dryRun || config.dryRun;
const devDir = resolveDevImageDir();
const prodDir = prodImagesPath(config);
if (!fs.existsSync(prodDir)) {
throw new Error(
`Prod image path not reachable: ${prodDir}. Map SMB share or set paths.prodImages in harmonize config.`,
);
}
if (!dryRun && process.env.CONFIRM_PROD !== '1') {
await confirmProdAction(
'Harmonize will copy image files between dev and prod (prod files may be overwritten).',
);
}
const devPaths = collectRelativePaths(devDir);
const prodPaths = collectRelativePaths(prodDir);
const allPaths = new Set([...devPaths, ...prodPaths]);
const stats = {
devToProd: 0,
prodToDev: 0,
skipped: 0,
actions: [],
};
for (const rel of allPaths) {
const devStat = statFile(devDir, rel);
const prodStat = statFile(prodDir, rel);
if (devStat && !prodStat) {
stats.devToProd += 1;
stats.actions.push({ path: rel, action: 'dev→prod copy' });
copyFile(path.join(devDir, rel), path.join(prodDir, rel), dryRun);
continue;
}
if (prodStat && !devStat) {
stats.prodToDev += 1;
stats.actions.push({ path: rel, action: 'prod→dev copy' });
copyFile(path.join(prodDir, rel), path.join(devDir, rel), dryRun);
continue;
}
if (!devStat || !prodStat) continue;
const sameSize = devStat.size === prodStat.size;
const mtimeClose = Math.abs(devStat.mtimeMs - prodStat.mtimeMs) <= MTIME_TOLERANCE_MS;
if (sameSize && mtimeClose) {
stats.skipped += 1;
continue;
}
if (devStat.mtimeMs > prodStat.mtimeMs) {
stats.devToProd += 1;
stats.actions.push({ path: rel, action: 'dev→prod update' });
copyFile(path.join(devDir, rel), path.join(prodDir, rel), dryRun);
} else if (prodStat.mtimeMs > devStat.mtimeMs) {
stats.prodToDev += 1;
stats.actions.push({ path: rel, action: 'prod→dev update' });
copyFile(path.join(prodDir, rel), path.join(devDir, rel), dryRun);
} else {
stats.skipped += 1;
}
}
const reportDir = path.join(rootDir, 'db', 'SyncReports');
fs.mkdirSync(reportDir, { recursive: true });
const reportPath = path.join(reportDir, `harmonize_images_${timestampSlug()}.json`);
fs.writeFileSync(reportPath, JSON.stringify({
generatedAt: new Date().toISOString(),
dryRun,
devDir,
prodDir,
stats: {
devToProd: stats.devToProd,
prodToDev: stats.prodToDev,
skipped: stats.skipped,
},
actions: options.verbose ? stats.actions : stats.actions.slice(0, 500),
}, null, 2));
const mode = dryRun ? ' (dry-run)' : '';
printCliResult({
script: 'harmonize-images',
ok: true,
summary: `Image harmonize complete${mode}: dev→prod ${stats.devToProd}, prod→dev ${stats.prodToDev}, skipped ${stats.skipped}.`,
details: [`Dev: ${devDir}`, `Prod: ${prodDir}`, `Report: ${reportPath}`],
});
}
main().catch((err) => {
printCliResult({
script: 'harmonize-images',
ok: false,
summary: err.message,
});
});
+54
View File
@@ -0,0 +1,54 @@
/**
* Migrate prod schema from dev migrations (schema dev → prod only).
*
* Usage: npm run harmonize:schema
*/
const { spawnSync } = require('child_process');
const path = require('path');
const { rootDir, loadProdPgConfig, confirmProdAction } = require('./db-env');
const { printCliResult } = require('./lib/cli-result');
async function main() {
if (process.env.CONFIRM_PROD !== '1') {
await confirmProdAction('Apply dev migrations to gallery_prod schema?');
}
const prodConfig = loadProdPgConfig();
const env = {
...process.env,
DB_HOST: prodConfig.host,
DB_PORT: String(prodConfig.port),
DB_USER: prodConfig.user,
DB_PASSWORD: prodConfig.password,
DB_NAME: prodConfig.database,
};
const result = spawnSync(process.execPath, [path.join(rootDir, 'server', 'migrate.js')], {
cwd: rootDir,
env,
stdio: 'inherit',
});
if (result.status !== 0) {
printCliResult({
script: 'harmonize-schema',
ok: false,
summary: 'Prod schema migration failed.',
});
return;
}
printCliResult({
script: 'harmonize-schema',
ok: true,
summary: 'Prod schema migrated from dev migration files (gallery_prod).',
});
}
main().catch((err) => {
printCliResult({
script: 'harmonize-schema',
ok: false,
summary: err.message,
});
});
+78
View File
@@ -0,0 +1,78 @@
const fs = require('fs');
const path = require('path');
const { rootDir } = require('./db-env');
const DEFAULT_CONFIG_PATH = path.join(rootDir, 'infra', 'deploy', 'harmonize.config.json');
const FALLBACK_CONFIG_PATH = path.join(rootDir, 'infra', 'deploy', 'devtoprod.config.json');
function loadHarmonizeConfig(configPath) {
const resolved = configPath
? path.resolve(configPath)
: (fs.existsSync(DEFAULT_CONFIG_PATH) ? DEFAULT_CONFIG_PATH : FALLBACK_CONFIG_PATH);
if (!fs.existsSync(resolved)) {
return {
configPath: resolved,
dryRun: false,
prefer: null,
schemaChanged: false,
steps: {
backupDev: true,
backupProd: true,
schema: false,
db: true,
images: true,
verify: false,
},
smb: {
host: '192.168.10.122',
share: 'Gallery',
user: '',
password: '',
},
paths: {
prodImages: '\\\\192.168.10.122\\Gallery\\data\\images',
},
verify: {},
};
}
const raw = JSON.parse(fs.readFileSync(resolved, 'utf8'));
const harmonize = raw.harmonize || raw;
return {
configPath: resolved,
dryRun: Boolean(harmonize.dryRun),
prefer: harmonize.prefer || null,
schemaChanged: Boolean(harmonize.schemaChanged ?? raw.schemaChanged),
steps: {
backupDev: harmonize.steps?.backupDev ?? true,
backupProd: harmonize.steps?.backupProd ?? true,
schema: harmonize.steps?.schema ?? Boolean(harmonize.schemaChanged ?? raw.schemaChanged),
db: harmonize.steps?.db ?? true,
images: harmonize.steps?.images ?? true,
verify: harmonize.steps?.verify ?? false,
},
smb: {
host: harmonize.smb?.host ?? raw.smb?.host ?? '192.168.10.122',
share: harmonize.smb?.share ?? raw.smb?.share ?? 'Gallery',
user: harmonize.smb?.user ?? raw.smb?.user ?? '',
password: harmonize.smb?.password ?? raw.smb?.password ?? '',
},
paths: {
prodImages: harmonize.paths?.prodImages
?? `\\\\${harmonize.smb?.host ?? raw.smb?.host ?? '192.168.10.122'}\\${harmonize.smb?.share ?? raw.smb?.share ?? 'Gallery'}\\data\\images`,
},
verify: harmonize.verify ?? raw.verify ?? {},
};
}
function prodImagesPath(config) {
return config.paths?.prodImages
|| `\\\\${config.smb.host}\\${config.smb.share}\\data\\images`;
}
module.exports = {
DEFAULT_CONFIG_PATH,
loadHarmonizeConfig,
prodImagesPath,
};
+17
View File
@@ -0,0 +1,17 @@
require('dotenv').config();
const fs = require('fs');
const path = require('path');
const pool = require('../server/db');
async function main() {
const sqlPath = path.join(__dirname, '../db/migrate-sync-timestamps.sql');
const sql = fs.readFileSync(sqlPath, 'utf8');
await pool.query(sql);
console.log('Catalog sync timestamps ready (updated_at + triggers)');
await pool.end();
}
main().catch((err) => {
console.error(err);
process.exit(1);
});
+1
View File
@@ -13,6 +13,7 @@ const INCREMENTAL_MIGRATIONS = [
'migrate-portrait-thumbs.sql',
'migrate-perf-indexes.sql',
'migrate-search.sql',
'migrate-sync-timestamps.sql',
];
async function bootstrapCurator() {