diff --git a/.gitignore b/.gitignore index 09dda19..ec7bed2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ client/node_modules/ .env.*.local infra/docker/.env.prod infra/deploy/devtoprod.config.json +infra/deploy/harmonize.config.json +db/SyncReports/ infra/deploy/last-docker-release.json # DB backups (may contain data) diff --git a/Documentation/FAC.md b/Documentation/FAC.md index b5a74d6..be6d254 100644 --- a/Documentation/FAC.md +++ b/Documentation/FAC.md @@ -138,6 +138,10 @@ Run `npm run dev:migrate` against prod DB after first deploy with auth vars set | `npm run prod:db:backup` | Prod backup (reads `infra/docker/.env.prod`) | | `npm run dev:db:restore -- --file ` | Restore backup into **dev** (truncates tables first; prompts `yes`) | | `npm run devtoprod:db:restore -- --file ` | Restore into **prod** (requires confirmation) | +| `npm run harmonize` | Bidirectional catalog DB + image merge by `updated_at` / file mtime — [harmonize-dev-prod.md](harmonize-dev-prod.md) | +| `npm run harmonize:schema` | Apply dev migrations to prod schema only (dev → prod) | +| `npm run harmonize:db` / `harmonize:images` | DB or image merge only | +| `npm run dev:backfill-updated-at` | Backfill catalog `updated_at` from image mtimes (dev) | **One-time split (recommended):** pgAdmin on dev PC → open [`db/split-dev-prod-pgadmin.sql`](../db/split-dev-prod-pgadmin.sql) → run each STEP on database `postgres`, then verify on `gallery_dev`. @@ -237,6 +241,8 @@ npm run devtoprod:release # full promote from infra/deploy/devtoprod.conf npm run devtoprod:thumbnails # rebuild thumb files + DB paths on dev (before backup) npm run devtoprod:images # dev repo → TrueNAS (promote / first deploy) npm run prodto:dev:images # TrueNAS → dev repo +npm run harmonize # bidirectional merge (newer wins) — see harmonize-dev-prod.md +npm run harmonize:dry-run # preview DB + image changes only ``` Type `yes` when prompted (or set `autoConfirm: true` in release config). Robocopy exit codes **0–7** = success. Deploy scripts print a final **`===== SUCCESS =====`** or **`===== FAILED =====`** banner. diff --git a/Documentation/Plans.md b/Documentation/Plans.md index f3332e9..9565cfc 100644 --- a/Documentation/Plans.md +++ b/Documentation/Plans.md @@ -3,7 +3,7 @@ this file contains draft for future releases and features 1. Multy language support, russian version at least, search for best implementation, preferably story in db and easily expandable. tool to check and correct translation 2. tool to manage links (influence/influenced by ) import csv's ( define format), edit ,add, delete, visualize, map to pictures/ entities 3. tool to monitor/manage (plan actions) of curator actions, markers to check painting/text ? -4. tool to sync prod /env resources (both ways), db structure, db data, images, users etc +4. ~~tool to sync prod /env resources (both ways), db structure, db data, images, users etc~~ — done for catalog DB + images: `npm run harmonize` (schema dev→prod only; users/audit excluded) — [harmonize-dev-prod.md](harmonize-dev-prod.md) 5. curator_audit_log should contain log of actions like fixit, checked, upload etc with details for which entity it was made and details what was the action and outcome 6. ~~create search by entity (painting, artist, movement)~~ — done: timeline header + `GET /api/search` 7. create guided tours (with text/extra infor, set of entities) diff --git a/Documentation/basics.md b/Documentation/basics.md index 73bffb3..7979f9a 100644 --- a/Documentation/basics.md +++ b/Documentation/basics.md @@ -471,6 +471,7 @@ See [API.md](API.md#authentication) and [data-and-images.md](data-and-images.md# |----------|----------| | [setup.md](setup.md) | Install, database, npm scripts | | [deploy-dev-to-prod.md](deploy-dev-to-prod.md) | Release runbook + one-command `devtoprod:release` | +| [harmonize-dev-prod.md](harmonize-dev-prod.md) | Incremental dev ↔ prod merge (catalog DB + images) | | [DB_structure.md](DB_structure.md) | Tables and relationships | | [API.md](API.md) | REST endpoints | | [data-and-images.md](data-and-images.md) | Image pipeline and seeding | diff --git a/Documentation/data-and-images.md b/Documentation/data-and-images.md index 7d0a910..ca0d97f 100644 --- a/Documentation/data-and-images.md +++ b/Documentation/data-and-images.md @@ -31,7 +31,7 @@ File names are sanitised `{Artist}_{Title}.{ext}`. The image service can redisco | **Development** | `./data/images/` in repo | Working copy on dev PC | | **Production** | `/mnt/BasePool/Applications/Gallery/data/images` on TrueNAS | SMB `\\192.168.10.122\Gallery\data\images` | -Promote dev → prod files: `npm run devtoprod:images` (after `net use \\192.168.10.122\Gallery`). Refresh dev from prod: `npm run prodto:dev:images`. See [environments.md](environments.md). +**One-direction promote:** `npm run devtoprod:images` (dev → prod, skip older). **Refresh dev from prod:** `npm run prodto:dev:images`. **Bidirectional merge** (newer file wins): `npm run harmonize:images` or full `npm run harmonize` — see [harmonize-dev-prod.md](harmonize-dev-prod.md). General sync reference: [environments.md](environments.md). ## Scripts overview @@ -49,6 +49,8 @@ Promote dev → prod files: `npm run devtoprod:images` (after `net use \\192.168 | `fetch-missing-images.js` | `npm run dev:fetch-images` | Downloads files for paintings missing on disk | | `image-fetcher.js` | *(library)* | Wikimedia / museum resolution used by fetch scripts and API | | `sync-images-to-prod.ps1` / `sync-images-from-prod.ps1` | `npm run devtoprod:images` / `npm run prodto:dev:images` | Robocopy via SMB `\\192.168.10.122\Gallery` | +| `harmonize-db.js` / `harmonize-images.js` | `npm run harmonize:db` / `harmonize:images` | Bidirectional merge by `updated_at` / file mtime | +| `harmonize.ps1` | `npm run harmonize` | Orchestrator: backups, optional schema, DB + image merge | | `regenerate-thumbnails.js` | `npm run dev:regenerate-thumbnails` | Rebuild painting thumbs from full images via `sharp` | | `regenerate-portrait-thumbs.js` | `npm run dev:regenerate-portrait-thumbs` | Rebuild timeline portrait thumbs (~256px) and set `portrait_thumb_path` | | `audit-painting-images.js` | `npm run dev:audit-painting-images` | Detect thumb/full aspect-ratio mismatches | diff --git a/Documentation/deploy-dev-to-prod.md b/Documentation/deploy-dev-to-prod.md index 012cead..756c166 100644 --- a/Documentation/deploy-dev-to-prod.md +++ b/Documentation/deploy-dev-to-prod.md @@ -2,7 +2,11 @@ Step-by-step guide for promoting the **development** version of Gallery to **production**. Covers code, database schema, database data, thumbnail generation, and image files. -> **Default is dev.** This runbook is for a **scheduled release** (~weekly, or when you explicitly decide to ship). Day-to-day work stays on dev — see [environments.md](environments.md#development-first-workflow-default). If the one-time prod install is not done yet, follow [environments.md → One-time setup](environments.md#one-time-setup-full-walkthrough) and [infra/docker/DEPLOY-truenas.md](../infra/docker/DEPLOY-truenas.md) first. +> **Default is dev.** This runbook is for a **scheduled release** (~weekly, or when you explicitly decide to ship). Day-to-day work stays on dev — see [environments.md](environments.md#development-first-workflow-default). +> +> **Mid-week merge:** If both dev and prod have catalog or image edits and you need **last-write-wins** sync instead of a full prod overwrite, use [harmonize-dev-prod.md](harmonize-dev-prod.md) (`npm run harmonize`). Keep this runbook for releases where prod should exactly match dev. +> +> If the one-time prod install is not done yet, follow [environments.md → One-time setup](environments.md#one-time-setup-full-walkthrough) and [infra/docker/DEPLOY-truenas.md](../infra/docker/DEPLOY-truenas.md) first. | | Dev (source) | Prod (target) | |---|---|---| diff --git a/Documentation/environments.md b/Documentation/environments.md index 18d9a71..a0b9e49 100644 --- a/Documentation/environments.md +++ b/Documentation/environments.md @@ -305,6 +305,7 @@ See also [Drunkmeyou gitea-https-keenetic-npm-setup.md](../../Drunkmeyou/Documen | Fast local HMR (no Keenetic) | `npm run dev:server` + `npm run dev:client` | | Refresh dev DB from prod | `npm run prodto:dev:db` | | Pull prod images to dev | `npm run prodto:dev:images` | +| Merge dev ↔ prod catalog + images (incremental) | `npm run harmonize` — see [harmonize-dev-prod.md](harmonize-dev-prod.md) | --- @@ -312,6 +313,8 @@ See also [Drunkmeyou gitea-https-keenetic-npm-setup.md](../../Drunkmeyou/Documen Run this when you are ready to ship dev to production — **not** after every small change. Typical cadence: **about once a week**. +For **incremental** dev ↔ prod merge (both sides edited), use [harmonize-dev-prod.md](harmonize-dev-prod.md) instead of full restore. + **One command (recommended):** copy [`infra/deploy/devtoprod.config.example.json`](../infra/deploy/devtoprod.config.example.json) to `infra/deploy/devtoprod.config.json`, edit it, then `npm run devtoprod:release` or `deploy-dev-to-prod.cmd`. See [deploy-dev-to-prod.md → One-command release](deploy-dev-to-prod.md#one-command-release-automated). **Manual steps:** detailed runbook (per-change decision matrix, schema migration, rollback): [deploy-dev-to-prod.md](deploy-dev-to-prod.md). @@ -338,11 +341,15 @@ Run this when you are ready to ship dev to production — **not** after every sm | `npm run prodto:dev:db` | Dev PC PowerShell | Clone prod → dev | | `npm run dev:db:backup` | Dev PC PowerShell | Dev backup | | `npm run devtoprod:db:restore` | Dev PC PowerShell | Restore into prod | +| `npm run harmonize` | Dev PC PowerShell | Bidirectional catalog + image merge — [harmonize-dev-prod.md](harmonize-dev-prod.md) | +| `npm run harmonize:db` | Dev PC PowerShell | DB merge only | +| `npm run harmonize:images` | Dev PC PowerShell | Image merge only | ## Image sync | Command | Where | Direction | |---------|-------|-----------| +| `npm run harmonize` | Dev PC PowerShell | Bidirectional merge (mtime newer wins) — [harmonize-dev-prod.md](harmonize-dev-prod.md) | | `npm run devtoprod:release` | Dev PC PowerShell | Full config-driven promote (see [deploy-dev-to-prod.md](deploy-dev-to-prod.md#one-command-release-automated)) | | `npm run devtoprod:thumbnails` | Dev PC PowerShell | Rebuild painting + portrait thumbs on dev before promote | | `npm run devtoprod:images` | Dev PC PowerShell | Dev → TrueNAS volume | diff --git a/Documentation/harmonize-dev-prod.md b/Documentation/harmonize-dev-prod.md new file mode 100644 index 0000000..1408a92 --- /dev/null +++ b/Documentation/harmonize-dev-prod.md @@ -0,0 +1,173 @@ +# Harmonize dev and prod (incremental merge) + +Bidirectional **catalog data** and **image** sync between `gallery_dev` and `gallery_prod`, with **last-write-wins** by timestamp. Use this when **both** environments may have curator edits since the last release — not when prod should become an exact copy of dev. + +For a full prod replace (weekly release), use [deploy-dev-to-prod.md](deploy-dev-to-prod.md) (`npm run devtoprod:release`). + +For refreshing dev from prod entirely, use `npm run prodto:dev:db` (destructive to dev). + +--- + +## Rules + +| Layer | Direction | Conflict resolution | +|-------|-----------|---------------------| +| **Schema** | dev → prod only | Run `npm run harmonize:schema` (same as `dev:migrate` on `gallery_prod`) | +| **Catalog DB** | dev ↔ prod | Newer `updated_at` wins; missing rows copied to the other side (union merge) | +| **Images** | dev ↔ prod | Newer file mtime wins; missing files copied to the other side | +| **Users / sessions / audit** | not synced | `users`, `session`, `curator_audit_log` stay env-local | + +**Never auto-deletes** rows or files that exist on only one side. + +--- + +## Prerequisites + +1. Both databases on Postgres `192.168.10.122` with `updated_at` columns applied: + + ```powershell + npm run dev:migrate + $env:DB_NAME = "gallery_prod"; npm run dev:migrate; Remove-Item Env:\DB_NAME + ``` + +2. Backfill `updated_at` from image file mtimes (recommended once after migration): + + ```powershell + npm run dev:backfill-updated-at + npm run harmonize:backfill-updated-at + ``` + +3. [`infra/docker/.env.prod`](../infra/docker/.env.prod) present with `DB_NAME=gallery_prod`. + +4. SMB share reachable for prod images: + + ```powershell + net use \\192.168.10.122\Gallery /user:YOUR_TRUENAS_USER + ``` + +5. Copy harmonize config: + + ```powershell + Copy-Item infra/deploy/harmonize.config.example.json infra/deploy/harmonize.config.json + ``` + + Edit `harmonize.config.json` (gitignored) — optional `smb.user` / `smb.password`, `prefer` for tie-breaks (`dev` | `prod`), `schemaChanged: true` when new migrations shipped. + +--- + +## One-command harmonize + +```powershell +npm run harmonize +``` + +Dry-run (report only, no writes): + +```powershell +npm run harmonize -- -DryRun +``` + +Or: + +```powershell +npm run harmonize:dry-run +``` + +### Orchestrator steps + +| Step | npm script | Purpose | +|------|------------|---------| +| `backupDev` | `dev:db:backup` | Safety snapshot | +| `backupProd` | `prod:db:backup` | Safety snapshot | +| `schema` | `harmonize:schema` | Apply dev migrations to prod (when `schemaChanged: true`) | +| `db` | `harmonize:db` | Row-level catalog merge | +| `images` | `harmonize:images` | Bidirectional file merge | +| `verify` | curl `/api/bounds` | Optional smoke check | + +Reports are written to `db/SyncReports/harmonize_db_*.json` and `harmonize_images_*.json` (gitignored). + +--- + +## Individual commands + +| Command | Purpose | +|---------|---------| +| `npm run harmonize:schema` | Migrate prod schema from dev migration files | +| `npm run harmonize:db` | Merge catalog rows by `updated_at` | +| `npm run harmonize:db -- --dry-run` | Preview DB changes | +| `npm run harmonize:db -- --prefer=dev` | On equal `updated_at`, dev wins | +| `npm run harmonize:images` | Merge image files by mtime | +| `npm run dev:migrate:sync-timestamps` | Apply `updated_at` migration on dev only | +| `npm run dev:backfill-updated-at` | Backfill dev `updated_at` from image mtimes | +| `npm run harmonize:backfill-updated-at` | Same backfill on prod | + +--- + +## Catalog tables synced + +Processed in FK order: + +`historical_eras` → `art_movements` → `artists` → `artist_periods` → `paintings` → `painting_influences` → `painting_influence_sources` → `painting_annotations` + +--- + +## Conflict handling + +Harmonize reports conflicts in the JSON report and skips those rows: + +| Conflict | Cause | Resolution | +|----------|-------|------------| +| `id_collision` | Same `id` but different natural key (e.g. artist name) | Manual fix in pgAdmin; environments diverged too far | +| `equal_updated_at` | Same timestamp, different row content | Re-run with `--prefer=dev` or `--prefer=prod`, or edit one side and re-run | + +**Tip:** Harmonize regularly from a shared baseline (e.g. after each weekly release) to avoid ID/natural-key collisions from independent inserts. + +--- + +## When to use what + +| Situation | Tool | +|-----------|------| +| Weekly release — prod should match dev exactly | `npm run devtoprod:release` | +| Mid-week prod curator fix + dev also changed | `npm run harmonize` | +| Dev workspace stale — full prod copy | `npm run prodto:dev:db` | +| New migration in repo | `harmonize:schema` or deploy step 4 | +| Only images changed on one side | `npm run harmonize:images` | +| Only DB metadata changed | `npm run harmonize:db` | + +--- + +## Example flows + +### Prod curator uploaded a painting; dev also edited metadata + +```powershell +npm run harmonize +``` + +DB rows merge by `updated_at`; image files merge by mtime. Both sides receive the latest version of each entity. + +### Schema change + mixed edits + +1. Finish and test on dev: `npm run dev:migrate` +2. Set `schemaChanged: true` in `harmonize.config.json` (or enable `steps.schema`) +3. `npm run harmonize` + +### Preview before writing + +```powershell +npm run harmonize:dry-run +# Review db/SyncReports/harmonize_*.json +npm run harmonize +``` + +--- + +## Safety + +- Pre-flight backups of dev and prod DB (configurable; on by default) +- Prod writes require `yes` or `CONFIRM_PROD=1` (orchestrator sets `CONFIRM_PROD=1` when `autoConfirm`-style run) +- No TRUNCATE — harmonize only inserts/updates changed rows +- Rollback: restore from `db/DataBackup/gallery_*_data_*.txt` using `dev:db:restore` or `devtoprod:db:restore` + +See also [environments.md](environments.md) and [deploy-dev-to-prod.md](deploy-dev-to-prod.md). diff --git a/Documentation/setup.md b/Documentation/setup.md index 7d2a79b..0168764 100644 --- a/Documentation/setup.md +++ b/Documentation/setup.md @@ -110,13 +110,14 @@ Image fetch can take hours if you run it for the entire catalog. The first line | `npm run prodto:dev:images` | Copy prod images → dev repo | | `npm run prodto:dev:db` | Clone `gallery_prod` → `gallery_dev` | | `npm run dev:db:backup` / `devtoprod:db:restore` | Dev backup / promote DB to prod | +| `npm run harmonize` | Bidirectional catalog + image merge (last-write-wins) — [harmonize-dev-prod.md](harmonize-dev-prod.md) | | `npm run prod:build` | Build production SPA into `client/dist` | | `npm run dev:start` | API + static SPA on `HOST`:`PORT` (uses root `.env`) | | `npm run prod:start` | Build client, then start server | | `npm run dev:server` | API with nodemon reload (local HMR workflow) | | `npm run dev:client` | Vite dev server on :5173 | -See [environments.md](environments.md) for dev/prod URLs, database split, Docker deploy, and sync commands. Quick reference: [FAC.md](FAC.md). +See [environments.md](environments.md) for dev/prod URLs, database split, Docker deploy, and sync commands. For incremental dev ↔ prod merge (both sides edited), see [harmonize-dev-prod.md](harmonize-dev-prod.md). Quick reference: [FAC.md](FAC.md). **Production frontend:** build the client, then start the server: diff --git a/db/migrate-sync-timestamps.sql b/db/migrate-sync-timestamps.sql new file mode 100644 index 0000000..e754886 --- /dev/null +++ b/db/migrate-sync-timestamps.sql @@ -0,0 +1,39 @@ +-- Catalog row timestamps for dev/prod harmonize (last-write-wins merge). +-- Idempotent: safe to re-run on existing databases. + +CREATE OR REPLACE FUNCTION sync_touch_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = now(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DO $$ +DECLARE + tbl TEXT; +BEGIN + FOREACH tbl IN ARRAY ARRAY[ + 'historical_eras', + 'art_movements', + 'artists', + 'artist_periods', + 'paintings', + 'painting_influences', + 'painting_influence_sources', + 'painting_annotations' + ] + LOOP + EXECUTE format( + 'ALTER TABLE %I ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now()', + tbl + ); + + EXECUTE format('DROP TRIGGER IF EXISTS %I ON %I', tbl || '_updated_at', tbl); + EXECUTE format( + 'CREATE TRIGGER %I BEFORE UPDATE ON %I FOR EACH ROW EXECUTE PROCEDURE sync_touch_updated_at()', + tbl || '_updated_at', + tbl + ); + END LOOP; +END $$; diff --git a/infra/deploy/harmonize.config.example.json b/infra/deploy/harmonize.config.example.json new file mode 100644 index 0000000..26cd60d --- /dev/null +++ b/infra/deploy/harmonize.config.example.json @@ -0,0 +1,27 @@ +{ + "_comment": "Copy to harmonize.config.json (gitignored) before running npm run harmonize", + "dryRun": false, + "prefer": null, + "schemaChanged": false, + "steps": { + "backupDev": true, + "backupProd": true, + "schema": false, + "db": true, + "images": true, + "verify": false + }, + "smb": { + "host": "192.168.10.122", + "share": "Gallery", + "user": "", + "password": "" + }, + "paths": { + "prodImages": "\\\\192.168.10.122\\Gallery\\data\\images" + }, + "verify": { + "devUrl": "https://devgallery.mysuperlab.netcraze.pro/api/bounds", + "publicUrl": "https://gallery.mysuperlab.netcraze.pro/api/bounds" + } +} diff --git a/infra/scripts/harmonize-images.ps1 b/infra/scripts/harmonize-images.ps1 new file mode 100644 index 0000000..37b8233 --- /dev/null +++ b/infra/scripts/harmonize-images.ps1 @@ -0,0 +1,27 @@ +# Map SMB share for harmonize image step (optional wrapper). +param( + [switch]$SkipConfirm +) + +$ErrorActionPreference = "Stop" +. (Join-Path $PSScriptRoot "lib\Deploy-CliResult.ps1") + +$smbRoot = "\\192.168.10.122\Gallery" +if ($env:SMB_USER -and $env:SMB_PASSWORD) { + Write-Host "Mapping $smbRoot ..." + net use $smbRoot /user:$env:SMB_USER $env:SMB_PASSWORD 2>&1 | Out-Host +} elseif (-not (Test-Path $smbRoot)) { + Write-Warning "Cannot access $smbRoot (not authenticated)." + Write-Warning "Map the share first or set `$env:SMB_USER and `$env:SMB_PASSWORD." +} + +$npmArgs = @('run', 'harmonize:images') +if ($args.Count -gt 0) { + $npmArgs += '--' + $npmArgs += $args +} + +$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path +Set-Location $RepoRoot +& npm @npmArgs +exit $LASTEXITCODE diff --git a/infra/scripts/harmonize.ps1 b/infra/scripts/harmonize.ps1 new file mode 100644 index 0000000..ea9dafe --- /dev/null +++ b/infra/scripts/harmonize.ps1 @@ -0,0 +1,234 @@ +# Dev <-> prod harmonize orchestrator (incremental merge; not full replace). +# +# Usage (from repo root): +# npm run harmonize +# npm run harmonize -- -DryRun +# npm run harmonize -- -Config infra/deploy/harmonize.config.json +# +# Requires infra/deploy/harmonize.config.json (copy from harmonize.config.example.json) +# or infra/deploy/devtoprod.config.json with optional "harmonize" section. + +param( + [string]$Config = (Join-Path $PSScriptRoot "..\deploy\harmonize.config.json"), + [switch]$DryRun +) + +$ErrorActionPreference = "Stop" +. (Join-Path $PSScriptRoot "lib\Deploy-CliResult.ps1") + +$RepoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..\..")).Path +Set-Location $RepoRoot + +$BannerWidth = 64 +$CompletedSteps = [System.Collections.Generic.List[string]]::new() +$FailedStep = $null + +function Write-HarmonizeBanner { + param([bool]$Success, [string]$FailedAt = '') + + Write-Host '' + Write-Host ('=' * $BannerWidth) + if ($Success) { + Write-Host ' HARMONIZE SUCCEEDED - dev and prod converged' + Write-Host (' Steps: ' + ($CompletedSteps -join ', ')) + } else { + Write-Host " HARMONIZE FAILED at step: $FailedAt" + if ($CompletedSteps.Count -gt 0) { + Write-Host (' Completed: ' + ($CompletedSteps -join ', ')) + } + } + Write-Host ('=' * $BannerWidth) +} + +function Invoke-Npm { + param([string]$Script, [string[]]$ExtraArgs = @()) + $npmArgs = @('run', $Script) + if ($ExtraArgs.Count -gt 0) { + $npmArgs += '--' + $npmArgs += $ExtraArgs + } + Write-Host ("Running npm: npm " + ($npmArgs -join ' ')) -ForegroundColor DarkGray + $prevEap = $ErrorActionPreference + $ErrorActionPreference = 'Continue' + $output = & npm @npmArgs 2>&1 + $exit = $LASTEXITCODE + $ErrorActionPreference = $prevEap + foreach ($line in $output) { Write-Host $line } + return $exit +} + +function Invoke-Step { + param([string]$Name, [scriptblock]$Action) + + if ($DryRun) { + Write-Host "[dry-run] Would run: $Name" + $script:CompletedSteps.Add($Name) + return $true + } + + Write-Host '' + Write-Host "--- Step: $Name ---" -ForegroundColor Cyan + try { + $code = & $Action + if ($null -eq $code) { $code = if ($null -ne $LASTEXITCODE) { $LASTEXITCODE } else { 0 } } + if ($code -ne 0) { + $script:FailedStep = $Name + return $false + } + $script:CompletedSteps.Add($Name) + return $true + } catch { + Write-Host $_.Exception.Message -ForegroundColor Red + $script:FailedStep = $Name + return $false + } +} + +function Resolve-ConfigPath { + if (Test-Path $Config) { return (Resolve-Path $Config).Path } + $fallback = Join-Path $PSScriptRoot "..\deploy\devtoprod.config.json" + if (Test-Path $fallback) { + Write-Host "Using fallback config: $fallback" -ForegroundColor DarkYellow + return (Resolve-Path $fallback).Path + } + throw "Missing harmonize config. Copy infra/deploy/harmonize.config.example.json to infra/deploy/harmonize.config.json" +} + +function Get-HarmonizeSteps { + param([object]$Raw) + + $harmonize = $Raw.harmonize + if ($null -eq $harmonize) { $harmonize = $Raw } + + $schemaChanged = $false + if ($null -ne $harmonize.schemaChanged) { $schemaChanged = [bool]$harmonize.schemaChanged } + elseif ($null -ne $Raw.schemaChanged) { $schemaChanged = [bool]$Raw.schemaChanged } + + $steps = @{ + backupDev = $true + backupProd = $true + schema = $schemaChanged + db = $true + images = $true + verify = $false + } + + $cfgSteps = $harmonize.steps + if ($null -ne $cfgSteps) { + foreach ($key in $cfgSteps.PSObject.Properties.Name) { + $steps[$key] = [bool]$cfgSteps.$key + } + } + + return $steps +} + +function Set-SmbCredentials { + param([object]$Smb) + + if ($Smb.user -and $Smb.password) { + $env:SMB_USER = [string]$Smb.user + $env:SMB_PASSWORD = [string]$Smb.password + } +} + +function Test-ApiUrl { + param([string]$Url) + if (-not $Url) { return $true } + try { + $resp = Invoke-WebRequest -Uri $Url -UseBasicParsing -TimeoutSec 20 + Write-Host " OK $Url ($($resp.StatusCode))" -ForegroundColor DarkGreen + return $true + } catch { + Write-Host " FAIL $Url : $($_.Exception.Message)" -ForegroundColor Red + return $false + } +} + +$configPath = Resolve-ConfigPath +$raw = Get-Content -LiteralPath $configPath -Raw | ConvertFrom-Json +$steps = Get-HarmonizeSteps -Raw $raw +$harmonize = if ($raw.harmonize) { $raw.harmonize } else { $raw } + +if ($DryRun -or ($harmonize.dryRun -eq $true)) { + $DryRun = $true + $env:CONFIRM_PROD = '1' +} + +Set-SmbCredentials -Smb $raw.smb +if ($harmonize.smb) { Set-SmbCredentials -Smb $harmonize.smb } + +$preferArg = @() +if ($harmonize.prefer) { + $preferArg = @("--prefer=$($harmonize.prefer)") +} + +$dryRunArgs = @() +if ($DryRun) { $dryRunArgs = @('--dry-run') } + +Write-Host "Harmonize config: $configPath" +Write-Host ("Steps enabled: " + (($steps.GetEnumerator() | Where-Object { $_.Value } | ForEach-Object { $_.Key }) -join ', ')) + +if (-not $DryRun) { + $env:CONFIRM_PROD = '1' +} + +if ($steps.backupDev) { + if (-not (Invoke-Step 'backupDev' { Invoke-Npm 'dev:db:backup' })) { + Write-HarmonizeBanner -Success $false -FailedAt 'backupDev' + Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at backupDev.' + exit 1 + } +} + +if ($steps.backupProd) { + if (-not (Invoke-Step 'backupProd' { Invoke-Npm 'prod:db:backup' })) { + Write-HarmonizeBanner -Success $false -FailedAt 'backupProd' + Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at backupProd.' + exit 1 + } +} + +if ($steps.schema) { + if (-not (Invoke-Step 'schema' { Invoke-Npm 'harmonize:schema' })) { + Write-HarmonizeBanner -Success $false -FailedAt 'schema' + Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at schema migrate.' + exit 1 + } +} + +if ($steps.db) { + if (-not (Invoke-Step 'db' { Invoke-Npm 'harmonize:db' @dryRunArgs @preferArg })) { + Write-HarmonizeBanner -Success $false -FailedAt 'db' + Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at DB harmonize.' + exit 1 + } +} + +if ($steps.images) { + if (-not (Invoke-Step 'images' { Invoke-Npm 'harmonize:images' @dryRunArgs })) { + Write-HarmonizeBanner -Success $false -FailedAt 'images' + Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Failed at image harmonize.' + exit 1 + } +} + +if ($steps.verify -and -not $DryRun) { + $verify = $harmonize.verify + if (-not $verify) { $verify = $raw.verify } + $ok = $true + if (-not (Invoke-Step 'verify' { + if ($verify.devUrl -and -not (Test-ApiUrl $verify.devUrl)) { return 1 } + if ($verify.publicUrl -and -not (Test-ApiUrl $verify.publicUrl)) { return 1 } + return 0 + })) { + Write-HarmonizeBanner -Success $false -FailedAt 'verify' + Write-DeployCliResult -Script 'harmonize' -Success $false -Summary 'Verify step failed.' + exit 1 + } +} + +Write-HarmonizeBanner -Success $true +$summary = if ($DryRun) { 'Harmonize dry-run complete (no changes written).' } else { 'Harmonize complete.' } +Write-DeployCliResult -Script 'harmonize' -Success $true -Summary $summary -Details @("Config: $configPath") +exit 0 diff --git a/package.json b/package.json index 966e8bf..fa2c023 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,8 @@ "dev:migrate:painting-annotations": "node scripts/migrate-painting-annotations.js", "dev:migrate:artist-palette": "node scripts/migrate-artist-palette.js", "dev:migrate:search": "node scripts/migrate-search.js", + "dev:migrate:sync-timestamps": "node scripts/migrate-sync-timestamps.js", + "dev:backfill-updated-at": "node scripts/backfill-updated-at.js", "dev:import-painter-palette": "node scripts/import-painter-palette.js", "dev:analyze-painter-palette": "node scripts/analyze-painter-palette.js", "dev:export-paintings": "node scripts/export-paintings-csv.js", @@ -48,6 +50,12 @@ "devtoprod:release": "powershell -NoProfile -ExecutionPolicy Bypass -File infra/scripts/deploy-dev-to-prod.ps1", "prodto:dev:db": "node scripts/sync-prod-to-dev.js", "prodto:dev:images": "powershell -NoProfile -ExecutionPolicy Bypass -File infra/scripts/sync-images-from-prod.ps1", + "harmonize:schema": "node scripts/harmonize-schema.js", + "harmonize:db": "node scripts/harmonize-db.js", + "harmonize:images": "node scripts/harmonize-images.js", + "harmonize:backfill-updated-at": "node scripts/backfill-updated-at.js --prod", + "harmonize:dry-run": "node scripts/harmonize-db.js --dry-run && node scripts/harmonize-images.js --dry-run", + "harmonize": "powershell -NoProfile -ExecutionPolicy Bypass -File infra/scripts/harmonize.ps1", "infra:db:split-dev-prod": "node scripts/split-dev-prod-databases.js" }, "keywords": [], diff --git a/scripts/backfill-updated-at.js b/scripts/backfill-updated-at.js new file mode 100644 index 0000000..5e3c421 --- /dev/null +++ b/scripts/backfill-updated-at.js @@ -0,0 +1,151 @@ +/** + * Backfill catalog updated_at from linked image file mtimes where possible. + * + * Dev: npm run dev:backfill-updated-at + * Prod: npm run harmonize:backfill-updated-at + */ +const fs = require('fs'); +const path = require('path'); +const pg = require('pg'); +const { + assertDevDatabase, + assertProdDatabase, + loadDevPgConfig, + loadProdPgConfig, + rootDir, +} = require('./db-env'); +const { printCliResult } = require('./lib/cli-result'); + +const { Client } = pg; + +function parseArgs(argv) { + return { prod: argv.includes('--prod') }; +} + +function resolveImageDir(prod) { + if (prod) { + const envPath = path.join(rootDir, 'infra', 'docker', '.env.prod'); + if (fs.existsSync(envPath)) { + for (const line of fs.readFileSync(envPath, 'utf8').split(/\r?\n/)) { + const trimmed = line.trim(); + if (trimmed.startsWith('IMAGE_DIR=')) { + const val = trimmed.slice('IMAGE_DIR='.length).trim().replace(/^["']|["']$/g, ''); + if (val) return path.resolve(val); + } + } + } + return path.resolve('\\\\192.168.10.122\\Gallery\\data\\images'); + } + require('dotenv').config({ path: path.join(rootDir, '.env') }); + return path.resolve(process.env.IMAGE_DIR || path.join(rootDir, 'data', 'images')); +} + +function fileMtimeMs(imageDir, relPath) { + if (!relPath || typeof relPath !== 'string') return null; + const abs = path.join(imageDir, relPath.replace(/^\//, '')); + try { + if (!fs.existsSync(abs)) return null; + return fs.statSync(abs).mtimeMs; + } catch { + return null; + } +} + +function maxMtime(imageDir, paths) { + let max = null; + for (const rel of paths) { + const ms = fileMtimeMs(imageDir, rel); + if (ms != null && (max == null || ms > max)) max = ms; + } + return max; +} + +async function main() { + const { prod } = parseArgs(process.argv.slice(2)); + const config = prod ? loadProdPgConfig() : loadDevPgConfig(); + if (prod) assertProdDatabase(config.database); + else assertDevDatabase(config.database); + + const imageDir = resolveImageDir(prod); + const client = new Client(config); + await client.connect(); + + const fallback = new Date(); + let updated = 0; + + const { rows: artists } = await client.query( + 'SELECT id, portrait_path, portrait_thumb_path FROM artists', + ); + for (const row of artists) { + const ms = maxMtime(imageDir, [row.portrait_path, row.portrait_thumb_path]); + const ts = ms != null ? new Date(ms) : fallback; + await client.query('UPDATE artists SET updated_at = $1 WHERE id = $2', [ts, row.id]); + updated += 1; + } + + const { rows: paintings } = await client.query( + 'SELECT id, image_path, thumbnail_path FROM paintings', + ); + for (const row of paintings) { + const ms = maxMtime(imageDir, [row.image_path, row.thumbnail_path]); + const ts = ms != null ? new Date(ms) : fallback; + await client.query('UPDATE paintings SET updated_at = $1 WHERE id = $2', [ts, row.id]); + updated += 1; + } + + const childUpdates = [ + ['artist_periods', 'artist_id', 'artists'], + ['painting_influences', 'painting_id', 'paintings'], + ['painting_influence_sources', 'painting_id', 'paintings'], + ['painting_annotations', 'painting_id', 'paintings'], + ]; + + for (const [child, fk, parent] of childUpdates) { + const res = await client.query( + `UPDATE ${child} c + SET updated_at = GREATEST(c.updated_at, p.updated_at) + FROM ${parent} p + WHERE c.${fk} = p.id`, + ); + updated += res.rowCount || 0; + } + + for (const table of ['historical_eras', 'art_movements']) { + const res = await client.query( + `UPDATE ${table} SET updated_at = $1 WHERE updated_at IS NOT NULL`, + [fallback], + ); + updated += res.rowCount || 0; + } + + await client.query( + `UPDATE art_movements m + SET updated_at = GREATEST(m.updated_at, e.updated_at) + FROM historical_eras e + WHERE m.era_id = e.id`, + ); + + await client.query( + `UPDATE artists a + SET updated_at = GREATEST(a.updated_at, m.updated_at) + FROM art_movements m + WHERE a.movement_id = m.id`, + ); + + await client.end(); + + printCliResult({ + script: 'backfill-updated-at', + ok: true, + summary: `Backfilled updated_at on ${config.database} (${updated} row touches).`, + details: [`Image dir: ${imageDir}`], + }); +} + +main().catch((err) => { + printCliResult({ + script: 'backfill-updated-at', + ok: false, + summary: err.message, + }); +}); diff --git a/scripts/harmonize-db.js b/scripts/harmonize-db.js new file mode 100644 index 0000000..6ff37e7 --- /dev/null +++ b/scripts/harmonize-db.js @@ -0,0 +1,304 @@ +/** + * Bidirectional catalog DB harmonize (dev <-> prod) by updated_at. + * + * Usage: + * npm run harmonize:db + * npm run harmonize:db -- --dry-run + * npm run harmonize:db -- --prefer=dev + */ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const pg = require('pg'); +const { + loadDevPgConfig, + loadProdPgConfig, + confirmProdAction, + rootDir, +} = require('./db-env'); +const { printCliResult } = require('./lib/cli-result'); + +const { Client } = pg; + +const CATALOG_TABLES = [ + 'historical_eras', + 'art_movements', + 'artists', + 'artist_periods', + 'paintings', + 'painting_influences', + 'painting_influence_sources', + 'painting_annotations', +]; + +const NATURAL_KEY_FN = { + historical_eras: (r) => String(r.name || '').trim().toLowerCase(), + art_movements: (r) => String(r.name || '').trim().toLowerCase(), + artists: (r) => String(r.name || '').trim().toLowerCase(), + artist_periods: (r) => `${r.artist_id}:${String(r.name || '').trim().toLowerCase()}`, + paintings: (r) => `${r.artist_id}:${String(r.title || '').trim().toLowerCase()}`, + painting_influences: (r) => `${r.painting_id}:${r.influenced_by_painting_id}`, + painting_influence_sources: (r) => `${r.painting_id}:${r.source_type}:${r.source_painting_id || 0}:${r.source_artist_id || 0}:${r.source_movement_id || 0}`, + painting_annotations: (r) => `${r.painting_id}:${String(r.label || '').trim().toLowerCase()}:${r.sort_order}`, +}; + +function parseArgs(argv) { + const tablesArg = argv.find((a) => a.startsWith('--tables=')); + const preferArg = argv.find((a) => a.startsWith('--prefer=')); + return { + dryRun: argv.includes('--dry-run'), + verbose: argv.includes('--verbose'), + tables: tablesArg ? tablesArg.slice('--tables='.length).split(',').map((t) => t.trim()).filter(Boolean) : null, + prefer: preferArg ? preferArg.slice('--prefer='.length).trim().toLowerCase() : null, + }; +} + +function stableRowHash(row, columns) { + const payload = {}; + for (const col of columns) { + const val = row[col]; + if (val instanceof Date) payload[col] = val.toISOString(); + else if (val != null && typeof val === 'object') payload[col] = JSON.stringify(val); + else payload[col] = val; + } + return crypto.createHash('sha256').update(JSON.stringify(payload)).digest('hex'); +} + +function compareUpdatedAt(a, b) { + const ta = a?.updated_at ? new Date(a.updated_at).getTime() : 0; + const tb = b?.updated_at ? new Date(b.updated_at).getTime() : 0; + if (ta === tb) return 0; + return ta > tb ? 1 : -1; +} + +async function getTableColumns(client, tableName) { + const { rows } = await client.query( + `SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = $1 + ORDER BY ordinal_position`, + [tableName], + ); + return rows.map((r) => r.column_name); +} + +async function fetchTableRows(client, tableName) { + const { rows } = await client.query(`SELECT * FROM ${tableName} ORDER BY id`); + const map = new Map(); + for (const row of rows) map.set(row.id, row); + return map; +} + +function buildInsertSql(tableName, columns) { + const cols = columns.map((c) => `"${c}"`).join(', '); + const placeholders = columns.map((_, i) => `$${i + 1}`).join(', '); + return `INSERT INTO ${tableName} (${cols}) OVERRIDING SYSTEM VALUE VALUES (${placeholders})`; +} + +function buildUpsertSql(tableName, columns) { + const insertCols = columns.map((c) => `"${c}"`).join(', '); + const placeholders = columns.map((_, i) => `$${i + 1}`).join(', '); + const updates = columns + .filter((c) => c !== 'id') + .map((c) => `"${c}" = EXCLUDED."${c}"`) + .join(', '); + return `INSERT INTO ${tableName} (${insertCols}) OVERRIDING SYSTEM VALUE VALUES (${placeholders}) + ON CONFLICT (id) DO UPDATE SET ${updates} + WHERE ${tableName}.updated_at < EXCLUDED.updated_at`; +} + +function rowValues(row, columns) { + return columns.map((col) => row[col]); +} + +async function insertRow(client, tableName, columns, row, dryRun) { + if (dryRun) return; + const sql = buildInsertSql(tableName, columns); + await client.query(sql, rowValues(row, columns)); +} + +async function upsertRow(client, tableName, columns, row, dryRun) { + if (dryRun) return; + const sql = buildUpsertSql(tableName, columns); + await client.query(sql, rowValues(row, columns)); +} + +async function fixSequence(client, tableName) { + await client.query( + `SELECT setval(pg_get_serial_sequence($1, 'id'), COALESCE((SELECT MAX(id) FROM ${tableName}), 1), true)`, + [tableName], + ); +} + +async function harmonizeTable(tableName, devClient, prodClient, options, stats) { + const naturalKeyFn = NATURAL_KEY_FN[tableName]; + const columns = await getTableColumns(devClient, tableName); + if (!columns.includes('updated_at')) { + throw new Error(`Table ${tableName} missing updated_at — run migrate-sync-timestamps first`); + } + + const devRows = await fetchTableRows(devClient, tableName); + const prodRows = await fetchTableRows(prodClient, tableName); + const allIds = new Set([...devRows.keys(), ...prodRows.keys()]); + + for (const id of allIds) { + const devRow = devRows.get(id); + const prodRow = prodRows.get(id); + + if (devRow && !prodRow) { + stats.devToProd += 1; + stats.actions.push({ table: tableName, id, action: 'dev→prod insert' }); + await insertRow(prodClient, tableName, columns, devRow, options.dryRun); + continue; + } + + if (prodRow && !devRow) { + stats.prodToDev += 1; + stats.actions.push({ table: tableName, id, action: 'prod→dev insert' }); + await insertRow(devClient, tableName, columns, prodRow, options.dryRun); + continue; + } + + const devKey = naturalKeyFn(devRow); + const prodKey = naturalKeyFn(prodRow); + if (devKey !== prodKey) { + stats.conflicts += 1; + stats.conflictDetails.push({ + table: tableName, + id, + reason: 'id_collision', + devKey, + prodKey, + }); + continue; + } + + const devHash = stableRowHash(devRow, columns); + const prodHash = stableRowHash(prodRow, columns); + if (devHash === prodHash) { + stats.skipped += 1; + continue; + } + + const cmp = compareUpdatedAt(devRow, prodRow); + if (cmp > 0) { + stats.devToProd += 1; + stats.actions.push({ table: tableName, id, action: 'dev→prod update' }); + await upsertRow(prodClient, tableName, columns, devRow, options.dryRun); + } else if (cmp < 0) { + stats.prodToDev += 1; + stats.actions.push({ table: tableName, id, action: 'prod→dev update' }); + await upsertRow(devClient, tableName, columns, prodRow, options.dryRun); + } else if (options.prefer === 'dev') { + stats.devToProd += 1; + stats.actions.push({ table: tableName, id, action: 'dev→prod update (prefer tie)' }); + await upsertRow(prodClient, tableName, columns, devRow, options.dryRun); + } else if (options.prefer === 'prod') { + stats.prodToDev += 1; + stats.actions.push({ table: tableName, id, action: 'prod→dev update (prefer tie)' }); + await upsertRow(devClient, tableName, columns, prodRow, options.dryRun); + } else { + stats.conflicts += 1; + stats.conflictDetails.push({ + table: tableName, + id, + reason: 'equal_updated_at', + devUpdatedAt: devRow.updated_at, + prodUpdatedAt: prodRow.updated_at, + }); + } + } + + if (!options.dryRun) { + await fixSequence(devClient, tableName); + await fixSequence(prodClient, tableName); + } +} + +function timestampSlug(date = new Date()) { + const pad = (n) => String(n).padStart(2, '0'); + return [ + date.getFullYear(), + pad(date.getMonth() + 1), + pad(date.getDate()), + '_', + pad(date.getHours()), + pad(date.getMinutes()), + pad(date.getSeconds()), + ].join(''); +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const tables = options.tables || CATALOG_TABLES; + + for (const table of tables) { + if (!CATALOG_TABLES.includes(table)) { + throw new Error(`Unknown catalog table: ${table}`); + } + } + + if (!options.dryRun && process.env.CONFIRM_PROD !== '1') { + await confirmProdAction( + 'Harmonize will merge catalog rows between gallery_dev and gallery_prod (prod may be modified).', + ); + } + + const devClient = new Client(loadDevPgConfig()); + const prodClient = new Client(loadProdPgConfig()); + await devClient.connect(); + await prodClient.connect(); + + const stats = { + devToProd: 0, + prodToDev: 0, + skipped: 0, + conflicts: 0, + actions: [], + conflictDetails: [], + }; + + try { + for (const table of tables) { + if (options.verbose) console.log(`Harmonizing ${table}…`); + await harmonizeTable(table, devClient, prodClient, options, stats); + } + } finally { + await devClient.end(); + await prodClient.end(); + } + + const reportDir = path.join(rootDir, 'db', 'SyncReports'); + fs.mkdirSync(reportDir, { recursive: true }); + const reportPath = path.join(reportDir, `harmonize_db_${timestampSlug()}.json`); + fs.writeFileSync(reportPath, JSON.stringify({ + generatedAt: new Date().toISOString(), + dryRun: options.dryRun, + prefer: options.prefer, + tables, + stats: { + devToProd: stats.devToProd, + prodToDev: stats.prodToDev, + skipped: stats.skipped, + conflicts: stats.conflicts, + }, + conflicts: stats.conflictDetails, + actions: options.verbose ? stats.actions : stats.actions.slice(0, 200), + }, null, 2)); + + const mode = options.dryRun ? ' (dry-run)' : ''; + printCliResult({ + script: 'harmonize-db', + ok: true, + summary: `DB harmonize complete${mode}: dev→prod ${stats.devToProd}, prod→dev ${stats.prodToDev}, skipped ${stats.skipped}, conflicts ${stats.conflicts}.`, + details: [`Report: ${reportPath}`], + }); +} + +main().catch((err) => { + printCliResult({ + script: 'harmonize-db', + ok: false, + summary: err.message, + }); +}); diff --git a/scripts/harmonize-images.js b/scripts/harmonize-images.js new file mode 100644 index 0000000..bba57ef --- /dev/null +++ b/scripts/harmonize-images.js @@ -0,0 +1,196 @@ +/** + * Bidirectional image harmonize (dev <-> prod) by file mtime. + * + * Usage: + * npm run harmonize:images + * npm run harmonize:images -- --dry-run + */ +const fs = require('fs'); +const path = require('path'); +const { rootDir, confirmProdAction } = require('./db-env'); +const { loadHarmonizeConfig, prodImagesPath } = require('./lib/harmonize-config'); +const { printCliResult } = require('./lib/cli-result'); + +const IMAGE_SUBDIRS = ['portraits', 'paintings']; +const MTIME_TOLERANCE_MS = 1000; + +function parseArgs(argv) { + const configArg = argv.find((a) => a.startsWith('--config=')); + return { + dryRun: argv.includes('--dry-run'), + verbose: argv.includes('--verbose'), + configPath: configArg ? configArg.slice('--config='.length) : null, + }; +} + +function resolveDevImageDir() { + require('dotenv').config({ path: path.join(rootDir, '.env') }); + return path.resolve(process.env.IMAGE_DIR || path.join(rootDir, 'data', 'images')); +} + +function walkImageFiles(rootDir, baseRel = '') { + const files = []; + const abs = path.join(rootDir, baseRel); + if (!fs.existsSync(abs)) return files; + + for (const entry of fs.readdirSync(abs, { withFileTypes: true })) { + const rel = baseRel ? path.join(baseRel, entry.name).replace(/\\/g, '/') : entry.name; + if (entry.isDirectory()) { + files.push(...walkImageFiles(rootDir, rel)); + } else if (entry.isFile()) { + files.push(rel); + } + } + return files; +} + +function collectRelativePaths(imageRoot) { + const relPaths = new Set(); + for (const sub of IMAGE_SUBDIRS) { + const subRoot = path.join(imageRoot, sub); + if (!fs.existsSync(subRoot)) continue; + for (const rel of walkImageFiles(subRoot, sub)) { + relPaths.add(rel.replace(/\\/g, '/')); + } + } + return relPaths; +} + +function statFile(imageRoot, relPath) { + const abs = path.join(imageRoot, relPath); + try { + if (!fs.existsSync(abs)) return null; + const st = fs.statSync(abs); + if (!st.isFile()) return null; + return { size: st.size, mtimeMs: st.mtimeMs }; + } catch { + return null; + } +} + +function ensureDirForFile(filePath) { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); +} + +function copyFile(src, dest, dryRun) { + if (dryRun) return; + ensureDirForFile(dest); + fs.copyFileSync(src, dest); +} + +function timestampSlug(date = new Date()) { + const pad = (n) => String(n).padStart(2, '0'); + return [ + date.getFullYear(), + pad(date.getMonth() + 1), + pad(date.getDate()), + '_', + pad(date.getHours()), + pad(date.getMinutes()), + pad(date.getSeconds()), + ].join(''); +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const config = loadHarmonizeConfig(options.configPath); + const dryRun = options.dryRun || config.dryRun; + + const devDir = resolveDevImageDir(); + const prodDir = prodImagesPath(config); + + if (!fs.existsSync(prodDir)) { + throw new Error( + `Prod image path not reachable: ${prodDir}. Map SMB share or set paths.prodImages in harmonize config.`, + ); + } + + if (!dryRun && process.env.CONFIRM_PROD !== '1') { + await confirmProdAction( + 'Harmonize will copy image files between dev and prod (prod files may be overwritten).', + ); + } + + const devPaths = collectRelativePaths(devDir); + const prodPaths = collectRelativePaths(prodDir); + const allPaths = new Set([...devPaths, ...prodPaths]); + + const stats = { + devToProd: 0, + prodToDev: 0, + skipped: 0, + actions: [], + }; + + for (const rel of allPaths) { + const devStat = statFile(devDir, rel); + const prodStat = statFile(prodDir, rel); + + if (devStat && !prodStat) { + stats.devToProd += 1; + stats.actions.push({ path: rel, action: 'dev→prod copy' }); + copyFile(path.join(devDir, rel), path.join(prodDir, rel), dryRun); + continue; + } + + if (prodStat && !devStat) { + stats.prodToDev += 1; + stats.actions.push({ path: rel, action: 'prod→dev copy' }); + copyFile(path.join(prodDir, rel), path.join(devDir, rel), dryRun); + continue; + } + + if (!devStat || !prodStat) continue; + + const sameSize = devStat.size === prodStat.size; + const mtimeClose = Math.abs(devStat.mtimeMs - prodStat.mtimeMs) <= MTIME_TOLERANCE_MS; + if (sameSize && mtimeClose) { + stats.skipped += 1; + continue; + } + + if (devStat.mtimeMs > prodStat.mtimeMs) { + stats.devToProd += 1; + stats.actions.push({ path: rel, action: 'dev→prod update' }); + copyFile(path.join(devDir, rel), path.join(prodDir, rel), dryRun); + } else if (prodStat.mtimeMs > devStat.mtimeMs) { + stats.prodToDev += 1; + stats.actions.push({ path: rel, action: 'prod→dev update' }); + copyFile(path.join(prodDir, rel), path.join(devDir, rel), dryRun); + } else { + stats.skipped += 1; + } + } + + const reportDir = path.join(rootDir, 'db', 'SyncReports'); + fs.mkdirSync(reportDir, { recursive: true }); + const reportPath = path.join(reportDir, `harmonize_images_${timestampSlug()}.json`); + fs.writeFileSync(reportPath, JSON.stringify({ + generatedAt: new Date().toISOString(), + dryRun, + devDir, + prodDir, + stats: { + devToProd: stats.devToProd, + prodToDev: stats.prodToDev, + skipped: stats.skipped, + }, + actions: options.verbose ? stats.actions : stats.actions.slice(0, 500), + }, null, 2)); + + const mode = dryRun ? ' (dry-run)' : ''; + printCliResult({ + script: 'harmonize-images', + ok: true, + summary: `Image harmonize complete${mode}: dev→prod ${stats.devToProd}, prod→dev ${stats.prodToDev}, skipped ${stats.skipped}.`, + details: [`Dev: ${devDir}`, `Prod: ${prodDir}`, `Report: ${reportPath}`], + }); +} + +main().catch((err) => { + printCliResult({ + script: 'harmonize-images', + ok: false, + summary: err.message, + }); +}); diff --git a/scripts/harmonize-schema.js b/scripts/harmonize-schema.js new file mode 100644 index 0000000..7516b63 --- /dev/null +++ b/scripts/harmonize-schema.js @@ -0,0 +1,54 @@ +/** + * Migrate prod schema from dev migrations (schema dev → prod only). + * + * Usage: npm run harmonize:schema + */ +const { spawnSync } = require('child_process'); +const path = require('path'); +const { rootDir, loadProdPgConfig, confirmProdAction } = require('./db-env'); +const { printCliResult } = require('./lib/cli-result'); + +async function main() { + if (process.env.CONFIRM_PROD !== '1') { + await confirmProdAction('Apply dev migrations to gallery_prod schema?'); + } + + const prodConfig = loadProdPgConfig(); + const env = { + ...process.env, + DB_HOST: prodConfig.host, + DB_PORT: String(prodConfig.port), + DB_USER: prodConfig.user, + DB_PASSWORD: prodConfig.password, + DB_NAME: prodConfig.database, + }; + + const result = spawnSync(process.execPath, [path.join(rootDir, 'server', 'migrate.js')], { + cwd: rootDir, + env, + stdio: 'inherit', + }); + + if (result.status !== 0) { + printCliResult({ + script: 'harmonize-schema', + ok: false, + summary: 'Prod schema migration failed.', + }); + return; + } + + printCliResult({ + script: 'harmonize-schema', + ok: true, + summary: 'Prod schema migrated from dev migration files (gallery_prod).', + }); +} + +main().catch((err) => { + printCliResult({ + script: 'harmonize-schema', + ok: false, + summary: err.message, + }); +}); diff --git a/scripts/lib/harmonize-config.js b/scripts/lib/harmonize-config.js new file mode 100644 index 0000000..22bec3c --- /dev/null +++ b/scripts/lib/harmonize-config.js @@ -0,0 +1,78 @@ +const fs = require('fs'); +const path = require('path'); +const { rootDir } = require('./db-env'); + +const DEFAULT_CONFIG_PATH = path.join(rootDir, 'infra', 'deploy', 'harmonize.config.json'); +const FALLBACK_CONFIG_PATH = path.join(rootDir, 'infra', 'deploy', 'devtoprod.config.json'); + +function loadHarmonizeConfig(configPath) { + const resolved = configPath + ? path.resolve(configPath) + : (fs.existsSync(DEFAULT_CONFIG_PATH) ? DEFAULT_CONFIG_PATH : FALLBACK_CONFIG_PATH); + + if (!fs.existsSync(resolved)) { + return { + configPath: resolved, + dryRun: false, + prefer: null, + schemaChanged: false, + steps: { + backupDev: true, + backupProd: true, + schema: false, + db: true, + images: true, + verify: false, + }, + smb: { + host: '192.168.10.122', + share: 'Gallery', + user: '', + password: '', + }, + paths: { + prodImages: '\\\\192.168.10.122\\Gallery\\data\\images', + }, + verify: {}, + }; + } + + const raw = JSON.parse(fs.readFileSync(resolved, 'utf8')); + const harmonize = raw.harmonize || raw; + return { + configPath: resolved, + dryRun: Boolean(harmonize.dryRun), + prefer: harmonize.prefer || null, + schemaChanged: Boolean(harmonize.schemaChanged ?? raw.schemaChanged), + steps: { + backupDev: harmonize.steps?.backupDev ?? true, + backupProd: harmonize.steps?.backupProd ?? true, + schema: harmonize.steps?.schema ?? Boolean(harmonize.schemaChanged ?? raw.schemaChanged), + db: harmonize.steps?.db ?? true, + images: harmonize.steps?.images ?? true, + verify: harmonize.steps?.verify ?? false, + }, + smb: { + host: harmonize.smb?.host ?? raw.smb?.host ?? '192.168.10.122', + share: harmonize.smb?.share ?? raw.smb?.share ?? 'Gallery', + user: harmonize.smb?.user ?? raw.smb?.user ?? '', + password: harmonize.smb?.password ?? raw.smb?.password ?? '', + }, + paths: { + prodImages: harmonize.paths?.prodImages + ?? `\\\\${harmonize.smb?.host ?? raw.smb?.host ?? '192.168.10.122'}\\${harmonize.smb?.share ?? raw.smb?.share ?? 'Gallery'}\\data\\images`, + }, + verify: harmonize.verify ?? raw.verify ?? {}, + }; +} + +function prodImagesPath(config) { + return config.paths?.prodImages + || `\\\\${config.smb.host}\\${config.smb.share}\\data\\images`; +} + +module.exports = { + DEFAULT_CONFIG_PATH, + loadHarmonizeConfig, + prodImagesPath, +}; diff --git a/scripts/migrate-sync-timestamps.js b/scripts/migrate-sync-timestamps.js new file mode 100644 index 0000000..5ce9468 --- /dev/null +++ b/scripts/migrate-sync-timestamps.js @@ -0,0 +1,17 @@ +require('dotenv').config(); +const fs = require('fs'); +const path = require('path'); +const pool = require('../server/db'); + +async function main() { + const sqlPath = path.join(__dirname, '../db/migrate-sync-timestamps.sql'); + const sql = fs.readFileSync(sqlPath, 'utf8'); + await pool.query(sql); + console.log('Catalog sync timestamps ready (updated_at + triggers)'); + await pool.end(); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/server/migrate.js b/server/migrate.js index 3c8e0c6..95ef49c 100644 --- a/server/migrate.js +++ b/server/migrate.js @@ -13,6 +13,7 @@ const INCREMENTAL_MIGRATIONS = [ 'migrate-portrait-thumbs.sql', 'migrate-perf-indexes.sql', 'migrate-search.sql', + 'migrate-sync-timestamps.sql', ]; async function bootstrapCurator() {