From 89e39d408afbb610533ccc3f454efab63eff5c9e Mon Sep 17 00:00:00 2001 From: Danila Khodjaef Date: Mon, 27 Jul 2026 12:23:25 +0300 Subject: [PATCH] Fix harmonize:images thumb rebuild and allow oversized museum scans. Co-authored-by: Cursor --- Documentation/FAC.md | 2 +- Documentation/data-and-images.md | 4 +- Documentation/environments.md | 2 +- Documentation/harmonize-dev-prod.md | 6 ++- scripts/harmonize-images.js | 79 +++++++++++++++++++++++++++-- scripts/image-fetcher.js | 8 ++- scripts/lib/harmonize-config.js | 2 +- scripts/regenerate-thumbnails.js | 2 +- 8 files changed, 92 insertions(+), 13 deletions(-) diff --git a/Documentation/FAC.md b/Documentation/FAC.md index c9c7363..1fcd0f4 100644 --- a/Documentation/FAC.md +++ b/Documentation/FAC.md @@ -143,7 +143,7 @@ Run `npm run dev:migrate` against prod DB after first deploy with auth vars set | `npm run devtoprod:db:restore -- --file ` | Restore into **prod** (requires confirmation) | | `npm run harmonize` | Bidirectional catalog DB + image merge by `updated_at` / file mtime — [harmonize-dev-prod.md](harmonize-dev-prod.md) | | `npm run harmonize:schema` | Apply dev migrations to prod schema only (dev → prod) | -| `npm run harmonize:db` / `harmonize:images` | DB or image merge only | +| `npm run harmonize:db` / `harmonize:images` | DB or image merge only (`harmonize:images` also regenerates painting + portrait thumbs on both sides) | | `npm run dev:backfill-updated-at` | Backfill catalog `updated_at` from image mtimes (dev) | **One-time split (recommended):** pgAdmin on dev PC → open [`db/split-dev-prod-pgadmin.sql`](../db/split-dev-prod-pgadmin.sql) → run each STEP on database `postgres`, then verify on `gallery_dev`. diff --git a/Documentation/data-and-images.md b/Documentation/data-and-images.md index 794ad3f..69245d3 100644 --- a/Documentation/data-and-images.md +++ b/Documentation/data-and-images.md @@ -31,7 +31,7 @@ File names are sanitised `{Artist}_{Title}.{ext}`. The image service can redisco | **Development** | `./data/images/` in repo | Working copy on dev PC | | **Production** | `/mnt/BasePool/Applications/Gallery/data/images` on TrueNAS | SMB `\\192.168.10.122\Gallery\data\images` | -**One-direction promote:** `npm run devtoprod:images` (dev → prod, skip older). **Refresh all images on dev from prod:** `npm run prodto:dev:images`. **One artist only:** map the SMB share, then `robocopy` that artist’s `ArtistName*` files under `paintings/` and `paintings/thumbs/` (see [FAC.md](FAC.md#dev--prod-image-sync-smb)). **Bidirectional merge** (newer file wins): `npm run harmonize:images` or full `npm run harmonize` — see [harmonize-dev-prod.md](harmonize-dev-prod.md). General sync reference: [environments.md](environments.md). +**One-direction promote:** `npm run devtoprod:images` (dev → prod, skip older). **Refresh all images on dev from prod:** `npm run prodto:dev:images`. **One artist only:** map the SMB share, then `robocopy` that artist’s `ArtistName*` files under `paintings/` and `paintings/thumbs/` (see [FAC.md](FAC.md#dev--prod-image-sync-smb)). **Bidirectional merge** (newer file wins, then rebuild painting + portrait thumbs on both sides): `npm run harmonize:images` or full `npm run harmonize` — see [harmonize-dev-prod.md](harmonize-dev-prod.md). General sync reference: [environments.md](environments.md). ## Scripts overview @@ -49,7 +49,7 @@ File names are sanitised `{Artist}_{Title}.{ext}`. The image service can redisco | `fetch-missing-images.js` | `npm run dev:fetch-images` | Downloads files for paintings missing on disk | | `image-fetcher.js` | *(library)* | Wikimedia / museum resolution used by fetch scripts and API | | `sync-images-to-prod.ps1` / `sync-images-from-prod.ps1` | `npm run devtoprod:images` / `npm run prodto:dev:images` | Robocopy via SMB `\\192.168.10.122\Gallery` | -| `harmonize-db.js` / `harmonize-images.js` | `npm run harmonize:db` / `harmonize:images` | Bidirectional merge by `updated_at` / file mtime | +| `harmonize-db.js` / `harmonize-images.js` | `npm run harmonize:db` / `harmonize:images` | Bidirectional merge by `updated_at` / file mtime; images step then rebuilds thumbs on both sides | | `harmonize.ps1` | `npm run harmonize` | Orchestrator: backups, optional schema, DB + image merge | | `regenerate-thumbnails.js` | `npm run dev:regenerate-thumbnails` | Rebuild painting thumbs from full images via `sharp` | | `regenerate-portrait-thumbs.js` | `npm run dev:regenerate-portrait-thumbs` | Rebuild timeline portrait thumbs (~256px) and set `portrait_thumb_path` | diff --git a/Documentation/environments.md b/Documentation/environments.md index 670e9dc..cc93548 100644 --- a/Documentation/environments.md +++ b/Documentation/environments.md @@ -341,7 +341,7 @@ For **incremental** dev ↔ prod merge (both sides edited), use [harmonize-dev-p | `npm run devtoprod:db:restore` | Dev PC PowerShell | Restore into prod | | `npm run harmonize` | Dev PC PowerShell | Bidirectional catalog + image merge — [harmonize-dev-prod.md](harmonize-dev-prod.md) | | `npm run harmonize:db` | Dev PC PowerShell | DB merge only | -| `npm run harmonize:images` | Dev PC PowerShell | Image merge only | +| `npm run harmonize:images` | Dev PC PowerShell | Image merge + regenerate thumbs on both sides | ## Image sync diff --git a/Documentation/harmonize-dev-prod.md b/Documentation/harmonize-dev-prod.md index 43c5ff2..3d7c8ac 100644 --- a/Documentation/harmonize-dev-prod.md +++ b/Documentation/harmonize-dev-prod.md @@ -83,7 +83,7 @@ npm run harmonize:dry-run | `backupProd` | `prod:db:backup` | Safety snapshot | | `schema` | `harmonize:schema` | Apply dev migrations to prod (when `schemaChanged: true`) | | `db` | `harmonize:db` | Row-level catalog merge | -| `images` | `harmonize:images` | Bidirectional file merge | +| `images` | `harmonize:images` | Bidirectional file merge, then regenerate painting + portrait thumbs on **dev and prod** | | `verify` | curl `/api/bounds` | Optional smoke check | Reports are written to `db/SyncReports/harmonize_db_*.json` and `harmonize_images_*.json` (gitignored). @@ -98,7 +98,9 @@ Reports are written to `db/SyncReports/harmonize_db_*.json` and `harmonize_image | `npm run harmonize:db` | Merge catalog rows by `updated_at` | | `npm run harmonize:db -- --dry-run` | Preview DB changes | | `npm run harmonize:db -- --prefer=dev` | On equal `updated_at`, dev wins | -| `npm run harmonize:images` | Merge image files by mtime | +| `npm run harmonize:images` | Merge image files by mtime, then regenerate painting + portrait thumbnails on both sides | +| `npm run harmonize:images -- --skip-thumbnails` | File merge only (no thumb rebuild) | +| `npm run harmonize:images -- --dry-run` | Preview file copies only (skips thumbnails) | | `npm run dev:migrate:sync-timestamps` | Apply `updated_at` migration on dev only | | `npm run dev:backfill-updated-at` | Backfill dev `updated_at` from image mtimes | | `npm run harmonize:backfill-updated-at` | Same backfill on prod | diff --git a/scripts/harmonize-images.js b/scripts/harmonize-images.js index bba57ef..b14e1f1 100644 --- a/scripts/harmonize-images.js +++ b/scripts/harmonize-images.js @@ -1,24 +1,33 @@ /** - * Bidirectional image harmonize (dev <-> prod) by file mtime. + * Bidirectional image harmonize (dev <-> prod) by file mtime, + * then regenerate painting + portrait thumbnails on both image roots. * * Usage: * npm run harmonize:images * npm run harmonize:images -- --dry-run + * npm run harmonize:images -- --skip-thumbnails */ const fs = require('fs'); const path = require('path'); -const { rootDir, confirmProdAction } = require('./db-env'); +const { spawnSync } = require('child_process'); +const { rootDir, confirmProdAction, PROD_DB_NAME } = require('./db-env'); const { loadHarmonizeConfig, prodImagesPath } = require('./lib/harmonize-config'); const { printCliResult } = require('./lib/cli-result'); const IMAGE_SUBDIRS = ['portraits', 'paintings']; const MTIME_TOLERANCE_MS = 1000; +const THUMB_SCRIPTS = [ + 'regenerate-thumbnails.js', + 'regenerate-portrait-thumbs.js', +]; + function parseArgs(argv) { const configArg = argv.find((a) => a.startsWith('--config=')); return { dryRun: argv.includes('--dry-run'), verbose: argv.includes('--verbose'), + skipThumbnails: argv.includes('--skip-thumbnails'), configPath: configArg ? configArg.slice('--config='.length) : null, }; } @@ -78,6 +87,36 @@ function copyFile(src, dest, dryRun) { fs.copyFileSync(src, dest); } +/** + * Rebuild thumbs from full images for one environment. + * Child scripts honour existing env (dotenv will not override DB_NAME / IMAGE_DIR). + */ +function regenerateThumbnails({ label, imageDir, database }) { + console.log(`\n--- Thumbnails (${label}) ---`); + console.log(` IMAGE_DIR=${imageDir}`); + console.log(` DB_NAME=${database}`); + + const env = { + ...process.env, + IMAGE_DIR: imageDir, + DB_NAME: database, + }; + + for (const file of THUMB_SCRIPTS) { + const scriptPath = path.join(__dirname, file); + console.log(` Running ${file}…`); + const result = spawnSync(process.execPath, [scriptPath], { + cwd: rootDir, + stdio: 'inherit', + env, + }); + const code = result.status ?? 1; + if (code !== 0) { + throw new Error(`Thumbnail regeneration failed (${label}): ${file} exited ${code}`); + } + } +} + function timestampSlug(date = new Date()) { const pad = (n) => String(n).padStart(2, '0'); return [ @@ -107,7 +146,7 @@ async function main() { if (!dryRun && process.env.CONFIRM_PROD !== '1') { await confirmProdAction( - 'Harmonize will copy image files between dev and prod (prod files may be overwritten).', + 'Harmonize will copy image files between dev and prod, then regenerate painting and portrait thumbnails on both sides (prod files and gallery_prod paths may be updated).', ); } @@ -165,9 +204,35 @@ async function main() { const reportDir = path.join(rootDir, 'db', 'SyncReports'); fs.mkdirSync(reportDir, { recursive: true }); const reportPath = path.join(reportDir, `harmonize_images_${timestampSlug()}.json`); + + const thumbDetails = []; + if (!dryRun && !options.skipThumbnails) { + require('dotenv').config({ path: path.join(rootDir, '.env') }); + const devDb = process.env.DB_NAME || 'gallery_dev'; + + regenerateThumbnails({ + label: 'dev', + imageDir: devDir, + database: devDb, + }); + thumbDetails.push(`Dev thumbs regenerated (${devDb} @ ${devDir})`); + + regenerateThumbnails({ + label: 'prod', + imageDir: prodDir, + database: PROD_DB_NAME, + }); + thumbDetails.push(`Prod thumbs regenerated (${PROD_DB_NAME} @ ${prodDir})`); + } else if (options.skipThumbnails) { + thumbDetails.push('Thumbnails skipped (--skip-thumbnails)'); + } else { + thumbDetails.push('Thumbnails skipped (dry-run)'); + } + fs.writeFileSync(reportPath, JSON.stringify({ generatedAt: new Date().toISOString(), dryRun, + skipThumbnails: options.skipThumbnails, devDir, prodDir, stats: { @@ -175,6 +240,7 @@ async function main() { prodToDev: stats.prodToDev, skipped: stats.skipped, }, + thumbnails: thumbDetails, actions: options.verbose ? stats.actions : stats.actions.slice(0, 500), }, null, 2)); @@ -183,7 +249,12 @@ async function main() { script: 'harmonize-images', ok: true, summary: `Image harmonize complete${mode}: dev→prod ${stats.devToProd}, prod→dev ${stats.prodToDev}, skipped ${stats.skipped}.`, - details: [`Dev: ${devDir}`, `Prod: ${prodDir}`, `Report: ${reportPath}`], + details: [ + `Dev: ${devDir}`, + `Prod: ${prodDir}`, + `Report: ${reportPath}`, + ...thumbDetails, + ], }); } diff --git a/scripts/image-fetcher.js b/scripts/image-fetcher.js index 0da90ec..9c32fb8 100644 --- a/scripts/image-fetcher.js +++ b/scripts/image-fetcher.js @@ -1691,7 +1691,13 @@ async function generateThumbnailFromFull(fullDest, thumbDest, width = THUMB_WIDT if (!fs.existsSync(fullDest)) return false; const ext = path.extname(thumbDest).toLowerCase(); - let pipeline = sharp(fullDest).rotate().resize({ width, withoutEnlargement: true }); + // Museum scans can exceed Sharp's default ~268MP input cap; we only resize down. + let pipeline = sharp(fullDest, { + limitInputPixels: false, + sequentialRead: true, + }) + .rotate() + .resize({ width, withoutEnlargement: true }); if (ext === '.png') { pipeline = pipeline.png({ quality: 85 }); diff --git a/scripts/lib/harmonize-config.js b/scripts/lib/harmonize-config.js index 22bec3c..7e7767a 100644 --- a/scripts/lib/harmonize-config.js +++ b/scripts/lib/harmonize-config.js @@ -1,6 +1,6 @@ const fs = require('fs'); const path = require('path'); -const { rootDir } = require('./db-env'); +const { rootDir } = require('../db-env'); const DEFAULT_CONFIG_PATH = path.join(rootDir, 'infra', 'deploy', 'harmonize.config.json'); const FALLBACK_CONFIG_PATH = path.join(rootDir, 'infra', 'deploy', 'devtoprod.config.json'); diff --git a/scripts/regenerate-thumbnails.js b/scripts/regenerate-thumbnails.js index 3c78e48..1676944 100644 --- a/scripts/regenerate-thumbnails.js +++ b/scripts/regenerate-thumbnails.js @@ -51,7 +51,7 @@ async function unlinkOldThumbRel(thumbRel, imageRel) { } async function aspectRatio(filePath) { - const meta = await sharp(filePath).metadata(); + const meta = await sharp(filePath, { limitInputPixels: false }).metadata(); if (!meta.width || !meta.height) return null; return meta.width / meta.height; }