Fix harmonize:images thumb rebuild and allow oversized museum scans.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-27 12:23:25 +03:00
parent 6656f91f25
commit 89e39d408a
8 changed files with 92 additions and 13 deletions
@@ -143,7 +143,7 @@ Run `npm run dev:migrate` against prod DB after first deploy with auth vars set
 | `npm run devtoprod:db:restore -- --file <path>` | Restore into **prod** (requires confirmation) |
 | `npm run harmonize` | Bidirectional catalog DB + image merge by `updated_at` / file mtime — [harmonize-dev-prod.md](harmonize-dev-prod.md) |
 | `npm run harmonize:schema` | Apply dev migrations to prod schema only (dev → prod) |
-| `npm run harmonize:db` / `harmonize:images` | DB or image merge only |
+| `npm run harmonize:db` / `harmonize:images` | DB or image merge only (`harmonize:images` also regenerates painting + portrait thumbs on both sides) |
 | `npm run dev:backfill-updated-at` | Backfill catalog `updated_at` from image mtimes (dev) |

 **One-time split (recommended):** pgAdmin on dev PC → open [`db/split-dev-prod-pgadmin.sql`](../db/split-dev-prod-pgadmin.sql) → run each STEP on database `postgres`, then verify on `gallery_dev`.
@@ -31,7 +31,7 @@ File names are sanitised `{Artist}_{Title}.{ext}`. The image service can redisco
 | **Development** | `./data/images/` in repo | Working copy on dev PC |
 | **Production** | `/mnt/BasePool/Applications/Gallery/data/images` on TrueNAS | SMB `\\192.168.10.122\Gallery\data\images` |

-**One-direction promote:** `npm run devtoprod:images` (dev → prod, skip older). **Refresh all images on dev from prod:** `npm run prodto:dev:images`. **One artist only:** map the SMB share, then `robocopy` that artist’s `ArtistName*` files under `paintings/` and `paintings/thumbs/` (see [FAC.md](FAC.md#dev--prod-image-sync-smb)). **Bidirectional merge** (newer file wins): `npm run harmonize:images` or full `npm run harmonize` — see [harmonize-dev-prod.md](harmonize-dev-prod.md). General sync reference: [environments.md](environments.md).
+**One-direction promote:** `npm run devtoprod:images` (dev → prod, skip older). **Refresh all images on dev from prod:** `npm run prodto:dev:images`. **One artist only:** map the SMB share, then `robocopy` that artist’s `ArtistName*` files under `paintings/` and `paintings/thumbs/` (see [FAC.md](FAC.md#dev--prod-image-sync-smb)). **Bidirectional merge** (newer file wins, then rebuild painting + portrait thumbs on both sides): `npm run harmonize:images` or full `npm run harmonize` — see [harmonize-dev-prod.md](harmonize-dev-prod.md). General sync reference: [environments.md](environments.md).

 ## Scripts overview

@@ -49,7 +49,7 @@ File names are sanitised `{Artist}_{Title}.{ext}`. The image service can redisco
 | `fetch-missing-images.js` | `npm run dev:fetch-images` | Downloads files for paintings missing on disk |
 | `image-fetcher.js` | *(library)* | Wikimedia / museum resolution used by fetch scripts and API |
 | `sync-images-to-prod.ps1` / `sync-images-from-prod.ps1` | `npm run devtoprod:images` / `npm run prodto:dev:images` | Robocopy via SMB `\\192.168.10.122\Gallery` |
-| `harmonize-db.js` / `harmonize-images.js` | `npm run harmonize:db` / `harmonize:images` | Bidirectional merge by `updated_at` / file mtime |
+| `harmonize-db.js` / `harmonize-images.js` | `npm run harmonize:db` / `harmonize:images` | Bidirectional merge by `updated_at` / file mtime; images step then rebuilds thumbs on both sides |
 | `harmonize.ps1` | `npm run harmonize` | Orchestrator: backups, optional schema, DB + image merge |
 | `regenerate-thumbnails.js` | `npm run dev:regenerate-thumbnails` | Rebuild painting thumbs from full images via `sharp` |
 | `regenerate-portrait-thumbs.js` | `npm run dev:regenerate-portrait-thumbs` | Rebuild timeline portrait thumbs (~256px) and set `portrait_thumb_path` |
@@ -341,7 +341,7 @@ For **incremental** dev ↔ prod merge (both sides edited), use [harmonize-dev-p
 | `npm run devtoprod:db:restore` | Dev PC PowerShell | Restore into prod |
 | `npm run harmonize` | Dev PC PowerShell | Bidirectional catalog + image merge — [harmonize-dev-prod.md](harmonize-dev-prod.md) |
 | `npm run harmonize:db` | Dev PC PowerShell | DB merge only |
-| `npm run harmonize:images` | Dev PC PowerShell | Image merge only |
+| `npm run harmonize:images` | Dev PC PowerShell | Image merge + regenerate thumbs on both sides |

 ## Image sync

@@ -83,7 +83,7 @@ npm run harmonize:dry-run
 | `backupProd` | `prod:db:backup` | Safety snapshot |
 | `schema` | `harmonize:schema` | Apply dev migrations to prod (when `schemaChanged: true`) |
 | `db` | `harmonize:db` | Row-level catalog merge |
-| `images` | `harmonize:images` | Bidirectional file merge |
+| `images` | `harmonize:images` | Bidirectional file merge, then regenerate painting + portrait thumbs on **dev and prod** |
 | `verify` | curl `/api/bounds` | Optional smoke check |

 Reports are written to `db/SyncReports/harmonize_db_*.json` and `harmonize_images_*.json` (gitignored).
@@ -98,7 +98,9 @@ Reports are written to `db/SyncReports/harmonize_db_*.json` and `harmonize_image
 | `npm run harmonize:db` | Merge catalog rows by `updated_at` |
 | `npm run harmonize:db -- --dry-run` | Preview DB changes |
 | `npm run harmonize:db -- --prefer=dev` | On equal `updated_at`, dev wins |
-| `npm run harmonize:images` | Merge image files by mtime |
+| `npm run harmonize:images` | Merge image files by mtime, then regenerate painting + portrait thumbnails on both sides |
+| `npm run harmonize:images -- --skip-thumbnails` | File merge only (no thumb rebuild) |
+| `npm run harmonize:images -- --dry-run` | Preview file copies only (skips thumbnails) |
 | `npm run dev:migrate:sync-timestamps` | Apply `updated_at` migration on dev only |
 | `npm run dev:backfill-updated-at` | Backfill dev `updated_at` from image mtimes |
 | `npm run harmonize:backfill-updated-at` | Same backfill on prod |
@@ -1,24 +1,33 @@
 /**
- * Bidirectional image harmonize (dev <-> prod) by file mtime.
+ * Bidirectional image harmonize (dev <-> prod) by file mtime,
+ * then regenerate painting + portrait thumbnails on both image roots.
 *
 * Usage:
 *   npm run harmonize:images
 *   npm run harmonize:images -- --dry-run
+ *   npm run harmonize:images -- --skip-thumbnails
 */
 const fs = require('fs');
 const path = require('path');
-const { rootDir, confirmProdAction } = require('./db-env');
+const { spawnSync } = require('child_process');
+const { rootDir, confirmProdAction, PROD_DB_NAME } = require('./db-env');
 const { loadHarmonizeConfig, prodImagesPath } = require('./lib/harmonize-config');
 const { printCliResult } = require('./lib/cli-result');

 const IMAGE_SUBDIRS = ['portraits', 'paintings'];
 const MTIME_TOLERANCE_MS = 1000;

+const THUMB_SCRIPTS = [
+  'regenerate-thumbnails.js',
+  'regenerate-portrait-thumbs.js',
+];
+
 function parseArgs(argv) {
  const configArg = argv.find((a) => a.startsWith('--config='));
  return {
    dryRun: argv.includes('--dry-run'),
    verbose: argv.includes('--verbose'),
+    skipThumbnails: argv.includes('--skip-thumbnails'),
    configPath: configArg ? configArg.slice('--config='.length) : null,
  };
 }
@@ -78,6 +87,36 @@ function copyFile(src, dest, dryRun) {
  fs.copyFileSync(src, dest);
 }

+/**
+ * Rebuild thumbs from full images for one environment.
+ * Child scripts honour existing env (dotenv will not override DB_NAME / IMAGE_DIR).
+ */
+function regenerateThumbnails({ label, imageDir, database }) {
+  console.log(`\n--- Thumbnails (${label}) ---`);
+  console.log(`  IMAGE_DIR=${imageDir}`);
+  console.log(`  DB_NAME=${database}`);
+
+  const env = {
+    ...process.env,
+    IMAGE_DIR: imageDir,
+    DB_NAME: database,
+  };
+
+  for (const file of THUMB_SCRIPTS) {
+    const scriptPath = path.join(__dirname, file);
+    console.log(`  Running ${file}…`);
+    const result = spawnSync(process.execPath, [scriptPath], {
+      cwd: rootDir,
+      stdio: 'inherit',
+      env,
+    });
+    const code = result.status ?? 1;
+    if (code !== 0) {
+      throw new Error(`Thumbnail regeneration failed (${label}): ${file} exited ${code}`);
+    }
+  }
+}
+
 function timestampSlug(date = new Date()) {
  const pad = (n) => String(n).padStart(2, '0');
  return [
@@ -107,7 +146,7 @@ async function main() {

  if (!dryRun && process.env.CONFIRM_PROD !== '1') {
    await confirmProdAction(
-      'Harmonize will copy image files between dev and prod (prod files may be overwritten).',
+      'Harmonize will copy image files between dev and prod, then regenerate painting and portrait thumbnails on both sides (prod files and gallery_prod paths may be updated).',
    );
  }

@@ -165,9 +204,35 @@ async function main() {
  const reportDir = path.join(rootDir, 'db', 'SyncReports');
  fs.mkdirSync(reportDir, { recursive: true });
  const reportPath = path.join(reportDir, `harmonize_images_${timestampSlug()}.json`);
+
+  const thumbDetails = [];
+  if (!dryRun && !options.skipThumbnails) {
+    require('dotenv').config({ path: path.join(rootDir, '.env') });
+    const devDb = process.env.DB_NAME || 'gallery_dev';
+
+    regenerateThumbnails({
+      label: 'dev',
+      imageDir: devDir,
+      database: devDb,
+    });
+    thumbDetails.push(`Dev thumbs regenerated (${devDb} @ ${devDir})`);
+
+    regenerateThumbnails({
+      label: 'prod',
+      imageDir: prodDir,
+      database: PROD_DB_NAME,
+    });
+    thumbDetails.push(`Prod thumbs regenerated (${PROD_DB_NAME} @ ${prodDir})`);
+  } else if (options.skipThumbnails) {
+    thumbDetails.push('Thumbnails skipped (--skip-thumbnails)');
+  } else {
+    thumbDetails.push('Thumbnails skipped (dry-run)');
+  }
+
  fs.writeFileSync(reportPath, JSON.stringify({
    generatedAt: new Date().toISOString(),
    dryRun,
+    skipThumbnails: options.skipThumbnails,
    devDir,
    prodDir,
    stats: {
@@ -175,6 +240,7 @@ async function main() {
      prodToDev: stats.prodToDev,
      skipped: stats.skipped,
    },
+    thumbnails: thumbDetails,
    actions: options.verbose ? stats.actions : stats.actions.slice(0, 500),
  }, null, 2));

@@ -183,7 +249,12 @@ async function main() {
    script: 'harmonize-images',
    ok: true,
    summary: `Image harmonize complete${mode}: dev→prod ${stats.devToProd}, prod→dev ${stats.prodToDev}, skipped ${stats.skipped}.`,
-    details: [`Dev: ${devDir}`, `Prod: ${prodDir}`, `Report: ${reportPath}`],
+    details: [
+      `Dev: ${devDir}`,
+      `Prod: ${prodDir}`,
+      `Report: ${reportPath}`,
+      ...thumbDetails,
+    ],
  });
 }

@@ -1691,7 +1691,13 @@ async function generateThumbnailFromFull(fullDest, thumbDest, width = THUMB_WIDT
  if (!fs.existsSync(fullDest)) return false;

  const ext = path.extname(thumbDest).toLowerCase();
-  let pipeline = sharp(fullDest).rotate().resize({ width, withoutEnlargement: true });
+  // Museum scans can exceed Sharp's default ~268MP input cap; we only resize down.
+  let pipeline = sharp(fullDest, {
+    limitInputPixels: false,
+    sequentialRead: true,
+  })
+    .rotate()
+    .resize({ width, withoutEnlargement: true });

  if (ext === '.png') {
    pipeline = pipeline.png({ quality: 85 });
@@ -1,6 +1,6 @@
 const fs = require('fs');
 const path = require('path');
-const { rootDir } = require('./db-env');
+const { rootDir } = require('../db-env');

 const DEFAULT_CONFIG_PATH = path.join(rootDir, 'infra', 'deploy', 'harmonize.config.json');
 const FALLBACK_CONFIG_PATH = path.join(rootDir, 'infra', 'deploy', 'devtoprod.config.json');
@@ -51,7 +51,7 @@ async function unlinkOldThumbRel(thumbRel, imageRel) {
 }

 async function aspectRatio(filePath) {
-  const meta = await sharp(filePath).metadata();
+  const meta = await sharp(filePath, { limitInputPixels: false }).metadata();
  if (!meta.width || !meta.height) return null;
  return meta.width / meta.height;
 }