Art-gallery/scripts/fetch-missing-images.js

require('dotenv').config();
const fs = require('fs');
const path = require('path');
const pool = require('../server/db');
const { savePaintingImages } = require('./image-fetcher');

const IMAGE_DIR = path.resolve(process.env.IMAGE_DIR || path.join(__dirname, '../data/images'));
const LIMIT = parseInt(process.argv.find((a) => a.startsWith('--limit='))?.split('=')[1] || '0', 10);
const ARTIST = process.argv.find((a) => a.startsWith('--artist='))?.split('=')[1];
const MAX_WAIT_SEC = parseInt(
  process.argv.find((a) => a.startsWith('--max-wait='))?.split('=')[1] ||
    process.env.FETCH_MAX_WAIT_SEC ||
    '10',
  10
);
const MAX_WAIT_MS = MAX_WAIT_SEC * 1000;
const DISCOVER_ONLY = process.argv.includes('--discover-only');
const WEB_SEARCH_ONLY = process.argv.includes('--web-search-only');

function localExists(relPath) {
  if (!relPath) return false;
  return fs.existsSync(path.join(IMAGE_DIR, relPath));
}

function shuffleArray(items) {
  const arr = [...items];
  for (let i = arr.length - 1; i > 0; i--) {
    const j = Math.floor(Math.random() * (i + 1));
    [arr[i], arr[j]] = [arr[j], arr[i]];
  }
  return arr;
}

async function main() {
  let query = `
    SELECT p.id, p.title, p.wikipedia_title, p.image_path, p.thumbnail_path, a.name AS artist_name
    FROM paintings p
    JOIN artists a ON a.id = p.artist_id
  `;
  const params = [];
  if (ARTIST) {
    params.push(ARTIST);
    query += ` WHERE a.name = $${params.length}`;
  }
  query += ' ORDER BY a.name, p.sort_order, p.year NULLS LAST';

  const { rows } = await pool.query(query, params);
  const missing = rows.filter((r) => !localExists(r.image_path) && !localExists(r.thumbnail_path));

  const targets =
    LIMIT > 0 ? shuffleArray(missing).slice(0, LIMIT) : missing;
  console.log(
    `Missing local files: ${missing.length}, processing: ${targets.length}` +
      `${LIMIT > 0 ? ' (random sample)' : ''}` +
      `${DISCOVER_ONLY ? ' (discover-only)' : ''}` +
      `${WEB_SEARCH_ONLY ? ' (web-search-only)' : ''}`
  );
  console.log(
    'Sources: Wikipedia/Wikidata, Wikimedia Commons, Google Arts & Culture, Louvre, DE/FR/IT/RU Wikipedia, web search (DuckDuckGo)' +
      ', Met, Art Institute, Cleveland, Rijksmuseum' +
      (process.env.EUROPEANA_API_KEY ? ', Europeana' : '') +
      (process.env.SMITHSONIAN_API_KEY ? ', Smithsonian' : '') +
      (process.env.HARVARD_ART_API_KEY ? ', Harvard' : '')
  );
  console.log(`Max wait per painting: ${MAX_WAIT_SEC}s`);

  let ok = 0;
  let fail = 0;
  let discovered = 0;

  for (const row of targets) {
    const wikiTitle = row.wikipedia_title || row.title;
    try {
      const base = `${row.artist_name}_${row.title}`.replace(/[^a-zA-Z0-9_-]/g, '_');
      const saved = await savePaintingImages(wikiTitle, base, IMAGE_DIR, {
        artistName: row.artist_name,
        paintingTitle: row.title,
        webSearchOnly: WEB_SEARCH_ONLY,
        maxWaitMs: MAX_WAIT_MS,
      });

      if (saved.resolvedWikiTitle && saved.resolvedWikiTitle !== row.wikipedia_title) {
        discovered += 1;
        if (DISCOVER_ONLY || saved.imagePath || saved.thumbnailPath) {
          await pool.query(`UPDATE paintings SET wikipedia_title = $1 WHERE id = $2`, [
            saved.resolvedWikiTitle,
            row.id,
          ]);
        }
      }

      if (DISCOVER_ONLY) {
        if (saved.resolvedWikiTitle) {
          console.log(`~ ${row.artist_name} — ${row.title} → wiki: ${saved.resolvedWikiTitle}`);
          ok += 1;
        } else {
          fail += 1;
          console.warn(`✗ no wiki match: ${row.artist_name} — ${row.title}`);
        }
        continue;
      }

      if (saved.imagePath || saved.thumbnailPath) {
        if (saved.resolvedWikiTitle && saved.resolvedWikiTitle !== row.wikipedia_title) {
          await pool.query(
            `UPDATE paintings SET image_path = $1, thumbnail_path = $2, wikipedia_title = $3 WHERE id = $4`,
            [saved.imagePath, saved.thumbnailPath, saved.resolvedWikiTitle, row.id]
          );
        } else {
          await pool.query(
            `UPDATE paintings SET image_path = $1, thumbnail_path = $2 WHERE id = $3`,
            [saved.imagePath, saved.thumbnailPath, row.id]
          );
        }
        ok += 1;
        const src = saved.source ? ` [${saved.source}]` : '';
        const wiki =
          saved.resolvedWikiTitle && saved.resolvedWikiTitle !== row.wikipedia_title
            ? ` (wiki→${saved.resolvedWikiTitle})`
            : '';
        console.log(`✓ ${row.artist_name} — ${row.title}${wiki}${src}`);
      } else {
        fail += 1;
        console.warn(`✗ no source: ${row.artist_name} — ${row.title}`);
      }
    } catch (err) {
      fail += 1;
      if (err.code === 'FETCH_TIME_LIMIT') {
        console.warn(`⏱ timeout (${MAX_WAIT_SEC}s): ${row.artist_name} — ${row.title}`);
      } else {
        console.warn(`✗ ${row.artist_name} — ${row.title}: ${err.message}`);
      }
    }
  }

  console.log(`\nDone: ${ok} ok, ${fail} failed, ${discovered} wikipedia titles improved`);
  await pool.end();
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});