require('dotenv').config(); const fs = require('fs'); const path = require('path'); const pool = require('../server/db'); const { savePaintingImages } = require('./image-fetcher'); const IMAGE_DIR = path.resolve(process.env.IMAGE_DIR || path.join(__dirname, '../data/images')); const LIMIT = parseInt(process.argv.find((a) => a.startsWith('--limit='))?.split('=')[1] || '0', 10); const ARTIST = process.argv.find((a) => a.startsWith('--artist='))?.split('=')[1]; const MAX_WAIT_SEC = parseInt( process.argv.find((a) => a.startsWith('--max-wait='))?.split('=')[1] || process.env.FETCH_MAX_WAIT_SEC || '10', 10 ); const MAX_WAIT_MS = MAX_WAIT_SEC * 1000; const DISCOVER_ONLY = process.argv.includes('--discover-only'); const WEB_SEARCH_ONLY = process.argv.includes('--web-search-only'); function localExists(relPath) { if (!relPath) return false; return fs.existsSync(path.join(IMAGE_DIR, relPath)); } function shuffleArray(items) { const arr = [...items]; for (let i = arr.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)); [arr[i], arr[j]] = [arr[j], arr[i]]; } return arr; } async function main() { let query = ` SELECT p.id, p.title, p.wikipedia_title, p.image_path, p.thumbnail_path, a.name AS artist_name FROM paintings p JOIN artists a ON a.id = p.artist_id `; const params = []; if (ARTIST) { params.push(ARTIST); query += ` WHERE a.name = $${params.length}`; } query += ' ORDER BY a.name, p.sort_order, p.year NULLS LAST'; const { rows } = await pool.query(query, params); const missing = rows.filter((r) => !localExists(r.image_path) && !localExists(r.thumbnail_path)); const targets = LIMIT > 0 ? shuffleArray(missing).slice(0, LIMIT) : missing; console.log( `Missing local files: ${missing.length}, processing: ${targets.length}` + `${LIMIT > 0 ? ' (random sample)' : ''}` + `${DISCOVER_ONLY ? ' (discover-only)' : ''}` + `${WEB_SEARCH_ONLY ? ' (web-search-only)' : ''}` ); console.log( 'Sources: Wikipedia/Wikidata, Wikimedia Commons, Google Arts & Culture, Louvre, DE/FR/IT/RU Wikipedia, web search (DuckDuckGo)' + ', Met, Art Institute, Cleveland, Rijksmuseum' + (process.env.EUROPEANA_API_KEY ? ', Europeana' : '') + (process.env.SMITHSONIAN_API_KEY ? ', Smithsonian' : '') + (process.env.HARVARD_ART_API_KEY ? ', Harvard' : '') ); console.log(`Max wait per painting: ${MAX_WAIT_SEC}s`); let ok = 0; let fail = 0; let discovered = 0; for (const row of targets) { const wikiTitle = row.wikipedia_title || row.title; try { const base = `${row.artist_name}_${row.title}`.replace(/[^a-zA-Z0-9_-]/g, '_'); const saved = await savePaintingImages(wikiTitle, base, IMAGE_DIR, { artistName: row.artist_name, paintingTitle: row.title, webSearchOnly: WEB_SEARCH_ONLY, maxWaitMs: MAX_WAIT_MS, }); if (saved.resolvedWikiTitle && saved.resolvedWikiTitle !== row.wikipedia_title) { discovered += 1; if (DISCOVER_ONLY || saved.imagePath || saved.thumbnailPath) { await pool.query(`UPDATE paintings SET wikipedia_title = $1 WHERE id = $2`, [ saved.resolvedWikiTitle, row.id, ]); } } if (DISCOVER_ONLY) { if (saved.resolvedWikiTitle) { console.log(`~ ${row.artist_name} — ${row.title} → wiki: ${saved.resolvedWikiTitle}`); ok += 1; } else { fail += 1; console.warn(`✗ no wiki match: ${row.artist_name} — ${row.title}`); } continue; } if (saved.imagePath || saved.thumbnailPath) { if (saved.resolvedWikiTitle && saved.resolvedWikiTitle !== row.wikipedia_title) { await pool.query( `UPDATE paintings SET image_path = $1, thumbnail_path = $2, wikipedia_title = $3 WHERE id = $4`, [saved.imagePath, saved.thumbnailPath, saved.resolvedWikiTitle, row.id] ); } else { await pool.query( `UPDATE paintings SET image_path = $1, thumbnail_path = $2 WHERE id = $3`, [saved.imagePath, saved.thumbnailPath, row.id] ); } ok += 1; const src = saved.source ? ` [${saved.source}]` : ''; const wiki = saved.resolvedWikiTitle && saved.resolvedWikiTitle !== row.wikipedia_title ? ` (wiki→${saved.resolvedWikiTitle})` : ''; console.log(`✓ ${row.artist_name} — ${row.title}${wiki}${src}`); } else { fail += 1; console.warn(`✗ no source: ${row.artist_name} — ${row.title}`); } } catch (err) { fail += 1; if (err.code === 'FETCH_TIME_LIMIT') { console.warn(`⏱ timeout (${MAX_WAIT_SEC}s): ${row.artist_name} — ${row.title}`); } else { console.warn(`✗ ${row.artist_name} — ${row.title}: ${err.message}`); } } } console.log(`\nDone: ${ok} ok, ${fail} failed, ${discovered} wikipedia titles improved`); await pool.end(); } main().catch((err) => { console.error(err); process.exit(1); });