Improve batch image fetch speed and document the pipeline.

Add random queue sampling and a 10s per-painting deadline for fetch-images batches, cap HTTP timeouts to the remaining budget, and update docs plus newly fetched artwork files.

Co-authored-by: Cursor <cursoragent@cursor.com>

scripts/fetch-missing-images.js

@@ -7,6 +7,13 @@ const { savePaintingImages } = require('./image-fetcher');
 const IMAGE_DIR = path.resolve(process.env.IMAGE_DIR || path.join(__dirname, '../data/images'));
 const LIMIT = parseInt(process.argv.find((a) => a.startsWith('--limit='))?.split('=')[1] || '0', 10);
 const ARTIST = process.argv.find((a) => a.startsWith('--artist='))?.split('=')[1];
+const MAX_WAIT_SEC = parseInt(
+  process.argv.find((a) => a.startsWith('--max-wait='))?.split('=')[1] ||
+    process.env.FETCH_MAX_WAIT_SEC ||
+    '10',
+  10
+);
+const MAX_WAIT_MS = MAX_WAIT_SEC * 1000;
 const DISCOVER_ONLY = process.argv.includes('--discover-only');
 const WEB_SEARCH_ONLY = process.argv.includes('--web-search-only');
 
@@ -15,6 +22,15 @@ function localExists(relPath) {
   return fs.existsSync(path.join(IMAGE_DIR, relPath));
 }
 
+function shuffleArray(items) {
+  const arr = [...items];
+  for (let i = arr.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [arr[i], arr[j]] = [arr[j], arr[i]];
+  }
+  return arr;
+}
+
 async function main() {
   let query = `
     SELECT p.id, p.title, p.wikipedia_title, p.image_path, p.thumbnail_path, a.name AS artist_name
@@ -31,9 +47,11 @@ async function main() {
   const { rows } = await pool.query(query, params);
   const missing = rows.filter((r) => !localExists(r.image_path) && !localExists(r.thumbnail_path));
 
-  const targets = LIMIT > 0 ? missing.slice(0, LIMIT) : missing;
+  const targets =
+    LIMIT > 0 ? shuffleArray(missing).slice(0, LIMIT) : missing;
   console.log(
     `Missing local files: ${missing.length}, processing: ${targets.length}` +
+      `${LIMIT > 0 ? ' (random sample)' : ''}` +
       `${DISCOVER_ONLY ? ' (discover-only)' : ''}` +
       `${WEB_SEARCH_ONLY ? ' (web-search-only)' : ''}`
   );
@@ -44,6 +62,7 @@ async function main() {
       (process.env.SMITHSONIAN_API_KEY ? ', Smithsonian' : '') +
       (process.env.HARVARD_ART_API_KEY ? ', Harvard' : '')
   );
+  console.log(`Max wait per painting: ${MAX_WAIT_SEC}s`);
 
   let ok = 0;
   let fail = 0;
@@ -57,6 +76,7 @@ async function main() {
         artistName: row.artist_name,
         paintingTitle: row.title,
         webSearchOnly: WEB_SEARCH_ONLY,
+        maxWaitMs: MAX_WAIT_MS,
       });
 
       if (saved.resolvedWikiTitle && saved.resolvedWikiTitle !== row.wikipedia_title) {
@@ -105,7 +125,11 @@ async function main() {
       }
     } catch (err) {
       fail += 1;
-      console.warn(`✗ ${row.artist_name} — ${row.title}: ${err.message}`);
+      if (err.code === 'FETCH_TIME_LIMIT') {
+        console.warn(`⏱ timeout (${MAX_WAIT_SEC}s): ${row.artist_name} — ${row.title}`);
+      } else {
+        console.warn(`✗ ${row.artist_name} — ${row.title}: ${err.message}`);
+      }
     }
   }