Art-gallery/scripts/fetch-artist-bios.js

require('dotenv').config();
const https = require('https');
const pool = require('../server/db');

const USER_AGENT = 'VirtualArtGallery/1.0 (educational art history project; local museum gallery)';
const MIN_DELAY_MS = 3500;
const MAX_RETRIES = 8;

const ARTIST_WIKI_OVERRIDES = {
  Zeuxis: 'Zeuxis (painter)',
  'Ivan Klyun': 'Ivan Kliun',
  'Jean-Antoine Watteau': 'Antoine Watteau',
};

let lastRequestTime = 0;

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function throttle() {
  const elapsed = Date.now() - lastRequestTime;
  if (elapsed < MIN_DELAY_MS) await sleep(MIN_DELAY_MS - elapsed);
  lastRequestTime = Date.now();
}

function fetchJson(url) {
  return new Promise((resolve, reject) => {
    https
      .get(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => {
        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
          return fetchJson(res.headers.location).then(resolve).catch(reject);
        }
        let data = '';
        res.on('data', (c) => (data += c));
        res.on('end', () => {
          if (res.statusCode === 429) {
            return reject(new Error('HTTP 429'));
          }
          if (res.statusCode !== 200) {
            return reject(new Error(`HTTP ${res.statusCode}`));
          }
          try {
            resolve(JSON.parse(data));
          } catch (e) {
            reject(new Error(`Invalid JSON response (${data.slice(0, 40)}…)`));
          }
        });
      })
      .on('error', reject);
  });
}

async function withRetry(fn) {
  for (let i = 0; i < MAX_RETRIES; i++) {
    try {
      return await fn();
    } catch (err) {
      if (i === MAX_RETRIES - 1) throw err;
      const wait = err.message.includes('429') || err.message.includes('Invalid JSON')
        ? 12000 * (i + 1)
        : 4000 * (i + 1);
      console.warn(`  retry ${i + 1}/${MAX_RETRIES} in ${wait / 1000}s (${err.message})`);
      await sleep(wait);
    }
  }
}

function isDisambiguation(page, extract) {
  if (page?.pageprops?.disambiguation !== undefined) return true;
  if (!extract) return true;
  return /^.+\s+may refer to:\s*$/m.test(extract.split('\n')[0]);
}

function firstSentences(text, count = 2) {
  const parts = text.match(/[^.!?]+[.!?]+(?:\s|$)/g);
  if (!parts?.length) return text.trim();
  return parts.slice(0, count).join('').trim();
}

function uniqueTitles(candidates) {
  const seen = new Set();
  return candidates.filter((title) => {
    if (!title || seen.has(title)) return false;
    seen.add(title);
    return true;
  });
}

function candidateTitles(artist) {
  const { name, wikipedia_title: wikiTitle } = artist;
  return uniqueTitles([
    ARTIST_WIKI_OVERRIDES[name],
    ARTIST_WIKI_OVERRIDES[wikiTitle],
    wikiTitle,
    name,
    `${name} (painter)`,
    `${wikiTitle} (painter)`,
    `${name} (artist)`,
    `${wikiTitle} (artist)`,
  ]);
}

async function fetchPages(titles) {
  if (!titles.length) return {};
  return withRetry(async () => {
    await throttle();
    const params = new URLSearchParams({
      action: 'query',
      format: 'json',
      prop: 'extracts|pageprops',
      explaintext: '1',
      exintro: '1',
      ppprop: 'disambiguation',
      titles: titles.join('|'),
    });
    const data = await fetchJson(`https://en.wikipedia.org/w/api.php?${params}`);
    return data.query?.pages || {};
  });
}

async function resolveArtistBio(artist) {
  for (const title of candidateTitles(artist)) {
    const pages = await fetchPages([title]);
    const page = Object.values(pages)[0];
    if (!page || page.missing !== undefined) continue;

    const extract = page.extract?.trim();
    if (!extract || isDisambiguation(page, extract)) continue;

    return {
      bio_short: firstSentences(extract, 2),
      bio_full: extract,
      wikipedia_title: page.title,
    };
  }
  return null;
}

function needsBio(artist, force) {
  if (force) return true;
  if (!artist.bio_full) return true;
  if (artist.bio_full.trim() === '') return true;
  if (isDisambiguation(null, artist.bio_short || artist.bio_full)) return true;
  return false;
}

async function main() {
  const force = process.argv.includes('--force');

  const { rows: artists } = await pool.query(`
    SELECT id, name, wikipedia_title, bio_short, bio_full
    FROM artists
    ORDER BY name
  `);

  const targets = artists.filter((a) => needsBio(a, force));
  console.log(`Artists total: ${artists.length}, to fetch: ${targets.length}${force ? ' (force)' : ''}`);

  let updated = 0;
  let failed = 0;

  for (const artist of targets) {
    try {
      const bio = await resolveArtistBio(artist);
      if (!bio) {
        console.warn(`✗ ${artist.name} — no suitable Wikipedia article found`);
        failed += 1;
        continue;
      }

      await pool.query(
        `UPDATE artists
         SET bio_short = $1, bio_full = $2, wikipedia_title = $3
         WHERE id = $4`,
        [bio.bio_short, bio.bio_full, bio.wikipedia_title, artist.id]
      );

      console.log(`✓ ${artist.name} (${bio.wikipedia_title}, ${bio.bio_full.length} chars)`);
      updated += 1;
    } catch (err) {
      console.error(`✗ ${artist.name} — ${err.message}`);
      failed += 1;
    }
  }

  console.log(`\nDone: ${updated} updated, ${failed} failed`);
  await pool.end();
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});