Art-gallery/scripts/update-painting-annotations.js

require('dotenv').config();
const https = require('https');
const pool = require('../server/db');
const ANNOTATIONS = require('./painting-annotations-data');
const { findArtist, findPainting } = require('./influence-resolver');

const FROM_WIKIPEDIA = process.argv.includes('--wikipedia');
const REPLACE = !process.argv.includes('--no-replace');
const WIKI_DELAY_MS = parseInt(process.argv.find((a) => a.startsWith('--wiki-delay='))?.split('=')[1] || '1200', 10);
const USER_AGENT = 'VirtualArtGallery/1.0 (educational art history project; local museum gallery)';

function sleep(ms) {
  return new Promise((r) => setTimeout(r, ms));
}

function fetchJson(url, attempt = 0) {
  return new Promise((resolve, reject) => {
    https
      .get(url, { headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' } }, (res) => {
        let data = '';
        res.on('data', (chunk) => {
          data += chunk;
        });
        res.on('end', async () => {
          if (res.statusCode === 429 && attempt < 4) {
            const wait = 1500 * (attempt + 1);
            console.warn(`Wikipedia rate limit; retrying in ${wait}ms…`);
            await sleep(wait);
            fetchJson(url, attempt + 1).then(resolve).catch(reject);
            return;
          }
          if (res.statusCode && res.statusCode >= 400) {
            reject(new Error(`HTTP ${res.statusCode}: ${data.slice(0, 80)}`));
            return;
          }
          try {
            resolve(JSON.parse(data));
          } catch {
            reject(new Error(`Invalid JSON from Wikipedia: ${data.slice(0, 80)}`));
          }
        });
      })
      .on('error', reject);
  });
}

function wikiTitleKey(title) {
  return String(title || '').toLowerCase().replace(/ /g, '_');
}

function extractIntroFromPage(page) {
  const extract = page?.extract?.trim();
  if (!extract || page?.missing) return null;
  const sentence = extract.split(/(?<=[.!?])\s+/).find((s) => s.length > 40);
  return sentence || extract.slice(0, 280);
}

async function fetchWikipediaIntro(wikipediaTitle) {
  if (!wikipediaTitle) return null;
  const url =
    `https://en.wikipedia.org/w/api.php?action=query&titles=${encodeURIComponent(wikipediaTitle)}` +
    '&prop=extracts&explaintext=1&exintro=1&format=json';
  try {
    const data = await fetchJson(url);
    const page = Object.values(data.query?.pages || {})[0];
    return extractIntroFromPage(page);
  } catch (err) {
    console.warn(`✗ Wikipedia fetch failed for "${wikipediaTitle}": ${err.message}`);
    return null;
  }
}

async function fetchWikipediaIntroBatch(titles) {
  if (!titles.length) return new Map();
  const url =
    `https://en.wikipedia.org/w/api.php?action=query&titles=${titles.map(encodeURIComponent).join('|')}` +
    '&prop=extracts&explaintext=1&exintro=1&format=json';
  const data = await fetchJson(url);
  const out = new Map();
  for (const page of Object.values(data.query?.pages || {})) {
    if (!page?.title) continue;
    const body = extractIntroFromPage(page);
    if (body) out.set(wikiTitleKey(page.title), body);
  }
  return out;
}

async function insertAnnotation(paintingId, row, sortOrder) {
  await pool.query(
    `INSERT INTO painting_annotations
       (painting_id, label, body, category, pos_x, pos_y, source_author, source, source_url, sort_order, confidence)
     VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
    [
      paintingId,
      row.label || null,
      row.body,
      row.category || 'subject',
      row.pos_x ?? null,
      row.pos_y ?? null,
      row.source_author || null,
      row.source || null,
      row.source_url || null,
      sortOrder,
      row.confidence || 'curated',
    ]
  );
}

async function clearCurated(paintingId) {
  await pool.query(
    `DELETE FROM painting_annotations WHERE painting_id = $1 AND confidence = 'curated'`,
    [paintingId]
  );
}

async function applyCuratedEntry(entry, stats) {
  const artist = await findArtist(pool, entry.artist);
  if (!artist) {
    stats.missing += 1;
    console.warn(`✗ artist not found: ${entry.artist}`);
    return;
  }

  const painting = await findPainting(pool, artist.id, entry.title);
  if (!painting) {
    stats.missing += 1;
    console.warn(`✗ painting not found: ${entry.artist} / ${entry.title}`);
    return;
  }

  if (REPLACE) {
    await clearCurated(painting.id);
  }

  let order = 0;
  for (const ann of entry.annotations || []) {
    await insertAnnotation(painting.id, ann, order++);
    stats.added += 1;
  }
  console.log(`→ ${entry.artist} / ${painting.title}: ${entry.annotations.length} annotation(s)`);
}

async function applyWikipediaFallback(stats) {
  const { rows } = await pool.query(`
    SELECT p.id, p.title, p.wikipedia_title, a.name AS artist_name
    FROM paintings p
    JOIN artists a ON a.id = p.artist_id
    WHERE p.wikipedia_title IS NOT NULL
      AND NOT EXISTS (SELECT 1 FROM painting_annotations pa WHERE pa.painting_id = p.id)
    ORDER BY p.id
  `);

  const BATCH_SIZE = 20;
  for (let i = 0; i < rows.length; i += BATCH_SIZE) {
    const batch = rows.slice(i, i + BATCH_SIZE);
    let extracts = new Map();
    try {
      extracts = await fetchWikipediaIntroBatch(batch.map((r) => r.wikipedia_title));
    } catch (err) {
      console.warn(`✗ Wikipedia batch failed (${i + 1}-${i + batch.length}): ${err.message}`);
      stats.wikiFailed += batch.length;
      await sleep(WIKI_DELAY_MS * 4);
      continue;
    }

    for (const row of batch) {
      try {
        const body = extracts.get(wikiTitleKey(row.wikipedia_title)) || null;
        if (!body) {
          stats.wikiSkipped += 1;
          continue;
        }
        await insertAnnotation(
          row.id,
          {
            label: 'Overview',
            body,
            category: 'subject',
            source_author: 'Wikipedia',
            source: row.wikipedia_title,
            source_url: `https://en.wikipedia.org/wiki/${encodeURIComponent(row.wikipedia_title.replace(/ /g, '_'))}`,
            confidence: 'discovered',
          },
          0
        );
        stats.wiki += 1;
        console.log(`+ wiki: ${row.artist_name} / ${row.title}`);
      } catch (err) {
        stats.wikiFailed += 1;
        console.warn(`✗ wiki insert failed for ${row.artist_name} / ${row.title}: ${err.message}`);
      }
    }

    if (i + BATCH_SIZE < rows.length) {
      await sleep(WIKI_DELAY_MS);
    }
  }
}

async function main() {
  const stats = { added: 0, missing: 0, wiki: 0, wikiSkipped: 0, wikiFailed: 0 };

  for (const entry of ANNOTATIONS) {
    await applyCuratedEntry(entry, stats);
  }

  if (FROM_WIKIPEDIA) {
    await applyWikipediaFallback(stats);
  }

  const { rows } = await pool.query(`
    SELECT COUNT(*)::int AS total,
           COUNT(DISTINCT painting_id)::int AS paintings
    FROM painting_annotations
  `);
  console.log(
    `Done: ${stats.added} curated inserted, ${stats.missing} unmatched, ${stats.wiki} from Wikipedia` +
      (stats.wikiSkipped ? ` (${stats.wikiSkipped} skipped)` : '') +
      (stats.wikiFailed ? ` (${stats.wikiFailed} failed)` : '') +
      `; ${rows[0].total} total on ${rows[0].paintings} painting(s)`
  );
  await pool.end();
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});