Art-gallery/scripts/fetch-artist-bios-ru.js

require('dotenv').config();
const https = require('https');
const pool = require('../server/db');
const { upsertTranslation } = require('../server/translation-service');

const USER_AGENT = 'VirtualArtGallery/1.0 (educational art history project; local museum gallery)';
const MIN_DELAY_MS = 3500;
const MAX_RETRIES = 8;
const LOCALE = 'ru';
const WIKI_API = 'https://ru.wikipedia.org/w/api.php';

const ARTIST_WIKI_OVERRIDES = {
  Zeuxis: 'Zeuxis (painter)',
  'Ivan Klyun': 'Ivan Kliun',
  'Jean-Antoine Watteau': 'Antoine Watteau',
};

let lastRequestTime = 0;

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function throttle() {
  const elapsed = Date.now() - lastRequestTime;
  if (elapsed < MIN_DELAY_MS) await sleep(MIN_DELAY_MS - elapsed);
  lastRequestTime = Date.now();
}

function fetchJson(url) {
  return new Promise((resolve, reject) => {
    https
      .get(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => {
        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
          return fetchJson(res.headers.location).then(resolve).catch(reject);
        }
        let data = '';
        res.on('data', (c) => (data += c));
        res.on('end', () => {
          if (res.statusCode !== 200) return reject(new Error(`HTTP ${res.statusCode}`));
          try {
            resolve(JSON.parse(data));
          } catch (e) {
            reject(e);
          }
        });
      })
      .on('error', reject);
  });
}

async function withRetry(fn) {
  for (let i = 0; i < MAX_RETRIES; i++) {
    try {
      return await fn();
    } catch (err) {
      if (i === MAX_RETRIES - 1) throw err;
      await sleep(4000 * (i + 1));
    }
  }
}

function firstSentences(text, count = 2) {
  const parts = text.match(/[^.!?]+[.!?]+(?:\s|$)/g);
  if (!parts?.length) return text.trim();
  return parts.slice(0, count).join('').trim();
}

async function resolveRuPageTitle(enTitle) {
  await throttle();
  const params = new URLSearchParams({
    action: 'query',
    format: 'json',
    titles: enTitle,
    prop: 'langlinks',
    lllang: 'ru',
    lllimit: '1',
  });
  const data = await withRetry(() => fetchJson(`https://en.wikipedia.org/w/api.php?${params}`));
  const pages = data.query?.pages || {};
  const page = Object.values(pages)[0];
  const link = page?.langlinks?.[0];
  if (link?.['*']) return link['*'];
  return enTitle;
}

async function fetchRuExtract(title) {
  await throttle();
  const params = new URLSearchParams({
    action: 'query',
    format: 'json',
    prop: 'extracts|pageprops',
    explaintext: '1',
    exintro: '0',
    ppprop: 'disambiguation',
    titles: title,
  });
  const data = await withRetry(() => fetchJson(`${WIKI_API}?${params}`));
  const pages = data.query?.pages || {};
  const page = Object.values(pages)[0];
  if (!page || page.missing !== undefined) return null;
  const extract = page.extract?.trim();
  if (!extract) return null;
  return { title: page.title, extract };
}

async function resolveArtistBioRu(artist) {
  const enTitle = ARTIST_WIKI_OVERRIDES[artist.wikipedia_title]
    || ARTIST_WIKI_OVERRIDES[artist.name]
    || artist.wikipedia_title
    || artist.name;
  const ruTitle = await resolveRuPageTitle(enTitle);
  const page = await fetchRuExtract(ruTitle);
  if (!page) return null;
  return {
    name: page.title,
    bio_short: firstSentences(page.extract, 2),
    bio_full: page.extract,
  };
}

async function main() {
  const force = process.argv.includes('--force');
  const limitArg = process.argv.find((a) => a.startsWith('--limit='));
  const limit = limitArg ? parseInt(limitArg.split('=')[1], 10) : null;

  const { rows: artists } = await pool.query(`
    SELECT id, name, wikipedia_title FROM artists ORDER BY name
  `);

  let targets = artists;
  if (limit) targets = targets.slice(0, limit);

  let updated = 0;
  let skipped = 0;
  let failed = 0;

  for (const artist of targets) {
    if (!force) {
      const { rows: existing } = await pool.query(
        `SELECT 1 FROM entity_translations
         WHERE entity_type = 'artist' AND entity_id = $1 AND locale = $2 AND field_name = 'bio_full'`,
        [artist.id, LOCALE],
      );
      if (existing.length) {
        skipped += 1;
        continue;
      }
    }

    try {
      const bio = await resolveArtistBioRu(artist);
      if (!bio) {
        console.warn(`✗ ${artist.name} — no ru.wikipedia article`);
        failed += 1;
        continue;
      }

      await upsertTranslation({
        entityType: 'artist',
        entityId: artist.id,
        locale: LOCALE,
        fieldName: 'name',
        value: bio.name,
        status: 'draft',
        source: 'wikipedia_ru',
      });
      await upsertTranslation({
        entityType: 'artist',
        entityId: artist.id,
        locale: LOCALE,
        fieldName: 'bio_short',
        value: bio.bio_short,
        status: 'draft',
        source: 'wikipedia_ru',
      });
      await upsertTranslation({
        entityType: 'artist',
        entityId: artist.id,
        locale: LOCALE,
        fieldName: 'bio_full',
        value: bio.bio_full,
        status: 'draft',
        source: 'wikipedia_ru',
      });

      console.log(`✓ ${artist.name} → ${bio.name}`);
      updated += 1;
    } catch (err) {
      console.error(`✗ ${artist.name} — ${err.message}`);
      failed += 1;
    }
  }

  console.log(`\nDone: ${updated} updated, ${skipped} skipped, ${failed} failed`);
  await pool.end();
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});