require('dotenv').config(); const https = require('https'); const pool = require('../server/db'); const { upsertTranslation } = require('../server/translation-service'); const USER_AGENT = 'VirtualArtGallery/1.0 (educational art history project; local museum gallery)'; const MIN_DELAY_MS = 3500; const MAX_RETRIES = 8; const LOCALE = 'ru'; const WIKI_API = 'https://ru.wikipedia.org/w/api.php'; const ARTIST_WIKI_OVERRIDES = { Zeuxis: 'Zeuxis (painter)', 'Ivan Klyun': 'Ivan Kliun', 'Jean-Antoine Watteau': 'Antoine Watteau', }; let lastRequestTime = 0; function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } async function throttle() { const elapsed = Date.now() - lastRequestTime; if (elapsed < MIN_DELAY_MS) await sleep(MIN_DELAY_MS - elapsed); lastRequestTime = Date.now(); } function fetchJson(url) { return new Promise((resolve, reject) => { https .get(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { return fetchJson(res.headers.location).then(resolve).catch(reject); } let data = ''; res.on('data', (c) => (data += c)); res.on('end', () => { if (res.statusCode !== 200) return reject(new Error(`HTTP ${res.statusCode}`)); try { resolve(JSON.parse(data)); } catch (e) { reject(e); } }); }) .on('error', reject); }); } async function withRetry(fn) { for (let i = 0; i < MAX_RETRIES; i++) { try { return await fn(); } catch (err) { if (i === MAX_RETRIES - 1) throw err; await sleep(4000 * (i + 1)); } } } function firstSentences(text, count = 2) { const parts = text.match(/[^.!?]+[.!?]+(?:\s|$)/g); if (!parts?.length) return text.trim(); return parts.slice(0, count).join('').trim(); } async function resolveRuPageTitle(enTitle) { await throttle(); const params = new URLSearchParams({ action: 'query', format: 'json', titles: enTitle, prop: 'langlinks', lllang: 'ru', lllimit: '1', }); const data = await withRetry(() => fetchJson(`https://en.wikipedia.org/w/api.php?${params}`)); const pages = data.query?.pages || {}; const page = Object.values(pages)[0]; const link = page?.langlinks?.[0]; if (link?.['*']) return link['*']; return enTitle; } async function fetchRuExtract(title) { await throttle(); const params = new URLSearchParams({ action: 'query', format: 'json', prop: 'extracts|pageprops', explaintext: '1', exintro: '0', ppprop: 'disambiguation', titles: title, }); const data = await withRetry(() => fetchJson(`${WIKI_API}?${params}`)); const pages = data.query?.pages || {}; const page = Object.values(pages)[0]; if (!page || page.missing !== undefined) return null; const extract = page.extract?.trim(); if (!extract) return null; return { title: page.title, extract }; } async function resolveArtistBioRu(artist) { const enTitle = ARTIST_WIKI_OVERRIDES[artist.wikipedia_title] || ARTIST_WIKI_OVERRIDES[artist.name] || artist.wikipedia_title || artist.name; const ruTitle = await resolveRuPageTitle(enTitle); const page = await fetchRuExtract(ruTitle); if (!page) return null; return { name: page.title, bio_short: firstSentences(page.extract, 2), bio_full: page.extract, }; } async function main() { const force = process.argv.includes('--force'); const limitArg = process.argv.find((a) => a.startsWith('--limit=')); const limit = limitArg ? parseInt(limitArg.split('=')[1], 10) : null; const { rows: artists } = await pool.query(` SELECT id, name, wikipedia_title FROM artists ORDER BY name `); let targets = artists; if (limit) targets = targets.slice(0, limit); let updated = 0; let skipped = 0; let failed = 0; for (const artist of targets) { if (!force) { const { rows: existing } = await pool.query( `SELECT 1 FROM entity_translations WHERE entity_type = 'artist' AND entity_id = $1 AND locale = $2 AND field_name = 'bio_full'`, [artist.id, LOCALE], ); if (existing.length) { skipped += 1; continue; } } try { const bio = await resolveArtistBioRu(artist); if (!bio) { console.warn(`✗ ${artist.name} — no ru.wikipedia article`); failed += 1; continue; } await upsertTranslation({ entityType: 'artist', entityId: artist.id, locale: LOCALE, fieldName: 'name', value: bio.name, status: 'draft', source: 'wikipedia_ru', }); await upsertTranslation({ entityType: 'artist', entityId: artist.id, locale: LOCALE, fieldName: 'bio_short', value: bio.bio_short, status: 'draft', source: 'wikipedia_ru', }); await upsertTranslation({ entityType: 'artist', entityId: artist.id, locale: LOCALE, fieldName: 'bio_full', value: bio.bio_full, status: 'draft', source: 'wikipedia_ru', }); console.log(`✓ ${artist.name} → ${bio.name}`); updated += 1; } catch (err) { console.error(`✗ ${artist.name} — ${err.message}`); failed += 1; } } console.log(`\nDone: ${updated} updated, ${skipped} skipped, ${failed} failed`); await pool.end(); } main().catch((err) => { console.error(err); process.exit(1); });