Add Russian i18n with DB translations, locale API, and curator review UI.
UI chrome via react-i18next, catalog text in entity_translations with ru.wikipedia seeding, locale-aware search, and Translations page for publish workflow. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
f247b418d8
commit
ca58c43648
@@ -0,0 +1,202 @@
|
||||
require('dotenv').config();
|
||||
const https = require('https');
|
||||
const pool = require('../server/db');
|
||||
const { upsertTranslation } = require('../server/translation-service');
|
||||
|
||||
const USER_AGENT = 'VirtualArtGallery/1.0 (educational art history project; local museum gallery)';
|
||||
const MIN_DELAY_MS = 3500;
|
||||
const MAX_RETRIES = 8;
|
||||
const LOCALE = 'ru';
|
||||
const WIKI_API = 'https://ru.wikipedia.org/w/api.php';
|
||||
|
||||
const ARTIST_WIKI_OVERRIDES = {
|
||||
Zeuxis: 'Zeuxis (painter)',
|
||||
'Ivan Klyun': 'Ivan Kliun',
|
||||
'Jean-Antoine Watteau': 'Antoine Watteau',
|
||||
};
|
||||
|
||||
let lastRequestTime = 0;
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function throttle() {
|
||||
const elapsed = Date.now() - lastRequestTime;
|
||||
if (elapsed < MIN_DELAY_MS) await sleep(MIN_DELAY_MS - elapsed);
|
||||
lastRequestTime = Date.now();
|
||||
}
|
||||
|
||||
function fetchJson(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
https
|
||||
.get(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => {
|
||||
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
||||
return fetchJson(res.headers.location).then(resolve).catch(reject);
|
||||
}
|
||||
let data = '';
|
||||
res.on('data', (c) => (data += c));
|
||||
res.on('end', () => {
|
||||
if (res.statusCode !== 200) return reject(new Error(`HTTP ${res.statusCode}`));
|
||||
try {
|
||||
resolve(JSON.parse(data));
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
})
|
||||
.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function withRetry(fn) {
|
||||
for (let i = 0; i < MAX_RETRIES; i++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
if (i === MAX_RETRIES - 1) throw err;
|
||||
await sleep(4000 * (i + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function firstSentences(text, count = 2) {
|
||||
const parts = text.match(/[^.!?]+[.!?]+(?:\s|$)/g);
|
||||
if (!parts?.length) return text.trim();
|
||||
return parts.slice(0, count).join('').trim();
|
||||
}
|
||||
|
||||
async function resolveRuPageTitle(enTitle) {
|
||||
await throttle();
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: enTitle,
|
||||
prop: 'langlinks',
|
||||
lllang: 'ru',
|
||||
lllimit: '1',
|
||||
});
|
||||
const data = await withRetry(() => fetchJson(`https://en.wikipedia.org/w/api.php?${params}`));
|
||||
const pages = data.query?.pages || {};
|
||||
const page = Object.values(pages)[0];
|
||||
const link = page?.langlinks?.[0];
|
||||
if (link?.['*']) return link['*'];
|
||||
return enTitle;
|
||||
}
|
||||
|
||||
async function fetchRuExtract(title) {
|
||||
await throttle();
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
prop: 'extracts|pageprops',
|
||||
explaintext: '1',
|
||||
exintro: '0',
|
||||
ppprop: 'disambiguation',
|
||||
titles: title,
|
||||
});
|
||||
const data = await withRetry(() => fetchJson(`${WIKI_API}?${params}`));
|
||||
const pages = data.query?.pages || {};
|
||||
const page = Object.values(pages)[0];
|
||||
if (!page || page.missing !== undefined) return null;
|
||||
const extract = page.extract?.trim();
|
||||
if (!extract) return null;
|
||||
return { title: page.title, extract };
|
||||
}
|
||||
|
||||
async function resolveArtistBioRu(artist) {
|
||||
const enTitle = ARTIST_WIKI_OVERRIDES[artist.wikipedia_title]
|
||||
|| ARTIST_WIKI_OVERRIDES[artist.name]
|
||||
|| artist.wikipedia_title
|
||||
|| artist.name;
|
||||
const ruTitle = await resolveRuPageTitle(enTitle);
|
||||
const page = await fetchRuExtract(ruTitle);
|
||||
if (!page) return null;
|
||||
return {
|
||||
name: page.title,
|
||||
bio_short: firstSentences(page.extract, 2),
|
||||
bio_full: page.extract,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const force = process.argv.includes('--force');
|
||||
const limitArg = process.argv.find((a) => a.startsWith('--limit='));
|
||||
const limit = limitArg ? parseInt(limitArg.split('=')[1], 10) : null;
|
||||
|
||||
const { rows: artists } = await pool.query(`
|
||||
SELECT id, name, wikipedia_title FROM artists ORDER BY name
|
||||
`);
|
||||
|
||||
let targets = artists;
|
||||
if (limit) targets = targets.slice(0, limit);
|
||||
|
||||
let updated = 0;
|
||||
let skipped = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const artist of targets) {
|
||||
if (!force) {
|
||||
const { rows: existing } = await pool.query(
|
||||
`SELECT 1 FROM entity_translations
|
||||
WHERE entity_type = 'artist' AND entity_id = $1 AND locale = $2 AND field_name = 'bio_full'`,
|
||||
[artist.id, LOCALE],
|
||||
);
|
||||
if (existing.length) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const bio = await resolveArtistBioRu(artist);
|
||||
if (!bio) {
|
||||
console.warn(`✗ ${artist.name} — no ru.wikipedia article`);
|
||||
failed += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
await upsertTranslation({
|
||||
entityType: 'artist',
|
||||
entityId: artist.id,
|
||||
locale: LOCALE,
|
||||
fieldName: 'name',
|
||||
value: bio.name,
|
||||
status: 'draft',
|
||||
source: 'wikipedia_ru',
|
||||
});
|
||||
await upsertTranslation({
|
||||
entityType: 'artist',
|
||||
entityId: artist.id,
|
||||
locale: LOCALE,
|
||||
fieldName: 'bio_short',
|
||||
value: bio.bio_short,
|
||||
status: 'draft',
|
||||
source: 'wikipedia_ru',
|
||||
});
|
||||
await upsertTranslation({
|
||||
entityType: 'artist',
|
||||
entityId: artist.id,
|
||||
locale: LOCALE,
|
||||
fieldName: 'bio_full',
|
||||
value: bio.bio_full,
|
||||
status: 'draft',
|
||||
source: 'wikipedia_ru',
|
||||
});
|
||||
|
||||
console.log(`✓ ${artist.name} → ${bio.name}`);
|
||||
updated += 1;
|
||||
} catch (err) {
|
||||
console.error(`✗ ${artist.name} — ${err.message}`);
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nDone: ${updated} updated, ${skipped} skipped, ${failed} failed`);
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user