Add Russian i18n with DB translations, locale API, and curator review UI.
UI chrome via react-i18next, catalog text in entity_translations with ru.wikipedia seeding, locale-aware search, and Translations page for publish workflow. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Danila KhodjaefandCursor committed
1 parent
f247b418d8
commit
ca58c43648
51 files changed
+2252
-101
No files matched your search
@@ -0,0 +1,202 @@
|
||||
require('dotenv').config();
|
||||
const https = require('https');
|
||||
const pool = require('../server/db');
|
||||
const { upsertTranslation } = require('../server/translation-service');
|
||||
|
||||
const USER_AGENT = 'VirtualArtGallery/1.0 (educational art history project; local museum gallery)';
|
||||
const MIN_DELAY_MS = 3500;
|
||||
const MAX_RETRIES = 8;
|
||||
const LOCALE = 'ru';
|
||||
const WIKI_API = 'https://ru.wikipedia.org/w/api.php';
|
||||
|
||||
const ARTIST_WIKI_OVERRIDES = {
|
||||
Zeuxis: 'Zeuxis (painter)',
|
||||
'Ivan Klyun': 'Ivan Kliun',
|
||||
'Jean-Antoine Watteau': 'Antoine Watteau',
|
||||
};
|
||||
|
||||
let lastRequestTime = 0;
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function throttle() {
|
||||
const elapsed = Date.now() - lastRequestTime;
|
||||
if (elapsed < MIN_DELAY_MS) await sleep(MIN_DELAY_MS - elapsed);
|
||||
lastRequestTime = Date.now();
|
||||
}
|
||||
|
||||
function fetchJson(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
https
|
||||
.get(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => {
|
||||
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
||||
return fetchJson(res.headers.location).then(resolve).catch(reject);
|
||||
}
|
||||
let data = '';
|
||||
res.on('data', (c) => (data += c));
|
||||
res.on('end', () => {
|
||||
if (res.statusCode !== 200) return reject(new Error(`HTTP ${res.statusCode}`));
|
||||
try {
|
||||
resolve(JSON.parse(data));
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
})
|
||||
.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
async function withRetry(fn) {
|
||||
for (let i = 0; i < MAX_RETRIES; i++) {
|
||||
try {
|
||||
return await fn();
|
||||
} catch (err) {
|
||||
if (i === MAX_RETRIES - 1) throw err;
|
||||
await sleep(4000 * (i + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function firstSentences(text, count = 2) {
|
||||
const parts = text.match(/[^.!?]+[.!?]+(?:\s|$)/g);
|
||||
if (!parts?.length) return text.trim();
|
||||
return parts.slice(0, count).join('').trim();
|
||||
}
|
||||
|
||||
async function resolveRuPageTitle(enTitle) {
|
||||
await throttle();
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
titles: enTitle,
|
||||
prop: 'langlinks',
|
||||
lllang: 'ru',
|
||||
lllimit: '1',
|
||||
});
|
||||
const data = await withRetry(() => fetchJson(`https://en.wikipedia.org/w/api.php?${params}`));
|
||||
const pages = data.query?.pages || {};
|
||||
const page = Object.values(pages)[0];
|
||||
const link = page?.langlinks?.[0];
|
||||
if (link?.['*']) return link['*'];
|
||||
return enTitle;
|
||||
}
|
||||
|
||||
async function fetchRuExtract(title) {
|
||||
await throttle();
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
prop: 'extracts|pageprops',
|
||||
explaintext: '1',
|
||||
exintro: '0',
|
||||
ppprop: 'disambiguation',
|
||||
titles: title,
|
||||
});
|
||||
const data = await withRetry(() => fetchJson(`${WIKI_API}?${params}`));
|
||||
const pages = data.query?.pages || {};
|
||||
const page = Object.values(pages)[0];
|
||||
if (!page || page.missing !== undefined) return null;
|
||||
const extract = page.extract?.trim();
|
||||
if (!extract) return null;
|
||||
return { title: page.title, extract };
|
||||
}
|
||||
|
||||
async function resolveArtistBioRu(artist) {
|
||||
const enTitle = ARTIST_WIKI_OVERRIDES[artist.wikipedia_title]
|
||||
|| ARTIST_WIKI_OVERRIDES[artist.name]
|
||||
|| artist.wikipedia_title
|
||||
|| artist.name;
|
||||
const ruTitle = await resolveRuPageTitle(enTitle);
|
||||
const page = await fetchRuExtract(ruTitle);
|
||||
if (!page) return null;
|
||||
return {
|
||||
name: page.title,
|
||||
bio_short: firstSentences(page.extract, 2),
|
||||
bio_full: page.extract,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const force = process.argv.includes('--force');
|
||||
const limitArg = process.argv.find((a) => a.startsWith('--limit='));
|
||||
const limit = limitArg ? parseInt(limitArg.split('=')[1], 10) : null;
|
||||
|
||||
const { rows: artists } = await pool.query(`
|
||||
SELECT id, name, wikipedia_title FROM artists ORDER BY name
|
||||
`);
|
||||
|
||||
let targets = artists;
|
||||
if (limit) targets = targets.slice(0, limit);
|
||||
|
||||
let updated = 0;
|
||||
let skipped = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (const artist of targets) {
|
||||
if (!force) {
|
||||
const { rows: existing } = await pool.query(
|
||||
`SELECT 1 FROM entity_translations
|
||||
WHERE entity_type = 'artist' AND entity_id = $1 AND locale = $2 AND field_name = 'bio_full'`,
|
||||
[artist.id, LOCALE],
|
||||
);
|
||||
if (existing.length) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const bio = await resolveArtistBioRu(artist);
|
||||
if (!bio) {
|
||||
console.warn(`✗ ${artist.name} — no ru.wikipedia article`);
|
||||
failed += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
await upsertTranslation({
|
||||
entityType: 'artist',
|
||||
entityId: artist.id,
|
||||
locale: LOCALE,
|
||||
fieldName: 'name',
|
||||
value: bio.name,
|
||||
status: 'draft',
|
||||
source: 'wikipedia_ru',
|
||||
});
|
||||
await upsertTranslation({
|
||||
entityType: 'artist',
|
||||
entityId: artist.id,
|
||||
locale: LOCALE,
|
||||
fieldName: 'bio_short',
|
||||
value: bio.bio_short,
|
||||
status: 'draft',
|
||||
source: 'wikipedia_ru',
|
||||
});
|
||||
await upsertTranslation({
|
||||
entityType: 'artist',
|
||||
entityId: artist.id,
|
||||
locale: LOCALE,
|
||||
fieldName: 'bio_full',
|
||||
value: bio.bio_full,
|
||||
status: 'draft',
|
||||
source: 'wikipedia_ru',
|
||||
});
|
||||
|
||||
console.log(`✓ ${artist.name} → ${bio.name}`);
|
||||
updated += 1;
|
||||
} catch (err) {
|
||||
console.error(`✗ ${artist.name} — ${err.message}`);
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\nDone: ${updated} updated, ${skipped} skipped, ${failed} failed`);
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -29,6 +29,7 @@ const CATALOG_TABLES = [
|
||||
'painting_influences',
|
||||
'painting_influence_sources',
|
||||
'painting_annotations',
|
||||
'entity_translations',
|
||||
];
|
||||
|
||||
const NATURAL_KEY_FN = {
|
||||
@@ -40,6 +41,7 @@ const NATURAL_KEY_FN = {
|
||||
painting_influences: (r) => `${r.painting_id}:${r.influenced_by_painting_id}`,
|
||||
painting_influence_sources: (r) => `${r.painting_id}:${r.source_type}:${r.source_painting_id || 0}:${r.source_artist_id || 0}:${r.source_movement_id || 0}`,
|
||||
painting_annotations: (r) => `${r.painting_id}:${String(r.label || '').trim().toLowerCase()}:${r.sort_order}`,
|
||||
entity_translations: (r) => `${r.entity_type}:${r.entity_id}:${r.locale}:${r.field_name}`,
|
||||
};
|
||||
|
||||
function parseArgs(argv) {
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
/**
|
||||
* Import translation rows from JSON or CSV.
|
||||
*
|
||||
* JSON format: [{ "entity_type": "artist", "entity_id": 1, "field_name": "name", "value": "...", "status": "draft" }]
|
||||
* CSV format: entity_type,entity_id,field_name,value[,status][,source]
|
||||
*/
|
||||
require('dotenv').config();
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { upsertTranslation } = require('../server/translation-service');
|
||||
const pool = require('../server/db');
|
||||
|
||||
function parseArgs(argv) {
|
||||
const fileIdx = argv.indexOf('--file');
|
||||
const localeIdx = argv.indexOf('--locale');
|
||||
if (fileIdx === -1 || !argv[fileIdx + 1]) {
|
||||
throw new Error('--file <path> is required');
|
||||
}
|
||||
return {
|
||||
filePath: path.resolve(argv[fileIdx + 1]),
|
||||
locale: localeIdx !== -1 && argv[localeIdx + 1] ? argv[localeIdx + 1] : 'ru',
|
||||
publish: argv.includes('--publish'),
|
||||
};
|
||||
}
|
||||
|
||||
function parseCsv(content) {
|
||||
const lines = content.split(/\r?\n/).filter((l) => l.trim() && !l.trim().startsWith('#'));
|
||||
const rows = [];
|
||||
for (const line of lines) {
|
||||
const parts = line.split(',').map((p) => p.trim().replace(/^"|"$/g, ''));
|
||||
if (parts[0] === 'entity_type') continue;
|
||||
if (parts.length < 4) continue;
|
||||
rows.push({
|
||||
entity_type: parts[0],
|
||||
entity_id: parseInt(parts[1], 10),
|
||||
field_name: parts[2],
|
||||
value: parts[3],
|
||||
status: parts[4] || 'draft',
|
||||
source: parts[5] || 'import_csv',
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { filePath, locale, publish } = parseArgs(process.argv.slice(2));
|
||||
const raw = fs.readFileSync(filePath, 'utf8');
|
||||
let rows;
|
||||
if (filePath.endsWith('.json')) {
|
||||
rows = JSON.parse(raw);
|
||||
} else {
|
||||
rows = parseCsv(raw);
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
for (const row of rows) {
|
||||
if (!row.entity_type || !row.entity_id || !row.field_name || row.value == null) continue;
|
||||
const status = publish ? 'published' : (row.status || 'draft');
|
||||
await upsertTranslation({
|
||||
entityType: row.entity_type,
|
||||
entityId: row.entity_id,
|
||||
locale: row.locale || locale,
|
||||
fieldName: row.field_name,
|
||||
value: String(row.value),
|
||||
status,
|
||||
source: row.source || 'import',
|
||||
});
|
||||
count += 1;
|
||||
}
|
||||
|
||||
console.log(`Imported ${count} translation rows from ${filePath}`);
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err.message);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
require('dotenv').config();
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const pool = require('../server/db');
|
||||
|
||||
async function main() {
|
||||
const sqlPath = path.join(__dirname, '../db/migrate-i18n.sql');
|
||||
const sql = fs.readFileSync(sqlPath, 'utf8');
|
||||
await pool.query(sql);
|
||||
console.log('entity_translations table ready');
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in new issue
Block a user