Danila KhodjaefandCursor
40db99d734
Express API with PostgreSQL, React/Vite/Three.js frontend, and locally cached artwork images. Co-authored-by: Cursor <cursoragent@cursor.com>
455 lines
16 KiB
JavaScript
455 lines
16 KiB
JavaScript
const https = require('https');
|
|
const http = require('http');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
const USER_AGENT = 'VirtualArtGallery/1.0 (educational art history project; local museum gallery)';
|
|
|
|
let lastRequestTime = 0;
|
|
const MIN_DELAY_MS = 2500;
|
|
const THUMB_WIDTH = 400;
|
|
const FULL_WIDTH = 1600;
|
|
|
|
const PAINTING_WIKI_OVERRIDES = {
|
|
'Job Cigarette Papers': 'Job (advertising)',
|
|
'Charing Cross Bridge': 'Charing Cross Bridge (Derain)',
|
|
'Cut with the Dada Kitchen Knife':
|
|
'Cut with the Dada Kitchen Knife through the Last Weimar Beer-Belly Cultural Epoch in Germany',
|
|
};
|
|
|
|
const DIRECT_IMAGE_OVERRIDES = {
|
|
'The Persistence of Memory': {
|
|
thumbUrl: 'https://upload.wikimedia.org/wikipedia/en/d/dd/The_Persistence_of_Memory.jpg',
|
|
fullUrl: 'https://upload.wikimedia.org/wikipedia/en/d/dd/The_Persistence_of_Memory.jpg',
|
|
source: 'Wikimedia Commons (Museum of Modern Art, New York)',
|
|
},
|
|
};
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async function throttle() {
|
|
const elapsed = Date.now() - lastRequestTime;
|
|
if (elapsed < MIN_DELAY_MS) await sleep(MIN_DELAY_MS - elapsed);
|
|
lastRequestTime = Date.now();
|
|
}
|
|
|
|
function fetchBuffer(url, redirectCount = 0, referer = null) {
|
|
return new Promise((resolve, reject) => {
|
|
if (redirectCount > 8) return reject(new Error('Too many redirects'));
|
|
const client = url.startsWith('https') ? https : http;
|
|
const headers = {
|
|
'User-Agent': USER_AGENT,
|
|
Accept: 'image/*,*/*',
|
|
};
|
|
if (referer) headers.Referer = referer;
|
|
if (url.includes('wikimedia.org') || url.includes('wikipedia.org')) {
|
|
headers.Referer = 'https://commons.wikimedia.org/';
|
|
}
|
|
if (url.includes('artic.edu')) {
|
|
headers.Referer = 'https://www.artic.edu/';
|
|
}
|
|
const req = client.get(url, { headers }, (res) => {
|
|
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
const next = res.headers.location.startsWith('http')
|
|
? res.headers.location
|
|
: new URL(res.headers.location, url).href;
|
|
return resolve(fetchBuffer(next, redirectCount + 1, referer));
|
|
}
|
|
if (res.statusCode === 429) {
|
|
res.resume();
|
|
return reject(new Error('HTTP 429'));
|
|
}
|
|
if (res.statusCode !== 200) {
|
|
res.resume();
|
|
return reject(new Error(`HTTP ${res.statusCode}`));
|
|
}
|
|
const chunks = [];
|
|
res.on('data', (c) => chunks.push(c));
|
|
res.on('end', () => resolve(Buffer.concat(chunks)));
|
|
}
|
|
);
|
|
req.on('error', reject);
|
|
});
|
|
}
|
|
|
|
async function fetchJson(url) {
|
|
await throttle();
|
|
return new Promise((resolve, reject) => {
|
|
const client = url.startsWith('https') ? https : http;
|
|
client
|
|
.get(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => {
|
|
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
return fetchJson(res.headers.location).then(resolve).catch(reject);
|
|
}
|
|
let data = '';
|
|
res.on('data', (c) => (data += c));
|
|
res.on('end', () => {
|
|
try {
|
|
resolve(JSON.parse(data));
|
|
} catch (e) {
|
|
reject(e);
|
|
}
|
|
});
|
|
})
|
|
.on('error', reject);
|
|
});
|
|
}
|
|
|
|
async function withRetry(fn, retries = 6) {
|
|
for (let i = 0; i < retries; i++) {
|
|
try {
|
|
return await fn();
|
|
} catch (err) {
|
|
if (i === retries - 1) throw err;
|
|
const wait = err.message.includes('429') ? 8000 * (i + 1) : 2000 * (i + 1);
|
|
await sleep(wait);
|
|
}
|
|
}
|
|
}
|
|
|
|
function pickExt(url) {
|
|
try {
|
|
const ext = path.extname(new URL(url).pathname) || '.jpg';
|
|
return ext.length > 5 ? '.jpg' : ext;
|
|
} catch {
|
|
return '.jpg';
|
|
}
|
|
}
|
|
|
|
async function getWikidataImageFilename(wikiTitle) {
|
|
const propsUrl = `https://en.wikipedia.org/w/api.php?action=query&titles=${encodeURIComponent(wikiTitle)}&prop=pageprops&ppprop=wikibase_item&format=json`;
|
|
const props = await fetchJson(propsUrl);
|
|
const page = Object.values(props.query?.pages || {})[0];
|
|
const qid = page?.pageprops?.wikibase_item;
|
|
if (!qid) return null;
|
|
|
|
const entityUrl = `https://www.wikidata.org/wiki/Special:EntityData/${qid}.json`;
|
|
const entity = await fetchJson(entityUrl);
|
|
const claims = entity.entities?.[qid]?.claims?.P18;
|
|
if (!claims?.length) return null;
|
|
return claims[0].mainsnak?.datavalue?.value;
|
|
}
|
|
|
|
async function getCommonsFileUrls(filename) {
|
|
const fileTitle = filename.startsWith('File:') ? filename : `File:${filename}`;
|
|
const url = `https://commons.wikimedia.org/w/api.php?action=query&titles=${encodeURIComponent(fileTitle)}&prop=imageinfo&iiprop=url&iiurlwidth=${THUMB_WIDTH}&format=json`;
|
|
const data = await fetchJson(url);
|
|
const page = Object.values(data.query?.pages || {})[0];
|
|
const info = page?.imageinfo?.[0];
|
|
if (!info) return null;
|
|
return {
|
|
thumbUrl: info.thumburl || info.url,
|
|
fullUrl: info.url || info.thumburl,
|
|
source: 'Wikimedia Commons',
|
|
};
|
|
}
|
|
|
|
async function getWikipediaImages(wikiTitle) {
|
|
const url = `https://en.wikipedia.org/w/api.php?action=query&titles=${encodeURIComponent(wikiTitle)}&prop=pageimages&piprop=original|thumbnail&pithumbsize=${THUMB_WIDTH}&format=json`;
|
|
const data = await fetchJson(url);
|
|
const page = Object.values(data.query?.pages || {})[0];
|
|
if (!page) return null;
|
|
const thumbUrl = page.thumbnail?.source;
|
|
const fullUrl = page.original?.source || thumbUrl;
|
|
if (!fullUrl) return null;
|
|
return {
|
|
thumbUrl: thumbUrl || fullUrl,
|
|
fullUrl,
|
|
source: 'Wikimedia Commons (via Wikipedia)',
|
|
};
|
|
}
|
|
|
|
async function searchCommonsImages(artistName, paintingTitle) {
|
|
const queries = [
|
|
[artistName, paintingTitle].filter(Boolean).join(' '),
|
|
paintingTitle,
|
|
[artistName, ...(paintingTitle || '').split(/\s+/).slice(0, 4)].join(' '),
|
|
[artistName, 'painting', (paintingTitle || '').split('(')[0].trim()].filter(Boolean).join(' '),
|
|
];
|
|
const seen = new Set();
|
|
|
|
for (const query of queries) {
|
|
if (!query || seen.has(query)) continue;
|
|
seen.add(query);
|
|
|
|
const url = `https://commons.wikimedia.org/w/api.php?action=query&generator=search&gssearch=${encodeURIComponent(query)}&gsnamespace=6&gslimit=8&prop=imageinfo&iiprop=url&iiurlwidth=${THUMB_WIDTH}&format=json`;
|
|
try {
|
|
const data = await fetchJson(url);
|
|
const pages = data.query?.pages;
|
|
if (!pages) continue;
|
|
|
|
const artistToken = (artistName || '').toLowerCase().split(' ').pop() || '';
|
|
const titleTokens = (paintingTitle || '')
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9\s]/g, ' ')
|
|
.split(/\s+/)
|
|
.filter((t) => t.length > 3)
|
|
.slice(0, 4);
|
|
|
|
for (const page of Object.values(pages)) {
|
|
const info = page.imageinfo?.[0];
|
|
if (!info?.url) continue;
|
|
const fileTitle = (page.title || '').toLowerCase();
|
|
if (artistToken && !fileTitle.includes(artistToken) && !fileTitle.includes('raphael') && !fileTitle.includes('raffael')) {
|
|
continue;
|
|
}
|
|
const matchCount = titleTokens.filter((t) => fileTitle.includes(t)).length;
|
|
if (titleTokens.length > 0 && matchCount === 0 && !fileTitle.includes('uffizi')) continue;
|
|
|
|
return {
|
|
thumbUrl: info.thumburl || info.url,
|
|
fullUrl: info.url,
|
|
source: 'Wikimedia Commons (search)',
|
|
};
|
|
}
|
|
} catch {
|
|
continue;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getMetMuseumImages(searchTerm, artistName) {
|
|
const query = [artistName, searchTerm].filter(Boolean).join(' ');
|
|
const searchUrl = `https://collectionapi.metmuseum.org/public/collection/v1/search?q=${encodeURIComponent(query)}&hasImages=true`;
|
|
try {
|
|
await throttle();
|
|
const search = await fetchJson(searchUrl);
|
|
if (!search.objectIDs?.length) return null;
|
|
|
|
for (const id of search.objectIDs.slice(0, 10)) {
|
|
await throttle();
|
|
const obj = await fetchJson(
|
|
`https://collectionapi.metmuseum.org/public/collection/v1/objects/${id}`
|
|
);
|
|
if (!obj.isPublicDomain || !obj.primaryImage) continue;
|
|
const titleMatch =
|
|
!searchTerm ||
|
|
(obj.title || '').toLowerCase().includes(searchTerm.toLowerCase().slice(0, 10));
|
|
if (!titleMatch && search.objectIDs.length > 3) continue;
|
|
return {
|
|
thumbUrl: obj.primaryImageSmall || obj.primaryImage,
|
|
fullUrl: obj.primaryImage,
|
|
source: 'The Metropolitan Museum of Art (Open Access)',
|
|
};
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getArtInstituteImages(searchTerm, artistName) {
|
|
const query = [artistName, searchTerm].filter(Boolean).join(' ');
|
|
const url = `https://api.artic.edu/api/v1/artworks/search?q=${encodeURIComponent(query)}&fields=id,title,artist_title,image_id&limit=6`;
|
|
try {
|
|
const data = await fetchJson(url);
|
|
for (const art of data.data || []) {
|
|
if (!art.image_id) continue;
|
|
const titleOk =
|
|
!searchTerm ||
|
|
(art.title || '').toLowerCase().includes(searchTerm.toLowerCase().slice(0, 8));
|
|
const artistOk =
|
|
!artistName ||
|
|
(art.artist_title || '').toLowerCase().includes(artistName.toLowerCase().split(' ').pop());
|
|
if (!titleOk && !artistOk) continue;
|
|
const base = `https://www.artic.edu/iiif/2/${art.image_id}`;
|
|
return {
|
|
thumbUrl: `${base}/full/200,/0/default.jpg`,
|
|
fullUrl: `${base}/full/1686,/0/default.jpg`,
|
|
source: 'Art Institute of Chicago (Open Access)',
|
|
};
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getRijksmuseumImages(searchTerm, artistName) {
|
|
const query = [artistName, searchTerm].filter(Boolean).join(' ');
|
|
const url = `https://www.rijksmuseum.nl/api/en/collection?key=0&format=json&ps=8&s=relevance&q=${encodeURIComponent(query)}&imgonly=true`;
|
|
try {
|
|
const data = await fetchJson(url);
|
|
for (const item of data.artObjects || []) {
|
|
if (!item.webImage?.url) continue;
|
|
return {
|
|
thumbUrl: item.webImage.url,
|
|
fullUrl: item.webImage.url,
|
|
source: 'Rijksmuseum',
|
|
};
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getWikidataImagesBySearch(artistName, paintingTitle) {
|
|
const label = (paintingTitle || '').replace(/"/g, '\\"').slice(0, 120);
|
|
const query = `
|
|
SELECT ?img ?imgLabel WHERE {
|
|
?work rdfs:label ?workLabel .
|
|
FILTER(LANG(?workLabel) = "en")
|
|
FILTER(CONTAINS(LCASE(?workLabel), LCASE("${label.slice(0, 40)}")))
|
|
?work wdt:P170 ?artist .
|
|
?artist rdfs:label ?artistLabel .
|
|
FILTER(CONTAINS(LCASE(?artistLabel), "${(artistName || '').split(' ').pop().toLowerCase()}"))
|
|
?work wdt:P18 ?img .
|
|
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
|
|
}
|
|
LIMIT 3`;
|
|
try {
|
|
const url = `https://query.wikidata.org/sparql?format=json&query=${encodeURIComponent(query)}`;
|
|
const data = await fetchJson(url);
|
|
const row = data.results?.bindings?.[0];
|
|
if (!row?.img?.value) return null;
|
|
const filename = decodeURIComponent(row.img.value.split('/').pop());
|
|
return getCommonsFileUrls(filename);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function resolvePaintingImages(wikiTitle, options = {}) {
|
|
const { artistName, paintingTitle } = options;
|
|
const searchTitle = paintingTitle || wikiTitle;
|
|
const lookupTitle = PAINTING_WIKI_OVERRIDES[searchTitle] || wikiTitle;
|
|
|
|
if (DIRECT_IMAGE_OVERRIDES[searchTitle]) {
|
|
return DIRECT_IMAGE_OVERRIDES[searchTitle];
|
|
}
|
|
|
|
const sources = [
|
|
async () => {
|
|
const filename = await getWikidataImageFilename(lookupTitle);
|
|
if (!filename) return null;
|
|
const commons = await getCommonsFileUrls(filename);
|
|
if (commons) return commons;
|
|
const filePathUrl = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(filename)}`;
|
|
return { thumbUrl: filePathUrl, fullUrl: filePathUrl, source: 'Wikimedia Commons (Wikidata)' };
|
|
},
|
|
() => getWikipediaImages(lookupTitle),
|
|
() => searchCommonsImages(artistName, searchTitle),
|
|
() => getMetMuseumImages(searchTitle, artistName),
|
|
() => getArtInstituteImages(searchTitle, artistName),
|
|
() => getRijksmuseumImages(searchTitle, artistName),
|
|
() => getWikidataImagesBySearch(artistName, searchTitle),
|
|
() => getWikipediaImages(searchTitle),
|
|
() => searchCommonsImages(artistName, lookupTitle),
|
|
];
|
|
|
|
for (const fn of sources) {
|
|
try {
|
|
const result = await fn();
|
|
if (result?.fullUrl || result?.thumbUrl) {
|
|
return {
|
|
thumbUrl: result.thumbUrl || result.fullUrl,
|
|
fullUrl: result.fullUrl || result.thumbUrl,
|
|
source: result.source,
|
|
};
|
|
}
|
|
} catch {}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/** @deprecated use resolvePaintingImages */
|
|
async function resolveImageUrl(wikiTitle, options = {}) {
|
|
const pair = await resolvePaintingImages(wikiTitle, options);
|
|
if (!pair) return null;
|
|
return { url: pair.fullUrl, source: pair.source };
|
|
}
|
|
|
|
async function downloadImageToFile(url, destPath) {
|
|
if (fs.existsSync(destPath)) return destPath;
|
|
await throttle();
|
|
const buffer = await withRetry(() => fetchBuffer(url));
|
|
fs.writeFileSync(destPath, buffer);
|
|
return destPath;
|
|
}
|
|
|
|
async function savePaintingImages(wikiTitle, baseFilename, imageDir, options = {}) {
|
|
const resolved = await resolvePaintingImages(wikiTitle, options);
|
|
if (!resolved) return { imagePath: null, thumbnailPath: null, source: null };
|
|
|
|
const paintingsDir = path.join(imageDir, 'paintings');
|
|
const thumbsDir = path.join(imageDir, 'paintings', 'thumbs');
|
|
if (!fs.existsSync(paintingsDir)) fs.mkdirSync(paintingsDir, { recursive: true });
|
|
if (!fs.existsSync(thumbsDir)) fs.mkdirSync(thumbsDir, { recursive: true });
|
|
|
|
const safeBase = baseFilename.replace(/[^a-zA-Z0-9_-]/g, '_');
|
|
const fullExt = pickExt(resolved.fullUrl);
|
|
const thumbExt = pickExt(resolved.thumbUrl);
|
|
|
|
const fullDest = path.join(paintingsDir, safeBase + fullExt);
|
|
const thumbDest = path.join(thumbsDir, safeBase + '_thumb' + thumbExt);
|
|
|
|
let imagePath = null;
|
|
let thumbnailPath = null;
|
|
|
|
try {
|
|
await downloadImageToFile(resolved.fullUrl, fullDest);
|
|
imagePath = path.join('paintings', safeBase + fullExt).replace(/\\/g, '/');
|
|
} catch (err) {
|
|
console.warn(` Full image download failed: ${err.message}`);
|
|
}
|
|
|
|
try {
|
|
const thumbSrc =
|
|
resolved.thumbUrl !== resolved.fullUrl ? resolved.thumbUrl : resolved.fullUrl;
|
|
await downloadImageToFile(thumbSrc, thumbDest);
|
|
thumbnailPath = path.join('paintings', 'thumbs', safeBase + '_thumb' + thumbExt).replace(/\\/g, '/');
|
|
} catch (err) {
|
|
console.warn(` Thumbnail download failed: ${err.message}`);
|
|
if (imagePath) thumbnailPath = imagePath;
|
|
}
|
|
|
|
if (!imagePath && thumbnailPath) imagePath = thumbnailPath;
|
|
|
|
return { imagePath, thumbnailPath, source: resolved.source };
|
|
}
|
|
|
|
async function saveImageForItem(wikiTitle, subdir, filename, imageDir, options = {}) {
|
|
if (options.type === 'painting') {
|
|
const result = await savePaintingImages(wikiTitle, filename, imageDir, options);
|
|
return {
|
|
path: result.imagePath,
|
|
thumbnailPath: result.thumbnailPath,
|
|
source: result.source,
|
|
};
|
|
}
|
|
|
|
const resolved = await resolveImageUrl(wikiTitle, options);
|
|
if (!resolved) return { path: null, source: null };
|
|
|
|
const dir = path.join(imageDir, subdir);
|
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
|
|
const ext = pickExt(resolved.url);
|
|
const safeName = filename.replace(/[^a-zA-Z0-9_-]/g, '_') + ext;
|
|
const destPath = path.join(dir, safeName);
|
|
|
|
try {
|
|
await downloadImageToFile(resolved.url, destPath);
|
|
return {
|
|
path: path.join(subdir, safeName).replace(/\\/g, '/'),
|
|
source: resolved.source,
|
|
};
|
|
} catch (err) {
|
|
console.warn(` Download failed (${resolved.source}): ${err.message}`);
|
|
return { path: null, source: resolved.source };
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
resolveImageUrl,
|
|
resolvePaintingImages,
|
|
saveImageForItem,
|
|
savePaintingImages,
|
|
downloadImageToFile,
|
|
sleep,
|
|
};
|