1922 lines
64 KiB
JavaScript
1922 lines
64 KiB
JavaScript
const https = require('https');
|
|
const http = require('http');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
const USER_AGENT = 'VirtualArtGallery/1.0 (educational art history project; local museum gallery)';
|
|
const BROWSER_USER_AGENT =
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
|
|
|
const WIKI_LANGS = ['de', 'fr', 'it', 'ru'];
|
|
|
|
const TRUSTED_IMAGE_HOSTS = [
|
|
'upload.wikimedia.org',
|
|
'commons.wikimedia.org',
|
|
'collections.louvre.fr',
|
|
'artic.edu',
|
|
'images.metmuseum.org',
|
|
'collectionapi.metmuseum.org',
|
|
'ids.si.edu',
|
|
'harvardartmuseums.org',
|
|
'rijksmuseum.nl',
|
|
'openaccess-api.clevelandart.org',
|
|
'api.europeana.eu',
|
|
'hermitagemuseum.org',
|
|
'pushkinmuseum.art',
|
|
'googleusercontent.com',
|
|
'artsandculture.google.com',
|
|
];
|
|
|
|
let lastRequestTime = 0;
|
|
const MIN_DELAY_MS = 2500;
|
|
const REQUEST_TIMEOUT_MS = 15000;
|
|
const DEFAULT_BATCH_MAX_WAIT_MS = 10000;
|
|
const THUMB_WIDTH = 400;
|
|
const FULL_WIDTH = 1600;
|
|
|
|
const PAINTING_WIKI_OVERRIDES = {
|
|
'Job Cigarette Papers': 'Job (advertising)',
|
|
'Charing Cross Bridge': 'Charing Cross Bridge (Derain)',
|
|
'Cut with the Dada Kitchen Knife':
|
|
'Cut with the Dada Kitchen Knife through the Last Weimar Beer-Belly Cultural Epoch in Germany',
|
|
'Cherubs of the Sistine Madonna': "Raphael's Cherubs",
|
|
'Madonna and Child (Madonna della Seggiola)': 'Madonna della Seggiola',
|
|
'Madonna della seggiola': 'Madonna della Seggiola',
|
|
'La Madonna della sedia': 'Madonna della Seggiola',
|
|
'Madonna della sedia': 'Madonna della Seggiola',
|
|
'Raphael::Madonna and Child': 'Small Cowper Madonna',
|
|
};
|
|
|
|
const MADONNA_SEGGIOLA_IMAGE = {
|
|
thumbUrl:
|
|
'https://upload.wikimedia.org/wikipedia/commons/thumb/0/09/Raphael_Madonna_della_seggiola.jpg/960px-Raphael_Madonna_della_seggiola.jpg',
|
|
fullUrl: 'https://upload.wikimedia.org/wikipedia/commons/0/09/Raphael_Madonna_della_seggiola.jpg',
|
|
source: 'Wikimedia Commons (Palazzo Pitti, Florence)',
|
|
};
|
|
|
|
const SMALL_COWPER_MADONNA_IMAGE = {
|
|
thumbUrl:
|
|
'https://upload.wikimedia.org/wikipedia/commons/thumb/b/b1/Raffaello_Madonna_Cowper.jpg/960px-Raffaello_Madonna_Cowper.jpg',
|
|
fullUrl: 'https://upload.wikimedia.org/wikipedia/commons/b/b1/Raffaello_Madonna_Cowper.jpg',
|
|
source: 'Wikimedia Commons (National Gallery of Art, Washington)',
|
|
};
|
|
|
|
function artistPaintingKey(artistName, title) {
|
|
return artistName && title ? `${artistName}::${title}` : title;
|
|
}
|
|
|
|
/** Strip duplicate parentheticals and museum catalog noise from seed titles. */
|
|
function simplifyPaintingTitle(title) {
|
|
if (!title) return '';
|
|
let t = title.trim();
|
|
const paren = t.match(/^(.+?)\s*\(([^)]+)\)\s*$/);
|
|
if (paren) {
|
|
const outer = paren[1].trim();
|
|
const inner = paren[2].trim();
|
|
const outerNorm = outer.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
const innerNorm = inner.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
if (innerNorm.includes(outerNorm.slice(0, 12)) || outerNorm.includes(innerNorm.slice(0, 12))) {
|
|
return inner.length >= outer.length ? inner : outer;
|
|
}
|
|
return outer;
|
|
}
|
|
return t;
|
|
}
|
|
|
|
function artistSurname(name) {
|
|
if (!name) return '';
|
|
const parts = name.trim().split(/\s+/);
|
|
return parts[parts.length - 1].toLowerCase();
|
|
}
|
|
|
|
function normalizeForMatch(text) {
|
|
return (text || '')
|
|
.normalize('NFD')
|
|
.replace(/\p{M}/gu, '')
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9\s]/g, ' ')
|
|
.replace(/\s+/g, ' ')
|
|
.trim();
|
|
}
|
|
|
|
function titleMatchScore(candidate, searchTitle, artistName) {
|
|
const c = normalizeForMatch(candidate);
|
|
const tokens = normalizeForMatch(simplifyPaintingTitle(searchTitle))
|
|
.split(/\s+/)
|
|
.filter((w) => w.length > 3);
|
|
if (!tokens.length) return 0;
|
|
const hits = tokens.filter((t) => c.includes(t)).length;
|
|
const surname = artistSurname(artistName);
|
|
const artistHit = surname && c.includes(normalizeForMatch(surname)) ? 1 : 0;
|
|
return hits + artistHit;
|
|
}
|
|
|
|
const DIRECT_IMAGE_OVERRIDES = {
|
|
'The Persistence of Memory': {
|
|
thumbUrl: 'https://upload.wikimedia.org/wikipedia/en/d/dd/The_Persistence_of_Memory.jpg',
|
|
fullUrl: 'https://upload.wikimedia.org/wikipedia/en/d/dd/The_Persistence_of_Memory.jpg',
|
|
source: 'Wikimedia Commons (Museum of Modern Art, New York)',
|
|
},
|
|
'Self-Portrait Hesitating': {
|
|
thumbUrl:
|
|
'https://upload.wikimedia.org/wikipedia/commons/thumb/2/2c/Angelica_Kauffman._Self-Portrait_Hesitating_Between_the_Arts_of_Music_and_Painting.jpg/960px-Angelica_Kauffman._Self-Portrait_Hesitating_Between_the_Arts_of_Music_and_Painting.jpg',
|
|
fullUrl:
|
|
'https://upload.wikimedia.org/wikipedia/commons/2/2c/Angelica_Kauffman._Self-Portrait_Hesitating_Between_the_Arts_of_Music_and_Painting.jpg',
|
|
source: 'Wikimedia Commons (National Trust, Nostell Priory)',
|
|
},
|
|
'Cherubs of the Sistine Madonna': {
|
|
thumbUrl:
|
|
'https://upload.wikimedia.org/wikipedia/commons/thumb/5/54/Raffaels_Angels.jpg/960px-Raffaels_Angels.jpg',
|
|
fullUrl: 'https://upload.wikimedia.org/wikipedia/commons/5/54/Raffaels_Angels.jpg',
|
|
source: 'Wikimedia Commons (detail from Sistine Madonna, Gemäldegalerie Alte Meister)',
|
|
},
|
|
'Madonna and Child (Madonna della Seggiola)': MADONNA_SEGGIOLA_IMAGE,
|
|
'Madonna della seggiola': MADONNA_SEGGIOLA_IMAGE,
|
|
'La Madonna della sedia': MADONNA_SEGGIOLA_IMAGE,
|
|
'Madonna della sedia': MADONNA_SEGGIOLA_IMAGE,
|
|
'Raphael::Madonna and Child': SMALL_COWPER_MADONNA_IMAGE,
|
|
};
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
class FetchDeadline {
|
|
constructor(maxMs) {
|
|
this.deadline = maxMs ? Date.now() + maxMs : null;
|
|
}
|
|
|
|
get active() {
|
|
return this.deadline !== null;
|
|
}
|
|
|
|
expired() {
|
|
return this.deadline !== null && Date.now() >= this.deadline;
|
|
}
|
|
|
|
remainingMs() {
|
|
return this.deadline ? Math.max(0, this.deadline - Date.now()) : Infinity;
|
|
}
|
|
|
|
throwIfExpired() {
|
|
if (this.expired()) {
|
|
const err = new Error('Painting fetch time limit exceeded');
|
|
err.code = 'FETCH_TIME_LIMIT';
|
|
throw err;
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Per-operation deadline for batch fetches; cleared after savePaintingImages finishes. */
|
|
let activeDeadline = null;
|
|
|
|
function deadlineFromOptions(options = {}) {
|
|
if (options.deadline instanceof FetchDeadline) return options.deadline;
|
|
if (options.maxWaitMs) return new FetchDeadline(options.maxWaitMs);
|
|
return new FetchDeadline(null);
|
|
}
|
|
|
|
function requestTimeoutMs() {
|
|
if (!activeDeadline?.active) return REQUEST_TIMEOUT_MS;
|
|
return Math.min(REQUEST_TIMEOUT_MS, Math.max(500, activeDeadline.remainingMs() - 50));
|
|
}
|
|
|
|
function attachRequestTimeout(req, reject) {
|
|
const timeoutMs = requestTimeoutMs();
|
|
req.setTimeout(timeoutMs, () => {
|
|
req.destroy();
|
|
reject(new Error('Request timeout'));
|
|
});
|
|
}
|
|
|
|
async function throttle() {
|
|
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
|
|
const elapsed = Date.now() - lastRequestTime;
|
|
const minDelay = activeDeadline?.active ? 0 : MIN_DELAY_MS;
|
|
let wait = Math.max(0, minDelay - elapsed);
|
|
if (activeDeadline?.active) {
|
|
wait = Math.min(wait, activeDeadline.remainingMs() - 50);
|
|
if (wait <= 0) activeDeadline.throwIfExpired();
|
|
}
|
|
if (wait > 0) await sleep(wait);
|
|
lastRequestTime = Date.now();
|
|
}
|
|
|
|
function fetchBuffer(url, redirectCount = 0, referer = null, options = {}) {
|
|
return new Promise((resolve, reject) => {
|
|
if (redirectCount > 8) return reject(new Error('Too many redirects'));
|
|
const client = url.startsWith('https') ? https : http;
|
|
const useBrowser = options.browser !== false && (options.browser === true || !!referer);
|
|
const headers = {
|
|
'User-Agent': useBrowser ? BROWSER_USER_AGENT : USER_AGENT,
|
|
Accept: 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
};
|
|
if (referer) headers.Referer = referer;
|
|
if (url.includes('wikimedia.org') || url.includes('wikipedia.org')) {
|
|
headers.Referer = 'https://commons.wikimedia.org/';
|
|
headers['User-Agent'] = USER_AGENT;
|
|
}
|
|
if (url.includes('artic.edu')) {
|
|
headers.Referer = 'https://www.artic.edu/';
|
|
}
|
|
if (url.includes('collections.louvre.fr')) {
|
|
headers.Referer = 'https://collections.louvre.fr/';
|
|
}
|
|
if (url.includes('hermitagemuseum.org') || url.includes('pushkinmuseum.art')) {
|
|
headers.Referer = url.split('/').slice(0, 3).join('/') + '/';
|
|
}
|
|
if (url.includes('googleusercontent.com') || url.includes('gstatic.com') || url.includes('artsandculture.google.com')) {
|
|
headers.Referer = headers.Referer || 'https://artsandculture.google.com/';
|
|
}
|
|
const req = client.get(url, { headers }, (res) => {
|
|
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
const next = res.headers.location.startsWith('http')
|
|
? res.headers.location
|
|
: new URL(res.headers.location, url).href;
|
|
return resolve(fetchBuffer(next, redirectCount + 1, referer, options));
|
|
}
|
|
if (res.statusCode === 429) {
|
|
res.resume();
|
|
return reject(new Error('HTTP 429'));
|
|
}
|
|
if (res.statusCode !== 200) {
|
|
res.resume();
|
|
return reject(new Error(`HTTP ${res.statusCode}`));
|
|
}
|
|
const chunks = [];
|
|
res.on('data', (c) => chunks.push(c));
|
|
res.on('end', () => resolve(Buffer.concat(chunks)));
|
|
}
|
|
);
|
|
attachRequestTimeout(req, reject);
|
|
req.on('error', reject);
|
|
});
|
|
}
|
|
|
|
async function fetchJson(url) {
|
|
await throttle();
|
|
return new Promise((resolve, reject) => {
|
|
const client = url.startsWith('https') ? https : http;
|
|
const req = client
|
|
.get(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => {
|
|
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
return fetchJson(res.headers.location).then(resolve).catch(reject);
|
|
}
|
|
let data = '';
|
|
res.on('data', (c) => (data += c));
|
|
res.on('end', () => {
|
|
try {
|
|
resolve(JSON.parse(data));
|
|
} catch (e) {
|
|
reject(e);
|
|
}
|
|
});
|
|
})
|
|
.on('error', reject);
|
|
attachRequestTimeout(req, reject);
|
|
});
|
|
}
|
|
|
|
async function fetchHtml(url, options = {}) {
|
|
await throttle();
|
|
const { method = 'GET', body = null, headers = {} } = options;
|
|
return new Promise((resolve, reject) => {
|
|
const parsed = new URL(url);
|
|
const client = parsed.protocol === 'https:' ? https : http;
|
|
const req = client.request(
|
|
{
|
|
hostname: parsed.hostname,
|
|
port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80),
|
|
path: parsed.pathname + parsed.search,
|
|
method,
|
|
headers: {
|
|
'User-Agent': USER_AGENT,
|
|
Accept: 'text/html,application/xhtml+xml',
|
|
...headers,
|
|
},
|
|
},
|
|
(res) => {
|
|
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
const next = res.headers.location.startsWith('http')
|
|
? res.headers.location
|
|
: new URL(res.headers.location, url).href;
|
|
return fetchHtml(next, options).then(resolve).catch(reject);
|
|
}
|
|
let data = '';
|
|
res.on('data', (c) => (data += c));
|
|
res.on('end', () => {
|
|
if (res.statusCode !== 200) {
|
|
return reject(new Error(`HTTP ${res.statusCode}`));
|
|
}
|
|
resolve(data);
|
|
});
|
|
}
|
|
);
|
|
attachRequestTimeout(req, reject);
|
|
req.on('error', reject);
|
|
if (body) req.write(body);
|
|
req.end();
|
|
});
|
|
}
|
|
|
|
function wikiHost(lang) {
|
|
return lang === 'en' ? 'en.wikipedia.org' : `${lang}.wikipedia.org`;
|
|
}
|
|
|
|
function isTrustedImageUrl(url) {
|
|
try {
|
|
const host = new URL(url).hostname.replace(/^www\./, '');
|
|
return TRUSTED_IMAGE_HOSTS.some((h) => host === h || host.endsWith('.' + h));
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function pickBestImageUrl(urls) {
|
|
for (const url of urls) {
|
|
if (isTrustedImageUrl(url) && /\.(jpe?g|png|webp)(\?|$)/i.test(url)) {
|
|
return url;
|
|
}
|
|
}
|
|
for (const url of urls) {
|
|
if (isTrustedImageUrl(url)) return url;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function extractImageUrlsFromText(text) {
|
|
const urls = new Set();
|
|
for (const match of text.matchAll(/https?:\/\/[^\s"'<>\\]+?\.(?:jpe?g|png|webp)(?:\?[^\s"'<>\\]*)?/gi)) {
|
|
urls.add(match[0].replace(/\\$/g, ''));
|
|
}
|
|
for (const match of text.matchAll(/https?:\/\/upload\.wikimedia\.org\/[^\s"'<>\\]+/gi)) {
|
|
urls.add(match[0].replace(/\\$/g, ''));
|
|
}
|
|
for (const match of text.matchAll(/https?:\/\/collections\.louvre\.fr\/media\/[^\s"'<>\\]+/gi)) {
|
|
urls.add(match[0].replace(/\\$/g, ''));
|
|
}
|
|
return [...urls];
|
|
}
|
|
|
|
async function withRetry(fn, retries = 6) {
|
|
const maxRetries = activeDeadline?.active ? Math.min(retries, 2) : retries;
|
|
for (let i = 0; i < maxRetries; i++) {
|
|
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
|
|
try {
|
|
return await fn();
|
|
} catch (err) {
|
|
if (err.code === 'FETCH_TIME_LIMIT') throw err;
|
|
if (i === maxRetries - 1) throw err;
|
|
const wait = err.message.includes('429') ? 8000 * (i + 1) : 2000 * (i + 1);
|
|
if (activeDeadline?.active) {
|
|
const capped = Math.min(wait, activeDeadline.remainingMs() - 50);
|
|
if (capped <= 0) activeDeadline.throwIfExpired();
|
|
await sleep(capped);
|
|
} else {
|
|
await sleep(wait);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function pickExt(url) {
|
|
try {
|
|
const ext = path.extname(new URL(url).pathname) || '.jpg';
|
|
return ext.length > 5 ? '.jpg' : ext;
|
|
} catch {
|
|
return '.jpg';
|
|
}
|
|
}
|
|
|
|
async function getWikidataImageFilename(wikiTitle) {
|
|
const propsUrl = `https://en.wikipedia.org/w/api.php?action=query&titles=${encodeURIComponent(wikiTitle)}&prop=pageprops&ppprop=wikibase_item&format=json`;
|
|
const props = await fetchJson(propsUrl);
|
|
const page = Object.values(props.query?.pages || {})[0];
|
|
const qid = page?.pageprops?.wikibase_item;
|
|
if (!qid) return null;
|
|
|
|
const entityUrl = `https://www.wikidata.org/wiki/Special:EntityData/${qid}.json`;
|
|
const entity = await fetchJson(entityUrl);
|
|
const claims = entity.entities?.[qid]?.claims?.P18;
|
|
if (!claims?.length) return null;
|
|
return claims[0].mainsnak?.datavalue?.value;
|
|
}
|
|
|
|
async function getCommonsFileUrls(filename) {
|
|
const fileTitle = filename.startsWith('File:') ? filename : `File:${filename}`;
|
|
const url = `https://commons.wikimedia.org/w/api.php?action=query&titles=${encodeURIComponent(fileTitle)}&prop=imageinfo&iiprop=url&iiurlwidth=${THUMB_WIDTH}&format=json`;
|
|
const data = await fetchJson(url);
|
|
const page = Object.values(data.query?.pages || {})[0];
|
|
const info = page?.imageinfo?.[0];
|
|
if (!info) return null;
|
|
return {
|
|
thumbUrl: info.thumburl || info.url,
|
|
fullUrl: info.url || info.thumburl,
|
|
source: 'Wikimedia Commons',
|
|
};
|
|
}
|
|
|
|
async function getWikipediaImages(wikiTitle) {
|
|
return getWikipediaImagesLang(wikiTitle, 'en');
|
|
}
|
|
|
|
async function getWikipediaImagesLang(wikiTitle, lang = 'en') {
|
|
const host = wikiHost(lang);
|
|
const langLabel = lang === 'en' ? 'English' : lang.toUpperCase();
|
|
const url = `https://${host}/w/api.php?action=query&titles=${encodeURIComponent(wikiTitle)}&prop=pageimages&piprop=original|thumbnail&pithumbsize=${THUMB_WIDTH}&format=json`;
|
|
const data = await fetchJson(url);
|
|
const page = Object.values(data.query?.pages || {})[0];
|
|
if (!page || page.missing) return null;
|
|
const thumbUrl = page.thumbnail?.source;
|
|
const fullUrl = page.original?.source || thumbUrl;
|
|
if (!fullUrl) return null;
|
|
return {
|
|
thumbUrl: thumbUrl || fullUrl,
|
|
fullUrl,
|
|
source: `Wikipedia ${langLabel} (Wikimedia Commons)`,
|
|
resolvedWikiTitle: wikiTitle,
|
|
wikiLang: lang,
|
|
};
|
|
}
|
|
|
|
async function searchCommonsImages(artistName, paintingTitle) {
|
|
const queries = [
|
|
[artistName, paintingTitle].filter(Boolean).join(' '),
|
|
paintingTitle,
|
|
[artistName, ...(paintingTitle || '').split(/\s+/).slice(0, 4)].join(' '),
|
|
[artistName, 'painting', (paintingTitle || '').split('(')[0].trim()].filter(Boolean).join(' '),
|
|
];
|
|
const seen = new Set();
|
|
|
|
for (const query of queries) {
|
|
if (!query || seen.has(query)) continue;
|
|
seen.add(query);
|
|
|
|
const url = `https://commons.wikimedia.org/w/api.php?action=query&generator=search&gssearch=${encodeURIComponent(query)}&gsnamespace=6&gslimit=8&prop=imageinfo&iiprop=url&iiurlwidth=${THUMB_WIDTH}&format=json`;
|
|
try {
|
|
const data = await fetchJson(url);
|
|
const pages = data.query?.pages;
|
|
if (!pages) continue;
|
|
|
|
const artistToken = (artistName || '').toLowerCase().split(' ').pop() || '';
|
|
const titleTokens = (paintingTitle || '')
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9\s]/g, ' ')
|
|
.split(/\s+/)
|
|
.filter((t) => t.length > 3)
|
|
.slice(0, 4);
|
|
|
|
for (const page of Object.values(pages)) {
|
|
const info = page.imageinfo?.[0];
|
|
if (!info?.url) continue;
|
|
const fileTitle = (page.title || '').toLowerCase();
|
|
if (artistToken && !fileTitle.includes(artistToken) && !fileTitle.includes('raphael') && !fileTitle.includes('raffael')) {
|
|
continue;
|
|
}
|
|
const matchCount = titleTokens.filter((t) => fileTitle.includes(t)).length;
|
|
if (titleTokens.length > 0 && matchCount === 0 && !fileTitle.includes('uffizi')) continue;
|
|
|
|
return {
|
|
thumbUrl: info.thumburl || info.url,
|
|
fullUrl: info.url,
|
|
source: 'Wikimedia Commons (search)',
|
|
};
|
|
}
|
|
} catch {
|
|
continue;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getMetMuseumImages(searchTerm, artistName) {
|
|
const query = [artistName, searchTerm].filter(Boolean).join(' ');
|
|
const searchUrl = `https://collectionapi.metmuseum.org/public/collection/v1/search?q=${encodeURIComponent(query)}&hasImages=true`;
|
|
try {
|
|
await throttle();
|
|
const search = await fetchJson(searchUrl);
|
|
if (!search.objectIDs?.length) return null;
|
|
|
|
for (const id of search.objectIDs.slice(0, 10)) {
|
|
await throttle();
|
|
const obj = await fetchJson(
|
|
`https://collectionapi.metmuseum.org/public/collection/v1/objects/${id}`
|
|
);
|
|
if (!obj.isPublicDomain || !obj.primaryImage) continue;
|
|
const titleMatch =
|
|
!searchTerm ||
|
|
(obj.title || '').toLowerCase().includes(searchTerm.toLowerCase().slice(0, 10));
|
|
if (!titleMatch && search.objectIDs.length > 3) continue;
|
|
return {
|
|
thumbUrl: obj.primaryImageSmall || obj.primaryImage,
|
|
fullUrl: obj.primaryImage,
|
|
source: 'The Metropolitan Museum of Art (Open Access)',
|
|
};
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getArtInstituteImages(searchTerm, artistName) {
|
|
const query = [artistName, searchTerm].filter(Boolean).join(' ');
|
|
const url = `https://api.artic.edu/api/v1/artworks/search?q=${encodeURIComponent(query)}&fields=id,title,artist_title,image_id&limit=6`;
|
|
try {
|
|
const data = await fetchJson(url);
|
|
for (const art of data.data || []) {
|
|
if (!art.image_id) continue;
|
|
const titleOk =
|
|
!searchTerm ||
|
|
(art.title || '').toLowerCase().includes(searchTerm.toLowerCase().slice(0, 8));
|
|
const artistOk =
|
|
!artistName ||
|
|
(art.artist_title || '').toLowerCase().includes(artistName.toLowerCase().split(' ').pop());
|
|
if (!titleOk && !artistOk) continue;
|
|
const base = `https://www.artic.edu/iiif/2/${art.image_id}`;
|
|
return {
|
|
thumbUrl: `${base}/full/200,/0/default.jpg`,
|
|
fullUrl: `${base}/full/1686,/0/default.jpg`,
|
|
source: 'Art Institute of Chicago (Open Access)',
|
|
};
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getRijksmuseumImages(searchTerm, artistName) {
|
|
const query = [artistName, searchTerm].filter(Boolean).join(' ');
|
|
const url = `https://www.rijksmuseum.nl/api/en/collection?key=0&format=json&ps=8&s=relevance&q=${encodeURIComponent(query)}&imgonly=true`;
|
|
try {
|
|
const data = await fetchJson(url);
|
|
for (const item of data.artObjects || []) {
|
|
if (!item.webImage?.url) continue;
|
|
return {
|
|
thumbUrl: item.webImage.url,
|
|
fullUrl: item.webImage.url,
|
|
source: 'Rijksmuseum',
|
|
};
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getWikidataImagesBySearch(artistName, paintingTitle) {
|
|
const label = (paintingTitle || '').replace(/"/g, '\\"').slice(0, 120);
|
|
const query = `
|
|
SELECT ?img ?imgLabel WHERE {
|
|
?work rdfs:label ?workLabel .
|
|
FILTER(LANG(?workLabel) = "en")
|
|
FILTER(CONTAINS(LCASE(?workLabel), LCASE("${label.slice(0, 40)}")))
|
|
?work wdt:P170 ?artist .
|
|
?artist rdfs:label ?artistLabel .
|
|
FILTER(CONTAINS(LCASE(?artistLabel), "${(artistName || '').split(' ').pop().toLowerCase()}"))
|
|
?work wdt:P18 ?img .
|
|
SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
|
|
}
|
|
LIMIT 3`;
|
|
try {
|
|
const url = `https://query.wikidata.org/sparql?format=json&query=${encodeURIComponent(query)}`;
|
|
const data = await fetchJson(url);
|
|
const row = data.results?.bindings?.[0];
|
|
if (!row?.img?.value) return null;
|
|
const filename = decodeURIComponent(row.img.value.split('/').pop());
|
|
return getCommonsFileUrls(filename);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function searchWikipediaTitle(artistName, paintingTitle) {
|
|
const hit = await searchWikipediaTitleMultilingual(artistName, paintingTitle);
|
|
return hit?.title || null;
|
|
}
|
|
|
|
async function searchWikipediaTitleLang(artistName, paintingTitle, lang = 'en') {
|
|
const simple = simplifyPaintingTitle(paintingTitle);
|
|
const host = wikiHost(lang);
|
|
const queries = [
|
|
`${artistName} ${simple}`,
|
|
`${simple} ${artistSurname(artistName)}`,
|
|
`${artistName} ${simple.split(' ').slice(0, 4).join(' ')}`,
|
|
simple,
|
|
];
|
|
const seen = new Set();
|
|
let best = null;
|
|
let bestScore = 0;
|
|
|
|
for (const query of queries) {
|
|
if (!query || seen.has(query)) continue;
|
|
seen.add(query);
|
|
|
|
const apiUrl = `https://${host}/w/api.php?action=query&list=search&srsearch=${encodeURIComponent(query)}&srlimit=8&format=json`;
|
|
const restUrl = `https://${host}/w/rest.php/v1/search/page?q=${encodeURIComponent(query)}&limit=8`;
|
|
|
|
for (const url of [apiUrl, restUrl]) {
|
|
try {
|
|
const data = await fetchJson(url);
|
|
const hits = data.query?.search || data.pages || [];
|
|
for (const hit of hits) {
|
|
const title = hit.title || '';
|
|
if (/disambiguation|list of|category:|liste von|liste des|elenco|categoría:/i.test(title)) {
|
|
continue;
|
|
}
|
|
const score =
|
|
titleMatchScore(title, paintingTitle, artistName) + (hit.score || hit.key || 0) / 1000;
|
|
if (score > bestScore) {
|
|
bestScore = score;
|
|
best = title;
|
|
}
|
|
}
|
|
} catch {
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
return bestScore >= 2 ? best : null;
|
|
}
|
|
|
|
async function searchWikipediaTitleMultilingual(artistName, paintingTitle) {
|
|
const langs = ['en', ...WIKI_LANGS];
|
|
let best = null;
|
|
let bestScore = 0;
|
|
|
|
for (const lang of langs) {
|
|
const title = await searchWikipediaTitleLang(artistName, paintingTitle, lang);
|
|
if (!title) continue;
|
|
const score = titleMatchScore(title, paintingTitle, artistName);
|
|
if (score > bestScore) {
|
|
bestScore = score;
|
|
best = { title, lang };
|
|
}
|
|
}
|
|
return bestScore >= 2 ? best : null;
|
|
}
|
|
|
|
async function getMultilingualWikipediaImages(artistName, searchTitle) {
|
|
for (const lang of WIKI_LANGS) {
|
|
const title = await searchWikipediaTitleLang(artistName, searchTitle, lang);
|
|
if (!title) continue;
|
|
const images = await getWikipediaImagesLang(title, lang);
|
|
if (images) return images;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getClevelandArtImages(searchTerm, artistName) {
|
|
const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
|
|
const url = `https://openaccess-api.clevelandart.org/api/artworks/?q=${encodeURIComponent(query)}&has_image=1&cc0=1&limit=12`;
|
|
try {
|
|
const data = await fetchJson(url);
|
|
for (const item of data.data || []) {
|
|
const img = item.images?.web?.url || item.images?.full?.url || item.images?.print?.url;
|
|
if (!img) continue;
|
|
const score = titleMatchScore(`${item.title || ''}`, searchTerm, artistName);
|
|
if (score < 1 && (data.data?.length || 0) > 2) continue;
|
|
return {
|
|
thumbUrl: item.images?.web?.url || img,
|
|
fullUrl: item.images?.full?.url || item.images?.web?.url || img,
|
|
source: 'Cleveland Museum of Art (Open Access)',
|
|
};
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getSmithsonianImages(searchTerm, artistName) {
|
|
const apiKey = process.env.SMITHSONIAN_API_KEY;
|
|
if (!apiKey) return null;
|
|
const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
|
|
const url = `https://api.si.edu/openaccess/api/v1.0/search?q=${encodeURIComponent(query)}&api_key=${apiKey}&rows=12&sort=Relevance`;
|
|
try {
|
|
const data = await fetchJson(url);
|
|
for (const row of data.response?.rows || []) {
|
|
const content = row.content || {};
|
|
const title = content.title?.content || content.label?.content || '';
|
|
const score = titleMatchScore(title, searchTerm, artistName);
|
|
if (score < 1 && (data.response?.rows?.length || 0) > 3) continue;
|
|
|
|
const onlineMedia = content.online_media?.content || [];
|
|
for (const media of onlineMedia) {
|
|
const ids = media.ids || media.id || [];
|
|
const idList = Array.isArray(ids) ? ids : [ids];
|
|
for (const id of idList) {
|
|
if (!id || typeof id !== 'string') continue;
|
|
const imgUrl = id.startsWith('http')
|
|
? id
|
|
: `https://ids.si.edu/ids/deliveryService?id=${encodeURIComponent(id)}&max=1600`;
|
|
return {
|
|
thumbUrl: imgUrl.replace('max=1600', 'max=400'),
|
|
fullUrl: imgUrl,
|
|
source: 'Smithsonian Open Access',
|
|
};
|
|
}
|
|
}
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getHarvardArtMuseumsImages(searchTerm, artistName) {
|
|
const apiKey = process.env.HARVARD_ART_API_KEY;
|
|
if (!apiKey) return null;
|
|
const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
|
|
const url = `https://api.harvardartmuseums.org/object?q=${encodeURIComponent(query)}&apikey=${apiKey}&size=10&hasimage=1&fields=id,title,primaryimageurl,people`;
|
|
try {
|
|
const data = await fetchJson(url);
|
|
for (const rec of data.records || []) {
|
|
if (!rec.primaryimageurl) continue;
|
|
const people = (rec.people || []).map((p) => p.name).join(' ');
|
|
const score = titleMatchScore(`${rec.title} ${people}`, searchTerm, artistName);
|
|
if (score < 1 && (data.records?.length || 0) > 3) continue;
|
|
return {
|
|
thumbUrl: rec.primaryimageurl,
|
|
fullUrl: rec.primaryimageurl,
|
|
source: 'Harvard Art Museums',
|
|
};
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getLouvreImages(searchTerm, artistName) {
|
|
const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
|
|
try {
|
|
const html = await fetchHtml(
|
|
`https://collections.louvre.fr/recherche?q=${encodeURIComponent(query)}&page=1`,
|
|
{ headers: { Referer: 'https://collections.louvre.fr/' } }
|
|
);
|
|
const arks = [
|
|
...new Set([...html.matchAll(/href="(\/ark:[^"]+)"/g)].map((m) => m[1])),
|
|
].slice(0, 8);
|
|
|
|
for (const ark of arks) {
|
|
const data = await fetchJson(`https://collections.louvre.fr${ark}.json`);
|
|
const title = data.title || data.denominationTitle || '';
|
|
const score = titleMatchScore(`${title} ${(data.creator || []).map((c) => c.label).join(' ')}`, searchTerm, artistName);
|
|
const images = data.image || [];
|
|
const primary = images.find((img) => img.urlImage) || images[0];
|
|
if (!primary?.urlImage) continue;
|
|
if (score < 1 && arks.length > 2) continue;
|
|
|
|
const fullUrl = primary.urlImage.startsWith('http')
|
|
? primary.urlImage
|
|
: `https://collections.louvre.fr/${primary.urlImage.replace(/^\//, '')}`;
|
|
const thumbUrl = primary.urlThumbnail
|
|
? primary.urlThumbnail.startsWith('http')
|
|
? primary.urlThumbnail
|
|
: `https://collections.louvre.fr/${primary.urlThumbnail.replace(/^\//, '')}`
|
|
: fullUrl;
|
|
|
|
return {
|
|
thumbUrl,
|
|
fullUrl,
|
|
source: 'Musée du Louvre (collections.louvre.fr)',
|
|
};
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getEuropeanaImages(searchTerm, artistName) {
|
|
const apiKey = process.env.EUROPEANA_API_KEY;
|
|
if (!apiKey) return null;
|
|
const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
|
|
const url =
|
|
`https://api.europeana.eu/record/v2/search.json?wskey=${encodeURIComponent(apiKey)}` +
|
|
`&query=${encodeURIComponent(query)}&qf=TYPE:IMAGE&reusability=open&rows=12&profile=rich`;
|
|
try {
|
|
const data = await fetchJson(url);
|
|
for (const item of data.items || []) {
|
|
const title = item.title?.[0] || item.dcTitle?.[0] || '';
|
|
const score = titleMatchScore(`${title} ${item.dcCreator?.join(' ') || ''}`, searchTerm, artistName);
|
|
if (score < 1 && (data.items?.length || 0) > 3) continue;
|
|
|
|
const fullUrl =
|
|
item.edmIsShownBy?.[0] ||
|
|
item.edmPreview?.[0] ||
|
|
item.link ||
|
|
item.guid;
|
|
if (!fullUrl || !isTrustedImageUrl(fullUrl)) continue;
|
|
|
|
return {
|
|
thumbUrl: item.edmPreview?.[0] || fullUrl,
|
|
fullUrl,
|
|
source: 'Europeana (European museums)',
|
|
};
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function upgradeGoogleUserContentUrl(url, width = FULL_WIDTH) {
|
|
if (!url || !url.includes('googleusercontent.com')) return url;
|
|
const base = url.replace(/=w[^=]*$/, '').replace(/=s\d+$/, '');
|
|
return `${base}=w${width}-no`;
|
|
}
|
|
|
|
function extractGoogleArtsAssetLinks(html) {
|
|
const assets = [];
|
|
const seen = new Set();
|
|
for (const match of html.matchAll(/\/asset\/([^/"'?]+)\/([A-Za-z0-9_-]{8,})/g)) {
|
|
const slug = match[1];
|
|
const id = match[2];
|
|
const key = `${slug}/${id}`;
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
assets.push({ slug, id, path: `/asset/${slug}/${id}` });
|
|
}
|
|
return assets;
|
|
}
|
|
|
|
async function getGoogleArtsCultureAssetImage(assetPath) {
|
|
const html = await fetchHtml(`https://artsandculture.google.com${assetPath}`, {
|
|
headers: { Referer: 'https://artsandculture.google.com/' },
|
|
});
|
|
const ogImage = html.match(/property="og:image" content="([^"]+)"/)?.[1];
|
|
const jsonImage = html.match(/"image":"(https:\/\/lh3\.googleusercontent\.com\/[^"]+)"/)?.[1];
|
|
const imageUrl = ogImage || jsonImage;
|
|
if (!imageUrl) return null;
|
|
|
|
const fullUrl = upgradeGoogleUserContentUrl(imageUrl, FULL_WIDTH);
|
|
const thumbUrl = upgradeGoogleUserContentUrl(imageUrl, THUMB_WIDTH);
|
|
return { thumbUrl, fullUrl };
|
|
}
|
|
|
|
async function getGoogleArtsCultureImages(searchTerm, artistName) {
|
|
const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
|
|
try {
|
|
const html = await fetchHtml(
|
|
`https://artsandculture.google.com/search?q=${encodeURIComponent(query)}`,
|
|
{ headers: { Referer: 'https://artsandculture.google.com/' } }
|
|
);
|
|
|
|
const assets = extractGoogleArtsAssetLinks(html);
|
|
const ranked = assets
|
|
.map((asset) => ({
|
|
...asset,
|
|
score: titleMatchScore(asset.slug.replace(/-/g, ' '), searchTerm, artistName),
|
|
}))
|
|
.filter((a) => a.score >= 1)
|
|
.sort((a, b) => b.score - a.score);
|
|
|
|
const candidates = ranked.length ? ranked : assets.slice(0, 6).map((a) => ({ ...a, score: 0 }));
|
|
|
|
for (const asset of candidates.slice(0, 5)) {
|
|
const images = await getGoogleArtsCultureAssetImage(asset.path);
|
|
if (!images) continue;
|
|
if (asset.score < 1 && candidates.length > 2) continue;
|
|
return {
|
|
...images,
|
|
source: 'Google Arts & Culture',
|
|
};
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function getWikidataImagesMultilingual(artistName, paintingTitle) {
|
|
const label = simplifyPaintingTitle(paintingTitle).replace(/"/g, '\\"').slice(0, 80);
|
|
const surname = (artistName || '').split(' ').pop().toLowerCase();
|
|
const langs = ['en', ...WIKI_LANGS];
|
|
const langFilter = langs.map((l) => `"${l}"`).join(', ');
|
|
const query = `
|
|
SELECT ?img ?workLabel WHERE {
|
|
?work rdfs:label ?workLabel .
|
|
FILTER(LANG(?workLabel) IN (${langFilter}))
|
|
FILTER(CONTAINS(LCASE(STR(?workLabel)), LCASE("${label.slice(0, 40)}")))
|
|
?work wdt:P170 ?artist .
|
|
?artist rdfs:label ?artistLabel .
|
|
FILTER(LANG(?artistLabel) IN (${langFilter}))
|
|
FILTER(CONTAINS(LCASE(STR(?artistLabel)), "${surname}"))
|
|
?work wdt:P18 ?img .
|
|
}
|
|
LIMIT 5`;
|
|
try {
|
|
const url = `https://query.wikidata.org/sparql?format=json&query=${encodeURIComponent(query)}`;
|
|
const data = await fetchJson(url);
|
|
for (const row of data.results?.bindings || []) {
|
|
if (!row.img?.value) continue;
|
|
const filename = decodeURIComponent(row.img.value.split('/').pop());
|
|
const commons = await getCommonsFileUrls(filename);
|
|
if (commons) {
|
|
return { ...commons, source: 'Wikimedia Commons (Wikidata multilingual)' };
|
|
}
|
|
}
|
|
} catch {
|
|
return null;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function extractDdgResultLinks(html) {
|
|
const links = [];
|
|
for (const match of html.matchAll(/class="result__a"[^>]*href="([^"]+)"/g)) {
|
|
links.push(match[1].replace(/&/g, '&'));
|
|
}
|
|
for (const match of html.matchAll(/uddg=([^&"]+)/g)) {
|
|
try {
|
|
links.push(decodeURIComponent(match[1]));
|
|
} catch {
|
|
continue;
|
|
}
|
|
}
|
|
return links;
|
|
}
|
|
|
|
async function resolveCommonsFilePageUrl(pageUrl) {
|
|
if (!pageUrl || !/commons\.wikimedia\.org\/wiki\/File:/i.test(pageUrl)) return null;
|
|
const fileTitle = decodeURIComponent(pageUrl.split('/wiki/')[1].replace(/_/g, ' '));
|
|
const commons = await getCommonsFileUrls(fileTitle);
|
|
if (commons) return commons;
|
|
const filePathUrl = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(fileTitle.replace(/^File:/i, ''))}`;
|
|
return {
|
|
thumbUrl: filePathUrl,
|
|
fullUrl: filePathUrl,
|
|
source: 'Web search → Wikimedia Commons',
|
|
};
|
|
}
|
|
|
|
async function searchWebForPaintingImages(artistName, searchTitle) {
|
|
const simple = simplifyPaintingTitle(searchTitle);
|
|
const queries = [
|
|
`${artistName} ${simple} site:commons.wikimedia.org`,
|
|
`${artistName} ${simple} site:artsandculture.google.com`,
|
|
`${artistName} ${simple} painting filetype:jpg`,
|
|
`${artistName} ${simple} site:collections.louvre.fr`,
|
|
`${artistName} ${simple} Gemälde`,
|
|
`${artistName} ${simple} peinture`,
|
|
`${artistName} ${simple} quadro`,
|
|
`${artistName} ${simple} картина`,
|
|
];
|
|
|
|
const seenQueries = new Set();
|
|
const candidateUrls = [];
|
|
|
|
for (const query of queries.slice(0, 4)) {
|
|
if (!query || seenQueries.has(query)) continue;
|
|
seenQueries.add(query);
|
|
|
|
try {
|
|
const body = `q=${encodeURIComponent(query)}`;
|
|
const html = await fetchHtml('https://html.duckduckgo.com/html/', {
|
|
method: 'POST',
|
|
body,
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (compatible; VirtualArtGallery/1.0)',
|
|
'Content-Type': 'application/x-www-form-urlencoded',
|
|
'Content-Length': String(Buffer.byteLength(body)),
|
|
},
|
|
});
|
|
|
|
candidateUrls.push(...extractDdgResultLinks(html));
|
|
candidateUrls.push(...extractImageUrlsFromText(html));
|
|
} catch {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
const seenUrls = new Set();
|
|
for (const pageUrl of candidateUrls) {
|
|
if (!pageUrl?.startsWith('http') || seenUrls.has(pageUrl)) continue;
|
|
seenUrls.add(pageUrl);
|
|
|
|
try {
|
|
if (/commons\.wikimedia\.org\/wiki\/File:/i.test(pageUrl)) {
|
|
const commons = await resolveCommonsFilePageUrl(pageUrl);
|
|
if (commons) return commons;
|
|
continue;
|
|
}
|
|
|
|
if (/\.(jpe?g|png|webp)(\?|$)/i.test(pageUrl) && isTrustedImageUrl(pageUrl)) {
|
|
return {
|
|
thumbUrl: pageUrl,
|
|
fullUrl: pageUrl,
|
|
source: 'Web search (image URL)',
|
|
};
|
|
}
|
|
|
|
if (!/wikipedia\.org|collections\.louvre|artic\.edu|metmuseum|europeana|artsandculture\.google/i.test(pageUrl)) {
|
|
continue;
|
|
}
|
|
|
|
if (/artsandculture\.google\.com\/asset\//i.test(pageUrl)) {
|
|
const assetPath = pageUrl.replace(/^https?:\/\/artsandculture\.google\.com/i, '').split('?')[0];
|
|
const gac = await getGoogleArtsCultureAssetImage(assetPath);
|
|
if (gac) return { ...gac, source: 'Web search → Google Arts & Culture' };
|
|
continue;
|
|
}
|
|
|
|
const html = await fetchHtml(pageUrl);
|
|
const imageUrl = pickBestImageUrl(extractImageUrlsFromText(html));
|
|
if (imageUrl) {
|
|
return {
|
|
thumbUrl: imageUrl,
|
|
fullUrl: imageUrl,
|
|
source: 'Web search (museum/Wikipedia page)',
|
|
};
|
|
}
|
|
|
|
const wikiMatch = pageUrl.match(/\/wiki\/([^#?]+)/);
|
|
if (wikiMatch && /wikipedia\.org/.test(pageUrl)) {
|
|
const lang = pageUrl.match(/\/\/([a-z]{2})\.wikipedia\.org/)?.[1] || 'en';
|
|
const title = decodeURIComponent(wikiMatch[1].replace(/_/g, ' '));
|
|
const images = await getWikipediaImagesLang(title, lang);
|
|
if (images) return { ...images, source: `Web search → Wikipedia ${lang.toUpperCase()}` };
|
|
}
|
|
} catch {
|
|
continue;
|
|
}
|
|
}
|
|
|
|
for (const lang of WIKI_LANGS) {
|
|
const title = await searchWikipediaTitleLang(artistName, searchTitle, lang);
|
|
if (!title) continue;
|
|
const images = await getWikipediaImagesLang(title, lang);
|
|
if (images) return { ...images, source: `Web search → Wikipedia ${lang.toUpperCase()}` };
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
async function resolveFromLookupTitle(lookupTitle, options) {
|
|
const { artistName, paintingTitle } = options;
|
|
const searchTitle = paintingTitle || lookupTitle;
|
|
|
|
const sources = [
|
|
async () => {
|
|
const filename = await getWikidataImageFilename(lookupTitle);
|
|
if (!filename) return null;
|
|
const commons = await getCommonsFileUrls(filename);
|
|
if (commons) return commons;
|
|
const filePathUrl = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(filename)}`;
|
|
return { thumbUrl: filePathUrl, fullUrl: filePathUrl, source: 'Wikimedia Commons (Wikidata)' };
|
|
},
|
|
() => getWikipediaImages(lookupTitle),
|
|
() => searchCommonsImages(artistName, searchTitle),
|
|
() => getMetMuseumImages(searchTitle, artistName),
|
|
() => getArtInstituteImages(searchTitle, artistName),
|
|
() => getClevelandArtImages(searchTitle, artistName),
|
|
() => getRijksmuseumImages(searchTitle, artistName),
|
|
() => getLouvreImages(searchTitle, artistName),
|
|
() => getGoogleArtsCultureImages(searchTitle, artistName),
|
|
() => getEuropeanaImages(searchTitle, artistName),
|
|
() => getMultilingualWikipediaImages(artistName, searchTitle),
|
|
() => getSmithsonianImages(searchTitle, artistName),
|
|
() => getHarvardArtMuseumsImages(searchTitle, artistName),
|
|
() => getWikidataImagesBySearch(artistName, searchTitle),
|
|
() => getWikidataImagesMultilingual(artistName, searchTitle),
|
|
() => getWikipediaImages(searchTitle),
|
|
() => searchCommonsImages(artistName, lookupTitle),
|
|
() => searchWebForPaintingImages(artistName, searchTitle),
|
|
];
|
|
|
|
for (const fn of sources) {
|
|
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
|
|
try {
|
|
const result = await fn();
|
|
if (result?.fullUrl || result?.thumbUrl) {
|
|
return {
|
|
thumbUrl: result.thumbUrl || result.fullUrl,
|
|
fullUrl: result.fullUrl || result.thumbUrl,
|
|
source: result.source,
|
|
resolvedWikiTitle: lookupTitle,
|
|
};
|
|
}
|
|
} catch (err) {
|
|
if (err.code === 'FETCH_TIME_LIMIT') throw err;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function resolvePaintingImages(wikiTitle, options = {}) {
|
|
const { artistName, paintingTitle, webSearchOnly } = options;
|
|
const searchTitle = paintingTitle || wikiTitle;
|
|
|
|
if (webSearchOnly) {
|
|
return searchWebForPaintingImages(artistName, searchTitle);
|
|
}
|
|
|
|
const scopedKey = artistPaintingKey(artistName, searchTitle);
|
|
|
|
const directOverride =
|
|
DIRECT_IMAGE_OVERRIDES[scopedKey] || DIRECT_IMAGE_OVERRIDES[searchTitle];
|
|
if (directOverride) {
|
|
return { ...directOverride, resolvedWikiTitle: wikiTitle };
|
|
}
|
|
|
|
const overrideTitle =
|
|
PAINTING_WIKI_OVERRIDES[scopedKey] || PAINTING_WIKI_OVERRIDES[searchTitle];
|
|
const simplified = simplifyPaintingTitle(searchTitle);
|
|
|
|
const lookupCandidates = [];
|
|
const seen = new Set();
|
|
for (const t of [overrideTitle, wikiTitle, simplified, searchTitle]) {
|
|
if (t && !seen.has(t)) {
|
|
seen.add(t);
|
|
lookupCandidates.push(t);
|
|
}
|
|
}
|
|
|
|
for (const title of lookupCandidates) {
|
|
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
|
|
const result = await resolveFromLookupTitle(title, options);
|
|
if (result) return result;
|
|
}
|
|
|
|
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
|
|
|
|
const discovered = await searchWikipediaTitleMultilingual(artistName, searchTitle);
|
|
if (discovered && !seen.has(discovered.title)) {
|
|
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
|
|
if (discovered.lang === 'en') {
|
|
const result = await resolveFromLookupTitle(discovered.title, options);
|
|
if (result) return { ...result, resolvedWikiTitle: discovered.title };
|
|
} else {
|
|
const images = await getWikipediaImagesLang(discovered.title, discovered.lang);
|
|
if (images) {
|
|
return {
|
|
...images,
|
|
resolvedWikiTitle: discovered.title,
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|
|
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
|
|
|
|
const web = await searchWebForPaintingImages(artistName, searchTitle);
|
|
if (web) return web;
|
|
|
|
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
|
|
|
|
return null;
|
|
}
|
|
|
|
/** @deprecated use resolvePaintingImages */
|
|
async function resolveImageUrl(wikiTitle, options = {}) {
|
|
const pair = await resolvePaintingImages(wikiTitle, options);
|
|
if (!pair) return null;
|
|
return { url: pair.fullUrl, source: pair.source };
|
|
}
|
|
|
|
async function downloadImageToFile(url, destPath, { force = false } = {}) {
|
|
if (!force && fs.existsSync(destPath)) return destPath;
|
|
await throttle();
|
|
const buffer = await withRetry(() => fetchBuffer(url));
|
|
fs.mkdirSync(path.dirname(destPath), { recursive: true });
|
|
fs.writeFileSync(destPath, buffer);
|
|
return destPath;
|
|
}
|
|
|
|
function upgradeWikimediaToFull(url) {
|
|
const thumbMatch = url.match(
|
|
/^(https?:\/\/upload\.wikimedia\.org\/wikipedia\/(?:commons|en)\/)thumb\/(.+)\/\d+px-[^/]+$/
|
|
);
|
|
if (thumbMatch) return `${thumbMatch[1]}${thumbMatch[2]}`;
|
|
return url;
|
|
}
|
|
|
|
function prepareDownloadCandidates(url, context = {}) {
|
|
const candidates = [url];
|
|
if (context.thumbUrl && context.thumbUrl !== url) candidates.push(context.thumbUrl);
|
|
const google = upgradeGoogleUserContentUrl(url, FULL_WIDTH);
|
|
if (google !== url) candidates.push(google);
|
|
const wiki = upgradeWikimediaToFull(url);
|
|
if (wiki !== url) candidates.push(wiki);
|
|
try {
|
|
const parsed = new URL(url);
|
|
if (parsed.search) candidates.push(`${parsed.origin}${parsed.pathname}`);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
return [...new Set(candidates)];
|
|
}
|
|
|
|
function referersForDownload(url, context = {}) {
|
|
const refs = [];
|
|
if (context.pageUrl) refs.push(context.pageUrl);
|
|
if (context.searchUrl) refs.push(context.searchUrl);
|
|
if (context.source === 'google-arts-culture') refs.push('https://artsandculture.google.com/');
|
|
if (context.source === 'google-images' || context.source === 'google-custom-search') {
|
|
refs.push('https://www.google.com/');
|
|
}
|
|
if (context.source === 'duckduckgo-images') refs.push('https://duckduckgo.com/');
|
|
if (/googleusercontent\.com|gstatic\.com/i.test(url)) refs.push('https://artsandculture.google.com/');
|
|
if (/wikimedia\.org|wikipedia\.org/i.test(url)) refs.push('https://commons.wikimedia.org/');
|
|
try {
|
|
refs.push(new URL(url).origin + '/');
|
|
} catch {
|
|
// ignore
|
|
}
|
|
return [...new Set(refs)];
|
|
}
|
|
|
|
/** Download with browser headers, referer fallbacks, and URL variants (fix / debug). */
|
|
async function downloadImageForFix(url, destPath, context = {}) {
|
|
const candidates = prepareDownloadCandidates(url, context);
|
|
const referers = referersForDownload(url, context);
|
|
let lastError = null;
|
|
|
|
await throttle();
|
|
|
|
for (const candidate of candidates) {
|
|
for (const referer of referers) {
|
|
try {
|
|
const buffer = await fetchBuffer(candidate, 0, referer, { browser: true });
|
|
fs.mkdirSync(path.dirname(destPath), { recursive: true });
|
|
fs.writeFileSync(destPath, buffer);
|
|
return destPath;
|
|
} catch (err) {
|
|
lastError = err;
|
|
await sleep(300);
|
|
}
|
|
}
|
|
}
|
|
|
|
throw lastError || new Error('Download failed');
|
|
}
|
|
|
|
async function fetchImageBuffer(url, context = {}) {
|
|
const candidates = prepareDownloadCandidates(url, context);
|
|
const referers = referersForDownload(url, context);
|
|
let lastError = null;
|
|
|
|
await throttle();
|
|
|
|
for (const candidate of candidates) {
|
|
for (const referer of referers) {
|
|
try {
|
|
return await fetchBuffer(candidate, 0, referer, { browser: true });
|
|
} catch (err) {
|
|
lastError = err;
|
|
await sleep(300);
|
|
}
|
|
}
|
|
}
|
|
|
|
throw lastError || new Error('Download failed');
|
|
}
|
|
|
|
function friendlyImageFetchError(err) {
|
|
const msg = err?.message || '';
|
|
if (/ECONNRESET|ETIMEDOUT|EPIPE|socket hang up/i.test(msg)) {
|
|
return 'Could not download this image — the host closed the connection. Try Search again; Wikimedia and Google Arts results work best.';
|
|
}
|
|
if (/HTTP 403|HTTP 401|HTTP 429/i.test(msg)) {
|
|
return 'Could not download this image — the host blocked the request. Try Search again or pick a different preview.';
|
|
}
|
|
if (/Too many redirects/i.test(msg)) {
|
|
return 'Could not download this image — too many redirects from the image URL.';
|
|
}
|
|
return msg || 'Could not download image';
|
|
}
|
|
|
|
function decodeGoogleEmbeddedUrl(raw) {
|
|
return raw
|
|
.replace(/\\u003d/g, '=')
|
|
.replace(/\\u0026/g, '&')
|
|
.replace(/\\u0027/g, "'")
|
|
.replace(/\\\//g, '/');
|
|
}
|
|
|
|
function isLikelyImageUrl(url) {
|
|
if (!url || !/^https?:\/\//i.test(url)) return false;
|
|
if (/^data:/i.test(url)) return false;
|
|
if (/gstatic\.com\/images\?/i.test(url)) return false;
|
|
return true;
|
|
}
|
|
|
|
function scoreImageUrl(url) {
|
|
let score = 0;
|
|
if (/upload\.wikimedia\.org/i.test(url)) score += 40;
|
|
if (/googleusercontent\.com/i.test(url)) score += 30;
|
|
if (/\.(jpe?g|png|webp)(\?|$)/i.test(url)) score += 20;
|
|
if (/thumb|thumbnail|small|icon|logo|avatar/i.test(url)) score -= 25;
|
|
if (/=\s*[sS]\d{2,3}(-c|-rw)?(\||$)/.test(url) || /[?&]w=\d{2,3}(&|$)/.test(url)) score -= 15;
|
|
return score;
|
|
}
|
|
|
|
function rankImageCandidates(candidates, limit = 20) {
|
|
const unique = [...new Set(candidates.filter(isLikelyImageUrl))];
|
|
return unique
|
|
.map((url) => ({ url, score: scoreImageUrl(url) }))
|
|
.sort((a, b) => b.score - a.score)
|
|
.slice(0, limit)
|
|
.map((item) => item.url);
|
|
}
|
|
|
|
function pickBestGoogleImageUrl(candidates) {
|
|
return rankImageCandidates(candidates, 1)[0] || null;
|
|
}
|
|
|
|
function extractGoogleImageCandidates(html) {
|
|
const candidates = [];
|
|
|
|
for (const match of html.matchAll(/"ou":"((?:https?:\\\/\\\/|\\\/\\\/)[^"\\]+)"/g)) {
|
|
candidates.push(decodeGoogleEmbeddedUrl(match[1]));
|
|
}
|
|
|
|
for (const match of html.matchAll(/"ou":\s*"((?:https?:\/\/)[^"]+)"/g)) {
|
|
candidates.push(match[1]);
|
|
}
|
|
|
|
for (const match of html.matchAll(/\["(https:\\\/\\\/[^"]+)",(?:\d+,\d+|\d+)\]/g)) {
|
|
candidates.push(decodeGoogleEmbeddedUrl(match[1]));
|
|
}
|
|
|
|
for (const match of html.matchAll(/imgurl=([^&"]+)/g)) {
|
|
try {
|
|
candidates.push(decodeURIComponent(match[1]));
|
|
} catch {
|
|
// skip
|
|
}
|
|
}
|
|
|
|
return candidates;
|
|
}
|
|
|
|
async function searchGoogleCustomSearchImagesMany(query, limit = 20) {
|
|
const apiKey = process.env.GOOGLE_CSE_API_KEY;
|
|
const cx = process.env.GOOGLE_CSE_CX;
|
|
if (!apiKey || !cx) return [];
|
|
|
|
const out = [];
|
|
for (let start = 1; start <= 91 && out.length < limit; start += 10) {
|
|
const num = Math.min(10, limit - out.length);
|
|
const url =
|
|
`https://www.googleapis.com/customsearch/v1?key=${encodeURIComponent(apiKey)}` +
|
|
`&cx=${encodeURIComponent(cx)}&q=${encodeURIComponent(query)}&searchType=image` +
|
|
`&num=${num}&start=${start}&safe=active`;
|
|
const data = await fetchJson(url);
|
|
for (const item of data?.items || []) {
|
|
if (!item?.link) continue;
|
|
out.push({
|
|
imageUrl: item.link,
|
|
thumbUrl: item.image?.thumbnailLink || item.link,
|
|
source: 'google-custom-search',
|
|
width: item.image?.width,
|
|
height: item.image?.height,
|
|
});
|
|
if (out.length >= limit) break;
|
|
}
|
|
if (!data?.items?.length) break;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function searchGoogleCustomSearchImagesFirst(query) {
|
|
const apiKey = process.env.GOOGLE_CSE_API_KEY;
|
|
const cx = process.env.GOOGLE_CSE_CX;
|
|
if (!apiKey || !cx) return null;
|
|
|
|
const url =
|
|
`https://www.googleapis.com/customsearch/v1?key=${encodeURIComponent(apiKey)}` +
|
|
`&cx=${encodeURIComponent(cx)}&q=${encodeURIComponent(query)}&searchType=image&num=1&safe=active`;
|
|
const data = await fetchJson(url);
|
|
const item = data?.items?.[0];
|
|
if (!item?.link) return null;
|
|
return {
|
|
imageUrl: item.link,
|
|
source: 'google-custom-search',
|
|
sourceLabel: 'Google Custom Search',
|
|
};
|
|
}
|
|
|
|
async function searchDuckDuckGoImagesMany(query, limit = 20) {
|
|
const html = await fetchHtml(
|
|
`https://duckduckgo.com/?q=${encodeURIComponent(query)}&iar=images&iax=images&ia=images`,
|
|
{
|
|
headers: {
|
|
'User-Agent':
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
},
|
|
}
|
|
);
|
|
const vqd =
|
|
html.match(/vqd=['"]([^'"]+)['"]/)?.[1] ||
|
|
html.match(/vqd=([\d-]+)/)?.[1] ||
|
|
null;
|
|
if (!vqd) return [];
|
|
|
|
const out = [];
|
|
for (let page = 1; page <= 4 && out.length < limit; page++) {
|
|
await throttle();
|
|
const jsUrl = `https://duckduckgo.com/i.js?o=json&q=${encodeURIComponent(query)}&l=us-en&vqd=${encodeURIComponent(vqd)}&f=,,,&p=${page}`;
|
|
const payload = await new Promise((resolve, reject) => {
|
|
https
|
|
.get(
|
|
jsUrl,
|
|
{ headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' } },
|
|
(res) => {
|
|
let data = '';
|
|
res.on('data', (chunk) => {
|
|
data += chunk;
|
|
});
|
|
res.on('end', () => {
|
|
try {
|
|
resolve(JSON.parse(data));
|
|
} catch (err) {
|
|
reject(err);
|
|
}
|
|
});
|
|
}
|
|
)
|
|
.on('error', reject);
|
|
});
|
|
|
|
for (const item of payload?.results || []) {
|
|
const imageUrl = item?.image || item?.thumbnail;
|
|
if (!imageUrl) continue;
|
|
out.push({
|
|
imageUrl,
|
|
thumbUrl: item?.thumbnail || item?.image,
|
|
source: 'duckduckgo-images',
|
|
width: item?.width,
|
|
height: item?.height,
|
|
});
|
|
if (out.length >= limit) break;
|
|
}
|
|
if (!payload?.results?.length) break;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function searchDuckDuckGoImagesFirst(query) {
|
|
const html = await fetchHtml(
|
|
`https://duckduckgo.com/?q=${encodeURIComponent(query)}&iar=images&iax=images&ia=images`,
|
|
{
|
|
headers: {
|
|
'User-Agent':
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
},
|
|
}
|
|
);
|
|
const vqd =
|
|
html.match(/vqd=['"]([^'"]+)['"]/)?.[1] ||
|
|
html.match(/vqd=([\d-]+)/)?.[1] ||
|
|
null;
|
|
if (!vqd) return null;
|
|
|
|
await throttle();
|
|
const jsUrl = `https://duckduckgo.com/i.js?o=json&q=${encodeURIComponent(query)}&l=us-en&vqd=${encodeURIComponent(vqd)}&f=,,,&p=1`;
|
|
const payload = await new Promise((resolve, reject) => {
|
|
https
|
|
.get(
|
|
jsUrl,
|
|
{ headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' } },
|
|
(res) => {
|
|
let data = '';
|
|
res.on('data', (chunk) => {
|
|
data += chunk;
|
|
});
|
|
res.on('end', () => {
|
|
try {
|
|
resolve(JSON.parse(data));
|
|
} catch (err) {
|
|
reject(err);
|
|
}
|
|
});
|
|
}
|
|
)
|
|
.on('error', reject);
|
|
});
|
|
|
|
const first = payload?.results?.[0];
|
|
const imageUrl = first?.image || first?.thumbnail;
|
|
if (!imageUrl) return null;
|
|
return {
|
|
imageUrl,
|
|
source: 'duckduckgo-images',
|
|
sourceLabel: 'DuckDuckGo Images (fallback)',
|
|
};
|
|
}
|
|
|
|
/** First Google-family image result for artist + painting title (developer image audit). */
|
|
async function searchGoogleImagesFirst(artistName, paintingTitle) {
|
|
const simple = simplifyPaintingTitle(paintingTitle);
|
|
const query = `${artistName} ${simple} painting`.trim();
|
|
const searchUrl =
|
|
`https://www.google.com/search?q=${encodeURIComponent(query)}&tbm=isch&hl=en&ijn=0`;
|
|
|
|
const custom = await searchGoogleCustomSearchImagesFirst(query).catch(() => null);
|
|
if (custom?.imageUrl) {
|
|
return { query, searchUrl, ...custom };
|
|
}
|
|
|
|
const gac = await getGoogleArtsCultureImages(simple, artistName).catch(() => null);
|
|
if (gac?.fullUrl) {
|
|
return {
|
|
query,
|
|
imageUrl: gac.fullUrl,
|
|
thumbUrl: gac.thumbUrl,
|
|
searchUrl: `https://artsandculture.google.com/search?q=${encodeURIComponent(query)}`,
|
|
source: 'google-arts-culture',
|
|
sourceLabel: 'Google Arts & Culture',
|
|
};
|
|
}
|
|
|
|
try {
|
|
const html = await fetchHtml(searchUrl, {
|
|
headers: {
|
|
'User-Agent':
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
Accept: 'text/html,application/xhtml+xml',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
},
|
|
});
|
|
const imageUrl = pickBestGoogleImageUrl(extractGoogleImageCandidates(html));
|
|
if (imageUrl) {
|
|
return {
|
|
query,
|
|
imageUrl,
|
|
searchUrl,
|
|
source: 'google-images',
|
|
sourceLabel: 'Google Images',
|
|
};
|
|
}
|
|
} catch {
|
|
// continue to fallback
|
|
}
|
|
|
|
const ddg = await searchDuckDuckGoImagesFirst(query).catch(() => null);
|
|
if (ddg?.imageUrl) {
|
|
return { query, searchUrl, ...ddg };
|
|
}
|
|
|
|
return { query, imageUrl: null, searchUrl, source: 'google-images', sourceLabel: 'Google Images' };
|
|
}
|
|
|
|
/** First Google-family portrait result for an artist (developer portrait audit). */
|
|
async function searchArtistPortraitFirst(artistName) {
|
|
const query = `${artistName} portrait`.trim();
|
|
const searchUrl =
|
|
`https://www.google.com/search?q=${encodeURIComponent(query)}&tbm=isch&hl=en&ijn=0`;
|
|
|
|
const custom = await searchGoogleCustomSearchImagesFirst(query).catch(() => null);
|
|
if (custom?.imageUrl) {
|
|
return { query, searchUrl, ...custom };
|
|
}
|
|
|
|
const gac = await getGoogleArtsCultureImages(artistName, artistName).catch(() => null);
|
|
if (gac?.fullUrl) {
|
|
return {
|
|
query,
|
|
imageUrl: gac.fullUrl,
|
|
thumbUrl: gac.thumbUrl,
|
|
searchUrl: `https://artsandculture.google.com/search?q=${encodeURIComponent(query)}`,
|
|
source: 'google-arts-culture',
|
|
sourceLabel: 'Google Arts & Culture',
|
|
};
|
|
}
|
|
|
|
try {
|
|
const html = await fetchHtml(searchUrl, {
|
|
headers: {
|
|
'User-Agent':
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
Accept: 'text/html,application/xhtml+xml',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
},
|
|
});
|
|
const imageUrl = pickBestGoogleImageUrl(extractGoogleImageCandidates(html));
|
|
if (imageUrl) {
|
|
return {
|
|
query,
|
|
imageUrl,
|
|
searchUrl,
|
|
source: 'google-images',
|
|
sourceLabel: 'Google Images',
|
|
};
|
|
}
|
|
} catch {
|
|
// continue to fallback
|
|
}
|
|
|
|
const ddg = await searchDuckDuckGoImagesFirst(query).catch(() => null);
|
|
if (ddg?.imageUrl) {
|
|
return { query, searchUrl, ...ddg };
|
|
}
|
|
|
|
return { query, imageUrl: null, searchUrl, source: 'google-images', sourceLabel: 'Google Images' };
|
|
}
|
|
|
|
const GOOGLE_ISCH_HEADERS = {
|
|
'User-Agent':
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
Accept: 'text/html,application/xhtml+xml',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
};
|
|
|
|
async function searchDebugImagesMany(query, limit = 20) {
|
|
const searchUrl =
|
|
`https://www.google.com/search?q=${encodeURIComponent(query)}&tbm=isch&hl=en&ijn=0`;
|
|
const merged = [];
|
|
const seen = new Set();
|
|
|
|
const push = (item) => {
|
|
const url = item?.imageUrl;
|
|
if (!url || !isLikelyImageUrl(url) || seen.has(url)) return;
|
|
seen.add(url);
|
|
merged.push({
|
|
imageUrl: url,
|
|
thumbUrl: item.thumbUrl || url,
|
|
source: item.source || 'image-search',
|
|
width: item.width,
|
|
height: item.height,
|
|
});
|
|
};
|
|
|
|
for (const item of await searchGoogleCustomSearchImagesMany(query, limit).catch(() => [])) {
|
|
push(item);
|
|
}
|
|
|
|
if (merged.length < limit) {
|
|
try {
|
|
const html = await fetchHtml(searchUrl, { headers: GOOGLE_ISCH_HEADERS });
|
|
for (const url of rankImageCandidates(extractGoogleImageCandidates(html), limit - merged.length)) {
|
|
push({ imageUrl: url, source: 'google-images' });
|
|
}
|
|
} catch {
|
|
// continue
|
|
}
|
|
}
|
|
|
|
if (merged.length < limit) {
|
|
for (const item of await searchDuckDuckGoImagesMany(query, limit - merged.length).catch(() => [])) {
|
|
push(item);
|
|
}
|
|
}
|
|
|
|
return {
|
|
query,
|
|
searchUrl,
|
|
results: merged.slice(0, limit),
|
|
source: 'mixed',
|
|
sourceLabel: 'Image search results',
|
|
};
|
|
}
|
|
|
|
async function searchPaintingImagesMany(artistName, paintingTitle, limit = 20) {
|
|
const simple = simplifyPaintingTitle(paintingTitle);
|
|
const query = `${artistName} ${simple} painting`.trim();
|
|
return searchDebugImagesMany(query, limit);
|
|
}
|
|
|
|
async function searchArtistPortraitMany(artistName, limit = 20) {
|
|
const query = `${artistName} portrait`.trim();
|
|
return searchDebugImagesMany(query, limit);
|
|
}
|
|
|
|
async function generateThumbnailFromFull(fullDest, thumbDest, width = THUMB_WIDTH) {
|
|
const sharp = require('sharp');
|
|
if (!fs.existsSync(fullDest)) return false;
|
|
|
|
const ext = path.extname(thumbDest).toLowerCase();
|
|
// Museum scans can exceed Sharp's default ~268MP input cap; we only resize down.
|
|
let pipeline = sharp(fullDest, {
|
|
limitInputPixels: false,
|
|
sequentialRead: true,
|
|
})
|
|
.rotate()
|
|
.resize({ width, withoutEnlargement: true });
|
|
|
|
if (ext === '.png') {
|
|
pipeline = pipeline.png({ quality: 85 });
|
|
} else if (ext === '.webp') {
|
|
pipeline = pipeline.webp({ quality: 85 });
|
|
} else {
|
|
pipeline = pipeline.jpeg({ quality: 85 });
|
|
}
|
|
|
|
await pipeline.toFile(thumbDest);
|
|
return true;
|
|
}
|
|
|
|
async function savePaintingImages(wikiTitle, baseFilename, imageDir, options = {}) {
|
|
const deadline = deadlineFromOptions(options);
|
|
activeDeadline = deadline.active ? deadline : null;
|
|
|
|
try {
|
|
const resolved = await resolvePaintingImages(wikiTitle, options);
|
|
if (!resolved) return { imagePath: null, thumbnailPath: null, source: null, resolvedWikiTitle: null };
|
|
|
|
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
|
|
|
|
const paintingsDir = path.join(imageDir, 'paintings');
|
|
const thumbsDir = path.join(imageDir, 'paintings', 'thumbs');
|
|
if (!fs.existsSync(paintingsDir)) fs.mkdirSync(paintingsDir, { recursive: true });
|
|
if (!fs.existsSync(thumbsDir)) fs.mkdirSync(thumbsDir, { recursive: true });
|
|
|
|
const safeBase = baseFilename.replace(/[^a-zA-Z0-9_-]/g, '_');
|
|
const fullExt = pickExt(resolved.fullUrl);
|
|
|
|
const fullDest = path.join(paintingsDir, safeBase + fullExt);
|
|
const thumbDest = path.join(thumbsDir, safeBase + '_thumb.jpg');
|
|
|
|
let imagePath = null;
|
|
let thumbnailPath = null;
|
|
|
|
try {
|
|
await downloadImageToFile(resolved.fullUrl, fullDest);
|
|
imagePath = path.join('paintings', safeBase + fullExt).replace(/\\/g, '/');
|
|
} catch (err) {
|
|
if (err.code === 'FETCH_TIME_LIMIT') throw err;
|
|
console.warn(` Full image download failed: ${err.message}`);
|
|
}
|
|
|
|
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
|
|
|
|
if (imagePath) {
|
|
try {
|
|
await generateThumbnailFromFull(fullDest, thumbDest);
|
|
thumbnailPath = path.join('paintings', 'thumbs', safeBase + '_thumb.jpg').replace(/\\/g, '/');
|
|
} catch (err) {
|
|
console.warn(` Thumbnail generation failed: ${err.message}`);
|
|
thumbnailPath = imagePath;
|
|
}
|
|
} else {
|
|
try {
|
|
const thumbSrc =
|
|
resolved.thumbUrl !== resolved.fullUrl ? resolved.thumbUrl : resolved.fullUrl;
|
|
const thumbExt = pickExt(resolved.thumbUrl);
|
|
const fallbackThumbDest = path.join(thumbsDir, safeBase + '_thumb' + thumbExt);
|
|
await downloadImageToFile(thumbSrc, fallbackThumbDest);
|
|
thumbnailPath = path
|
|
.join('paintings', 'thumbs', safeBase + '_thumb' + thumbExt)
|
|
.replace(/\\/g, '/');
|
|
imagePath = thumbnailPath;
|
|
} catch (err) {
|
|
if (err.code === 'FETCH_TIME_LIMIT') throw err;
|
|
console.warn(` Thumbnail download failed: ${err.message}`);
|
|
}
|
|
}
|
|
|
|
return {
|
|
imagePath,
|
|
thumbnailPath,
|
|
source: resolved.source,
|
|
resolvedWikiTitle: resolved.resolvedWikiTitle || wikiTitle,
|
|
};
|
|
} finally {
|
|
activeDeadline = null;
|
|
}
|
|
}
|
|
|
|
async function saveImageForItem(wikiTitle, subdir, filename, imageDir, options = {}) {
|
|
if (options.type === 'painting') {
|
|
const result = await savePaintingImages(wikiTitle, filename, imageDir, options);
|
|
return {
|
|
path: result.imagePath,
|
|
thumbnailPath: result.thumbnailPath,
|
|
source: result.source,
|
|
};
|
|
}
|
|
|
|
const resolved = await resolveImageUrl(wikiTitle, options);
|
|
if (!resolved) return { path: null, source: null };
|
|
|
|
const dir = path.join(imageDir, subdir);
|
|
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
|
|
const ext = pickExt(resolved.url);
|
|
const safeName = filename.replace(/[^a-zA-Z0-9_-]/g, '_') + ext;
|
|
const destPath = path.join(dir, safeName);
|
|
|
|
try {
|
|
await downloadImageToFile(resolved.url, destPath);
|
|
return {
|
|
path: path.join(subdir, safeName).replace(/\\/g, '/'),
|
|
source: resolved.source,
|
|
};
|
|
} catch (err) {
|
|
console.warn(` Download failed (${resolved.source}): ${err.message}`);
|
|
return { path: null, source: resolved.source };
|
|
}
|
|
}
|
|
|
|
const ARTIST_PORTRAIT_WIKI_OVERRIDES = {
|
|
Zeuxis: 'Zeuxis (painter)',
|
|
'Ivan Klyun': 'Ivan Kliun',
|
|
'Jean-Antoine Watteau': 'Antoine Watteau',
|
|
};
|
|
|
|
function artistPortraitWikiCandidates(artistName, wikiTitle) {
|
|
const seen = new Set();
|
|
const out = [];
|
|
for (const title of [
|
|
ARTIST_PORTRAIT_WIKI_OVERRIDES[artistName],
|
|
ARTIST_PORTRAIT_WIKI_OVERRIDES[wikiTitle],
|
|
wikiTitle,
|
|
artistName,
|
|
`${artistName} (painter)`,
|
|
`${wikiTitle} (painter)`,
|
|
]) {
|
|
if (title && !seen.has(title)) {
|
|
seen.add(title);
|
|
out.push(title);
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function resolveArtistPortrait(artistName, wikiTitle) {
|
|
for (const title of artistPortraitWikiCandidates(artistName, wikiTitle)) {
|
|
const images = await getWikipediaImages(title);
|
|
if (images?.fullUrl) {
|
|
return { ...images, wikipedia_title: title };
|
|
}
|
|
}
|
|
|
|
const search = await searchArtistPortraitFirst(artistName);
|
|
if (search?.imageUrl) {
|
|
return {
|
|
fullUrl: search.imageUrl,
|
|
source: search.sourceLabel || search.source || 'web search',
|
|
wikipedia_title: wikiTitle,
|
|
};
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function findLocalPortraitPath(artistName, imageDir) {
|
|
const safeBase = artistName.replace(/[^a-zA-Z0-9_-]/g, '_');
|
|
for (const ext of ['.jpg', '.jpeg', '.png', '.webp', '.JPG']) {
|
|
const rel = `portraits/${safeBase}${ext}`;
|
|
if (fs.existsSync(path.join(imageDir, rel))) return rel.replace(/\\/g, '/');
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function saveArtistPortrait(artistName, wikiTitle, imageDir) {
|
|
const local = findLocalPortraitPath(artistName, imageDir);
|
|
if (local) return { path: local, source: 'local disk' };
|
|
|
|
const resolved = await resolveArtistPortrait(artistName, wikiTitle);
|
|
if (!resolved?.fullUrl) return { path: null, source: null };
|
|
|
|
const portraitsDir = path.join(imageDir, 'portraits');
|
|
if (!fs.existsSync(portraitsDir)) fs.mkdirSync(portraitsDir, { recursive: true });
|
|
|
|
const safeBase = artistName.replace(/[^a-zA-Z0-9_-]/g, '_');
|
|
const ext = pickExt(resolved.fullUrl);
|
|
const destPath = path.join(portraitsDir, safeBase + ext);
|
|
|
|
try {
|
|
await downloadImageToFile(resolved.fullUrl, destPath);
|
|
return {
|
|
path: path.join('portraits', safeBase + ext).replace(/\\/g, '/'),
|
|
source: resolved.source,
|
|
};
|
|
} catch (err) {
|
|
console.warn(` Portrait download failed: ${err.message}`);
|
|
return { path: null, source: resolved.source };
|
|
}
|
|
}
|
|
|
|
module.exports = {
|
|
resolveImageUrl,
|
|
resolvePaintingImages,
|
|
saveImageForItem,
|
|
savePaintingImages,
|
|
saveArtistPortrait,
|
|
findLocalPortraitPath,
|
|
resolveArtistPortrait,
|
|
downloadImageToFile,
|
|
downloadImageForFix,
|
|
generateThumbnailFromFull,
|
|
searchWikipediaTitle,
|
|
searchWebForPaintingImages,
|
|
searchGoogleImagesFirst,
|
|
searchArtistPortraitFirst,
|
|
searchPaintingImagesMany,
|
|
searchArtistPortraitMany,
|
|
fetchImageBuffer,
|
|
friendlyImageFetchError,
|
|
pickExt,
|
|
getGoogleArtsCultureImages,
|
|
simplifyPaintingTitle,
|
|
fetchHtml,
|
|
DEFAULT_BATCH_MAX_WAIT_MS,
|
|
FetchDeadline,
|
|
sleep,
|
|
};
|