const https = require('https'); const http = require('http'); const fs = require('fs'); const path = require('path'); const USER_AGENT = 'VirtualArtGallery/1.0 (educational art history project; local museum gallery)'; let lastRequestTime = 0; const MIN_DELAY_MS = 2500; const THUMB_WIDTH = 400; const FULL_WIDTH = 1600; const PAINTING_WIKI_OVERRIDES = { 'Job Cigarette Papers': 'Job (advertising)', 'Charing Cross Bridge': 'Charing Cross Bridge (Derain)', 'Cut with the Dada Kitchen Knife': 'Cut with the Dada Kitchen Knife through the Last Weimar Beer-Belly Cultural Epoch in Germany', }; const DIRECT_IMAGE_OVERRIDES = { 'The Persistence of Memory': { thumbUrl: 'https://upload.wikimedia.org/wikipedia/en/d/dd/The_Persistence_of_Memory.jpg', fullUrl: 'https://upload.wikimedia.org/wikipedia/en/d/dd/The_Persistence_of_Memory.jpg', source: 'Wikimedia Commons (Museum of Modern Art, New York)', }, }; function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } async function throttle() { const elapsed = Date.now() - lastRequestTime; if (elapsed < MIN_DELAY_MS) await sleep(MIN_DELAY_MS - elapsed); lastRequestTime = Date.now(); } function fetchBuffer(url, redirectCount = 0, referer = null) { return new Promise((resolve, reject) => { if (redirectCount > 8) return reject(new Error('Too many redirects')); const client = url.startsWith('https') ? https : http; const headers = { 'User-Agent': USER_AGENT, Accept: 'image/*,*/*', }; if (referer) headers.Referer = referer; if (url.includes('wikimedia.org') || url.includes('wikipedia.org')) { headers.Referer = 'https://commons.wikimedia.org/'; } if (url.includes('artic.edu')) { headers.Referer = 'https://www.artic.edu/'; } const req = client.get(url, { headers }, (res) => { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { const next = res.headers.location.startsWith('http') ? res.headers.location : new URL(res.headers.location, url).href; return resolve(fetchBuffer(next, redirectCount + 1, referer)); } if (res.statusCode === 429) { res.resume(); return reject(new Error('HTTP 429')); } if (res.statusCode !== 200) { res.resume(); return reject(new Error(`HTTP ${res.statusCode}`)); } const chunks = []; res.on('data', (c) => chunks.push(c)); res.on('end', () => resolve(Buffer.concat(chunks))); } ); req.on('error', reject); }); } async function fetchJson(url) { await throttle(); return new Promise((resolve, reject) => { const client = url.startsWith('https') ? https : http; client .get(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { return fetchJson(res.headers.location).then(resolve).catch(reject); } let data = ''; res.on('data', (c) => (data += c)); res.on('end', () => { try { resolve(JSON.parse(data)); } catch (e) { reject(e); } }); }) .on('error', reject); }); } async function withRetry(fn, retries = 6) { for (let i = 0; i < retries; i++) { try { return await fn(); } catch (err) { if (i === retries - 1) throw err; const wait = err.message.includes('429') ? 8000 * (i + 1) : 2000 * (i + 1); await sleep(wait); } } } function pickExt(url) { try { const ext = path.extname(new URL(url).pathname) || '.jpg'; return ext.length > 5 ? '.jpg' : ext; } catch { return '.jpg'; } } async function getWikidataImageFilename(wikiTitle) { const propsUrl = `https://en.wikipedia.org/w/api.php?action=query&titles=${encodeURIComponent(wikiTitle)}&prop=pageprops&ppprop=wikibase_item&format=json`; const props = await fetchJson(propsUrl); const page = Object.values(props.query?.pages || {})[0]; const qid = page?.pageprops?.wikibase_item; if (!qid) return null; const entityUrl = `https://www.wikidata.org/wiki/Special:EntityData/${qid}.json`; const entity = await fetchJson(entityUrl); const claims = entity.entities?.[qid]?.claims?.P18; if (!claims?.length) return null; return claims[0].mainsnak?.datavalue?.value; } async function getCommonsFileUrls(filename) { const fileTitle = filename.startsWith('File:') ? filename : `File:${filename}`; const url = `https://commons.wikimedia.org/w/api.php?action=query&titles=${encodeURIComponent(fileTitle)}&prop=imageinfo&iiprop=url&iiurlwidth=${THUMB_WIDTH}&format=json`; const data = await fetchJson(url); const page = Object.values(data.query?.pages || {})[0]; const info = page?.imageinfo?.[0]; if (!info) return null; return { thumbUrl: info.thumburl || info.url, fullUrl: info.url || info.thumburl, source: 'Wikimedia Commons', }; } async function getWikipediaImages(wikiTitle) { const url = `https://en.wikipedia.org/w/api.php?action=query&titles=${encodeURIComponent(wikiTitle)}&prop=pageimages&piprop=original|thumbnail&pithumbsize=${THUMB_WIDTH}&format=json`; const data = await fetchJson(url); const page = Object.values(data.query?.pages || {})[0]; if (!page) return null; const thumbUrl = page.thumbnail?.source; const fullUrl = page.original?.source || thumbUrl; if (!fullUrl) return null; return { thumbUrl: thumbUrl || fullUrl, fullUrl, source: 'Wikimedia Commons (via Wikipedia)', }; } async function searchCommonsImages(artistName, paintingTitle) { const queries = [ [artistName, paintingTitle].filter(Boolean).join(' '), paintingTitle, [artistName, ...(paintingTitle || '').split(/\s+/).slice(0, 4)].join(' '), [artistName, 'painting', (paintingTitle || '').split('(')[0].trim()].filter(Boolean).join(' '), ]; const seen = new Set(); for (const query of queries) { if (!query || seen.has(query)) continue; seen.add(query); const url = `https://commons.wikimedia.org/w/api.php?action=query&generator=search&gssearch=${encodeURIComponent(query)}&gsnamespace=6&gslimit=8&prop=imageinfo&iiprop=url&iiurlwidth=${THUMB_WIDTH}&format=json`; try { const data = await fetchJson(url); const pages = data.query?.pages; if (!pages) continue; const artistToken = (artistName || '').toLowerCase().split(' ').pop() || ''; const titleTokens = (paintingTitle || '') .toLowerCase() .replace(/[^a-z0-9\s]/g, ' ') .split(/\s+/) .filter((t) => t.length > 3) .slice(0, 4); for (const page of Object.values(pages)) { const info = page.imageinfo?.[0]; if (!info?.url) continue; const fileTitle = (page.title || '').toLowerCase(); if (artistToken && !fileTitle.includes(artistToken) && !fileTitle.includes('raphael') && !fileTitle.includes('raffael')) { continue; } const matchCount = titleTokens.filter((t) => fileTitle.includes(t)).length; if (titleTokens.length > 0 && matchCount === 0 && !fileTitle.includes('uffizi')) continue; return { thumbUrl: info.thumburl || info.url, fullUrl: info.url, source: 'Wikimedia Commons (search)', }; } } catch { continue; } } return null; } async function getMetMuseumImages(searchTerm, artistName) { const query = [artistName, searchTerm].filter(Boolean).join(' '); const searchUrl = `https://collectionapi.metmuseum.org/public/collection/v1/search?q=${encodeURIComponent(query)}&hasImages=true`; try { await throttle(); const search = await fetchJson(searchUrl); if (!search.objectIDs?.length) return null; for (const id of search.objectIDs.slice(0, 10)) { await throttle(); const obj = await fetchJson( `https://collectionapi.metmuseum.org/public/collection/v1/objects/${id}` ); if (!obj.isPublicDomain || !obj.primaryImage) continue; const titleMatch = !searchTerm || (obj.title || '').toLowerCase().includes(searchTerm.toLowerCase().slice(0, 10)); if (!titleMatch && search.objectIDs.length > 3) continue; return { thumbUrl: obj.primaryImageSmall || obj.primaryImage, fullUrl: obj.primaryImage, source: 'The Metropolitan Museum of Art (Open Access)', }; } } catch { return null; } return null; } async function getArtInstituteImages(searchTerm, artistName) { const query = [artistName, searchTerm].filter(Boolean).join(' '); const url = `https://api.artic.edu/api/v1/artworks/search?q=${encodeURIComponent(query)}&fields=id,title,artist_title,image_id&limit=6`; try { const data = await fetchJson(url); for (const art of data.data || []) { if (!art.image_id) continue; const titleOk = !searchTerm || (art.title || '').toLowerCase().includes(searchTerm.toLowerCase().slice(0, 8)); const artistOk = !artistName || (art.artist_title || '').toLowerCase().includes(artistName.toLowerCase().split(' ').pop()); if (!titleOk && !artistOk) continue; const base = `https://www.artic.edu/iiif/2/${art.image_id}`; return { thumbUrl: `${base}/full/200,/0/default.jpg`, fullUrl: `${base}/full/1686,/0/default.jpg`, source: 'Art Institute of Chicago (Open Access)', }; } } catch { return null; } return null; } async function getRijksmuseumImages(searchTerm, artistName) { const query = [artistName, searchTerm].filter(Boolean).join(' '); const url = `https://www.rijksmuseum.nl/api/en/collection?key=0&format=json&ps=8&s=relevance&q=${encodeURIComponent(query)}&imgonly=true`; try { const data = await fetchJson(url); for (const item of data.artObjects || []) { if (!item.webImage?.url) continue; return { thumbUrl: item.webImage.url, fullUrl: item.webImage.url, source: 'Rijksmuseum', }; } } catch { return null; } return null; } async function getWikidataImagesBySearch(artistName, paintingTitle) { const label = (paintingTitle || '').replace(/"/g, '\\"').slice(0, 120); const query = ` SELECT ?img ?imgLabel WHERE { ?work rdfs:label ?workLabel . FILTER(LANG(?workLabel) = "en") FILTER(CONTAINS(LCASE(?workLabel), LCASE("${label.slice(0, 40)}"))) ?work wdt:P170 ?artist . ?artist rdfs:label ?artistLabel . FILTER(CONTAINS(LCASE(?artistLabel), "${(artistName || '').split(' ').pop().toLowerCase()}")) ?work wdt:P18 ?img . SERVICE wikibase:label { bd:serviceParam wikibase:language "en". } } LIMIT 3`; try { const url = `https://query.wikidata.org/sparql?format=json&query=${encodeURIComponent(query)}`; const data = await fetchJson(url); const row = data.results?.bindings?.[0]; if (!row?.img?.value) return null; const filename = decodeURIComponent(row.img.value.split('/').pop()); return getCommonsFileUrls(filename); } catch { return null; } } async function resolvePaintingImages(wikiTitle, options = {}) { const { artistName, paintingTitle } = options; const searchTitle = paintingTitle || wikiTitle; const lookupTitle = PAINTING_WIKI_OVERRIDES[searchTitle] || wikiTitle; if (DIRECT_IMAGE_OVERRIDES[searchTitle]) { return DIRECT_IMAGE_OVERRIDES[searchTitle]; } const sources = [ async () => { const filename = await getWikidataImageFilename(lookupTitle); if (!filename) return null; const commons = await getCommonsFileUrls(filename); if (commons) return commons; const filePathUrl = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(filename)}`; return { thumbUrl: filePathUrl, fullUrl: filePathUrl, source: 'Wikimedia Commons (Wikidata)' }; }, () => getWikipediaImages(lookupTitle), () => searchCommonsImages(artistName, searchTitle), () => getMetMuseumImages(searchTitle, artistName), () => getArtInstituteImages(searchTitle, artistName), () => getRijksmuseumImages(searchTitle, artistName), () => getWikidataImagesBySearch(artistName, searchTitle), () => getWikipediaImages(searchTitle), () => searchCommonsImages(artistName, lookupTitle), ]; for (const fn of sources) { try { const result = await fn(); if (result?.fullUrl || result?.thumbUrl) { return { thumbUrl: result.thumbUrl || result.fullUrl, fullUrl: result.fullUrl || result.thumbUrl, source: result.source, }; } } catch {} } return null; } /** @deprecated use resolvePaintingImages */ async function resolveImageUrl(wikiTitle, options = {}) { const pair = await resolvePaintingImages(wikiTitle, options); if (!pair) return null; return { url: pair.fullUrl, source: pair.source }; } async function downloadImageToFile(url, destPath) { if (fs.existsSync(destPath)) return destPath; await throttle(); const buffer = await withRetry(() => fetchBuffer(url)); fs.writeFileSync(destPath, buffer); return destPath; } async function savePaintingImages(wikiTitle, baseFilename, imageDir, options = {}) { const resolved = await resolvePaintingImages(wikiTitle, options); if (!resolved) return { imagePath: null, thumbnailPath: null, source: null }; const paintingsDir = path.join(imageDir, 'paintings'); const thumbsDir = path.join(imageDir, 'paintings', 'thumbs'); if (!fs.existsSync(paintingsDir)) fs.mkdirSync(paintingsDir, { recursive: true }); if (!fs.existsSync(thumbsDir)) fs.mkdirSync(thumbsDir, { recursive: true }); const safeBase = baseFilename.replace(/[^a-zA-Z0-9_-]/g, '_'); const fullExt = pickExt(resolved.fullUrl); const thumbExt = pickExt(resolved.thumbUrl); const fullDest = path.join(paintingsDir, safeBase + fullExt); const thumbDest = path.join(thumbsDir, safeBase + '_thumb' + thumbExt); let imagePath = null; let thumbnailPath = null; try { await downloadImageToFile(resolved.fullUrl, fullDest); imagePath = path.join('paintings', safeBase + fullExt).replace(/\\/g, '/'); } catch (err) { console.warn(` Full image download failed: ${err.message}`); } try { const thumbSrc = resolved.thumbUrl !== resolved.fullUrl ? resolved.thumbUrl : resolved.fullUrl; await downloadImageToFile(thumbSrc, thumbDest); thumbnailPath = path.join('paintings', 'thumbs', safeBase + '_thumb' + thumbExt).replace(/\\/g, '/'); } catch (err) { console.warn(` Thumbnail download failed: ${err.message}`); if (imagePath) thumbnailPath = imagePath; } if (!imagePath && thumbnailPath) imagePath = thumbnailPath; return { imagePath, thumbnailPath, source: resolved.source }; } async function saveImageForItem(wikiTitle, subdir, filename, imageDir, options = {}) { if (options.type === 'painting') { const result = await savePaintingImages(wikiTitle, filename, imageDir, options); return { path: result.imagePath, thumbnailPath: result.thumbnailPath, source: result.source, }; } const resolved = await resolveImageUrl(wikiTitle, options); if (!resolved) return { path: null, source: null }; const dir = path.join(imageDir, subdir); if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); const ext = pickExt(resolved.url); const safeName = filename.replace(/[^a-zA-Z0-9_-]/g, '_') + ext; const destPath = path.join(dir, safeName); try { await downloadImageToFile(resolved.url, destPath); return { path: path.join(subdir, safeName).replace(/\\/g, '/'), source: resolved.source, }; } catch (err) { console.warn(` Download failed (${resolved.source}): ${err.message}`); return { path: null, source: resolved.source }; } } module.exports = { resolveImageUrl, resolvePaintingImages, saveImageForItem, savePaintingImages, downloadImageToFile, sleep, };