Add influence rework, image checkup, debug mode, and fetched paintings.
Support artist and movement influence links with web discovery, a developer checkup table with gallery/detail thumbnails, and debug image search with fix-it workflow. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Danila Khodjaef
co-authored by
Cursor
Cursor
parent
0ece1195fa
commit
bf7db9b25e
+191
-2
@@ -1173,14 +1173,199 @@ async function resolveImageUrl(wikiTitle, options = {}) {
|
||||
return { url: pair.fullUrl, source: pair.source };
|
||||
}
|
||||
|
||||
async function downloadImageToFile(url, destPath) {
|
||||
if (fs.existsSync(destPath)) return destPath;
|
||||
async function downloadImageToFile(url, destPath, { force = false } = {}) {
|
||||
if (!force && fs.existsSync(destPath)) return destPath;
|
||||
await throttle();
|
||||
const buffer = await withRetry(() => fetchBuffer(url));
|
||||
fs.mkdirSync(path.dirname(destPath), { recursive: true });
|
||||
fs.writeFileSync(destPath, buffer);
|
||||
return destPath;
|
||||
}
|
||||
|
||||
function decodeGoogleEmbeddedUrl(raw) {
|
||||
return raw
|
||||
.replace(/\\u003d/g, '=')
|
||||
.replace(/\\u0026/g, '&')
|
||||
.replace(/\\u0027/g, "'")
|
||||
.replace(/\\\//g, '/');
|
||||
}
|
||||
|
||||
function isLikelyImageUrl(url) {
|
||||
if (!url || !/^https?:\/\//i.test(url)) return false;
|
||||
if (/^data:/i.test(url)) return false;
|
||||
if (/gstatic\.com\/images\?/i.test(url)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function pickBestGoogleImageUrl(candidates) {
|
||||
const unique = [...new Set(candidates.filter(isLikelyImageUrl))];
|
||||
const scored = unique.map((url) => {
|
||||
let score = 0;
|
||||
if (/upload\.wikimedia\.org/i.test(url)) score += 40;
|
||||
if (/googleusercontent\.com/i.test(url)) score += 30;
|
||||
if (/\.(jpe?g|png|webp)(\?|$)/i.test(url)) score += 20;
|
||||
if (/thumb|thumbnail|small|icon|logo|avatar/i.test(url)) score -= 25;
|
||||
if (/=\s*[sS]\d{2,3}(-c|-rw)?(\||$)/.test(url) || /[?&]w=\d{2,3}(&|$)/.test(url)) score -= 15;
|
||||
return { url, score };
|
||||
});
|
||||
scored.sort((a, b) => b.score - a.score);
|
||||
return scored[0]?.url || unique[0] || null;
|
||||
}
|
||||
|
||||
function extractGoogleImageCandidates(html) {
|
||||
const candidates = [];
|
||||
|
||||
for (const match of html.matchAll(/"ou":"((?:https?:\\\/\\\/|\\\/\\\/)[^"\\]+)"/g)) {
|
||||
candidates.push(decodeGoogleEmbeddedUrl(match[1]));
|
||||
}
|
||||
|
||||
for (const match of html.matchAll(/"ou":\s*"((?:https?:\/\/)[^"]+)"/g)) {
|
||||
candidates.push(match[1]);
|
||||
}
|
||||
|
||||
for (const match of html.matchAll(/\["(https:\\\/\\\/[^"]+)",(?:\d+,\d+|\d+)\]/g)) {
|
||||
candidates.push(decodeGoogleEmbeddedUrl(match[1]));
|
||||
}
|
||||
|
||||
for (const match of html.matchAll(/imgurl=([^&"]+)/g)) {
|
||||
try {
|
||||
candidates.push(decodeURIComponent(match[1]));
|
||||
} catch {
|
||||
// skip
|
||||
}
|
||||
}
|
||||
|
||||
return candidates;
|
||||
}
|
||||
|
||||
async function searchGoogleCustomSearchImagesFirst(query) {
|
||||
const apiKey = process.env.GOOGLE_CSE_API_KEY;
|
||||
const cx = process.env.GOOGLE_CSE_CX;
|
||||
if (!apiKey || !cx) return null;
|
||||
|
||||
const url =
|
||||
`https://www.googleapis.com/customsearch/v1?key=${encodeURIComponent(apiKey)}` +
|
||||
`&cx=${encodeURIComponent(cx)}&q=${encodeURIComponent(query)}&searchType=image&num=1&safe=active`;
|
||||
const data = await fetchJson(url);
|
||||
const item = data?.items?.[0];
|
||||
if (!item?.link) return null;
|
||||
return {
|
||||
imageUrl: item.link,
|
||||
source: 'google-custom-search',
|
||||
sourceLabel: 'Google Custom Search',
|
||||
};
|
||||
}
|
||||
|
||||
async function searchDuckDuckGoImagesFirst(query) {
|
||||
const html = await fetchHtml(
|
||||
`https://duckduckgo.com/?q=${encodeURIComponent(query)}&iar=images&iax=images&ia=images`,
|
||||
{
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
},
|
||||
}
|
||||
);
|
||||
const vqd =
|
||||
html.match(/vqd=['"]([^'"]+)['"]/)?.[1] ||
|
||||
html.match(/vqd=([\d-]+)/)?.[1] ||
|
||||
null;
|
||||
if (!vqd) return null;
|
||||
|
||||
await throttle();
|
||||
const jsUrl = `https://duckduckgo.com/i.js?o=json&q=${encodeURIComponent(query)}&l=us-en&vqd=${encodeURIComponent(vqd)}&f=,,,&p=1`;
|
||||
const payload = await new Promise((resolve, reject) => {
|
||||
https
|
||||
.get(
|
||||
jsUrl,
|
||||
{ headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' } },
|
||||
(res) => {
|
||||
let data = '';
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
res.on('end', () => {
|
||||
try {
|
||||
resolve(JSON.parse(data));
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
}
|
||||
)
|
||||
.on('error', reject);
|
||||
});
|
||||
|
||||
const first = payload?.results?.[0];
|
||||
const imageUrl = first?.image || first?.thumbnail;
|
||||
if (!imageUrl) return null;
|
||||
return {
|
||||
imageUrl,
|
||||
source: 'duckduckgo-images',
|
||||
sourceLabel: 'DuckDuckGo Images (fallback)',
|
||||
};
|
||||
}
|
||||
|
||||
/** First Google-family image result for artist + painting title (developer image audit). */
|
||||
async function searchGoogleImagesFirst(artistName, paintingTitle) {
|
||||
const simple = simplifyPaintingTitle(paintingTitle);
|
||||
const query = `${artistName} ${simple} painting`.trim();
|
||||
const searchUrl =
|
||||
`https://www.google.com/search?q=${encodeURIComponent(query)}&tbm=isch&hl=en&ijn=0`;
|
||||
|
||||
const custom = await searchGoogleCustomSearchImagesFirst(query).catch(() => null);
|
||||
if (custom?.imageUrl) {
|
||||
return { query, searchUrl, ...custom };
|
||||
}
|
||||
|
||||
const gac = await getGoogleArtsCultureImages(simple, artistName).catch(() => null);
|
||||
if (gac?.fullUrl) {
|
||||
return {
|
||||
query,
|
||||
imageUrl: gac.fullUrl,
|
||||
thumbUrl: gac.thumbUrl,
|
||||
searchUrl: `https://artsandculture.google.com/search?q=${encodeURIComponent(query)}`,
|
||||
source: 'google-arts-culture',
|
||||
sourceLabel: 'Google Arts & Culture',
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const html = await fetchHtml(searchUrl, {
|
||||
headers: {
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
Accept: 'text/html,application/xhtml+xml',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
},
|
||||
});
|
||||
const imageUrl = pickBestGoogleImageUrl(extractGoogleImageCandidates(html));
|
||||
if (imageUrl) {
|
||||
return {
|
||||
query,
|
||||
imageUrl,
|
||||
searchUrl,
|
||||
source: 'google-images',
|
||||
sourceLabel: 'Google Images',
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// continue to fallback
|
||||
}
|
||||
|
||||
const ddg = await searchDuckDuckGoImagesFirst(query).catch(() => null);
|
||||
if (ddg?.imageUrl) {
|
||||
return { query, searchUrl, ...ddg };
|
||||
}
|
||||
|
||||
return { query, imageUrl: null, searchUrl, source: 'google-images', sourceLabel: 'Google Images' };
|
||||
}
|
||||
|
||||
async function fetchImageBuffer(url) {
|
||||
await throttle();
|
||||
return withRetry(() => fetchBuffer(url));
|
||||
}
|
||||
|
||||
async function generateThumbnailFromFull(fullDest, thumbDest, width = THUMB_WIDTH) {
|
||||
const sharp = require('sharp');
|
||||
if (!fs.existsSync(fullDest)) return false;
|
||||
@@ -1311,8 +1496,12 @@ module.exports = {
|
||||
generateThumbnailFromFull,
|
||||
searchWikipediaTitle,
|
||||
searchWebForPaintingImages,
|
||||
searchGoogleImagesFirst,
|
||||
fetchImageBuffer,
|
||||
pickExt,
|
||||
getGoogleArtsCultureImages,
|
||||
simplifyPaintingTitle,
|
||||
fetchHtml,
|
||||
DEFAULT_BATCH_MAX_WAIT_MS,
|
||||
FetchDeadline,
|
||||
sleep,
|
||||
|
||||
Reference in New Issue
Block a user