Danilka/Art-gallery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve batch image fetch speed and document the pipeline.

Browse Source 
Add random queue sampling and a 10s per-painting deadline for fetch-images batches, cap HTTP timeouts to the remaining budget, and update docs plus newly fetched artwork files.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Danila Khodjaef
2026-06-19 17:59:32 +03:00
co-authored by Cursor
parent d8385d83d6
commit 35c337253d
199 changed files with 206 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/image-fetcher.js
+151 -56
View File
@@ -27,6 +27,8 @@ const TRUSTED_IMAGE_HOSTS = [
let lastRequestTime = 0;
const MIN_DELAY_MS = 2500;
const REQUEST_TIMEOUT_MS = 15000;
const DEFAULT_BATCH_MAX_WAIT_MS = 10000;
const THUMB_WIDTH = 400;
const FULL_WIDTH = 1600;
@@ -137,9 +139,64 @@ function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
class FetchDeadline {
constructor(maxMs) {
this.deadline = maxMs ? Date.now() + maxMs : null;
}
get active() {
return this.deadline !== null;
}
expired() {
return this.deadline !== null && Date.now() >= this.deadline;
}
remainingMs() {
return this.deadline ? Math.max(0, this.deadline - Date.now()) : Infinity;
}
throwIfExpired() {
if (this.expired()) {
const err = new Error('Painting fetch time limit exceeded');
err.code = 'FETCH_TIME_LIMIT';
throw err;
}
}
}
/** Per-operation deadline for batch fetches; cleared after savePaintingImages finishes. */
let activeDeadline = null;
function deadlineFromOptions(options = {}) {
if (options.deadline instanceof FetchDeadline) return options.deadline;
if (options.maxWaitMs) return new FetchDeadline(options.maxWaitMs);
return new FetchDeadline(null);
}
function requestTimeoutMs() {
if (!activeDeadline?.active) return REQUEST_TIMEOUT_MS;
return Math.min(REQUEST_TIMEOUT_MS, Math.max(500, activeDeadline.remainingMs() - 50));
}
function attachRequestTimeout(req, reject) {
const timeoutMs = requestTimeoutMs();
req.setTimeout(timeoutMs, () => {
req.destroy();
reject(new Error('Request timeout'));
});
}
async function throttle() {
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
const elapsed = Date.now() - lastRequestTime;
if (elapsed < MIN_DELAY_MS) await sleep(MIN_DELAY_MS - elapsed);
const minDelay = activeDeadline?.active ? 0 : MIN_DELAY_MS;
let wait = Math.max(0, minDelay - elapsed);
if (activeDeadline?.active) {
wait = Math.min(wait, activeDeadline.remainingMs() - 50);
if (wait <= 0) activeDeadline.throwIfExpired();
}
if (wait > 0) await sleep(wait);
lastRequestTime = Date.now();
}
@@ -187,6 +244,7 @@ function fetchBuffer(url, redirectCount = 0, referer = null) {
res.on('end', () => resolve(Buffer.concat(chunks)));
}
);
attachRequestTimeout(req, reject);
req.on('error', reject);
});
}
@@ -195,7 +253,7 @@ async function fetchJson(url) {
await throttle();
return new Promise((resolve, reject) => {
const client = url.startsWith('https') ? https : http;
client
const req = client
.get(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
return fetchJson(res.headers.location).then(resolve).catch(reject);
@@ -211,6 +269,7 @@ async function fetchJson(url) {
});
})
.on('error', reject);
attachRequestTimeout(req, reject);
});
}
@@ -249,6 +308,7 @@ async function fetchHtml(url, options = {}) {
});
}
);
attachRequestTimeout(req, reject);
req.on('error', reject);
if (body) req.write(body);
req.end();
@@ -295,13 +355,22 @@ function extractImageUrlsFromText(text) {
}
async function withRetry(fn, retries = 6) {
for (let i = 0; i < retries; i++) {
const maxRetries = activeDeadline?.active ? Math.min(retries, 2) : retries;
for (let i = 0; i < maxRetries; i++) {
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
try {
return await fn();
} catch (err) {
if (i === retries - 1) throw err;
if (err.code === 'FETCH_TIME_LIMIT') throw err;
if (i === maxRetries - 1) throw err;
const wait = err.message.includes('429') ? 8000 * (i + 1) : 2000 * (i + 1);
await sleep(wait);
if (activeDeadline?.active) {
const capped = Math.min(wait, activeDeadline.remainingMs() - 50);
if (capped <= 0) activeDeadline.throwIfExpired();
await sleep(capped);
} else {
await sleep(wait);
}
}
}
}
@@ -1015,6 +1084,7 @@ async function resolveFromLookupTitle(lookupTitle, options) {
];
for (const fn of sources) {
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
try {
const result = await fn();
if (result?.fullUrl || result?.thumbUrl) {
@@ -1025,7 +1095,9 @@ async function resolveFromLookupTitle(lookupTitle, options) {
resolvedWikiTitle: lookupTitle,
};
}
} catch {}
} catch (err) {
if (err.code === 'FETCH_TIME_LIMIT') throw err;
}
}
return null;
}
@@ -1060,12 +1132,16 @@ async function resolvePaintingImages(wikiTitle, options = {}) {
}
for (const title of lookupCandidates) {
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
const result = await resolveFromLookupTitle(title, options);
if (result) return result;
}
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
const discovered = await searchWikipediaTitleMultilingual(artistName, searchTitle);
if (discovered && !seen.has(discovered.title)) {
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
if (discovered.lang === 'en') {
const result = await resolveFromLookupTitle(discovered.title, options);
if (result) return { ...result, resolvedWikiTitle: discovered.title };
@@ -1080,9 +1156,13 @@ async function resolvePaintingImages(wikiTitle, options = {}) {
}
}
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
const web = await searchWebForPaintingImages(artistName, searchTitle);
if (web) return web;
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
return null;
}
@@ -1121,60 +1201,73 @@ async function generateThumbnailFromFull(fullDest, thumbDest, width = THUMB_WIDT
}
async function savePaintingImages(wikiTitle, baseFilename, imageDir, options = {}) {
const resolved = await resolvePaintingImages(wikiTitle, options);
if (!resolved) return { imagePath: null, thumbnailPath: null, source: null, resolvedWikiTitle: null };
const paintingsDir = path.join(imageDir, 'paintings');
const thumbsDir = path.join(imageDir, 'paintings', 'thumbs');
if (!fs.existsSync(paintingsDir)) fs.mkdirSync(paintingsDir, { recursive: true });
if (!fs.existsSync(thumbsDir)) fs.mkdirSync(thumbsDir, { recursive: true });
const safeBase = baseFilename.replace(/[^a-zA-Z0-9_-]/g, '_');
const fullExt = pickExt(resolved.fullUrl);
const fullDest = path.join(paintingsDir, safeBase + fullExt);
const thumbDest = path.join(thumbsDir, safeBase + '_thumb.jpg');
let imagePath = null;
let thumbnailPath = null;
const deadline = deadlineFromOptions(options);
activeDeadline = deadline.active ? deadline : null;
try {
await downloadImageToFile(resolved.fullUrl, fullDest);
imagePath = path.join('paintings', safeBase + fullExt).replace(/\\/g, '/');
} catch (err) {
console.warn(` Full image download failed: ${err.message}`);
}
const resolved = await resolvePaintingImages(wikiTitle, options);
if (!resolved) return { imagePath: null, thumbnailPath: null, source: null, resolvedWikiTitle: null };
if (imagePath) {
try {
await generateThumbnailFromFull(fullDest, thumbDest);
thumbnailPath = path.join('paintings', 'thumbs', safeBase + '_thumb.jpg').replace(/\\/g, '/');
} catch (err) {
console.warn(` Thumbnail generation failed: ${err.message}`);
thumbnailPath = imagePath;
}
} else {
try {
const thumbSrc =
resolved.thumbUrl !== resolved.fullUrl ? resolved.thumbUrl : resolved.fullUrl;
const thumbExt = pickExt(resolved.thumbUrl);
const fallbackThumbDest = path.join(thumbsDir, safeBase + '_thumb' + thumbExt);
await downloadImageToFile(thumbSrc, fallbackThumbDest);
thumbnailPath = path
.join('paintings', 'thumbs', safeBase + '_thumb' + thumbExt)
.replace(/\\/g, '/');
imagePath = thumbnailPath;
} catch (err) {
console.warn(` Thumbnail download failed: ${err.message}`);
}
}
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
return {
imagePath,
thumbnailPath,
source: resolved.source,
resolvedWikiTitle: resolved.resolvedWikiTitle || null,
};
const paintingsDir = path.join(imageDir, 'paintings');
const thumbsDir = path.join(imageDir, 'paintings', 'thumbs');
if (!fs.existsSync(paintingsDir)) fs.mkdirSync(paintingsDir, { recursive: true });
if (!fs.existsSync(thumbsDir)) fs.mkdirSync(thumbsDir, { recursive: true });
const safeBase = baseFilename.replace(/[^a-zA-Z0-9_-]/g, '_');
const fullExt = pickExt(resolved.fullUrl);
const fullDest = path.join(paintingsDir, safeBase + fullExt);
const thumbDest = path.join(thumbsDir, safeBase + '_thumb.jpg');
let imagePath = null;
let thumbnailPath = null;
try {
await downloadImageToFile(resolved.fullUrl, fullDest);
imagePath = path.join('paintings', safeBase + fullExt).replace(/\\/g, '/');
} catch (err) {
if (err.code === 'FETCH_TIME_LIMIT') throw err;
console.warn(` Full image download failed: ${err.message}`);
}
if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
if (imagePath) {
try {
await generateThumbnailFromFull(fullDest, thumbDest);
thumbnailPath = path.join('paintings', 'thumbs', safeBase + '_thumb.jpg').replace(/\\/g, '/');
} catch (err) {
console.warn(` Thumbnail generation failed: ${err.message}`);
thumbnailPath = imagePath;
}
} else {
try {
const thumbSrc =
resolved.thumbUrl !== resolved.fullUrl ? resolved.thumbUrl : resolved.fullUrl;
const thumbExt = pickExt(resolved.thumbUrl);
const fallbackThumbDest = path.join(thumbsDir, safeBase + '_thumb' + thumbExt);
await downloadImageToFile(thumbSrc, fallbackThumbDest);
thumbnailPath = path
.join('paintings', 'thumbs', safeBase + '_thumb' + thumbExt)
.replace(/\\/g, '/');
imagePath = thumbnailPath;
} catch (err) {
if (err.code === 'FETCH_TIME_LIMIT') throw err;
console.warn(` Thumbnail download failed: ${err.message}`);
}
}
return {
imagePath,
thumbnailPath,
source: resolved.source,
resolvedWikiTitle: resolved.resolvedWikiTitle || wikiTitle,
};
} finally {
activeDeadline = null;
}
}
async function saveImageForItem(wikiTitle, subdir, filename, imageDir, options = {}) {
@@ -1220,5 +1313,7 @@ module.exports = {
searchWebForPaintingImages,
getGoogleArtsCultureImages,
simplifyPaintingTitle,
DEFAULT_BATCH_MAX_WAIT_MS,
FetchDeadline,
sleep,
};