const https = require('https');
const http = require('http');
const fs = require('fs');
const path = require('path');

const USER_AGENT = 'VirtualArtGallery/1.0 (educational art history project; local museum gallery)';
const BROWSER_USER_AGENT =
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';

const WIKI_LANGS = ['de', 'fr', 'it', 'ru'];

const TRUSTED_IMAGE_HOSTS = [
  'upload.wikimedia.org',
  'commons.wikimedia.org',
  'collections.louvre.fr',
  'artic.edu',
  'images.metmuseum.org',
  'collectionapi.metmuseum.org',
  'ids.si.edu',
  'harvardartmuseums.org',
  'rijksmuseum.nl',
  'openaccess-api.clevelandart.org',
  'api.europeana.eu',
  'hermitagemuseum.org',
  'pushkinmuseum.art',
  'googleusercontent.com',
  'artsandculture.google.com',
];

let lastRequestTime = 0;
const MIN_DELAY_MS = 2500;
const REQUEST_TIMEOUT_MS = 15000;
const DEFAULT_BATCH_MAX_WAIT_MS = 10000;
const THUMB_WIDTH = 400;
const FULL_WIDTH = 1600;

const PAINTING_WIKI_OVERRIDES = {
  'Job Cigarette Papers': 'Job (advertising)',
  'Charing Cross Bridge': 'Charing Cross Bridge (Derain)',
  'Cut with the Dada Kitchen Knife':
    'Cut with the Dada Kitchen Knife through the Last Weimar Beer-Belly Cultural Epoch in Germany',
  'Cherubs of the Sistine Madonna': "Raphael's Cherubs",
  'Madonna and Child (Madonna della Seggiola)': 'Madonna della Seggiola',
  'Madonna della seggiola': 'Madonna della Seggiola',
  'La Madonna della sedia': 'Madonna della Seggiola',
  'Madonna della sedia': 'Madonna della Seggiola',
  'Raphael::Madonna and Child': 'Small Cowper Madonna',
};

const MADONNA_SEGGIOLA_IMAGE = {
  thumbUrl:
    'https://upload.wikimedia.org/wikipedia/commons/thumb/0/09/Raphael_Madonna_della_seggiola.jpg/960px-Raphael_Madonna_della_seggiola.jpg',
  fullUrl: 'https://upload.wikimedia.org/wikipedia/commons/0/09/Raphael_Madonna_della_seggiola.jpg',
  source: 'Wikimedia Commons (Palazzo Pitti, Florence)',
};

const SMALL_COWPER_MADONNA_IMAGE = {
  thumbUrl:
    'https://upload.wikimedia.org/wikipedia/commons/thumb/b/b1/Raffaello_Madonna_Cowper.jpg/960px-Raffaello_Madonna_Cowper.jpg',
  fullUrl: 'https://upload.wikimedia.org/wikipedia/commons/b/b1/Raffaello_Madonna_Cowper.jpg',
  source: 'Wikimedia Commons (National Gallery of Art, Washington)',
};

function artistPaintingKey(artistName, title) {
  return artistName && title ? `${artistName}::${title}` : title;
}

/** Strip duplicate parentheticals and museum catalog noise from seed titles. */
function simplifyPaintingTitle(title) {
  if (!title) return '';
  let t = title.trim();
  const paren = t.match(/^(.+?)\s*\(([^)]+)\)\s*$/);
  if (paren) {
    const outer = paren[1].trim();
    const inner = paren[2].trim();
    const outerNorm = outer.toLowerCase().replace(/[^a-z0-9]/g, '');
    const innerNorm = inner.toLowerCase().replace(/[^a-z0-9]/g, '');
    if (innerNorm.includes(outerNorm.slice(0, 12)) || outerNorm.includes(innerNorm.slice(0, 12))) {
      return inner.length >= outer.length ? inner : outer;
    }
    return outer;
  }
  return t;
}

function artistSurname(name) {
  if (!name) return '';
  const parts = name.trim().split(/\s+/);
  return parts[parts.length - 1].toLowerCase();
}

function normalizeForMatch(text) {
  return (text || '')
    .normalize('NFD')
    .replace(/\p{M}/gu, '')
    .toLowerCase()
    .replace(/[^a-z0-9\s]/g, ' ')
    .replace(/\s+/g, ' ')
    .trim();
}

function titleMatchScore(candidate, searchTitle, artistName) {
  const c = normalizeForMatch(candidate);
  const tokens = normalizeForMatch(simplifyPaintingTitle(searchTitle))
    .split(/\s+/)
    .filter((w) => w.length > 3);
  if (!tokens.length) return 0;
  const hits = tokens.filter((t) => c.includes(t)).length;
  const surname = artistSurname(artistName);
  const artistHit = surname && c.includes(normalizeForMatch(surname)) ? 1 : 0;
  return hits + artistHit;
}

const DIRECT_IMAGE_OVERRIDES = {
  'The Persistence of Memory': {
    thumbUrl: 'https://upload.wikimedia.org/wikipedia/en/d/dd/The_Persistence_of_Memory.jpg',
    fullUrl: 'https://upload.wikimedia.org/wikipedia/en/d/dd/The_Persistence_of_Memory.jpg',
    source: 'Wikimedia Commons (Museum of Modern Art, New York)',
  },
  'Self-Portrait Hesitating': {
    thumbUrl:
      'https://upload.wikimedia.org/wikipedia/commons/thumb/2/2c/Angelica_Kauffman._Self-Portrait_Hesitating_Between_the_Arts_of_Music_and_Painting.jpg/960px-Angelica_Kauffman._Self-Portrait_Hesitating_Between_the_Arts_of_Music_and_Painting.jpg',
    fullUrl:
      'https://upload.wikimedia.org/wikipedia/commons/2/2c/Angelica_Kauffman._Self-Portrait_Hesitating_Between_the_Arts_of_Music_and_Painting.jpg',
    source: 'Wikimedia Commons (National Trust, Nostell Priory)',
  },
  'Cherubs of the Sistine Madonna': {
    thumbUrl:
      'https://upload.wikimedia.org/wikipedia/commons/thumb/5/54/Raffaels_Angels.jpg/960px-Raffaels_Angels.jpg',
    fullUrl: 'https://upload.wikimedia.org/wikipedia/commons/5/54/Raffaels_Angels.jpg',
    source: 'Wikimedia Commons (detail from Sistine Madonna, Gemäldegalerie Alte Meister)',
  },
  'Madonna and Child (Madonna della Seggiola)': MADONNA_SEGGIOLA_IMAGE,
  'Madonna della seggiola': MADONNA_SEGGIOLA_IMAGE,
  'La Madonna della sedia': MADONNA_SEGGIOLA_IMAGE,
  'Madonna della sedia': MADONNA_SEGGIOLA_IMAGE,
  'Raphael::Madonna and Child': SMALL_COWPER_MADONNA_IMAGE,
};

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

class FetchDeadline {
  constructor(maxMs) {
    this.deadline = maxMs ? Date.now() + maxMs : null;
  }

  get active() {
    return this.deadline !== null;
  }

  expired() {
    return this.deadline !== null && Date.now() >= this.deadline;
  }

  remainingMs() {
    return this.deadline ? Math.max(0, this.deadline - Date.now()) : Infinity;
  }

  throwIfExpired() {
    if (this.expired()) {
      const err = new Error('Painting fetch time limit exceeded');
      err.code = 'FETCH_TIME_LIMIT';
      throw err;
    }
  }
}

/** Per-operation deadline for batch fetches; cleared after savePaintingImages finishes. */
let activeDeadline = null;

function deadlineFromOptions(options = {}) {
  if (options.deadline instanceof FetchDeadline) return options.deadline;
  if (options.maxWaitMs) return new FetchDeadline(options.maxWaitMs);
  return new FetchDeadline(null);
}

function requestTimeoutMs() {
  if (!activeDeadline?.active) return REQUEST_TIMEOUT_MS;
  return Math.min(REQUEST_TIMEOUT_MS, Math.max(500, activeDeadline.remainingMs() - 50));
}

function attachRequestTimeout(req, reject) {
  const timeoutMs = requestTimeoutMs();
  req.setTimeout(timeoutMs, () => {
    req.destroy();
    reject(new Error('Request timeout'));
  });
}

async function throttle() {
  if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
  const elapsed = Date.now() - lastRequestTime;
  const minDelay = activeDeadline?.active ? 0 : MIN_DELAY_MS;
  let wait = Math.max(0, minDelay - elapsed);
  if (activeDeadline?.active) {
    wait = Math.min(wait, activeDeadline.remainingMs() - 50);
    if (wait <= 0) activeDeadline.throwIfExpired();
  }
  if (wait > 0) await sleep(wait);
  lastRequestTime = Date.now();
}

function fetchBuffer(url, redirectCount = 0, referer = null, options = {}) {
  return new Promise((resolve, reject) => {
    if (redirectCount > 8) return reject(new Error('Too many redirects'));
    const client = url.startsWith('https') ? https : http;
    const useBrowser = options.browser !== false && (options.browser === true || !!referer);
    const headers = {
      'User-Agent': useBrowser ? BROWSER_USER_AGENT : USER_AGENT,
      Accept: 'image/avif,image/webp,image/apng,image/*,*/*;q=0.8',
      'Accept-Language': 'en-US,en;q=0.9',
    };
    if (referer) headers.Referer = referer;
    if (url.includes('wikimedia.org') || url.includes('wikipedia.org')) {
      headers.Referer = 'https://commons.wikimedia.org/';
      headers['User-Agent'] = USER_AGENT;
    }
    if (url.includes('artic.edu')) {
      headers.Referer = 'https://www.artic.edu/';
    }
    if (url.includes('collections.louvre.fr')) {
      headers.Referer = 'https://collections.louvre.fr/';
    }
    if (url.includes('hermitagemuseum.org') || url.includes('pushkinmuseum.art')) {
      headers.Referer = url.split('/').slice(0, 3).join('/') + '/';
    }
    if (url.includes('googleusercontent.com') || url.includes('gstatic.com') || url.includes('artsandculture.google.com')) {
      headers.Referer = headers.Referer || 'https://artsandculture.google.com/';
    }
    const req = client.get(url, { headers }, (res) => {
        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
          const next = res.headers.location.startsWith('http')
            ? res.headers.location
            : new URL(res.headers.location, url).href;
          return resolve(fetchBuffer(next, redirectCount + 1, referer, options));
        }
        if (res.statusCode === 429) {
          res.resume();
          return reject(new Error('HTTP 429'));
        }
        if (res.statusCode !== 200) {
          res.resume();
          return reject(new Error(`HTTP ${res.statusCode}`));
        }
        const chunks = [];
        res.on('data', (c) => chunks.push(c));
        res.on('end', () => resolve(Buffer.concat(chunks)));
      }
    );
    attachRequestTimeout(req, reject);
    req.on('error', reject);
  });
}

async function fetchJson(url) {
  await throttle();
  return new Promise((resolve, reject) => {
    const client = url.startsWith('https') ? https : http;
    const req = client
      .get(url, { headers: { 'User-Agent': USER_AGENT } }, (res) => {
        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
          return fetchJson(res.headers.location).then(resolve).catch(reject);
        }
        let data = '';
        res.on('data', (c) => (data += c));
        res.on('end', () => {
          try {
            resolve(JSON.parse(data));
          } catch (e) {
            reject(e);
          }
        });
      })
      .on('error', reject);
    attachRequestTimeout(req, reject);
  });
}

async function fetchHtml(url, options = {}) {
  await throttle();
  const { method = 'GET', body = null, headers = {} } = options;
  return new Promise((resolve, reject) => {
    const parsed = new URL(url);
    const client = parsed.protocol === 'https:' ? https : http;
    const req = client.request(
      {
        hostname: parsed.hostname,
        port: parsed.port || (parsed.protocol === 'https:' ? 443 : 80),
        path: parsed.pathname + parsed.search,
        method,
        headers: {
          'User-Agent': USER_AGENT,
          Accept: 'text/html,application/xhtml+xml',
          ...headers,
        },
      },
      (res) => {
        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
          const next = res.headers.location.startsWith('http')
            ? res.headers.location
            : new URL(res.headers.location, url).href;
          return fetchHtml(next, options).then(resolve).catch(reject);
        }
        let data = '';
        res.on('data', (c) => (data += c));
        res.on('end', () => {
          if (res.statusCode !== 200) {
            return reject(new Error(`HTTP ${res.statusCode}`));
          }
          resolve(data);
        });
      }
    );
    attachRequestTimeout(req, reject);
    req.on('error', reject);
    if (body) req.write(body);
    req.end();
  });
}

function wikiHost(lang) {
  return lang === 'en' ? 'en.wikipedia.org' : `${lang}.wikipedia.org`;
}

function isTrustedImageUrl(url) {
  try {
    const host = new URL(url).hostname.replace(/^www\./, '');
    return TRUSTED_IMAGE_HOSTS.some((h) => host === h || host.endsWith('.' + h));
  } catch {
    return false;
  }
}

function pickBestImageUrl(urls) {
  for (const url of urls) {
    if (isTrustedImageUrl(url) && /\.(jpe?g|png|webp)(\?|$)/i.test(url)) {
      return url;
    }
  }
  for (const url of urls) {
    if (isTrustedImageUrl(url)) return url;
  }
  return null;
}

function extractImageUrlsFromText(text) {
  const urls = new Set();
  for (const match of text.matchAll(/https?:\/\/[^\s"'<>\\]+?\.(?:jpe?g|png|webp)(?:\?[^\s"'<>\\]*)?/gi)) {
    urls.add(match[0].replace(/\\$/g, ''));
  }
  for (const match of text.matchAll(/https?:\/\/upload\.wikimedia\.org\/[^\s"'<>\\]+/gi)) {
    urls.add(match[0].replace(/\\$/g, ''));
  }
  for (const match of text.matchAll(/https?:\/\/collections\.louvre\.fr\/media\/[^\s"'<>\\]+/gi)) {
    urls.add(match[0].replace(/\\$/g, ''));
  }
  return [...urls];
}

async function withRetry(fn, retries = 6) {
  const maxRetries = activeDeadline?.active ? Math.min(retries, 2) : retries;
  for (let i = 0; i < maxRetries; i++) {
    if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
    try {
      return await fn();
    } catch (err) {
      if (err.code === 'FETCH_TIME_LIMIT') throw err;
      if (i === maxRetries - 1) throw err;
      const wait = err.message.includes('429') ? 8000 * (i + 1) : 2000 * (i + 1);
      if (activeDeadline?.active) {
        const capped = Math.min(wait, activeDeadline.remainingMs() - 50);
        if (capped <= 0) activeDeadline.throwIfExpired();
        await sleep(capped);
      } else {
        await sleep(wait);
      }
    }
  }
}

function pickExt(url) {
  try {
    const ext = path.extname(new URL(url).pathname) || '.jpg';
    return ext.length > 5 ? '.jpg' : ext;
  } catch {
    return '.jpg';
  }
}

async function getWikidataImageFilename(wikiTitle) {
  const propsUrl = `https://en.wikipedia.org/w/api.php?action=query&titles=${encodeURIComponent(wikiTitle)}&prop=pageprops&ppprop=wikibase_item&format=json`;
  const props = await fetchJson(propsUrl);
  const page = Object.values(props.query?.pages || {})[0];
  const qid = page?.pageprops?.wikibase_item;
  if (!qid) return null;

  const entityUrl = `https://www.wikidata.org/wiki/Special:EntityData/${qid}.json`;
  const entity = await fetchJson(entityUrl);
  const claims = entity.entities?.[qid]?.claims?.P18;
  if (!claims?.length) return null;
  return claims[0].mainsnak?.datavalue?.value;
}

async function getCommonsFileUrls(filename) {
  const fileTitle = filename.startsWith('File:') ? filename : `File:${filename}`;
  const url = `https://commons.wikimedia.org/w/api.php?action=query&titles=${encodeURIComponent(fileTitle)}&prop=imageinfo&iiprop=url&iiurlwidth=${THUMB_WIDTH}&format=json`;
  const data = await fetchJson(url);
  const page = Object.values(data.query?.pages || {})[0];
  const info = page?.imageinfo?.[0];
  if (!info) return null;
  return {
    thumbUrl: info.thumburl || info.url,
    fullUrl: info.url || info.thumburl,
    source: 'Wikimedia Commons',
  };
}

async function getWikipediaImages(wikiTitle) {
  return getWikipediaImagesLang(wikiTitle, 'en');
}

async function getWikipediaImagesLang(wikiTitle, lang = 'en') {
  const host = wikiHost(lang);
  const langLabel = lang === 'en' ? 'English' : lang.toUpperCase();
  const url = `https://${host}/w/api.php?action=query&titles=${encodeURIComponent(wikiTitle)}&prop=pageimages&piprop=original|thumbnail&pithumbsize=${THUMB_WIDTH}&format=json`;
  const data = await fetchJson(url);
  const page = Object.values(data.query?.pages || {})[0];
  if (!page || page.missing) return null;
  const thumbUrl = page.thumbnail?.source;
  const fullUrl = page.original?.source || thumbUrl;
  if (!fullUrl) return null;
  return {
    thumbUrl: thumbUrl || fullUrl,
    fullUrl,
    source: `Wikipedia ${langLabel} (Wikimedia Commons)`,
    resolvedWikiTitle: wikiTitle,
    wikiLang: lang,
  };
}

async function searchCommonsImages(artistName, paintingTitle) {
  const queries = [
    [artistName, paintingTitle].filter(Boolean).join(' '),
    paintingTitle,
    [artistName, ...(paintingTitle || '').split(/\s+/).slice(0, 4)].join(' '),
    [artistName, 'painting', (paintingTitle || '').split('(')[0].trim()].filter(Boolean).join(' '),
  ];
  const seen = new Set();

  for (const query of queries) {
    if (!query || seen.has(query)) continue;
    seen.add(query);

    const url = `https://commons.wikimedia.org/w/api.php?action=query&generator=search&gssearch=${encodeURIComponent(query)}&gsnamespace=6&gslimit=8&prop=imageinfo&iiprop=url&iiurlwidth=${THUMB_WIDTH}&format=json`;
    try {
      const data = await fetchJson(url);
      const pages = data.query?.pages;
      if (!pages) continue;

      const artistToken = (artistName || '').toLowerCase().split(' ').pop() || '';
      const titleTokens = (paintingTitle || '')
        .toLowerCase()
        .replace(/[^a-z0-9\s]/g, ' ')
        .split(/\s+/)
        .filter((t) => t.length > 3)
        .slice(0, 4);

      for (const page of Object.values(pages)) {
        const info = page.imageinfo?.[0];
        if (!info?.url) continue;
        const fileTitle = (page.title || '').toLowerCase();
        if (artistToken && !fileTitle.includes(artistToken) && !fileTitle.includes('raphael') && !fileTitle.includes('raffael')) {
          continue;
        }
        const matchCount = titleTokens.filter((t) => fileTitle.includes(t)).length;
        if (titleTokens.length > 0 && matchCount === 0 && !fileTitle.includes('uffizi')) continue;

        return {
          thumbUrl: info.thumburl || info.url,
          fullUrl: info.url,
          source: 'Wikimedia Commons (search)',
        };
      }
    } catch {
      continue;
    }
  }
  return null;
}

async function getMetMuseumImages(searchTerm, artistName) {
  const query = [artistName, searchTerm].filter(Boolean).join(' ');
  const searchUrl = `https://collectionapi.metmuseum.org/public/collection/v1/search?q=${encodeURIComponent(query)}&hasImages=true`;
  try {
    await throttle();
    const search = await fetchJson(searchUrl);
    if (!search.objectIDs?.length) return null;

    for (const id of search.objectIDs.slice(0, 10)) {
      await throttle();
      const obj = await fetchJson(
        `https://collectionapi.metmuseum.org/public/collection/v1/objects/${id}`
      );
      if (!obj.isPublicDomain || !obj.primaryImage) continue;
      const titleMatch =
        !searchTerm ||
        (obj.title || '').toLowerCase().includes(searchTerm.toLowerCase().slice(0, 10));
      if (!titleMatch && search.objectIDs.length > 3) continue;
      return {
        thumbUrl: obj.primaryImageSmall || obj.primaryImage,
        fullUrl: obj.primaryImage,
        source: 'The Metropolitan Museum of Art (Open Access)',
      };
    }
  } catch {
    return null;
  }
  return null;
}

async function getArtInstituteImages(searchTerm, artistName) {
  const query = [artistName, searchTerm].filter(Boolean).join(' ');
  const url = `https://api.artic.edu/api/v1/artworks/search?q=${encodeURIComponent(query)}&fields=id,title,artist_title,image_id&limit=6`;
  try {
    const data = await fetchJson(url);
    for (const art of data.data || []) {
      if (!art.image_id) continue;
      const titleOk =
        !searchTerm ||
        (art.title || '').toLowerCase().includes(searchTerm.toLowerCase().slice(0, 8));
      const artistOk =
        !artistName ||
        (art.artist_title || '').toLowerCase().includes(artistName.toLowerCase().split(' ').pop());
      if (!titleOk && !artistOk) continue;
      const base = `https://www.artic.edu/iiif/2/${art.image_id}`;
      return {
        thumbUrl: `${base}/full/200,/0/default.jpg`,
        fullUrl: `${base}/full/1686,/0/default.jpg`,
        source: 'Art Institute of Chicago (Open Access)',
      };
    }
  } catch {
    return null;
  }
  return null;
}

async function getRijksmuseumImages(searchTerm, artistName) {
  const query = [artistName, searchTerm].filter(Boolean).join(' ');
  const url = `https://www.rijksmuseum.nl/api/en/collection?key=0&format=json&ps=8&s=relevance&q=${encodeURIComponent(query)}&imgonly=true`;
  try {
    const data = await fetchJson(url);
    for (const item of data.artObjects || []) {
      if (!item.webImage?.url) continue;
      return {
        thumbUrl: item.webImage.url,
        fullUrl: item.webImage.url,
        source: 'Rijksmuseum',
      };
    }
  } catch {
    return null;
  }
  return null;
}

async function getWikidataImagesBySearch(artistName, paintingTitle) {
  const label = (paintingTitle || '').replace(/"/g, '\\"').slice(0, 120);
  const query = `
SELECT ?img ?imgLabel WHERE {
  ?work rdfs:label ?workLabel .
  FILTER(LANG(?workLabel) = "en")
  FILTER(CONTAINS(LCASE(?workLabel), LCASE("${label.slice(0, 40)}")))
  ?work wdt:P170 ?artist .
  ?artist rdfs:label ?artistLabel .
  FILTER(CONTAINS(LCASE(?artistLabel), "${(artistName || '').split(' ').pop().toLowerCase()}"))
  ?work wdt:P18 ?img .
  SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
}
LIMIT 3`;
  try {
    const url = `https://query.wikidata.org/sparql?format=json&query=${encodeURIComponent(query)}`;
    const data = await fetchJson(url);
    const row = data.results?.bindings?.[0];
    if (!row?.img?.value) return null;
    const filename = decodeURIComponent(row.img.value.split('/').pop());
    return getCommonsFileUrls(filename);
  } catch {
    return null;
  }
}

async function searchWikipediaTitle(artistName, paintingTitle) {
  const hit = await searchWikipediaTitleMultilingual(artistName, paintingTitle);
  return hit?.title || null;
}

async function searchWikipediaTitleLang(artistName, paintingTitle, lang = 'en') {
  const simple = simplifyPaintingTitle(paintingTitle);
  const host = wikiHost(lang);
  const queries = [
    `${artistName} ${simple}`,
    `${simple} ${artistSurname(artistName)}`,
    `${artistName} ${simple.split(' ').slice(0, 4).join(' ')}`,
    simple,
  ];
  const seen = new Set();
  let best = null;
  let bestScore = 0;

  for (const query of queries) {
    if (!query || seen.has(query)) continue;
    seen.add(query);

    const apiUrl = `https://${host}/w/api.php?action=query&list=search&srsearch=${encodeURIComponent(query)}&srlimit=8&format=json`;
    const restUrl = `https://${host}/w/rest.php/v1/search/page?q=${encodeURIComponent(query)}&limit=8`;

    for (const url of [apiUrl, restUrl]) {
      try {
        const data = await fetchJson(url);
        const hits = data.query?.search || data.pages || [];
        for (const hit of hits) {
          const title = hit.title || '';
          if (/disambiguation|list of|category:|liste von|liste des|elenco|categoría:/i.test(title)) {
            continue;
          }
          const score =
            titleMatchScore(title, paintingTitle, artistName) + (hit.score || hit.key || 0) / 1000;
          if (score > bestScore) {
            bestScore = score;
            best = title;
          }
        }
      } catch {
        continue;
      }
    }
  }
  return bestScore >= 2 ? best : null;
}

async function searchWikipediaTitleMultilingual(artistName, paintingTitle) {
  const langs = ['en', ...WIKI_LANGS];
  let best = null;
  let bestScore = 0;

  for (const lang of langs) {
    const title = await searchWikipediaTitleLang(artistName, paintingTitle, lang);
    if (!title) continue;
    const score = titleMatchScore(title, paintingTitle, artistName);
    if (score > bestScore) {
      bestScore = score;
      best = { title, lang };
    }
  }
  return bestScore >= 2 ? best : null;
}

async function getMultilingualWikipediaImages(artistName, searchTitle) {
  for (const lang of WIKI_LANGS) {
    const title = await searchWikipediaTitleLang(artistName, searchTitle, lang);
    if (!title) continue;
    const images = await getWikipediaImagesLang(title, lang);
    if (images) return images;
  }
  return null;
}

async function getClevelandArtImages(searchTerm, artistName) {
  const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
  const url = `https://openaccess-api.clevelandart.org/api/artworks/?q=${encodeURIComponent(query)}&has_image=1&cc0=1&limit=12`;
  try {
    const data = await fetchJson(url);
    for (const item of data.data || []) {
      const img = item.images?.web?.url || item.images?.full?.url || item.images?.print?.url;
      if (!img) continue;
      const score = titleMatchScore(`${item.title || ''}`, searchTerm, artistName);
      if (score < 1 && (data.data?.length || 0) > 2) continue;
      return {
        thumbUrl: item.images?.web?.url || img,
        fullUrl: item.images?.full?.url || item.images?.web?.url || img,
        source: 'Cleveland Museum of Art (Open Access)',
      };
    }
  } catch {
    return null;
  }
  return null;
}

async function getSmithsonianImages(searchTerm, artistName) {
  const apiKey = process.env.SMITHSONIAN_API_KEY;
  if (!apiKey) return null;
  const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
  const url = `https://api.si.edu/openaccess/api/v1.0/search?q=${encodeURIComponent(query)}&api_key=${apiKey}&rows=12&sort=Relevance`;
  try {
    const data = await fetchJson(url);
    for (const row of data.response?.rows || []) {
      const content = row.content || {};
      const title = content.title?.content || content.label?.content || '';
      const score = titleMatchScore(title, searchTerm, artistName);
      if (score < 1 && (data.response?.rows?.length || 0) > 3) continue;

      const onlineMedia = content.online_media?.content || [];
      for (const media of onlineMedia) {
        const ids = media.ids || media.id || [];
        const idList = Array.isArray(ids) ? ids : [ids];
        for (const id of idList) {
          if (!id || typeof id !== 'string') continue;
          const imgUrl = id.startsWith('http')
            ? id
            : `https://ids.si.edu/ids/deliveryService?id=${encodeURIComponent(id)}&max=1600`;
          return {
            thumbUrl: imgUrl.replace('max=1600', 'max=400'),
            fullUrl: imgUrl,
            source: 'Smithsonian Open Access',
          };
        }
      }
    }
  } catch {
    return null;
  }
  return null;
}

async function getHarvardArtMuseumsImages(searchTerm, artistName) {
  const apiKey = process.env.HARVARD_ART_API_KEY;
  if (!apiKey) return null;
  const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
  const url = `https://api.harvardartmuseums.org/object?q=${encodeURIComponent(query)}&apikey=${apiKey}&size=10&hasimage=1&fields=id,title,primaryimageurl,people`;
  try {
    const data = await fetchJson(url);
    for (const rec of data.records || []) {
      if (!rec.primaryimageurl) continue;
      const people = (rec.people || []).map((p) => p.name).join(' ');
      const score = titleMatchScore(`${rec.title} ${people}`, searchTerm, artistName);
      if (score < 1 && (data.records?.length || 0) > 3) continue;
      return {
        thumbUrl: rec.primaryimageurl,
        fullUrl: rec.primaryimageurl,
        source: 'Harvard Art Museums',
      };
    }
  } catch {
    return null;
  }
  return null;
}

async function getLouvreImages(searchTerm, artistName) {
  const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
  try {
    const html = await fetchHtml(
      `https://collections.louvre.fr/recherche?q=${encodeURIComponent(query)}&page=1`,
      { headers: { Referer: 'https://collections.louvre.fr/' } }
    );
    const arks = [
      ...new Set([...html.matchAll(/href="(\/ark:[^"]+)"/g)].map((m) => m[1])),
    ].slice(0, 8);

    for (const ark of arks) {
      const data = await fetchJson(`https://collections.louvre.fr${ark}.json`);
      const title = data.title || data.denominationTitle || '';
      const score = titleMatchScore(`${title} ${(data.creator || []).map((c) => c.label).join(' ')}`, searchTerm, artistName);
      const images = data.image || [];
      const primary = images.find((img) => img.urlImage) || images[0];
      if (!primary?.urlImage) continue;
      if (score < 1 && arks.length > 2) continue;

      const fullUrl = primary.urlImage.startsWith('http')
        ? primary.urlImage
        : `https://collections.louvre.fr/${primary.urlImage.replace(/^\//, '')}`;
      const thumbUrl = primary.urlThumbnail
        ? primary.urlThumbnail.startsWith('http')
          ? primary.urlThumbnail
          : `https://collections.louvre.fr/${primary.urlThumbnail.replace(/^\//, '')}`
        : fullUrl;

      return {
        thumbUrl,
        fullUrl,
        source: 'Musée du Louvre (collections.louvre.fr)',
      };
    }
  } catch {
    return null;
  }
  return null;
}

async function getEuropeanaImages(searchTerm, artistName) {
  const apiKey = process.env.EUROPEANA_API_KEY;
  if (!apiKey) return null;
  const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
  const url =
    `https://api.europeana.eu/record/v2/search.json?wskey=${encodeURIComponent(apiKey)}` +
    `&query=${encodeURIComponent(query)}&qf=TYPE:IMAGE&reusability=open&rows=12&profile=rich`;
  try {
    const data = await fetchJson(url);
    for (const item of data.items || []) {
      const title = item.title?.[0] || item.dcTitle?.[0] || '';
      const score = titleMatchScore(`${title} ${item.dcCreator?.join(' ') || ''}`, searchTerm, artistName);
      if (score < 1 && (data.items?.length || 0) > 3) continue;

      const fullUrl =
        item.edmIsShownBy?.[0] ||
        item.edmPreview?.[0] ||
        item.link ||
        item.guid;
      if (!fullUrl || !isTrustedImageUrl(fullUrl)) continue;

      return {
        thumbUrl: item.edmPreview?.[0] || fullUrl,
        fullUrl,
        source: 'Europeana (European museums)',
      };
    }
  } catch {
    return null;
  }
  return null;
}

function upgradeGoogleUserContentUrl(url, width = FULL_WIDTH) {
  if (!url || !url.includes('googleusercontent.com')) return url;
  const base = url.replace(/=w[^=]*$/, '').replace(/=s\d+$/, '');
  return `${base}=w${width}-no`;
}

function extractGoogleArtsAssetLinks(html) {
  const assets = [];
  const seen = new Set();
  for (const match of html.matchAll(/\/asset\/([^/"'?]+)\/([A-Za-z0-9_-]{8,})/g)) {
    const slug = match[1];
    const id = match[2];
    const key = `${slug}/${id}`;
    if (seen.has(key)) continue;
    seen.add(key);
    assets.push({ slug, id, path: `/asset/${slug}/${id}` });
  }
  return assets;
}

async function getGoogleArtsCultureAssetImage(assetPath) {
  const html = await fetchHtml(`https://artsandculture.google.com${assetPath}`, {
    headers: { Referer: 'https://artsandculture.google.com/' },
  });
  const ogImage = html.match(/property="og:image" content="([^"]+)"/)?.[1];
  const jsonImage = html.match(/"image":"(https:\/\/lh3\.googleusercontent\.com\/[^"]+)"/)?.[1];
  const imageUrl = ogImage || jsonImage;
  if (!imageUrl) return null;

  const fullUrl = upgradeGoogleUserContentUrl(imageUrl, FULL_WIDTH);
  const thumbUrl = upgradeGoogleUserContentUrl(imageUrl, THUMB_WIDTH);
  return { thumbUrl, fullUrl };
}

async function getGoogleArtsCultureImages(searchTerm, artistName) {
  const query = [artistName, simplifyPaintingTitle(searchTerm)].filter(Boolean).join(' ');
  try {
    const html = await fetchHtml(
      `https://artsandculture.google.com/search?q=${encodeURIComponent(query)}`,
      { headers: { Referer: 'https://artsandculture.google.com/' } }
    );

    const assets = extractGoogleArtsAssetLinks(html);
    const ranked = assets
      .map((asset) => ({
        ...asset,
        score: titleMatchScore(asset.slug.replace(/-/g, ' '), searchTerm, artistName),
      }))
      .filter((a) => a.score >= 1)
      .sort((a, b) => b.score - a.score);

    const candidates = ranked.length ? ranked : assets.slice(0, 6).map((a) => ({ ...a, score: 0 }));

    for (const asset of candidates.slice(0, 5)) {
      const images = await getGoogleArtsCultureAssetImage(asset.path);
      if (!images) continue;
      if (asset.score < 1 && candidates.length > 2) continue;
      return {
        ...images,
        source: 'Google Arts & Culture',
      };
    }
  } catch {
    return null;
  }
  return null;
}

async function getWikidataImagesMultilingual(artistName, paintingTitle) {
  const label = simplifyPaintingTitle(paintingTitle).replace(/"/g, '\\"').slice(0, 80);
  const surname = (artistName || '').split(' ').pop().toLowerCase();
  const langs = ['en', ...WIKI_LANGS];
  const langFilter = langs.map((l) => `"${l}"`).join(', ');
  const query = `
SELECT ?img ?workLabel WHERE {
  ?work rdfs:label ?workLabel .
  FILTER(LANG(?workLabel) IN (${langFilter}))
  FILTER(CONTAINS(LCASE(STR(?workLabel)), LCASE("${label.slice(0, 40)}")))
  ?work wdt:P170 ?artist .
  ?artist rdfs:label ?artistLabel .
  FILTER(LANG(?artistLabel) IN (${langFilter}))
  FILTER(CONTAINS(LCASE(STR(?artistLabel)), "${surname}"))
  ?work wdt:P18 ?img .
}
LIMIT 5`;
  try {
    const url = `https://query.wikidata.org/sparql?format=json&query=${encodeURIComponent(query)}`;
    const data = await fetchJson(url);
    for (const row of data.results?.bindings || []) {
      if (!row.img?.value) continue;
      const filename = decodeURIComponent(row.img.value.split('/').pop());
      const commons = await getCommonsFileUrls(filename);
      if (commons) {
        return { ...commons, source: 'Wikimedia Commons (Wikidata multilingual)' };
      }
    }
  } catch {
    return null;
  }
  return null;
}

function extractDdgResultLinks(html) {
  const links = [];
  for (const match of html.matchAll(/class="result__a"[^>]*href="([^"]+)"/g)) {
    links.push(match[1].replace(/&amp;/g, '&'));
  }
  for (const match of html.matchAll(/uddg=([^&"]+)/g)) {
    try {
      links.push(decodeURIComponent(match[1]));
    } catch {
      continue;
    }
  }
  return links;
}

async function resolveCommonsFilePageUrl(pageUrl) {
  if (!pageUrl || !/commons\.wikimedia\.org\/wiki\/File:/i.test(pageUrl)) return null;
  const fileTitle = decodeURIComponent(pageUrl.split('/wiki/')[1].replace(/_/g, ' '));
  const commons = await getCommonsFileUrls(fileTitle);
  if (commons) return commons;
  const filePathUrl = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(fileTitle.replace(/^File:/i, ''))}`;
  return {
    thumbUrl: filePathUrl,
    fullUrl: filePathUrl,
    source: 'Web search → Wikimedia Commons',
  };
}

async function searchWebForPaintingImages(artistName, searchTitle) {
  const simple = simplifyPaintingTitle(searchTitle);
  const queries = [
    `${artistName} ${simple} site:commons.wikimedia.org`,
    `${artistName} ${simple} site:artsandculture.google.com`,
    `${artistName} ${simple} painting filetype:jpg`,
    `${artistName} ${simple} site:collections.louvre.fr`,
    `${artistName} ${simple} Gemälde`,
    `${artistName} ${simple} peinture`,
    `${artistName} ${simple} quadro`,
    `${artistName} ${simple} картина`,
  ];

  const seenQueries = new Set();
  const candidateUrls = [];

  for (const query of queries.slice(0, 4)) {
    if (!query || seenQueries.has(query)) continue;
    seenQueries.add(query);

    try {
      const body = `q=${encodeURIComponent(query)}`;
      const html = await fetchHtml('https://html.duckduckgo.com/html/', {
        method: 'POST',
        body,
        headers: {
          'User-Agent': 'Mozilla/5.0 (compatible; VirtualArtGallery/1.0)',
          'Content-Type': 'application/x-www-form-urlencoded',
          'Content-Length': String(Buffer.byteLength(body)),
        },
      });

      candidateUrls.push(...extractDdgResultLinks(html));
      candidateUrls.push(...extractImageUrlsFromText(html));
    } catch {
      continue;
    }
  }

  const seenUrls = new Set();
  for (const pageUrl of candidateUrls) {
    if (!pageUrl?.startsWith('http') || seenUrls.has(pageUrl)) continue;
    seenUrls.add(pageUrl);

    try {
      if (/commons\.wikimedia\.org\/wiki\/File:/i.test(pageUrl)) {
        const commons = await resolveCommonsFilePageUrl(pageUrl);
        if (commons) return commons;
        continue;
      }

      if (/\.(jpe?g|png|webp)(\?|$)/i.test(pageUrl) && isTrustedImageUrl(pageUrl)) {
        return {
          thumbUrl: pageUrl,
          fullUrl: pageUrl,
          source: 'Web search (image URL)',
        };
      }

      if (!/wikipedia\.org|collections\.louvre|artic\.edu|metmuseum|europeana|artsandculture\.google/i.test(pageUrl)) {
        continue;
      }

      if (/artsandculture\.google\.com\/asset\//i.test(pageUrl)) {
        const assetPath = pageUrl.replace(/^https?:\/\/artsandculture\.google\.com/i, '').split('?')[0];
        const gac = await getGoogleArtsCultureAssetImage(assetPath);
        if (gac) return { ...gac, source: 'Web search → Google Arts & Culture' };
        continue;
      }

      const html = await fetchHtml(pageUrl);
      const imageUrl = pickBestImageUrl(extractImageUrlsFromText(html));
      if (imageUrl) {
        return {
          thumbUrl: imageUrl,
          fullUrl: imageUrl,
          source: 'Web search (museum/Wikipedia page)',
        };
      }

      const wikiMatch = pageUrl.match(/\/wiki\/([^#?]+)/);
      if (wikiMatch && /wikipedia\.org/.test(pageUrl)) {
        const lang = pageUrl.match(/\/\/([a-z]{2})\.wikipedia\.org/)?.[1] || 'en';
        const title = decodeURIComponent(wikiMatch[1].replace(/_/g, ' '));
        const images = await getWikipediaImagesLang(title, lang);
        if (images) return { ...images, source: `Web search → Wikipedia ${lang.toUpperCase()}` };
      }
    } catch {
      continue;
    }
  }

  for (const lang of WIKI_LANGS) {
    const title = await searchWikipediaTitleLang(artistName, searchTitle, lang);
    if (!title) continue;
    const images = await getWikipediaImagesLang(title, lang);
    if (images) return { ...images, source: `Web search → Wikipedia ${lang.toUpperCase()}` };
  }

  return null;
}

async function resolveFromLookupTitle(lookupTitle, options) {
  const { artistName, paintingTitle } = options;
  const searchTitle = paintingTitle || lookupTitle;

  const sources = [
    async () => {
      const filename = await getWikidataImageFilename(lookupTitle);
      if (!filename) return null;
      const commons = await getCommonsFileUrls(filename);
      if (commons) return commons;
      const filePathUrl = `https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(filename)}`;
      return { thumbUrl: filePathUrl, fullUrl: filePathUrl, source: 'Wikimedia Commons (Wikidata)' };
    },
    () => getWikipediaImages(lookupTitle),
    () => searchCommonsImages(artistName, searchTitle),
    () => getMetMuseumImages(searchTitle, artistName),
    () => getArtInstituteImages(searchTitle, artistName),
    () => getClevelandArtImages(searchTitle, artistName),
    () => getRijksmuseumImages(searchTitle, artistName),
    () => getLouvreImages(searchTitle, artistName),
    () => getGoogleArtsCultureImages(searchTitle, artistName),
    () => getEuropeanaImages(searchTitle, artistName),
    () => getMultilingualWikipediaImages(artistName, searchTitle),
    () => getSmithsonianImages(searchTitle, artistName),
    () => getHarvardArtMuseumsImages(searchTitle, artistName),
    () => getWikidataImagesBySearch(artistName, searchTitle),
    () => getWikidataImagesMultilingual(artistName, searchTitle),
    () => getWikipediaImages(searchTitle),
    () => searchCommonsImages(artistName, lookupTitle),
    () => searchWebForPaintingImages(artistName, searchTitle),
  ];

  for (const fn of sources) {
    if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
    try {
      const result = await fn();
      if (result?.fullUrl || result?.thumbUrl) {
        return {
          thumbUrl: result.thumbUrl || result.fullUrl,
          fullUrl: result.fullUrl || result.thumbUrl,
          source: result.source,
          resolvedWikiTitle: lookupTitle,
        };
      }
    } catch (err) {
      if (err.code === 'FETCH_TIME_LIMIT') throw err;
    }
  }
  return null;
}

async function resolvePaintingImages(wikiTitle, options = {}) {
  const { artistName, paintingTitle, webSearchOnly } = options;
  const searchTitle = paintingTitle || wikiTitle;

  if (webSearchOnly) {
    return searchWebForPaintingImages(artistName, searchTitle);
  }

  const scopedKey = artistPaintingKey(artistName, searchTitle);

  const directOverride =
    DIRECT_IMAGE_OVERRIDES[scopedKey] || DIRECT_IMAGE_OVERRIDES[searchTitle];
  if (directOverride) {
    return { ...directOverride, resolvedWikiTitle: wikiTitle };
  }

  const overrideTitle =
    PAINTING_WIKI_OVERRIDES[scopedKey] || PAINTING_WIKI_OVERRIDES[searchTitle];
  const simplified = simplifyPaintingTitle(searchTitle);

  const lookupCandidates = [];
  const seen = new Set();
  for (const t of [overrideTitle, wikiTitle, simplified, searchTitle]) {
    if (t && !seen.has(t)) {
      seen.add(t);
      lookupCandidates.push(t);
    }
  }

  for (const title of lookupCandidates) {
    if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
    const result = await resolveFromLookupTitle(title, options);
    if (result) return result;
  }

  if (activeDeadline?.expired()) activeDeadline.throwIfExpired();

  const discovered = await searchWikipediaTitleMultilingual(artistName, searchTitle);
  if (discovered && !seen.has(discovered.title)) {
    if (activeDeadline?.expired()) activeDeadline.throwIfExpired();
    if (discovered.lang === 'en') {
      const result = await resolveFromLookupTitle(discovered.title, options);
      if (result) return { ...result, resolvedWikiTitle: discovered.title };
    } else {
      const images = await getWikipediaImagesLang(discovered.title, discovered.lang);
      if (images) {
        return {
          ...images,
          resolvedWikiTitle: discovered.title,
        };
      }
    }
  }

  if (activeDeadline?.expired()) activeDeadline.throwIfExpired();

  const web = await searchWebForPaintingImages(artistName, searchTitle);
  if (web) return web;

  if (activeDeadline?.expired()) activeDeadline.throwIfExpired();

  return null;
}

/** @deprecated use resolvePaintingImages */
async function resolveImageUrl(wikiTitle, options = {}) {
  const pair = await resolvePaintingImages(wikiTitle, options);
  if (!pair) return null;
  return { url: pair.fullUrl, source: pair.source };
}

async function downloadImageToFile(url, destPath, { force = false } = {}) {
  if (!force && fs.existsSync(destPath)) return destPath;
  await throttle();
  const buffer = await withRetry(() => fetchBuffer(url));
  fs.mkdirSync(path.dirname(destPath), { recursive: true });
  fs.writeFileSync(destPath, buffer);
  return destPath;
}

function upgradeWikimediaToFull(url) {
  const thumbMatch = url.match(
    /^(https?:\/\/upload\.wikimedia\.org\/wikipedia\/(?:commons|en)\/)thumb\/(.+)\/\d+px-[^/]+$/
  );
  if (thumbMatch) return `${thumbMatch[1]}${thumbMatch[2]}`;
  return url;
}

function prepareDownloadCandidates(url, context = {}) {
  const candidates = [url];
  if (context.thumbUrl && context.thumbUrl !== url) candidates.push(context.thumbUrl);
  const google = upgradeGoogleUserContentUrl(url, FULL_WIDTH);
  if (google !== url) candidates.push(google);
  const wiki = upgradeWikimediaToFull(url);
  if (wiki !== url) candidates.push(wiki);
  try {
    const parsed = new URL(url);
    if (parsed.search) candidates.push(`${parsed.origin}${parsed.pathname}`);
  } catch {
    // ignore
  }
  return [...new Set(candidates)];
}

function referersForDownload(url, context = {}) {
  const refs = [];
  if (context.pageUrl) refs.push(context.pageUrl);
  if (context.searchUrl) refs.push(context.searchUrl);
  if (context.source === 'google-arts-culture') refs.push('https://artsandculture.google.com/');
  if (context.source === 'google-images' || context.source === 'google-custom-search') {
    refs.push('https://www.google.com/');
  }
  if (context.source === 'duckduckgo-images') refs.push('https://duckduckgo.com/');
  if (/googleusercontent\.com|gstatic\.com/i.test(url)) refs.push('https://artsandculture.google.com/');
  if (/wikimedia\.org|wikipedia\.org/i.test(url)) refs.push('https://commons.wikimedia.org/');
  try {
    refs.push(new URL(url).origin + '/');
  } catch {
    // ignore
  }
  return [...new Set(refs)];
}

/** Download with browser headers, referer fallbacks, and URL variants (fix / debug). */
async function downloadImageForFix(url, destPath, context = {}) {
  const candidates = prepareDownloadCandidates(url, context);
  const referers = referersForDownload(url, context);
  let lastError = null;

  await throttle();

  for (const candidate of candidates) {
    for (const referer of referers) {
      try {
        const buffer = await fetchBuffer(candidate, 0, referer, { browser: true });
        fs.mkdirSync(path.dirname(destPath), { recursive: true });
        fs.writeFileSync(destPath, buffer);
        return destPath;
      } catch (err) {
        lastError = err;
        await sleep(300);
      }
    }
  }

  throw lastError || new Error('Download failed');
}

async function fetchImageBuffer(url, context = {}) {
  const candidates = prepareDownloadCandidates(url, context);
  const referers = referersForDownload(url, context);
  let lastError = null;

  await throttle();

  for (const candidate of candidates) {
    for (const referer of referers) {
      try {
        return await fetchBuffer(candidate, 0, referer, { browser: true });
      } catch (err) {
        lastError = err;
        await sleep(300);
      }
    }
  }

  throw lastError || new Error('Download failed');
}

function friendlyImageFetchError(err) {
  const msg = err?.message || '';
  if (/ECONNRESET|ETIMEDOUT|EPIPE|socket hang up/i.test(msg)) {
    return 'Could not download this image — the host closed the connection. Try Search again; Wikimedia and Google Arts results work best.';
  }
  if (/HTTP 403|HTTP 401|HTTP 429/i.test(msg)) {
    return 'Could not download this image — the host blocked the request. Try Search again or pick a different preview.';
  }
  if (/Too many redirects/i.test(msg)) {
    return 'Could not download this image — too many redirects from the image URL.';
  }
  return msg || 'Could not download image';
}

function decodeGoogleEmbeddedUrl(raw) {
  return raw
    .replace(/\\u003d/g, '=')
    .replace(/\\u0026/g, '&')
    .replace(/\\u0027/g, "'")
    .replace(/\\\//g, '/');
}

function isLikelyImageUrl(url) {
  if (!url || !/^https?:\/\//i.test(url)) return false;
  if (/^data:/i.test(url)) return false;
  if (/gstatic\.com\/images\?/i.test(url)) return false;
  return true;
}

function scoreImageUrl(url) {
  let score = 0;
  if (/upload\.wikimedia\.org/i.test(url)) score += 40;
  if (/googleusercontent\.com/i.test(url)) score += 30;
  if (/\.(jpe?g|png|webp)(\?|$)/i.test(url)) score += 20;
  if (/thumb|thumbnail|small|icon|logo|avatar/i.test(url)) score -= 25;
  if (/=\s*[sS]\d{2,3}(-c|-rw)?(\||$)/.test(url) || /[?&]w=\d{2,3}(&|$)/.test(url)) score -= 15;
  return score;
}

function rankImageCandidates(candidates, limit = 20) {
  const unique = [...new Set(candidates.filter(isLikelyImageUrl))];
  return unique
    .map((url) => ({ url, score: scoreImageUrl(url) }))
    .sort((a, b) => b.score - a.score)
    .slice(0, limit)
    .map((item) => item.url);
}

function pickBestGoogleImageUrl(candidates) {
  return rankImageCandidates(candidates, 1)[0] || null;
}

function extractGoogleImageCandidates(html) {
  const candidates = [];

  for (const match of html.matchAll(/"ou":"((?:https?:\\\/\\\/|\\\/\\\/)[^"\\]+)"/g)) {
    candidates.push(decodeGoogleEmbeddedUrl(match[1]));
  }

  for (const match of html.matchAll(/"ou":\s*"((?:https?:\/\/)[^"]+)"/g)) {
    candidates.push(match[1]);
  }

  for (const match of html.matchAll(/\["(https:\\\/\\\/[^"]+)",(?:\d+,\d+|\d+)\]/g)) {
    candidates.push(decodeGoogleEmbeddedUrl(match[1]));
  }

  for (const match of html.matchAll(/imgurl=([^&"]+)/g)) {
    try {
      candidates.push(decodeURIComponent(match[1]));
    } catch {
      // skip
    }
  }

  return candidates;
}

async function searchGoogleCustomSearchImagesMany(query, limit = 20) {
  const apiKey = process.env.GOOGLE_CSE_API_KEY;
  const cx = process.env.GOOGLE_CSE_CX;
  if (!apiKey || !cx) return [];

  const out = [];
  for (let start = 1; start <= 91 && out.length < limit; start += 10) {
    const num = Math.min(10, limit - out.length);
    const url =
      `https://www.googleapis.com/customsearch/v1?key=${encodeURIComponent(apiKey)}` +
      `&cx=${encodeURIComponent(cx)}&q=${encodeURIComponent(query)}&searchType=image` +
      `&num=${num}&start=${start}&safe=active`;
    const data = await fetchJson(url);
    for (const item of data?.items || []) {
      if (!item?.link) continue;
      out.push({
        imageUrl: item.link,
        thumbUrl: item.image?.thumbnailLink || item.link,
        source: 'google-custom-search',
        width: item.image?.width,
        height: item.image?.height,
      });
      if (out.length >= limit) break;
    }
    if (!data?.items?.length) break;
  }
  return out;
}

async function searchGoogleCustomSearchImagesFirst(query) {
  const apiKey = process.env.GOOGLE_CSE_API_KEY;
  const cx = process.env.GOOGLE_CSE_CX;
  if (!apiKey || !cx) return null;

  const url =
    `https://www.googleapis.com/customsearch/v1?key=${encodeURIComponent(apiKey)}` +
    `&cx=${encodeURIComponent(cx)}&q=${encodeURIComponent(query)}&searchType=image&num=1&safe=active`;
  const data = await fetchJson(url);
  const item = data?.items?.[0];
  if (!item?.link) return null;
  return {
    imageUrl: item.link,
    source: 'google-custom-search',
    sourceLabel: 'Google Custom Search',
  };
}

async function searchDuckDuckGoImagesMany(query, limit = 20) {
  const html = await fetchHtml(
    `https://duckduckgo.com/?q=${encodeURIComponent(query)}&iar=images&iax=images&ia=images`,
    {
      headers: {
        'User-Agent':
          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
      },
    }
  );
  const vqd =
    html.match(/vqd=['"]([^'"]+)['"]/)?.[1] ||
    html.match(/vqd=([\d-]+)/)?.[1] ||
    null;
  if (!vqd) return [];

  const out = [];
  for (let page = 1; page <= 4 && out.length < limit; page++) {
    await throttle();
    const jsUrl = `https://duckduckgo.com/i.js?o=json&q=${encodeURIComponent(query)}&l=us-en&vqd=${encodeURIComponent(vqd)}&f=,,,&p=${page}`;
    const payload = await new Promise((resolve, reject) => {
      https
        .get(
          jsUrl,
          { headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' } },
          (res) => {
            let data = '';
            res.on('data', (chunk) => {
              data += chunk;
            });
            res.on('end', () => {
              try {
                resolve(JSON.parse(data));
              } catch (err) {
                reject(err);
              }
            });
          }
        )
        .on('error', reject);
    });

    for (const item of payload?.results || []) {
      const imageUrl = item?.image || item?.thumbnail;
      if (!imageUrl) continue;
      out.push({
        imageUrl,
        thumbUrl: item?.thumbnail || item?.image,
        source: 'duckduckgo-images',
        width: item?.width,
        height: item?.height,
      });
      if (out.length >= limit) break;
    }
    if (!payload?.results?.length) break;
  }
  return out;
}

async function searchDuckDuckGoImagesFirst(query) {
  const html = await fetchHtml(
    `https://duckduckgo.com/?q=${encodeURIComponent(query)}&iar=images&iax=images&ia=images`,
    {
      headers: {
        'User-Agent':
          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
      },
    }
  );
  const vqd =
    html.match(/vqd=['"]([^'"]+)['"]/)?.[1] ||
    html.match(/vqd=([\d-]+)/)?.[1] ||
    null;
  if (!vqd) return null;

  await throttle();
  const jsUrl = `https://duckduckgo.com/i.js?o=json&q=${encodeURIComponent(query)}&l=us-en&vqd=${encodeURIComponent(vqd)}&f=,,,&p=1`;
  const payload = await new Promise((resolve, reject) => {
    https
      .get(
        jsUrl,
        { headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' } },
        (res) => {
          let data = '';
          res.on('data', (chunk) => {
            data += chunk;
          });
          res.on('end', () => {
            try {
              resolve(JSON.parse(data));
            } catch (err) {
              reject(err);
            }
          });
        }
      )
      .on('error', reject);
  });

  const first = payload?.results?.[0];
  const imageUrl = first?.image || first?.thumbnail;
  if (!imageUrl) return null;
  return {
    imageUrl,
    source: 'duckduckgo-images',
    sourceLabel: 'DuckDuckGo Images (fallback)',
  };
}

/** First Google-family image result for artist + painting title (developer image audit). */
async function searchGoogleImagesFirst(artistName, paintingTitle) {
  const simple = simplifyPaintingTitle(paintingTitle);
  const query = `${artistName} ${simple} painting`.trim();
  const searchUrl =
    `https://www.google.com/search?q=${encodeURIComponent(query)}&tbm=isch&hl=en&ijn=0`;

  const custom = await searchGoogleCustomSearchImagesFirst(query).catch(() => null);
  if (custom?.imageUrl) {
    return { query, searchUrl, ...custom };
  }

  const gac = await getGoogleArtsCultureImages(simple, artistName).catch(() => null);
  if (gac?.fullUrl) {
    return {
      query,
      imageUrl: gac.fullUrl,
      thumbUrl: gac.thumbUrl,
      searchUrl: `https://artsandculture.google.com/search?q=${encodeURIComponent(query)}`,
      source: 'google-arts-culture',
      sourceLabel: 'Google Arts & Culture',
    };
  }

  try {
    const html = await fetchHtml(searchUrl, {
      headers: {
        'User-Agent':
          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        Accept: 'text/html,application/xhtml+xml',
        'Accept-Language': 'en-US,en;q=0.9',
      },
    });
    const imageUrl = pickBestGoogleImageUrl(extractGoogleImageCandidates(html));
    if (imageUrl) {
      return {
        query,
        imageUrl,
        searchUrl,
        source: 'google-images',
        sourceLabel: 'Google Images',
      };
    }
  } catch {
    // continue to fallback
  }

  const ddg = await searchDuckDuckGoImagesFirst(query).catch(() => null);
  if (ddg?.imageUrl) {
    return { query, searchUrl, ...ddg };
  }

  return { query, imageUrl: null, searchUrl, source: 'google-images', sourceLabel: 'Google Images' };
}

/** First Google-family portrait result for an artist (developer portrait audit). */
async function searchArtistPortraitFirst(artistName) {
  const query = `${artistName} portrait`.trim();
  const searchUrl =
    `https://www.google.com/search?q=${encodeURIComponent(query)}&tbm=isch&hl=en&ijn=0`;

  const custom = await searchGoogleCustomSearchImagesFirst(query).catch(() => null);
  if (custom?.imageUrl) {
    return { query, searchUrl, ...custom };
  }

  const gac = await getGoogleArtsCultureImages(artistName, artistName).catch(() => null);
  if (gac?.fullUrl) {
    return {
      query,
      imageUrl: gac.fullUrl,
      thumbUrl: gac.thumbUrl,
      searchUrl: `https://artsandculture.google.com/search?q=${encodeURIComponent(query)}`,
      source: 'google-arts-culture',
      sourceLabel: 'Google Arts & Culture',
    };
  }

  try {
    const html = await fetchHtml(searchUrl, {
      headers: {
        'User-Agent':
          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        Accept: 'text/html,application/xhtml+xml',
        'Accept-Language': 'en-US,en;q=0.9',
      },
    });
    const imageUrl = pickBestGoogleImageUrl(extractGoogleImageCandidates(html));
    if (imageUrl) {
      return {
        query,
        imageUrl,
        searchUrl,
        source: 'google-images',
        sourceLabel: 'Google Images',
      };
    }
  } catch {
    // continue to fallback
  }

  const ddg = await searchDuckDuckGoImagesFirst(query).catch(() => null);
  if (ddg?.imageUrl) {
    return { query, searchUrl, ...ddg };
  }

  return { query, imageUrl: null, searchUrl, source: 'google-images', sourceLabel: 'Google Images' };
}

const GOOGLE_ISCH_HEADERS = {
  'User-Agent':
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
  Accept: 'text/html,application/xhtml+xml',
  'Accept-Language': 'en-US,en;q=0.9',
};

async function searchDebugImagesMany(query, limit = 20) {
  const searchUrl =
    `https://www.google.com/search?q=${encodeURIComponent(query)}&tbm=isch&hl=en&ijn=0`;
  const merged = [];
  const seen = new Set();

  const push = (item) => {
    const url = item?.imageUrl;
    if (!url || !isLikelyImageUrl(url) || seen.has(url)) return;
    seen.add(url);
    merged.push({
      imageUrl: url,
      thumbUrl: item.thumbUrl || url,
      source: item.source || 'image-search',
      width: item.width,
      height: item.height,
    });
  };

  for (const item of await searchGoogleCustomSearchImagesMany(query, limit).catch(() => [])) {
    push(item);
  }

  if (merged.length < limit) {
    try {
      const html = await fetchHtml(searchUrl, { headers: GOOGLE_ISCH_HEADERS });
      for (const url of rankImageCandidates(extractGoogleImageCandidates(html), limit - merged.length)) {
        push({ imageUrl: url, source: 'google-images' });
      }
    } catch {
      // continue
    }
  }

  if (merged.length < limit) {
    for (const item of await searchDuckDuckGoImagesMany(query, limit - merged.length).catch(() => [])) {
      push(item);
    }
  }

  return {
    query,
    searchUrl,
    results: merged.slice(0, limit),
    source: 'mixed',
    sourceLabel: 'Image search results',
  };
}

async function searchPaintingImagesMany(artistName, paintingTitle, limit = 20) {
  const simple = simplifyPaintingTitle(paintingTitle);
  const query = `${artistName} ${simple} painting`.trim();
  return searchDebugImagesMany(query, limit);
}

async function searchArtistPortraitMany(artistName, limit = 20) {
  const query = `${artistName} portrait`.trim();
  return searchDebugImagesMany(query, limit);
}

async function generateThumbnailFromFull(fullDest, thumbDest, width = THUMB_WIDTH) {
  const sharp = require('sharp');
  if (!fs.existsSync(fullDest)) return false;

  const ext = path.extname(thumbDest).toLowerCase();
  let pipeline = sharp(fullDest).rotate().resize({ width, withoutEnlargement: true });

  if (ext === '.png') {
    pipeline = pipeline.png({ quality: 85 });
  } else if (ext === '.webp') {
    pipeline = pipeline.webp({ quality: 85 });
  } else {
    pipeline = pipeline.jpeg({ quality: 85 });
  }

  await pipeline.toFile(thumbDest);
  return true;
}

async function savePaintingImages(wikiTitle, baseFilename, imageDir, options = {}) {
  const deadline = deadlineFromOptions(options);
  activeDeadline = deadline.active ? deadline : null;

  try {
    const resolved = await resolvePaintingImages(wikiTitle, options);
    if (!resolved) return { imagePath: null, thumbnailPath: null, source: null, resolvedWikiTitle: null };

    if (activeDeadline?.expired()) activeDeadline.throwIfExpired();

    const paintingsDir = path.join(imageDir, 'paintings');
    const thumbsDir = path.join(imageDir, 'paintings', 'thumbs');
    if (!fs.existsSync(paintingsDir)) fs.mkdirSync(paintingsDir, { recursive: true });
    if (!fs.existsSync(thumbsDir)) fs.mkdirSync(thumbsDir, { recursive: true });

    const safeBase = baseFilename.replace(/[^a-zA-Z0-9_-]/g, '_');
    const fullExt = pickExt(resolved.fullUrl);

    const fullDest = path.join(paintingsDir, safeBase + fullExt);
    const thumbDest = path.join(thumbsDir, safeBase + '_thumb.jpg');

    let imagePath = null;
    let thumbnailPath = null;

    try {
      await downloadImageToFile(resolved.fullUrl, fullDest);
      imagePath = path.join('paintings', safeBase + fullExt).replace(/\\/g, '/');
    } catch (err) {
      if (err.code === 'FETCH_TIME_LIMIT') throw err;
      console.warn(`  Full image download failed: ${err.message}`);
    }

    if (activeDeadline?.expired()) activeDeadline.throwIfExpired();

    if (imagePath) {
      try {
        await generateThumbnailFromFull(fullDest, thumbDest);
        thumbnailPath = path.join('paintings', 'thumbs', safeBase + '_thumb.jpg').replace(/\\/g, '/');
      } catch (err) {
        console.warn(`  Thumbnail generation failed: ${err.message}`);
        thumbnailPath = imagePath;
      }
    } else {
      try {
        const thumbSrc =
          resolved.thumbUrl !== resolved.fullUrl ? resolved.thumbUrl : resolved.fullUrl;
        const thumbExt = pickExt(resolved.thumbUrl);
        const fallbackThumbDest = path.join(thumbsDir, safeBase + '_thumb' + thumbExt);
        await downloadImageToFile(thumbSrc, fallbackThumbDest);
        thumbnailPath = path
          .join('paintings', 'thumbs', safeBase + '_thumb' + thumbExt)
          .replace(/\\/g, '/');
        imagePath = thumbnailPath;
      } catch (err) {
        if (err.code === 'FETCH_TIME_LIMIT') throw err;
        console.warn(`  Thumbnail download failed: ${err.message}`);
      }
    }

    return {
      imagePath,
      thumbnailPath,
      source: resolved.source,
      resolvedWikiTitle: resolved.resolvedWikiTitle || wikiTitle,
    };
  } finally {
    activeDeadline = null;
  }
}

async function saveImageForItem(wikiTitle, subdir, filename, imageDir, options = {}) {
  if (options.type === 'painting') {
    const result = await savePaintingImages(wikiTitle, filename, imageDir, options);
    return {
      path: result.imagePath,
      thumbnailPath: result.thumbnailPath,
      source: result.source,
    };
  }

  const resolved = await resolveImageUrl(wikiTitle, options);
  if (!resolved) return { path: null, source: null };

  const dir = path.join(imageDir, subdir);
  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });

  const ext = pickExt(resolved.url);
  const safeName = filename.replace(/[^a-zA-Z0-9_-]/g, '_') + ext;
  const destPath = path.join(dir, safeName);

  try {
    await downloadImageToFile(resolved.url, destPath);
    return {
      path: path.join(subdir, safeName).replace(/\\/g, '/'),
      source: resolved.source,
    };
  } catch (err) {
    console.warn(`  Download failed (${resolved.source}): ${err.message}`);
    return { path: null, source: resolved.source };
  }
}

const ARTIST_PORTRAIT_WIKI_OVERRIDES = {
  Zeuxis: 'Zeuxis (painter)',
  'Ivan Klyun': 'Ivan Kliun',
  'Jean-Antoine Watteau': 'Antoine Watteau',
};

function artistPortraitWikiCandidates(artistName, wikiTitle) {
  const seen = new Set();
  const out = [];
  for (const title of [
    ARTIST_PORTRAIT_WIKI_OVERRIDES[artistName],
    ARTIST_PORTRAIT_WIKI_OVERRIDES[wikiTitle],
    wikiTitle,
    artistName,
    `${artistName} (painter)`,
    `${wikiTitle} (painter)`,
  ]) {
    if (title && !seen.has(title)) {
      seen.add(title);
      out.push(title);
    }
  }
  return out;
}

async function resolveArtistPortrait(artistName, wikiTitle) {
  for (const title of artistPortraitWikiCandidates(artistName, wikiTitle)) {
    const images = await getWikipediaImages(title);
    if (images?.fullUrl) {
      return { ...images, wikipedia_title: title };
    }
  }

  const search = await searchArtistPortraitFirst(artistName);
  if (search?.imageUrl) {
    return {
      fullUrl: search.imageUrl,
      source: search.sourceLabel || search.source || 'web search',
      wikipedia_title: wikiTitle,
    };
  }
  return null;
}

function findLocalPortraitPath(artistName, imageDir) {
  const safeBase = artistName.replace(/[^a-zA-Z0-9_-]/g, '_');
  for (const ext of ['.jpg', '.jpeg', '.png', '.webp', '.JPG']) {
    const rel = `portraits/${safeBase}${ext}`;
    if (fs.existsSync(path.join(imageDir, rel))) return rel.replace(/\\/g, '/');
  }
  return null;
}

async function saveArtistPortrait(artistName, wikiTitle, imageDir) {
  const local = findLocalPortraitPath(artistName, imageDir);
  if (local) return { path: local, source: 'local disk' };

  const resolved = await resolveArtistPortrait(artistName, wikiTitle);
  if (!resolved?.fullUrl) return { path: null, source: null };

  const portraitsDir = path.join(imageDir, 'portraits');
  if (!fs.existsSync(portraitsDir)) fs.mkdirSync(portraitsDir, { recursive: true });

  const safeBase = artistName.replace(/[^a-zA-Z0-9_-]/g, '_');
  const ext = pickExt(resolved.fullUrl);
  const destPath = path.join(portraitsDir, safeBase + ext);

  try {
    await downloadImageToFile(resolved.fullUrl, destPath);
    return {
      path: path.join('portraits', safeBase + ext).replace(/\\/g, '/'),
      source: resolved.source,
    };
  } catch (err) {
    console.warn(`  Portrait download failed: ${err.message}`);
    return { path: null, source: resolved.source };
  }
}

module.exports = {
  resolveImageUrl,
  resolvePaintingImages,
  saveImageForItem,
  savePaintingImages,
  saveArtistPortrait,
  findLocalPortraitPath,
  resolveArtistPortrait,
  downloadImageToFile,
  downloadImageForFix,
  generateThumbnailFromFull,
  searchWikipediaTitle,
  searchWebForPaintingImages,
  searchGoogleImagesFirst,
  searchArtistPortraitFirst,
  searchPaintingImagesMany,
  searchArtistPortraitMany,
  fetchImageBuffer,
  friendlyImageFetchError,
  pickExt,
  getGoogleArtsCultureImages,
  simplifyPaintingTitle,
  fetchHtml,
  DEFAULT_BATCH_MAX_WAIT_MS,
  FetchDeadline,
  sleep,
};