Add checkup review flags, duplicate detection, and corrected painting images.
Introduce checkup_checked/checkup_fixed on paintings with Checkup page filters and API. Fixed paintings auto-mark as reviewed. Add find-duplicates tooling and document debug/checkup workflow. Include Botticelli and Michelangelo image fixes from checkup. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Danila Khodjaef
co-authored by
Cursor
Cursor
parent
4683089495
commit
4c6acd5a3a
@@ -0,0 +1,36 @@
|
||||
require('dotenv').config();
|
||||
const pool = require('../server/db');
|
||||
|
||||
async function main() {
|
||||
const { rows } = await pool.query(`
|
||||
WITH dups AS (
|
||||
SELECT artist_id, title, year, array_agg(id ORDER BY id) AS ids
|
||||
FROM paintings
|
||||
GROUP BY artist_id, title, year
|
||||
HAVING COUNT(*) > 1
|
||||
),
|
||||
expanded AS (
|
||||
SELECT artist_id, title, year, ids[1] AS keep_id, unnest(ids[2:]) AS drop_id
|
||||
FROM dups
|
||||
)
|
||||
SELECT e.drop_id,
|
||||
(SELECT COUNT(*) FROM painting_influences pi
|
||||
WHERE pi.painting_id = e.drop_id OR pi.influenced_by_painting_id = e.drop_id) AS legacy_links,
|
||||
(SELECT COUNT(*) FROM painting_influence_sources pis
|
||||
WHERE pis.painting_id = e.drop_id OR pis.source_painting_id = e.drop_id) AS source_links
|
||||
FROM expanded e
|
||||
ORDER BY e.drop_id
|
||||
`);
|
||||
|
||||
const withLinks = rows.filter((r) => Number(r.legacy_links) + Number(r.source_links) > 0);
|
||||
console.log(`Exact-duplicate drop candidates: ${rows.length}`);
|
||||
console.log(`With influence links (need merge before delete): ${withLinks.length}`);
|
||||
if (withLinks.length) console.table(withLinks);
|
||||
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
require('dotenv').config();
|
||||
const pool = require('../server/db');
|
||||
|
||||
async function main() {
|
||||
const sameWiki = await pool.query(`
|
||||
SELECT a.name AS artist, p.wikipedia_title, COUNT(*)::int AS cnt,
|
||||
array_agg(p.id ORDER BY p.id) AS ids,
|
||||
array_agg(p.title ORDER BY p.id) AS titles,
|
||||
array_agg(COALESCE(p.year::text, '?') ORDER BY p.id) AS years
|
||||
FROM paintings p
|
||||
JOIN artists a ON a.id = p.artist_id
|
||||
WHERE p.wikipedia_title IS NOT NULL AND p.wikipedia_title <> ''
|
||||
GROUP BY a.id, a.name, p.wikipedia_title
|
||||
HAVING COUNT(*) > 1
|
||||
ORDER BY cnt DESC, a.name
|
||||
LIMIT 30
|
||||
`);
|
||||
|
||||
const sameImage = await pool.query(`
|
||||
SELECT a.name AS artist, p.image_path, COUNT(*)::int AS cnt,
|
||||
array_agg(p.id ORDER BY p.id) AS ids,
|
||||
array_agg(p.title ORDER BY p.id) AS titles
|
||||
FROM paintings p
|
||||
JOIN artists a ON a.id = p.artist_id
|
||||
WHERE p.image_path IS NOT NULL AND p.image_path <> ''
|
||||
GROUP BY a.id, a.name, p.image_path
|
||||
HAVING COUNT(*) > 1
|
||||
ORDER BY cnt DESC, a.name
|
||||
LIMIT 20
|
||||
`);
|
||||
|
||||
console.log('=== Same Wikipedia title (different painting rows) ===');
|
||||
console.log(`Groups: ${sameWiki.rows.length} shown (top 30)`);
|
||||
for (const row of sameWiki.rows) {
|
||||
console.log(`\n[${row.cnt}x] ${row.artist} — wiki: "${row.wikipedia_title}"`);
|
||||
row.titles.forEach((t, i) => console.log(` id ${row.ids[i]}: "${t}" (${row.years[i]})`));
|
||||
}
|
||||
|
||||
console.log('\n=== Same image file (different painting rows) ===');
|
||||
for (const row of sameImage.rows) {
|
||||
console.log(`\n[${row.cnt}x] ${row.artist} — ${row.image_path}`);
|
||||
row.titles.forEach((t, i) => console.log(` id ${row.ids[i]}: "${t}"`));
|
||||
}
|
||||
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
require('dotenv').config();
|
||||
const pool = require('../server/db');
|
||||
|
||||
async function main() {
|
||||
const exact = await pool.query(`
|
||||
SELECT a.name AS artist, p.title, p.year, COUNT(*)::int AS cnt,
|
||||
array_agg(p.id ORDER BY p.id) AS ids,
|
||||
array_agg(COALESCE(p.image_path, '') ORDER BY p.id) AS image_paths,
|
||||
array_agg(COALESCE(p.wikipedia_title, '') ORDER BY p.id) AS wiki_titles
|
||||
FROM paintings p
|
||||
JOIN artists a ON a.id = p.artist_id
|
||||
GROUP BY a.id, a.name, p.title, p.year
|
||||
HAVING COUNT(*) > 1
|
||||
ORDER BY cnt DESC, a.name, p.title
|
||||
`);
|
||||
|
||||
const normalized = await pool.query(`
|
||||
SELECT a.name AS artist,
|
||||
lower(regexp_replace(regexp_replace(p.title, '[^a-zA-Z0-9 ]', '', 'g'), '\\s+', ' ', 'g')) AS norm_title,
|
||||
COUNT(*)::int AS cnt,
|
||||
array_agg(p.id ORDER BY p.id) AS ids,
|
||||
array_agg(p.title ORDER BY p.id) AS titles,
|
||||
array_agg(COALESCE(p.year::text, '?') ORDER BY p.id) AS years
|
||||
FROM paintings p
|
||||
JOIN artists a ON a.id = p.artist_id
|
||||
GROUP BY a.id, a.name, norm_title
|
||||
HAVING COUNT(*) > 1
|
||||
ORDER BY cnt DESC, a.name, norm_title
|
||||
`);
|
||||
|
||||
const rublev = await pool.query(`
|
||||
SELECT p.id, p.title, p.year, p.image_path, p.wikipedia_title,
|
||||
p.checkup_checked, p.checkup_fixed
|
||||
FROM paintings p
|
||||
JOIN artists a ON a.id = p.artist_id
|
||||
WHERE a.name ILIKE '%rublev%'
|
||||
OR p.title ILIKE '%trinity%'
|
||||
ORDER BY a.name, p.title, p.id
|
||||
`);
|
||||
|
||||
const stats = await pool.query(`
|
||||
SELECT COUNT(*)::int AS dup_groups,
|
||||
COALESCE(SUM(cnt - 1), 0)::int AS extra_rows
|
||||
FROM (
|
||||
SELECT COUNT(*)::int AS cnt
|
||||
FROM paintings
|
||||
GROUP BY artist_id, title, year
|
||||
HAVING COUNT(*) > 1
|
||||
) d
|
||||
`);
|
||||
|
||||
const byArtist = await pool.query(`
|
||||
SELECT a.name, COUNT(*)::int AS dup_groups, SUM(cnt - 1)::int AS extra_rows
|
||||
FROM (
|
||||
SELECT artist_id, title, year, COUNT(*)::int AS cnt
|
||||
FROM paintings GROUP BY artist_id, title, year HAVING COUNT(*) > 1
|
||||
) d
|
||||
JOIN artists a ON a.id = d.artist_id
|
||||
GROUP BY a.id, a.name ORDER BY extra_rows DESC
|
||||
`);
|
||||
|
||||
const rublevCluster = await pool.query(`
|
||||
SELECT p.id, p.title, p.year, p.wikipedia_title, p.image_path,
|
||||
(SELECT COUNT(*) FROM painting_influences pi
|
||||
WHERE pi.painting_id = p.id OR pi.influenced_by_painting_id = p.id) AS influence_edges,
|
||||
(SELECT COUNT(*) FROM painting_influence_sources pis
|
||||
WHERE pis.painting_id = p.id OR pis.source_painting_id = p.id) AS source_edges
|
||||
FROM paintings p
|
||||
JOIN artists a ON a.id = p.artist_id
|
||||
WHERE a.name ILIKE '%rublev%'
|
||||
AND (p.title ILIKE '%trinity%' OR p.title ILIKE '%hospitality%')
|
||||
ORDER BY p.id
|
||||
`);
|
||||
|
||||
const total = await pool.query('SELECT COUNT(*)::int AS n FROM paintings');
|
||||
|
||||
console.log('=== Summary ===');
|
||||
console.log(`Total paintings: ${total.rows[0].n}`);
|
||||
console.log(`Exact duplicate groups: ${stats.rows[0].dup_groups} (${stats.rows[0].extra_rows} rows could be removed)`);
|
||||
console.log('\nArtists with exact duplicates:');
|
||||
for (const row of byArtist.rows) {
|
||||
console.log(` ${row.extra_rows} extra — ${row.name} (${row.dup_groups} duplicated works)`);
|
||||
}
|
||||
|
||||
console.log('\n=== Exact duplicates (same artist + title + year) ===');
|
||||
console.log(`Groups: ${exact.rows.length}`);
|
||||
for (const row of exact.rows) {
|
||||
console.log(`\n[${row.cnt}x] ${row.artist} — "${row.title}" (${row.year ?? '?'})`);
|
||||
console.log(` ids: ${row.ids.join(', ')}`);
|
||||
console.log(` images: ${row.image_paths.join(' | ')}`);
|
||||
if (row.wiki_titles.some(Boolean)) console.log(` wiki: ${row.wiki_titles.join(' | ')}`);
|
||||
}
|
||||
|
||||
console.log('\n=== Normalized title duplicates (same artist, similar title) ===');
|
||||
console.log(`Groups: ${normalized.rows.length}`);
|
||||
for (const row of normalized.rows) {
|
||||
const alreadyExact = exact.rows.some(
|
||||
(e) => e.artist === row.artist && e.cnt === row.cnt && e.ids.join() === row.ids.join()
|
||||
);
|
||||
if (alreadyExact) continue;
|
||||
console.log(`\n[${row.cnt}x] ${row.artist}`);
|
||||
row.titles.forEach((t, i) => console.log(` id ${row.ids[i]}: "${t}" (${row.years[i]})`));
|
||||
}
|
||||
|
||||
console.log('\n=== Rublev Trinity / Hospitality cluster (likely same icon) ===');
|
||||
for (const row of rublevCluster.rows) {
|
||||
console.log(` id ${row.id}: "${row.title}" (${row.year ?? '?'})`);
|
||||
console.log(` wiki: ${row.wikipedia_title || '—'}`);
|
||||
console.log(` image: ${row.image_path || '—'}`);
|
||||
console.log(` influence links: ${row.influence_edges} legacy, ${row.source_edges} sources`);
|
||||
}
|
||||
|
||||
console.log('\n=== All Rublev / Trinity title matches ===');
|
||||
for (const row of rublev.rows) {
|
||||
console.log(` id ${row.id}: "${row.title}" (${row.year ?? '?'}) — ${row.image_path || 'no image'}`);
|
||||
}
|
||||
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
require('dotenv').config();
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const pool = require('../server/db');
|
||||
|
||||
async function main() {
|
||||
const sqlPath = path.join(__dirname, '../db/migrate-checkup-flags.sql');
|
||||
const sql = fs.readFileSync(sqlPath, 'utf8');
|
||||
await pool.query(sql);
|
||||
await pool.query(
|
||||
`UPDATE paintings SET checkup_checked = true WHERE checkup_fixed = true AND NOT checkup_checked`
|
||||
);
|
||||
const { rows } = await pool.query(`
|
||||
SELECT
|
||||
COUNT(*) FILTER (WHERE checkup_checked)::int AS checked,
|
||||
COUNT(*) FILTER (WHERE checkup_fixed)::int AS fixed
|
||||
FROM paintings
|
||||
`);
|
||||
console.log(`checkup flags ready (${rows[0].checked} checked, ${rows[0].fixed} fixed)`);
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user