From 5749e41148baa91adfc703f459f04a24db71ba03 Mon Sep 17 00:00:00 2001 From: Jos Vooges | STH Date: Wed, 2 Sep 2026 03:25:58 +0200 Subject: [PATCH] Fix IMDb backfill timing out by processing in batches with auto-retry in admin UI. Co-authored-by: Cursor --- apps/admin-ui/src/app/dashboard/page.tsx | 67 +++++++++++++++++++----- apps/master-api/src/admin/routes.ts | 10 ++-- apps/master-api/src/metadata/service.ts | 14 ++++- 3 files changed, 73 insertions(+), 18 deletions(-) diff --git a/apps/admin-ui/src/app/dashboard/page.tsx b/apps/admin-ui/src/app/dashboard/page.tsx index d15e547..08ae0e5 100644 --- a/apps/admin-ui/src/app/dashboard/page.tsx +++ b/apps/admin-ui/src/app/dashboard/page.tsx @@ -114,27 +114,68 @@ export default function DashboardPage() { async function backfillImdb() { if ( !confirm( - "IMDb-IDs ophalen via TMDB voor ALLE films én series die die nog missen? Dit kan enkele minuten duren. Series-afleveringenlijst blijft op privé-IDs; IMDb is alleen voor Cinemeta-bronnen." + "IMDb-IDs ophalen via TMDB voor films en series die die nog missen? Dit gebeurt in batches (duurt even bij grote bibliotheken)." ) ) { return; } setImdbBusy(true); - setImdbMsg("Bezig…"); + setImdbMsg("Batch 1 bezig…"); + let batch = 0; + let totalMoviesUpdated = 0; + let totalSeriesUpdated = 0; + let prevRemaining = Number.POSITIVE_INFINITY; try { - const res = await fetch("/api/v1/admin/library/backfill-imdb", { - method: "POST", - credentials: "include", - }); - const json = await res.json().catch(() => ({})); - if (!res.ok) { - setImdbMsg(json.error?.message ?? "Mislukt"); - return; + while (batch < 300) { + batch += 1; + setImdbMsg(`Batch ${batch} bezig…`); + const res = await fetch("/api/v1/admin/library/backfill-imdb", { + method: "POST", + credentials: "include", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ limit: 100 }), + }); + const json = await res.json().catch(() => ({})); + if (!res.ok) { + setImdbMsg(json.error?.message ?? "Mislukt"); + return; + } + const movies = json.movies ?? {}; + const series = json.series ?? {}; + const checked = (movies.checked ?? 0) + (series.checked ?? 0); + const updated = (movies.updated ?? 0) + (series.updated ?? 0); + totalMoviesUpdated += movies.updated ?? 0; + totalSeriesUpdated += series.updated ?? 0; + const remaining = (movies.remaining ?? 0) + (series.remaining ?? 0); + + if (remaining === 0) { + setImdbMsg( + `Klaar: films +${totalMoviesUpdated}, series +${totalSeriesUpdated} · alles ingevuld` + ); + load(); + return; + } + if (checked === 0) { + setImdbMsg( + `Klaar: films +${totalMoviesUpdated}, series +${totalSeriesUpdated} · nog ${remaining} open (geen IMDb op TMDB)` + ); + load(); + return; + } + if (remaining >= prevRemaining && updated === 0) { + setImdbMsg( + `Gestopt na batch ${batch}: films +${totalMoviesUpdated}, series +${totalSeriesUpdated} · nog ${remaining} zonder IMDb (niet beschikbaar op TMDB)` + ); + load(); + return; + } + prevRemaining = remaining; + setImdbMsg( + `Batch ${batch}: +${movies.updated ?? 0} films, +${series.updated ?? 0} series · nog ${remaining} open…` + ); } - const movies = json.movies ?? json; - const series = json.series ?? {}; setImdbMsg( - `Films: gecontroleerd ${movies.checked ?? 0}, bijgewerkt ${movies.updated ?? 0}, open ${movies.remaining ?? 0} · Series: gecontroleerd ${series.checked ?? 0}, bijgewerkt ${series.updated ?? 0}, open ${series.remaining ?? 0}` + `Batchlimiet bereikt: films +${totalMoviesUpdated}, series +${totalSeriesUpdated} · druk opnieuw op de knop om verder te gaan` ); load(); } catch (err) { diff --git a/apps/master-api/src/admin/routes.ts b/apps/master-api/src/admin/routes.ts index ccec73c..a018f0f 100644 --- a/apps/master-api/src/admin/routes.ts +++ b/apps/master-api/src/admin/routes.ts @@ -514,9 +514,13 @@ export async function registerAdminRoutes(app: FastifyInstance, config: Config) return result; }); - app.post("/api/v1/admin/library/backfill-imdb", { preHandler: requireAdmin }, async () => { - // Process ALL pending movies + series (batched). Meta for series stays mcseries-*. - const result = await metadata.backfillAllImdbIds({ delayMs: 120 }); + app.post("/api/v1/admin/library/backfill-imdb", { preHandler: requireAdmin }, async (request) => { + const body = (request.body ?? {}) as { limit?: number }; + const limit = + typeof body.limit === "number" && body.limit > 0 + ? Math.min(Math.floor(body.limit), 500) + : 100; + const result = await metadata.backfillAllImdbIds({ delayMs: 80, limit }); return result; }); diff --git a/apps/master-api/src/metadata/service.ts b/apps/master-api/src/metadata/service.ts index 7d1ce90..b2f72aa 100644 --- a/apps/master-api/src/metadata/service.ts +++ b/apps/master-api/src/metadata/service.ts @@ -1390,7 +1390,12 @@ export class MetadataService { const movie = await prisma.movie.findUnique({ where: { id: movieId } }); if (!movie?.tmdbId || movie.imdbId) return false; const imdbId = await this.fetchTmdbMovieImdbId(movie.tmdbId); - if (!imdbId) return false; + if (!imdbId) { + await prisma.movie + .update({ where: { id: movieId }, data: { updatedAt: new Date() } }) + .catch(() => undefined); + return false; + } try { await prisma.movie.update({ where: { id: movieId }, data: { imdbId } }); return true; @@ -1469,7 +1474,12 @@ export class MetadataService { const series = await prisma.series.findUnique({ where: { id: seriesId } }); if (!series?.tmdbId || series.imdbId) return false; const imdbId = await this.fetchTmdbSeriesImdbId(series.tmdbId); - if (!imdbId) return false; + if (!imdbId) { + await prisma.series + .update({ where: { id: seriesId }, data: { updatedAt: new Date() } }) + .catch(() => undefined); + return false; + } try { await prisma.series.update({ where: { id: seriesId }, data: { imdbId } }); return true;