From fc842aa14560074bcd32e9c625f188e51dfa77a9 Mon Sep 17 00:00:00 2001 From: Jos Vooges | STH Date: Tue, 25 Aug 2026 11:41:56 +0200 Subject: [PATCH] Process all pending movies in IMDb backfill instead of capping at 200. Co-authored-by: Cursor --- apps/admin-ui/src/app/dashboard/page.tsx | 4 +- apps/master-api/src/admin/routes.ts | 3 +- apps/master-api/src/app.ts | 6 +-- apps/master-api/src/metadata/service.ts | 66 +++++++++++++----------- apps/master-api/src/stremio/routes.ts | 2 +- 5 files changed, 45 insertions(+), 36 deletions(-) diff --git a/apps/admin-ui/src/app/dashboard/page.tsx b/apps/admin-ui/src/app/dashboard/page.tsx index 5e3aadb..fdc4254 100644 --- a/apps/admin-ui/src/app/dashboard/page.tsx +++ b/apps/admin-ui/src/app/dashboard/page.tsx @@ -70,7 +70,7 @@ export default function DashboardPage() { async function backfillImdb() { if ( !confirm( - "IMDb-IDs ophalen via TMDB voor films die die nog missen? Nodig zodat Cinemeta jouw lokale bronnen toont." + "IMDb-IDs ophalen via TMDB voor ALLE films die die nog missen? Dit kan enkele minuten duren bij een grote bibliotheek. Nodig zodat Cinemeta jouw lokale bronnen toont." ) ) { return; @@ -86,7 +86,7 @@ export default function DashboardPage() { return; } setImdbMsg( - `Gecontroleerd: ${json.checked ?? 0}, bijgewerkt: ${json.updated ?? 0}, nog open: ${json.skipped ?? 0}` + `Gecontroleerd: ${json.checked ?? 0}, bijgewerkt: ${json.updated ?? 0}, nog open: ${json.remaining ?? json.skipped ?? 0}, mislukt: ${json.failed ?? 0}` ); load(); } diff --git a/apps/master-api/src/admin/routes.ts b/apps/master-api/src/admin/routes.ts index c76536e..ef8e11e 100644 --- a/apps/master-api/src/admin/routes.ts +++ b/apps/master-api/src/admin/routes.ts @@ -445,7 +445,8 @@ export async function registerAdminRoutes(app: FastifyInstance, config: Config) }); app.post("/api/v1/admin/library/backfill-imdb", { preHandler: requireAdmin }, async () => { - const result = await metadata.backfillMovieImdbIds({ delayMs: 150, limit: 500 }); + // Process ALL pending movies (batched). Can take a few minutes for large libraries. + const result = await metadata.backfillMovieImdbIds({ delayMs: 120 }); return result; }); diff --git a/apps/master-api/src/app.ts b/apps/master-api/src/app.ts index 7c590f4..8938fff 100644 --- a/apps/master-api/src/app.ts +++ b/apps/master-api/src/app.ts @@ -126,16 +126,16 @@ async function main() { await app.listen({ port: config.PORT, host: config.HOST }); console.log(`Master API listening on ${config.HOST}:${config.PORT}`); - // Background: fill missing movie imdbIds so Cinemeta detail pages can resolve local streams. + // Background: fill ALL missing movie imdbIds (batched) so Cinemeta can resolve local streams. if (config.TMDB_API_KEY) { const metadata = new MetadataService(config.TMDB_API_KEY); setTimeout(() => { void metadata - .backfillMovieImdbIds({ delayMs: 200, limit: 200 }) + .backfillMovieImdbIds({ delayMs: 120 }) .then((r) => { if (r.checked > 0) { console.log( - `IMDb backfill: checked=${r.checked} updated=${r.updated} remaining=${r.skipped}` + `IMDb backfill: checked=${r.checked} updated=${r.updated} remaining=${r.remaining} failed=${r.failed}` ); } }) diff --git a/apps/master-api/src/metadata/service.ts b/apps/master-api/src/metadata/service.ts index 92d7f4c..ba6881f 100644 --- a/apps/master-api/src/metadata/service.ts +++ b/apps/master-api/src/metadata/service.ts @@ -666,50 +666,58 @@ export class MetadataService { } /** - * Fill missing imdbId for all movies that have a TMDB id. + * Fill missing imdbId for movies that have a TMDB id. + * Processes in batches until none remain (or `limit` is hit). * Rate-limited to avoid TMDB bans. Safe for series (movies only). */ async backfillMovieImdbIds(options?: { delayMs?: number; + /** Max movies to process this run. Omit / 0 = all pending. */ limit?: number; - }): Promise<{ checked: number; updated: number; skipped: number; failed: number }> { - const delayMs = options?.delayMs ?? 200; - const limit = options?.limit ?? 500; + batchSize?: number; + }): Promise<{ checked: number; updated: number; remaining: number; failed: number }> { + const delayMs = options?.delayMs ?? 150; + const batchSize = Math.min(Math.max(options?.batchSize ?? 100, 1), 500); + const maxTotal = options?.limit && options.limit > 0 ? options.limit : Number.POSITIVE_INFINITY; if (!this.tmdbApiKey) { - return { checked: 0, updated: 0, skipped: 0, failed: 0 }; + return { checked: 0, updated: 0, remaining: 0, failed: 0 }; } - const pending = await prisma.movie.findMany({ - where: { tmdbId: { not: null }, imdbId: null }, - select: { id: true, title: true, tmdbId: true }, - take: limit, - orderBy: { updatedAt: "asc" }, - }); - + let checked = 0; let updated = 0; let failed = 0; - for (const row of pending) { - try { - const ok = await this.ensureMovieImdbId(row.id); - if (ok) updated += 1; - else failed += 1; - } catch { - failed += 1; - } - if (delayMs > 0) { - await new Promise((r) => setTimeout(r, delayMs)); + + while (checked < maxTotal) { + const take = Math.min(batchSize, maxTotal - checked); + const pending = await prisma.movie.findMany({ + where: { tmdbId: { not: null }, imdbId: null }, + select: { id: true }, + take, + orderBy: { updatedAt: "asc" }, + }); + if (pending.length === 0) break; + + for (const row of pending) { + try { + const ok = await this.ensureMovieImdbId(row.id); + if (ok) updated += 1; + else failed += 1; + } catch { + failed += 1; + } + checked += 1; + if (delayMs > 0) { + await new Promise((r) => setTimeout(r, delayMs)); + } } + + if (pending.length < take) break; } - const stillMissing = await prisma.movie.count({ + const remaining = await prisma.movie.count({ where: { tmdbId: { not: null }, imdbId: null }, }); - return { - checked: pending.length, - updated, - skipped: stillMissing, - failed, - }; + return { checked, updated, remaining, failed }; } private async absorbOrphanMovies( diff --git a/apps/master-api/src/stremio/routes.ts b/apps/master-api/src/stremio/routes.ts index ad37535..dcd7771 100644 --- a/apps/master-api/src/stremio/routes.ts +++ b/apps/master-api/src/stremio/routes.ts @@ -603,7 +603,7 @@ async function getStreams( ); // Do not block this request — fill IMDb ids in background for next try. if (isImdb && missingImdb > 0) { - void metadata.backfillMovieImdbIds({ delayMs: 150, limit: 80 }).catch(() => undefined); + void metadata.backfillMovieImdbIds({ delayMs: 120, limit: 30 }).catch(() => undefined); } return []; }