From 34130ab412b54885089e9e709b63d8a35928a6e0 Mon Sep 17 00:00:00 2001 From: Jos Vooges | STH Date: Thu, 17 Sep 2026 03:07:29 +0200 Subject: [PATCH] Cut full-sync Postgres load: skip no-op updates, batch lookup, one post-startup sync. Also cap startup IMDb backfill and bump media-node to 1.3.12 so upgrades no longer match the library twice. --- apps/master-api/src/app.ts | 6 +- apps/master-api/src/media/sync.ts | 30 ++++++- apps/master-api/src/metadata/service.ts | 88 +++++++++++++++------ node/media-node/cmd/media-node/main.go | 2 +- node/media-node/internal/scanner/scanner.go | 11 ++- 5 files changed, 105 insertions(+), 32 deletions(-) diff --git a/apps/master-api/src/app.ts b/apps/master-api/src/app.ts index e31450f..e1f8e21 100644 --- a/apps/master-api/src/app.ts +++ b/apps/master-api/src/app.ts @@ -610,13 +610,13 @@ async function main() { await app.listen({ port: config.PORT, host: config.HOST }); console.log(`Master API listening on ${config.HOST}:${config.PORT}`); - // Background: fill ALL missing movie + series imdbIds so Cinemeta can resolve local streams. - // Series catalog/meta stay on mcseries-* (tt only used for streams). + // Background: klein stukje IMDb backfill na start (niet de hele library — dat hield CPU/TMDB bezig). + // Rest via Admin → metadata tools of Stremio lazy backfill. if (config.TMDB_API_KEY) { const metadata = createMetadataService(config); setTimeout(() => { void metadata - .backfillAllImdbIds({ delayMs: 120 }) + .backfillAllImdbIds({ delayMs: 200, limit: 40 }) .then((r) => { if (r.movies.checked > 0 || r.series.checked > 0) { console.log( diff --git a/apps/master-api/src/media/sync.ts b/apps/master-api/src/media/sync.ts index 59befa9..9dbe3cd 100644 --- a/apps/master-api/src/media/sync.ts +++ b/apps/master-api/src/media/sync.ts @@ -53,10 +53,38 @@ export class LibrarySyncService { fileIdsSoFar: Set; } ): Promise { + // Eén findMany per batch i.p.v. N findUnique — scheelt enorm bij stabiele library. + const existingRows = + files.length > 0 + ? await prisma.mediaFile.findMany({ + where: { + nodeId, + localFileId: { in: files.map((f) => f.localFileId) }, + }, + select: { + id: true, + localFileId: true, + movieId: true, + episodeId: true, + shelfId: true, + sizeBytes: true, + path: true, + available: true, + container: true, + resolution: true, + videoCodec: true, + audioCodec: true, + releaseName: true, + }, + }) + : []; + const existingByLocal = new Map(existingRows.map((r) => [r.localFileId, r])); + for (const file of files) { options.fileIdsSoFar.add(file.localFileId); try { - await this.metadata.matchMediaFile(nodeId, file); + const preloaded = existingByLocal.get(file.localFileId) ?? null; + await this.metadata.matchMediaFile(nodeId, file, { existing: preloaded }); } catch (err) { console.error( `Library full sync: failed to match ${file.releaseName} on node ${nodeId}:`, diff --git a/apps/master-api/src/metadata/service.ts b/apps/master-api/src/metadata/service.ts index 1a0257e..1822631 100644 --- a/apps/master-api/src/metadata/service.ts +++ b/apps/master-api/src/metadata/service.ts @@ -443,21 +443,67 @@ export class MetadataService { async matchMediaFile( nodeId: string, - file: MediaFileInfo + file: MediaFileInfo, + options?: { + /** Vooraf geladen rij (full-sync batch); `null` = bestaat niet, weglaten = zelf ophalen. */ + existing?: { + id: string; + movieId: string | null; + episodeId: string | null; + shelfId: string | null; + sizeBytes: bigint; + path: string | null; + available: boolean; + container: string | null; + resolution: string | null; + videoCodec: string | null; + audioCodec: string | null; + releaseName: string | null; + } | null; + } ): Promise { - // Master herparst het pad (Plex S10E01 Title.mkv e.d.) — betrouwbaarder dan alleen node-parse. - file = this.enrichParsedFromPath(file); - - // Fast path: already matched → only refresh shelf / availability (no TMDB). - const existing = await prisma.mediaFile.findUnique({ - where: { - nodeId_localFileId: { nodeId, localFileId: file.localFileId }, - }, - select: { id: true, movieId: true, episodeId: true, shelfId: true, sizeBytes: true, path: true }, - }); + // Fast path eerst — geen pad-herparse nodig als het bestand al gekoppeld is. + const existing = + options && "existing" in options + ? options.existing ?? null + : await prisma.mediaFile.findUnique({ + where: { + nodeId_localFileId: { nodeId, localFileId: file.localFileId }, + }, + select: { + id: true, + movieId: true, + episodeId: true, + shelfId: true, + sizeBytes: true, + path: true, + available: true, + container: true, + resolution: true, + videoCodec: true, + audioCodec: true, + releaseName: true, + }, + }); if (existing && (existing.movieId || existing.episodeId)) { const nextShelf = file.shelfId || null; const sizeBytes = BigInt(file.sizeBytes); + const unchanged = + existing.available === true && + existing.sizeBytes === sizeBytes && + (existing.shelfId ?? null) === nextShelf && + (file.path == null || file.path === "" || existing.path === file.path) && + (file.container == null || existing.container === file.container) && + (file.resolution == null || existing.resolution === file.resolution) && + (file.videoCodec == null || existing.videoCodec === file.videoCodec) && + (file.audioCodec == null || existing.audioCodec === file.audioCodec) && + (file.releaseName == null || existing.releaseName === file.releaseName); + + // Full sync van een stabiele library: geen WRITE-storm op Postgres. + if (unchanged) { + return existing.movieId ?? existing.episodeId; + } + const sizeChanged = existing.sizeBytes !== sizeBytes; const discoveredAt = sizeChanged ? new Date() : undefined; await prisma.mediaFile.update({ @@ -486,6 +532,9 @@ export class MetadataService { return existing.movieId ?? existing.episodeId; } + // Alleen herparsen voor nieuwe / unmatched files. + file = this.enrichParsedFromPath(file); + if (file.mediaType === "movie" && file.parsedMovie) { return this.matchMovie(nodeId, file); } @@ -2604,7 +2653,8 @@ export class MetadataService { /** Alleen series met TMDB of IMDb — orphan titel-records tellen niet als match. */ private async findExistingCanonicalSeries(title: string) { const cleaned = stripTrailingSeasonLabel(title.trim()); - const direct = await prisma.series.findFirst({ + if (!cleaned) return null; + return prisma.series.findFirst({ where: { title: { equals: cleaned, mode: "insensitive" }, OR: [{ tmdbId: { not: null } }, { imdbId: { not: null } }], @@ -2615,20 +2665,6 @@ export class MetadataService { { createdAt: "asc" }, ], }); - if (direct) return direct; - - const key = normalizeSeriesKey(cleaned); - if (!key) return null; - const candidates = await prisma.series.findMany({ - where: { OR: [{ tmdbId: { not: null } }, { imdbId: { not: null } }] }, - orderBy: [ - { imdbId: { sort: "desc", nulls: "last" } }, - { tmdbId: { sort: "desc", nulls: "last" } }, - { createdAt: "asc" }, - ], - take: 5000, - }); - return candidates.find((s) => normalizeSeriesKey(s.title) === key) ?? null; } /** OMDb titel-match voor scanner (na TMDB). Alleen bij confident title match. */ diff --git a/node/media-node/cmd/media-node/main.go b/node/media-node/cmd/media-node/main.go index 529f3b8..0b4be2e 100644 --- a/node/media-node/cmd/media-node/main.go +++ b/node/media-node/cmd/media-node/main.go @@ -24,7 +24,7 @@ import ( "github.com/sthmedia/media-node/internal/streaming" ) -var version = "1.3.11" +var version = "1.3.12" func main() { if len(os.Args) < 2 { diff --git a/node/media-node/internal/scanner/scanner.go b/node/media-node/internal/scanner/scanner.go index 06cf9e4..205db0f 100644 --- a/node/media-node/internal/scanner/scanner.go +++ b/node/media-node/internal/scanner/scanner.go @@ -183,8 +183,17 @@ func mediaRootsEqual(a, b []config.MediaRoot) bool { func (s *Scanner) runStartupScan() { log.Println("Startup library inventory scan...") - s.FullScan() + // Geen onScanComplete-sync hier: HELLO wacht op startupScanDone en pusht + // precies één full sync. Anders matcht master de hele library twee keer. s.mu.Lock() + hook := s.onScanComplete + s.onScanComplete = nil + s.mu.Unlock() + + s.FullScan() + + s.mu.Lock() + s.onScanComplete = hook s.startupScanDone = true s.mu.Unlock() }