jos/stremio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Cut full-sync Postgres load: skip no-op updates, batch lookup, one post-startup sync.

Browse source 
Also cap startup IMDb backfill and bump media-node to 1.3.12 so upgrades no longer match the library twice.
This commit is contained in:
Jos Vooges | STH 2026-09-17 03:07:29 +02:00
parent 7e2dd68da9
commit 34130ab412
5 changed files with 105 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
apps/master-api/src/app.ts
View file

@ -610,13 +610,13 @@ async function main() {
await app.listen({ port: config.PORT, host: config.HOST });
console.log(`Master API listening on ${config.HOST}:${config.PORT}`);
// Background: fill ALL missing movie + series imdbIds so Cinemeta can resolve local streams.
// Series catalog/meta stay on mcseries-* (tt only used for streams).
// Background: klein stukje IMDb backfill na start (niet de hele library — dat hield CPU/TMDB bezig).
// Rest via Admin → metadata tools of Stremio lazy backfill.
if (config.TMDB_API_KEY) {
const metadata = createMetadataService(config);
setTimeout(() => {
void metadata
.backfillAllImdbIds({ delayMs: 120 })
.backfillAllImdbIds({ delayMs: 200, limit: 40 })
.then((r) => {
if (r.movies.checked > 0 || r.series.checked > 0) {
console.log(

30
apps/master-api/src/media/sync.ts
View file

@ -53,10 +53,38 @@ export class LibrarySyncService {
fileIdsSoFar: Set<string>;
}
): Promise<void> {
// Eén findMany per batch i.p.v. N findUnique — scheelt enorm bij stabiele library.
const existingRows =
files.length > 0
? await prisma.mediaFile.findMany({
where: {
nodeId,
localFileId: { in: files.map((f) => f.localFileId) },
},
select: {
id: true,
localFileId: true,
movieId: true,
episodeId: true,
shelfId: true,
sizeBytes: true,
path: true,
available: true,
container: true,
resolution: true,
videoCodec: true,
audioCodec: true,
releaseName: true,
},
})
: [];
const existingByLocal = new Map(existingRows.map((r) => [r.localFileId, r]));
for (const file of files) {
options.fileIdsSoFar.add(file.localFileId);
try {
await this.metadata.matchMediaFile(nodeId, file);
const preloaded = existingByLocal.get(file.localFileId) ?? null;
await this.metadata.matchMediaFile(nodeId, file, { existing: preloaded });
} catch (err) {
console.error(
`Library full sync: failed to match ${file.releaseName} on node ${nodeId}:`,

80
apps/master-api/src/metadata/service.ts
View file

@ -443,21 +443,67 @@ export class MetadataService {
async matchMediaFile(
nodeId: string,
file: MediaFileInfo
file: MediaFileInfo,
options?: {
/** Vooraf geladen rij (full-sync batch); `null` = bestaat niet, weglaten = zelf ophalen. */
existing?: {
id: string;
movieId: string | null;
episodeId: string | null;
shelfId: string | null;
sizeBytes: bigint;
path: string | null;
available: boolean;
container: string | null;
resolution: string | null;
videoCodec: string | null;
audioCodec: string | null;
releaseName: string | null;
} | null;
}
): Promise<string | null> {
// Master herparst het pad (Plex S10E01 Title.mkv e.d.) — betrouwbaarder dan alleen node-parse.
file = this.enrichParsedFromPath(file);
// Fast path: already matched → only refresh shelf / availability (no TMDB).
const existing = await prisma.mediaFile.findUnique({
// Fast path eerst — geen pad-herparse nodig als het bestand al gekoppeld is.
const existing =
options && "existing" in options
? options.existing ?? null
: await prisma.mediaFile.findUnique({
where: {
nodeId_localFileId: { nodeId, localFileId: file.localFileId },
},
select: { id: true, movieId: true, episodeId: true, shelfId: true, sizeBytes: true, path: true },
select: {
id: true,
movieId: true,
episodeId: true,
shelfId: true,
sizeBytes: true,
path: true,
available: true,
container: true,
resolution: true,
videoCodec: true,
audioCodec: true,
releaseName: true,
},
});
if (existing && (existing.movieId || existing.episodeId)) {
const nextShelf = file.shelfId || null;
const sizeBytes = BigInt(file.sizeBytes);
const unchanged =
existing.available === true &&
existing.sizeBytes === sizeBytes &&
(existing.shelfId ?? null) === nextShelf &&
(file.path == null || file.path === "" || existing.path === file.path) &&
(file.container == null || existing.container === file.container) &&
(file.resolution == null || existing.resolution === file.resolution) &&
(file.videoCodec == null || existing.videoCodec === file.videoCodec) &&
(file.audioCodec == null || existing.audioCodec === file.audioCodec) &&
(file.releaseName == null || existing.releaseName === file.releaseName);
// Full sync van een stabiele library: geen WRITE-storm op Postgres.
if (unchanged) {
return existing.movieId ?? existing.episodeId;
}
const sizeChanged = existing.sizeBytes !== sizeBytes;
const discoveredAt = sizeChanged ? new Date() : undefined;
await prisma.mediaFile.update({
@ -486,6 +532,9 @@ export class MetadataService {
return existing.movieId ?? existing.episodeId;
}
// Alleen herparsen voor nieuwe / unmatched files.
file = this.enrichParsedFromPath(file);
if (file.mediaType === "movie" && file.parsedMovie) {
return this.matchMovie(nodeId, file);
}
@ -2604,7 +2653,8 @@ export class MetadataService {
/** Alleen series met TMDB of IMDb — orphan titel-records tellen niet als match. */
private async findExistingCanonicalSeries(title: string) {
const cleaned = stripTrailingSeasonLabel(title.trim());
const direct = await prisma.series.findFirst({
if (!cleaned) return null;
return prisma.series.findFirst({
where: {
title: { equals: cleaned, mode: "insensitive" },
OR: [{ tmdbId: { not: null } }, { imdbId: { not: null } }],
@ -2615,20 +2665,6 @@ export class MetadataService {
{ createdAt: "asc" },
],
});
if (direct) return direct;
const key = normalizeSeriesKey(cleaned);
if (!key) return null;
const candidates = await prisma.series.findMany({
where: { OR: [{ tmdbId: { not: null } }, { imdbId: { not: null } }] },
orderBy: [
{ imdbId: { sort: "desc", nulls: "last" } },
{ tmdbId: { sort: "desc", nulls: "last" } },
{ createdAt: "asc" },
],
take: 5000,
});
return candidates.find((s) => normalizeSeriesKey(s.title) === key) ?? null;
}
/** OMDb titel-match voor scanner (na TMDB). Alleen bij confident title match. */

2
node/media-node/cmd/media-node/main.go
View file

@ -24,7 +24,7 @@ import (
"github.com/sthmedia/media-node/internal/streaming"
)
var version = "1.3.11"
var version = "1.3.12"
func main() {
if len(os.Args) < 2 {

11
node/media-node/internal/scanner/scanner.go
View file

@ -183,8 +183,17 @@ func mediaRootsEqual(a, b []config.MediaRoot) bool {
func (s *Scanner) runStartupScan() {
log.Println("Startup library inventory scan...")
s.FullScan()
// Geen onScanComplete-sync hier: HELLO wacht op startupScanDone en pusht
// precies één full sync. Anders matcht master de hele library twee keer.
s.mu.Lock()
hook := s.onScanComplete
s.onScanComplete = nil
s.mu.Unlock()
s.FullScan()
s.mu.Lock()
s.onScanComplete = hook
s.startupScanDone = true
s.mu.Unlock()
}