From 1cc6fedd956f9e89228a9f5136985b083b5c4bf2 Mon Sep 17 00:00:00 2001 From: Jos Vooges | STH Date: Sun, 13 Sep 2026 17:36:11 +0200 Subject: [PATCH] Harden OpenSubtitles.org IMDb search and surface subtitle search warnings. --- apps/admin-ui/src/app/downloads/page.tsx | 18 ++- apps/master-api/src/downloads/service.ts | 24 +++- apps/master-api/src/opensubtitles/client.ts | 115 ++++++++++-------- .../src/opensubtitles/org-client.test.ts | 18 +++ .../src/opensubtitles/org-client.ts | 90 ++++++++++---- apps/master-api/src/settings/opensubtitles.ts | 4 +- 6 files changed, 188 insertions(+), 81 deletions(-) create mode 100644 apps/master-api/src/opensubtitles/org-client.test.ts diff --git a/apps/admin-ui/src/app/downloads/page.tsx b/apps/admin-ui/src/app/downloads/page.tsx index 6e09d64..2d8acb4 100644 --- a/apps/admin-ui/src/app/downloads/page.tsx +++ b/apps/admin-ui/src/app/downloads/page.tsx @@ -417,12 +417,20 @@ export default function DownloadsPage() { } const results: SubtitleHit[] = data.results ?? []; setSubs(results); + const warnings: string[] = Array.isArray(data.warnings) ? data.warnings : []; if (results[0]) { const preferred = results.find((r) => r.source === "org") || results[0]; setSelectedSubId(preferred.fileId); setSelectedSubSource(preferred.source === "org" ? "org" : "com"); + if (warnings.length) { + setSubsMsg(`Let op: ${warnings.join(" · ")}`); + } } else { - setSubsMsg("Geen NL-ondertitels gevonden"); + setSubsMsg( + warnings.length + ? `Geen NL-ondertitels gevonden (${warnings.join(" · ")})` + : "Geen NL-ondertitels gevonden" + ); } } finally { setSubsBusy(false); @@ -666,11 +674,17 @@ export default function DownloadsPage() { return (b.downloadCount || 0) - (a.downloadCount || 0); }); setSubs(results); + const warnings: string[] = Array.isArray(data.warnings) ? data.warnings : []; if (results[0]) { setSelectedSubId(results[0].fileId); setSelectedSubSource(results[0].source === "org" ? "org" : "com"); + if (warnings.length) setSubsMsg(`Let op: ${warnings.join(" · ")}`); } else { - setSubsMsg("Geen ondertitels gevonden"); + setSubsMsg( + warnings.length + ? `Geen ondertitels gevonden (${warnings.join(" · ")})` + : "Geen ondertitels gevonden" + ); } } finally { setSubsBusy(false); diff --git a/apps/master-api/src/downloads/service.ts b/apps/master-api/src/downloads/service.ts index 2463f30..c78058b 100644 --- a/apps/master-api/src/downloads/service.ts +++ b/apps/master-api/src/downloads/service.ts @@ -154,6 +154,7 @@ export class DownloadService { imdbId: string | null; results: SubtitleHit[]; configured: boolean; + warnings?: string[]; }> { const enabled = await isOpenSubtitlesEnabled(); if (!enabled) { @@ -163,6 +164,7 @@ export class DownloadService { const comClient = await this.opensubtitles(); const orgClient = this.orgClient(); const configured = true; + const warnings: string[] = []; const isEpisode = opts.mediaKind === "episode" || @@ -198,7 +200,9 @@ export class DownloadService { const safe = (label: string, p: Promise) => p.catch((err) => { - console.warn(`[downloads] subtitle search ${label}:`, err instanceof Error ? err.message : err); + const msg = err instanceof Error ? err.message : String(err); + console.warn(`[downloads] subtitle search ${label}:`, msg); + warnings.push(`${label}: ${msg}`); return [] as SubtitleHit[]; }); @@ -225,7 +229,7 @@ export class DownloadService { } if (tasks.length === 0) { - return { imdbId, results: [], configured }; + return { imdbId, results: [], configured, warnings: warnings.length ? warnings : undefined }; } const batches = await Promise.all(tasks); @@ -253,7 +257,21 @@ export class DownloadService { .sort((a, b) => b.score - a.score) .map((x) => x.r); - return { imdbId, results: scored.slice(0, 40), configured }; + // Als .org stil faalde en we alleen .com-hits hebben: expliciet waarschuwen + const hasOrg = scored.some((r) => r.source === "org"); + const orgAttempted = !!imdbId || !!query; + if (orgAttempted && !hasOrg && !warnings.some((w) => w.startsWith("org/"))) { + warnings.push( + "org: geen resultaten (controleer IMDb-padding / rest.opensubtitles.org)" + ); + } + + return { + imdbId, + results: scored.slice(0, 40), + configured, + warnings: warnings.length ? warnings : undefined, + }; } /** TV/client: resolve IMDb (+ episode query) from media file, then search OS.com/OS.org + local sidecars. */ diff --git a/apps/master-api/src/opensubtitles/client.ts b/apps/master-api/src/opensubtitles/client.ts index f701e7d..8bfa6cd 100644 --- a/apps/master-api/src/opensubtitles/client.ts +++ b/apps/master-api/src/opensubtitles/client.ts @@ -138,68 +138,77 @@ export class OpenSubtitlesClient { token ? { Authorization: `Bearer ${token}` } : undefined ); - const res = await fetch(`${this.root()}/api/v1/subtitles?${params}`, { headers }); - if (!res.ok) { - const text = await res.text().catch(() => ""); - if (res.status === 403 || /cannot consume/i.test(text)) { + const out: SubtitleHit[] = []; + const maxPages = 3; + for (let page = 1; page <= maxPages; page++) { + params.set("page", String(page)); + const res = await fetch(`${this.root()}/api/v1/subtitles?${params}`, { headers }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + if (res.status === 403 || /cannot consume/i.test(text)) { + throw new AppError( + "OPENSUBTITLES", + `OpenSubtitles.com zoeken geweigerd (403). User-Agent "${this.userAgent}" moet exact je API Consumer-naam zijn, of gebruik alleen OS.org-hits.`, + 400 + ); + } throw new AppError( "OPENSUBTITLES", - `OpenSubtitles.com zoeken geweigerd (403). User-Agent "${this.userAgent}" moet exact je API Consumer-naam zijn, of gebruik alleen OS.org-hits.`, + `OpenSubtitles.com zoeken mislukt (${res.status}): ${text.slice(0, 160)}`, 400 ); } - throw new AppError( - "OPENSUBTITLES", - `OpenSubtitles.com zoeken mislukt (${res.status}): ${text.slice(0, 160)}`, - 400 - ); - } - const body = (await res.json()) as { - data?: Array<{ - attributes?: { - language?: string; - release?: string; - download_count?: number; - hearing_impaired?: boolean; - fps?: number; - feature_details?: { - imdb_id?: number | string; - title?: string; - year?: number; + const body = (await res.json()) as { + total_pages?: number; + data?: Array<{ + attributes?: { + language?: string; + release?: string; + download_count?: number; + hearing_impaired?: boolean; + fps?: number; + feature_details?: { + imdb_id?: number | string; + title?: string; + year?: number; + }; + files?: Array<{ file_id?: number; file_name?: string }>; + uploader?: { name?: string }; }; - files?: Array<{ file_id?: number; file_name?: string }>; - uploader?: { name?: string }; - }; - }>; - }; + }>; + }; - const out: SubtitleHit[] = []; - for (const row of body.data ?? []) { - const a = row.attributes; - const file = a?.files?.[0]; - if (!file?.file_id) continue; - const featImdb = a?.feature_details?.imdb_id - ? String(a.feature_details.imdb_id).replace(/^tt/i, "").replace(/^0+/, "") - : null; - // Movie search: keep IMDb strict. Episode search: parent/series IMDb ≠ episode IMDb. - if (!hasEpisode && featImdb && imdb && featImdb !== imdb) continue; - out.push({ - fileId: file.file_id, - language: (a?.language || languages).toLowerCase(), - release: a?.release || file.file_name || "", - fileName: file.file_name || a?.release || `sub-${file.file_id}.srt`, - downloadCount: a?.download_count ?? 0, - hearingImpaired: !!a?.hearing_impaired, - fps: a?.fps ?? null, - uploader: a?.uploader?.name ?? null, - source: "com", - movieName: a?.feature_details?.title ?? null, - movieYear: a?.feature_details?.year ?? null, - imdbId: featImdb || imdb, - }); + for (const row of body.data ?? []) { + const a = row.attributes; + const file = a?.files?.[0]; + if (!file?.file_id) continue; + const featImdb = a?.feature_details?.imdb_id + ? String(a.feature_details.imdb_id).replace(/^tt/i, "").replace(/^0+/, "") + : null; + // Movie search: keep IMDb strict. Episode search: parent/series IMDb ≠ episode IMDb. + if (!hasEpisode && featImdb && imdb && featImdb !== imdb) continue; + out.push({ + fileId: file.file_id, + language: (a?.language || languages).toLowerCase(), + release: a?.release || file.file_name || "", + fileName: file.file_name || a?.release || `sub-${file.file_id}.srt`, + downloadCount: a?.download_count ?? 0, + hearingImpaired: !!a?.hearing_impaired, + fps: a?.fps ?? null, + uploader: a?.uploader?.name ?? null, + source: "com", + movieName: a?.feature_details?.title ?? null, + movieYear: a?.feature_details?.year ?? null, + imdbId: featImdb || imdb, + }); + } + + const totalPages = body.total_pages ?? page; + if (page >= totalPages || out.length >= 50) break; } - return out.slice(0, 40); + + return out.slice(0, 50); } async downloadFile(fileId: number): Promise<{ content: Buffer; fileName: string }> { diff --git a/apps/master-api/src/opensubtitles/org-client.test.ts b/apps/master-api/src/opensubtitles/org-client.test.ts new file mode 100644 index 0000000..23895e9 --- /dev/null +++ b/apps/master-api/src/opensubtitles/org-client.test.ts @@ -0,0 +1,18 @@ +import { imdbIdsMatch, normalizeImdbForUrl } from "./org-client"; + +function assert(cond: unknown, msg: string): asserts cond { + if (!cond) throw new Error(msg); +} + +assert(normalizeImdbForUrl("tt0468569") === "0468569", "pad Dark Knight"); +assert(normalizeImdbForUrl("468569") === "0468569", "pad short id"); +assert(normalizeImdbForUrl("0468569") === "0468569", "keep padded"); +assert(normalizeImdbForUrl("tt1375666") === "1375666", "no over-pad Inception"); +assert(normalizeImdbForUrl("1375666") === "1375666", "7-digit unchanged"); +assert(normalizeImdbForUrl("tt0111161") === "0111161", "Shawshank pad"); +assert(normalizeImdbForUrl(null) === null, "null"); +assert(imdbIdsMatch("0468569", "468569"), "match padded vs short"); +assert(imdbIdsMatch("0111161", "111161"), "match shawshank forms"); +assert(!imdbIdsMatch("0468569", "1375666"), "different titles"); + +console.log("opensubtitles/org-client imdb helpers: ok"); diff --git a/apps/master-api/src/opensubtitles/org-client.ts b/apps/master-api/src/opensubtitles/org-client.ts index 7c63883..63568da 100644 --- a/apps/master-api/src/opensubtitles/org-client.ts +++ b/apps/master-api/src/opensubtitles/org-client.ts @@ -22,6 +22,10 @@ type OrgRow = { /** * OpenSubtitles.org legacy REST (rest.opensubtitles.org). * Dutch language id is `dut`. IMDb without "tt". No API key. + * + * Important: unpadded IMDb IDs (e.g. `468569` instead of `0468569`) make OS.org + * respond with `302 Location: https://_/search/imdbid-0468569/...`. Following that + * redirect fails (ENOTFOUND) and searches look empty while .com still works. */ export class OpenSubtitlesOrgClient { private readonly ua = "MediaCluster v1.3.2"; @@ -31,17 +35,21 @@ export class OpenSubtitlesOrgClient { query?: string; languages?: string; }): Promise { - const lang = toOrgLang(opts.languages || "nl"); - const imdbDigits = normalizeImdb(opts.imdbId); + const langs = toOrgLangs(opts.languages || "nl"); + const imdbDigits = normalizeImdbForUrl(opts.imdbId); const urls: string[] = []; if (imdbDigits) { - urls.push( - `https://rest.opensubtitles.org/search/imdbid-${imdbDigits}/sublanguageid-${lang}` - ); + for (const lang of langs) { + urls.push( + `https://rest.opensubtitles.org/search/imdbid-${imdbDigits}/sublanguageid-${lang}` + ); + } } else if (opts.query?.trim()) { const q = encodeURIComponent(opts.query.trim()).replace(/%20/g, "+"); - urls.push(`https://rest.opensubtitles.org/search/query-${q}/sublanguageid-${lang}`); + for (const lang of langs) { + urls.push(`https://rest.opensubtitles.org/search/query-${q}/sublanguageid-${lang}`); + } } if (!urls.length) return []; @@ -54,10 +62,11 @@ export class OpenSubtitlesOrgClient { const rows = await this.fetchSearch(url); console.info(`[opensubtitles.org] ${url} -> ${rows.length} raw`); for (const row of rows) { - const hit = mapRow(row, lang); + const hit = mapRow(row, langs[0] || "dut"); if (!hit || seen.has(hit.fileId)) continue; - // Alleen droppen bij expliciet andere IMDb — ontbrekende IDMovieImdb behouden - if (imdbDigits && hit.imdbId && hit.imdbId !== imdbDigits) continue; + // Alleen droppen bij expliciet andere IMDb — ontbrekende IDMovieImdb behouden. + // Vergelijk canoniek (zonder leading zeros): API geeft vaak "468569", URL "0468569". + if (imdbDigits && hit.imdbId && !imdbIdsMatch(hit.imdbId, imdbDigits)) continue; seen.add(hit.fileId); out.push(hit); } @@ -73,15 +82,36 @@ export class OpenSubtitlesOrgClient { return out.slice(0, 50); } - private async fetchSearch(url: string): Promise { + private async fetchSearch(url: string, redirectHop = 0): Promise { const res = await fetch(url, { headers: { Accept: "application/json", "User-Agent": this.ua, "X-User-Agent": this.ua, }, - redirect: "follow", + // Manual: OS.org sometimes 302's to host "_" with the *correct* padded path. + redirect: "manual", }); + + if (res.status >= 300 && res.status < 400) { + const loc = res.headers.get("location") || ""; + const pathMatch = loc.match(/\/search\/[^?#\s]+/); + if (pathMatch && redirectHop < 2) { + const fixed = `https://rest.opensubtitles.org${pathMatch[0]}`; + if (fixed !== url) { + console.warn( + `[opensubtitles.org] broken redirect (${res.status} → ${loc}); retry ${fixed}` + ); + return this.fetchSearch(fixed, redirectHop + 1); + } + } + throw new AppError( + "OPENSUBTITLES", + `OpenSubtitles.org zoeken mislukt (${res.status} redirect: ${loc.slice(0, 120)})`, + 400 + ); + } + if (!res.ok) { const text = await res.text().catch(() => ""); throw new AppError( @@ -131,7 +161,7 @@ function mapRow(row: OrgRow, fallbackLang: string): SubtitleHit | null { if (!Number.isFinite(fileId) || fileId <= 0) return null; const fps = row.MovieFPS ? parseFloat(row.MovieFPS) : null; const yearRaw = row.MovieYear ? parseInt(String(row.MovieYear), 10) : NaN; - const imdb = normalizeImdb(row.IDMovieImdb || undefined); + const imdb = normalizeImdbForUrl(row.IDMovieImdb || undefined); return { fileId, language: (row.ISO639 || row.SubLanguageID || fallbackLang || "nl").toLowerCase(), @@ -148,22 +178,38 @@ function mapRow(row: OrgRow, fallbackLang: string): SubtitleHit | null { }; } -function normalizeImdb(id?: string | null): string | null { +/** Digits only, zero-padded to ≥7 for OS.org URL paths. */ +export function normalizeImdbForUrl(id?: string | null): string | null { if (!id) return null; const digits = String(id).trim().replace(/^tt/i, "").replace(/\D/g, ""); if (!digits) return null; - // OS.org URLs expect ≥7-digit zero-padded IDs; stripping zeros causes a - // broken 302 to host "_" and silent empty search results. - return digits.padStart(7, "0"); + return digits.length >= 7 ? digits : digits.padStart(7, "0"); +} + +/** Canonical compare: "0468569" === "468569". */ +export function imdbIdsMatch(a: string, b: string): boolean { + const ca = a.replace(/^0+/, "") || "0"; + const cb = b.replace(/^0+/, "") || "0"; + return ca === cb; } /** Legacy REST uses OpenSubtitles language ids (`dut` = Dutch). */ -function toOrgLang(languages: string): string { - const first = languages.split(",")[0]?.trim().toLowerCase() || "nl"; - if (first === "nl" || first === "nld" || first === "dut" || first === "dutch") return "dut"; - if (first === "en" || first === "eng") return "eng"; - if (first.length === 3) return first; - return first; +function toOrgLangs(languages: string): string[] { + const out: string[] = []; + const seen = new Set(); + for (const part of languages.split(",")) { + const first = part.trim().toLowerCase(); + if (!first) continue; + let code: string; + if (first === "nl" || first === "nld" || first === "dut" || first === "dutch") code = "dut"; + else if (first === "en" || first === "eng") code = "eng"; + else if (first.length === 3) code = first; + else code = first; + if (seen.has(code)) continue; + seen.add(code); + out.push(code); + } + return out.length ? out : ["dut"]; } function isGzip(buf: Buffer): boolean { diff --git a/apps/master-api/src/settings/opensubtitles.ts b/apps/master-api/src/settings/opensubtitles.ts index 263e1bf..8458772 100644 --- a/apps/master-api/src/settings/opensubtitles.ts +++ b/apps/master-api/src/settings/opensubtitles.ts @@ -103,7 +103,9 @@ export async function upsertOpenSubtitlesSettings( try { const { OpenSubtitlesOrgClient } = await import("../opensubtitles/org-client"); - await new OpenSubtitlesOrgClient().search({ imdbId: "tt1375666", languages: "nl" }); + // tt0111161 (Shawshank) has a leading zero — catches the OS.org padding/redirect bug + // that tt1375666 (Inception) does not. + await new OpenSubtitlesOrgClient().search({ imdbId: "tt0111161", languages: "nl" }); orgOk = true; } catch (err) { const msg = err instanceof Error ? err.message : "OpenSubtitles.org test mislukt";