jos/stremio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Harden OpenSubtitles.org IMDb search and surface subtitle search warnings.

Browse source
This commit is contained in:
Jos Vooges | STH 2026-09-13 17:36:11 +02:00
parent 55695aa47c
commit 1cc6fedd95
6 changed files with 188 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
apps/admin-ui/src/app/downloads/page.tsx
View file

@ -417,12 +417,20 @@ export default function DownloadsPage() {
}
const results: SubtitleHit[] = data.results ?? [];
setSubs(results);
const warnings: string[] = Array.isArray(data.warnings) ? data.warnings : [];
if (results[0]) {
const preferred = results.find((r) => r.source === "org") || results[0];
setSelectedSubId(preferred.fileId);
setSelectedSubSource(preferred.source === "org" ? "org" : "com");
if (warnings.length) {
setSubsMsg(`Let op: ${warnings.join(" · ")}`);
}
} else {
setSubsMsg("Geen NL-ondertitels gevonden");
setSubsMsg(
warnings.length
? `Geen NL-ondertitels gevonden (${warnings.join(" · ")})`
: "Geen NL-ondertitels gevonden"
);
}
} finally {
setSubsBusy(false);
@ -666,11 +674,17 @@ export default function DownloadsPage() {
return (b.downloadCount || 0) - (a.downloadCount || 0);
});
setSubs(results);
const warnings: string[] = Array.isArray(data.warnings) ? data.warnings : [];
if (results[0]) {
setSelectedSubId(results[0].fileId);
setSelectedSubSource(results[0].source === "org" ? "org" : "com");
if (warnings.length) setSubsMsg(`Let op: ${warnings.join(" · ")}`);
} else {
setSubsMsg("Geen ondertitels gevonden");
setSubsMsg(
warnings.length
? `Geen ondertitels gevonden (${warnings.join(" · ")})`
: "Geen ondertitels gevonden"
);
}
} finally {
setSubsBusy(false);

24
apps/master-api/src/downloads/service.ts
View file

@ -154,6 +154,7 @@ export class DownloadService {
imdbId: string | null;
results: SubtitleHit[];
configured: boolean;
warnings?: string[];
}> {
const enabled = await isOpenSubtitlesEnabled();
if (!enabled) {
@ -163,6 +164,7 @@ export class DownloadService {
const comClient = await this.opensubtitles();
const orgClient = this.orgClient();
const configured = true;
const warnings: string[] = [];
const isEpisode =
opts.mediaKind === "episode" ||
@ -198,7 +200,9 @@ export class DownloadService {
const safe = (label: string, p: Promise<SubtitleHit[]>) =>
p.catch((err) => {
console.warn(`[downloads] subtitle search ${label}:`, err instanceof Error ? err.message : err);
const msg = err instanceof Error ? err.message : String(err);
console.warn(`[downloads] subtitle search ${label}:`, msg);
warnings.push(`${label}: ${msg}`);
return [] as SubtitleHit[];
});
@ -225,7 +229,7 @@ export class DownloadService {
}
if (tasks.length === 0) {
return { imdbId, results: [], configured };
return { imdbId, results: [], configured, warnings: warnings.length ? warnings : undefined };
}
const batches = await Promise.all(tasks);
@ -253,7 +257,21 @@ export class DownloadService {
.sort((a, b) => b.score - a.score)
.map((x) => x.r);
return { imdbId, results: scored.slice(0, 40), configured };
// Als .org stil faalde en we alleen .com-hits hebben: expliciet waarschuwen
const hasOrg = scored.some((r) => r.source === "org");
const orgAttempted = !!imdbId || !!query;
if (orgAttempted && !hasOrg && !warnings.some((w) => w.startsWith("org/"))) {
warnings.push(
"org: geen resultaten (controleer IMDb-padding / rest.opensubtitles.org)"
);
}
return {
imdbId,
results: scored.slice(0, 40),
configured,
warnings: warnings.length ? warnings : undefined,
};
}
/** TV/client: resolve IMDb (+ episode query) from media file, then search OS.com/OS.org + local sidecars. */

13
apps/master-api/src/opensubtitles/client.ts
View file

@ -138,6 +138,10 @@ export class OpenSubtitlesClient {
token ? { Authorization: `Bearer ${token}` } : undefined
);
const out: SubtitleHit[] = [];
const maxPages = 3;
for (let page = 1; page <= maxPages; page++) {
params.set("page", String(page));
const res = await fetch(`${this.root()}/api/v1/subtitles?${params}`, { headers });
if (!res.ok) {
const text = await res.text().catch(() => "");
@ -156,6 +160,7 @@ export class OpenSubtitlesClient {
}
const body = (await res.json()) as {
total_pages?: number;
data?: Array<{
attributes?: {
language?: string;
@ -174,7 +179,6 @@ export class OpenSubtitlesClient {
}>;
};
const out: SubtitleHit[] = [];
for (const row of body.data ?? []) {
const a = row.attributes;
const file = a?.files?.[0];
@ -199,7 +203,12 @@ export class OpenSubtitlesClient {
imdbId: featImdb || imdb,
});
}
return out.slice(0, 40);
const totalPages = body.total_pages ?? page;
if (page >= totalPages || out.length >= 50) break;
}
return out.slice(0, 50);
}
async downloadFile(fileId: number): Promise<{ content: Buffer; fileName: string }> {

18
apps/master-api/src/opensubtitles/org-client.test.ts Normal file
View file

@ -0,0 +1,18 @@
import { imdbIdsMatch, normalizeImdbForUrl } from "./org-client";
function assert(cond: unknown, msg: string): asserts cond {
if (!cond) throw new Error(msg);
}
assert(normalizeImdbForUrl("tt0468569") === "0468569", "pad Dark Knight");
assert(normalizeImdbForUrl("468569") === "0468569", "pad short id");
assert(normalizeImdbForUrl("0468569") === "0468569", "keep padded");
assert(normalizeImdbForUrl("tt1375666") === "1375666", "no over-pad Inception");
assert(normalizeImdbForUrl("1375666") === "1375666", "7-digit unchanged");
assert(normalizeImdbForUrl("tt0111161") === "0111161", "Shawshank pad");
assert(normalizeImdbForUrl(null) === null, "null");
assert(imdbIdsMatch("0468569", "468569"), "match padded vs short");
assert(imdbIdsMatch("0111161", "111161"), "match shawshank forms");
assert(!imdbIdsMatch("0468569", "1375666"), "different titles");
console.log("opensubtitles/org-client imdb helpers: ok");

82
apps/master-api/src/opensubtitles/org-client.ts
View file

@ -22,6 +22,10 @@ type OrgRow = {
/**
* OpenSubtitles.org legacy REST (rest.opensubtitles.org).
* Dutch language id is `dut`. IMDb without "tt". No API key.
*
* Important: unpadded IMDb IDs (e.g. `468569` instead of `0468569`) make OS.org
* respond with `302 Location: https://_/search/imdbid-0468569/...`. Following that
* redirect fails (ENOTFOUND) and searches look empty while .com still works.
*/
export class OpenSubtitlesOrgClient {
private readonly ua = "MediaCluster v1.3.2";
@ -31,18 +35,22 @@ export class OpenSubtitlesOrgClient {
query?: string;
languages?: string;
}): Promise<SubtitleHit[]> {
const lang = toOrgLang(opts.languages || "nl");
const imdbDigits = normalizeImdb(opts.imdbId);
const langs = toOrgLangs(opts.languages || "nl");
const imdbDigits = normalizeImdbForUrl(opts.imdbId);
const urls: string[] = [];
if (imdbDigits) {
for (const lang of langs) {
urls.push(
`https://rest.opensubtitles.org/search/imdbid-${imdbDigits}/sublanguageid-${lang}`
);
}
} else if (opts.query?.trim()) {
const q = encodeURIComponent(opts.query.trim()).replace(/%20/g, "+");
for (const lang of langs) {
urls.push(`https://rest.opensubtitles.org/search/query-${q}/sublanguageid-${lang}`);
}
}
if (!urls.length) return [];
let lastErr: string | null = null;
@ -54,10 +62,11 @@ export class OpenSubtitlesOrgClient {
const rows = await this.fetchSearch(url);
console.info(`[opensubtitles.org] ${url} -> ${rows.length} raw`);
for (const row of rows) {
const hit = mapRow(row, lang);
const hit = mapRow(row, langs[0] || "dut");
if (!hit || seen.has(hit.fileId)) continue;
// Alleen droppen bij expliciet andere IMDb — ontbrekende IDMovieImdb behouden
if (imdbDigits && hit.imdbId && hit.imdbId !== imdbDigits) continue;
// Alleen droppen bij expliciet andere IMDb — ontbrekende IDMovieImdb behouden.
// Vergelijk canoniek (zonder leading zeros): API geeft vaak "468569", URL "0468569".
if (imdbDigits && hit.imdbId && !imdbIdsMatch(hit.imdbId, imdbDigits)) continue;
seen.add(hit.fileId);
out.push(hit);
}
@ -73,15 +82,36 @@ export class OpenSubtitlesOrgClient {
return out.slice(0, 50);
}
private async fetchSearch(url: string): Promise<OrgRow[]> {
private async fetchSearch(url: string, redirectHop = 0): Promise<OrgRow[]> {
const res = await fetch(url, {
headers: {
Accept: "application/json",
"User-Agent": this.ua,
"X-User-Agent": this.ua,
},
redirect: "follow",
// Manual: OS.org sometimes 302's to host "_" with the *correct* padded path.
redirect: "manual",
});
if (res.status >= 300 && res.status < 400) {
const loc = res.headers.get("location") || "";
const pathMatch = loc.match(/\/search\/[^?#\s]+/);
if (pathMatch && redirectHop < 2) {
const fixed = `https://rest.opensubtitles.org${pathMatch[0]}`;
if (fixed !== url) {
console.warn(
`[opensubtitles.org] broken redirect (${res.status} → ${loc}); retry ${fixed}`
);
return this.fetchSearch(fixed, redirectHop + 1);
}
}
throw new AppError(
"OPENSUBTITLES",
`OpenSubtitles.org zoeken mislukt (${res.status} redirect: ${loc.slice(0, 120)})`,
400
);
}
if (!res.ok) {
const text = await res.text().catch(() => "");
throw new AppError(
@ -131,7 +161,7 @@ function mapRow(row: OrgRow, fallbackLang: string): SubtitleHit | null {
if (!Number.isFinite(fileId) || fileId <= 0) return null;
const fps = row.MovieFPS ? parseFloat(row.MovieFPS) : null;
const yearRaw = row.MovieYear ? parseInt(String(row.MovieYear), 10) : NaN;
const imdb = normalizeImdb(row.IDMovieImdb || undefined);
const imdb = normalizeImdbForUrl(row.IDMovieImdb || undefined);
return {
fileId,
language: (row.ISO639 || row.SubLanguageID || fallbackLang || "nl").toLowerCase(),
@ -148,22 +178,38 @@ function mapRow(row: OrgRow, fallbackLang: string): SubtitleHit | null {
};
}
function normalizeImdb(id?: string | null): string | null {
/** Digits only, zero-padded to ≥7 for OS.org URL paths. */
export function normalizeImdbForUrl(id?: string | null): string | null {
if (!id) return null;
const digits = String(id).trim().replace(/^tt/i, "").replace(/\D/g, "");
if (!digits) return null;
// OS.org URLs expect ≥7-digit zero-padded IDs; stripping zeros causes a
// broken 302 to host "_" and silent empty search results.
return digits.padStart(7, "0");
return digits.length >= 7 ? digits : digits.padStart(7, "0");
}
/** Canonical compare: "0468569" === "468569". */
export function imdbIdsMatch(a: string, b: string): boolean {
const ca = a.replace(/^0+/, "") || "0";
const cb = b.replace(/^0+/, "") || "0";
return ca === cb;
}
/** Legacy REST uses OpenSubtitles language ids (`dut` = Dutch). */
function toOrgLang(languages: string): string {
const first = languages.split(",")[0]?.trim().toLowerCase() || "nl";
if (first === "nl" || first === "nld" || first === "dut" || first === "dutch") return "dut";
if (first === "en" || first === "eng") return "eng";
if (first.length === 3) return first;
return first;
function toOrgLangs(languages: string): string[] {
const out: string[] = [];
const seen = new Set<string>();
for (const part of languages.split(",")) {
const first = part.trim().toLowerCase();
if (!first) continue;
let code: string;
if (first === "nl" || first === "nld" || first === "dut" || first === "dutch") code = "dut";
else if (first === "en" || first === "eng") code = "eng";
else if (first.length === 3) code = first;
else code = first;
if (seen.has(code)) continue;
seen.add(code);
out.push(code);
}
return out.length ? out : ["dut"];
}
function isGzip(buf: Buffer): boolean {

4
apps/master-api/src/settings/opensubtitles.ts
View file

@ -103,7 +103,9 @@ export async function upsertOpenSubtitlesSettings(
try {
const { OpenSubtitlesOrgClient } = await import("../opensubtitles/org-client");
await new OpenSubtitlesOrgClient().search({ imdbId: "tt1375666", languages: "nl" });
// tt0111161 (Shawshank) has a leading zero — catches the OS.org padding/redirect bug
// that tt1375666 (Inception) does not.
await new OpenSubtitlesOrgClient().search({ imdbId: "tt0111161", languages: "nl" });
orgOk = true;
} catch (err) {
const msg = err instanceof Error ? err.message : "OpenSubtitles.org test mislukt";