From 15a678b835f49eb8a4987ed9b8e3012836103d5f Mon Sep 17 00:00:00 2001 From: Jos Vooges | STH Date: Mon, 21 Sep 2026 21:37:20 +0200 Subject: [PATCH] Auto-match Odido channels to XMLTV EPG across all configured sources. --- apps/admin-ui/src/app/live-lists/page.tsx | 2 + apps/master-api/src/admin/routes.ts | 2 +- apps/master-api/src/odido/sync.ts | 63 +++++++++- apps/master-api/src/odido/tweaks.ts | 10 ++ apps/master-api/src/viewer/epg-match.test.ts | 34 ++++++ apps/master-api/src/viewer/epg-match.ts | 118 +++++++++++++++++++ apps/master-api/src/viewer/live-channels.ts | 88 +++++++++----- 7 files changed, 284 insertions(+), 33 deletions(-) create mode 100644 apps/master-api/src/viewer/epg-match.test.ts create mode 100644 apps/master-api/src/viewer/epg-match.ts diff --git a/apps/admin-ui/src/app/live-lists/page.tsx b/apps/admin-ui/src/app/live-lists/page.tsx index 924c557..bd8e8e0 100644 --- a/apps/admin-ui/src/app/live-lists/page.tsx +++ b/apps/admin-ui/src/app/live-lists/page.tsx @@ -328,6 +328,8 @@ export default function LiveListsPage() { d.disabled != null ? `${d.disabled} uit` : null, d.skipped != null ? `${d.skipped} overgeslagen` : null, d.failed != null ? `${d.failed} mislukt` : null, + d.epgMatched != null ? `EPG ${d.epgMatched}` : null, + d.epgUnmatched != null && d.epgUnmatched > 0 ? `EPG-miss ${d.epgUnmatched}` : null, d.durationMs != null ? `${Math.round(d.durationMs / 1000)}s` : null, ].filter(Boolean); setMsg(`Odido-sync klaar: ${parts.join(" · ") || "ok"}`); diff --git a/apps/master-api/src/admin/routes.ts b/apps/master-api/src/admin/routes.ts index 7657302..1868211 100644 --- a/apps/master-api/src/admin/routes.ts +++ b/apps/master-api/src/admin/routes.ts @@ -1713,7 +1713,7 @@ export async function registerAdminRoutes(app: FastifyInstance, config: Config) async (request) => { const { id } = request.params as { id: string }; const { syncOdidoList } = await import("../odido/sync"); - return syncOdidoList(id, config.SESSION_SECRET); + return syncOdidoList(id, config.SESSION_SECRET, config); } ); diff --git a/apps/master-api/src/odido/sync.ts b/apps/master-api/src/odido/sync.ts index 1e1aa91..7ef77e6 100644 --- a/apps/master-api/src/odido/sync.ts +++ b/apps/master-api/src/odido/sync.ts @@ -1,6 +1,7 @@ import { prisma } from "../database/client"; import { encryptSecret } from "../security/crypto"; import { AppError } from "../security/errors"; +import type { Config } from "../config"; import { extractOdidoKeys } from "./cdm"; import { getPssh, @@ -10,6 +11,11 @@ import { type OdidoChannel, } from "./client"; import { loadOdidoSettings, saveOdidoSyncResult } from "./settings"; +import { + buildEpgNameIndex, + matchEpgChannelId, +} from "../viewer/epg-match"; +import { loadXmltvChannelIndex } from "../viewer/live-channels"; export type OdidoSyncSummary = { added: number; @@ -17,6 +23,8 @@ export type OdidoSyncSummary = { skipped: number; failed: number; disabled: number; + epgMatched: number; + epgUnmatched: number; durationMs: number; }; @@ -67,7 +75,8 @@ async function mapLimit( export async function syncOdidoList( listId: string, - sessionSecret: string + sessionSecret: string, + config?: Config ): Promise { const started = Date.now(); const list = await prisma.liveChannelList.findUnique({ @@ -96,7 +105,29 @@ export async function syncOdidoList( .map((channel) => [channel.externalId!, channel]) ); const successful = new Set(); - const counts = { added: 0, updated: 0, skipped: 0, failed: 0 }; + const counts = { + added: 0, + updated: 0, + skipped: 0, + failed: 0, + epgMatched: 0, + epgUnmatched: 0, + }; + + let epgIdSet = new Set(); + let epgByName = new Map(); + if (config && tweaks.syncAutoMatchEpg) { + try { + const index = await loadXmltvChannelIndex(config); + epgIdSet = new Set(index.keys()); + epgByName = buildEpgNameIndex( + [...index.entries()].map(([id, name]) => ({ id, name })), + "odido/" + ); + } catch (err) { + console.warn("[odido] EPG-index laden mislukt:", err); + } + } await mapLimit(videos, tweaks.syncConcurrency, async ({ channel, apiIndex }) => { const externalId = channel.ID?.trim(); @@ -147,13 +178,39 @@ export async function syncOdidoList( const name = channel.name?.trim() || channel.channelName?.trim() || physical?.mediaName?.trim() || externalId; + const resolvedContentId = play.contentId ?? contentId(channel); const now = new Date(); + + let epgChannelId = previous?.epgChannelId ?? null; + const mayOverwrite = tweaks.syncOverwriteEpgMatch; + if (epgIdSet.size > 0 && (mayOverwrite || !epgChannelId)) { + const matched = matchEpgChannelId( + { + name, + externalId, + contentId: resolvedContentId, + provider: "odido", + }, + epgIdSet, + epgByName + ); + if (matched) { + epgChannelId = matched; + counts.epgMatched += 1; + } else if (!epgChannelId) { + counts.epgUnmatched += 1; + } + } else if (!epgChannelId) { + counts.epgUnmatched += 1; + } + const data = { name, logoUrl: logoUrl(channel), mpdUrl: `odido://${externalId}/${mediaId}`, mediaId, - contentId: play.contentId ?? contentId(channel), + contentId: resolvedContentId, + epgChannelId, kidEnc, keyEnc, sortOrder: channelSortOrder(channel, apiIndex), diff --git a/apps/master-api/src/odido/tweaks.ts b/apps/master-api/src/odido/tweaks.ts index ce4d5ad..4df5c24 100644 --- a/apps/master-api/src/odido/tweaks.ts +++ b/apps/master-api/src/odido/tweaks.ts @@ -32,6 +32,10 @@ export type OdidoTweaks = { syncFetchKeysIfMissing: boolean; /** Tijdens sync de MPD-cache vullen. */ syncRefreshMpdCache: boolean; + /** Tijdens sync automatisch EPG-id koppelen (alle XMLTV-bronnen). */ + syncAutoMatchEpg: boolean; + /** Bestaande handmatige EPG-koppeling overschrijven bij sync. */ + syncOverwriteEpgMatch: boolean; /** Bij play ontbrekende keys alsnog via CDM. */ refetchKeysIfMissingOnPlay: boolean; heartbeatEnabled: boolean; @@ -57,6 +61,8 @@ export const ODIDO_TWEAK_DEFAULTS: OdidoTweaks = { syncConcurrency: 4, syncFetchKeysIfMissing: true, syncRefreshMpdCache: true, + syncAutoMatchEpg: true, + syncOverwriteEpgMatch: false, refetchKeysIfMissingOnPlay: true, heartbeatEnabled: false, heartbeatIntervalSeconds: 300, @@ -97,6 +103,8 @@ export function mergeOdidoTweaks(raw: Partial | null | undefined): syncConcurrency: clampInt(r.syncConcurrency, 1, 16, d.syncConcurrency), syncFetchKeysIfMissing: asBool(r.syncFetchKeysIfMissing, d.syncFetchKeysIfMissing), syncRefreshMpdCache: asBool(r.syncRefreshMpdCache, d.syncRefreshMpdCache), + syncAutoMatchEpg: asBool(r.syncAutoMatchEpg, d.syncAutoMatchEpg), + syncOverwriteEpgMatch: asBool(r.syncOverwriteEpgMatch, d.syncOverwriteEpgMatch), refetchKeysIfMissingOnPlay: asBool(r.refetchKeysIfMissingOnPlay, d.refetchKeysIfMissingOnPlay), heartbeatEnabled: asBool(r.heartbeatEnabled, d.heartbeatEnabled), heartbeatIntervalSeconds: clampInt(r.heartbeatIntervalSeconds, 30, 3600, d.heartbeatIntervalSeconds), @@ -139,6 +147,8 @@ export const ODIDO_TWEAK_META: Array<{ { key: "syncConcurrency", label: "Sync parallel", hint: "Gelijktijdige zenders bij volledig ophalen", kind: "number" }, { key: "syncFetchKeysIfMissing", label: "Sync: keys ophalen", hint: "CDM tijdens sync als keys ontbreken", kind: "boolean" }, { key: "syncRefreshMpdCache", label: "Sync: MPD-cache vullen", hint: "Sla playURL op tijdens sync", kind: "boolean" }, + { key: "syncAutoMatchEpg", label: "Sync: EPG auto-match", hint: "Koppel XMLTV-id via alle EPG-bronnen", kind: "boolean" }, + { key: "syncOverwriteEpgMatch", label: "Sync: EPG overschrijven", hint: "Ook bestaande handmatige EPG-koppeling vervangen", kind: "boolean" }, { key: "refetchKeysIfMissingOnPlay", label: "Play: keys als ontbrekend", hint: "CDM bij play zonder opgeslagen keys", kind: "boolean" }, { key: "heartbeatEnabled", label: "Odido heartbeat", hint: "Sessie warm houden via OnLineHeartbeat", kind: "boolean" }, { key: "heartbeatIntervalSeconds", label: "Heartbeat-interval", hint: "Alleen als heartbeat aan staat", kind: "number", unit: "sec" }, diff --git a/apps/master-api/src/viewer/epg-match.test.ts b/apps/master-api/src/viewer/epg-match.test.ts new file mode 100644 index 0000000..474dbd6 --- /dev/null +++ b/apps/master-api/src/viewer/epg-match.test.ts @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import { + buildEpgNameIndex, + matchEpgChannelId, + normalizeEpgLabel, +} from "./epg-match"; + +assert.equal(normalizeEpgLabel("1. NPO 1"), "npo1"); +assert.equal(normalizeEpgLabel("NPO 1 HD"), "npo1"); +assert.equal(normalizeEpgLabel("BBC One"), "bbcone"); + +const options = [ + { id: "odido/141430714", name: "1. NPO 1" }, + { id: "odido/141429798", name: "150. ESPN" }, + { id: "NPO1.nl", name: "NPO 1" }, + { id: "other/espn", name: "ESPN HD" }, +]; +const byName = buildEpgNameIndex(options); +const idSet = new Set(options.map((o) => o.id)); + +assert.equal( + matchEpgChannelId({ name: "NPO 1", externalId: "141430714", provider: "odido" }, idSet, byName), + "odido/141430714" +); +assert.equal( + matchEpgChannelId({ name: "ESPN", externalId: "141429798", provider: "odido" }, idSet, byName), + "odido/141429798" +); +assert.equal( + matchEpgChannelId({ name: "NPO 1", externalId: null, provider: null }, idSet, byName), + "odido/141430714" +); + +console.log("epg-match tests ok"); diff --git a/apps/master-api/src/viewer/epg-match.ts b/apps/master-api/src/viewer/epg-match.ts new file mode 100644 index 0000000..f1b1177 --- /dev/null +++ b/apps/master-api/src/viewer/epg-match.ts @@ -0,0 +1,118 @@ +/** + * Match live-lijst zenders → XMLTV channel id over alle geconfigureerde EPG-bronnen. + * Preferentie: exacte id (odido/) → contentId → genormaliseerde display-name. + */ + +export type EpgChannelOption = { id: string; name: string }; + +export function normalizeEpgLabel(raw: string | null | undefined): string { + let s = String(raw ?? "") + .toLowerCase() + .normalize("NFD") + .replace(/\p{M}/gu, ""); + // "1. NPO 1" / "150. ESPN" + s = s.replace(/^\d+\.\s*/, ""); + s = s.replace(/\b(hd|fhd|uhd|4k|hevc|hdr|sd)\b/g, " "); + s = s.replace(/&/g, " and "); + s = s.replace(/[^a-z0-9+]+/g, ""); + return s; +} + +/** Bouw lookup: genormaliseerde naam → voorkeurs-id (stabiele eerste winnaar). */ +export function buildEpgNameIndex( + options: Iterable, + preferIdPrefix?: string +): Map { + const byName = new Map(); + const prefer = preferIdPrefix?.toLowerCase(); + const preferBetter = (existing: string, next: string) => { + if (!prefer) return false; + const e = existing.toLowerCase().startsWith(prefer); + const n = next.toLowerCase().startsWith(prefer); + return n && !e; + }; + for (const { id, name } of options) { + const keys = new Set(); + const nName = normalizeEpgLabel(name); + const nId = normalizeEpgLabel(id); + if (nName) keys.add(nName); + if (nId) keys.add(nId); + const bare = id.includes("/") ? id.slice(id.lastIndexOf("/") + 1) : ""; + if (bare) { + const nBare = normalizeEpgLabel(bare); + if (nBare) keys.add(nBare); + } + for (const key of keys) { + const prev = byName.get(key); + if (!prev || preferBetter(prev, id)) byName.set(key, id); + } + } + return byName; +} + +export type MatchEpgInput = { + name: string; + externalId?: string | null; + contentId?: string | null; + /** provider hint: bij "odido" eerst odido/ proberen */ + provider?: string | null; +}; + +/** + * Zoek beste XMLTV-id. `idSet` bevat alle bekende ids (case-sensitive + lower aliases ok). + * `byName` uit buildEpgNameIndex. + */ +export function matchEpgChannelId( + input: MatchEpgInput, + idSet: Set, + byName: Map +): string | null { + const candidates: string[] = []; + const externalId = input.externalId?.trim(); + const contentId = input.contentId?.trim(); + const provider = (input.provider ?? "").trim().toLowerCase(); + + if (externalId) { + if (provider === "odido" || !provider) { + candidates.push(`odido/${externalId}`); + } + candidates.push(externalId); + } + if (contentId) { + if (provider === "odido" || !provider) { + candidates.push(`odido/${contentId}`); + } + candidates.push(contentId); + } + + for (const c of candidates) { + if (idSet.has(c)) return c; + const lower = c.toLowerCase(); + if (lower !== c && idSet.has(lower)) return lower; + // case-insensitive scan + for (const id of idSet) { + if (id.toLowerCase() === lower) return id; + } + } + + const nameKey = normalizeEpgLabel(input.name); + if (nameKey && byName.has(nameKey)) return byName.get(nameKey)!; + + // Lichte fuzzy: name bevat of zit in EPG-label (alleen bij unieke hit) + if (nameKey && nameKey.length >= 4) { + const hits: string[] = []; + for (const [key, id] of byName) { + if (key === nameKey) return id; + if (key.includes(nameKey) || nameKey.includes(key)) hits.push(id); + } + const unique = [...new Set(hits)]; + if (unique.length === 1) return unique[0]; + // Bij Odido: prefer odido/-ids + if (provider === "odido") { + const odidoHits = unique.filter((id) => id.toLowerCase().startsWith("odido/")); + if (odidoHits.length === 1) return odidoHits[0]; + } + } + + return null; +} diff --git a/apps/master-api/src/viewer/live-channels.ts b/apps/master-api/src/viewer/live-channels.ts index edfe125..77df15c 100644 --- a/apps/master-api/src/viewer/live-channels.ts +++ b/apps/master-api/src/viewer/live-channels.ts @@ -604,15 +604,72 @@ export async function getCustomGuideChannels( }; } -/** Admin EPG-picker: XMLTV channel id + display name. */ +/** Admin EPG-picker: XMLTV channel id + display name (alle geconfigureerde bronnen). */ export async function listEpgChannelOptions(config: Config) { - await applyIptvEpgUrlsToRuntime(config); const index = await loadXmltvChannelIndex(config); return [...index.entries()] .map(([id, name]) => ({ id, name })) .sort((a, b) => a.name.localeCompare(b.name, "nl")); } +/** + * Volledige EPG channel-index over alle XMLTV-bronnen (iptv epgXmlUrls + XUI indien aanwezig). + * Exporteerbaar voor Odido/auto-match. + */ +export async function loadXmltvChannelIndex(config: Config): Promise> { + await applyIptvEpgUrlsToRuntime(config); + const settings = await getIptvSettings(config); + const urls = [...settings.epgXmlUrls]; + + const index = new Map(); + const mergeParsed = (parsed: Map) => { + for (const [id, name] of parsed) { + if (!index.has(id)) index.set(id, name); + const lower = id.toLowerCase(); + if (lower !== id && !index.has(lower)) index.set(lower, name); + } + }; + + for (const url of urls) { + try { + const xml = await fetchXmltvText([url]); + mergeParsed(parseXmltvChannelIndex(xml)); + } catch { + /* skip broken source — andere bronnen blijven gelden */ + } + } + + // XUI xmltv.php meenemen als er een live-lijn is (extra bron naast externe feeds) + try { + const line = await prisma.iptvLine.findFirst({ + where: { enableLive: true }, + orderBy: { updatedAt: "desc" }, + select: { id: true }, + }); + if (line) { + const xt = await loadXtreamByLineId(line.id, config); + if (xt) { + const guides = await xt.client.getXmltvGuidesByChannelId().catch(() => null); + if (guides) { + for (const id of guides.keys()) { + if (!index.has(id)) index.set(id, id); + } + } + } + } + } catch { + /* XUI optioneel */ + } + + if (index.size === 0) { + const guides = await loadXmltvGuidesForEpg(config).catch(() => new Map()); + for (const id of guides.keys()) { + index.set(id, id); + } + } + return index; +} + const TV_LOGOS_GH = "https://api.github.com/repos/tv-logo/tv-logos/contents/countries/netherlands"; const TV_LOGOS_CDN = @@ -721,33 +778,6 @@ async function loadXmltvGuidesForEpg( return merged; } -async function loadXmltvChannelIndex(config: Config): Promise> { - const settings = await getIptvSettings(config); - const urls = [...settings.epgXmlUrls]; - - const index = new Map(); - for (const url of urls) { - try { - const xml = await fetchXmltvText([url]); - const parsed = parseXmltvChannelIndex(xml); - for (const [id, name] of parsed) { - if (!index.has(id)) index.set(id, name); - } - } catch { - /* skip */ - } - } - - // Fallback: keys from guides without nice names - if (index.size === 0) { - const guides = await loadXmltvGuidesForEpg(config).catch(() => new Map()); - for (const id of guides.keys()) { - index.set(id, id); - } - } - return index; -} - /** Lightweight check used by IPTV status. */ export async function countCustomLiveForViewer(viewerId: string): Promise { return prisma.liveChannel.count({