Fix storage double-count, add incremental catch-up scan, improve 4K detection.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Jos Vooges | STH 2026-08-27 07:39:43 +02:00
parent a26d1a4c45
commit 106aa996a4
9 changed files with 147 additions and 21 deletions

View file

@ -39,8 +39,15 @@ export async function registerAdminRoutes(app: FastifyInstance, config: Config)
prisma.episode.count(), prisma.episode.count(),
prisma.playbackSession.count({ where: { status: "ACTIVE", revoked: false } }), prisma.playbackSession.count({ where: { status: "ACTIVE", revoked: false } }),
prisma.node.findMany({ prisma.node.findMany({
where: { revoked: false }, where: { revoked: false, status: "ONLINE" },
select: { totalStorage: true, freeStorage: true, currentBandwidth: true, libraryFileCount: true }, select: {
id: true,
locationId: true,
totalStorage: true,
freeStorage: true,
currentBandwidth: true,
libraryFileCount: true,
},
}), }),
prisma.movie.count({ where: { mediaFiles: { some: { available: true } } } }), prisma.movie.count({ where: { mediaFiles: { some: { available: true } } } }),
prisma.series.count({ prisma.series.count({
@ -64,8 +71,21 @@ export async function registerAdminRoutes(app: FastifyInstance, config: Config)
}), }),
]); ]);
const totalStorage = nodes.reduce((sum, n) => sum + n.totalStorage, 0n); // Per locatie één keer tellen (nodes op dezelfde NAS rapporteren hetzelfde volume).
const freeStorage = nodes.reduce((sum, n) => sum + n.freeStorage, 0n); const storageByKey = new Map<string, { total: bigint; free: bigint }>();
for (const n of nodes) {
const key = n.locationId ?? n.id;
const cur = storageByKey.get(key);
if (!cur || n.totalStorage > cur.total) {
storageByKey.set(key, { total: n.totalStorage, free: n.freeStorage });
}
}
let totalStorage = 0n;
let freeStorage = 0n;
for (const s of storageByKey.values()) {
totalStorage += s.total;
freeStorage += s.free;
}
const bandwidth = nodes.reduce((sum, n) => sum + n.currentBandwidth, 0n); const bandwidth = nodes.reduce((sum, n) => sum + n.currentBandwidth, 0n);
const scannedFiles = nodes.reduce((sum, n) => sum + n.libraryFileCount, 0); const scannedFiles = nodes.reduce((sum, n) => sum + n.libraryFileCount, 0);

View file

@ -409,6 +409,7 @@ async function getMovieCatalog(
OR: [ OR: [
{ resolution: { contains: "2160" } }, { resolution: { contains: "2160" } },
{ resolution: { contains: "4K", mode: "insensitive" } }, { resolution: { contains: "4K", mode: "insensitive" } },
{ resolution: { contains: "UHD", mode: "insensitive" } },
], ],
}, },
}, },

View file

@ -7,8 +7,8 @@ WORKDIR /src
RUN apk add --no-cache git ca-certificates RUN apk add --no-cache git ca-certificates
COPY node/media-node/ ./ COPY node/media-node/ ./
RUN go mod tidy \ RUN go mod tidy \
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=1.2.6" -o /media-node-linux-amd64 ./cmd/media-node \ && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=1.2.7" -o /media-node-linux-amd64 ./cmd/media-node \
&& CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=1.2.6" -o /media-node-linux-arm64 ./cmd/media-node && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=1.2.7" -o /media-node-linux-arm64 ./cmd/media-node
FROM base AS builder FROM base AS builder
COPY package.json pnpm-workspace.yaml ./ COPY package.json pnpm-workspace.yaml ./

View file

@ -4,12 +4,19 @@ package health
import "syscall" import "syscall"
func diskStat(path string) (total, free uint64) { type fsKey struct {
typ int32
val0 int32
val1 int32
}
func diskStat(path string) (total, free uint64, fs fsKey, ok bool) {
var st syscall.Statfs_t var st syscall.Statfs_t
if err := syscall.Statfs(path, &st); err != nil { if err := syscall.Statfs(path, &st); err != nil {
return 0, 0 return 0, 0, fsKey{}, false
} }
fs = fsKey{typ: st.Type, val0: st.Fsid.X__val[0], val1: st.Fsid.X__val[1]}
total = st.Blocks * uint64(st.Bsize) total = st.Blocks * uint64(st.Bsize)
free = st.Bavail * uint64(st.Bsize) free = st.Bavail * uint64(st.Bsize)
return total, free return total, free, fs, true
} }

View file

@ -2,6 +2,6 @@
package health package health
func diskStat(_ string) (total, free uint64) { func diskStat(_ string) (total, free uint64, _ fsKey, ok bool) {
return 0, 0 return 0, 0, fsKey{}, false
} }

View file

@ -28,17 +28,25 @@ func MemoryUsagePercent() float64 {
} }
func DiskUsage(paths []string) (total, free uint64) { func DiskUsage(paths []string) (total, free uint64) {
seen := map[string]bool{} seenPaths := map[string]bool{}
seenFS := map[fsKey]bool{}
for _, p := range paths { for _, p := range paths {
if p == "" { if p == "" {
continue continue
} }
key := filepath.Clean(p) key := filepath.Clean(p)
if seen[key] { if seenPaths[key] {
continue continue
} }
seen[key] = true seenPaths[key] = true
t, f := diskStat(p) t, f, fsid, ok := diskStat(p)
if !ok {
continue
}
if seenFS[fsid] {
continue
}
seenFS[fsid] = true
total += t total += t
free += f free += f
} }

View file

@ -10,7 +10,7 @@ var (
yearSepRe = regexp.MustCompile(`[\.\s_-]((19|20)\d{2})[\.\s_-]`) yearSepRe = regexp.MustCompile(`[\.\s_-]((19|20)\d{2})[\.\s_-]`)
yearParenRe = regexp.MustCompile(`(?i)^(.+?)\s*\(((?:19|20)\d{2})\)\s*(?:\[.*\])?$`) yearParenRe = regexp.MustCompile(`(?i)^(.+?)\s*\(((?:19|20)\d{2})\)\s*(?:\[.*\])?$`)
yearEndRe = regexp.MustCompile(`^(.+?)[\.\s_-]((?:19|20)\d{2})$`) yearEndRe = regexp.MustCompile(`^(.+?)[\.\s_-]((?:19|20)\d{2})$`)
resolutionRe = regexp.MustCompile(`(?i)\b(2160p|1080p|720p|480p|4K)\b`) resolutionRe = regexp.MustCompile(`(?i)\b(2160p|1080p|720p|480p|4K|UHD)\b`)
sourceRe = regexp.MustCompile(`(?i)\b(UHD\s*BluRay|BluRay|WEB-DL|WEBRip|HDTV|DVDRip)\b`) sourceRe = regexp.MustCompile(`(?i)\b(UHD\s*BluRay|BluRay|WEB-DL|WEBRip|HDTV|DVDRip)\b`)
releaseRe = regexp.MustCompile(`(?i)\b(REMUX|PROPER|REPACK)\b`) releaseRe = regexp.MustCompile(`(?i)\b(REMUX|PROPER|REPACK)\b`)
videoRe = regexp.MustCompile(`(?i)\b(HEVC|H\.?264|H\.?265|x264|x265|AV1|XviD)\b`) videoRe = regexp.MustCompile(`(?i)\b(HEVC|H\.?264|H\.?265|x264|x265|AV1|XviD)\b`)
@ -205,8 +205,15 @@ func stripYearParen(s string) string {
func attachTags(result map[string]interface{}, base string) { func attachTags(result map[string]interface{}, base string) {
if m := resolutionRe.FindStringSubmatch(base); len(m) > 1 { if m := resolutionRe.FindStringSubmatch(base); len(m) > 1 {
res := strings.ToUpper(m[1])
if res == "UHD" || res == "4K" {
result["resolution"] = "2160p"
} else {
result["resolution"] = m[1] result["resolution"] = m[1]
} }
} else if strings.Contains(strings.ToUpper(base), "UHD") {
result["resolution"] = "2160p"
}
if m := sourceRe.FindStringSubmatch(base); len(m) > 1 { if m := sourceRe.FindStringSubmatch(base); len(m) > 1 {
result["source"] = m[1] result["source"] = m[1]
} }

View file

@ -13,6 +13,19 @@ func TestParseMovie(t *testing.T) {
if p["year"] != 1999 { if p["year"] != 1999 {
t.Fatalf("year=%v", p["year"]) t.Fatalf("year=%v", p["year"])
} }
if p["resolution"] != "2160p" {
t.Fatalf("resolution=%v", p["resolution"])
}
}
func TestParseMovieUHDOnly(t *testing.T) {
p := parseMovie("Avatar.UHD.BluRay.mkv")
if p == nil {
t.Fatal("expected parse result")
}
if p["resolution"] != "2160p" {
t.Fatalf("resolution=%v", p["resolution"])
}
} }
func TestParseMovieParenFolder(t *testing.T) { func TestParseMovieParenFolder(t *testing.T) {

View file

@ -96,6 +96,7 @@ func (s *Scanner) Start() {
go s.runStartupScan() go s.runStartupScan()
go s.runNightlyFullScan() go s.runNightlyFullScan()
go s.runIntervalFullScan() go s.runIntervalFullScan()
go s.runIncrementalCatchUp()
go s.watchFilesystem() go s.watchFilesystem()
} }
@ -246,6 +247,66 @@ func nextLocalClock(now time.Time, hour, minute int) time.Time {
return next return next
} }
const incrementalCatchUpInterval = 10 * time.Minute
// runIncrementalCatchUp picks up new/changed files when fsnotify misses events
// (common with SMB uploads or inotify limits on large libraries).
func (s *Scanner) runIncrementalCatchUp() {
timer := time.NewTimer(incrementalCatchUpInterval)
defer timer.Stop()
for {
<-timer.C
s.incrementalScan()
timer.Reset(incrementalCatchUpInterval)
}
}
func (s *Scanner) incrementalScan() {
s.mu.Lock()
if s.scanning {
s.mu.Unlock()
return
}
roots := s.cfg.Media.EffectiveRoots()
s.mu.Unlock()
if len(roots) == 0 {
return
}
newCount := 0
for _, root := range roots {
kind := root.Kind
if kind == "series" {
kind = "episode"
}
if root.Path == "" {
continue
}
_ = filepath.Walk(root.Path, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() || !isVideo(path) {
return nil
}
existing, _ := s.store.GetFileByPath(path)
if existing != nil &&
existing.ModifiedAt.Equal(info.ModTime()) &&
existing.SizeBytes == info.Size() {
return nil
}
eventType := "FILE_ADDED"
if existing != nil {
eventType = "FILE_UPDATED"
}
shelf := shelfForPath(path, roots)
s.indexFile(path, info, eventType, kind, shelf, true)
newCount++
return nil
})
}
if newCount > 0 {
log.Printf("Incremental scan indexed %d new/changed file(s)", newCount)
}
}
func (s *Scanner) FullScan() { func (s *Scanner) FullScan() {
for { for {
s.mu.Lock() s.mu.Lock()
@ -358,6 +419,7 @@ func (s *Scanner) runWatchSession() {
rootType := map[string]string{} rootType := map[string]string{}
rootShelf := map[string]string{} rootShelf := map[string]string{}
watchDirs := 0
addTree := func(dir, kind, shelfID string) { addTree := func(dir, kind, shelfID string) {
if dir == "" { if dir == "" {
return return
@ -369,7 +431,11 @@ func (s *Scanner) runWatchSession() {
} }
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err == nil && info.IsDir() { if err == nil && info.IsDir() {
_ = watcher.Add(path) if err := watcher.Add(path); err != nil {
log.Printf("watch add failed for %s: %v", path, err)
} else {
watchDirs++
}
} }
return nil return nil
}) })
@ -386,7 +452,7 @@ func (s *Scanner) runWatchSession() {
} }
addTree(root.Path, kind, root.ShelfID) addTree(root.Path, kind, root.ShelfID)
} }
log.Printf("Filesystem watch active on %d roots", len(roots)) log.Printf("Filesystem watch active on %d roots (%d directories)", len(roots), watchDirs)
debounce := time.NewTimer(0) debounce := time.NewTimer(0)
<-debounce.C <-debounce.C
@ -407,13 +473,17 @@ func (s *Scanner) runWatchSession() {
if event.Op&fsnotify.Create == fsnotify.Create { if event.Op&fsnotify.Create == fsnotify.Create {
if info, err := os.Stat(name); err == nil && info.IsDir() { if info, err := os.Stat(name); err == nil && info.IsDir() {
_ = watcher.Add(name) if err := watcher.Add(name); err != nil {
log.Printf("watch add failed for %s: %v", name, err)
}
filepath.Walk(name, func(path string, info os.FileInfo, err error) error { filepath.Walk(name, func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
return nil return nil
} }
if info.IsDir() { if info.IsDir() {
_ = watcher.Add(path) if err := watcher.Add(path); err != nil {
log.Printf("watch add failed for %s: %v", path, err)
}
return nil return nil
} }
if isVideo(path) { if isVideo(path) {