Fix storage double-count, add incremental catch-up scan, improve 4K detection.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
a26d1a4c45
commit
106aa996a4
9 changed files with 147 additions and 21 deletions
|
|
@ -39,8 +39,15 @@ export async function registerAdminRoutes(app: FastifyInstance, config: Config)
|
||||||
prisma.episode.count(),
|
prisma.episode.count(),
|
||||||
prisma.playbackSession.count({ where: { status: "ACTIVE", revoked: false } }),
|
prisma.playbackSession.count({ where: { status: "ACTIVE", revoked: false } }),
|
||||||
prisma.node.findMany({
|
prisma.node.findMany({
|
||||||
where: { revoked: false },
|
where: { revoked: false, status: "ONLINE" },
|
||||||
select: { totalStorage: true, freeStorage: true, currentBandwidth: true, libraryFileCount: true },
|
select: {
|
||||||
|
id: true,
|
||||||
|
locationId: true,
|
||||||
|
totalStorage: true,
|
||||||
|
freeStorage: true,
|
||||||
|
currentBandwidth: true,
|
||||||
|
libraryFileCount: true,
|
||||||
|
},
|
||||||
}),
|
}),
|
||||||
prisma.movie.count({ where: { mediaFiles: { some: { available: true } } } }),
|
prisma.movie.count({ where: { mediaFiles: { some: { available: true } } } }),
|
||||||
prisma.series.count({
|
prisma.series.count({
|
||||||
|
|
@ -64,8 +71,21 @@ export async function registerAdminRoutes(app: FastifyInstance, config: Config)
|
||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const totalStorage = nodes.reduce((sum, n) => sum + n.totalStorage, 0n);
|
// Per locatie één keer tellen (nodes op dezelfde NAS rapporteren hetzelfde volume).
|
||||||
const freeStorage = nodes.reduce((sum, n) => sum + n.freeStorage, 0n);
|
const storageByKey = new Map<string, { total: bigint; free: bigint }>();
|
||||||
|
for (const n of nodes) {
|
||||||
|
const key = n.locationId ?? n.id;
|
||||||
|
const cur = storageByKey.get(key);
|
||||||
|
if (!cur || n.totalStorage > cur.total) {
|
||||||
|
storageByKey.set(key, { total: n.totalStorage, free: n.freeStorage });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let totalStorage = 0n;
|
||||||
|
let freeStorage = 0n;
|
||||||
|
for (const s of storageByKey.values()) {
|
||||||
|
totalStorage += s.total;
|
||||||
|
freeStorage += s.free;
|
||||||
|
}
|
||||||
const bandwidth = nodes.reduce((sum, n) => sum + n.currentBandwidth, 0n);
|
const bandwidth = nodes.reduce((sum, n) => sum + n.currentBandwidth, 0n);
|
||||||
const scannedFiles = nodes.reduce((sum, n) => sum + n.libraryFileCount, 0);
|
const scannedFiles = nodes.reduce((sum, n) => sum + n.libraryFileCount, 0);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -409,6 +409,7 @@ async function getMovieCatalog(
|
||||||
OR: [
|
OR: [
|
||||||
{ resolution: { contains: "2160" } },
|
{ resolution: { contains: "2160" } },
|
||||||
{ resolution: { contains: "4K", mode: "insensitive" } },
|
{ resolution: { contains: "4K", mode: "insensitive" } },
|
||||||
|
{ resolution: { contains: "UHD", mode: "insensitive" } },
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -7,8 +7,8 @@ WORKDIR /src
|
||||||
RUN apk add --no-cache git ca-certificates
|
RUN apk add --no-cache git ca-certificates
|
||||||
COPY node/media-node/ ./
|
COPY node/media-node/ ./
|
||||||
RUN go mod tidy \
|
RUN go mod tidy \
|
||||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=1.2.6" -o /media-node-linux-amd64 ./cmd/media-node \
|
&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=1.2.7" -o /media-node-linux-amd64 ./cmd/media-node \
|
||||||
&& CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=1.2.6" -o /media-node-linux-arm64 ./cmd/media-node
|
&& CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=1.2.7" -o /media-node-linux-arm64 ./cmd/media-node
|
||||||
|
|
||||||
FROM base AS builder
|
FROM base AS builder
|
||||||
COPY package.json pnpm-workspace.yaml ./
|
COPY package.json pnpm-workspace.yaml ./
|
||||||
|
|
|
||||||
|
|
@ -4,12 +4,19 @@ package health
|
||||||
|
|
||||||
import "syscall"
|
import "syscall"
|
||||||
|
|
||||||
func diskStat(path string) (total, free uint64) {
|
type fsKey struct {
|
||||||
|
typ int32
|
||||||
|
val0 int32
|
||||||
|
val1 int32
|
||||||
|
}
|
||||||
|
|
||||||
|
func diskStat(path string) (total, free uint64, fs fsKey, ok bool) {
|
||||||
var st syscall.Statfs_t
|
var st syscall.Statfs_t
|
||||||
if err := syscall.Statfs(path, &st); err != nil {
|
if err := syscall.Statfs(path, &st); err != nil {
|
||||||
return 0, 0
|
return 0, 0, fsKey{}, false
|
||||||
}
|
}
|
||||||
|
fs = fsKey{typ: st.Type, val0: st.Fsid.X__val[0], val1: st.Fsid.X__val[1]}
|
||||||
total = st.Blocks * uint64(st.Bsize)
|
total = st.Blocks * uint64(st.Bsize)
|
||||||
free = st.Bavail * uint64(st.Bsize)
|
free = st.Bavail * uint64(st.Bsize)
|
||||||
return total, free
|
return total, free, fs, true
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,6 @@
|
||||||
|
|
||||||
package health
|
package health
|
||||||
|
|
||||||
func diskStat(_ string) (total, free uint64) {
|
func diskStat(_ string) (total, free uint64, _ fsKey, ok bool) {
|
||||||
return 0, 0
|
return 0, 0, fsKey{}, false
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -28,17 +28,25 @@ func MemoryUsagePercent() float64 {
|
||||||
}
|
}
|
||||||
|
|
||||||
func DiskUsage(paths []string) (total, free uint64) {
|
func DiskUsage(paths []string) (total, free uint64) {
|
||||||
seen := map[string]bool{}
|
seenPaths := map[string]bool{}
|
||||||
|
seenFS := map[fsKey]bool{}
|
||||||
for _, p := range paths {
|
for _, p := range paths {
|
||||||
if p == "" {
|
if p == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
key := filepath.Clean(p)
|
key := filepath.Clean(p)
|
||||||
if seen[key] {
|
if seenPaths[key] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seen[key] = true
|
seenPaths[key] = true
|
||||||
t, f := diskStat(p)
|
t, f, fsid, ok := diskStat(p)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if seenFS[fsid] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seenFS[fsid] = true
|
||||||
total += t
|
total += t
|
||||||
free += f
|
free += f
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ var (
|
||||||
yearSepRe = regexp.MustCompile(`[\.\s_-]((19|20)\d{2})[\.\s_-]`)
|
yearSepRe = regexp.MustCompile(`[\.\s_-]((19|20)\d{2})[\.\s_-]`)
|
||||||
yearParenRe = regexp.MustCompile(`(?i)^(.+?)\s*\(((?:19|20)\d{2})\)\s*(?:\[.*\])?$`)
|
yearParenRe = regexp.MustCompile(`(?i)^(.+?)\s*\(((?:19|20)\d{2})\)\s*(?:\[.*\])?$`)
|
||||||
yearEndRe = regexp.MustCompile(`^(.+?)[\.\s_-]((?:19|20)\d{2})$`)
|
yearEndRe = regexp.MustCompile(`^(.+?)[\.\s_-]((?:19|20)\d{2})$`)
|
||||||
resolutionRe = regexp.MustCompile(`(?i)\b(2160p|1080p|720p|480p|4K)\b`)
|
resolutionRe = regexp.MustCompile(`(?i)\b(2160p|1080p|720p|480p|4K|UHD)\b`)
|
||||||
sourceRe = regexp.MustCompile(`(?i)\b(UHD\s*BluRay|BluRay|WEB-DL|WEBRip|HDTV|DVDRip)\b`)
|
sourceRe = regexp.MustCompile(`(?i)\b(UHD\s*BluRay|BluRay|WEB-DL|WEBRip|HDTV|DVDRip)\b`)
|
||||||
releaseRe = regexp.MustCompile(`(?i)\b(REMUX|PROPER|REPACK)\b`)
|
releaseRe = regexp.MustCompile(`(?i)\b(REMUX|PROPER|REPACK)\b`)
|
||||||
videoRe = regexp.MustCompile(`(?i)\b(HEVC|H\.?264|H\.?265|x264|x265|AV1|XviD)\b`)
|
videoRe = regexp.MustCompile(`(?i)\b(HEVC|H\.?264|H\.?265|x264|x265|AV1|XviD)\b`)
|
||||||
|
|
@ -205,8 +205,15 @@ func stripYearParen(s string) string {
|
||||||
|
|
||||||
func attachTags(result map[string]interface{}, base string) {
|
func attachTags(result map[string]interface{}, base string) {
|
||||||
if m := resolutionRe.FindStringSubmatch(base); len(m) > 1 {
|
if m := resolutionRe.FindStringSubmatch(base); len(m) > 1 {
|
||||||
|
res := strings.ToUpper(m[1])
|
||||||
|
if res == "UHD" || res == "4K" {
|
||||||
|
result["resolution"] = "2160p"
|
||||||
|
} else {
|
||||||
result["resolution"] = m[1]
|
result["resolution"] = m[1]
|
||||||
}
|
}
|
||||||
|
} else if strings.Contains(strings.ToUpper(base), "UHD") {
|
||||||
|
result["resolution"] = "2160p"
|
||||||
|
}
|
||||||
if m := sourceRe.FindStringSubmatch(base); len(m) > 1 {
|
if m := sourceRe.FindStringSubmatch(base); len(m) > 1 {
|
||||||
result["source"] = m[1]
|
result["source"] = m[1]
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,19 @@ func TestParseMovie(t *testing.T) {
|
||||||
if p["year"] != 1999 {
|
if p["year"] != 1999 {
|
||||||
t.Fatalf("year=%v", p["year"])
|
t.Fatalf("year=%v", p["year"])
|
||||||
}
|
}
|
||||||
|
if p["resolution"] != "2160p" {
|
||||||
|
t.Fatalf("resolution=%v", p["resolution"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseMovieUHDOnly(t *testing.T) {
|
||||||
|
p := parseMovie("Avatar.UHD.BluRay.mkv")
|
||||||
|
if p == nil {
|
||||||
|
t.Fatal("expected parse result")
|
||||||
|
}
|
||||||
|
if p["resolution"] != "2160p" {
|
||||||
|
t.Fatalf("resolution=%v", p["resolution"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseMovieParenFolder(t *testing.T) {
|
func TestParseMovieParenFolder(t *testing.T) {
|
||||||
|
|
|
||||||
|
|
@ -96,6 +96,7 @@ func (s *Scanner) Start() {
|
||||||
go s.runStartupScan()
|
go s.runStartupScan()
|
||||||
go s.runNightlyFullScan()
|
go s.runNightlyFullScan()
|
||||||
go s.runIntervalFullScan()
|
go s.runIntervalFullScan()
|
||||||
|
go s.runIncrementalCatchUp()
|
||||||
go s.watchFilesystem()
|
go s.watchFilesystem()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -246,6 +247,66 @@ func nextLocalClock(now time.Time, hour, minute int) time.Time {
|
||||||
return next
|
return next
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const incrementalCatchUpInterval = 10 * time.Minute
|
||||||
|
|
||||||
|
// runIncrementalCatchUp picks up new/changed files when fsnotify misses events
|
||||||
|
// (common with SMB uploads or inotify limits on large libraries).
|
||||||
|
func (s *Scanner) runIncrementalCatchUp() {
|
||||||
|
timer := time.NewTimer(incrementalCatchUpInterval)
|
||||||
|
defer timer.Stop()
|
||||||
|
for {
|
||||||
|
<-timer.C
|
||||||
|
s.incrementalScan()
|
||||||
|
timer.Reset(incrementalCatchUpInterval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Scanner) incrementalScan() {
|
||||||
|
s.mu.Lock()
|
||||||
|
if s.scanning {
|
||||||
|
s.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
roots := s.cfg.Media.EffectiveRoots()
|
||||||
|
s.mu.Unlock()
|
||||||
|
if len(roots) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
newCount := 0
|
||||||
|
for _, root := range roots {
|
||||||
|
kind := root.Kind
|
||||||
|
if kind == "series" {
|
||||||
|
kind = "episode"
|
||||||
|
}
|
||||||
|
if root.Path == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_ = filepath.Walk(root.Path, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil || info.IsDir() || !isVideo(path) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
existing, _ := s.store.GetFileByPath(path)
|
||||||
|
if existing != nil &&
|
||||||
|
existing.ModifiedAt.Equal(info.ModTime()) &&
|
||||||
|
existing.SizeBytes == info.Size() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
eventType := "FILE_ADDED"
|
||||||
|
if existing != nil {
|
||||||
|
eventType = "FILE_UPDATED"
|
||||||
|
}
|
||||||
|
shelf := shelfForPath(path, roots)
|
||||||
|
s.indexFile(path, info, eventType, kind, shelf, true)
|
||||||
|
newCount++
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if newCount > 0 {
|
||||||
|
log.Printf("Incremental scan indexed %d new/changed file(s)", newCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Scanner) FullScan() {
|
func (s *Scanner) FullScan() {
|
||||||
for {
|
for {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
|
|
@ -358,6 +419,7 @@ func (s *Scanner) runWatchSession() {
|
||||||
|
|
||||||
rootType := map[string]string{}
|
rootType := map[string]string{}
|
||||||
rootShelf := map[string]string{}
|
rootShelf := map[string]string{}
|
||||||
|
watchDirs := 0
|
||||||
addTree := func(dir, kind, shelfID string) {
|
addTree := func(dir, kind, shelfID string) {
|
||||||
if dir == "" {
|
if dir == "" {
|
||||||
return
|
return
|
||||||
|
|
@ -369,7 +431,11 @@ func (s *Scanner) runWatchSession() {
|
||||||
}
|
}
|
||||||
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||||
if err == nil && info.IsDir() {
|
if err == nil && info.IsDir() {
|
||||||
_ = watcher.Add(path)
|
if err := watcher.Add(path); err != nil {
|
||||||
|
log.Printf("watch add failed for %s: %v", path, err)
|
||||||
|
} else {
|
||||||
|
watchDirs++
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
@ -386,7 +452,7 @@ func (s *Scanner) runWatchSession() {
|
||||||
}
|
}
|
||||||
addTree(root.Path, kind, root.ShelfID)
|
addTree(root.Path, kind, root.ShelfID)
|
||||||
}
|
}
|
||||||
log.Printf("Filesystem watch active on %d roots", len(roots))
|
log.Printf("Filesystem watch active on %d roots (%d directories)", len(roots), watchDirs)
|
||||||
|
|
||||||
debounce := time.NewTimer(0)
|
debounce := time.NewTimer(0)
|
||||||
<-debounce.C
|
<-debounce.C
|
||||||
|
|
@ -407,13 +473,17 @@ func (s *Scanner) runWatchSession() {
|
||||||
|
|
||||||
if event.Op&fsnotify.Create == fsnotify.Create {
|
if event.Op&fsnotify.Create == fsnotify.Create {
|
||||||
if info, err := os.Stat(name); err == nil && info.IsDir() {
|
if info, err := os.Stat(name); err == nil && info.IsDir() {
|
||||||
_ = watcher.Add(name)
|
if err := watcher.Add(name); err != nil {
|
||||||
|
log.Printf("watch add failed for %s: %v", name, err)
|
||||||
|
}
|
||||||
filepath.Walk(name, func(path string, info os.FileInfo, err error) error {
|
filepath.Walk(name, func(path string, info os.FileInfo, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if info.IsDir() {
|
if info.IsDir() {
|
||||||
_ = watcher.Add(path)
|
if err := watcher.Add(path); err != nil {
|
||||||
|
log.Printf("watch add failed for %s: %v", path, err)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if isVideo(path) {
|
if isVideo(path) {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue