From 351dd647a9bb8c4c192e22832233b76652c452a7 Mon Sep 17 00:00:00 2001
From: Mert <101130780+mertalev@users.noreply.github.com>
Date: Mon, 27 May 2024 12:08:38 -0400
Subject: [PATCH] feat(server): better video thumbnails (#9784)

---
 server/src/services/media.service.spec.ts | 18 ++++++++++++------
 server/src/utils/media.ts                 | 12 ++++++++----
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts
index 9fe0038232..5ff1b135dd 100644
--- a/server/src/services/media.service.spec.ts
+++ b/server/src/services/media.service.spec.ts
@@ -294,11 +294,13 @@ describe(MediaService.name, () => {
         '/original/path.ext',
         'upload/thumbs/user-id/as/se/asset-id-preview.jpeg',
         {
-          inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
+          inputOptions: ['-skip_frame nokey', '-sws_flags accurate_rnd+full_chroma_int'],
           outputOptions: [
+            '-fps_mode vfr',
             '-frames:v 1',
+            '-update 1',
             '-v verbose',
-            '-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p',
+            `-vf fps=12,thumbnail=12,select=gt(scene\\,0.1)+gt(n\\,20),scale=-2:1440:flags=lanczos+accurate_rnd+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p`,
           ],
           twoPass: false,
         },
@@ -319,11 +321,13 @@ describe(MediaService.name, () => {
         '/original/path.ext',
         'upload/thumbs/user-id/as/se/asset-id-preview.jpeg',
         {
-          inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
+          inputOptions: ['-skip_frame nokey', '-sws_flags accurate_rnd+full_chroma_int'],
           outputOptions: [
+            '-fps_mode vfr',
             '-frames:v 1',
+            '-update 1',
             '-v verbose',
-            '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p',
+            `-vf fps=12,thumbnail=12,select=gt(scene\\,0.1)+gt(n\\,20),zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p`,
           ],
           twoPass: false,
         },
@@ -346,11 +350,13 @@ describe(MediaService.name, () => {
         '/original/path.ext',
         'upload/thumbs/user-id/as/se/asset-id-preview.jpeg',
         {
-          inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
+          inputOptions: ['-skip_frame nokey', '-sws_flags accurate_rnd+full_chroma_int'],
           outputOptions: [
+            '-fps_mode vfr',
             '-frames:v 1',
+            '-update 1',
             '-v verbose',
-            '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p',
+            `-vf fps=12,thumbnail=12,select=gt(scene\\,0.1)+gt(n\\,20),zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p`,
           ],
           twoPass: false,
         },
diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts
index 5a57f0f0cf..419b7e2d60 100644
--- a/server/src/utils/media.ts
+++ b/server/src/utils/media.ts
@@ -319,10 +319,15 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
 
 export class ThumbnailConfig extends BaseConfig {
   getBaseInputOptions(): string[] {
-    return ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'];
+    return ['-skip_frame nokey', '-sws_flags accurate_rnd+full_chroma_int'];
   }
+
   getBaseOutputOptions() {
-    return ['-frames:v 1'];
+    return ['-fps_mode vfr', '-frames:v 1', '-update 1'];
+  }
+
+  getFilterOptions(videoStream: VideoStreamInfo): string[] {
+    return ['fps=12', 'thumbnail=12', `select=gt(scene\\,0.1)+gt(n\\,20)`, ...super.getFilterOptions(videoStream)];
   }
 
   getPresetOptions() {
@@ -338,8 +343,7 @@ export class ThumbnailConfig extends BaseConfig {
   }
 
   getScaling(videoStream: VideoStreamInfo) {
-    let options = super.getScaling(videoStream);
-    options += ':flags=lanczos+accurate_rnd+bitexact+full_chroma_int';
+    let options = super.getScaling(videoStream) + ':flags=lanczos+accurate_rnd+full_chroma_int';
     if (!this.shouldToneMap(videoStream)) {
       options += ':out_color_matrix=601:out_range=pc';
     }