feat(server): hardware HDR tonemapping for RKMPP (#7655)

* feat(server): hardware HDR tonemapping for RKMPP

* review feedback

docker/hwaccel.transcoding.yml

@@ -1,9 +1,9 @@
 version: "3.8"
 
-# Configurations for hardware-accelerated transcoding 
+# Configurations for hardware-accelerated transcoding
 
 # If using Unraid or another platform that doesn't allow multiple Compose files,
-# you can inline the config for a backend by copying its contents 
+# you can inline the config for a backend by copying its contents
 # into the immich-microservices service in the docker-compose.yml file.
 
 # See https://immich.app/docs/features/hardware-transcoding for more info on using hardware transcoding.
@@ -38,6 +38,10 @@ services:
       - /dev/dri:/dev/dri
       - /dev/dma_heap:/dev/dma_heap
       - /dev/mpp_service:/dev/mpp_service
+      #- /dev/mali0:/dev/mali0 # only required to enable OpenCL-accelerated HDR -> SDR tonemapping
+    volumes:
+      #- /etc/OpenCL:/etc/OpenCL:ro # only required to enable OpenCL-accelerated HDR -> SDR tonemapping
+      #- /usr/lib/aarch64-linux-gnu/libmali.so.1:/usr/lib/aarch64-linux-gnu/libmali.so.1:ro # only required to enable OpenCL-accelerated HDR -> SDR tonemapping
 
   vaapi:
     devices:

docs/docs/features/hardware-transcoding.md

@@ -42,6 +42,18 @@ You do not need to redo any transcoding jobs after enabling hardware acceleratio
   - If you have an 11th gen CPU or older, then you may need to follow [these][jellyfin-lp] instructions as Low-Power mode is required
   - Additionally, if the server specifically has an 11th gen CPU and is running kernel 5.15 (shipped with Ubuntu 22.04 LTS), then you will need to upgrade this kernel (from [Jellyfin docs][jellyfin-kernel-bug])
 
+#### RKMPP
+
+For RKMPP to work:
+
+- You must have a supported Rockchip ARM SoC.
+- Only RK3588 supports hardware tonemapping, other SoCs use slower software tonemapping while still using hardware encoding.
+- Tonemapping requires `/usr/lib/aarch64-linux-gnu/libmali.so.1` to be present on your host system. Install [`libmali-valhall-g610-g6p0-gbm`][libmali-rockchip] and modify the [`hwaccel.transcoding.yml`][hw-file] file:
+  - under `rkmpp` uncomment the 3 lines required for OpenCL tonemapping by removing the `#` symbol at the beginning of each line
+  - `- /dev/mali0:/dev/mali0`
+  - `- /etc/OpenCL:/etc/OpenCL:ro`
+  - `- /usr/lib/aarch64-linux-gnu/libmali.so.1:/usr/lib/aarch64-linux-gnu/libmali.so.1:ro`
+
 ## Setup
 
 #### Basic Setup
@@ -106,3 +118,4 @@ Once this is done, you can continue to step 3 of "Basic Setup".
 [nvcr]: https://github.com/NVIDIA/nvidia-container-runtime/
 [jellyfin-lp]: https://jellyfin.org/docs/general/administration/hardware-acceleration/intel/#configure-and-verify-lp-mode-on-linux
 [jellyfin-kernel-bug]: https://jellyfin.org/docs/general/administration/hardware-acceleration/intel/#known-issues-and-limitations
+[libmali-rockchip]: https://github.com/tsukumijima/libmali-rockchip/releases

server/src/domain/media/media.service.spec.ts

@@ -23,6 +23,7 @@ import {
   personStub,
   probeStub,
 } from '@test';
+import { Stats } from 'node:fs';
 import { JobName } from '../job';
 import {
   IAssetRepository,
@@ -1853,6 +1854,41 @@ describe(MediaService.name, () => {
         },
       );
     });
+
+    it('should set OpenCL tonemapping options for rkmpp when OpenCL is available', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      storageMock.stat.mockResolvedValue({ ...new Stats(), isFile: () => true, isCharacterDevice: () => true });
+      mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
+        { key: SystemConfigKey.FFMPEG_CRF, value: 30 },
+        { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '0' },
+      ]);
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        {
+          inputOptions: ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga'],
+          outputOptions: [
+            `-c:v h264_rkmpp`,
+            '-c:a copy',
+            '-movflags faststart',
+            '-fps_mode passthrough',
+            '-map 0:0',
+            '-map 0:1',
+            '-g 256',
+            '-v verbose',
+            '-vf scale_rkrga=-2:720:format=p010:afbc=1,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime',
+            '-level 51',
+            '-rc_mode CQP',
+            '-qp_init 30',
+          ],
+          twoPass: false,
+        },
+      );
+    });
   });
 
   it('should tonemap when policy is required and video is hdr', async () => {

server/src/domain/media/media.service.ts

@@ -47,6 +47,7 @@ export class MediaService {
   private logger = new ImmichLogger(MediaService.name);
   private configCore: SystemConfigCore;
   private storageCore: StorageCore;
+  private hasOpenCL?: boolean = undefined;
 
   constructor(
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
@@ -456,8 +457,19 @@ export class MediaService {
         break;
       }
       case TranscodeHWAccel.RKMPP: {
+        if (this.hasOpenCL === undefined) {
+          try {
+            const maliIcdStat = await this.storageRepository.stat('/etc/OpenCL/vendors/mali.icd');
+            const maliDeviceStat = await this.storageRepository.stat('/dev/mali0');
+            this.hasOpenCL = maliIcdStat.isFile() && maliDeviceStat.isCharacterDevice();
+          } catch {
+            this.logger.warn('OpenCL not available for transcoding, using CPU instead.');
+            this.hasOpenCL = false;
+          }
+        }
+
         devices = await this.storageRepository.readdir('/dev/dri');
-        handler = new RKMPPConfig(config, devices);
+        handler = new RKMPPConfig(config, devices, this.hasOpenCL);
         break;
       }
       default: {

server/src/domain/media/media.util.ts

@@ -608,6 +608,17 @@ export class VAAPIConfig extends BaseHWConfig {
 }
 
 export class RKMPPConfig extends BaseHWConfig {
+  private hasOpenCL: boolean;
+
+  constructor(
+    protected config: SystemConfigFFmpegDto,
+    devices: string[] = [],
+    hasOpenCL: boolean = false,
+  ) {
+    super(config, devices);
+    this.hasOpenCL = hasOpenCL;
+  }
+
   eligibleForTwoPass(): boolean {
     return false;
   }
@@ -616,19 +627,25 @@ export class RKMPPConfig extends BaseHWConfig {
     if (this.devices.length === 0) {
       throw new Error('No RKMPP device found');
     }
-    if (this.shouldToneMap(videoStream)) {
-      // disable hardware decoding
-      return [];
-    }
-    return ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga'];
+    return this.shouldToneMap(videoStream) && !this.hasOpenCL
+      ? [] // disable hardware decoding & filters
+      : ['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga'];
   }
 
   getFilterOptions(videoStream: VideoStreamInfo) {
     if (this.shouldToneMap(videoStream)) {
-      // use software filter options
-      return super.getFilterOptions(videoStream);
-    }
-    if (this.shouldScale(videoStream)) {
+      if (!this.hasOpenCL) {
+        return super.getFilterOptions(videoStream);
+      }
+      const colors = this.getColors();
+      return [
+        `scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1`,
+        'hwmap=derive_device=opencl:mode=read',
+        `tonemap_opencl=format=nv12:r=pc:p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:tonemap=${this.config.tonemap}:desat=0`,
+        'hwmap=derive_device=rkmpp:mode=write:reverse=1',
+        'format=drm_prime',
+      ];
+    } else if (this.shouldScale(videoStream)) {
       return [`scale_rkrga=${this.getScaling(videoStream)}:format=nv12:afbc=1`];
     }
     return [];