From f1ca1794a182d71e45340a262a450e694f4f1600 Mon Sep 17 00:00:00 2001 From: N00MKRAD <61149547+n00mkrad@users.noreply.github.com> Date: Thu, 11 Apr 2024 07:26:27 +0200 Subject: [PATCH] Add AV1 transcoding support (#8491) * Add AV1 transcoding support - AV1 encoding on CPU via SVT-AV1 (libsvtav1 in ffmpeg) - Supports CRF and optionally capped CRF (max bitrate) - Tested playback successfully in Chrome Win+Android, Firefox Win+Linux, Android app * AV1: Add support for encoding threads option * Revert previous commit; specifying params multiple times is bad We need to specify all svtav1-params at once, so putting the thread option into getThreadOptions is not possible. * AV1: Override VAAPI getSupportedCodecs as it does not yet support AV1 unlike nvenc, qsv, amf * Change BaseHWConfig supported codecs to only H264/HEVC Configs that support VP9 and/or AV1 need to override getSupportedCodecs() * Set SVT-AV1 threads with svtav1-params, remove duplicate block in NVENCConfig * AV1Config: Fix empty svtav1-params array being added to options * add tests * update api * allow crf-based two-pass mode * formatting * suggest 35 --------- Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> --- mobile/openapi/lib/model/video_codec.dart | Bin 2629 -> 2725 bytes open-api/immich-openapi-specs.json | 3 +- open-api/typescript-sdk/src/fetch-client.ts | 3 +- server/src/entities/system-config.entity.ts | 1 + server/src/services/media.service.spec.ts | 151 ++++++++++++++++++ server/src/services/media.service.ts | 4 + server/src/utils/.media.ts.kate-swp | Bin 0 -> 63 bytes server/src/utils/media.ts | 50 +++++- .../settings/ffmpeg/ffmpeg-settings.svelte | 6 +- 9 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 server/src/utils/.media.ts.kate-swp diff --git a/mobile/openapi/lib/model/video_codec.dart b/mobile/openapi/lib/model/video_codec.dart index 2cc18d1ae0c8beaa9d0aed229902ea895f90b29b..36e1c681a67a2d2e3aa55b52753baa26f0206f8c 100644 GIT binary patch delta 86 zcmX>qvQ%`#5+-#8h2oOLlFVd<)MUMQjUsiR;N%5N g9jpo<>CGz4GHhJQiN&c35Sfo0qRb$1GfsX+0N { ); }); + it('should use av1 if specified', async () => { + mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 }]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: [], + outputOptions: [ + '-c:v av1', + '-c:a copy', + '-movflags faststart', + '-fps_mode passthrough', + '-map 0:0', + '-map 0:1', + '-v verbose', + '-vf scale=-2:720,format=yuv420p', + '-preset 12', + '-crf 23', + ], + twoPass: false, + }, + ); + }); + + it('should map `veryslow` preset to 4 for av1', async () => { + mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 }, + { key: SystemConfigKey.FFMPEG_PRESET, value: 'veryslow' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: [], + outputOptions: [ + '-c:v av1', + '-c:a copy', + '-movflags faststart', + '-fps_mode passthrough', + '-map 0:0', + '-map 0:1', + '-v verbose', + '-vf scale=-2:720,format=yuv420p', + '-preset 4', + '-crf 23', + ], + twoPass: false, + }, + ); + }); + + it('should set max bitrate for av1 if specified', async () => { + mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 }, + { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '2M' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: [], + outputOptions: [ + '-c:v av1', + '-c:a copy', + '-movflags faststart', + '-fps_mode passthrough', + '-map 0:0', + '-map 0:1', + '-v verbose', + '-vf scale=-2:720,format=yuv420p', + '-preset 12', + '-crf 23', + '-svtav1-params mbr=2M', + ], + twoPass: false, + }, + ); + }); + + it('should set threads for av1 if specified', async () => { + mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 }, + { key: SystemConfigKey.FFMPEG_THREADS, value: 4 }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: [], + outputOptions: [ + '-c:v av1', + '-c:a copy', + '-movflags faststart', + '-fps_mode passthrough', + '-map 0:0', + '-map 0:1', + '-v verbose', + '-vf scale=-2:720,format=yuv420p', + '-preset 12', + '-crf 23', + '-svtav1-params lp=4', + ], + twoPass: false, + }, + ); + }); + + it('should set both bitrate and threads for av1 if specified', async () => { + mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9); + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 }, + { key: SystemConfigKey.FFMPEG_THREADS, value: 4 }, + { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '2M' }, + ]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: [], + outputOptions: [ + '-c:v av1', + '-c:a copy', + '-movflags faststart', + '-fps_mode passthrough', + '-map 0:0', + '-map 0:1', + '-v verbose', + '-vf scale=-2:720,format=yuv420p', + '-preset 12', + '-crf 23', + '-svtav1-params lp=4:mbr=2M', + ], + twoPass: false, + }, + ); + }); + it('should skip transcoding for audioless videos with optimal policy if video codec is correct', async () => { mediaMock.probe.mockResolvedValue(probeStub.noAudioStreams); configMock.load.mockResolvedValue([ diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index 8ca2486941..3c86c72bd7 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -32,6 +32,7 @@ import { IStorageRepository } from 'src/interfaces/storage.interface'; import { ISystemConfigRepository } from 'src/interfaces/system-config.interface'; import { ImmichLogger } from 'src/utils/logger'; import { + AV1Config, H264Config, HEVCConfig, NVENCConfig, @@ -439,6 +440,9 @@ export class MediaService { case VideoCodec.VP9: { return new VP9Config(config); } + case VideoCodec.AV1: { + return new AV1Config(config); + } default: { throw new UnsupportedMediaTypeException(`Codec '${config.targetVideoCodec}' is unsupported`); } diff --git a/server/src/utils/.media.ts.kate-swp b/server/src/utils/.media.ts.kate-swp new file mode 100644 index 0000000000000000000000000000000000000000..126d6519e091976b523282d8ddb054bca7bba4ad GIT binary patch literal 63 zcmZQzU=Z?7EJ;-eE>A2_aLdd|RWQ;sU|?Vn(KEYRC)O(eD*1`3x%rtpjgfMj?1McS Q7#LZBvU)(wq@dsm03f0d)Bpeg literal 0 HcmV?d00001 diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index 261977028a..d0a6d4d740 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -124,7 +124,7 @@ class BaseConfig implements VideoCodecSWConfig { return false; } - return this.isBitrateConstrained() || this.config.targetVideoCodec === VideoCodec.VP9; + return this.isBitrateConstrained(); } getBitrateDistribution() { @@ -265,7 +265,7 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { } getSupportedCodecs() { - return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9]; + return [VideoCodec.H264, VideoCodec.HEVC]; } validateDevices(devices: string[]) { @@ -394,6 +394,44 @@ export class VP9Config extends BaseConfig { getThreadOptions() { return ['-row-mt 1', ...super.getThreadOptions()]; } + + eligibleForTwoPass() { + return this.config.twoPass; + } +} + +export class AV1Config extends BaseConfig { + getPresetOptions() { + const speed = this.getPresetIndex() + 4; // Use 4 as slowest, giving us an effective range of 4-12 which is far more useful than 0-8 + if (speed >= 0) { + return [`-preset ${speed}`]; + } + return []; + } + + getBitrateOptions() { + const options = [`-crf ${this.config.crf}`]; + const bitrates = this.getBitrateDistribution(); + const svtparams = []; + if (this.config.threads > 0) { + svtparams.push(`lp=${this.config.threads}`); + } + if (bitrates.max > 0) { + svtparams.push(`mbr=${bitrates.max}${bitrates.unit}`); + } + if (svtparams.length > 0) { + options.push(`-svtav1-params ${svtparams.join(':')}`); + } + return options; + } + + getThreadOptions() { + return []; // Already set above with svtav1-params + } + + eligibleForTwoPass() { + return this.config.twoPass; + } } export class NVENCConfig extends BaseHWConfig { @@ -527,6 +565,10 @@ export class QSVConfig extends BaseHWConfig { return options; } + getSupportedCodecs() { + return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9]; + } + // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md getBFrames() { if (this.config.bframes < 0) { @@ -605,6 +647,10 @@ export class VAAPIConfig extends BaseHWConfig { return options; } + getSupportedCodecs() { + return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9]; + } + useCQP() { return this.config.cqMode !== CQMode.ICQ || this.config.targetVideoCodec === VideoCodec.VP9; } diff --git a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte index fba4c2bafc..bc91c2c993 100644 --- a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte +++ b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte @@ -54,7 +54,7 @@ inputType={SettingInputFieldType.NUMBER} {disabled} label="CONSTANT RATE FACTOR (-crf)" - desc="Video quality level. Typical values are 23 for H.264, 28 for HEVC, and 31 for VP9. Lower is better, but takes longer to encode and produces larger files." + desc="Video quality level. Typical values are 23 for H.264, 28 for HEVC, 31 for VP9, and 35 for AV1. Lower is better, but produces larger files." bind:value={config.ffmpeg.crf} required={true} isEdited={config.ffmpeg.crf !== savedConfig.ffmpeg.crf} @@ -115,12 +115,13 @@