c-l/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-29 15:11:58 +00:00
Code Releases Activity

Add AV1 transcoding support (#8491)

Browse source 
* Add AV1 transcoding support

- AV1 encoding on CPU via SVT-AV1 (libsvtav1 in ffmpeg)
- Supports CRF and optionally capped CRF (max bitrate)
- Tested playback successfully in Chrome Win+Android, Firefox Win+Linux, Android app

* AV1: Add support for encoding threads option

* Revert previous commit; specifying params multiple times is bad

We need to specify all svtav1-params at once, so putting the thread option into getThreadOptions is not possible.

* AV1: Override VAAPI getSupportedCodecs as it does not yet support AV1 unlike nvenc, qsv, amf

* Change BaseHWConfig supported codecs to only H264/HEVC

Configs that support VP9 and/or AV1 need to override getSupportedCodecs()

* Set SVT-AV1 threads with svtav1-params, remove duplicate block in NVENCConfig

* AV1Config: Fix empty svtav1-params array being added to options

* add tests

* update api

* allow crf-based two-pass mode

* formatting

* suggest 35

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
This commit is contained in:
N00MKRAD 2024-04-11 07:26:27 +02:00 committed by GitHub
parent ad5d115abe
commit f1ca1794a1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 212 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
mobile/openapi/lib/model/video_codec.dart generated
View file

Binary file not shown.

3
open-api/immich-openapi-specs.json
View file

@ -11231,7 +11231,8 @@
"enum": [
"h264",
"hevc",
"vp9"
"vp9",
"av1"
],
"type": "string"
}

3
open-api/typescript-sdk/src/fetch-client.ts
View file

@ -2984,7 +2984,8 @@ export enum AudioCodec {
export enum VideoCodec {
H264 = "h264",
Hevc = "hevc",
Vp9 = "vp9"
Vp9 = "vp9",
Av1 = "av1"
}
export enum CQMode {
Auto = "auto",

1
server/src/entities/system-config.entity.ts
View file

@ -153,6 +153,7 @@ export enum VideoCodec {
H264 = 'h264',
HEVC = 'hevc',
VP9 = 'vp9',
AV1 = 'av1',
}
export enum AudioCodec {

151
server/src/services/media.service.spec.ts
View file

@ -1268,6 +1268,157 @@ describe(MediaService.name, () => {
);
});
it('should use av1 if specified', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-c:v av1',
'-c:a copy',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-v verbose',
'-vf scale=-2:720,format=yuv420p',
'-preset 12',
'-crf 23',
],
twoPass: false,
},
);
});
it('should map `veryslow` preset to 4 for av1', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
{ key: SystemConfigKey.FFMPEG_PRESET, value: 'veryslow' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-c:v av1',
'-c:a copy',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-v verbose',
'-vf scale=-2:720,format=yuv420p',
'-preset 4',
'-crf 23',
],
twoPass: false,
},
);
});
it('should set max bitrate for av1 if specified', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '2M' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-c:v av1',
'-c:a copy',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-v verbose',
'-vf scale=-2:720,format=yuv420p',
'-preset 12',
'-crf 23',
'-svtav1-params mbr=2M',
],
twoPass: false,
},
);
});
it('should set threads for av1 if specified', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
{ key: SystemConfigKey.FFMPEG_THREADS, value: 4 },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-c:v av1',
'-c:a copy',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-v verbose',
'-vf scale=-2:720,format=yuv420p',
'-preset 12',
'-crf 23',
'-svtav1-params lp=4',
],
twoPass: false,
},
);
});
it('should set both bitrate and threads for av1 if specified', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.AV1 },
{ key: SystemConfigKey.FFMPEG_THREADS, value: 4 },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '2M' },
]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-c:v av1',
'-c:a copy',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-v verbose',
'-vf scale=-2:720,format=yuv420p',
'-preset 12',
'-crf 23',
'-svtav1-params lp=4:mbr=2M',
],
twoPass: false,
},
);
});
it('should skip transcoding for audioless videos with optimal policy if video codec is correct', async () => {
mediaMock.probe.mockResolvedValue(probeStub.noAudioStreams);
configMock.load.mockResolvedValue([

4
server/src/services/media.service.ts
View file

@ -32,6 +32,7 @@ import { IStorageRepository } from 'src/interfaces/storage.interface';
import { ISystemConfigRepository } from 'src/interfaces/system-config.interface';
import { ImmichLogger } from 'src/utils/logger';
import {
AV1Config,
H264Config,
HEVCConfig,
NVENCConfig,
@ -439,6 +440,9 @@ export class MediaService {
case VideoCodec.VP9: {
return new VP9Config(config);
}
case VideoCodec.AV1: {
return new AV1Config(config);
}
default: {
throw new UnsupportedMediaTypeException(`Codec '${config.targetVideoCodec}' is unsupported`);
}

BIN
server/src/utils/.media.ts.kate-swp Normal file
View file

Binary file not shown.

50
server/src/utils/media.ts
View file

@ -124,7 +124,7 @@ class BaseConfig implements VideoCodecSWConfig {
return false;
}
return this.isBitrateConstrained() || this.config.targetVideoCodec === VideoCodec.VP9;
return this.isBitrateConstrained();
}
getBitrateDistribution() {
@ -265,7 +265,7 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
}
getSupportedCodecs() {
return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
return [VideoCodec.H264, VideoCodec.HEVC];
}
validateDevices(devices: string[]) {
@ -394,6 +394,44 @@ export class VP9Config extends BaseConfig {
getThreadOptions() {
return ['-row-mt 1', ...super.getThreadOptions()];
}
eligibleForTwoPass() {
return this.config.twoPass;
}
}
export class AV1Config extends BaseConfig {
getPresetOptions() {
const speed = this.getPresetIndex() + 4; // Use 4 as slowest, giving us an effective range of 4-12 which is far more useful than 0-8
if (speed >= 0) {
return [`-preset ${speed}`];
}
return [];
}
getBitrateOptions() {
const options = [`-crf ${this.config.crf}`];
const bitrates = this.getBitrateDistribution();
const svtparams = [];
if (this.config.threads > 0) {
svtparams.push(`lp=${this.config.threads}`);
}
if (bitrates.max > 0) {
svtparams.push(`mbr=${bitrates.max}${bitrates.unit}`);
}
if (svtparams.length > 0) {
options.push(`-svtav1-params ${svtparams.join(':')}`);
}
return options;
}
getThreadOptions() {
return []; // Already set above with svtav1-params
}
eligibleForTwoPass() {
return this.config.twoPass;
}
}
export class NVENCConfig extends BaseHWConfig {
@ -527,6 +565,10 @@ export class QSVConfig extends BaseHWConfig {
return options;
}
getSupportedCodecs() {
return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
}
// recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
getBFrames() {
if (this.config.bframes < 0) {
@ -605,6 +647,10 @@ export class VAAPIConfig extends BaseHWConfig {
return options;
}
getSupportedCodecs() {
return [VideoCodec.H264, VideoCodec.HEVC, VideoCodec.VP9];
}
useCQP() {
return this.config.cqMode !== CQMode.ICQ || this.config.targetVideoCodec === VideoCodec.VP9;
}

6
web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte
View file

@ -54,7 +54,7 @@
inputType={SettingInputFieldType.NUMBER}
{disabled}
label="CONSTANT RATE FACTOR (-crf)"
desc="Video quality level. Typical values are 23 for H.264, 28 for HEVC, and 31 for VP9. Lower is better, but takes longer to encode and produces larger files."
desc="Video quality level. Typical values are 23 for H.264, 28 for HEVC, 31 for VP9, and 35 for AV1. Lower is better, but produces larger files."
bind:value={config.ffmpeg.crf}
required={true}
isEdited={config.ffmpeg.crf !== savedConfig.ffmpeg.crf}
@ -115,12 +115,13 @@
<SettingSelect
label="VIDEO CODEC"
{disabled}
desc="VP9 has high efficiency and web compatibility, but takes longer to transcode. HEVC performs similarly, but has lower web compatibility. H.264 is widely compatible and quick to transcode, but produces much larger files."
desc="VP9 has high efficiency and web compatibility, but takes longer to transcode. HEVC performs similarly, but has lower web compatibility. H.264 is widely compatible and quick to transcode, but produces much larger files. AV1 is the most efficient codec but lacks support on older devices."
bind:value={config.ffmpeg.targetVideoCodec}
options={[
{ value: VideoCodec.H264, text: 'h264' },
{ value: VideoCodec.Hevc, text: 'hevc' },
{ value: VideoCodec.Vp9, text: 'vp9' },
{ value: VideoCodec.Av1, text: 'av1' },
]}
name="vcodec"
isEdited={config.ffmpeg.targetVideoCodec !== savedConfig.ffmpeg.targetVideoCodec}
@ -137,6 +138,7 @@
{ value: VideoCodec.H264, text: 'H.264' },
{ value: VideoCodec.Hevc, text: 'HEVC' },
{ value: VideoCodec.Vp9, text: 'VP9' },
{ value: VideoCodec.Av1, text: 'AV1' },
]}
isEdited={!isEqual(sortBy(config.ffmpeg.acceptedVideoCodecs), sortBy(savedConfig.ffmpeg.acceptedVideoCodecs))}
/>