c-l/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-29 15:11:58 +00:00
Code Releases Activity

feat(server): only transcode streams that require it (#7106)

Browse source
This commit is contained in:
Mert 2024-02-14 11:24:39 -05:00 committed by GitHub
parent b823dfffdc
commit 5ff68d4cdb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 169 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
server/src/domain/media/media.service.spec.ts
View file

@ -704,8 +704,35 @@ describe(MediaService.name, () => {
);
});
it('should transcode when audio doesnt match target', async () => {
mediaMock.probe.mockResolvedValue(probeStub.audioStreamMp3);
it('should copy video stream when video matches target', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
expect(mediaMock.transcode).toHaveBeenCalledWith(
'/original/path.ext',
'upload/encoded-video/user-id/as/se/asset-id.mp4',
{
inputOptions: [],
outputOptions: [
'-c:v copy',
'-c:a aac',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
'-map 0:1',
'-tag:v hvc1',
'-v verbose',
'-preset ultrafast',
'-crf 23',
],
twoPass: false,
},
);
});
it('should copy audio stream when audio matches target', async () => {
mediaMock.probe.mockResolvedValue(probeStub.audioStreamAac);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.OPTIMAL }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
@ -716,7 +743,7 @@ describe(MediaService.name, () => {
inputOptions: [],
outputOptions: [
'-c:v h264',
'-c:a aac',
'-c:a copy',
'-movflags faststart',
'-fps_mode passthrough',
'-map 0:0',
@ -758,11 +785,11 @@ describe(MediaService.name, () => {
);
});
it('should not transcode an invalid transcode value', async () => {
it('should throw an exception if transcode value is invalid', async () => {
mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p);
configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'invalid' }]);
assetMock.getByIds.mockResolvedValue([assetStub.video]);
await sut.handleVideoConversion({ id: assetStub.video.id });
await expect(sut.handleVideoConversion({ id: assetStub.video.id })).rejects.toThrow();
expect(mediaMock.transcode).not.toHaveBeenCalled();
});
@ -1106,7 +1133,7 @@ describe(MediaService.name, () => {
});
it('should disable thread pooling for hevc if thread limit is above 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_THREADS, value: 2 },
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC },
@ -1140,7 +1167,7 @@ describe(MediaService.name, () => {
});
it('should omit thread flags for hevc if thread limit is at or below 0', async () => {
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_THREADS, value: 0 },
{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC },
@ -1756,7 +1783,7 @@ describe(MediaService.name, () => {
it('should set vbr options for rkmpp when max bitrate is enabled', async () => {
storageMock.readdir.mockResolvedValue(['renderD128']);
mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9);
configMock.load.mockResolvedValue([
{ key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP },
{ key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' },

126
server/src/domain/media/media.service.ts
View file

@ -6,6 +6,7 @@ import {
Colorspace,
TranscodeHWAccel,
TranscodePolicy,
TranscodeTarget,
VideoCodec,
} from '@app/infra/entities';
import { ImmichLogger } from '@app/infra/logger';
@ -197,7 +198,7 @@ export class MediaService {
}
const mainAudioStream = this.getMainStream(audioStreams);
const config = { ...ffmpeg, targetResolution: size.toString() };
const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream);
const options = new ThumbnailConfig(config).getOptions(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream);
await this.mediaRepository.transcode(asset.originalPath, path, options);
break;
}
@ -267,7 +268,6 @@ export class MediaService {
const mainVideoStream = this.getMainStream(videoStreams);
const mainAudioStream = this.getMainStream(audioStreams);
const containerExtension = format.formatName;
const bitrate = format.bitrate;
if (!mainVideoStream || !containerExtension) {
return false;
}
@ -279,15 +279,8 @@ export class MediaService {
const { ffmpeg: config } = await this.configCore.getConfig();
const required = this.isTranscodeRequired(
asset,
mainVideoStream,
mainAudioStream,
containerExtension,
config,
bitrate,
);
if (!required) {
const target = this.getTranscodeTarget(config, mainVideoStream, mainAudioStream);
if (target === TranscodeTarget.NONE) {
if (asset.encodedVideoPath) {
this.logger.log(`Transcoded video exists for asset ${asset.id}, but is no longer required. Deleting...`);
await this.jobRepository.queue({ name: JobName.DELETE_FILES, data: { files: [asset.encodedVideoPath] } });
@ -299,13 +292,15 @@ export class MediaService {
let transcodeOptions;
try {
transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream, mainAudioStream));
transcodeOptions = await this.getCodecConfig(config).then((c) =>
c.getOptions(target, mainVideoStream, mainAudioStream),
);
} catch (error) {
this.logger.error(`An error occurred while configuring transcoding options: ${error}`);
return false;
}
this.logger.log(`Start encoding video ${asset.id} ${JSON.stringify(transcodeOptions)}`);
this.logger.log(`Started encoding video ${asset.id} ${JSON.stringify(transcodeOptions)}`);
try {
await this.mediaRepository.transcode(input, output, transcodeOptions);
} catch (error) {
@ -316,11 +311,13 @@ export class MediaService {
);
}
config.accel = TranscodeHWAccel.DISABLED;
transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream, mainAudioStream));
transcodeOptions = await this.getCodecConfig(config).then((c) =>
c.getOptions(target, mainVideoStream, mainAudioStream),
);
await this.mediaRepository.transcode(input, output, transcodeOptions);
}
this.logger.log(`Encoding success ${asset.id}`);
this.logger.log(`Successfully encoded ${asset.id}`);
await this.assetRepository.save({ id: asset.id, encodedVideoPath: output });
@ -331,55 +328,88 @@ export class MediaService {
return streams.sort((stream1, stream2) => stream2.frameCount - stream1.frameCount)[0];
}
private isTranscodeRequired(
asset: AssetEntity,
videoStream: VideoStreamInfo,
private getTranscodeTarget(
config: SystemConfigFFmpegDto,
videoStream: VideoStreamInfo | null,
audioStream: AudioStreamInfo | null,
containerExtension: string,
ffmpegConfig: SystemConfigFFmpegDto,
bitrate: number,
): boolean {
const isTargetVideoCodec = ffmpegConfig.acceptedVideoCodecs.includes(videoStream.codecName as VideoCodec);
const isTargetContainer = ['mov,mp4,m4a,3gp,3g2,mj2', 'mp4', 'mov'].includes(containerExtension);
const isTargetAudioCodec =
audioStream == null || ffmpegConfig.acceptedAudioCodecs.includes(audioStream.codecName as AudioCodec);
): TranscodeTarget {
if (videoStream == null && audioStream == null) {
return TranscodeTarget.NONE;
}
this.logger.verbose(
`${asset.id}: AudioCodecName ${audioStream?.codecName ?? 'None'}, AudioStreamCodecType ${
audioStream?.codecType ?? 'None'
}, containerExtension ${containerExtension}`,
);
const isAudioTranscodeRequired = this.isAudioTranscodeRequired(config, audioStream);
const isVideoTranscodeRequired = this.isVideoTranscodeRequired(config, videoStream);
const allTargetsMatching = isTargetVideoCodec && isTargetAudioCodec && isTargetContainer;
const scalingEnabled = ffmpegConfig.targetResolution !== 'original';
const targetRes = Number.parseInt(ffmpegConfig.targetResolution);
const isLargerThanTargetRes = scalingEnabled && Math.min(videoStream.height, videoStream.width) > targetRes;
const isLargerThanTargetBitrate = bitrate > this.parseBitrateToBps(ffmpegConfig.maxBitrate);
if (isAudioTranscodeRequired && isVideoTranscodeRequired) {
return TranscodeTarget.ALL;
}
if (isAudioTranscodeRequired) {
return TranscodeTarget.AUDIO;
}
if (isVideoTranscodeRequired) {
return TranscodeTarget.VIDEO;
}
return TranscodeTarget.NONE;
}
private isAudioTranscodeRequired(ffmpegConfig: SystemConfigFFmpegDto, stream: AudioStreamInfo | null): boolean {
if (stream == null) {
return false;
}
switch (ffmpegConfig.transcode) {
case TranscodePolicy.DISABLED: {
return false;
}
case TranscodePolicy.ALL: {
return true;
}
case TranscodePolicy.REQUIRED: {
return !allTargetsMatching || videoStream.isHDR;
}
case TranscodePolicy.OPTIMAL: {
return !allTargetsMatching || isLargerThanTargetRes || videoStream.isHDR;
}
case TranscodePolicy.REQUIRED:
case TranscodePolicy.OPTIMAL:
case TranscodePolicy.BITRATE: {
return !allTargetsMatching || isLargerThanTargetBitrate || videoStream.isHDR;
return !ffmpegConfig.acceptedAudioCodecs.includes(stream.codecName as AudioCodec);
}
default: {
throw new Error(`Unsupported transcode policy: ${ffmpegConfig.transcode}`);
}
}
}
private isVideoTranscodeRequired(ffmpegConfig: SystemConfigFFmpegDto, stream: VideoStreamInfo | null): boolean {
if (stream == null) {
return false;
}
const scalingEnabled = ffmpegConfig.targetResolution !== 'original';
const targetRes = Number.parseInt(ffmpegConfig.targetResolution);
const isLargerThanTargetRes = scalingEnabled && Math.min(stream.height, stream.width) > targetRes;
const isLargerThanTargetBitrate = stream.bitrate > this.parseBitrateToBps(ffmpegConfig.maxBitrate);
const isTargetVideoCodec = ffmpegConfig.acceptedVideoCodecs.includes(stream.codecName as VideoCodec);
const isRequired = !isTargetVideoCodec || stream.isHDR;
switch (ffmpegConfig.transcode) {
case TranscodePolicy.DISABLED: {
return false;
}
case TranscodePolicy.ALL: {
return true;
}
case TranscodePolicy.REQUIRED: {
return isRequired;
}
case TranscodePolicy.OPTIMAL: {
return isRequired || isLargerThanTargetRes;
}
case TranscodePolicy.BITRATE: {
return isRequired || isLargerThanTargetBitrate;
}
default: {
throw new Error(`Unsupported transcode policy: ${ffmpegConfig.transcode}`);
}
}
}

37
server/src/domain/media/media.util.ts
View file

@ -1,4 +1,4 @@
import { CQMode, ToneMapping, TranscodeHWAccel, VideoCodec } from '@app/infra/entities';
import { CQMode, ToneMapping, TranscodeHWAccel, TranscodeTarget, VideoCodec } from '@app/infra/entities';
import {
AudioStreamInfo,
BitrateDistribution,
@ -12,16 +12,19 @@ class BaseConfig implements VideoCodecSWConfig {
presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'];
constructor(protected config: SystemConfigFFmpegDto) {}
getOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
getOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
const options = {
inputOptions: this.getBaseInputOptions(),
outputOptions: [...this.getBaseOutputOptions(videoStream, audioStream), '-v verbose'],
outputOptions: [...this.getBaseOutputOptions(target, videoStream, audioStream), '-v verbose'],
twoPass: this.eligibleForTwoPass(),
} as TranscodeOptions;
const filters = this.getFilterOptions(videoStream);
if (filters.length > 0) {
options.outputOptions.push(`-vf ${filters.join(',')}`);
if ([TranscodeTarget.ALL, TranscodeTarget.VIDEO].includes(target)) {
const filters = this.getFilterOptions(videoStream);
if (filters.length > 0) {
options.outputOptions.push(`-vf ${filters.join(',')}`);
}
}
options.outputOptions.push(...this.getPresetOptions(), ...this.getThreadOptions(), ...this.getBitrateOptions());
return options;
@ -31,10 +34,10 @@ class BaseConfig implements VideoCodecSWConfig {
return [];
}
getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
const options = [
`-c:v ${this.getVideoCodec()}`,
`-c:a ${this.getAudioCodec()}`,
`-c:v ${[TranscodeTarget.ALL, TranscodeTarget.VIDEO].includes(target) ? this.getVideoCodec() : 'copy'}`,
`-c:a ${[TranscodeTarget.ALL, TranscodeTarget.AUDIO].includes(target) ? this.getAudioCodec() : 'copy'}`,
// Makes a second pass moving the moov atom to the
// beginning of the file for improved playback speed.
'-movflags faststart',
@ -398,14 +401,14 @@ export class NVENCConfig extends BaseHWConfig {
return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'];
}
getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
const options = [
// below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding
'-tune hq',
'-qmin 0',
'-rc-lookahead 20',
'-i_qfactor 0.75',
...super.getBaseOutputOptions(videoStream, audioStream),
...super.getBaseOutputOptions(target, videoStream, audioStream),
];
if (this.getBFrames() > 0) {
options.push('-b_ref_mode middle', '-b_qfactor 1.1');
@ -483,8 +486,8 @@ export class QSVConfig extends BaseHWConfig {
return [`-init_hw_device qsv=hw${qsvString}`, '-filter_hw_device hw'];
}
getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
const options = super.getBaseOutputOptions(videoStream, audioStream);
getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
const options = super.getBaseOutputOptions(target, videoStream, audioStream);
// VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a
if (this.config.targetVideoCodec === VideoCodec.VP9) {
options.push('-low_power 1');
@ -604,11 +607,13 @@ export class VAAPIConfig extends BaseHWConfig {
}
export class RKMPPConfig extends BaseHWConfig {
getOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo): TranscodeOptions {
const options = super.getOptions(videoStream, audioStream);
getOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo): TranscodeOptions {
const options = super.getOptions(target, videoStream, audioStream);
options.ffmpegPath = 'ffmpeg_mpp';
options.ldLibraryPath = '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp';
options.outputOptions.push(...this.getSizeOptions(videoStream));
if ([TranscodeTarget.ALL, TranscodeTarget.VIDEO].includes(target)) {
options.outputOptions.push(...this.getSizeOptions(videoStream));
}
return options;
}

7
server/src/domain/repositories/media.repository.ts
View file

@ -1,4 +1,4 @@
import { VideoCodec } from '@app/infra/entities';
import { TranscodeTarget, VideoCodec } from '@app/infra/entities';
import { Writable } from 'node:stream';
export const IMediaRepository = 'IMediaRepository';
@ -16,15 +16,14 @@ export interface VideoStreamInfo {
width: number;
rotation: number;
codecName?: string;
codecType?: string;
frameCount: number;
isHDR: boolean;
bitrate: number;
}
export interface AudioStreamInfo {
index: number;
codecName?: string;
codecType?: string;
frameCount: number;
}
@ -64,7 +63,7 @@ export interface BitrateDistribution {
}
export interface VideoCodecSWConfig {
getOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeOptions;
getOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeOptions;
}
export interface VideoCodecHWConfig extends VideoCodecSWConfig {

7
server/src/infra/entities/system-config.entity.ts
View file

@ -118,6 +118,13 @@ export enum TranscodePolicy {
DISABLED = 'disabled',
}
export enum TranscodeTarget {
NONE,
AUDIO,
VIDEO,
ALL,
}
export enum VideoCodec {
H264 = 'h264',
HEVC = 'hevc',

1
server/src/infra/repositories/media.repository.ts
View file

@ -60,6 +60,7 @@ export class MediaRepository implements IMediaRepository {
frameCount: Number.parseInt(stream.nb_frames ?? '0'),
rotation: Number.parseInt(`${stream.rotation ?? 0}`),
isHDR: stream.color_transfer === 'smpte2084' || stream.color_transfer === 'arib-std-b67',
bitrate: Number.parseInt(stream.bit_rate ?? '0'),
})),
audioStreams: results.streams
.filter((stream) => stream.codec_type === 'audio')

43
server/test/fixtures/media.stub.ts vendored
View file

@ -13,16 +13,14 @@ const probeStubDefaultVideoStream: VideoStreamInfo[] = [
height: 1080,
width: 1920,
codecName: 'hevc',
codecType: 'video',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
},
];
const probeStubDefaultAudioStream: AudioStreamInfo[] = [
{ index: 1, codecName: 'aac', codecType: 'audio', frameCount: 100 },
];
const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ index: 1, codecName: 'mp3', frameCount: 100 }];
const probeStubDefault: VideoInfo = {
format: probeStubDefaultFormat,
@ -41,20 +39,20 @@ export const probeStub = {
height: 1080,
width: 400,
codecName: 'hevc',
codecType: 'video',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
},
{
index: 1,
height: 1080,
width: 400,
codecName: 'h7000',
codecType: 'video',
frameCount: 99,
rotation: 0,
isHDR: false,
bitrate: 0,
},
],
}),
@ -66,10 +64,10 @@ export const probeStub = {
height: 0,
width: 400,
codecName: 'hevc',
codecType: 'video',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
},
],
}),
@ -81,21 +79,16 @@ export const probeStub = {
height: 2160,
width: 3840,
codecName: 'h264',
codecType: 'video',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
},
],
}),
videoStream40Mbps: Object.freeze<VideoInfo>({
...probeStubDefault,
format: {
formatName: 'mov,mp4,m4a,3gp,3g2,mj2',
formatLongName: 'QuickTime / MOV',
duration: 0,
bitrate: 40_000_000,
},
videoStreams: [{ ...probeStubDefaultVideoStream[0], bitrate: 40_000_000 }],
}),
videoStreamHDR: Object.freeze<VideoInfo>({
...probeStubDefault,
@ -105,10 +98,10 @@ export const probeStub = {
height: 480,
width: 480,
codecName: 'h264',
codecType: 'video',
frameCount: 100,
rotation: 0,
isHDR: true,
bitrate: 0,
},
],
}),
@ -120,10 +113,10 @@ export const probeStub = {
height: 2160,
width: 3840,
codecName: 'h264',
codecType: 'video',
frameCount: 100,
rotation: 90,
isHDR: false,
bitrate: 0,
},
],
}),
@ -135,10 +128,10 @@ export const probeStub = {
height: 355,
width: 1586,
codecName: 'h264',
codecType: 'video',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
},
],
}),
@ -150,16 +143,16 @@ export const probeStub = {
height: 1586,
width: 355,
codecName: 'h264',
codecType: 'video',
frameCount: 100,
rotation: 0,
isHDR: false,
bitrate: 0,
},
],
}),
audioStreamMp3: Object.freeze<VideoInfo>({
audioStreamAac: Object.freeze<VideoInfo>({
...probeStubDefault,
audioStreams: [{ index: 1, codecType: 'audio', codecName: 'aac', frameCount: 100 }],
audioStreams: [{ index: 1, codecName: 'aac', frameCount: 100 }],
}),
matroskaContainer: Object.freeze<VideoInfo>({
...probeStubDefault,
@ -170,4 +163,14 @@ export const probeStub = {
bitrate: 0,
},
}),
videoStreamVp9: Object.freeze<VideoInfo>({
...probeStubDefault,
videoStreams: [{ ...probeStubDefaultVideoStream[0], codecName: 'vp9' }],
format: {
formatName: 'matroska,webm',
formatLongName: 'Matroska / WebM',
duration: 0,
bitrate: 0,
},
}),
};