From 5ff68d4cdb6dba44a959df58fc2d017f3892195a Mon Sep 17 00:00:00 2001 From: Mert <101130780+mertalev@users.noreply.github.com> Date: Wed, 14 Feb 2024 11:24:39 -0500 Subject: [PATCH] feat(server): only transcode streams that require it (#7106) --- server/src/domain/media/media.service.spec.ts | 45 +++++-- server/src/domain/media/media.service.ts | 126 +++++++++++------- server/src/domain/media/media.util.ts | 37 ++--- .../domain/repositories/media.repository.ts | 7 +- .../infra/entities/system-config.entity.ts | 7 + .../infra/repositories/media.repository.ts | 1 + server/test/fixtures/media.stub.ts | 43 +++--- 7 files changed, 169 insertions(+), 97 deletions(-) diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index 6406b2887a..aa48568b90 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -704,8 +704,35 @@ describe(MediaService.name, () => { ); }); - it('should transcode when audio doesnt match target', async () => { - mediaMock.probe.mockResolvedValue(probeStub.audioStreamMp3); + it('should copy video stream when video matches target', async () => { + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC }]); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + { + inputOptions: [], + outputOptions: [ + '-c:v copy', + '-c:a aac', + '-movflags faststart', + '-fps_mode passthrough', + '-map 0:0', + '-map 0:1', + '-tag:v hvc1', + '-v verbose', + '-preset ultrafast', + '-crf 23', + ], + twoPass: false, + }, + ); + }); + + it('should copy audio stream when audio matches target', async () => { + mediaMock.probe.mockResolvedValue(probeStub.audioStreamAac); configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: TranscodePolicy.OPTIMAL }]); assetMock.getByIds.mockResolvedValue([assetStub.video]); await sut.handleVideoConversion({ id: assetStub.video.id }); @@ -716,7 +743,7 @@ describe(MediaService.name, () => { inputOptions: [], outputOptions: [ '-c:v h264', - '-c:a aac', + '-c:a copy', '-movflags faststart', '-fps_mode passthrough', '-map 0:0', @@ -758,11 +785,11 @@ describe(MediaService.name, () => { ); }); - it('should not transcode an invalid transcode value', async () => { + it('should throw an exception if transcode value is invalid', async () => { mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p); configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'invalid' }]); - assetMock.getByIds.mockResolvedValue([assetStub.video]); - await sut.handleVideoConversion({ id: assetStub.video.id }); + + await expect(sut.handleVideoConversion({ id: assetStub.video.id })).rejects.toThrow(); expect(mediaMock.transcode).not.toHaveBeenCalled(); }); @@ -1106,7 +1133,7 @@ describe(MediaService.name, () => { }); it('should disable thread pooling for hevc if thread limit is above 0', async () => { - mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9); configMock.load.mockResolvedValue([ { key: SystemConfigKey.FFMPEG_THREADS, value: 2 }, { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC }, @@ -1140,7 +1167,7 @@ describe(MediaService.name, () => { }); it('should omit thread flags for hevc if thread limit is at or below 0', async () => { - mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9); configMock.load.mockResolvedValue([ { key: SystemConfigKey.FFMPEG_THREADS, value: 0 }, { key: SystemConfigKey.FFMPEG_TARGET_VIDEO_CODEC, value: VideoCodec.HEVC }, @@ -1756,7 +1783,7 @@ describe(MediaService.name, () => { it('should set vbr options for rkmpp when max bitrate is enabled', async () => { storageMock.readdir.mockResolvedValue(['renderD128']); - mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + mediaMock.probe.mockResolvedValue(probeStub.videoStreamVp9); configMock.load.mockResolvedValue([ { key: SystemConfigKey.FFMPEG_ACCEL, value: TranscodeHWAccel.RKMPP }, { key: SystemConfigKey.FFMPEG_MAX_BITRATE, value: '10000k' }, diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index 68f861d7e2..6a5c8ff9d3 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -6,6 +6,7 @@ import { Colorspace, TranscodeHWAccel, TranscodePolicy, + TranscodeTarget, VideoCodec, } from '@app/infra/entities'; import { ImmichLogger } from '@app/infra/logger'; @@ -197,7 +198,7 @@ export class MediaService { } const mainAudioStream = this.getMainStream(audioStreams); const config = { ...ffmpeg, targetResolution: size.toString() }; - const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream); + const options = new ThumbnailConfig(config).getOptions(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream); await this.mediaRepository.transcode(asset.originalPath, path, options); break; } @@ -267,7 +268,6 @@ export class MediaService { const mainVideoStream = this.getMainStream(videoStreams); const mainAudioStream = this.getMainStream(audioStreams); const containerExtension = format.formatName; - const bitrate = format.bitrate; if (!mainVideoStream || !containerExtension) { return false; } @@ -279,15 +279,8 @@ export class MediaService { const { ffmpeg: config } = await this.configCore.getConfig(); - const required = this.isTranscodeRequired( - asset, - mainVideoStream, - mainAudioStream, - containerExtension, - config, - bitrate, - ); - if (!required) { + const target = this.getTranscodeTarget(config, mainVideoStream, mainAudioStream); + if (target === TranscodeTarget.NONE) { if (asset.encodedVideoPath) { this.logger.log(`Transcoded video exists for asset ${asset.id}, but is no longer required. Deleting...`); await this.jobRepository.queue({ name: JobName.DELETE_FILES, data: { files: [asset.encodedVideoPath] } }); @@ -299,13 +292,15 @@ export class MediaService { let transcodeOptions; try { - transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream, mainAudioStream)); + transcodeOptions = await this.getCodecConfig(config).then((c) => + c.getOptions(target, mainVideoStream, mainAudioStream), + ); } catch (error) { this.logger.error(`An error occurred while configuring transcoding options: ${error}`); return false; } - this.logger.log(`Start encoding video ${asset.id} ${JSON.stringify(transcodeOptions)}`); + this.logger.log(`Started encoding video ${asset.id} ${JSON.stringify(transcodeOptions)}`); try { await this.mediaRepository.transcode(input, output, transcodeOptions); } catch (error) { @@ -316,11 +311,13 @@ export class MediaService { ); } config.accel = TranscodeHWAccel.DISABLED; - transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream, mainAudioStream)); + transcodeOptions = await this.getCodecConfig(config).then((c) => + c.getOptions(target, mainVideoStream, mainAudioStream), + ); await this.mediaRepository.transcode(input, output, transcodeOptions); } - this.logger.log(`Encoding success ${asset.id}`); + this.logger.log(`Successfully encoded ${asset.id}`); await this.assetRepository.save({ id: asset.id, encodedVideoPath: output }); @@ -331,55 +328,88 @@ export class MediaService { return streams.sort((stream1, stream2) => stream2.frameCount - stream1.frameCount)[0]; } - private isTranscodeRequired( - asset: AssetEntity, - videoStream: VideoStreamInfo, + private getTranscodeTarget( + config: SystemConfigFFmpegDto, + videoStream: VideoStreamInfo | null, audioStream: AudioStreamInfo | null, - containerExtension: string, - ffmpegConfig: SystemConfigFFmpegDto, - bitrate: number, - ): boolean { - const isTargetVideoCodec = ffmpegConfig.acceptedVideoCodecs.includes(videoStream.codecName as VideoCodec); - const isTargetContainer = ['mov,mp4,m4a,3gp,3g2,mj2', 'mp4', 'mov'].includes(containerExtension); - const isTargetAudioCodec = - audioStream == null || ffmpegConfig.acceptedAudioCodecs.includes(audioStream.codecName as AudioCodec); + ): TranscodeTarget { + if (videoStream == null && audioStream == null) { + return TranscodeTarget.NONE; + } - this.logger.verbose( - `${asset.id}: AudioCodecName ${audioStream?.codecName ?? 'None'}, AudioStreamCodecType ${ - audioStream?.codecType ?? 'None' - }, containerExtension ${containerExtension}`, - ); + const isAudioTranscodeRequired = this.isAudioTranscodeRequired(config, audioStream); + const isVideoTranscodeRequired = this.isVideoTranscodeRequired(config, videoStream); - const allTargetsMatching = isTargetVideoCodec && isTargetAudioCodec && isTargetContainer; - const scalingEnabled = ffmpegConfig.targetResolution !== 'original'; - const targetRes = Number.parseInt(ffmpegConfig.targetResolution); - const isLargerThanTargetRes = scalingEnabled && Math.min(videoStream.height, videoStream.width) > targetRes; - const isLargerThanTargetBitrate = bitrate > this.parseBitrateToBps(ffmpegConfig.maxBitrate); + if (isAudioTranscodeRequired && isVideoTranscodeRequired) { + return TranscodeTarget.ALL; + } + + if (isAudioTranscodeRequired) { + return TranscodeTarget.AUDIO; + } + + if (isVideoTranscodeRequired) { + return TranscodeTarget.VIDEO; + } + + return TranscodeTarget.NONE; + } + + private isAudioTranscodeRequired(ffmpegConfig: SystemConfigFFmpegDto, stream: AudioStreamInfo | null): boolean { + if (stream == null) { + return false; + } switch (ffmpegConfig.transcode) { case TranscodePolicy.DISABLED: { return false; } - case TranscodePolicy.ALL: { return true; } - - case TranscodePolicy.REQUIRED: { - return !allTargetsMatching || videoStream.isHDR; - } - - case TranscodePolicy.OPTIMAL: { - return !allTargetsMatching || isLargerThanTargetRes || videoStream.isHDR; - } - + case TranscodePolicy.REQUIRED: + case TranscodePolicy.OPTIMAL: case TranscodePolicy.BITRATE: { - return !allTargetsMatching || isLargerThanTargetBitrate || videoStream.isHDR; + return !ffmpegConfig.acceptedAudioCodecs.includes(stream.codecName as AudioCodec); } - default: { + throw new Error(`Unsupported transcode policy: ${ffmpegConfig.transcode}`); + } + } + } + + private isVideoTranscodeRequired(ffmpegConfig: SystemConfigFFmpegDto, stream: VideoStreamInfo | null): boolean { + if (stream == null) { + return false; + } + + const scalingEnabled = ffmpegConfig.targetResolution !== 'original'; + const targetRes = Number.parseInt(ffmpegConfig.targetResolution); + const isLargerThanTargetRes = scalingEnabled && Math.min(stream.height, stream.width) > targetRes; + const isLargerThanTargetBitrate = stream.bitrate > this.parseBitrateToBps(ffmpegConfig.maxBitrate); + + const isTargetVideoCodec = ffmpegConfig.acceptedVideoCodecs.includes(stream.codecName as VideoCodec); + const isRequired = !isTargetVideoCodec || stream.isHDR; + + switch (ffmpegConfig.transcode) { + case TranscodePolicy.DISABLED: { return false; } + case TranscodePolicy.ALL: { + return true; + } + case TranscodePolicy.REQUIRED: { + return isRequired; + } + case TranscodePolicy.OPTIMAL: { + return isRequired || isLargerThanTargetRes; + } + case TranscodePolicy.BITRATE: { + return isRequired || isLargerThanTargetBitrate; + } + default: { + throw new Error(`Unsupported transcode policy: ${ffmpegConfig.transcode}`); + } } } diff --git a/server/src/domain/media/media.util.ts b/server/src/domain/media/media.util.ts index ab3e43ec9f..c9483c3736 100644 --- a/server/src/domain/media/media.util.ts +++ b/server/src/domain/media/media.util.ts @@ -1,4 +1,4 @@ -import { CQMode, ToneMapping, TranscodeHWAccel, VideoCodec } from '@app/infra/entities'; +import { CQMode, ToneMapping, TranscodeHWAccel, TranscodeTarget, VideoCodec } from '@app/infra/entities'; import { AudioStreamInfo, BitrateDistribution, @@ -12,16 +12,19 @@ class BaseConfig implements VideoCodecSWConfig { presets = ['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast']; constructor(protected config: SystemConfigFFmpegDto) {} - getOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { + getOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { const options = { inputOptions: this.getBaseInputOptions(), - outputOptions: [...this.getBaseOutputOptions(videoStream, audioStream), '-v verbose'], + outputOptions: [...this.getBaseOutputOptions(target, videoStream, audioStream), '-v verbose'], twoPass: this.eligibleForTwoPass(), } as TranscodeOptions; - const filters = this.getFilterOptions(videoStream); - if (filters.length > 0) { - options.outputOptions.push(`-vf ${filters.join(',')}`); + if ([TranscodeTarget.ALL, TranscodeTarget.VIDEO].includes(target)) { + const filters = this.getFilterOptions(videoStream); + if (filters.length > 0) { + options.outputOptions.push(`-vf ${filters.join(',')}`); + } } + options.outputOptions.push(...this.getPresetOptions(), ...this.getThreadOptions(), ...this.getBitrateOptions()); return options; @@ -31,10 +34,10 @@ class BaseConfig implements VideoCodecSWConfig { return []; } - getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { + getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { const options = [ - `-c:v ${this.getVideoCodec()}`, - `-c:a ${this.getAudioCodec()}`, + `-c:v ${[TranscodeTarget.ALL, TranscodeTarget.VIDEO].includes(target) ? this.getVideoCodec() : 'copy'}`, + `-c:a ${[TranscodeTarget.ALL, TranscodeTarget.AUDIO].includes(target) ? this.getAudioCodec() : 'copy'}`, // Makes a second pass moving the moov atom to the // beginning of the file for improved playback speed. '-movflags faststart', @@ -398,14 +401,14 @@ export class NVENCConfig extends BaseHWConfig { return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda']; } - getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { + getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { const options = [ // below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding '-tune hq', '-qmin 0', '-rc-lookahead 20', '-i_qfactor 0.75', - ...super.getBaseOutputOptions(videoStream, audioStream), + ...super.getBaseOutputOptions(target, videoStream, audioStream), ]; if (this.getBFrames() > 0) { options.push('-b_ref_mode middle', '-b_qfactor 1.1'); @@ -483,8 +486,8 @@ export class QSVConfig extends BaseHWConfig { return [`-init_hw_device qsv=hw${qsvString}`, '-filter_hw_device hw']; } - getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { - const options = super.getBaseOutputOptions(videoStream, audioStream); + getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { + const options = super.getBaseOutputOptions(target, videoStream, audioStream); // VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a if (this.config.targetVideoCodec === VideoCodec.VP9) { options.push('-low_power 1'); @@ -604,11 +607,13 @@ export class VAAPIConfig extends BaseHWConfig { } export class RKMPPConfig extends BaseHWConfig { - getOptions(videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo): TranscodeOptions { - const options = super.getOptions(videoStream, audioStream); + getOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo): TranscodeOptions { + const options = super.getOptions(target, videoStream, audioStream); options.ffmpegPath = 'ffmpeg_mpp'; options.ldLibraryPath = '/lib/aarch64-linux-gnu:/lib/ffmpeg-mpp'; - options.outputOptions.push(...this.getSizeOptions(videoStream)); + if ([TranscodeTarget.ALL, TranscodeTarget.VIDEO].includes(target)) { + options.outputOptions.push(...this.getSizeOptions(videoStream)); + } return options; } diff --git a/server/src/domain/repositories/media.repository.ts b/server/src/domain/repositories/media.repository.ts index 60135e62dc..846b6156d6 100644 --- a/server/src/domain/repositories/media.repository.ts +++ b/server/src/domain/repositories/media.repository.ts @@ -1,4 +1,4 @@ -import { VideoCodec } from '@app/infra/entities'; +import { TranscodeTarget, VideoCodec } from '@app/infra/entities'; import { Writable } from 'node:stream'; export const IMediaRepository = 'IMediaRepository'; @@ -16,15 +16,14 @@ export interface VideoStreamInfo { width: number; rotation: number; codecName?: string; - codecType?: string; frameCount: number; isHDR: boolean; + bitrate: number; } export interface AudioStreamInfo { index: number; codecName?: string; - codecType?: string; frameCount: number; } @@ -64,7 +63,7 @@ export interface BitrateDistribution { } export interface VideoCodecSWConfig { - getOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeOptions; + getOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeOptions; } export interface VideoCodecHWConfig extends VideoCodecSWConfig { diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index b0aef8f2be..1515630cea 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -118,6 +118,13 @@ export enum TranscodePolicy { DISABLED = 'disabled', } +export enum TranscodeTarget { + NONE, + AUDIO, + VIDEO, + ALL, +} + export enum VideoCodec { H264 = 'h264', HEVC = 'hevc', diff --git a/server/src/infra/repositories/media.repository.ts b/server/src/infra/repositories/media.repository.ts index bb65dd25c8..a981bbc072 100644 --- a/server/src/infra/repositories/media.repository.ts +++ b/server/src/infra/repositories/media.repository.ts @@ -60,6 +60,7 @@ export class MediaRepository implements IMediaRepository { frameCount: Number.parseInt(stream.nb_frames ?? '0'), rotation: Number.parseInt(`${stream.rotation ?? 0}`), isHDR: stream.color_transfer === 'smpte2084' || stream.color_transfer === 'arib-std-b67', + bitrate: Number.parseInt(stream.bit_rate ?? '0'), })), audioStreams: results.streams .filter((stream) => stream.codec_type === 'audio') diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts index 30dfec1669..ad9a4baf72 100644 --- a/server/test/fixtures/media.stub.ts +++ b/server/test/fixtures/media.stub.ts @@ -13,16 +13,14 @@ const probeStubDefaultVideoStream: VideoStreamInfo[] = [ height: 1080, width: 1920, codecName: 'hevc', - codecType: 'video', frameCount: 100, rotation: 0, isHDR: false, + bitrate: 0, }, ]; -const probeStubDefaultAudioStream: AudioStreamInfo[] = [ - { index: 1, codecName: 'aac', codecType: 'audio', frameCount: 100 }, -]; +const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ index: 1, codecName: 'mp3', frameCount: 100 }]; const probeStubDefault: VideoInfo = { format: probeStubDefaultFormat, @@ -41,20 +39,20 @@ export const probeStub = { height: 1080, width: 400, codecName: 'hevc', - codecType: 'video', frameCount: 100, rotation: 0, isHDR: false, + bitrate: 0, }, { index: 1, height: 1080, width: 400, codecName: 'h7000', - codecType: 'video', frameCount: 99, rotation: 0, isHDR: false, + bitrate: 0, }, ], }), @@ -66,10 +64,10 @@ export const probeStub = { height: 0, width: 400, codecName: 'hevc', - codecType: 'video', frameCount: 100, rotation: 0, isHDR: false, + bitrate: 0, }, ], }), @@ -81,21 +79,16 @@ export const probeStub = { height: 2160, width: 3840, codecName: 'h264', - codecType: 'video', frameCount: 100, rotation: 0, isHDR: false, + bitrate: 0, }, ], }), videoStream40Mbps: Object.freeze({ ...probeStubDefault, - format: { - formatName: 'mov,mp4,m4a,3gp,3g2,mj2', - formatLongName: 'QuickTime / MOV', - duration: 0, - bitrate: 40_000_000, - }, + videoStreams: [{ ...probeStubDefaultVideoStream[0], bitrate: 40_000_000 }], }), videoStreamHDR: Object.freeze({ ...probeStubDefault, @@ -105,10 +98,10 @@ export const probeStub = { height: 480, width: 480, codecName: 'h264', - codecType: 'video', frameCount: 100, rotation: 0, isHDR: true, + bitrate: 0, }, ], }), @@ -120,10 +113,10 @@ export const probeStub = { height: 2160, width: 3840, codecName: 'h264', - codecType: 'video', frameCount: 100, rotation: 90, isHDR: false, + bitrate: 0, }, ], }), @@ -135,10 +128,10 @@ export const probeStub = { height: 355, width: 1586, codecName: 'h264', - codecType: 'video', frameCount: 100, rotation: 0, isHDR: false, + bitrate: 0, }, ], }), @@ -150,16 +143,16 @@ export const probeStub = { height: 1586, width: 355, codecName: 'h264', - codecType: 'video', frameCount: 100, rotation: 0, isHDR: false, + bitrate: 0, }, ], }), - audioStreamMp3: Object.freeze({ + audioStreamAac: Object.freeze({ ...probeStubDefault, - audioStreams: [{ index: 1, codecType: 'audio', codecName: 'aac', frameCount: 100 }], + audioStreams: [{ index: 1, codecName: 'aac', frameCount: 100 }], }), matroskaContainer: Object.freeze({ ...probeStubDefault, @@ -170,4 +163,14 @@ export const probeStub = { bitrate: 0, }, }), + videoStreamVp9: Object.freeze({ + ...probeStubDefault, + videoStreams: [{ ...probeStubDefaultVideoStream[0], codecName: 'vp9' }], + format: { + formatName: 'matroska,webm', + formatLongName: 'Matroska / WebM', + duration: 0, + bitrate: 0, + }, + }), };