mirror of
https://github.com/immich-app/immich.git
synced 2024-12-28 06:31:58 +00:00
fix(server): use hw decoding for rkmpp w/o OpenCL if possible (#13848)
* Set hardware decoding options for rkmpp when hardware decoding is enabled with no OpenCL on non-HDR file * Use hw decoding, sw tone-mapping on HDR files using RKMPP w/o OpenCL * fallback to software decoding if is hdr video * if hw decoding failed with hw dec config enabled, try sw dec+hw enc first, then full sw dec+enc * fix unit test * fix format, adjust log message * formatting --------- Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
This commit is contained in:
San
GitHub
parent
1935b88d13
commit
1c82804f63
3 changed files with 81 additions and 21 deletions
|
|
@ -2149,7 +2149,7 @@ describe(MediaService.name, () => {
|
|||
'-map 0:1',
|
||||
'-g 256',
|
||||
'-v verbose',
|
||||
'-vf scale_rkrga=-2:720:format=nv12:afbc=1',
|
||||
'-vf scale_rkrga=-2:720:format=nv12:afbc=1:async_depth=4',
|
||||
'-level 51',
|
||||
'-rc_mode CQP',
|
||||
'-qp_init 23',
|
||||
|
|
@ -2220,7 +2220,7 @@ describe(MediaService.name, () => {
|
|||
inputOptions: expect.arrayContaining(['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga']),
|
||||
outputOptions: expect.arrayContaining([
|
||||
expect.stringContaining(
|
||||
'scale_rkrga=-2:720:format=p010:afbc=1,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0:tonemap_mode=lum:peak=100,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime',
|
||||
'scale_rkrga=-2:720:format=p010:afbc=1:async_depth=4,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0:tonemap_mode=lum:peak=100,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime',
|
||||
),
|
||||
]),
|
||||
twoPass: false,
|
||||
|
|
@ -2228,6 +2228,28 @@ describe(MediaService.name, () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('should set hardware decoding options for rkmpp when hardware decoding is enabled with no OpenCL on non-HDR file', async () => {
|
||||
storageMock.readdir.mockResolvedValue(['renderD128']);
|
||||
storageMock.stat.mockResolvedValue({ isFile: () => false, isCharacterDevice: () => false } as Stats);
|
||||
mediaMock.probe.mockResolvedValue(probeStub.noAudioStreams);
|
||||
systemMock.get.mockResolvedValue({
|
||||
ffmpeg: { accel: TranscodeHWAccel.RKMPP, accelDecode: true, crf: 30, maxBitrate: '0' },
|
||||
});
|
||||
assetMock.getByIds.mockResolvedValue([assetStub.video]);
|
||||
await sut.handleVideoConversion({ id: assetStub.video.id });
|
||||
expect(mediaMock.transcode).toHaveBeenCalledWith(
|
||||
'/original/path.ext',
|
||||
'upload/encoded-video/user-id/as/se/asset-id.mp4',
|
||||
expect.objectContaining({
|
||||
inputOptions: expect.arrayContaining(['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga']),
|
||||
outputOptions: expect.arrayContaining([
|
||||
expect.stringContaining('scale_rkrga=-2:720:format=nv12:afbc=1:async_depth=4'),
|
||||
]),
|
||||
twoPass: false,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('should use software decoding and tone-mapping if hardware decoding is disabled', async () => {
|
||||
storageMock.readdir.mockResolvedValue(['renderD128']);
|
||||
storageMock.stat.mockResolvedValue({ isFile: () => true, isCharacterDevice: () => true } as Stats);
|
||||
|
|
@ -2252,7 +2274,7 @@ describe(MediaService.name, () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('should use software decoding and tone-mapping if opencl is not available', async () => {
|
||||
it('should use software tone-mapping if opencl is not available', async () => {
|
||||
storageMock.readdir.mockResolvedValue(['renderD128']);
|
||||
storageMock.stat.mockResolvedValue({ isFile: () => false, isCharacterDevice: () => false } as Stats);
|
||||
mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
|
||||
|
|
@ -2265,7 +2287,7 @@ describe(MediaService.name, () => {
|
|||
'/original/path.ext',
|
||||
'upload/encoded-video/user-id/as/se/asset-id.mp4',
|
||||
expect.objectContaining({
|
||||
inputOptions: [],
|
||||
inputOptions: expect.any(Array),
|
||||
outputOptions: expect.arrayContaining([
|
||||
expect.stringContaining(
|
||||
'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p',
|
||||
|
|
|
|||
|
|
@ -335,9 +335,11 @@ export class MediaService extends BaseService {
|
|||
}
|
||||
|
||||
if (ffmpeg.accel === TranscodeHWAccel.DISABLED) {
|
||||
this.logger.log(`Encoding video ${asset.id} without hardware acceleration`);
|
||||
this.logger.log(`Transcoding video ${asset.id} without hardware acceleration`);
|
||||
} else {
|
||||
this.logger.log(`Encoding video ${asset.id} with ${ffmpeg.accel.toUpperCase()} acceleration`);
|
||||
this.logger.log(
|
||||
`Transcoding video ${asset.id} with ${ffmpeg.accel.toUpperCase()}-accelerated encoding and${ffmpeg.accelDecode ? '' : ' software'} decoding`,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
|
|
@ -347,10 +349,26 @@ export class MediaService extends BaseService {
|
|||
if (ffmpeg.accel === TranscodeHWAccel.DISABLED) {
|
||||
return JobStatus.FAILED;
|
||||
}
|
||||
this.logger.error(`Retrying with ${ffmpeg.accel.toUpperCase()} acceleration disabled`);
|
||||
const config = BaseConfig.create({ ...ffmpeg, accel: TranscodeHWAccel.DISABLED });
|
||||
command = config.getCommand(target, mainVideoStream, mainAudioStream);
|
||||
await this.mediaRepository.transcode(input, output, command);
|
||||
|
||||
let partialFallbackSuccess = false;
|
||||
if (ffmpeg.accelDecode) {
|
||||
try {
|
||||
this.logger.error(`Retrying with ${ffmpeg.accel.toUpperCase()}-accelerated encoding and software decoding`);
|
||||
const config = BaseConfig.create({ ...ffmpeg, accelDecode: false });
|
||||
command = config.getCommand(target, mainVideoStream, mainAudioStream);
|
||||
await this.mediaRepository.transcode(input, output, command);
|
||||
partialFallbackSuccess = true;
|
||||
} catch (error: any) {
|
||||
this.logger.error(`Error occurred during transcoding: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!partialFallbackSuccess) {
|
||||
this.logger.error(`Retrying with ${ffmpeg.accel.toUpperCase()} acceleration disabled`);
|
||||
const config = BaseConfig.create({ ...ffmpeg, accel: TranscodeHWAccel.DISABLED });
|
||||
command = config.getCommand(target, mainVideoStream, mainAudioStream);
|
||||
await this.mediaRepository.transcode(input, output, command);
|
||||
}
|
||||
}
|
||||
|
||||
this.logger.log(`Successfully encoded ${asset.id}`);
|
||||
|
|
@ -508,7 +526,7 @@ export class MediaService extends BaseService {
|
|||
const maliDeviceStat = await this.storageRepository.stat('/dev/mali0');
|
||||
this.maliOpenCL = maliIcdStat.isFile() && maliDeviceStat.isCharacterDevice();
|
||||
} catch {
|
||||
this.logger.debug('OpenCL not available for transcoding, so RKMPP acceleration will use CPU decoding');
|
||||
this.logger.debug('OpenCL not available for transcoding, so RKMPP acceleration will use CPU tonemapping');
|
||||
this.maliOpenCL = false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -59,10 +59,9 @@ export class BaseConfig implements VideoCodecSWConfig {
|
|||
break;
|
||||
}
|
||||
case TranscodeHWAccel.RKMPP: {
|
||||
handler =
|
||||
config.accelDecode && hasMaliOpenCL
|
||||
? new RkmppHwDecodeConfig(config, devices)
|
||||
: new RkmppSwDecodeConfig(config, devices);
|
||||
handler = config.accelDecode
|
||||
? new RkmppHwDecodeConfig(config, devices, hasMaliOpenCL)
|
||||
: new RkmppSwDecodeConfig(config, devices);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
|
|
@ -977,6 +976,16 @@ export class RkmppSwDecodeConfig extends BaseHWConfig {
|
|||
}
|
||||
|
||||
export class RkmppHwDecodeConfig extends RkmppSwDecodeConfig {
|
||||
protected hasMaliOpenCL: boolean;
|
||||
constructor(
|
||||
protected config: SystemConfigFFmpegDto,
|
||||
devices: string[] = [],
|
||||
hasMaliOpenCL = false,
|
||||
) {
|
||||
super(config, devices);
|
||||
this.hasMaliOpenCL = hasMaliOpenCL;
|
||||
}
|
||||
|
||||
getBaseInputOptions() {
|
||||
if (this.devices.length === 0) {
|
||||
throw new Error('No RKMPP device found');
|
||||
|
|
@ -988,15 +997,26 @@ export class RkmppHwDecodeConfig extends RkmppSwDecodeConfig {
|
|||
getFilterOptions(videoStream: VideoStreamInfo) {
|
||||
if (this.shouldToneMap(videoStream)) {
|
||||
const { primaries, transfer, matrix } = this.getColors();
|
||||
if (this.hasMaliOpenCL) {
|
||||
return [
|
||||
// use RKMPP for scaling, OpenCL for tone mapping
|
||||
`scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1:async_depth=4`,
|
||||
'hwmap=derive_device=opencl:mode=read',
|
||||
`tonemap_opencl=format=nv12:r=pc:p=${primaries}:t=${transfer}:m=${matrix}:tonemap=${this.config.tonemap}:desat=0:tonemap_mode=lum:peak=100`,
|
||||
'hwmap=derive_device=rkmpp:mode=write:reverse=1',
|
||||
'format=drm_prime',
|
||||
];
|
||||
}
|
||||
return [
|
||||
`scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1`,
|
||||
'hwmap=derive_device=opencl:mode=read',
|
||||
`tonemap_opencl=format=nv12:r=pc:p=${primaries}:t=${transfer}:m=${matrix}:tonemap=${this.config.tonemap}:desat=0:tonemap_mode=lum:peak=100`,
|
||||
'hwmap=derive_device=rkmpp:mode=write:reverse=1',
|
||||
'format=drm_prime',
|
||||
// use RKMPP for scaling, CPU for tone mapping (only works on RK3588, which supports 10-bit output)
|
||||
`scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1:async_depth=4`,
|
||||
'hwdownload',
|
||||
'format=p010',
|
||||
`tonemapx=tonemap=${this.config.tonemap}:desat=0:p=${primaries}:t=${transfer}:m=${matrix}:r=pc:peak=100:format=yuv420p`,
|
||||
'hwupload',
|
||||
];
|
||||
} else if (this.shouldScale(videoStream)) {
|
||||
return [`scale_rkrga=${this.getScaling(videoStream)}:format=nv12:afbc=1`];
|
||||
return [`scale_rkrga=${this.getScaling(videoStream)}:format=nv12:afbc=1:async_depth=4`];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue