c-l/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-28 06:31:58 +00:00
Code Releases Activity

use old filters for cpu and when vulkan gpu is not available

Browse source
This commit is contained in:
mertalev 2024-05-12 22:47:19 -04:00
parent 86f113cc96
commit 9ffc2e59ef
No known key found for this signature in database
GPG key ID: 13C97EF14A338984
6 changed files with 250 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

54
server/package-lock.json generated
View file

@ -58,6 +58,7 @@
"rxjs": "^7.8.1",
"sanitize-filename": "^1.6.3",
"sharp": "^0.33.0",
"shelljs": "^0.8.5",
"sirv": "^2.0.4",
"thumbhash": "^0.1.1",
"typeorm": "^0.3.17",
@ -83,6 +84,7 @@
"@types/node": "^20.5.7",
"@types/nodemailer": "^6.4.14",
"@types/picomatch": "^2.3.3",
"@types/shelljs": "^0.8.15",
"@types/supertest": "^6.0.0",
"@types/ua-parser-js": "^0.7.36",
"@typescript-eslint/eslint-plugin": "^7.0.0",
@ -5741,6 +5743,16 @@
"@types/node": "*"
}
},
"node_modules/@types/glob": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz",
"integrity": "sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==",
"dev": true,
"dependencies": {
"@types/minimatch": "*",
"@types/node": "*"
}
},
"node_modules/@types/http-assert": {
"version": "1.5.5",
"resolved": "https://registry.npmjs.org/@types/http-assert/-/http-assert-1.5.5.tgz",
@ -5847,6 +5859,12 @@
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz",
"integrity": "sha512-Ys+/St+2VF4+xuY6+kDIXGxbNRO0mesVg0bbxEfB97Od1Vjpjx9KD1qxs64Gcb3CWPirk9Xe+PT4YiiHQ9T+eg=="
},
"node_modules/@types/minimatch": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz",
"integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==",
"dev": true
},
"node_modules/@types/mock-fs": {
"version": "4.13.4",
"resolved": "https://registry.npmjs.org/@types/mock-fs/-/mock-fs-4.13.4.tgz",
@ -6043,6 +6061,16 @@
"@types/node": "*"
}
},
"node_modules/@types/shelljs": {
"version": "0.8.15",
"resolved": "https://registry.npmjs.org/@types/shelljs/-/shelljs-0.8.15.tgz",
"integrity": "sha512-vzmnCHl6hViPu9GNLQJ+DZFd6BQI2DBTUeOvYHqkWQLMfKAAQYMb/xAmZkTogZI/vqXHCWkqDRymDI5p0QTi5Q==",
"dev": true,
"dependencies": {
"@types/glob": "~7.2.0",
"@types/node": "*"
}
},
"node_modules/@types/shimmer": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@types/shimmer/-/shimmer-1.0.5.tgz",
@ -20045,6 +20073,16 @@
"@types/node": "*"
}
},
"@types/glob": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz",
"integrity": "sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==",
"dev": true,
"requires": {
"@types/minimatch": "*",
"@types/node": "*"
}
},
"@types/http-assert": {
"version": "1.5.5",
"resolved": "https://registry.npmjs.org/@types/http-assert/-/http-assert-1.5.5.tgz",
@ -20151,6 +20189,12 @@
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz",
"integrity": "sha512-Ys+/St+2VF4+xuY6+kDIXGxbNRO0mesVg0bbxEfB97Od1Vjpjx9KD1qxs64Gcb3CWPirk9Xe+PT4YiiHQ9T+eg=="
},
"@types/minimatch": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz",
"integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==",
"dev": true
},
"@types/mock-fs": {
"version": "4.13.4",
"resolved": "https://registry.npmjs.org/@types/mock-fs/-/mock-fs-4.13.4.tgz",
@ -20334,6 +20378,16 @@
"@types/node": "*"
}
},
"@types/shelljs": {
"version": "0.8.15",
"resolved": "https://registry.npmjs.org/@types/shelljs/-/shelljs-0.8.15.tgz",
"integrity": "sha512-vzmnCHl6hViPu9GNLQJ+DZFd6BQI2DBTUeOvYHqkWQLMfKAAQYMb/xAmZkTogZI/vqXHCWkqDRymDI5p0QTi5Q==",
"dev": true,
"requires": {
"@types/glob": "~7.2.0",
"@types/node": "*"
}
},
"@types/shimmer": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@types/shimmer/-/shimmer-1.0.5.tgz",

2
server/package.json
View file

@ -82,6 +82,7 @@
"rxjs": "^7.8.1",
"sanitize-filename": "^1.6.3",
"sharp": "^0.33.0",
"shelljs": "^0.8.5",
"sirv": "^2.0.4",
"thumbhash": "^0.1.1",
"typeorm": "^0.3.17",
@ -107,6 +108,7 @@
"@types/node": "^20.5.7",
"@types/nodemailer": "^6.4.14",
"@types/picomatch": "^2.3.3",
"@types/shelljs": "^0.8.15",
"@types/supertest": "^6.0.0",
"@types/ua-parser-js": "^0.7.36",
"@typescript-eslint/eslint-plugin": "^7.0.0",

20
server/src/interfaces/media.interface.ts
View file

@ -66,6 +66,25 @@ export interface BitrateDistribution {
unit: string;
}
export enum VulkanDeviceType {
OTHER = 'OTHER',
INTEGRATED_GPU = 'INTEGRATED_GPU',
DISCRETE_GPU = 'DISCRETE_GPU',
VIRTUAL_GPU = 'VIRTUAL_GPU',
CPU = 'CPU',
}
export interface VulkanDevice {
index: number;
type: VulkanDeviceType;
}
export interface DeviceSummary {
driDevices: string[];
hasOpenCL: boolean;
vulkanDevices: VulkanDevice[];
}
export interface VideoCodecSWConfig {
getOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeOptions;
}
@ -84,4 +103,5 @@ export interface IMediaRepository {
// video
probe(input: string): Promise<VideoInfo>;
transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise<void>;
getVulkanDevices(): Promise<VulkanDevice[]>;
}

29
server/src/repositories/media.repository.ts
View file

@ -5,6 +5,7 @@ import fs from 'node:fs/promises';
import { Writable } from 'node:stream';
import { promisify } from 'node:util';
import sharp from 'sharp';
import shell, { ShellString } from 'shelljs';
import { Colorspace } from 'src/entities/system-config.entity';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import {
@ -13,11 +14,14 @@ import {
ThumbnailOptions,
TranscodeOptions,
VideoInfo,
VulkanDevice,
VulkanDeviceType,
} from 'src/interfaces/media.interface';
import { Instrumentation } from 'src/utils/instrumentation';
import { handlePromiseError } from 'src/utils/misc';
const probe = promisify<string, FfprobeData>(ffmpeg.ffprobe);
const exec = promisify<string, ShellString>(shell.exec);
sharp.concurrency(0);
sharp.cache({ files: 0 });
@ -150,6 +154,31 @@ export class MediaRepository implements IMediaRepository {
return { width, height };
}
async getVulkanDevices(): Promise<VulkanDevice[]> {
return [
{ index: 0, type: VulkanDeviceType.DISCRETE_GPU },
{ index: 1, type: VulkanDeviceType.CPU },
];
const devices = [];
let i = 0;
while (true) {
try {
const vulkanInfo = JSON.parse(await exec(`vulkaninfo --json=${i} -o /dev/tty`));
const deviceType =
vulkanInfo['capabilities']['device']['properties']['VkPhysicalDeviceProperties']['deviceType'];
devices.push({
index: i,
type: deviceType.replace('VK_PHYSICAL_DEVICE_TYPE_', ''),
});
i++;
} catch {
break;
}
}
return devices;
}
private configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) {
return ffmpeg(input, { niceness: 10 })
.inputOptions(options.inputOptions)

66
server/src/services/media.service.ts
View file

@ -27,7 +27,13 @@ import {
QueueName,
} from 'src/interfaces/job.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from 'src/interfaces/media.interface';
import {
AudioStreamInfo,
DeviceSummary,
IMediaRepository,
VideoCodecHWConfig,
VideoStreamInfo,
} from 'src/interfaces/media.interface';
import { IMoveRepository } from 'src/interfaces/move.interface';
import { IPersonRepository } from 'src/interfaces/person.interface';
import { IStorageRepository } from 'src/interfaces/storage.interface';
@ -50,8 +56,7 @@ import { usePagination } from 'src/utils/pagination';
export class MediaService {
private configCore: SystemConfigCore;
private storageCore: StorageCore;
private openCL: boolean | null = null;
private devices: string[] | null = null;
private deviceSummary: DeviceSummary | null = null;
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@ -492,22 +497,23 @@ export class MediaService {
}
private async getHWCodecConfig(config: SystemConfigFFmpegDto) {
const deviceSummary = await this.getDeviceSummary();
let handler: VideoCodecHWConfig;
switch (config.accel) {
case TranscodeHWAccel.NVENC: {
handler = new NVENCConfig(config);
handler = new NVENCConfig(config, deviceSummary);
break;
}
case TranscodeHWAccel.QSV: {
handler = new QSVConfig(config, await this.getDevices());
handler = new QSVConfig(config, deviceSummary);
break;
}
case TranscodeHWAccel.VAAPI: {
handler = new VAAPIConfig(config, await this.getDevices());
handler = new VAAPIConfig(config, deviceSummary);
break;
}
case TranscodeHWAccel.RKMPP: {
handler = new RKMPPConfig(config, await this.getDevices());
handler = new RKMPPConfig(config, deviceSummary);
break;
}
default: {
@ -559,26 +565,42 @@ export class MediaService {
return extractedSize >= targetSize;
}
private async getDevices() {
if (!this.devices) {
this.devices = await this.storageRepository.readdir('/dev/dri');
private async getDeviceSummary(): Promise<DeviceSummary> {
if (!this.deviceSummary) {
this.deviceSummary = {
driDevices: await this.getDriDevices(),
hasOpenCL: await this.hasOpenCL(),
vulkanDevices: await this.mediaRepository.getVulkanDevices(),
};
}
return this.devices;
return this.deviceSummary;
}
private async getDriDevices() {
const devices = await this.storageRepository.readdir('/dev/dri');
return devices
.filter((device) => device.startsWith('renderD') || device.startsWith('card'))
.sort((a, b) => {
// order GPU devices first
if (a.startsWith('card') && b.startsWith('renderD')) {
return -1;
}
if (a.startsWith('renderD') && b.startsWith('card')) {
return 1;
}
return -a.localeCompare(b);
});
}
private async hasOpenCL() {
if (this.openCL === null) {
try {
const maliIcdStat = await this.storageRepository.stat('/etc/OpenCL/vendors/mali.icd');
const maliDeviceStat = await this.storageRepository.stat('/dev/mali0');
this.openCL = maliIcdStat.isFile() && maliDeviceStat.isCharacterDevice();
} catch {
this.logger.warn('OpenCL not available for transcoding, using CPU instead.');
this.openCL = false;
}
try {
const maliIcdStat = await this.storageRepository.stat('/etc/OpenCL/vendors/mali.icd');
const maliDeviceStat = await this.storageRepository.stat('/dev/mali0');
return maliIcdStat.isFile() && maliDeviceStat.isCharacterDevice();
} catch {
this.logger.warn('OpenCL not available for transcoding, using CPU instead.');
return false;
}
return this.openCL;
}
}

187
server/src/utils/media.ts
View file

@ -3,10 +3,12 @@ import { CQMode, ToneMapping, TranscodeHWAccel, TranscodeTarget, VideoCodec } fr
import {
AudioStreamInfo,
BitrateDistribution,
DeviceSummary,
TranscodeOptions,
VideoCodecHWConfig,
VideoCodecSWConfig,
VideoStreamInfo,
VulkanDeviceType,
} from 'src/interfaces/media.interface';
class BaseConfig implements VideoCodecSWConfig {
@ -37,7 +39,7 @@ class BaseConfig implements VideoCodecSWConfig {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
getBaseInputOptions(videoStream: VideoStreamInfo): string[] {
return [...this.getDeviceOptions(), ...this.getInputThreadOptions()];
return this.getInputThreadOptions();
}
getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
@ -79,39 +81,20 @@ class BaseConfig implements VideoCodecSWConfig {
}
getFilterOptions(videoStream: VideoStreamInfo) {
const options = ['hwupload=derive_device=vulkan'];
const options = [];
if (this.shouldScale(videoStream)) {
const { width, height } = this.getSize(videoStream);
options.push(`scale_vulkan=w=${width}:h=${height}`);
options.push(`scale=w=${width}:h=${height}`);
}
const colors = this.getColors();
const libplaceboOptions = [
'format=yuv420p',
'upscaler=none',
'downscaler=none',
`tonemapping=${this.shouldToneMap(videoStream) ? this.config.tonemap : 'clip'}`,
`colorspace=${colors.matrix}`,
`color_primaries=${colors.primaries}`,
`color_trc=${colors.transfer}`,
];
// use faster settings on cpu, nicer settings on gpu
if (this.config.accel === TranscodeHWAccel.DISABLED) {
libplaceboOptions.push('peak_detect=false');
} else {
libplaceboOptions.push('deband=true', 'deband_iterations=3', 'deband_radius=8', 'deband_threshold=6');
if (this.shouldToneMap(videoStream)) {
options.push(...this.getToneMapping(videoStream));
}
options.push('format=yuv420p');
const libplacebo = `libplacebo=${libplaceboOptions.join(':')}`;
options.push(libplacebo, this.getFilterEnd(), 'format=yuv420p');
return options;
}
getFilterEnd(): string {
return 'hwdownload';
}
getPresetOptions() {
return [`-preset ${this.config.preset}`];
}
@ -243,34 +226,17 @@ class BaseConfig implements VideoCodecSWConfig {
}
}
getDeviceOptions() {
return [
`-init_hw_device ${this.getAccel()}=${this.getDevice()}`,
`-filter_hw_device ${this.getAccel()}`,
`-hwaccel ${this.getAccel()}`,
`-hwaccel_output_format ${this.getOutputFormat()}`,
];
}
getDevice() {
let device = this.getAccel();
if (this.getDeviceSpecifier() !== null) {
device += `:${this.getDeviceSpecifier()}`;
getToneMapping(videoStream: VideoStreamInfo) {
if (!this.shouldToneMap(videoStream)) {
return [];
}
return device;
}
getDeviceSpecifier(): string | null {
return null;
}
getAccel() {
return 'vulkan';
}
getOutputFormat() {
return this.getAccel();
const colors = this.getColors();
return [
`zscale=t=linear:npl=${this.getNPL()}`,
`tonemap=${this.config.tonemap}:desat=0`,
`zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`,
];
}
getAudioCodec(): string {
@ -299,35 +265,50 @@ class BaseConfig implements VideoCodecSWConfig {
}
export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
protected devices: string[];
protected deviceSummary: DeviceSummary;
constructor(
protected config: SystemConfigFFmpegDto,
devices: string[] = [],
deviceSummary: DeviceSummary,
) {
super(config);
this.devices = this.validateDevices(devices);
this.deviceSummary = deviceSummary;
}
getFilterOptions(videoStream: VideoStreamInfo) {
const options = ['hwupload=derive_device=vulkan'];
if (this.shouldScale(videoStream)) {
const { width, height } = this.getSize(videoStream);
options.push(`scale_vulkan=w=${width}:h=${height}`);
}
options.push(...this.getToneMapping(videoStream), `hwupload=derive_device=${this.getAccel()}`);
return options;
}
getToneMapping(videoStream: VideoStreamInfo) {
const colors = this.getColors();
const libplaceboOptions = [
`tonemapping=${this.shouldToneMap(videoStream) ? this.config.tonemap : 'clip'}`,
`colorspace=${colors.matrix}`,
`color_primaries=${colors.primaries}`,
`color_trc=${colors.transfer}`,
'format=yuv420p',
'deband=true',
'deband_iterations=3',
'deband_radius=8',
'deband_threshold=6',
'upscaler=none',
'downscaler=none',
];
return [`libplacebo=${libplaceboOptions.join(':')}`];
}
getSupportedCodecs() {
return [VideoCodec.H264, VideoCodec.HEVC];
}
validateDevices(devices: string[]) {
return devices
.filter((device) => device.startsWith('renderD') || device.startsWith('card'))
.sort((a, b) => {
// order GPU devices first
if (a.startsWith('card') && b.startsWith('renderD')) {
return -1;
}
if (a.startsWith('renderD') && b.startsWith('card')) {
return 1;
}
return -a.localeCompare(b);
});
}
getVideoCodec(): string {
return `${this.config.targetVideoCodec}_${this.config.accel}`;
}
@ -346,23 +327,61 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
}
const deviceName = device.replace('/dev/dri/', '');
if (!this.devices.includes(deviceName)) {
if (!this.deviceSummary.driDevices.includes(deviceName)) {
throw new Error(`Device '${device}' does not exist`);
}
return device;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
getBaseInputOptions(videoStream: VideoStreamInfo): string[] {
return [...this.getDeviceOptions(), ...this.getInputThreadOptions()];
}
getInputThreadOptions() {
return [`-threads ${this.config.threads <= 0 ? 1 : this.config.threads}`];
return [];
}
getOutputThreadOptions() {
return [];
}
getFilterEnd(): string {
return `hwupload=derive_device=${this.getAccel()}`;
getDeviceOptions() {
return [
`-init_hw_device ${this.getAccel()}=${this.getDevice()}`,
`-filter_hw_device ${this.getAccel()}`,
`-hwaccel ${this.getAccel()}`,
`-hwaccel_output_format ${this.getOutputFormat()}`,
];
}
getDevice() {
let device = this.getAccel();
if (this.getDeviceSpecifier() !== null) {
device += `:${this.getDeviceSpecifier()}`;
}
return device;
}
getDeviceSpecifier(): string | null {
return null;
}
getAccel() {
if (!this.deviceSummary.vulkanDevices.some((device) => device.type !== VulkanDeviceType.CPU)) {
return 'vulkan';
}
return this.getPreferredAccel();
}
getPreferredAccel() {
return 'vulkan';
}
getOutputFormat() {
return this.getAccel();
}
}
@ -488,7 +507,7 @@ export class AV1Config extends BaseConfig {
}
export class NVENCConfig extends BaseHWConfig {
getAccel() {
getPreferredAccel() {
return 'cuda';
}
@ -561,18 +580,18 @@ export class NVENCConfig extends BaseHWConfig {
}
export class QSVConfig extends BaseHWConfig {
getAccel() {
getPreferredAccel() {
return 'qsv';
}
getDeviceSpecifier() {
if (this.devices.length === 0) {
if (this.deviceSummary.driDevices.length === 0) {
throw new Error('No VAAPI device found');
}
let hwDevice = this.getPreferredDevice();
if (hwDevice === null) {
hwDevice = `/dev/dri/${this.devices[0]}`;
hwDevice = `/dev/dri/${this.deviceSummary.driDevices[0]}`;
}
return hwDevice;
@ -631,18 +650,18 @@ export class QSVConfig extends BaseHWConfig {
}
export class VAAPIConfig extends BaseHWConfig {
getAccel() {
getPreferredAccel() {
return 'vaapi';
}
getDeviceSpecifier() {
if (this.devices.length === 0) {
if (this.deviceSummary.driDevices.length === 0) {
throw new Error('No VAAPI device found');
}
let hwDevice = this.getPreferredDevice();
if (hwDevice === null) {
hwDevice = `/dev/dri/${this.devices[0]}`;
hwDevice = `/dev/dri/${this.deviceSummary.driDevices[0]}`;
}
return hwDevice;
@ -697,14 +716,14 @@ export class RKMPPConfig extends BaseHWConfig {
}
getDeviceOptions(): string[] {
if (this.devices.length === 0) {
if (this.deviceSummary.driDevices.length === 0) {
throw new Error('No RKMPP device found');
}
return [...super.getDeviceOptions(), '-afbc rga'];
}
getAccel() {
getPreferredAccel() {
return 'rkmpp';
}
@ -741,8 +760,4 @@ export class RKMPPConfig extends BaseHWConfig {
getSupportedCodecs() {
return [VideoCodec.H264, VideoCodec.HEVC];
}
getVideoCodec(): string {
return `${this.config.targetVideoCodec}_rkmpp`;
}
}