diff --git a/server/package-lock.json b/server/package-lock.json index 3beec64001..6773dd30a5 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -58,6 +58,7 @@ "rxjs": "^7.8.1", "sanitize-filename": "^1.6.3", "sharp": "^0.33.0", + "shelljs": "^0.8.5", "sirv": "^2.0.4", "thumbhash": "^0.1.1", "typeorm": "^0.3.17", @@ -83,6 +84,7 @@ "@types/node": "^20.5.7", "@types/nodemailer": "^6.4.14", "@types/picomatch": "^2.3.3", + "@types/shelljs": "^0.8.15", "@types/supertest": "^6.0.0", "@types/ua-parser-js": "^0.7.36", "@typescript-eslint/eslint-plugin": "^7.0.0", @@ -5741,6 +5743,16 @@ "@types/node": "*" } }, + "node_modules/@types/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==", + "dev": true, + "dependencies": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, "node_modules/@types/http-assert": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@types/http-assert/-/http-assert-1.5.5.tgz", @@ -5847,6 +5859,12 @@ "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz", "integrity": "sha512-Ys+/St+2VF4+xuY6+kDIXGxbNRO0mesVg0bbxEfB97Od1Vjpjx9KD1qxs64Gcb3CWPirk9Xe+PT4YiiHQ9T+eg==" }, + "node_modules/@types/minimatch": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", + "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==", + "dev": true + }, "node_modules/@types/mock-fs": { "version": "4.13.4", "resolved": "https://registry.npmjs.org/@types/mock-fs/-/mock-fs-4.13.4.tgz", @@ -6043,6 +6061,16 @@ "@types/node": "*" } }, + "node_modules/@types/shelljs": { + "version": "0.8.15", + "resolved": "https://registry.npmjs.org/@types/shelljs/-/shelljs-0.8.15.tgz", + "integrity": "sha512-vzmnCHl6hViPu9GNLQJ+DZFd6BQI2DBTUeOvYHqkWQLMfKAAQYMb/xAmZkTogZI/vqXHCWkqDRymDI5p0QTi5Q==", + "dev": true, + "dependencies": { + "@types/glob": "~7.2.0", + "@types/node": "*" + } + }, "node_modules/@types/shimmer": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@types/shimmer/-/shimmer-1.0.5.tgz", @@ -20045,6 +20073,16 @@ "@types/node": "*" } }, + "@types/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==", + "dev": true, + "requires": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, "@types/http-assert": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@types/http-assert/-/http-assert-1.5.5.tgz", @@ -20151,6 +20189,12 @@ "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz", "integrity": "sha512-Ys+/St+2VF4+xuY6+kDIXGxbNRO0mesVg0bbxEfB97Od1Vjpjx9KD1qxs64Gcb3CWPirk9Xe+PT4YiiHQ9T+eg==" }, + "@types/minimatch": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-5.1.2.tgz", + "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==", + "dev": true + }, "@types/mock-fs": { "version": "4.13.4", "resolved": "https://registry.npmjs.org/@types/mock-fs/-/mock-fs-4.13.4.tgz", @@ -20334,6 +20378,16 @@ "@types/node": "*" } }, + "@types/shelljs": { + "version": "0.8.15", + "resolved": "https://registry.npmjs.org/@types/shelljs/-/shelljs-0.8.15.tgz", + "integrity": "sha512-vzmnCHl6hViPu9GNLQJ+DZFd6BQI2DBTUeOvYHqkWQLMfKAAQYMb/xAmZkTogZI/vqXHCWkqDRymDI5p0QTi5Q==", + "dev": true, + "requires": { + "@types/glob": "~7.2.0", + "@types/node": "*" + } + }, "@types/shimmer": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@types/shimmer/-/shimmer-1.0.5.tgz", diff --git a/server/package.json b/server/package.json index 328b58fdd7..cc2d0efcd5 100644 --- a/server/package.json +++ b/server/package.json @@ -82,6 +82,7 @@ "rxjs": "^7.8.1", "sanitize-filename": "^1.6.3", "sharp": "^0.33.0", + "shelljs": "^0.8.5", "sirv": "^2.0.4", "thumbhash": "^0.1.1", "typeorm": "^0.3.17", @@ -107,6 +108,7 @@ "@types/node": "^20.5.7", "@types/nodemailer": "^6.4.14", "@types/picomatch": "^2.3.3", + "@types/shelljs": "^0.8.15", "@types/supertest": "^6.0.0", "@types/ua-parser-js": "^0.7.36", "@typescript-eslint/eslint-plugin": "^7.0.0", diff --git a/server/src/interfaces/media.interface.ts b/server/src/interfaces/media.interface.ts index 092536b026..5cdae5b54d 100644 --- a/server/src/interfaces/media.interface.ts +++ b/server/src/interfaces/media.interface.ts @@ -66,6 +66,25 @@ export interface BitrateDistribution { unit: string; } +export enum VulkanDeviceType { + OTHER = 'OTHER', + INTEGRATED_GPU = 'INTEGRATED_GPU', + DISCRETE_GPU = 'DISCRETE_GPU', + VIRTUAL_GPU = 'VIRTUAL_GPU', + CPU = 'CPU', +} + +export interface VulkanDevice { + index: number; + type: VulkanDeviceType; +} + +export interface DeviceSummary { + driDevices: string[]; + hasOpenCL: boolean; + vulkanDevices: VulkanDevice[]; +} + export interface VideoCodecSWConfig { getOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeOptions; } @@ -84,4 +103,5 @@ export interface IMediaRepository { // video probe(input: string): Promise; transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise; + getVulkanDevices(): Promise; } diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts index 71c58fd91e..b662c55650 100644 --- a/server/src/repositories/media.repository.ts +++ b/server/src/repositories/media.repository.ts @@ -5,6 +5,7 @@ import fs from 'node:fs/promises'; import { Writable } from 'node:stream'; import { promisify } from 'node:util'; import sharp from 'sharp'; +import shell, { ShellString } from 'shelljs'; import { Colorspace } from 'src/entities/system-config.entity'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { @@ -13,11 +14,14 @@ import { ThumbnailOptions, TranscodeOptions, VideoInfo, + VulkanDevice, + VulkanDeviceType, } from 'src/interfaces/media.interface'; import { Instrumentation } from 'src/utils/instrumentation'; import { handlePromiseError } from 'src/utils/misc'; const probe = promisify(ffmpeg.ffprobe); +const exec = promisify(shell.exec); sharp.concurrency(0); sharp.cache({ files: 0 }); @@ -150,6 +154,31 @@ export class MediaRepository implements IMediaRepository { return { width, height }; } + async getVulkanDevices(): Promise { + return [ + { index: 0, type: VulkanDeviceType.DISCRETE_GPU }, + { index: 1, type: VulkanDeviceType.CPU }, + ]; + const devices = []; + let i = 0; + while (true) { + try { + const vulkanInfo = JSON.parse(await exec(`vulkaninfo --json=${i} -o /dev/tty`)); + const deviceType = + vulkanInfo['capabilities']['device']['properties']['VkPhysicalDeviceProperties']['deviceType']; + devices.push({ + index: i, + type: deviceType.replace('VK_PHYSICAL_DEVICE_TYPE_', ''), + }); + i++; + } catch { + break; + } + } + + return devices; + } + private configureFfmpegCall(input: string, output: string | Writable, options: TranscodeOptions) { return ffmpeg(input, { niceness: 10 }) .inputOptions(options.inputOptions) diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index 58322cc0b7..32867b1a4a 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -27,7 +27,13 @@ import { QueueName, } from 'src/interfaces/job.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; -import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from 'src/interfaces/media.interface'; +import { + AudioStreamInfo, + DeviceSummary, + IMediaRepository, + VideoCodecHWConfig, + VideoStreamInfo, +} from 'src/interfaces/media.interface'; import { IMoveRepository } from 'src/interfaces/move.interface'; import { IPersonRepository } from 'src/interfaces/person.interface'; import { IStorageRepository } from 'src/interfaces/storage.interface'; @@ -50,8 +56,7 @@ import { usePagination } from 'src/utils/pagination'; export class MediaService { private configCore: SystemConfigCore; private storageCore: StorageCore; - private openCL: boolean | null = null; - private devices: string[] | null = null; + private deviceSummary: DeviceSummary | null = null; constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, @@ -492,22 +497,23 @@ export class MediaService { } private async getHWCodecConfig(config: SystemConfigFFmpegDto) { + const deviceSummary = await this.getDeviceSummary(); let handler: VideoCodecHWConfig; switch (config.accel) { case TranscodeHWAccel.NVENC: { - handler = new NVENCConfig(config); + handler = new NVENCConfig(config, deviceSummary); break; } case TranscodeHWAccel.QSV: { - handler = new QSVConfig(config, await this.getDevices()); + handler = new QSVConfig(config, deviceSummary); break; } case TranscodeHWAccel.VAAPI: { - handler = new VAAPIConfig(config, await this.getDevices()); + handler = new VAAPIConfig(config, deviceSummary); break; } case TranscodeHWAccel.RKMPP: { - handler = new RKMPPConfig(config, await this.getDevices()); + handler = new RKMPPConfig(config, deviceSummary); break; } default: { @@ -559,26 +565,42 @@ export class MediaService { return extractedSize >= targetSize; } - private async getDevices() { - if (!this.devices) { - this.devices = await this.storageRepository.readdir('/dev/dri'); + private async getDeviceSummary(): Promise { + if (!this.deviceSummary) { + this.deviceSummary = { + driDevices: await this.getDriDevices(), + hasOpenCL: await this.hasOpenCL(), + vulkanDevices: await this.mediaRepository.getVulkanDevices(), + }; } - return this.devices; + return this.deviceSummary; + } + + private async getDriDevices() { + const devices = await this.storageRepository.readdir('/dev/dri'); + return devices + .filter((device) => device.startsWith('renderD') || device.startsWith('card')) + .sort((a, b) => { + // order GPU devices first + if (a.startsWith('card') && b.startsWith('renderD')) { + return -1; + } + if (a.startsWith('renderD') && b.startsWith('card')) { + return 1; + } + return -a.localeCompare(b); + }); } private async hasOpenCL() { - if (this.openCL === null) { - try { - const maliIcdStat = await this.storageRepository.stat('/etc/OpenCL/vendors/mali.icd'); - const maliDeviceStat = await this.storageRepository.stat('/dev/mali0'); - this.openCL = maliIcdStat.isFile() && maliDeviceStat.isCharacterDevice(); - } catch { - this.logger.warn('OpenCL not available for transcoding, using CPU instead.'); - this.openCL = false; - } + try { + const maliIcdStat = await this.storageRepository.stat('/etc/OpenCL/vendors/mali.icd'); + const maliDeviceStat = await this.storageRepository.stat('/dev/mali0'); + return maliIcdStat.isFile() && maliDeviceStat.isCharacterDevice(); + } catch { + this.logger.warn('OpenCL not available for transcoding, using CPU instead.'); + return false; } - - return this.openCL; } } diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index 04eae65e6a..802bd3fb67 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -3,10 +3,12 @@ import { CQMode, ToneMapping, TranscodeHWAccel, TranscodeTarget, VideoCodec } fr import { AudioStreamInfo, BitrateDistribution, + DeviceSummary, TranscodeOptions, VideoCodecHWConfig, VideoCodecSWConfig, VideoStreamInfo, + VulkanDeviceType, } from 'src/interfaces/media.interface'; class BaseConfig implements VideoCodecSWConfig { @@ -37,7 +39,7 @@ class BaseConfig implements VideoCodecSWConfig { // eslint-disable-next-line @typescript-eslint/no-unused-vars getBaseInputOptions(videoStream: VideoStreamInfo): string[] { - return [...this.getDeviceOptions(), ...this.getInputThreadOptions()]; + return this.getInputThreadOptions(); } getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { @@ -79,39 +81,20 @@ class BaseConfig implements VideoCodecSWConfig { } getFilterOptions(videoStream: VideoStreamInfo) { - const options = ['hwupload=derive_device=vulkan']; + const options = []; if (this.shouldScale(videoStream)) { const { width, height } = this.getSize(videoStream); - options.push(`scale_vulkan=w=${width}:h=${height}`); + options.push(`scale=w=${width}:h=${height}`); } - const colors = this.getColors(); - const libplaceboOptions = [ - 'format=yuv420p', - 'upscaler=none', - 'downscaler=none', - `tonemapping=${this.shouldToneMap(videoStream) ? this.config.tonemap : 'clip'}`, - `colorspace=${colors.matrix}`, - `color_primaries=${colors.primaries}`, - `color_trc=${colors.transfer}`, - ]; - - // use faster settings on cpu, nicer settings on gpu - if (this.config.accel === TranscodeHWAccel.DISABLED) { - libplaceboOptions.push('peak_detect=false'); - } else { - libplaceboOptions.push('deband=true', 'deband_iterations=3', 'deband_radius=8', 'deband_threshold=6'); + if (this.shouldToneMap(videoStream)) { + options.push(...this.getToneMapping(videoStream)); } + options.push('format=yuv420p'); - const libplacebo = `libplacebo=${libplaceboOptions.join(':')}`; - options.push(libplacebo, this.getFilterEnd(), 'format=yuv420p'); return options; } - getFilterEnd(): string { - return 'hwdownload'; - } - getPresetOptions() { return [`-preset ${this.config.preset}`]; } @@ -243,34 +226,17 @@ class BaseConfig implements VideoCodecSWConfig { } } - getDeviceOptions() { - return [ - `-init_hw_device ${this.getAccel()}=${this.getDevice()}`, - `-filter_hw_device ${this.getAccel()}`, - `-hwaccel ${this.getAccel()}`, - `-hwaccel_output_format ${this.getOutputFormat()}`, - ]; - } - - getDevice() { - let device = this.getAccel(); - if (this.getDeviceSpecifier() !== null) { - device += `:${this.getDeviceSpecifier()}`; + getToneMapping(videoStream: VideoStreamInfo) { + if (!this.shouldToneMap(videoStream)) { + return []; } - return device; - } - - getDeviceSpecifier(): string | null { - return null; - } - - getAccel() { - return 'vulkan'; - } - - getOutputFormat() { - return this.getAccel(); + const colors = this.getColors(); + return [ + `zscale=t=linear:npl=${this.getNPL()}`, + `tonemap=${this.config.tonemap}:desat=0`, + `zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`, + ]; } getAudioCodec(): string { @@ -299,35 +265,50 @@ class BaseConfig implements VideoCodecSWConfig { } export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { - protected devices: string[]; + protected deviceSummary: DeviceSummary; constructor( protected config: SystemConfigFFmpegDto, - devices: string[] = [], + deviceSummary: DeviceSummary, ) { super(config); - this.devices = this.validateDevices(devices); + this.deviceSummary = deviceSummary; + } + + getFilterOptions(videoStream: VideoStreamInfo) { + const options = ['hwupload=derive_device=vulkan']; + if (this.shouldScale(videoStream)) { + const { width, height } = this.getSize(videoStream); + options.push(`scale_vulkan=w=${width}:h=${height}`); + } + + options.push(...this.getToneMapping(videoStream), `hwupload=derive_device=${this.getAccel()}`); + return options; + } + + getToneMapping(videoStream: VideoStreamInfo) { + const colors = this.getColors(); + const libplaceboOptions = [ + `tonemapping=${this.shouldToneMap(videoStream) ? this.config.tonemap : 'clip'}`, + `colorspace=${colors.matrix}`, + `color_primaries=${colors.primaries}`, + `color_trc=${colors.transfer}`, + 'format=yuv420p', + 'deband=true', + 'deband_iterations=3', + 'deband_radius=8', + 'deband_threshold=6', + 'upscaler=none', + 'downscaler=none', + ]; + + return [`libplacebo=${libplaceboOptions.join(':')}`]; } getSupportedCodecs() { return [VideoCodec.H264, VideoCodec.HEVC]; } - validateDevices(devices: string[]) { - return devices - .filter((device) => device.startsWith('renderD') || device.startsWith('card')) - .sort((a, b) => { - // order GPU devices first - if (a.startsWith('card') && b.startsWith('renderD')) { - return -1; - } - if (a.startsWith('renderD') && b.startsWith('card')) { - return 1; - } - return -a.localeCompare(b); - }); - } - getVideoCodec(): string { return `${this.config.targetVideoCodec}_${this.config.accel}`; } @@ -346,23 +327,61 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { } const deviceName = device.replace('/dev/dri/', ''); - if (!this.devices.includes(deviceName)) { + if (!this.deviceSummary.driDevices.includes(deviceName)) { throw new Error(`Device '${device}' does not exist`); } return device; } + // eslint-disable-next-line @typescript-eslint/no-unused-vars + getBaseInputOptions(videoStream: VideoStreamInfo): string[] { + return [...this.getDeviceOptions(), ...this.getInputThreadOptions()]; + } + getInputThreadOptions() { - return [`-threads ${this.config.threads <= 0 ? 1 : this.config.threads}`]; + return []; } getOutputThreadOptions() { return []; } - getFilterEnd(): string { - return `hwupload=derive_device=${this.getAccel()}`; + getDeviceOptions() { + return [ + `-init_hw_device ${this.getAccel()}=${this.getDevice()}`, + `-filter_hw_device ${this.getAccel()}`, + `-hwaccel ${this.getAccel()}`, + `-hwaccel_output_format ${this.getOutputFormat()}`, + ]; + } + + getDevice() { + let device = this.getAccel(); + if (this.getDeviceSpecifier() !== null) { + device += `:${this.getDeviceSpecifier()}`; + } + + return device; + } + + getDeviceSpecifier(): string | null { + return null; + } + + getAccel() { + if (!this.deviceSummary.vulkanDevices.some((device) => device.type !== VulkanDeviceType.CPU)) { + return 'vulkan'; + } + return this.getPreferredAccel(); + } + + getPreferredAccel() { + return 'vulkan'; + } + + getOutputFormat() { + return this.getAccel(); } } @@ -488,7 +507,7 @@ export class AV1Config extends BaseConfig { } export class NVENCConfig extends BaseHWConfig { - getAccel() { + getPreferredAccel() { return 'cuda'; } @@ -561,18 +580,18 @@ export class NVENCConfig extends BaseHWConfig { } export class QSVConfig extends BaseHWConfig { - getAccel() { + getPreferredAccel() { return 'qsv'; } getDeviceSpecifier() { - if (this.devices.length === 0) { + if (this.deviceSummary.driDevices.length === 0) { throw new Error('No VAAPI device found'); } let hwDevice = this.getPreferredDevice(); if (hwDevice === null) { - hwDevice = `/dev/dri/${this.devices[0]}`; + hwDevice = `/dev/dri/${this.deviceSummary.driDevices[0]}`; } return hwDevice; @@ -631,18 +650,18 @@ export class QSVConfig extends BaseHWConfig { } export class VAAPIConfig extends BaseHWConfig { - getAccel() { + getPreferredAccel() { return 'vaapi'; } getDeviceSpecifier() { - if (this.devices.length === 0) { + if (this.deviceSummary.driDevices.length === 0) { throw new Error('No VAAPI device found'); } let hwDevice = this.getPreferredDevice(); if (hwDevice === null) { - hwDevice = `/dev/dri/${this.devices[0]}`; + hwDevice = `/dev/dri/${this.deviceSummary.driDevices[0]}`; } return hwDevice; @@ -697,14 +716,14 @@ export class RKMPPConfig extends BaseHWConfig { } getDeviceOptions(): string[] { - if (this.devices.length === 0) { + if (this.deviceSummary.driDevices.length === 0) { throw new Error('No RKMPP device found'); } return [...super.getDeviceOptions(), '-afbc rga']; } - getAccel() { + getPreferredAccel() { return 'rkmpp'; } @@ -741,8 +760,4 @@ export class RKMPPConfig extends BaseHWConfig { getSupportedCodecs() { return [VideoCodec.H264, VideoCodec.HEVC]; } - - getVideoCodec(): string { - return `${this.config.targetVideoCodec}_rkmpp`; - } }