diff --git a/server/src/config.ts b/server/src/config.ts index 3317351f9f..53374d581f 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -20,7 +20,7 @@ import { VideoContainer, } from 'src/enum'; import { ConcurrentQueueName, QueueName } from 'src/interfaces/job.interface'; -import { ImageOutputConfig } from 'src/interfaces/media.interface'; +import { ImageOptions } from 'src/interfaces/media.interface'; export interface SystemConfig { ffmpeg: { @@ -110,8 +110,8 @@ export interface SystemConfig { template: string; }; image: { - thumbnail: ImageOutputConfig; - preview: ImageOutputConfig; + thumbnail: ImageOptions; + preview: ImageOptions; colorspace: Colorspace; extractEmbedded: boolean; }; diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index c12a54cd61..039dbd20ff 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -492,7 +492,7 @@ class SystemConfigGeneratedImageDto { size!: number; } -class SystemConfigImageDto { +export class SystemConfigImageDto { @Type(() => SystemConfigGeneratedImageDto) @ValidateNested() @IsObject() diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index c6808e3aa8..750a852094 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -141,6 +141,12 @@ export interface AssetUpdateDuplicateOptions { duplicateIds: string[]; } +export interface UpsertFileOptions { + assetId: string; + type: AssetFileType; + path: string; +} + export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>; export const IAssetRepository = 'IAssetRepository'; @@ -194,5 +200,6 @@ export interface IAssetRepository { getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]>; getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]>; getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>; - upsertFile(options: { assetId: string; type: AssetFileType; path: string }): Promise<void>; + upsertFile(file: UpsertFileOptions): Promise<void>; + upsertFiles(files: UpsertFileOptions[]): Promise<void>; } diff --git a/server/src/interfaces/job.interface.ts b/server/src/interfaces/job.interface.ts index af2726b858..aa3090675e 100644 --- a/server/src/interfaces/job.interface.ts +++ b/server/src/interfaces/job.interface.ts @@ -37,9 +37,7 @@ export enum JobName { // thumbnails QUEUE_GENERATE_THUMBNAILS = 'queue-generate-thumbnails', - GENERATE_PREVIEW = 'generate-preview', - GENERATE_THUMBNAIL = 'generate-thumbnail', - GENERATE_THUMBHASH = 'generate-thumbhash', + GENERATE_THUMBNAILS = 'generate-thumbnails', GENERATE_PERSON_THUMBNAIL = 'generate-person-thumbnail', // metadata @@ -212,9 +210,7 @@ export type JobItem = // Thumbnails | { name: JobName.QUEUE_GENERATE_THUMBNAILS; data: IBaseJob } - | { name: JobName.GENERATE_PREVIEW; data: IEntityJob } - | { name: JobName.GENERATE_THUMBNAIL; data: IEntityJob } - | { name: JobName.GENERATE_THUMBHASH; data: IEntityJob } + | { name: JobName.GENERATE_THUMBNAILS; data: IEntityJob } // User | { name: JobName.USER_DELETE_CHECK; data?: IBaseJob } diff --git a/server/src/interfaces/media.interface.ts b/server/src/interfaces/media.interface.ts index 64ba6236e8..2bc8ccde36 100644 --- a/server/src/interfaces/media.interface.ts +++ b/server/src/interfaces/media.interface.ts @@ -10,16 +10,44 @@ export interface CropOptions { height: number; } -export interface ImageOutputConfig { +export interface ImageOptions { format: ImageFormat; quality: number; size: number; } -export interface ThumbnailOptions extends ImageOutputConfig { +export interface RawImageInfo { + width: number; + height: number; + channels: 1 | 2 | 3 | 4; +} + +interface DecodeImageOptions { colorspace: string; crop?: CropOptions; processInvalidImages: boolean; + raw?: RawImageInfo; +} + +export interface DecodeToBufferOptions extends DecodeImageOptions { + size: number; +} + +export type GenerateThumbnailOptions = ImageOptions & DecodeImageOptions; + +export type GenerateThumbnailFromBufferOptions = GenerateThumbnailOptions & { raw: RawImageInfo }; + +export type GenerateThumbhashOptions = DecodeImageOptions; + +export type GenerateThumbhashFromBufferOptions = GenerateThumbhashOptions & { raw: RawImageInfo }; + +export interface GenerateThumbnailsOptions { + colorspace: string; + crop?: CropOptions; + preview?: ImageOptions; + processInvalidImages: boolean; + thumbhash?: boolean; + thumbnail?: ImageOptions; } export interface VideoStreamInfo { @@ -78,6 +106,11 @@ export interface BitrateDistribution { unit: string; } +export interface ImageBuffer { + data: Buffer; + info: RawImageInfo; +} + export interface VideoCodecSWConfig { getCommand(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeCommand; } @@ -93,8 +126,11 @@ export interface ProbeOptions { export interface IMediaRepository { // image extract(input: string, output: string): Promise<boolean>; - generateThumbnail(input: string | Buffer, output: string, options: ThumbnailOptions): Promise<void>; - generateThumbhash(imagePath: string): Promise<Buffer>; + decodeImage(input: string, options: DecodeToBufferOptions): Promise<ImageBuffer>; + generateThumbnail(input: string, options: GenerateThumbnailOptions, outputFile: string): Promise<void>; + generateThumbnail(input: Buffer, options: GenerateThumbnailFromBufferOptions, outputFile: string): Promise<void>; + generateThumbhash(input: string, options: GenerateThumbhashOptions): Promise<Buffer>; + generateThumbhash(input: Buffer, options: GenerateThumbhashFromBufferOptions): Promise<Buffer>; getImageDimensions(input: string): Promise<ImageDimensions>; // video diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index 6930932584..eda91482bb 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -1132,3 +1132,27 @@ RETURNING "id", "createdAt", "updatedAt" + +-- AssetRepository.upsertFiles +INSERT INTO + "asset_files" ( + "id", + "assetId", + "createdAt", + "updatedAt", + "type", + "path" + ) +VALUES + (DEFAULT, $1, DEFAULT, DEFAULT, $2, $3) +ON CONFLICT ("assetId", "type") DO +UPDATE +SET + "assetId" = EXCLUDED."assetId", + "type" = EXCLUDED."type", + "path" = EXCLUDED."path", + "updatedAt" = DEFAULT +RETURNING + "id", + "createdAt", + "updatedAt" diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 0ec347ed77..8bca755c32 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -801,7 +801,12 @@ export class AssetRepository implements IAssetRepository { } @GenerateSql({ params: [{ assetId: DummyValue.UUID, type: AssetFileType.PREVIEW, path: '/path/to/file' }] }) - async upsertFile({ assetId, type, path }: { assetId: string; type: AssetFileType; path: string }): Promise<void> { - await this.fileRepository.upsert({ assetId, type, path }, { conflictPaths: ['assetId', 'type'] }); + async upsertFile(file: { assetId: string; type: AssetFileType; path: string }): Promise<void> { + await this.fileRepository.upsert(file, { conflictPaths: ['assetId', 'type'] }); + } + + @GenerateSql({ params: [{ assetId: DummyValue.UUID, type: AssetFileType.PREVIEW, path: '/path/to/file' }] }) + async upsertFiles(files: { assetId: string; type: AssetFileType; path: string }[]): Promise<void> { + await this.fileRepository.upsert(files, { conflictPaths: ['assetId', 'type'] }); } } diff --git a/server/src/repositories/job.repository.ts b/server/src/repositories/job.repository.ts index cd4c7135be..3f154ee016 100644 --- a/server/src/repositories/job.repository.ts +++ b/server/src/repositories/job.repository.ts @@ -36,9 +36,7 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = { // thumbnails [JobName.QUEUE_GENERATE_THUMBNAILS]: QueueName.THUMBNAIL_GENERATION, - [JobName.GENERATE_PREVIEW]: QueueName.THUMBNAIL_GENERATION, - [JobName.GENERATE_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION, - [JobName.GENERATE_THUMBHASH]: QueueName.THUMBNAIL_GENERATION, + [JobName.GENERATE_THUMBNAILS]: QueueName.THUMBNAIL_GENERATION, [JobName.GENERATE_PERSON_THUMBNAIL]: QueueName.THUMBNAIL_GENERATION, // tags diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts index d001aa3158..cca87f44f2 100644 --- a/server/src/repositories/media.repository.ts +++ b/server/src/repositories/media.repository.ts @@ -8,10 +8,12 @@ import sharp from 'sharp'; import { Colorspace, LogLevel } from 'src/enum'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { + DecodeToBufferOptions, + GenerateThumbhashOptions, + GenerateThumbnailOptions, IMediaRepository, ImageDimensions, ProbeOptions, - ThumbnailOptions, TranscodeCommand, VideoInfo, } from 'src/interfaces/media.interface'; @@ -57,19 +59,12 @@ export class MediaRepository implements IMediaRepository { return true; } - async generateThumbnail(input: string | Buffer, output: string, options: ThumbnailOptions): Promise<void> { - // some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes - const pipeline = sharp(input, { failOn: options.processInvalidImages ? 'none' : 'error', limitInputPixels: false }) - .pipelineColorspace(options.colorspace === Colorspace.SRGB ? 'srgb' : 'rgb16') - .rotate(); + decodeImage(input: string, options: DecodeToBufferOptions) { + return this.getImageDecodingPipeline(input, options).raw().toBuffer({ resolveWithObject: true }); + } - if (options.crop) { - pipeline.extract(options.crop); - } - - await pipeline - .resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true }) - .withIccProfile(options.colorspace) + async generateThumbnail(input: string | Buffer, options: GenerateThumbnailOptions, output: string): Promise<void> { + await this.getImageDecodingPipeline(input, options) .toFormat(options.format, { quality: options.quality, // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp @@ -78,6 +73,40 @@ export class MediaRepository implements IMediaRepository { .toFile(output); } + private getImageDecodingPipeline(input: string | Buffer, options: DecodeToBufferOptions) { + let pipeline = sharp(input, { + // some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes + failOn: options.processInvalidImages ? 'none' : 'error', + limitInputPixels: false, + raw: options.raw, + }); + + if (!options.raw) { + pipeline = pipeline + .pipelineColorspace(options.colorspace === Colorspace.SRGB ? 'srgb' : 'rgb16') + .withIccProfile(options.colorspace) + .rotate(); + } + + if (options.crop) { + pipeline = pipeline.extract(options.crop); + } + + return pipeline.resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true }); + } + + async generateThumbhash(input: string | Buffer, options: GenerateThumbhashOptions): Promise<Buffer> { + const [{ rgbaToThumbHash }, { data, info }] = await Promise.all([ + import('thumbhash'), + sharp(input, options) + .resize(100, 100, { fit: 'inside', withoutEnlargement: true }) + .raw() + .ensureAlpha() + .toBuffer({ resolveWithObject: true }), + ]); + return Buffer.from(rgbaToThumbHash(info.width, info.height, data)); + } + async probe(input: string, options?: ProbeOptions): Promise<VideoInfo> { const results = await probe(input, options?.countFrames ? ['-count_packets'] : []); // gets frame count quickly: https://stackoverflow.com/a/28376817 return { @@ -150,19 +179,6 @@ export class MediaRepository implements IMediaRepository { }); } - async generateThumbhash(imagePath: string): Promise<Buffer> { - const maxSize = 100; - - const { data, info } = await sharp(imagePath) - .resize(maxSize, maxSize, { fit: 'inside', withoutEnlargement: true }) - .raw() - .ensureAlpha() - .toBuffer({ resolveWithObject: true }); - - const thumbhash = await import('thumbhash'); - return Buffer.from(thumbhash.rgbaToThumbHash(info.width, info.height, data)); - } - async getImageDimensions(input: string): Promise<ImageDimensions> { const { width = 0, height = 0 } = await sharp(input).metadata(); return { width, height }; diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.ts index 2e2d676939..f36d26fa7c 100755 --- a/server/src/services/asset.service.spec.ts +++ b/server/src/services/asset.service.spec.ts @@ -395,7 +395,7 @@ describe(AssetService.name, () => { it('should run the refresh thumbnails job', async () => { accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1'])); await sut.run(authStub.admin, { assetIds: ['asset-1'], name: AssetJobName.REGENERATE_THUMBNAIL }); - expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.GENERATE_PREVIEW, data: { id: 'asset-1' } }]); + expect(jobMock.queueAll).toHaveBeenCalledWith([{ name: JobName.GENERATE_THUMBNAILS, data: { id: 'asset-1' } }]); }); it('should run the transcode video', async () => { diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts index b3f824f226..aa88eaf957 100644 --- a/server/src/services/asset.service.ts +++ b/server/src/services/asset.service.ts @@ -322,7 +322,7 @@ export class AssetService { } case AssetJobName.REGENERATE_THUMBNAIL: { - jobs.push({ name: JobName.GENERATE_PREVIEW, data: { id } }); + jobs.push({ name: JobName.GENERATE_THUMBNAILS, data: { id } }); break; } diff --git a/server/src/services/job.service.spec.ts b/server/src/services/job.service.spec.ts index 1c810facb4..c2d7a29b9f 100644 --- a/server/src/services/job.service.spec.ts +++ b/server/src/services/job.service.spec.ts @@ -288,7 +288,7 @@ describe(JobService.name, () => { }, { item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1', source: 'upload' } }, - jobs: [JobName.GENERATE_PREVIEW], + jobs: [JobName.GENERATE_THUMBNAILS], }, { item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1' } }, @@ -299,28 +299,16 @@ describe(JobService.name, () => { jobs: [], }, { - item: { name: JobName.GENERATE_PREVIEW, data: { id: 'asset-1' } }, - jobs: [JobName.GENERATE_THUMBNAIL, JobName.GENERATE_THUMBHASH], + item: { name: JobName.GENERATE_THUMBNAILS, data: { id: 'asset-1' } }, + jobs: [], }, { - item: { name: JobName.GENERATE_PREVIEW, data: { id: 'asset-1', source: 'upload' } }, - jobs: [ - JobName.GENERATE_THUMBNAIL, - JobName.GENERATE_THUMBHASH, - JobName.SMART_SEARCH, - JobName.FACE_DETECTION, - JobName.VIDEO_CONVERSION, - ], + item: { name: JobName.GENERATE_THUMBNAILS, data: { id: 'asset-1', source: 'upload' } }, + jobs: [JobName.SMART_SEARCH, JobName.FACE_DETECTION, JobName.VIDEO_CONVERSION], }, { - item: { name: JobName.GENERATE_PREVIEW, data: { id: 'asset-live-image', source: 'upload' } }, - jobs: [ - JobName.GENERATE_THUMBNAIL, - JobName.GENERATE_THUMBHASH, - JobName.SMART_SEARCH, - JobName.FACE_DETECTION, - JobName.VIDEO_CONVERSION, - ], + item: { name: JobName.GENERATE_THUMBNAILS, data: { id: 'asset-live-image', source: 'upload' } }, + jobs: [JobName.SMART_SEARCH, JobName.FACE_DETECTION, JobName.VIDEO_CONVERSION], }, { item: { name: JobName.SMART_SEARCH, data: { id: 'asset-1' } }, @@ -338,11 +326,11 @@ describe(JobService.name, () => { for (const { item, jobs } of tests) { it(`should queue ${jobs.length} jobs when a ${item.name} job finishes successfully`, async () => { - if (item.name === JobName.GENERATE_PREVIEW && item.data.source === 'upload') { + if (item.name === JobName.GENERATE_THUMBNAILS && item.data.source === 'upload') { if (item.data.id === 'asset-live-image') { - assetMock.getByIds.mockResolvedValue([assetStub.livePhotoStillAsset]); + assetMock.getByIdsWithAllRelations.mockResolvedValue([assetStub.livePhotoStillAsset]); } else { - assetMock.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]); + assetMock.getByIdsWithAllRelations.mockResolvedValue([assetStub.livePhotoMotionAsset]); } } @@ -361,7 +349,7 @@ describe(JobService.name, () => { } }); - it(`should not queue any jobs when ${item.name} finishes with 'false'`, async () => { + it(`should not queue any jobs when ${item.name} fails`, async () => { await sut.init(makeMockHandlers(JobStatus.FAILED)); await jobMock.addHandler.mock.calls[0][2](item); diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index f978f33410..9c73e71cbf 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -281,7 +281,7 @@ export class JobService { case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE: { if (item.data.source === 'upload' || item.data.source === 'copy') { - await this.jobRepository.queue({ name: JobName.GENERATE_PREVIEW, data: item.data }); + await this.jobRepository.queue({ name: JobName.GENERATE_THUMBNAILS, data: item.data }); } break; } @@ -295,40 +295,33 @@ export class JobService { break; } - case JobName.GENERATE_PREVIEW: { - const jobs: JobItem[] = [ - { name: JobName.GENERATE_THUMBNAIL, data: item.data }, - { name: JobName.GENERATE_THUMBHASH, data: item.data }, - ]; - - if (item.data.source === 'upload') { - jobs.push({ name: JobName.SMART_SEARCH, data: item.data }, { name: JobName.FACE_DETECTION, data: item.data }); - - const [asset] = await this.assetRepository.getByIds([item.data.id]); - if (asset) { - if (asset.type === AssetType.VIDEO) { - jobs.push({ name: JobName.VIDEO_CONVERSION, data: item.data }); - } else if (asset.livePhotoVideoId) { - jobs.push({ name: JobName.VIDEO_CONVERSION, data: { id: asset.livePhotoVideoId } }); - } - } - } - - await this.jobRepository.queueAll(jobs); - break; - } - - case JobName.GENERATE_THUMBNAIL: { - if (!(item.data.notify || item.data.source === 'upload')) { + case JobName.GENERATE_THUMBNAILS: { + if (!item.data.notify && item.data.source !== 'upload') { break; } const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]); + if (!asset) { + this.logger.warn(`Could not find asset ${item.data.id} after generating thumbnails`); + break; + } - // Only live-photo motion part will be marked as not visible immediately on upload. Skip notifying clients - if (asset && asset.isVisible) { + const jobs: JobItem[] = [ + { name: JobName.SMART_SEARCH, data: item.data }, + { name: JobName.FACE_DETECTION, data: item.data }, + ]; + + if (asset.type === AssetType.VIDEO) { + jobs.push({ name: JobName.VIDEO_CONVERSION, data: item.data }); + } else if (asset.livePhotoVideoId) { + jobs.push({ name: JobName.VIDEO_CONVERSION, data: { id: asset.livePhotoVideoId } }); + } + + await this.jobRepository.queueAll(jobs); + if (asset.isVisible) { this.eventRepository.clientSend(ClientEvent.UPLOAD_SUCCESS, asset.ownerId, mapAsset(asset)); } + break; } diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index c0903fa101..88e9f478bd 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -15,7 +15,7 @@ import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interfac import { ICryptoRepository } from 'src/interfaces/crypto.interface'; import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; -import { IMediaRepository } from 'src/interfaces/media.interface'; +import { IMediaRepository, RawImageInfo } from 'src/interfaces/media.interface'; import { IMoveRepository } from 'src/interfaces/move.interface'; import { IPersonRepository } from 'src/interfaces/person.interface'; import { IStorageRepository } from 'src/interfaces/storage.interface'; @@ -94,7 +94,7 @@ describe(MediaService.name, () => { expect(assetMock.getWithout).not.toHaveBeenCalled(); expect(jobMock.queueAll).toHaveBeenCalledWith([ { - name: JobName.GENERATE_PREVIEW, + name: JobName.GENERATE_THUMBNAILS, data: { id: assetStub.image.id }, }, ]); @@ -127,7 +127,7 @@ describe(MediaService.name, () => { expect(assetMock.getWithout).not.toHaveBeenCalled(); expect(jobMock.queueAll).toHaveBeenCalledWith([ { - name: JobName.GENERATE_PREVIEW, + name: JobName.GENERATE_THUMBNAILS, data: { id: assetStub.trashed.id }, }, ]); @@ -152,7 +152,7 @@ describe(MediaService.name, () => { expect(assetMock.getWithout).not.toHaveBeenCalled(); expect(jobMock.queueAll).toHaveBeenCalledWith([ { - name: JobName.GENERATE_PREVIEW, + name: JobName.GENERATE_THUMBNAILS, data: { id: assetStub.archived.id }, }, ]); @@ -202,7 +202,7 @@ describe(MediaService.name, () => { expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL); expect(jobMock.queueAll).toHaveBeenCalledWith([ { - name: JobName.GENERATE_PREVIEW, + name: JobName.GENERATE_THUMBNAILS, data: { id: assetStub.image.id }, }, ]); @@ -226,7 +226,7 @@ describe(MediaService.name, () => { expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL); expect(jobMock.queueAll).toHaveBeenCalledWith([ { - name: JobName.GENERATE_THUMBNAIL, + name: JobName.GENERATE_THUMBNAILS, data: { id: assetStub.image.id }, }, ]); @@ -250,7 +250,7 @@ describe(MediaService.name, () => { expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL); expect(jobMock.queueAll).toHaveBeenCalledWith([ { - name: JobName.GENERATE_THUMBHASH, + name: JobName.GENERATE_THUMBNAILS, data: { id: assetStub.image.id }, }, ]); @@ -259,10 +259,19 @@ describe(MediaService.name, () => { }); }); - describe('handleGeneratePreview', () => { + describe('handleGenerateThumbnails', () => { + let rawBuffer: Buffer; + let rawInfo: RawImageInfo; + + beforeEach(() => { + rawBuffer = Buffer.from('image data'); + rawInfo = { width: 100, height: 100, channels: 3 }; + mediaMock.decodeImage.mockResolvedValue({ data: rawBuffer, info: rawInfo }); + }); + it('should skip thumbnail generation if asset not found', async () => { - assetMock.getByIds.mockResolvedValue([]); - await sut.handleGeneratePreview({ id: assetStub.image.id }); + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); + expect(mediaMock.generateThumbnail).not.toHaveBeenCalled(); expect(assetMock.update).not.toHaveBeenCalledWith(); }); @@ -270,80 +279,100 @@ describe(MediaService.name, () => { it('should skip video thumbnail generation if no video stream', async () => { mediaMock.probe.mockResolvedValue(probeStub.noVideoStreams); assetMock.getByIds.mockResolvedValue([assetStub.video]); - await sut.handleGeneratePreview({ id: assetStub.image.id }); + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); expect(mediaMock.generateThumbnail).not.toHaveBeenCalled(); expect(assetMock.update).not.toHaveBeenCalledWith(); }); it('should skip invisible assets', async () => { - assetMock.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]); + assetMock.getById.mockResolvedValue(assetStub.livePhotoMotionAsset); - expect(await sut.handleGeneratePreview({ id: assetStub.livePhotoMotionAsset.id })).toEqual(JobStatus.SKIPPED); + expect(await sut.handleGenerateThumbnails({ id: assetStub.livePhotoMotionAsset.id })).toEqual(JobStatus.SKIPPED); expect(mediaMock.generateThumbnail).not.toHaveBeenCalled(); expect(assetMock.update).not.toHaveBeenCalledWith(); }); - it.each(Object.values(ImageFormat))('should generate a %s preview for an image when specified', async (format) => { - systemMock.get.mockResolvedValue({ image: { preview: { format } } }); - assetMock.getByIds.mockResolvedValue([assetStub.image]); - const previewPath = `upload/thumbs/user-id/as/se/asset-id-preview.${format}`; - - await sut.handleGeneratePreview({ id: assetStub.image.id }); - - expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se'); - expect(mediaMock.generateThumbnail).toHaveBeenCalledWith('/original/path.jpg', previewPath, { - size: 1440, - format, - quality: 80, - colorspace: Colorspace.SRGB, - processInvalidImages: false, - }); - expect(assetMock.upsertFile).toHaveBeenCalledWith({ - assetId: 'asset-id', - type: AssetFileType.PREVIEW, - path: previewPath, - }); - }); - it('should delete previous preview if different path', async () => { systemMock.get.mockResolvedValue({ image: { thumbnail: { format: ImageFormat.WEBP } } }); - assetMock.getByIds.mockResolvedValue([assetStub.image]); + assetMock.getById.mockResolvedValue(assetStub.image); - await sut.handleGeneratePreview({ id: assetStub.image.id }); + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); expect(storageMock.unlink).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg'); }); - it('should generate a P3 thumbnail for a wide gamut image', async () => { - assetMock.getByIds.mockResolvedValue([ - { ...assetStub.image, exifInfo: { profileDescription: 'Adobe RGB', bitsPerSample: 14 } as ExifEntity }, - ]); - await sut.handleGeneratePreview({ id: assetStub.image.id }); + it('should generate P3 thumbnails for a wide gamut image', async () => { + assetMock.getById.mockResolvedValue({ + ...assetStub.image, + exifInfo: { profileDescription: 'Adobe RGB', bitsPerSample: 14 } as ExifEntity, + }); + const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8'); + mediaMock.generateThumbhash.mockResolvedValue(thumbhashBuffer); + + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se'); - expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( - '/original/path.jpg', - 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', - { - size: 1440, - format: ImageFormat.JPEG, - quality: 80, - colorspace: Colorspace.P3, - processInvalidImages: false, - }, - ); - expect(assetMock.upsertFile).toHaveBeenCalledWith({ - assetId: 'asset-id', - type: AssetFileType.PREVIEW, - path: 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', + + expect(mediaMock.decodeImage).toHaveBeenCalledOnce(); + expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.image.originalPath, { + colorspace: Colorspace.P3, + processInvalidImages: false, + size: 1440, }); + + expect(mediaMock.generateThumbnail).toHaveBeenCalledTimes(2); + expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( + rawBuffer, + { + colorspace: Colorspace.P3, + format: ImageFormat.JPEG, + size: 1440, + quality: 80, + processInvalidImages: false, + raw: rawInfo, + }, + 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', + ); + expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( + rawBuffer, + { + colorspace: Colorspace.P3, + format: ImageFormat.WEBP, + size: 250, + quality: 80, + processInvalidImages: false, + raw: rawInfo, + }, + 'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp', + ); + + expect(mediaMock.generateThumbhash).toHaveBeenCalledOnce(); + expect(mediaMock.generateThumbhash).toHaveBeenCalledWith(rawBuffer, { + colorspace: Colorspace.P3, + processInvalidImages: false, + raw: rawInfo, + }); + + expect(assetMock.upsertFiles).toHaveBeenCalledWith([ + { + assetId: 'asset-id', + type: AssetFileType.PREVIEW, + path: 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', + }, + { + assetId: 'asset-id', + type: AssetFileType.THUMBNAIL, + path: 'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp', + }, + ]); + expect(assetMock.update).toHaveBeenCalledWith({ id: 'asset-id', thumbhash: thumbhashBuffer }); }); it('should generate a thumbnail for a video', async () => { mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p); - assetMock.getByIds.mockResolvedValue([assetStub.video]); - await sut.handleGeneratePreview({ id: assetStub.video.id }); + assetMock.getById.mockResolvedValue(assetStub.video); + await sut.handleGenerateThumbnails({ id: assetStub.video.id }); expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se'); expect(mediaMock.transcode).toHaveBeenCalledWith( @@ -361,17 +390,24 @@ describe(MediaService.name, () => { twoPass: false, }), ); - expect(assetMock.upsertFile).toHaveBeenCalledWith({ - assetId: 'asset-id', - type: AssetFileType.PREVIEW, - path: 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', - }); + expect(assetMock.upsertFiles).toHaveBeenCalledWith([ + { + assetId: 'asset-id', + type: AssetFileType.PREVIEW, + path: 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', + }, + { + assetId: 'asset-id', + type: AssetFileType.THUMBNAIL, + path: 'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp', + }, + ]); }); it('should tonemap thumbnail for hdr video', async () => { mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR); - assetMock.getByIds.mockResolvedValue([assetStub.video]); - await sut.handleGeneratePreview({ id: assetStub.video.id }); + assetMock.getById.mockResolvedValue(assetStub.video); + await sut.handleGenerateThumbnails({ id: assetStub.video.id }); expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se'); expect(mediaMock.transcode).toHaveBeenCalledWith( @@ -389,11 +425,18 @@ describe(MediaService.name, () => { twoPass: false, }), ); - expect(assetMock.upsertFile).toHaveBeenCalledWith({ - assetId: 'asset-id', - type: AssetFileType.PREVIEW, - path: 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', - }); + expect(assetMock.upsertFiles).toHaveBeenCalledWith([ + { + assetId: 'asset-id', + type: AssetFileType.PREVIEW, + path: 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', + }, + { + assetId: 'asset-id', + type: AssetFileType.THUMBNAIL, + path: 'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp', + }, + ]); }); it('should always generate video thumbnail in one pass', async () => { @@ -401,8 +444,8 @@ describe(MediaService.name, () => { systemMock.get.mockResolvedValue({ ffmpeg: { twoPass: true, maxBitrate: '5000k' }, }); - assetMock.getByIds.mockResolvedValue([assetStub.video]); - await sut.handleGeneratePreview({ id: assetStub.video.id }); + assetMock.getById.mockResolvedValue(assetStub.video); + await sut.handleGenerateThumbnails({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', @@ -424,8 +467,8 @@ describe(MediaService.name, () => { it('should use scaling divisible by 2 even when using quick sync', async () => { mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p); systemMock.get.mockResolvedValue({ ffmpeg: { accel: TranscodeHWAccel.QSV } }); - assetMock.getByIds.mockResolvedValue([assetStub.video]); - await sut.handleGeneratePreview({ id: assetStub.video.id }); + assetMock.getById.mockResolvedValue(assetStub.video); + await sut.handleGenerateThumbnails({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', @@ -438,233 +481,207 @@ describe(MediaService.name, () => { ); }); - it('should run successfully', async () => { - assetMock.getByIds.mockResolvedValue([assetStub.image]); - await sut.handleGeneratePreview({ id: assetStub.image.id }); - }); - }); + it.each(Object.values(ImageFormat))('should generate an image preview in %s format', async (format) => { + systemMock.get.mockResolvedValue({ image: { preview: { format } } }); + assetMock.getById.mockResolvedValue(assetStub.image); + const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8'); + mediaMock.generateThumbhash.mockResolvedValue(thumbhashBuffer); + const previewPath = `upload/thumbs/user-id/as/se/asset-id-preview.${format}`; + const thumbnailPath = `upload/thumbs/user-id/as/se/asset-id-thumbnail.webp`; - describe('handleGenerateThumbnail', () => { - it('should skip thumbnail generation if asset not found', async () => { - assetMock.getByIds.mockResolvedValue([]); - await sut.handleGenerateThumbnail({ id: assetStub.image.id }); - expect(mediaMock.generateThumbnail).not.toHaveBeenCalled(); - expect(assetMock.update).not.toHaveBeenCalledWith(); - }); + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); - it('should skip invisible assets', async () => { - assetMock.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]); + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se'); + expect(mediaMock.decodeImage).toHaveBeenCalledOnce(); + expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.image.originalPath, { + colorspace: Colorspace.SRGB, + processInvalidImages: false, + size: 1440, + }); - expect(await sut.handleGenerateThumbnail({ id: assetStub.livePhotoMotionAsset.id })).toEqual(JobStatus.SKIPPED); - - expect(mediaMock.generateThumbnail).not.toHaveBeenCalled(); - expect(assetMock.update).not.toHaveBeenCalledWith(); - }); - - it.each(Object.values(ImageFormat))( - 'should generate a %s thumbnail for an image when specified', - async (format) => { - systemMock.get.mockResolvedValue({ image: { thumbnail: { format } } }); - assetMock.getByIds.mockResolvedValue([assetStub.image]); - const thumbnailPath = `upload/thumbs/user-id/as/se/asset-id-thumbnail.${format}`; - - await sut.handleGenerateThumbnail({ id: assetStub.image.id }); - - expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se'); - expect(mediaMock.generateThumbnail).toHaveBeenCalledWith('/original/path.jpg', thumbnailPath, { - size: 250, - format, - quality: 80, + expect(mediaMock.generateThumbnail).toHaveBeenCalledTimes(2); + expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( + rawBuffer, + { colorspace: Colorspace.SRGB, + format, + size: 1440, + quality: 80, processInvalidImages: false, - }); - expect(assetMock.upsertFile).toHaveBeenCalledWith({ - assetId: 'asset-id', - type: AssetFileType.THUMBNAIL, - path: thumbnailPath, - }); - }, - ); + raw: rawInfo, + }, + previewPath, + ); + expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( + rawBuffer, + { + colorspace: Colorspace.SRGB, + format: ImageFormat.WEBP, + size: 250, + quality: 80, + processInvalidImages: false, + raw: rawInfo, + }, + thumbnailPath, + ); + }); + + it.each(Object.values(ImageFormat))('should generate an image thumbnail in %s format', async (format) => { + systemMock.get.mockResolvedValue({ image: { thumbnail: { format } } }); + assetMock.getById.mockResolvedValue(assetStub.image); + const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8'); + mediaMock.generateThumbhash.mockResolvedValue(thumbhashBuffer); + const previewPath = `upload/thumbs/user-id/as/se/asset-id-preview.jpeg`; + const thumbnailPath = `upload/thumbs/user-id/as/se/asset-id-thumbnail.${format}`; + + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); + + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se'); + expect(mediaMock.decodeImage).toHaveBeenCalledOnce(); + expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.image.originalPath, { + colorspace: Colorspace.SRGB, + processInvalidImages: false, + size: 1440, + }); + + expect(mediaMock.generateThumbnail).toHaveBeenCalledTimes(2); + expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( + rawBuffer, + { + colorspace: Colorspace.SRGB, + format: ImageFormat.JPEG, + size: 1440, + quality: 80, + processInvalidImages: false, + raw: rawInfo, + }, + previewPath, + ); + expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( + rawBuffer, + { + colorspace: Colorspace.SRGB, + format, + size: 250, + quality: 80, + processInvalidImages: false, + raw: rawInfo, + }, + thumbnailPath, + ); + }); it('should delete previous thumbnail if different path', async () => { systemMock.get.mockResolvedValue({ image: { thumbnail: { format: ImageFormat.WEBP } } }); - assetMock.getByIds.mockResolvedValue([assetStub.image]); + assetMock.getById.mockResolvedValue(assetStub.image); - await sut.handleGenerateThumbnail({ id: assetStub.image.id }); + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); expect(storageMock.unlink).toHaveBeenCalledWith('/uploads/user-id/webp/path.ext'); }); - }); - it('should generate a P3 thumbnail for a wide gamut image', async () => { - assetMock.getByIds.mockResolvedValue([assetStub.imageDng]); - await sut.handleGenerateThumbnail({ id: assetStub.image.id }); + it('should extract embedded image if enabled and available', async () => { + mediaMock.extract.mockResolvedValue(true); + mediaMock.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); + systemMock.get.mockResolvedValue({ image: { extractEmbedded: true } }); + assetMock.getById.mockResolvedValue(assetStub.imageDng); - expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se'); - expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( - assetStub.imageDng.originalPath, - 'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp', - { - format: ImageFormat.WEBP, - size: 250, - quality: 80, + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); + + const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString(); + expect(mediaMock.decodeImage).toHaveBeenCalledOnce(); + expect(mediaMock.decodeImage).toHaveBeenCalledWith(extractedPath, { colorspace: Colorspace.P3, processInvalidImages: false, - }, - ); - expect(assetMock.upsertFile).toHaveBeenCalledWith({ - assetId: 'asset-id', - type: AssetFileType.THUMBNAIL, - path: 'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp', + size: 1440, + }); + expect(extractedPath?.endsWith('.tmp')).toBe(true); + expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath); }); - }); - it('should extract embedded image if enabled and available', async () => { - mediaMock.extract.mockResolvedValue(true); - mediaMock.getImageDimensions.mockResolvedValue({ width: 3840, height: 2160 }); - systemMock.get.mockResolvedValue({ image: { extractEmbedded: true } }); - assetMock.getByIds.mockResolvedValue([assetStub.imageDng]); + it('should resize original image if embedded image is too small', async () => { + mediaMock.extract.mockResolvedValue(true); + mediaMock.getImageDimensions.mockResolvedValue({ width: 1000, height: 1000 }); + systemMock.get.mockResolvedValue({ image: { extractEmbedded: true } }); + assetMock.getById.mockResolvedValue(assetStub.imageDng); - await sut.handleGenerateThumbnail({ id: assetStub.image.id }); + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); - const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString(); - expect(mediaMock.generateThumbnail.mock.calls).toEqual([ - [ - extractedPath, - 'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp', - { - format: ImageFormat.WEBP, - size: 250, - quality: 80, - colorspace: Colorspace.P3, - processInvalidImages: false, - }, - ], - ]); - expect(extractedPath?.endsWith('.tmp')).toBe(true); - expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath); - }); + expect(mediaMock.decodeImage).toHaveBeenCalledOnce(); + expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, { + colorspace: Colorspace.P3, + processInvalidImages: false, + size: 1440, + }); + const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString(); + expect(extractedPath?.endsWith('.tmp')).toBe(true); + expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath); + }); - it('should resize original image if embedded image is too small', async () => { - mediaMock.extract.mockResolvedValue(true); - mediaMock.getImageDimensions.mockResolvedValue({ width: 1000, height: 1000 }); - systemMock.get.mockResolvedValue({ image: { extractEmbedded: true } }); - assetMock.getByIds.mockResolvedValue([assetStub.imageDng]); + it('should resize original image if embedded image not found', async () => { + systemMock.get.mockResolvedValue({ image: { extractEmbedded: true } }); + assetMock.getById.mockResolvedValue(assetStub.imageDng); - await sut.handleGenerateThumbnail({ id: assetStub.image.id }); + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); - expect(mediaMock.generateThumbnail.mock.calls).toEqual([ - [ + expect(mediaMock.decodeImage).toHaveBeenCalledOnce(); + expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, { + colorspace: Colorspace.P3, + processInvalidImages: false, + size: 1440, + }); + expect(mediaMock.getImageDimensions).not.toHaveBeenCalled(); + }); + + it('should resize original image if embedded image extraction is not enabled', async () => { + systemMock.get.mockResolvedValue({ image: { extractEmbedded: false } }); + assetMock.getById.mockResolvedValue(assetStub.imageDng); + + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); + + expect(mediaMock.extract).not.toHaveBeenCalled(); + expect(mediaMock.decodeImage).toHaveBeenCalledOnce(); + expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, { + colorspace: Colorspace.P3, + processInvalidImages: false, + size: 1440, + }); + expect(mediaMock.getImageDimensions).not.toHaveBeenCalled(); + }); + + it('should process invalid images if enabled', async () => { + vi.stubEnv('IMMICH_PROCESS_INVALID_IMAGES', 'true'); + + assetMock.getById.mockResolvedValue(assetStub.imageDng); + + await sut.handleGenerateThumbnails({ id: assetStub.image.id }); + + expect(mediaMock.decodeImage).toHaveBeenCalledOnce(); + expect(mediaMock.decodeImage).toHaveBeenCalledWith( assetStub.imageDng.originalPath, + expect.objectContaining({ processInvalidImages: true }), + ); + + expect(mediaMock.generateThumbnail).toHaveBeenCalledTimes(2); + expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( + rawBuffer, + expect.objectContaining({ processInvalidImages: true }), + 'upload/thumbs/user-id/as/se/asset-id-preview.jpeg', + ); + expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( + rawBuffer, + expect.objectContaining({ processInvalidImages: true }), 'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp', - { - format: ImageFormat.WEBP, - size: 250, - quality: 80, - colorspace: Colorspace.P3, - processInvalidImages: false, - }, - ], - ]); - const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString(); - expect(extractedPath?.endsWith('.tmp')).toBe(true); - expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath); - }); + ); - it('should resize original image if embedded image not found', async () => { - systemMock.get.mockResolvedValue({ image: { extractEmbedded: true } }); - assetMock.getByIds.mockResolvedValue([assetStub.imageDng]); + expect(mediaMock.generateThumbhash).toHaveBeenCalledOnce(); + expect(mediaMock.generateThumbhash).toHaveBeenCalledWith( + rawBuffer, + expect.objectContaining({ processInvalidImages: true }), + ); - await sut.handleGenerateThumbnail({ id: assetStub.image.id }); - - expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( - assetStub.imageDng.originalPath, - 'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp', - { - format: ImageFormat.WEBP, - size: 250, - quality: 80, - colorspace: Colorspace.P3, - processInvalidImages: false, - }, - ); - expect(mediaMock.getImageDimensions).not.toHaveBeenCalled(); - }); - - it('should resize original image if embedded image extraction is not enabled', async () => { - systemMock.get.mockResolvedValue({ image: { extractEmbedded: false } }); - assetMock.getByIds.mockResolvedValue([assetStub.imageDng]); - - await sut.handleGenerateThumbnail({ id: assetStub.image.id }); - - expect(mediaMock.extract).not.toHaveBeenCalled(); - expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( - assetStub.imageDng.originalPath, - 'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp', - { - format: ImageFormat.WEBP, - size: 250, - quality: 80, - colorspace: Colorspace.P3, - processInvalidImages: false, - }, - ); - expect(mediaMock.getImageDimensions).not.toHaveBeenCalled(); - }); - - it('should process invalid images if enabled', async () => { - vi.stubEnv('IMMICH_PROCESS_INVALID_IMAGES', 'true'); - - assetMock.getByIds.mockResolvedValue([assetStub.imageDng]); - - await sut.handleGenerateThumbnail({ id: assetStub.image.id }); - - expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( - assetStub.imageDng.originalPath, - 'upload/thumbs/user-id/as/se/asset-id-thumbnail.webp', - { - format: ImageFormat.WEBP, - size: 250, - quality: 80, - colorspace: Colorspace.P3, - processInvalidImages: true, - }, - ); - expect(mediaMock.getImageDimensions).not.toHaveBeenCalled(); - vi.unstubAllEnvs(); - }); - - describe('handleGenerateThumbhash', () => { - it('should skip thumbhash generation if asset not found', async () => { - assetMock.getByIds.mockResolvedValue([]); - await sut.handleGenerateThumbhash({ id: assetStub.image.id }); - expect(mediaMock.generateThumbhash).not.toHaveBeenCalled(); - }); - - it('should skip thumbhash generation if resize path is missing', async () => { - assetMock.getByIds.mockResolvedValue([assetStub.noResizePath]); - await sut.handleGenerateThumbhash({ id: assetStub.noResizePath.id }); - expect(mediaMock.generateThumbhash).not.toHaveBeenCalled(); - }); - - it('should skip invisible assets', async () => { - assetMock.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]); - - expect(await sut.handleGenerateThumbhash({ id: assetStub.livePhotoMotionAsset.id })).toEqual(JobStatus.SKIPPED); - - expect(mediaMock.generateThumbhash).not.toHaveBeenCalled(); - expect(assetMock.update).not.toHaveBeenCalledWith(); - }); - - it('should generate a thumbhash', async () => { - const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8'); - assetMock.getByIds.mockResolvedValue([assetStub.image]); - mediaMock.generateThumbhash.mockResolvedValue(thumbhashBuffer); - - await sut.handleGenerateThumbhash({ id: assetStub.image.id }); - - expect(mediaMock.generateThumbhash).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg'); - expect(assetMock.update).toHaveBeenCalledWith({ id: 'asset-id', thumbhash: thumbhashBuffer }); + expect(mediaMock.getImageDimensions).not.toHaveBeenCalled(); + vi.unstubAllEnvs(); }); }); diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index 1b69c5acd5..71f432e040 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -1,6 +1,7 @@ -import { Inject, Injectable, UnsupportedMediaTypeException } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { dirname } from 'node:path'; -import { GeneratedImageType, StorageCore } from 'src/cores/storage.core'; + +import { StorageCore } from 'src/cores/storage.core'; import { SystemConfigCore } from 'src/cores/system-config.core'; import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto'; import { AssetEntity } from 'src/entities/asset.entity'; @@ -18,7 +19,7 @@ import { VideoCodec, VideoContainer, } from 'src/enum'; -import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface'; +import { IAssetRepository, UpsertFileOptions, WithoutProperty } from 'src/interfaces/asset.interface'; import { ICryptoRepository } from 'src/interfaces/crypto.interface'; import { IBaseJob, @@ -95,18 +96,10 @@ export class MediaService { for (const asset of assets) { const { previewFile, thumbnailFile } = getAssetFiles(asset.files); - if (!previewFile || force) { - jobs.push({ name: JobName.GENERATE_PREVIEW, data: { id: asset.id } }); + if (!previewFile || !thumbnailFile || !asset.thumbhash || force) { + jobs.push({ name: JobName.GENERATE_THUMBNAILS, data: { id: asset.id } }); continue; } - - if (!thumbnailFile) { - jobs.push({ name: JobName.GENERATE_THUMBNAIL, data: { id: asset.id } }); - } - - if (!asset.thumbhash) { - jobs.push({ name: JobName.GENERATE_THUMBHASH, data: { id: asset.id } }); - } } await this.jobRepository.queueAll(jobs); @@ -181,141 +174,127 @@ export class MediaService { return JobStatus.SUCCESS; } - async handleGeneratePreview({ id }: IEntityJob): Promise<JobStatus> { - const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true, files: true }); + async handleGenerateThumbnails({ id }: IEntityJob): Promise<JobStatus> { + const asset = await this.assetRepository.getById(id, { exifInfo: true, files: true }); if (!asset) { + this.logger.warn(`Thumbnail generation failed for asset ${id}: not found`); return JobStatus.FAILED; } if (!asset.isVisible) { + this.logger.verbose(`Thumbnail generation skipped for asset ${id}: not visible`); return JobStatus.SKIPPED; } - const previewPath = await this.generateThumbnail(asset, AssetPathType.PREVIEW); - if (!previewPath) { + let generated: { previewPath: string; thumbnailPath: string; thumbhash: Buffer }; + if (asset.type === AssetType.IMAGE) { + generated = await this.generateImageThumbnails(asset); + } else if (asset.type === AssetType.VIDEO) { + generated = await this.generateVideoThumbnails(asset); + } else { + this.logger.warn(`Skipping thumbnail generation for asset ${id}: ${asset.type} is not an image or video`); return JobStatus.SKIPPED; } - const { previewFile } = getAssetFiles(asset.files); - if (previewFile && previewFile.path !== previewPath) { + const { previewFile, thumbnailFile } = getAssetFiles(asset.files); + const toUpsert: UpsertFileOptions[] = []; + if (previewFile?.path !== generated.previewPath) { + toUpsert.push({ assetId: asset.id, path: generated.previewPath, type: AssetFileType.PREVIEW }); + } + + if (thumbnailFile?.path !== generated.thumbnailPath) { + toUpsert.push({ assetId: asset.id, path: generated.thumbnailPath, type: AssetFileType.THUMBNAIL }); + } + + if (toUpsert.length > 0) { + await this.assetRepository.upsertFiles(toUpsert); + } + + const pathsToDelete = []; + if (previewFile && previewFile.path !== generated.previewPath) { this.logger.debug(`Deleting old preview for asset ${asset.id}`); - await this.storageRepository.unlink(previewFile.path); + pathsToDelete.push(previewFile.path); } - await this.assetRepository.upsertFile({ assetId: asset.id, type: AssetFileType.PREVIEW, path: previewPath }); - await this.assetRepository.update({ id: asset.id, updatedAt: new Date() }); - await this.assetRepository.upsertJobStatus({ assetId: asset.id, previewAt: new Date() }); - - return JobStatus.SUCCESS; - } - - private async generateThumbnail(asset: AssetEntity, type: GeneratedImageType) { - const { image, ffmpeg } = await this.configCore.getConfig({ withCache: true }); - const { size, format, quality } = image[type]; - const path = StorageCore.getImagePath(asset, type, format); - this.storageCore.ensureFolders(path); - - switch (asset.type) { - case AssetType.IMAGE: { - const shouldExtract = image.extractEmbedded && mimeTypes.isRaw(asset.originalPath); - const extractedPath = StorageCore.getTempPathInDir(dirname(path)); - const didExtract = shouldExtract && (await this.mediaRepository.extract(asset.originalPath, extractedPath)); - - try { - const useExtracted = didExtract && (await this.shouldUseExtractedImage(extractedPath, image.preview.size)); - const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace; - const imageOptions = { - format, - size, - colorspace, - quality, - processInvalidImages: process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true', - }; - - const outputPath = useExtracted ? extractedPath : asset.originalPath; - await this.mediaRepository.generateThumbnail(outputPath, path, imageOptions); - } finally { - if (didExtract) { - await this.storageRepository.unlink(extractedPath); - } - } - break; - } - - case AssetType.VIDEO: { - const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); - const mainVideoStream = this.getMainStream(videoStreams); - if (!mainVideoStream) { - this.logger.warn(`Skipped thumbnail generation for asset ${asset.id}: no video streams found`); - return; - } - const mainAudioStream = this.getMainStream(audioStreams); - const config = ThumbnailConfig.create({ ...ffmpeg, targetResolution: size.toString() }); - const options = config.getCommand(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream); - await this.mediaRepository.transcode(asset.originalPath, path, options); - break; - } - - default: { - throw new UnsupportedMediaTypeException(`Unsupported asset type for thumbnail generation: ${asset.type}`); - } - } - - const assetLabel = asset.isExternal ? asset.originalPath : asset.id; - this.logger.log( - `Successfully generated ${format.toUpperCase()} ${asset.type.toLowerCase()} ${type} for asset ${assetLabel}`, - ); - - return path; - } - - async handleGenerateThumbnail({ id }: IEntityJob): Promise<JobStatus> { - const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true, files: true }); - if (!asset) { - return JobStatus.FAILED; - } - - if (!asset.isVisible) { - return JobStatus.SKIPPED; - } - - const thumbnailPath = await this.generateThumbnail(asset, AssetPathType.THUMBNAIL); - if (!thumbnailPath) { - return JobStatus.SKIPPED; - } - - const { thumbnailFile } = getAssetFiles(asset.files); - if (thumbnailFile && thumbnailFile.path !== thumbnailPath) { + if (thumbnailFile && thumbnailFile.path !== generated.thumbnailPath) { this.logger.debug(`Deleting old thumbnail for asset ${asset.id}`); - await this.storageRepository.unlink(thumbnailFile.path); + pathsToDelete.push(thumbnailFile.path); } - await this.assetRepository.upsertFile({ assetId: asset.id, type: AssetFileType.THUMBNAIL, path: thumbnailPath }); - await this.assetRepository.update({ id: asset.id, updatedAt: new Date() }); - await this.assetRepository.upsertJobStatus({ assetId: asset.id, thumbnailAt: new Date() }); + if (pathsToDelete.length > 0) { + await Promise.all(pathsToDelete.map((path) => this.storageRepository.unlink(path))); + } + + if (asset.thumbhash != generated.thumbhash) { + await this.assetRepository.update({ id: asset.id, thumbhash: generated.thumbhash }); + } + + await this.assetRepository.upsertJobStatus({ assetId: asset.id, previewAt: new Date(), thumbnailAt: new Date() }); return JobStatus.SUCCESS; } - async handleGenerateThumbhash({ id }: IEntityJob): Promise<JobStatus> { - const [asset] = await this.assetRepository.getByIds([id], { files: true }); - if (!asset) { - return JobStatus.FAILED; + private async generateImageThumbnails(asset: AssetEntity) { + const { image } = await this.configCore.getConfig({ withCache: true }); + const previewPath = StorageCore.getImagePath(asset, AssetPathType.PREVIEW, image.preview.format); + const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.THUMBNAIL, image.thumbnail.format); + this.storageCore.ensureFolders(previewPath); + + const shouldExtract = image.extractEmbedded && mimeTypes.isRaw(asset.originalPath); + const extractedPath = StorageCore.getTempPathInDir(dirname(previewPath)); + const didExtract = shouldExtract && (await this.mediaRepository.extract(asset.originalPath, extractedPath)); + + try { + const useExtracted = didExtract && (await this.shouldUseExtractedImage(extractedPath, image.preview.size)); + const inputPath = useExtracted ? extractedPath : asset.originalPath; + const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace; + const processInvalidImages = process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true'; + + const decodeOptions = { colorspace, processInvalidImages, size: image.preview.size }; + const { data, info } = await this.mediaRepository.decodeImage(inputPath, decodeOptions); + + const options = { colorspace, processInvalidImages, raw: info }; + const outputs = await Promise.all([ + this.mediaRepository.generateThumbnail(data, { ...image.thumbnail, ...options }, thumbnailPath), + this.mediaRepository.generateThumbnail(data, { ...image.preview, ...options }, previewPath), + this.mediaRepository.generateThumbhash(data, options), + ]); + + return { previewPath, thumbnailPath, thumbhash: outputs[2] }; + } finally { + if (didExtract) { + await this.storageRepository.unlink(extractedPath); + } } + } - if (!asset.isVisible) { - return JobStatus.SKIPPED; + private async generateVideoThumbnails(asset: AssetEntity) { + const { image, ffmpeg } = await this.configCore.getConfig({ withCache: true }); + const previewPath = StorageCore.getImagePath(asset, AssetPathType.PREVIEW, image.preview.format); + const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.THUMBNAIL, image.thumbnail.format); + this.storageCore.ensureFolders(previewPath); + + const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); + const mainVideoStream = this.getMainStream(videoStreams); + if (!mainVideoStream) { + throw new Error(`No video streams found for asset ${asset.id}`); } + const mainAudioStream = this.getMainStream(audioStreams); - const { previewFile } = getAssetFiles(asset.files); - if (!previewFile) { - return JobStatus.FAILED; - } + const previewConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.preview.size.toString() }); + const thumbnailConfig = ThumbnailConfig.create({ ...ffmpeg, targetResolution: image.thumbnail.size.toString() }); - const thumbhash = await this.mediaRepository.generateThumbhash(previewFile.path); - await this.assetRepository.update({ id: asset.id, thumbhash }); + const previewOptions = previewConfig.getCommand(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream); + const thumbnailOptions = thumbnailConfig.getCommand(TranscodeTarget.VIDEO, mainVideoStream, mainAudioStream); + await this.mediaRepository.transcode(asset.originalPath, previewPath, previewOptions); + await this.mediaRepository.transcode(asset.originalPath, thumbnailPath, thumbnailOptions); - return JobStatus.SUCCESS; + const thumbhash = await this.mediaRepository.generateThumbhash(previewPath, { + colorspace: image.colorspace, + processInvalidImages: process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true', + }); + + return { previewPath, thumbnailPath, thumbhash }; } async handleQueueVideoConversion(job: IBaseJob): Promise<JobStatus> { diff --git a/server/src/services/microservices.service.ts b/server/src/services/microservices.service.ts index 80f1b2be41..0afefefff3 100644 --- a/server/src/services/microservices.service.ts +++ b/server/src/services/microservices.service.ts @@ -68,9 +68,7 @@ export class MicroservicesService { [JobName.MIGRATE_ASSET]: (data) => this.mediaService.handleAssetMigration(data), [JobName.MIGRATE_PERSON]: (data) => this.personService.handlePersonMigration(data), [JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data), - [JobName.GENERATE_PREVIEW]: (data) => this.mediaService.handleGeneratePreview(data), - [JobName.GENERATE_THUMBNAIL]: (data) => this.mediaService.handleGenerateThumbnail(data), - [JobName.GENERATE_THUMBHASH]: (data) => this.mediaService.handleGenerateThumbhash(data), + [JobName.GENERATE_THUMBNAILS]: (data) => this.mediaService.handleGenerateThumbnails(data), [JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data), [JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data), [JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataService.handleQueueMetadataExtraction(data), diff --git a/server/src/services/notification.service.spec.ts b/server/src/services/notification.service.spec.ts index a0b9436f75..b3a1e73541 100644 --- a/server/src/services/notification.service.spec.ts +++ b/server/src/services/notification.service.spec.ts @@ -155,7 +155,7 @@ describe(NotificationService.name, () => { it('should queue the generate thumbnail job', async () => { await sut.onAssetShow({ assetId: 'asset-id', userId: 'user-id' }); expect(jobMock.queue).toHaveBeenCalledWith({ - name: JobName.GENERATE_THUMBNAIL, + name: JobName.GENERATE_THUMBNAILS, data: { id: 'asset-id', notify: true }, }); }); diff --git a/server/src/services/notification.service.ts b/server/src/services/notification.service.ts index bdb23ce700..fdb8257ffa 100644 --- a/server/src/services/notification.service.ts +++ b/server/src/services/notification.service.ts @@ -65,7 +65,7 @@ export class NotificationService { @OnEmit({ event: 'asset.show' }) async onAssetShow({ assetId }: ArgOf<'asset.show'>) { - await this.jobRepository.queue({ name: JobName.GENERATE_THUMBNAIL, data: { id: assetId, notify: true } }); + await this.jobRepository.queue({ name: JobName.GENERATE_THUMBNAILS, data: { id: assetId, notify: true } }); } @OnEmit({ event: 'asset.trash' }) diff --git a/server/src/services/person.service.spec.ts b/server/src/services/person.service.spec.ts index 03da110ac6..c2b8f18221 100644 --- a/server/src/services/person.service.spec.ts +++ b/server/src/services/person.service.spec.ts @@ -2,7 +2,7 @@ import { BadRequestException, NotFoundException } from '@nestjs/common'; import { BulkIdErrorReason } from 'src/dtos/asset-ids.response.dto'; import { PersonResponseDto, mapFaces, mapPerson } from 'src/dtos/person.dto'; import { AssetFaceEntity } from 'src/entities/asset-face.entity'; -import { CacheControl, Colorspace, SourceType, SystemMetadataKey } from 'src/enum'; +import { CacheControl, Colorspace, ImageFormat, SourceType, SystemMetadataKey } from 'src/enum'; import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface'; import { ICryptoRepository } from 'src/interfaces/crypto.interface'; import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface'; @@ -961,12 +961,11 @@ describe(PersonService.name, () => { expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/admin_id/pe/rs'); expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( assetStub.primaryImage.originalPath, - 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', { - format: 'jpeg', + colorspace: Colorspace.P3, + format: ImageFormat.JPEG, size: 250, quality: 80, - colorspace: Colorspace.P3, crop: { left: 238, top: 163, @@ -975,6 +974,7 @@ describe(PersonService.name, () => { }, processInvalidImages: false, }, + 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', ); expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', @@ -990,13 +990,12 @@ describe(PersonService.name, () => { await sut.handleGeneratePersonThumbnail({ id: personStub.primaryPerson.id }); expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( - assetStub.image.originalPath, - 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', + assetStub.primaryImage.originalPath, { - format: 'jpeg', + colorspace: Colorspace.P3, + format: ImageFormat.JPEG, size: 250, quality: 80, - colorspace: Colorspace.P3, crop: { left: 0, top: 85, @@ -1005,6 +1004,7 @@ describe(PersonService.name, () => { }, processInvalidImages: false, }, + 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', ); }); @@ -1017,12 +1017,11 @@ describe(PersonService.name, () => { expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( assetStub.primaryImage.originalPath, - 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', { - format: 'jpeg', + colorspace: Colorspace.P3, + format: ImageFormat.JPEG, size: 250, quality: 80, - colorspace: Colorspace.P3, crop: { left: 591, top: 591, @@ -1031,33 +1030,7 @@ describe(PersonService.name, () => { }, processInvalidImages: false, }, - ); - }); - - it('should use preview path for videos', async () => { - personMock.getById.mockResolvedValue({ ...personStub.primaryPerson, faceAssetId: faceStub.end.assetId }); - personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.end); - assetMock.getById.mockResolvedValue(assetStub.video); - mediaMock.getImageDimensions.mockResolvedValue({ width: 2560, height: 1440 }); - - await sut.handleGeneratePersonThumbnail({ id: personStub.primaryPerson.id }); - - expect(mediaMock.generateThumbnail).toHaveBeenCalledWith( - '/uploads/user-id/thumbs/path.jpg', 'upload/thumbs/admin_id/pe/rs/person-1.jpeg', - { - format: 'jpeg', - size: 250, - quality: 80, - colorspace: Colorspace.P3, - crop: { - left: 1741, - top: 851, - width: 588, - height: 588, - }, - processInvalidImages: false, - }, ); }); }); diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index 651c8eebee..e8e16adb17 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -571,15 +571,15 @@ export class PersonService { this.storageCore.ensureFolders(thumbnailPath); const thumbnailOptions = { + colorspace: image.colorspace, format: ImageFormat.JPEG, size: FACE_THUMBNAIL_SIZE, - colorspace: image.colorspace, quality: image.thumbnail.quality, crop: this.getCrop({ old: { width: oldWidth, height: oldHeight }, new: { width, height } }, { x1, y1, x2, y2 }), processInvalidImages: process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true', - } as const; + }; - await this.mediaRepository.generateThumbnail(inputPath, thumbnailPath, thumbnailOptions); + await this.mediaRepository.generateThumbnail(inputPath, thumbnailOptions, thumbnailPath); await this.repository.update({ id: person.id, thumbnailPath }); return JobStatus.SUCCESS; diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index ba2f5e10d9..50fff31e55 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -39,5 +39,6 @@ export const newAssetRepositoryMock = (): Mocked<IAssetRepository> => { getChangedDeltaSync: vitest.fn(), getDuplicates: vitest.fn(), upsertFile: vitest.fn(), + upsertFiles: vitest.fn(), }; }; diff --git a/server/test/repositories/media.repository.mock.ts b/server/test/repositories/media.repository.mock.ts index 4c344a9866..a809b08162 100644 --- a/server/test/repositories/media.repository.mock.ts +++ b/server/test/repositories/media.repository.mock.ts @@ -3,8 +3,9 @@ import { Mocked, vitest } from 'vitest'; export const newMediaRepositoryMock = (): Mocked<IMediaRepository> => { return { - generateThumbnail: vitest.fn(), - generateThumbhash: vitest.fn(), + generateThumbnail: vitest.fn().mockImplementation(() => Promise.resolve()), + generateThumbhash: vitest.fn().mockImplementation(() => Promise.resolve()), + decodeImage: vitest.fn().mockResolvedValue({ data: Buffer.from(''), info: {} }), extract: vitest.fn().mockResolvedValue(false), probe: vitest.fn(), transcode: vitest.fn(),