feat(server): extract full-size previews from RAW images

open-api/immich-openapi-specs.json

@@ -8406,6 +8406,7 @@
       },
       "AssetMediaSize": {
         "enum": [
+          "original",
           "preview",
           "thumbnail"
         ],
@@ -10186,6 +10187,7 @@
       "PathType": {
         "enum": [
           "original",
+          "extracted",
           "preview",
           "thumbnail",
           "encoded_video",

open-api/typescript-sdk/src/fetch-client.ts

@@ -3464,6 +3464,7 @@ export enum AssetJobName {
     TranscodeVideo = "transcode-video"
 }
 export enum AssetMediaSize {
+    Original = "original",
     Preview = "preview",
     Thumbnail = "thumbnail"
 }
@@ -3514,6 +3515,7 @@ export enum PathEntityType {
 }
 export enum PathType {
     Original = "original",
+    Extracted = "extracted",
     Preview = "preview",
     Thumbnail = "thumbnail",
     EncodedVideo = "encoded_video",

server/src/cores/storage.core.ts

@@ -26,7 +26,7 @@ export interface MoveRequest {
   };
 }
 
-export type GeneratedImageType = AssetPathType.PREVIEW | AssetPathType.THUMBNAIL;
+export type GeneratedImageType = AssetPathType.PREVIEW | AssetPathType.THUMBNAIL | AssetPathType.EXTRACTED;
 export type GeneratedAssetType = GeneratedImageType | AssetPathType.ENCODED_VIDEO;
 
 let instance: StorageCore | null;

server/src/dtos/asset-media.dto.ts

@@ -4,6 +4,11 @@ import { ArrayNotEmpty, IsArray, IsEnum, IsNotEmpty, IsString, ValidateNested }
 import { Optional, ValidateBoolean, ValidateDate, ValidateUUID } from 'src/validation';
 
 export enum AssetMediaSize {
+  /**
+   * An original-sized JPG extracted from the RAW image,
+   * or otherwise the original non-RAW image itself.
+   */
+  ORIGINAL = 'original',
   PREVIEW = 'preview',
   THUMBNAIL = 'thumbnail',
 }

server/src/enum.ts

@@ -33,6 +33,10 @@ export enum AssetType {
 }
 
 export enum AssetFileType {
+  /**
+   * An full/large-size image extracted/converted from RAW photos
+   */
+  EXTRACTED = 'extracted',
   PREVIEW = 'preview',
   THUMBNAIL = 'thumbnail',
 }
@@ -237,6 +241,7 @@ export enum ManualJobName {
 
 export enum AssetPathType {
   ORIGINAL = 'original',
+  EXTRACTED = 'extracted',
   PREVIEW = 'preview',
   THUMBNAIL = 'thumbnail',
   ENCODED_VIDEO = 'encoded_video',

server/src/interfaces/asset.interface.ts

@@ -185,6 +185,7 @@ export interface IAssetRepository {
   updateDuplicates(options: AssetUpdateDuplicateOptions): Promise<void>;
   update(asset: AssetUpdateOptions): Promise<void>;
   remove(asset: AssetEntity): Promise<void>;
+  removeAssetFile(path: string): Promise<void>;
   findLivePhotoMatch(options: LivePhotoSearchOptions): Promise<AssetEntity | null>;
   getStatistics(ownerId: string, options: AssetStatsOptions): Promise<AssetStats>;
   getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>;

server/src/repositories/asset.repository.ts

@@ -298,6 +298,10 @@ export class AssetRepository implements IAssetRepository {
     await this.repository.remove(asset);
   }
 
+  async removeAssetFile(path: string): Promise<void> {
+    await this.fileRepository.delete({ path });
+  }
+
   @GenerateSql({ params: [{ ownerId: DummyValue.UUID, libraryId: DummyValue.UUID, checksum: DummyValue.BUFFER }] })
   getByChecksum({
     ownerId,

server/src/repositories/media.repository.ts

@@ -44,6 +44,11 @@ export class MediaRepository implements IMediaRepository {
 
   async extract(input: string, output: string): Promise<boolean> {
     try {
+      // remove existing output file if it exists
+      // as exiftool-vendord does not support overwriting via "-w!" flag
+      // and throws "1 files could not be read" error when the output file exists
+      await fs.unlink(output).catch(() => null);
+      this.logger.debug('Extracting JPEG from RAW image:', input);
       await exiftool.extractJpgFromRaw(input, output);
     } catch (error: any) {
       this.logger.debug('Could not extract JPEG from image, trying preview', error.message);
@@ -98,6 +103,10 @@ export class MediaRepository implements IMediaRepository {
       pipeline = pipeline.extract(options.crop);
     }
 
+    // Infinity is a special value that means no resizing
+    if (options.size === Infinity) {
+      return pipeline;
+    }
     return pipeline.resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true });
   }
 

server/src/services/asset-media.service.ts

@@ -208,10 +208,17 @@ export class AssetMediaService extends BaseService {
     const asset = await this.findOrFail(id);
     const size = dto.size ?? AssetMediaSize.THUMBNAIL;
 
-    const { thumbnailFile, previewFile } = getAssetFiles(asset.files);
+    const { thumbnailFile, previewFile, extractedFile } = getAssetFiles(asset.files);
     let filepath = previewFile?.path;
     if (size === AssetMediaSize.THUMBNAIL && thumbnailFile) {
       filepath = thumbnailFile.path;
+    } else if (size === AssetMediaSize.ORIGINAL) {
+      // eslint-disable-next-line unicorn/prefer-ternary
+      if (mimeTypes.isRaw(asset.originalPath)) {
+        filepath = extractedFile?.path ?? previewFile?.path;
+      } else {
+        filepath = asset.originalPath;
+      }
     }
 
     if (!filepath) {

server/src/services/media.service.spec.ts

@@ -269,7 +269,7 @@ describe(MediaService.name, () => {
       });
 
       await expect(sut.handleAssetMigration({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);
-      expect(moveMock.create).toHaveBeenCalledTimes(2);
+      expect(moveMock.create).toHaveBeenCalledTimes(3);
     });
   });
 
@@ -634,8 +634,6 @@ describe(MediaService.name, () => {
         processInvalidImages: false,
         size: 1440,
       });
-      expect(extractedPath?.endsWith('.tmp')).toBe(true);
-      expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath);
     });
 
     it('should resize original image if embedded image is too small', async () => {
@@ -646,15 +644,19 @@ describe(MediaService.name, () => {
 
       await sut.handleGenerateThumbnails({ id: assetStub.image.id });
 
+      const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString();
+      expect(mediaMock.generateThumbnail).toHaveBeenCalledWith(
+        assetStub.imageDng.originalPath,
+        expect.objectContaining({ size: Infinity }),
+        extractedPath,
+      );
+      expect(extractedPath).toMatch(/-extracted\.jpeg$/);
       expect(mediaMock.decodeImage).toHaveBeenCalledOnce();
-      expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, {
+      expect(mediaMock.decodeImage).toHaveBeenCalledWith(extractedPath, {
         colorspace: Colorspace.P3,
         processInvalidImages: false,
         size: 1440,
       });
-      const extractedPath = mediaMock.extract.mock.calls.at(-1)?.[1].toString();
-      expect(extractedPath?.endsWith('.tmp')).toBe(true);
-      expect(storageMock.unlink).toHaveBeenCalledWith(extractedPath);
     });
 
     it('should resize original image if embedded image not found', async () => {
@@ -664,7 +666,7 @@ describe(MediaService.name, () => {
       await sut.handleGenerateThumbnails({ id: assetStub.image.id });
 
       expect(mediaMock.decodeImage).toHaveBeenCalledOnce();
-      expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, {
+      expect(mediaMock.decodeImage).toHaveBeenCalledWith('upload/thumbs/user-id/as/se/asset-id-extracted.jpeg', {
         colorspace: Colorspace.P3,
         processInvalidImages: false,
         size: 1440,
@@ -680,7 +682,7 @@ describe(MediaService.name, () => {
 
       expect(mediaMock.extract).not.toHaveBeenCalled();
       expect(mediaMock.decodeImage).toHaveBeenCalledOnce();
-      expect(mediaMock.decodeImage).toHaveBeenCalledWith(assetStub.imageDng.originalPath, {
+      expect(mediaMock.decodeImage).toHaveBeenCalledWith('upload/thumbs/user-id/as/se/asset-id-extracted.jpeg', {
         colorspace: Colorspace.P3,
         processInvalidImages: false,
         size: 1440,
@@ -697,11 +699,16 @@ describe(MediaService.name, () => {
 
       expect(mediaMock.decodeImage).toHaveBeenCalledOnce();
       expect(mediaMock.decodeImage).toHaveBeenCalledWith(
-        assetStub.imageDng.originalPath,
+        'upload/thumbs/user-id/as/se/asset-id-extracted.jpeg',
         expect.objectContaining({ processInvalidImages: true }),
       );
 
-      expect(mediaMock.generateThumbnail).toHaveBeenCalledTimes(2);
+      expect(mediaMock.generateThumbnail).toHaveBeenCalledTimes(3);
+      expect(mediaMock.generateThumbnail).toHaveBeenCalledWith(
+        assetStub.imageDng.originalPath,
+        expect.objectContaining({ processInvalidImages: true }),
+        'upload/thumbs/user-id/as/se/asset-id-extracted.jpeg',
+      );
       expect(mediaMock.generateThumbnail).toHaveBeenCalledWith(
         rawBuffer,
         expect.objectContaining({ processInvalidImages: true }),

server/src/services/media.service.ts

@@ -1,5 +1,4 @@
 import { Injectable } from '@nestjs/common';
-import { dirname } from 'node:path';
 import { StorageCore } from 'src/cores/storage.core';
 import { OnEvent, OnJob } from 'src/decorators';
 import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
@@ -10,6 +9,7 @@ import {
   AssetType,
   AudioCodec,
   Colorspace,
+  ImageFormat,
   LogLevel,
   StorageFolder,
   TranscodeHWAccel,
@@ -135,6 +135,7 @@ export class MediaService extends BaseService {
       return JobStatus.FAILED;
     }
 
+    await this.storageCore.moveAssetImage(asset, AssetPathType.EXTRACTED, ImageFormat.JPEG);
     await this.storageCore.moveAssetImage(asset, AssetPathType.PREVIEW, image.preview.format);
     await this.storageCore.moveAssetImage(asset, AssetPathType.THUMBNAIL, image.thumbnail.format);
     await this.storageCore.moveAssetVideo(asset);
@@ -155,7 +156,12 @@ export class MediaService extends BaseService {
       return JobStatus.SKIPPED;
     }
 
-    let generated: { previewPath: string; thumbnailPath: string; thumbhash: Buffer };
+    let generated: {
+      previewPath: string;
+      thumbnailPath: string;
+      extractedPath?: string;
+      thumbhash: Buffer;
+    };
     if (asset.type === AssetType.VIDEO || asset.originalFileName.toLowerCase().endsWith('.gif')) {
       generated = await this.generateVideoThumbnails(asset);
     } else if (asset.type === AssetType.IMAGE) {
@@ -165,7 +171,7 @@ export class MediaService extends BaseService {
       return JobStatus.SKIPPED;
     }
 
-    const { previewFile, thumbnailFile } = getAssetFiles(asset.files);
+    const { previewFile, thumbnailFile, extractedFile } = getAssetFiles(asset.files);
     const toUpsert: UpsertFileOptions[] = [];
     if (previewFile?.path !== generated.previewPath) {
       toUpsert.push({ assetId: asset.id, path: generated.previewPath, type: AssetFileType.PREVIEW });
@@ -175,11 +181,15 @@ export class MediaService extends BaseService {
       toUpsert.push({ assetId: asset.id, path: generated.thumbnailPath, type: AssetFileType.THUMBNAIL });
     }
 
+    if (generated.extractedPath && extractedFile?.path !== generated.extractedPath) {
+      toUpsert.push({ assetId: asset.id, path: generated.extractedPath, type: AssetFileType.EXTRACTED });
+    }
+
     if (toUpsert.length > 0) {
       await this.assetRepository.upsertFiles(toUpsert);
     }
 
-    const pathsToDelete = [];
+    const pathsToDelete: string[] = [];
     if (previewFile && previewFile.path !== generated.previewPath) {
       this.logger.debug(`Deleting old preview for asset ${asset.id}`);
       pathsToDelete.push(previewFile.path);
@@ -190,8 +200,18 @@ export class MediaService extends BaseService {
       pathsToDelete.push(thumbnailFile.path);
     }
 
+    if (extractedFile && extractedFile.path !== generated.extractedPath) {
+      this.logger.debug(`Deleting old extracted image for asset ${asset.id}`);
+      pathsToDelete.push(extractedFile.path);
+    }
+
     if (pathsToDelete.length > 0) {
-      await Promise.all(pathsToDelete.map((path) => this.storageRepository.unlink(path)));
+      await Promise.all(
+        pathsToDelete.map(async (path) => {
+          await this.storageRepository.unlink(path);
+          await this.assetRepository.removeAssetFile(path);
+        }),
+      );
     }
 
     if (asset.thumbhash != generated.thumbhash) {
@@ -209,33 +229,50 @@ export class MediaService extends BaseService {
     const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.THUMBNAIL, image.thumbnail.format);
     this.storageCore.ensureFolders(previewPath);
 
-    const shouldExtract = image.extractEmbedded && mimeTypes.isRaw(asset.originalPath);
-    const extractedPath = StorageCore.getTempPathInDir(dirname(previewPath));
-    const didExtract = shouldExtract && (await this.mediaRepository.extract(asset.originalPath, extractedPath));
+    const processInvalidImages = process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true';
+    const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace;
+    const decodeOptions = { colorspace, processInvalidImages, size: image.preview.size };
 
-    try {
-      const useExtracted = didExtract && (await this.shouldUseExtractedImage(extractedPath, image.preview.size));
-      const inputPath = useExtracted ? extractedPath : asset.originalPath;
-      const colorspace = this.isSRGB(asset) ? Colorspace.SRGB : image.colorspace;
-      const processInvalidImages = process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true';
-
-      const orientation = useExtracted && asset.exifInfo?.orientation ? Number(asset.exifInfo.orientation) : undefined;
-      const decodeOptions = { colorspace, processInvalidImages, size: image.preview.size, orientation };
-      const { data, info } = await this.mediaRepository.decodeImage(inputPath, decodeOptions);
-
-      const options = { colorspace, processInvalidImages, raw: info };
-      const outputs = await Promise.all([
-        this.mediaRepository.generateThumbnail(data, { ...image.thumbnail, ...options }, thumbnailPath),
-        this.mediaRepository.generateThumbnail(data, { ...image.preview, ...options }, previewPath),
-        this.mediaRepository.generateThumbhash(data, options),
-      ]);
-
-      return { previewPath, thumbnailPath, thumbhash: outputs[2] };
-    } finally {
-      if (didExtract) {
-        await this.storageRepository.unlink(extractedPath);
+    let fullsizePath: string;
+    let extractedPath: string | undefined;
+    if (mimeTypes.isRaw(asset.originalPath)) {
+      let useExtracted = false;
+      extractedPath = StorageCore.getImagePath(asset, AssetPathType.EXTRACTED, ImageFormat.JPEG);
+      if (image.extractEmbedded) {
+        // try extracting embedded preview from RAW
+        // extracted image from RAW is always in JPEG format, as implied from the `jpgFromRaw` tag name
+        const didExtract = await this.mediaRepository.extract(asset.originalPath, extractedPath);
+        useExtracted = didExtract && (await this.shouldUseExtractedImage(extractedPath, image.preview.size));
       }
+
+      if (useExtracted) {
+        fullsizePath = extractedPath;
+      } else {
+        // did not extract or extracted preview is smaller than target size,
+        // convert a full-sized thumbnail from original instead
+        extractedPath = StorageCore.getImagePath(asset, AssetPathType.EXTRACTED, image.preview.format);
+        // const orientation = asset.exifInfo?.orientation ? Number(asset.exifInfo.orientation) : undefined;
+        await this.mediaRepository.generateThumbnail(
+          asset.originalPath,
+          { ...image.preview, colorspace, processInvalidImages, size: Infinity },
+          extractedPath,
+        );
+      }
+      fullsizePath = extractedPath;
+    } else {
+      fullsizePath = asset.originalPath;
     }
+
+    const { info, data } = await this.mediaRepository.decodeImage(fullsizePath, decodeOptions);
+
+    const thumbnailOptions = { colorspace, processInvalidImages, raw: info };
+    const outputs = await Promise.all([
+      this.mediaRepository.generateThumbnail(data, { ...image.thumbnail, ...thumbnailOptions }, thumbnailPath),
+      this.mediaRepository.generateThumbnail(data, { ...image.preview, ...thumbnailOptions }, previewPath),
+      this.mediaRepository.generateThumbhash(data, thumbnailOptions),
+    ]);
+
+    return { previewPath, thumbnailPath, extractedPath, thumbhash: outputs[2] };
   }
 
   private async generateVideoThumbnails(asset: AssetEntity) {

server/src/utils/asset.util.ts

@@ -25,6 +25,7 @@ const getFileByType = (files: AssetFileEntity[] | undefined, type: AssetFileType
 };
 
 export const getAssetFiles = (files?: AssetFileEntity[]) => ({
+  extractedFile: getFileByType(files, AssetFileType.EXTRACTED),
   previewFile: getFileByType(files, AssetFileType.PREVIEW),
   thumbnailFile: getFileByType(files, AssetFileType.THUMBNAIL),
 });

server/src/utils/file.ts

@@ -46,7 +46,7 @@ export const sendFile = async (
     const file = await handler();
     switch (file.cacheControl) {
       case CacheControl.PRIVATE_WITH_CACHE: {
-        res.set('Cache-Control', 'private, max-age=86400, no-transform');
+        res.set('Cache-Control', 'private, max-age=3600, stale-while-revalidate=86400, no-transform');
         break;
       }
 

server/test/repositories/asset.repository.mock.ts

@@ -28,6 +28,7 @@ export const newAssetRepositoryMock = (): Mocked<IAssetRepository> => {
     deleteAll: vitest.fn(),
     update: vitest.fn(),
     remove: vitest.fn(),
+    removeAssetFile: vitest.fn(),
     findLivePhotoMatch: vitest.fn(),
     getStatistics: vitest.fn(),
     getTimeBucket: vitest.fn(),