feat(server): immich checksum header (#9229)

* feat: dedupe by checksum header

* chore: open api

open-api/immich-openapi-specs.json

@@ -1682,6 +1682,15 @@
             "schema": {
               "type": "string"
             }
+          },
+          {
+            "name": "x-immich-checksum",
+            "in": "header",
+            "description": "sha1 checksum that can be used for duplicate detection before the file is uploaded",
+            "required": false,
+            "schema": {
+              "type": "string"
+            }
           }
         ],
         "requestBody": {

open-api/typescript-sdk/src/fetch-client.ts

@@ -1524,8 +1524,9 @@ export function getAssetThumbnail({ format, id, key }: {
         ...opts
     }));
 }
-export function uploadFile({ key, createAssetDto }: {
+export function uploadFile({ key, xImmichChecksum, createAssetDto }: {
     key?: string;
+    xImmichChecksum?: string;
     createAssetDto: CreateAssetDto;
 }, opts?: Oazapfts.RequestOpts) {
     return oazapfts.ok(oazapfts.fetchJson<{
@@ -1536,7 +1537,10 @@ export function uploadFile({ key, createAssetDto }: {
     }))}`, oazapfts.multipart({
         ...opts,
         method: "POST",
-        body: createAssetDto
+        body: createAssetDto,
+        headers: oazapfts.mergeHeaders(opts?.headers, {
+            "x-immich-checksum": xImmichChecksum
+        })
     })));
 }
 export function getAssetInfo({ id, key }: {

server/src/controllers/asset-v1.controller.ts

@@ -29,7 +29,8 @@ import {
   GetAssetThumbnailDto,
   ServeFileDto,
 } from 'src/dtos/asset-v1.dto';
-import { AuthDto } from 'src/dtos/auth.dto';
+import { AuthDto, ImmichHeader } from 'src/dtos/auth.dto';
+import { AssetUploadInterceptor } from 'src/middleware/asset-upload.interceptor';
 import { Auth, Authenticated, FileResponse, SharedLinkRoute } from 'src/middleware/auth.guard';
 import { FileUploadInterceptor, ImmichFile, Route, mapToUploadFile } from 'src/middleware/file-upload.interceptor';
 import { AssetServiceV1 } from 'src/services/asset-v1.service';
@@ -50,8 +51,13 @@ export class AssetControllerV1 {
 
   @SharedLinkRoute()
   @Post('upload')
-  @UseInterceptors(FileUploadInterceptor)
+  @UseInterceptors(AssetUploadInterceptor, FileUploadInterceptor)
   @ApiConsumes('multipart/form-data')
+  @ApiHeader({
+    name: ImmichHeader.CHECKSUM,
+    description: 'sha1 checksum that can be used for duplicate detection before the file is uploaded',
+    required: false,
+  })
   @ApiBody({
     description: 'Asset Upload Information',
     type: CreateAssetDto,

server/src/dtos/auth.dto.ts

@@ -18,6 +18,7 @@ export enum ImmichHeader {
   USER_TOKEN = 'x-immich-user-token',
   SESSION_TOKEN = 'x-immich-session-token',
   SHARED_LINK_TOKEN = 'x-immich-share-key',
+  CHECKSUM = 'x-immich-checksum',
 }
 
 export type CookieResponse = {

server/src/interfaces/asset.interface.ts

@@ -159,6 +159,7 @@ export interface IAssetRepository {
   getByIdsWithAllRelations(ids: string[]): Promise<AssetEntity[]>;
   getByDayOfYear(ownerIds: string[], monthDay: MonthDay): Promise<AssetEntity[]>;
   getByChecksum(libraryId: string, checksum: Buffer): Promise<AssetEntity | null>;
+  getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise<string | undefined>;
   getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
   getByUserId(pagination: PaginationOptions, userId: string, options?: AssetSearchOptions): Paginated<AssetEntity>;
   getById(id: string, relations?: FindOptionsRelations<AssetEntity>): Promise<AssetEntity | null>;

server/src/middleware/asset-upload.interceptor.ts

@@ -0,0 +1,25 @@
+import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
+import { Response } from 'express';
+import { AssetFileUploadResponseDto } from 'src/dtos/asset-v1-response.dto';
+import { ImmichHeader } from 'src/dtos/auth.dto';
+import { AuthenticatedRequest } from 'src/middleware/auth.guard';
+import { AssetService } from 'src/services/asset.service';
+import { fromMaybeArray } from 'src/utils/request';
+
+@Injectable()
+export class AssetUploadInterceptor implements NestInterceptor {
+  constructor(private service: AssetService) {}
+
+  async intercept(context: ExecutionContext, next: CallHandler<any>) {
+    const req = context.switchToHttp().getRequest<AuthenticatedRequest>();
+    const res = context.switchToHttp().getResponse<Response<AssetFileUploadResponseDto>>();
+
+    const checksum = fromMaybeArray(req.headers[ImmichHeader.CHECKSUM]);
+    const response = await this.service.getUploadAssetIdByChecksum(req.user, checksum);
+    if (response) {
+      res.status(200).send(response);
+    }
+
+    return next.handle();
+  }
+}

server/src/middleware/auth.guard.ts

@@ -78,6 +78,10 @@ export interface AuthRequest extends Request {
   user?: AuthDto;
 }
 
+export interface AuthenticatedRequest extends Request {
+  user: AuthDto;
+}
+
 @Injectable()
 export class AuthGuard implements CanActivate {
   constructor(

server/src/queries/asset.repository.sql

@@ -473,6 +473,30 @@ WHERE
 LIMIT
   1
 
+-- AssetRepository.getUploadAssetIdByChecksum
+SELECT DISTINCT
+  "distinctAlias"."AssetEntity_id" AS "ids_AssetEntity_id"
+FROM
+  (
+    SELECT
+      "AssetEntity"."id" AS "AssetEntity_id"
+    FROM
+      "assets" "AssetEntity"
+      LEFT JOIN "libraries" "AssetEntity__AssetEntity_library" ON "AssetEntity__AssetEntity_library"."id" = "AssetEntity"."libraryId"
+    WHERE
+      (
+        ("AssetEntity"."ownerId" = $1)
+        AND ("AssetEntity"."checksum" = $2)
+        AND (
+          (("AssetEntity__AssetEntity_library"."type" = $3))
+        )
+      )
+  ) "distinctAlias"
+ORDER BY
+  "AssetEntity_id" ASC
+LIMIT
+  1
+
 -- AssetRepository.getWithout (sidecar)
 SELECT
   "AssetEntity"."id" AS "AssetEntity_id",

server/src/repositories/asset.repository.ts

@@ -5,6 +5,7 @@ import { AlbumEntity, AssetOrder } from 'src/entities/album.entity';
 import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
 import { AssetEntity, AssetType } from 'src/entities/asset.entity';
 import { ExifEntity } from 'src/entities/exif.entity';
+import { LibraryType } from 'src/entities/library.entity';
 import { PartnerEntity } from 'src/entities/partner.entity';
 import { SmartInfoEntity } from 'src/entities/smart-info.entity';
 import {
@@ -273,6 +274,23 @@ export class AssetRepository implements IAssetRepository {
     return this.repository.findOne({ where: { libraryId, checksum } });
   }
 
+  @GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] })
+  async getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise<string | undefined> {
+    const asset = await this.repository.findOne({
+      select: { id: true },
+      where: {
+        ownerId,
+        checksum,
+        library: {
+          type: LibraryType.UPLOAD,
+        },
+      },
+      withDeleted: true,
+    });
+
+    return asset?.id;
+  }
+
   findLivePhotoMatch(options: LivePhotoSearchOptions): Promise<AssetEntity | null> {
     const { ownerId, otherAssetId, livePhotoCID, type } = options;
 

server/src/services/asset-v1.service.ts

@@ -37,6 +37,7 @@ import { IUserRepository } from 'src/interfaces/user.interface';
 import { UploadFile } from 'src/services/asset.service';
 import { CacheControl, ImmichFileResponse, getLivePhotoMotionFilename } from 'src/utils/file';
 import { mimeTypes } from 'src/utils/mime-types';
+import { fromChecksum } from 'src/utils/request';
 import { QueryFailedError } from 'typeorm';
 
 @Injectable()
@@ -164,14 +165,7 @@ export class AssetServiceV1 {
   }
 
   async bulkUploadCheck(auth: AuthDto, dto: AssetBulkUploadCheckDto): Promise<AssetBulkUploadCheckResponseDto> {
-    // support base64 and hex checksums
+    const checksums: Buffer[] = dto.assets.map((asset) => fromChecksum(asset.checksum));
-    for (const asset of dto.assets) {
-      if (asset.checksum.length === 28) {
-        asset.checksum = Buffer.from(asset.checksum, 'base64').toString('hex');
-      }
-    }
-
-    const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex'));
     const results = await this.assetRepositoryV1.getAssetsByChecksums(auth.user.id, checksums);
     const checksumMap: Record<string, string> = {};
 
@@ -181,7 +175,7 @@ export class AssetServiceV1 {
 
     return {
       results: dto.assets.map(({ id, checksum }) => {
-        const duplicate = checksumMap[checksum];
+        const duplicate = checksumMap[fromChecksum(checksum).toString('hex')];
         if (duplicate) {
           return {
             id,

server/src/services/asset.service.spec.ts

@@ -29,6 +29,8 @@ import { newSystemConfigRepositoryMock } from 'test/repositories/system-config.r
 import { newUserRepositoryMock } from 'test/repositories/user.repository.mock';
 import { Mocked, vitest } from 'vitest';
 
+const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex');
+
 const stats: AssetStats = {
   [AssetType.IMAGE]: 10,
   [AssetType.VIDEO]: 23,
@@ -198,6 +200,31 @@ describe(AssetService.name, () => {
     mockGetById([assetStub.livePhotoStillAsset, assetStub.livePhotoMotionAsset]);
   });
 
+  describe('getUploadAssetIdByChecksum', () => {
+    it('should handle a non-existent asset', async () => {
+      await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('hex'))).resolves.toBeUndefined();
+      expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
+    });
+
+    it('should find an existing asset', async () => {
+      assetMock.getUploadAssetIdByChecksum.mockResolvedValue('asset-id');
+      await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('hex'))).resolves.toEqual({
+        id: 'asset-id',
+        duplicate: true,
+      });
+      expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
+    });
+
+    it('should find an existing asset by base64', async () => {
+      assetMock.getUploadAssetIdByChecksum.mockResolvedValue('asset-id');
+      await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('base64'))).resolves.toEqual({
+        id: 'asset-id',
+        duplicate: true,
+      });
+      expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
+    });
+  });
+
   describe('canUpload', () => {
     it('should require an authenticated user', () => {
       expect(() => sut.canUploadFile(uploadFile.nullAuth)).toThrowError(UnauthorizedException);

server/src/services/asset.service.ts

@@ -12,6 +12,7 @@ import {
   SanitizedAssetResponseDto,
   mapAsset,
 } from 'src/dtos/asset-response.dto';
+import { AssetFileUploadResponseDto } from 'src/dtos/asset-v1-response.dto';
 import {
   AssetBulkDeleteDto,
   AssetBulkUpdateDto,
@@ -47,6 +48,7 @@ import { ISystemConfigRepository } from 'src/interfaces/system-config.interface'
 import { IUserRepository } from 'src/interfaces/user.interface';
 import { mimeTypes } from 'src/utils/mime-types';
 import { usePagination } from 'src/utils/pagination';
+import { fromChecksum } from 'src/utils/request';
 
 export interface UploadRequest {
   auth: AuthDto | null;
@@ -83,6 +85,19 @@ export class AssetService {
     this.configCore = SystemConfigCore.create(configRepository, this.logger);
   }
 
+  async getUploadAssetIdByChecksum(auth: AuthDto, checksum?: string): Promise<AssetFileUploadResponseDto | undefined> {
+    if (!checksum) {
+      return;
+    }
+
+    const assetId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, fromChecksum(checksum));
+    if (!assetId) {
+      return;
+    }
+
+    return { id: assetId, duplicate: true };
+  }
+
   canUploadFile({ auth, fieldName, file }: UploadRequest): true {
     this.access.requireUploadAccess(auth);
 

server/src/utils/request.ts

@@ -0,0 +1,5 @@
+export const fromChecksum = (checksum: string): Buffer => {
+  return Buffer.from(checksum, checksum.length === 28 ? 'base64' : 'hex');
+};
+
+export const fromMaybeArray = (param: string | string[] | undefined) => (Array.isArray(param) ? param[0] : param);

server/test/repositories/asset.repository.mock.ts

@@ -14,6 +14,7 @@ export const newAssetRepositoryMock = (): Mocked<IAssetRepository> => {
     getById: vitest.fn(),
     getWithout: vitest.fn(),
     getByChecksum: vitest.fn(),
+    getUploadAssetIdByChecksum: vitest.fn(),
     getWith: vitest.fn(),
     getRandom: vitest.fn(),
     getFirstAssetForAlbumId: vitest.fn(),