From ec4eb7cd19c09d57f9440bab1d9808c9a2e4e8b5 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 2 May 2024 15:42:26 -0400 Subject: [PATCH] feat(server): immich checksum header (#9229) * feat: dedupe by checksum header * chore: open api --- mobile/openapi/doc/AssetApi.md | Bin 39094 -> 39404 bytes mobile/openapi/lib/api/asset_api.dart | Bin 35922 -> 36389 bytes mobile/openapi/test/asset_api_test.dart | Bin 3820 -> 3844 bytes open-api/immich-openapi-specs.json | 9 ++++++ open-api/typescript-sdk/src/fetch-client.ts | 8 ++++-- server/src/controllers/asset-v1.controller.ts | 10 +++++-- server/src/dtos/auth.dto.ts | 1 + server/src/interfaces/asset.interface.ts | 1 + .../middleware/asset-upload.interceptor.ts | 25 ++++++++++++++++ server/src/middleware/auth.guard.ts | 4 +++ server/src/queries/asset.repository.sql | 24 ++++++++++++++++ server/src/repositories/asset.repository.ts | 18 ++++++++++++ server/src/services/asset-v1.service.ts | 12 ++------ server/src/services/asset.service.spec.ts | 27 ++++++++++++++++++ server/src/services/asset.service.ts | 15 ++++++++++ server/src/utils/request.ts | 5 ++++ .../repositories/asset.repository.mock.ts | 1 + 17 files changed, 147 insertions(+), 13 deletions(-) create mode 100644 server/src/middleware/asset-upload.interceptor.ts create mode 100644 server/src/utils/request.ts diff --git a/mobile/openapi/doc/AssetApi.md b/mobile/openapi/doc/AssetApi.md index f335ca58681d1b31df4c5bdf3927c294b34345fa..5c2af2594b31e0066c9908836b9dffc0477ada08 100644 GIT binary patch delta 313 zcmdnCk?GB5rVU2j{1u+LxtYls&Kar6*~O)~n+>|JvnrwqD%dKZiA-LoCl*wkk!Yxp z4ArPml95=VkeryOkd&%WTAZ4qkd|MhkWyNZlbM`YlB$rBT9TSvl9>;Z2g;-Z)u$?? zW#*(RWELv`Rplq9q^4||D8Yp8qR9r`Ih&g%A7HFUbAy(aLXCo!R&YsCW?s6MR*eGn V-2=2BI==wq>cpJb&9+l3nE|>SbqfFh delta 33 rcmV++0N(%XvjVoU0ikpo5XqHai(0*cauao`e$5rx-!kK9b8t4iZHV99iCCw*Tp_>VI zt}e`BlOHsQV;CoIqO)0{W*6rGc+P-vGFSTsbQf*j(C)&hfaWPHbb-yEIue)xNQ##y delta 51 zcmZ2Fhw0J`rVYFmn8QXKrq0a)xt8YI1gQY3}5Ay#D~8ZwvhZ delta 11 ScmZpXdn3EyJMUzEzOMitd<6vn diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index cfcdbfbaa9..31ca9b2937 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -1682,6 +1682,15 @@ "schema": { "type": "string" } + }, + { + "name": "x-immich-checksum", + "in": "header", + "description": "sha1 checksum that can be used for duplicate detection before the file is uploaded", + "required": false, + "schema": { + "type": "string" + } } ], "requestBody": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 55a8e78d50..fee15d0fbc 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -1524,8 +1524,9 @@ export function getAssetThumbnail({ format, id, key }: { ...opts })); } -export function uploadFile({ key, createAssetDto }: { +export function uploadFile({ key, xImmichChecksum, createAssetDto }: { key?: string; + xImmichChecksum?: string; createAssetDto: CreateAssetDto; }, opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchJson<{ @@ -1536,7 +1537,10 @@ export function uploadFile({ key, createAssetDto }: { }))}`, oazapfts.multipart({ ...opts, method: "POST", - body: createAssetDto + body: createAssetDto, + headers: oazapfts.mergeHeaders(opts?.headers, { + "x-immich-checksum": xImmichChecksum + }) }))); } export function getAssetInfo({ id, key }: { diff --git a/server/src/controllers/asset-v1.controller.ts b/server/src/controllers/asset-v1.controller.ts index e83df8d0d1..908569b58f 100644 --- a/server/src/controllers/asset-v1.controller.ts +++ b/server/src/controllers/asset-v1.controller.ts @@ -29,7 +29,8 @@ import { GetAssetThumbnailDto, ServeFileDto, } from 'src/dtos/asset-v1.dto'; -import { AuthDto } from 'src/dtos/auth.dto'; +import { AuthDto, ImmichHeader } from 'src/dtos/auth.dto'; +import { AssetUploadInterceptor } from 'src/middleware/asset-upload.interceptor'; import { Auth, Authenticated, FileResponse, SharedLinkRoute } from 'src/middleware/auth.guard'; import { FileUploadInterceptor, ImmichFile, Route, mapToUploadFile } from 'src/middleware/file-upload.interceptor'; import { AssetServiceV1 } from 'src/services/asset-v1.service'; @@ -50,8 +51,13 @@ export class AssetControllerV1 { @SharedLinkRoute() @Post('upload') - @UseInterceptors(FileUploadInterceptor) + @UseInterceptors(AssetUploadInterceptor, FileUploadInterceptor) @ApiConsumes('multipart/form-data') + @ApiHeader({ + name: ImmichHeader.CHECKSUM, + description: 'sha1 checksum that can be used for duplicate detection before the file is uploaded', + required: false, + }) @ApiBody({ description: 'Asset Upload Information', type: CreateAssetDto, diff --git a/server/src/dtos/auth.dto.ts b/server/src/dtos/auth.dto.ts index 5c1e01b818..73843729b9 100644 --- a/server/src/dtos/auth.dto.ts +++ b/server/src/dtos/auth.dto.ts @@ -18,6 +18,7 @@ export enum ImmichHeader { USER_TOKEN = 'x-immich-user-token', SESSION_TOKEN = 'x-immich-session-token', SHARED_LINK_TOKEN = 'x-immich-share-key', + CHECKSUM = 'x-immich-checksum', } export type CookieResponse = { diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index cad83f09d4..ad0bc1864f 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -159,6 +159,7 @@ export interface IAssetRepository { getByIdsWithAllRelations(ids: string[]): Promise; getByDayOfYear(ownerIds: string[], monthDay: MonthDay): Promise; getByChecksum(libraryId: string, checksum: Buffer): Promise; + getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise; getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated; getByUserId(pagination: PaginationOptions, userId: string, options?: AssetSearchOptions): Paginated; getById(id: string, relations?: FindOptionsRelations): Promise; diff --git a/server/src/middleware/asset-upload.interceptor.ts b/server/src/middleware/asset-upload.interceptor.ts new file mode 100644 index 0000000000..845e6f906d --- /dev/null +++ b/server/src/middleware/asset-upload.interceptor.ts @@ -0,0 +1,25 @@ +import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common'; +import { Response } from 'express'; +import { AssetFileUploadResponseDto } from 'src/dtos/asset-v1-response.dto'; +import { ImmichHeader } from 'src/dtos/auth.dto'; +import { AuthenticatedRequest } from 'src/middleware/auth.guard'; +import { AssetService } from 'src/services/asset.service'; +import { fromMaybeArray } from 'src/utils/request'; + +@Injectable() +export class AssetUploadInterceptor implements NestInterceptor { + constructor(private service: AssetService) {} + + async intercept(context: ExecutionContext, next: CallHandler) { + const req = context.switchToHttp().getRequest(); + const res = context.switchToHttp().getResponse>(); + + const checksum = fromMaybeArray(req.headers[ImmichHeader.CHECKSUM]); + const response = await this.service.getUploadAssetIdByChecksum(req.user, checksum); + if (response) { + res.status(200).send(response); + } + + return next.handle(); + } +} diff --git a/server/src/middleware/auth.guard.ts b/server/src/middleware/auth.guard.ts index 1253e99bbb..59e82c00aa 100644 --- a/server/src/middleware/auth.guard.ts +++ b/server/src/middleware/auth.guard.ts @@ -78,6 +78,10 @@ export interface AuthRequest extends Request { user?: AuthDto; } +export interface AuthenticatedRequest extends Request { + user: AuthDto; +} + @Injectable() export class AuthGuard implements CanActivate { constructor( diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index 7d49fb18df..8e8e55e6e3 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -473,6 +473,30 @@ WHERE LIMIT 1 +-- AssetRepository.getUploadAssetIdByChecksum +SELECT DISTINCT + "distinctAlias"."AssetEntity_id" AS "ids_AssetEntity_id" +FROM + ( + SELECT + "AssetEntity"."id" AS "AssetEntity_id" + FROM + "assets" "AssetEntity" + LEFT JOIN "libraries" "AssetEntity__AssetEntity_library" ON "AssetEntity__AssetEntity_library"."id" = "AssetEntity"."libraryId" + WHERE + ( + ("AssetEntity"."ownerId" = $1) + AND ("AssetEntity"."checksum" = $2) + AND ( + (("AssetEntity__AssetEntity_library"."type" = $3)) + ) + ) + ) "distinctAlias" +ORDER BY + "AssetEntity_id" ASC +LIMIT + 1 + -- AssetRepository.getWithout (sidecar) SELECT "AssetEntity"."id" AS "AssetEntity_id", diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index a961ab97d6..7618996fb1 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -5,6 +5,7 @@ import { AlbumEntity, AssetOrder } from 'src/entities/album.entity'; import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity'; import { AssetEntity, AssetType } from 'src/entities/asset.entity'; import { ExifEntity } from 'src/entities/exif.entity'; +import { LibraryType } from 'src/entities/library.entity'; import { PartnerEntity } from 'src/entities/partner.entity'; import { SmartInfoEntity } from 'src/entities/smart-info.entity'; import { @@ -273,6 +274,23 @@ export class AssetRepository implements IAssetRepository { return this.repository.findOne({ where: { libraryId, checksum } }); } + @GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] }) + async getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise { + const asset = await this.repository.findOne({ + select: { id: true }, + where: { + ownerId, + checksum, + library: { + type: LibraryType.UPLOAD, + }, + }, + withDeleted: true, + }); + + return asset?.id; + } + findLivePhotoMatch(options: LivePhotoSearchOptions): Promise { const { ownerId, otherAssetId, livePhotoCID, type } = options; diff --git a/server/src/services/asset-v1.service.ts b/server/src/services/asset-v1.service.ts index e7affe1073..22b2618541 100644 --- a/server/src/services/asset-v1.service.ts +++ b/server/src/services/asset-v1.service.ts @@ -37,6 +37,7 @@ import { IUserRepository } from 'src/interfaces/user.interface'; import { UploadFile } from 'src/services/asset.service'; import { CacheControl, ImmichFileResponse, getLivePhotoMotionFilename } from 'src/utils/file'; import { mimeTypes } from 'src/utils/mime-types'; +import { fromChecksum } from 'src/utils/request'; import { QueryFailedError } from 'typeorm'; @Injectable() @@ -164,14 +165,7 @@ export class AssetServiceV1 { } async bulkUploadCheck(auth: AuthDto, dto: AssetBulkUploadCheckDto): Promise { - // support base64 and hex checksums - for (const asset of dto.assets) { - if (asset.checksum.length === 28) { - asset.checksum = Buffer.from(asset.checksum, 'base64').toString('hex'); - } - } - - const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex')); + const checksums: Buffer[] = dto.assets.map((asset) => fromChecksum(asset.checksum)); const results = await this.assetRepositoryV1.getAssetsByChecksums(auth.user.id, checksums); const checksumMap: Record = {}; @@ -181,7 +175,7 @@ export class AssetServiceV1 { return { results: dto.assets.map(({ id, checksum }) => { - const duplicate = checksumMap[checksum]; + const duplicate = checksumMap[fromChecksum(checksum).toString('hex')]; if (duplicate) { return { id, diff --git a/server/src/services/asset.service.spec.ts b/server/src/services/asset.service.spec.ts index 49753923a2..27a7eeb88d 100755 --- a/server/src/services/asset.service.spec.ts +++ b/server/src/services/asset.service.spec.ts @@ -29,6 +29,8 @@ import { newSystemConfigRepositoryMock } from 'test/repositories/system-config.r import { newUserRepositoryMock } from 'test/repositories/user.repository.mock'; import { Mocked, vitest } from 'vitest'; +const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex'); + const stats: AssetStats = { [AssetType.IMAGE]: 10, [AssetType.VIDEO]: 23, @@ -198,6 +200,31 @@ describe(AssetService.name, () => { mockGetById([assetStub.livePhotoStillAsset, assetStub.livePhotoMotionAsset]); }); + describe('getUploadAssetIdByChecksum', () => { + it('should handle a non-existent asset', async () => { + await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('hex'))).resolves.toBeUndefined(); + expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1); + }); + + it('should find an existing asset', async () => { + assetMock.getUploadAssetIdByChecksum.mockResolvedValue('asset-id'); + await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('hex'))).resolves.toEqual({ + id: 'asset-id', + duplicate: true, + }); + expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1); + }); + + it('should find an existing asset by base64', async () => { + assetMock.getUploadAssetIdByChecksum.mockResolvedValue('asset-id'); + await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('base64'))).resolves.toEqual({ + id: 'asset-id', + duplicate: true, + }); + expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1); + }); + }); + describe('canUpload', () => { it('should require an authenticated user', () => { expect(() => sut.canUploadFile(uploadFile.nullAuth)).toThrowError(UnauthorizedException); diff --git a/server/src/services/asset.service.ts b/server/src/services/asset.service.ts index d9113f0787..afb167f902 100644 --- a/server/src/services/asset.service.ts +++ b/server/src/services/asset.service.ts @@ -12,6 +12,7 @@ import { SanitizedAssetResponseDto, mapAsset, } from 'src/dtos/asset-response.dto'; +import { AssetFileUploadResponseDto } from 'src/dtos/asset-v1-response.dto'; import { AssetBulkDeleteDto, AssetBulkUpdateDto, @@ -47,6 +48,7 @@ import { ISystemConfigRepository } from 'src/interfaces/system-config.interface' import { IUserRepository } from 'src/interfaces/user.interface'; import { mimeTypes } from 'src/utils/mime-types'; import { usePagination } from 'src/utils/pagination'; +import { fromChecksum } from 'src/utils/request'; export interface UploadRequest { auth: AuthDto | null; @@ -83,6 +85,19 @@ export class AssetService { this.configCore = SystemConfigCore.create(configRepository, this.logger); } + async getUploadAssetIdByChecksum(auth: AuthDto, checksum?: string): Promise { + if (!checksum) { + return; + } + + const assetId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, fromChecksum(checksum)); + if (!assetId) { + return; + } + + return { id: assetId, duplicate: true }; + } + canUploadFile({ auth, fieldName, file }: UploadRequest): true { this.access.requireUploadAccess(auth); diff --git a/server/src/utils/request.ts b/server/src/utils/request.ts new file mode 100644 index 0000000000..f6edb2f8b3 --- /dev/null +++ b/server/src/utils/request.ts @@ -0,0 +1,5 @@ +export const fromChecksum = (checksum: string): Buffer => { + return Buffer.from(checksum, checksum.length === 28 ? 'base64' : 'hex'); +}; + +export const fromMaybeArray = (param: string | string[] | undefined) => (Array.isArray(param) ? param[0] : param); diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index f09d6b619e..e22fc1f011 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -14,6 +14,7 @@ export const newAssetRepositoryMock = (): Mocked => { getById: vitest.fn(), getWithout: vitest.fn(), getByChecksum: vitest.fn(), + getUploadAssetIdByChecksum: vitest.fn(), getWith: vitest.fn(), getRandom: vitest.fn(), getFirstAssetForAlbumId: vitest.fn(),