1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-28 06:31:58 +00:00

feat(server): immich checksum header (#9229)

* feat: dedupe by checksum header

* chore: open api
This commit is contained in:
Jason Rasmussen 2024-05-02 15:42:26 -04:00 committed by GitHub
parent 16706f7f49
commit ec4eb7cd19
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 147 additions and 13 deletions

Binary file not shown.

Binary file not shown.

Binary file not shown.

View file

@ -1682,6 +1682,15 @@
"schema": {
"type": "string"
}
},
{
"name": "x-immich-checksum",
"in": "header",
"description": "sha1 checksum that can be used for duplicate detection before the file is uploaded",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": {

View file

@ -1524,8 +1524,9 @@ export function getAssetThumbnail({ format, id, key }: {
...opts
}));
}
export function uploadFile({ key, createAssetDto }: {
export function uploadFile({ key, xImmichChecksum, createAssetDto }: {
key?: string;
xImmichChecksum?: string;
createAssetDto: CreateAssetDto;
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
@ -1536,7 +1537,10 @@ export function uploadFile({ key, createAssetDto }: {
}))}`, oazapfts.multipart({
...opts,
method: "POST",
body: createAssetDto
body: createAssetDto,
headers: oazapfts.mergeHeaders(opts?.headers, {
"x-immich-checksum": xImmichChecksum
})
})));
}
export function getAssetInfo({ id, key }: {

View file

@ -29,7 +29,8 @@ import {
GetAssetThumbnailDto,
ServeFileDto,
} from 'src/dtos/asset-v1.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { AuthDto, ImmichHeader } from 'src/dtos/auth.dto';
import { AssetUploadInterceptor } from 'src/middleware/asset-upload.interceptor';
import { Auth, Authenticated, FileResponse, SharedLinkRoute } from 'src/middleware/auth.guard';
import { FileUploadInterceptor, ImmichFile, Route, mapToUploadFile } from 'src/middleware/file-upload.interceptor';
import { AssetServiceV1 } from 'src/services/asset-v1.service';
@ -50,8 +51,13 @@ export class AssetControllerV1 {
@SharedLinkRoute()
@Post('upload')
@UseInterceptors(FileUploadInterceptor)
@UseInterceptors(AssetUploadInterceptor, FileUploadInterceptor)
@ApiConsumes('multipart/form-data')
@ApiHeader({
name: ImmichHeader.CHECKSUM,
description: 'sha1 checksum that can be used for duplicate detection before the file is uploaded',
required: false,
})
@ApiBody({
description: 'Asset Upload Information',
type: CreateAssetDto,

View file

@ -18,6 +18,7 @@ export enum ImmichHeader {
USER_TOKEN = 'x-immich-user-token',
SESSION_TOKEN = 'x-immich-session-token',
SHARED_LINK_TOKEN = 'x-immich-share-key',
CHECKSUM = 'x-immich-checksum',
}
export type CookieResponse = {

View file

@ -159,6 +159,7 @@ export interface IAssetRepository {
getByIdsWithAllRelations(ids: string[]): Promise<AssetEntity[]>;
getByDayOfYear(ownerIds: string[], monthDay: MonthDay): Promise<AssetEntity[]>;
getByChecksum(libraryId: string, checksum: Buffer): Promise<AssetEntity | null>;
getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise<string | undefined>;
getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
getByUserId(pagination: PaginationOptions, userId: string, options?: AssetSearchOptions): Paginated<AssetEntity>;
getById(id: string, relations?: FindOptionsRelations<AssetEntity>): Promise<AssetEntity | null>;

View file

@ -0,0 +1,25 @@
import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
import { Response } from 'express';
import { AssetFileUploadResponseDto } from 'src/dtos/asset-v1-response.dto';
import { ImmichHeader } from 'src/dtos/auth.dto';
import { AuthenticatedRequest } from 'src/middleware/auth.guard';
import { AssetService } from 'src/services/asset.service';
import { fromMaybeArray } from 'src/utils/request';
@Injectable()
export class AssetUploadInterceptor implements NestInterceptor {
constructor(private service: AssetService) {}
async intercept(context: ExecutionContext, next: CallHandler<any>) {
const req = context.switchToHttp().getRequest<AuthenticatedRequest>();
const res = context.switchToHttp().getResponse<Response<AssetFileUploadResponseDto>>();
const checksum = fromMaybeArray(req.headers[ImmichHeader.CHECKSUM]);
const response = await this.service.getUploadAssetIdByChecksum(req.user, checksum);
if (response) {
res.status(200).send(response);
}
return next.handle();
}
}

View file

@ -78,6 +78,10 @@ export interface AuthRequest extends Request {
user?: AuthDto;
}
export interface AuthenticatedRequest extends Request {
user: AuthDto;
}
@Injectable()
export class AuthGuard implements CanActivate {
constructor(

View file

@ -473,6 +473,30 @@ WHERE
LIMIT
1
-- AssetRepository.getUploadAssetIdByChecksum
SELECT DISTINCT
"distinctAlias"."AssetEntity_id" AS "ids_AssetEntity_id"
FROM
(
SELECT
"AssetEntity"."id" AS "AssetEntity_id"
FROM
"assets" "AssetEntity"
LEFT JOIN "libraries" "AssetEntity__AssetEntity_library" ON "AssetEntity__AssetEntity_library"."id" = "AssetEntity"."libraryId"
WHERE
(
("AssetEntity"."ownerId" = $1)
AND ("AssetEntity"."checksum" = $2)
AND (
(("AssetEntity__AssetEntity_library"."type" = $3))
)
)
) "distinctAlias"
ORDER BY
"AssetEntity_id" ASC
LIMIT
1
-- AssetRepository.getWithout (sidecar)
SELECT
"AssetEntity"."id" AS "AssetEntity_id",

View file

@ -5,6 +5,7 @@ import { AlbumEntity, AssetOrder } from 'src/entities/album.entity';
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
import { AssetEntity, AssetType } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { LibraryType } from 'src/entities/library.entity';
import { PartnerEntity } from 'src/entities/partner.entity';
import { SmartInfoEntity } from 'src/entities/smart-info.entity';
import {
@ -273,6 +274,23 @@ export class AssetRepository implements IAssetRepository {
return this.repository.findOne({ where: { libraryId, checksum } });
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] })
async getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise<string | undefined> {
const asset = await this.repository.findOne({
select: { id: true },
where: {
ownerId,
checksum,
library: {
type: LibraryType.UPLOAD,
},
},
withDeleted: true,
});
return asset?.id;
}
findLivePhotoMatch(options: LivePhotoSearchOptions): Promise<AssetEntity | null> {
const { ownerId, otherAssetId, livePhotoCID, type } = options;

View file

@ -37,6 +37,7 @@ import { IUserRepository } from 'src/interfaces/user.interface';
import { UploadFile } from 'src/services/asset.service';
import { CacheControl, ImmichFileResponse, getLivePhotoMotionFilename } from 'src/utils/file';
import { mimeTypes } from 'src/utils/mime-types';
import { fromChecksum } from 'src/utils/request';
import { QueryFailedError } from 'typeorm';
@Injectable()
@ -164,14 +165,7 @@ export class AssetServiceV1 {
}
async bulkUploadCheck(auth: AuthDto, dto: AssetBulkUploadCheckDto): Promise<AssetBulkUploadCheckResponseDto> {
// support base64 and hex checksums
for (const asset of dto.assets) {
if (asset.checksum.length === 28) {
asset.checksum = Buffer.from(asset.checksum, 'base64').toString('hex');
}
}
const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex'));
const checksums: Buffer[] = dto.assets.map((asset) => fromChecksum(asset.checksum));
const results = await this.assetRepositoryV1.getAssetsByChecksums(auth.user.id, checksums);
const checksumMap: Record<string, string> = {};
@ -181,7 +175,7 @@ export class AssetServiceV1 {
return {
results: dto.assets.map(({ id, checksum }) => {
const duplicate = checksumMap[checksum];
const duplicate = checksumMap[fromChecksum(checksum).toString('hex')];
if (duplicate) {
return {
id,

View file

@ -29,6 +29,8 @@ import { newSystemConfigRepositoryMock } from 'test/repositories/system-config.r
import { newUserRepositoryMock } from 'test/repositories/user.repository.mock';
import { Mocked, vitest } from 'vitest';
const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex');
const stats: AssetStats = {
[AssetType.IMAGE]: 10,
[AssetType.VIDEO]: 23,
@ -198,6 +200,31 @@ describe(AssetService.name, () => {
mockGetById([assetStub.livePhotoStillAsset, assetStub.livePhotoMotionAsset]);
});
describe('getUploadAssetIdByChecksum', () => {
it('should handle a non-existent asset', async () => {
await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('hex'))).resolves.toBeUndefined();
expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
});
it('should find an existing asset', async () => {
assetMock.getUploadAssetIdByChecksum.mockResolvedValue('asset-id');
await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('hex'))).resolves.toEqual({
id: 'asset-id',
duplicate: true,
});
expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
});
it('should find an existing asset by base64', async () => {
assetMock.getUploadAssetIdByChecksum.mockResolvedValue('asset-id');
await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('base64'))).resolves.toEqual({
id: 'asset-id',
duplicate: true,
});
expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
});
});
describe('canUpload', () => {
it('should require an authenticated user', () => {
expect(() => sut.canUploadFile(uploadFile.nullAuth)).toThrowError(UnauthorizedException);

View file

@ -12,6 +12,7 @@ import {
SanitizedAssetResponseDto,
mapAsset,
} from 'src/dtos/asset-response.dto';
import { AssetFileUploadResponseDto } from 'src/dtos/asset-v1-response.dto';
import {
AssetBulkDeleteDto,
AssetBulkUpdateDto,
@ -47,6 +48,7 @@ import { ISystemConfigRepository } from 'src/interfaces/system-config.interface'
import { IUserRepository } from 'src/interfaces/user.interface';
import { mimeTypes } from 'src/utils/mime-types';
import { usePagination } from 'src/utils/pagination';
import { fromChecksum } from 'src/utils/request';
export interface UploadRequest {
auth: AuthDto | null;
@ -83,6 +85,19 @@ export class AssetService {
this.configCore = SystemConfigCore.create(configRepository, this.logger);
}
async getUploadAssetIdByChecksum(auth: AuthDto, checksum?: string): Promise<AssetFileUploadResponseDto | undefined> {
if (!checksum) {
return;
}
const assetId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, fromChecksum(checksum));
if (!assetId) {
return;
}
return { id: assetId, duplicate: true };
}
canUploadFile({ auth, fieldName, file }: UploadRequest): true {
this.access.requireUploadAccess(auth);

View file

@ -0,0 +1,5 @@
export const fromChecksum = (checksum: string): Buffer => {
return Buffer.from(checksum, checksum.length === 28 ? 'base64' : 'hex');
};
export const fromMaybeArray = (param: string | string[] | undefined) => (Array.isArray(param) ? param[0] : param);

View file

@ -14,6 +14,7 @@ export const newAssetRepositoryMock = (): Mocked<IAssetRepository> => {
getById: vitest.fn(),
getWithout: vitest.fn(),
getByChecksum: vitest.fn(),
getUploadAssetIdByChecksum: vitest.fn(),
getWith: vitest.fn(),
getRandom: vitest.fn(),
getFirstAssetForAlbumId: vitest.fn(),