mirror of
https://github.com/immich-app/immich.git
synced 2024-12-28 06:31:58 +00:00
feat(server): immich checksum header (#9229)
* feat: dedupe by checksum header * chore: open api
This commit is contained in:
parent
16706f7f49
commit
ec4eb7cd19
17 changed files with 147 additions and 13 deletions
BIN
mobile/openapi/doc/AssetApi.md
generated
BIN
mobile/openapi/doc/AssetApi.md
generated
Binary file not shown.
BIN
mobile/openapi/lib/api/asset_api.dart
generated
BIN
mobile/openapi/lib/api/asset_api.dart
generated
Binary file not shown.
BIN
mobile/openapi/test/asset_api_test.dart
generated
BIN
mobile/openapi/test/asset_api_test.dart
generated
Binary file not shown.
|
@ -1682,6 +1682,15 @@
|
|||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "x-immich-checksum",
|
||||
"in": "header",
|
||||
"description": "sha1 checksum that can be used for duplicate detection before the file is uploaded",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
|
|
|
@ -1524,8 +1524,9 @@ export function getAssetThumbnail({ format, id, key }: {
|
|||
...opts
|
||||
}));
|
||||
}
|
||||
export function uploadFile({ key, createAssetDto }: {
|
||||
export function uploadFile({ key, xImmichChecksum, createAssetDto }: {
|
||||
key?: string;
|
||||
xImmichChecksum?: string;
|
||||
createAssetDto: CreateAssetDto;
|
||||
}, opts?: Oazapfts.RequestOpts) {
|
||||
return oazapfts.ok(oazapfts.fetchJson<{
|
||||
|
@ -1536,7 +1537,10 @@ export function uploadFile({ key, createAssetDto }: {
|
|||
}))}`, oazapfts.multipart({
|
||||
...opts,
|
||||
method: "POST",
|
||||
body: createAssetDto
|
||||
body: createAssetDto,
|
||||
headers: oazapfts.mergeHeaders(opts?.headers, {
|
||||
"x-immich-checksum": xImmichChecksum
|
||||
})
|
||||
})));
|
||||
}
|
||||
export function getAssetInfo({ id, key }: {
|
||||
|
|
|
@ -29,7 +29,8 @@ import {
|
|||
GetAssetThumbnailDto,
|
||||
ServeFileDto,
|
||||
} from 'src/dtos/asset-v1.dto';
|
||||
import { AuthDto } from 'src/dtos/auth.dto';
|
||||
import { AuthDto, ImmichHeader } from 'src/dtos/auth.dto';
|
||||
import { AssetUploadInterceptor } from 'src/middleware/asset-upload.interceptor';
|
||||
import { Auth, Authenticated, FileResponse, SharedLinkRoute } from 'src/middleware/auth.guard';
|
||||
import { FileUploadInterceptor, ImmichFile, Route, mapToUploadFile } from 'src/middleware/file-upload.interceptor';
|
||||
import { AssetServiceV1 } from 'src/services/asset-v1.service';
|
||||
|
@ -50,8 +51,13 @@ export class AssetControllerV1 {
|
|||
|
||||
@SharedLinkRoute()
|
||||
@Post('upload')
|
||||
@UseInterceptors(FileUploadInterceptor)
|
||||
@UseInterceptors(AssetUploadInterceptor, FileUploadInterceptor)
|
||||
@ApiConsumes('multipart/form-data')
|
||||
@ApiHeader({
|
||||
name: ImmichHeader.CHECKSUM,
|
||||
description: 'sha1 checksum that can be used for duplicate detection before the file is uploaded',
|
||||
required: false,
|
||||
})
|
||||
@ApiBody({
|
||||
description: 'Asset Upload Information',
|
||||
type: CreateAssetDto,
|
||||
|
|
|
@ -18,6 +18,7 @@ export enum ImmichHeader {
|
|||
USER_TOKEN = 'x-immich-user-token',
|
||||
SESSION_TOKEN = 'x-immich-session-token',
|
||||
SHARED_LINK_TOKEN = 'x-immich-share-key',
|
||||
CHECKSUM = 'x-immich-checksum',
|
||||
}
|
||||
|
||||
export type CookieResponse = {
|
||||
|
|
|
@ -159,6 +159,7 @@ export interface IAssetRepository {
|
|||
getByIdsWithAllRelations(ids: string[]): Promise<AssetEntity[]>;
|
||||
getByDayOfYear(ownerIds: string[], monthDay: MonthDay): Promise<AssetEntity[]>;
|
||||
getByChecksum(libraryId: string, checksum: Buffer): Promise<AssetEntity | null>;
|
||||
getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise<string | undefined>;
|
||||
getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
|
||||
getByUserId(pagination: PaginationOptions, userId: string, options?: AssetSearchOptions): Paginated<AssetEntity>;
|
||||
getById(id: string, relations?: FindOptionsRelations<AssetEntity>): Promise<AssetEntity | null>;
|
||||
|
|
25
server/src/middleware/asset-upload.interceptor.ts
Normal file
25
server/src/middleware/asset-upload.interceptor.ts
Normal file
|
@ -0,0 +1,25 @@
|
|||
import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nestjs/common';
|
||||
import { Response } from 'express';
|
||||
import { AssetFileUploadResponseDto } from 'src/dtos/asset-v1-response.dto';
|
||||
import { ImmichHeader } from 'src/dtos/auth.dto';
|
||||
import { AuthenticatedRequest } from 'src/middleware/auth.guard';
|
||||
import { AssetService } from 'src/services/asset.service';
|
||||
import { fromMaybeArray } from 'src/utils/request';
|
||||
|
||||
@Injectable()
|
||||
export class AssetUploadInterceptor implements NestInterceptor {
|
||||
constructor(private service: AssetService) {}
|
||||
|
||||
async intercept(context: ExecutionContext, next: CallHandler<any>) {
|
||||
const req = context.switchToHttp().getRequest<AuthenticatedRequest>();
|
||||
const res = context.switchToHttp().getResponse<Response<AssetFileUploadResponseDto>>();
|
||||
|
||||
const checksum = fromMaybeArray(req.headers[ImmichHeader.CHECKSUM]);
|
||||
const response = await this.service.getUploadAssetIdByChecksum(req.user, checksum);
|
||||
if (response) {
|
||||
res.status(200).send(response);
|
||||
}
|
||||
|
||||
return next.handle();
|
||||
}
|
||||
}
|
|
@ -78,6 +78,10 @@ export interface AuthRequest extends Request {
|
|||
user?: AuthDto;
|
||||
}
|
||||
|
||||
export interface AuthenticatedRequest extends Request {
|
||||
user: AuthDto;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AuthGuard implements CanActivate {
|
||||
constructor(
|
||||
|
|
|
@ -473,6 +473,30 @@ WHERE
|
|||
LIMIT
|
||||
1
|
||||
|
||||
-- AssetRepository.getUploadAssetIdByChecksum
|
||||
SELECT DISTINCT
|
||||
"distinctAlias"."AssetEntity_id" AS "ids_AssetEntity_id"
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
"AssetEntity"."id" AS "AssetEntity_id"
|
||||
FROM
|
||||
"assets" "AssetEntity"
|
||||
LEFT JOIN "libraries" "AssetEntity__AssetEntity_library" ON "AssetEntity__AssetEntity_library"."id" = "AssetEntity"."libraryId"
|
||||
WHERE
|
||||
(
|
||||
("AssetEntity"."ownerId" = $1)
|
||||
AND ("AssetEntity"."checksum" = $2)
|
||||
AND (
|
||||
(("AssetEntity__AssetEntity_library"."type" = $3))
|
||||
)
|
||||
)
|
||||
) "distinctAlias"
|
||||
ORDER BY
|
||||
"AssetEntity_id" ASC
|
||||
LIMIT
|
||||
1
|
||||
|
||||
-- AssetRepository.getWithout (sidecar)
|
||||
SELECT
|
||||
"AssetEntity"."id" AS "AssetEntity_id",
|
||||
|
|
|
@ -5,6 +5,7 @@ import { AlbumEntity, AssetOrder } from 'src/entities/album.entity';
|
|||
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
|
||||
import { AssetEntity, AssetType } from 'src/entities/asset.entity';
|
||||
import { ExifEntity } from 'src/entities/exif.entity';
|
||||
import { LibraryType } from 'src/entities/library.entity';
|
||||
import { PartnerEntity } from 'src/entities/partner.entity';
|
||||
import { SmartInfoEntity } from 'src/entities/smart-info.entity';
|
||||
import {
|
||||
|
@ -273,6 +274,23 @@ export class AssetRepository implements IAssetRepository {
|
|||
return this.repository.findOne({ where: { libraryId, checksum } });
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID, DummyValue.BUFFER] })
|
||||
async getUploadAssetIdByChecksum(ownerId: string, checksum: Buffer): Promise<string | undefined> {
|
||||
const asset = await this.repository.findOne({
|
||||
select: { id: true },
|
||||
where: {
|
||||
ownerId,
|
||||
checksum,
|
||||
library: {
|
||||
type: LibraryType.UPLOAD,
|
||||
},
|
||||
},
|
||||
withDeleted: true,
|
||||
});
|
||||
|
||||
return asset?.id;
|
||||
}
|
||||
|
||||
findLivePhotoMatch(options: LivePhotoSearchOptions): Promise<AssetEntity | null> {
|
||||
const { ownerId, otherAssetId, livePhotoCID, type } = options;
|
||||
|
||||
|
|
|
@ -37,6 +37,7 @@ import { IUserRepository } from 'src/interfaces/user.interface';
|
|||
import { UploadFile } from 'src/services/asset.service';
|
||||
import { CacheControl, ImmichFileResponse, getLivePhotoMotionFilename } from 'src/utils/file';
|
||||
import { mimeTypes } from 'src/utils/mime-types';
|
||||
import { fromChecksum } from 'src/utils/request';
|
||||
import { QueryFailedError } from 'typeorm';
|
||||
|
||||
@Injectable()
|
||||
|
@ -164,14 +165,7 @@ export class AssetServiceV1 {
|
|||
}
|
||||
|
||||
async bulkUploadCheck(auth: AuthDto, dto: AssetBulkUploadCheckDto): Promise<AssetBulkUploadCheckResponseDto> {
|
||||
// support base64 and hex checksums
|
||||
for (const asset of dto.assets) {
|
||||
if (asset.checksum.length === 28) {
|
||||
asset.checksum = Buffer.from(asset.checksum, 'base64').toString('hex');
|
||||
}
|
||||
}
|
||||
|
||||
const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex'));
|
||||
const checksums: Buffer[] = dto.assets.map((asset) => fromChecksum(asset.checksum));
|
||||
const results = await this.assetRepositoryV1.getAssetsByChecksums(auth.user.id, checksums);
|
||||
const checksumMap: Record<string, string> = {};
|
||||
|
||||
|
@ -181,7 +175,7 @@ export class AssetServiceV1 {
|
|||
|
||||
return {
|
||||
results: dto.assets.map(({ id, checksum }) => {
|
||||
const duplicate = checksumMap[checksum];
|
||||
const duplicate = checksumMap[fromChecksum(checksum).toString('hex')];
|
||||
if (duplicate) {
|
||||
return {
|
||||
id,
|
||||
|
|
|
@ -29,6 +29,8 @@ import { newSystemConfigRepositoryMock } from 'test/repositories/system-config.r
|
|||
import { newUserRepositoryMock } from 'test/repositories/user.repository.mock';
|
||||
import { Mocked, vitest } from 'vitest';
|
||||
|
||||
const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex');
|
||||
|
||||
const stats: AssetStats = {
|
||||
[AssetType.IMAGE]: 10,
|
||||
[AssetType.VIDEO]: 23,
|
||||
|
@ -198,6 +200,31 @@ describe(AssetService.name, () => {
|
|||
mockGetById([assetStub.livePhotoStillAsset, assetStub.livePhotoMotionAsset]);
|
||||
});
|
||||
|
||||
describe('getUploadAssetIdByChecksum', () => {
|
||||
it('should handle a non-existent asset', async () => {
|
||||
await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('hex'))).resolves.toBeUndefined();
|
||||
expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
|
||||
});
|
||||
|
||||
it('should find an existing asset', async () => {
|
||||
assetMock.getUploadAssetIdByChecksum.mockResolvedValue('asset-id');
|
||||
await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('hex'))).resolves.toEqual({
|
||||
id: 'asset-id',
|
||||
duplicate: true,
|
||||
});
|
||||
expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
|
||||
});
|
||||
|
||||
it('should find an existing asset by base64', async () => {
|
||||
assetMock.getUploadAssetIdByChecksum.mockResolvedValue('asset-id');
|
||||
await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('base64'))).resolves.toEqual({
|
||||
id: 'asset-id',
|
||||
duplicate: true,
|
||||
});
|
||||
expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('canUpload', () => {
|
||||
it('should require an authenticated user', () => {
|
||||
expect(() => sut.canUploadFile(uploadFile.nullAuth)).toThrowError(UnauthorizedException);
|
||||
|
|
|
@ -12,6 +12,7 @@ import {
|
|||
SanitizedAssetResponseDto,
|
||||
mapAsset,
|
||||
} from 'src/dtos/asset-response.dto';
|
||||
import { AssetFileUploadResponseDto } from 'src/dtos/asset-v1-response.dto';
|
||||
import {
|
||||
AssetBulkDeleteDto,
|
||||
AssetBulkUpdateDto,
|
||||
|
@ -47,6 +48,7 @@ import { ISystemConfigRepository } from 'src/interfaces/system-config.interface'
|
|||
import { IUserRepository } from 'src/interfaces/user.interface';
|
||||
import { mimeTypes } from 'src/utils/mime-types';
|
||||
import { usePagination } from 'src/utils/pagination';
|
||||
import { fromChecksum } from 'src/utils/request';
|
||||
|
||||
export interface UploadRequest {
|
||||
auth: AuthDto | null;
|
||||
|
@ -83,6 +85,19 @@ export class AssetService {
|
|||
this.configCore = SystemConfigCore.create(configRepository, this.logger);
|
||||
}
|
||||
|
||||
async getUploadAssetIdByChecksum(auth: AuthDto, checksum?: string): Promise<AssetFileUploadResponseDto | undefined> {
|
||||
if (!checksum) {
|
||||
return;
|
||||
}
|
||||
|
||||
const assetId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, fromChecksum(checksum));
|
||||
if (!assetId) {
|
||||
return;
|
||||
}
|
||||
|
||||
return { id: assetId, duplicate: true };
|
||||
}
|
||||
|
||||
canUploadFile({ auth, fieldName, file }: UploadRequest): true {
|
||||
this.access.requireUploadAccess(auth);
|
||||
|
||||
|
|
5
server/src/utils/request.ts
Normal file
5
server/src/utils/request.ts
Normal file
|
@ -0,0 +1,5 @@
|
|||
export const fromChecksum = (checksum: string): Buffer => {
|
||||
return Buffer.from(checksum, checksum.length === 28 ? 'base64' : 'hex');
|
||||
};
|
||||
|
||||
export const fromMaybeArray = (param: string | string[] | undefined) => (Array.isArray(param) ? param[0] : param);
|
|
@ -14,6 +14,7 @@ export const newAssetRepositoryMock = (): Mocked<IAssetRepository> => {
|
|||
getById: vitest.fn(),
|
||||
getWithout: vitest.fn(),
|
||||
getByChecksum: vitest.fn(),
|
||||
getUploadAssetIdByChecksum: vitest.fn(),
|
||||
getWith: vitest.fn(),
|
||||
getRandom: vitest.fn(),
|
||||
getFirstAssetForAlbumId: vitest.fn(),
|
||||
|
|
Loading…
Reference in a new issue