diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 263687549f..b296bbcb55 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -24,6 +24,7 @@ doc/AssetBulkUploadCheckDto.md doc/AssetBulkUploadCheckItem.md doc/AssetBulkUploadCheckResponseDto.md doc/AssetBulkUploadCheckResult.md +doc/AssetDeltaSyncResponseDto.md doc/AssetFaceResponseDto.md doc/AssetFaceUpdateDto.md doc/AssetFaceUpdateItem.md @@ -149,6 +150,7 @@ doc/SharedLinkType.md doc/SignUpDto.md doc/SmartInfoResponseDto.md doc/SmartSearchDto.md +doc/SyncApi.md doc/SystemConfigApi.md doc/SystemConfigDto.md doc/SystemConfigFFmpegDto.md @@ -218,6 +220,7 @@ lib/api/person_api.dart lib/api/search_api.dart lib/api/server_info_api.dart lib/api/shared_link_api.dart +lib/api/sync_api.dart lib/api/system_config_api.dart lib/api/tag_api.dart lib/api/timeline_api.dart @@ -248,6 +251,7 @@ lib/model/asset_bulk_upload_check_dto.dart lib/model/asset_bulk_upload_check_item.dart lib/model/asset_bulk_upload_check_response_dto.dart lib/model/asset_bulk_upload_check_result.dart +lib/model/asset_delta_sync_response_dto.dart lib/model/asset_face_response_dto.dart lib/model/asset_face_update_dto.dart lib/model/asset_face_update_item.dart @@ -427,6 +431,7 @@ test/asset_bulk_upload_check_dto_test.dart test/asset_bulk_upload_check_item_test.dart test/asset_bulk_upload_check_response_dto_test.dart test/asset_bulk_upload_check_result_test.dart +test/asset_delta_sync_response_dto_test.dart test/asset_face_response_dto_test.dart test/asset_face_update_dto_test.dart test/asset_face_update_item_test.dart @@ -552,6 +557,7 @@ test/shared_link_type_test.dart test/sign_up_dto_test.dart test/smart_info_response_dto_test.dart test/smart_search_dto_test.dart +test/sync_api_test.dart test/system_config_api_test.dart test/system_config_dto_test.dart test/system_config_f_fmpeg_dto_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 9a03fbd61c..730307b9bf 100644 Binary files a/mobile/openapi/README.md and b/mobile/openapi/README.md differ diff --git a/mobile/openapi/doc/AssetDeltaSyncResponseDto.md b/mobile/openapi/doc/AssetDeltaSyncResponseDto.md new file mode 100644 index 0000000000..527203a4b8 Binary files /dev/null and b/mobile/openapi/doc/AssetDeltaSyncResponseDto.md differ diff --git a/mobile/openapi/doc/SyncApi.md b/mobile/openapi/doc/SyncApi.md new file mode 100644 index 0000000000..1b28e10c8c Binary files /dev/null and b/mobile/openapi/doc/SyncApi.md differ diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index ae458f5de1..e7320d5bb2 100644 Binary files a/mobile/openapi/lib/api.dart and b/mobile/openapi/lib/api.dart differ diff --git a/mobile/openapi/lib/api/sync_api.dart b/mobile/openapi/lib/api/sync_api.dart new file mode 100644 index 0000000000..fdfd8b9ac7 Binary files /dev/null and b/mobile/openapi/lib/api/sync_api.dart differ diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 757f475683..4bbae89285 100644 Binary files a/mobile/openapi/lib/api_client.dart and b/mobile/openapi/lib/api_client.dart differ diff --git a/mobile/openapi/lib/model/asset_delta_sync_response_dto.dart b/mobile/openapi/lib/model/asset_delta_sync_response_dto.dart new file mode 100644 index 0000000000..5d7679e734 Binary files /dev/null and b/mobile/openapi/lib/model/asset_delta_sync_response_dto.dart differ diff --git a/mobile/openapi/test/asset_delta_sync_response_dto_test.dart b/mobile/openapi/test/asset_delta_sync_response_dto_test.dart new file mode 100644 index 0000000000..20104c08c6 Binary files /dev/null and b/mobile/openapi/test/asset_delta_sync_response_dto_test.dart differ diff --git a/mobile/openapi/test/sync_api_test.dart b/mobile/openapi/test/sync_api_test.dart new file mode 100644 index 0000000000..ad9ef0f92f Binary files /dev/null and b/mobile/openapi/test/sync_api_test.dart differ diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index fbedfd359d..8f53f838b0 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -5566,6 +5566,140 @@ ] } }, + "/sync/delta-sync": { + "get": { + "operationId": "getDeltaSync", + "parameters": [ + { + "name": "updatedAfter", + "required": true, + "in": "query", + "schema": { + "format": "date-time", + "type": "string" + } + }, + { + "name": "userIds", + "required": true, + "in": "query", + "schema": { + "format": "uuid", + "type": "array", + "items": { + "type": "string" + } + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AssetDeltaSyncResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + } + }, + "/sync/full-sync": { + "get": { + "operationId": "getAllForUserFullSync", + "parameters": [ + { + "name": "lastCreationDate", + "required": false, + "in": "query", + "schema": { + "format": "date-time", + "type": "string" + } + }, + { + "name": "lastId", + "required": false, + "in": "query", + "schema": { + "format": "uuid", + "type": "string" + } + }, + { + "name": "limit", + "required": true, + "in": "query", + "schema": { + "type": "integer" + } + }, + { + "name": "updatedUntil", + "required": true, + "in": "query", + "schema": { + "format": "date-time", + "type": "string" + } + }, + { + "name": "userId", + "required": false, + "in": "query", + "schema": { + "format": "uuid", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/AssetResponseDto" + }, + "type": "array" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + } + }, "/system-config": { "get": { "operationId": "getConfig", @@ -7335,6 +7469,31 @@ ], "type": "object" }, + "AssetDeltaSyncResponseDto": { + "properties": { + "deleted": { + "items": { + "type": "string" + }, + "type": "array" + }, + "needsFullSync": { + "type": "boolean" + }, + "upserted": { + "items": { + "$ref": "#/components/schemas/AssetResponseDto" + }, + "type": "array" + } + }, + "required": [ + "deleted", + "needsFullSync", + "upserted" + ], + "type": "object" + }, "AssetFaceResponseDto": { "properties": { "boundingBoxX1": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 43f766dfe2..96b071f1f9 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -835,6 +835,11 @@ export type AssetIdsResponseDto = { error?: Error2; success: boolean; }; +export type AssetDeltaSyncResponseDto = { + deleted: string[]; + needsFullSync: boolean; + upserted: AssetResponseDto[]; +}; export type SystemConfigFFmpegDto = { accel: TranscodeHWAccel; acceptedAudioCodecs: AudioCodec[]; @@ -2507,6 +2512,40 @@ export function addSharedLinkAssets({ id, key, assetIdsDto }: { body: assetIdsDto }))); } +export function getDeltaSync({ updatedAfter, userIds }: { + updatedAfter: string; + userIds: string[]; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: AssetDeltaSyncResponseDto; + }>(`/sync/delta-sync${QS.query(QS.explode({ + updatedAfter, + userIds + }))}`, { + ...opts + })); +} +export function getAllForUserFullSync({ lastCreationDate, lastId, limit, updatedUntil, userId }: { + lastCreationDate?: string; + lastId?: string; + limit: number; + updatedUntil: string; + userId?: string; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: AssetResponseDto[]; + }>(`/sync/full-sync${QS.query(QS.explode({ + lastCreationDate, + lastId, + limit, + updatedUntil, + userId + }))}`, { + ...opts + })); +} export function getConfig(opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchJson<{ status: 200; diff --git a/server/src/controllers/index.ts b/server/src/controllers/index.ts index ce51aa4c01..d136a52b04 100644 --- a/server/src/controllers/index.ts +++ b/server/src/controllers/index.ts @@ -17,6 +17,7 @@ import { PersonController } from 'src/controllers/person.controller'; import { SearchController } from 'src/controllers/search.controller'; import { ServerInfoController } from 'src/controllers/server-info.controller'; import { SharedLinkController } from 'src/controllers/shared-link.controller'; +import { SyncController } from 'src/controllers/sync.controller'; import { SystemConfigController } from 'src/controllers/system-config.controller'; import { TagController } from 'src/controllers/tag.controller'; import { TimelineController } from 'src/controllers/timeline.controller'; @@ -43,6 +44,7 @@ export const controllers = [ SearchController, ServerInfoController, SharedLinkController, + SyncController, SystemConfigController, TagController, TimelineController, diff --git a/server/src/controllers/sync.controller.ts b/server/src/controllers/sync.controller.ts new file mode 100644 index 0000000000..c12d42df23 --- /dev/null +++ b/server/src/controllers/sync.controller.ts @@ -0,0 +1,24 @@ +import { Controller, Get, Query } from '@nestjs/common'; +import { ApiTags } from '@nestjs/swagger'; +import { AssetResponseDto } from 'src/dtos/asset-response.dto'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto'; +import { Auth, Authenticated } from 'src/middleware/auth.guard'; +import { SyncService } from 'src/services/sync.service'; + +@ApiTags('Sync') +@Controller('sync') +@Authenticated() +export class SyncController { + constructor(private service: SyncService) {} + + @Get('full-sync') + getAllForUserFullSync(@Auth() auth: AuthDto, @Query() dto: AssetFullSyncDto): Promise { + return this.service.getAllAssetsForUserFullSync(auth, dto); + } + + @Get('delta-sync') + getDeltaSync(@Auth() auth: AuthDto, @Query() dto: AssetDeltaSyncDto): Promise { + return this.service.getChangesForDeltaSync(auth, dto); + } +} diff --git a/server/src/dtos/sync.dto.ts b/server/src/dtos/sync.dto.ts new file mode 100644 index 0000000000..a69062ec2d --- /dev/null +++ b/server/src/dtos/sync.dto.ts @@ -0,0 +1,38 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { Type } from 'class-transformer'; +import { IsInt, IsPositive } from 'class-validator'; +import { AssetResponseDto } from 'src/dtos/asset-response.dto'; +import { ValidateDate, ValidateUUID } from 'src/validation'; + +export class AssetFullSyncDto { + @ValidateUUID({ optional: true }) + lastId?: string; + + @ValidateDate({ optional: true }) + lastCreationDate?: Date; + + @ValidateDate() + updatedUntil!: Date; + + @IsInt() + @IsPositive() + @Type(() => Number) + @ApiProperty({ type: 'integer' }) + limit!: number; + + @ValidateUUID({ optional: true }) + userId?: string; +} + +export class AssetDeltaSyncDto { + @ValidateDate() + updatedAfter!: Date; + @ValidateUUID({ each: true }) + userIds!: string[]; +} + +export class AssetDeltaSyncResponseDto { + needsFullSync!: boolean; + upserted!: AssetResponseDto[]; + deleted!: string[]; +} diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index c5905f3b55..9c2ebe3e73 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -133,6 +133,20 @@ export interface MetadataSearchOptions { numResults: number; } +export interface AssetFullSyncOptions { + ownerId: string; + lastCreationDate?: Date; + lastId?: string; + updatedUntil: Date; + limit: number; +} + +export interface AssetDeltaSyncOptions { + userIds: string[]; + updatedAfter: Date; + limit: number; +} + export type AssetPathEntity = Pick; export const IAssetRepository = 'IAssetRepository'; @@ -175,4 +189,6 @@ export interface IAssetRepository { getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise>; getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise>; searchMetadata(query: string, userIds: string[], options: MetadataSearchOptions): Promise; + getAllForUserFullSync(options: AssetFullSyncOptions): Promise; + getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise; } diff --git a/server/src/interfaces/audit.interface.ts b/server/src/interfaces/audit.interface.ts index 767a4bc2f6..b023d00d56 100644 --- a/server/src/interfaces/audit.interface.ts +++ b/server/src/interfaces/audit.interface.ts @@ -1,14 +1,14 @@ -import { AuditEntity, DatabaseAction, EntityType } from 'src/entities/audit.entity'; +import { DatabaseAction, EntityType } from 'src/entities/audit.entity'; export const IAuditRepository = 'IAuditRepository'; export interface AuditSearch { action?: DatabaseAction; entityType?: EntityType; - ownerId?: string; + userIds: string[]; } export interface IAuditRepository { - getAfter(since: Date, options: AuditSearch): Promise; + getAfter(since: Date, options: AuditSearch): Promise; removeBefore(before: Date): Promise; } diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index e223539dc0..86e9796faa 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -768,3 +768,151 @@ ORDER BY "asset"."fileCreatedAt" DESC LIMIT 250 + +-- AssetRepository.getAllForUserFullSync +SELECT + "asset"."id" AS "asset_id", + "asset"."deviceAssetId" AS "asset_deviceAssetId", + "asset"."ownerId" AS "asset_ownerId", + "asset"."libraryId" AS "asset_libraryId", + "asset"."deviceId" AS "asset_deviceId", + "asset"."type" AS "asset_type", + "asset"."originalPath" AS "asset_originalPath", + "asset"."previewPath" AS "asset_previewPath", + "asset"."thumbnailPath" AS "asset_thumbnailPath", + "asset"."thumbhash" AS "asset_thumbhash", + "asset"."encodedVideoPath" AS "asset_encodedVideoPath", + "asset"."createdAt" AS "asset_createdAt", + "asset"."updatedAt" AS "asset_updatedAt", + "asset"."deletedAt" AS "asset_deletedAt", + "asset"."fileCreatedAt" AS "asset_fileCreatedAt", + "asset"."localDateTime" AS "asset_localDateTime", + "asset"."fileModifiedAt" AS "asset_fileModifiedAt", + "asset"."isFavorite" AS "asset_isFavorite", + "asset"."isArchived" AS "asset_isArchived", + "asset"."isExternal" AS "asset_isExternal", + "asset"."isReadOnly" AS "asset_isReadOnly", + "asset"."isOffline" AS "asset_isOffline", + "asset"."checksum" AS "asset_checksum", + "asset"."duration" AS "asset_duration", + "asset"."isVisible" AS "asset_isVisible", + "asset"."livePhotoVideoId" AS "asset_livePhotoVideoId", + "asset"."originalFileName" AS "asset_originalFileName", + "asset"."sidecarPath" AS "asset_sidecarPath", + "asset"."stackId" AS "asset_stackId", + "exifInfo"."assetId" AS "exifInfo_assetId", + "exifInfo"."description" AS "exifInfo_description", + "exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth", + "exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight", + "exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte", + "exifInfo"."orientation" AS "exifInfo_orientation", + "exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal", + "exifInfo"."modifyDate" AS "exifInfo_modifyDate", + "exifInfo"."timeZone" AS "exifInfo_timeZone", + "exifInfo"."latitude" AS "exifInfo_latitude", + "exifInfo"."longitude" AS "exifInfo_longitude", + "exifInfo"."projectionType" AS "exifInfo_projectionType", + "exifInfo"."city" AS "exifInfo_city", + "exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID", + "exifInfo"."autoStackId" AS "exifInfo_autoStackId", + "exifInfo"."state" AS "exifInfo_state", + "exifInfo"."country" AS "exifInfo_country", + "exifInfo"."make" AS "exifInfo_make", + "exifInfo"."model" AS "exifInfo_model", + "exifInfo"."lensModel" AS "exifInfo_lensModel", + "exifInfo"."fNumber" AS "exifInfo_fNumber", + "exifInfo"."focalLength" AS "exifInfo_focalLength", + "exifInfo"."iso" AS "exifInfo_iso", + "exifInfo"."exposureTime" AS "exifInfo_exposureTime", + "exifInfo"."profileDescription" AS "exifInfo_profileDescription", + "exifInfo"."colorspace" AS "exifInfo_colorspace", + "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", + "exifInfo"."fps" AS "exifInfo_fps", + "stack"."id" AS "stack_id", + "stack"."primaryAssetId" AS "stack_primaryAssetId" +FROM + "assets" "asset" + LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" + LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" +WHERE + "asset"."ownerId" = $1 + AND ("asset"."fileCreatedAt", "asset"."id") < ($2, $3) + AND "asset"."updatedAt" <= $4 + AND "asset"."isVisible" = true +ORDER BY + "asset"."fileCreatedAt" DESC, + "asset"."id" DESC +LIMIT + 10 + +-- AssetRepository.getChangedDeltaSync +SELECT + "AssetEntity"."id" AS "AssetEntity_id", + "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId", + "AssetEntity"."ownerId" AS "AssetEntity_ownerId", + "AssetEntity"."libraryId" AS "AssetEntity_libraryId", + "AssetEntity"."deviceId" AS "AssetEntity_deviceId", + "AssetEntity"."type" AS "AssetEntity_type", + "AssetEntity"."originalPath" AS "AssetEntity_originalPath", + "AssetEntity"."previewPath" AS "AssetEntity_previewPath", + "AssetEntity"."thumbnailPath" AS "AssetEntity_thumbnailPath", + "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash", + "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath", + "AssetEntity"."createdAt" AS "AssetEntity_createdAt", + "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt", + "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt", + "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt", + "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime", + "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt", + "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite", + "AssetEntity"."isArchived" AS "AssetEntity_isArchived", + "AssetEntity"."isExternal" AS "AssetEntity_isExternal", + "AssetEntity"."isReadOnly" AS "AssetEntity_isReadOnly", + "AssetEntity"."isOffline" AS "AssetEntity_isOffline", + "AssetEntity"."checksum" AS "AssetEntity_checksum", + "AssetEntity"."duration" AS "AssetEntity_duration", + "AssetEntity"."isVisible" AS "AssetEntity_isVisible", + "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId", + "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName", + "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath", + "AssetEntity"."stackId" AS "AssetEntity_stackId", + "AssetEntity__AssetEntity_exifInfo"."assetId" AS "AssetEntity__AssetEntity_exifInfo_assetId", + "AssetEntity__AssetEntity_exifInfo"."description" AS "AssetEntity__AssetEntity_exifInfo_description", + "AssetEntity__AssetEntity_exifInfo"."exifImageWidth" AS "AssetEntity__AssetEntity_exifInfo_exifImageWidth", + "AssetEntity__AssetEntity_exifInfo"."exifImageHeight" AS "AssetEntity__AssetEntity_exifInfo_exifImageHeight", + "AssetEntity__AssetEntity_exifInfo"."fileSizeInByte" AS "AssetEntity__AssetEntity_exifInfo_fileSizeInByte", + "AssetEntity__AssetEntity_exifInfo"."orientation" AS "AssetEntity__AssetEntity_exifInfo_orientation", + "AssetEntity__AssetEntity_exifInfo"."dateTimeOriginal" AS "AssetEntity__AssetEntity_exifInfo_dateTimeOriginal", + "AssetEntity__AssetEntity_exifInfo"."modifyDate" AS "AssetEntity__AssetEntity_exifInfo_modifyDate", + "AssetEntity__AssetEntity_exifInfo"."timeZone" AS "AssetEntity__AssetEntity_exifInfo_timeZone", + "AssetEntity__AssetEntity_exifInfo"."latitude" AS "AssetEntity__AssetEntity_exifInfo_latitude", + "AssetEntity__AssetEntity_exifInfo"."longitude" AS "AssetEntity__AssetEntity_exifInfo_longitude", + "AssetEntity__AssetEntity_exifInfo"."projectionType" AS "AssetEntity__AssetEntity_exifInfo_projectionType", + "AssetEntity__AssetEntity_exifInfo"."city" AS "AssetEntity__AssetEntity_exifInfo_city", + "AssetEntity__AssetEntity_exifInfo"."livePhotoCID" AS "AssetEntity__AssetEntity_exifInfo_livePhotoCID", + "AssetEntity__AssetEntity_exifInfo"."autoStackId" AS "AssetEntity__AssetEntity_exifInfo_autoStackId", + "AssetEntity__AssetEntity_exifInfo"."state" AS "AssetEntity__AssetEntity_exifInfo_state", + "AssetEntity__AssetEntity_exifInfo"."country" AS "AssetEntity__AssetEntity_exifInfo_country", + "AssetEntity__AssetEntity_exifInfo"."make" AS "AssetEntity__AssetEntity_exifInfo_make", + "AssetEntity__AssetEntity_exifInfo"."model" AS "AssetEntity__AssetEntity_exifInfo_model", + "AssetEntity__AssetEntity_exifInfo"."lensModel" AS "AssetEntity__AssetEntity_exifInfo_lensModel", + "AssetEntity__AssetEntity_exifInfo"."fNumber" AS "AssetEntity__AssetEntity_exifInfo_fNumber", + "AssetEntity__AssetEntity_exifInfo"."focalLength" AS "AssetEntity__AssetEntity_exifInfo_focalLength", + "AssetEntity__AssetEntity_exifInfo"."iso" AS "AssetEntity__AssetEntity_exifInfo_iso", + "AssetEntity__AssetEntity_exifInfo"."exposureTime" AS "AssetEntity__AssetEntity_exifInfo_exposureTime", + "AssetEntity__AssetEntity_exifInfo"."profileDescription" AS "AssetEntity__AssetEntity_exifInfo_profileDescription", + "AssetEntity__AssetEntity_exifInfo"."colorspace" AS "AssetEntity__AssetEntity_exifInfo_colorspace", + "AssetEntity__AssetEntity_exifInfo"."bitsPerSample" AS "AssetEntity__AssetEntity_exifInfo_bitsPerSample", + "AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps", + "AssetEntity__AssetEntity_stack"."id" AS "AssetEntity__AssetEntity_stack_id", + "AssetEntity__AssetEntity_stack"."primaryAssetId" AS "AssetEntity__AssetEntity_stack_primaryAssetId" +FROM + "assets" "AssetEntity" + LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id" + LEFT JOIN "asset_stack" "AssetEntity__AssetEntity_stack" ON "AssetEntity__AssetEntity_stack"."id" = "AssetEntity"."stackId" +WHERE + ( + ("AssetEntity"."ownerId" IN ($1)) + AND ("AssetEntity"."isVisible" = $2) + AND ("AssetEntity"."updatedAt" > $3) + ) diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 724c1b0c0f..ddc666edd3 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -2,15 +2,18 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import path from 'node:path'; import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators'; -import { AssetOrder } from 'src/entities/album.entity'; +import { AlbumEntity, AssetOrder } from 'src/entities/album.entity'; import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity'; import { AssetEntity, AssetType } from 'src/entities/asset.entity'; import { ExifEntity } from 'src/entities/exif.entity'; +import { PartnerEntity } from 'src/entities/partner.entity'; import { SmartInfoEntity } from 'src/entities/smart-info.entity'; import { AssetBuilderOptions, AssetCreate, + AssetDeltaSyncOptions, AssetExploreFieldOptions, + AssetFullSyncOptions, AssetPathEntity, AssetStats, AssetStatsOptions, @@ -39,6 +42,7 @@ import { FindOptionsWhere, In, IsNull, + MoreThan, Not, Repository, } from 'typeorm'; @@ -61,6 +65,8 @@ export class AssetRepository implements IAssetRepository { @InjectRepository(ExifEntity) private exifRepository: Repository, @InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository, @InjectRepository(SmartInfoEntity) private smartInfoRepository: Repository, + @InjectRepository(PartnerEntity) private partnerRepository: Repository, + @InjectRepository(AlbumEntity) private albumRepository: Repository, ) {} async upsertExif(exif: Partial): Promise { @@ -781,4 +787,55 @@ export class AssetRepository implements IAssetRepository { }) as AssetEntity, ); } + + @GenerateSql({ + params: [ + { + ownerId: DummyValue.UUID, + lastCreationDate: DummyValue.DATE, + lastId: DummyValue.STRING, + updatedUntil: DummyValue.DATE, + limit: 10, + }, + ], + }) + getAllForUserFullSync(options: AssetFullSyncOptions): Promise { + const { ownerId, lastCreationDate, lastId, updatedUntil, limit } = options; + let builder = this.repository + .createQueryBuilder('asset') + .leftJoinAndSelect('asset.exifInfo', 'exifInfo') + .leftJoinAndSelect('asset.stack', 'stack') + .where('asset.ownerId = :ownerId', { ownerId }); + if (lastCreationDate !== undefined && lastId !== undefined) { + builder = builder.andWhere('(asset.fileCreatedAt, asset.id) < (:lastCreationDate, :lastId)', { + lastCreationDate, + lastId, + }); + } + return builder + .andWhere('asset.updatedAt <= :updatedUntil', { updatedUntil }) + .andWhere('asset.isVisible = true') + .orderBy('asset.fileCreatedAt', 'DESC') + .addOrderBy('asset.id', 'DESC') + .limit(limit) + .withDeleted() + .getMany(); + } + + @GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE }] }) + getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise { + return this.repository.find({ + where: { + ownerId: In(options.userIds), + isVisible: true, + updatedAt: MoreThan(options.updatedAfter), + }, + relations: { + exifInfo: true, + stack: true, + }, + take: options.limit, + withDeleted: true, + }); + } } diff --git a/server/src/repositories/audit.repository.ts b/server/src/repositories/audit.repository.ts index 50f5631f3a..3332b06fe1 100644 --- a/server/src/repositories/audit.repository.ts +++ b/server/src/repositories/audit.repository.ts @@ -2,24 +2,25 @@ import { InjectRepository } from '@nestjs/typeorm'; import { AuditEntity } from 'src/entities/audit.entity'; import { AuditSearch, IAuditRepository } from 'src/interfaces/audit.interface'; import { Instrumentation } from 'src/utils/instrumentation'; -import { LessThan, MoreThan, Repository } from 'typeorm'; +import { In, LessThan, MoreThan, Repository } from 'typeorm'; @Instrumentation() export class AuditRepository implements IAuditRepository { constructor(@InjectRepository(AuditEntity) private repository: Repository) {} - getAfter(since: Date, options: AuditSearch): Promise { + getAfter(since: Date, options: AuditSearch): Promise { return this.repository .createQueryBuilder('audit') .where({ createdAt: MoreThan(since), action: options.action, entityType: options.entityType, - ownerId: options.ownerId, + ownerId: In(options.userIds), }) .distinctOn(['audit.entityId', 'audit.entityType']) .orderBy('audit.entityId, audit.entityType, audit.createdAt', 'DESC') - .getMany(); + .select('audit.entityId') + .getRawMany(); } async removeBefore(before: Date): Promise { diff --git a/server/src/services/audit.service.spec.ts b/server/src/services/audit.service.spec.ts index 4af5c1f94d..8d6a3ea2e8 100644 --- a/server/src/services/audit.service.spec.ts +++ b/server/src/services/audit.service.spec.ts @@ -61,13 +61,13 @@ describe(AuditService.name, () => { expect(auditMock.getAfter).toHaveBeenCalledWith(date, { action: DatabaseAction.DELETE, - ownerId: authStub.admin.user.id, + userIds: [authStub.admin.user.id], entityType: EntityType.ASSET, }); }); it('should get any new or updated assets and deleted ids', async () => { - auditMock.getAfter.mockResolvedValue([auditStub.delete]); + auditMock.getAfter.mockResolvedValue([auditStub.delete.entityId]); const date = new Date(); await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({ @@ -77,7 +77,7 @@ describe(AuditService.name, () => { expect(auditMock.getAfter).toHaveBeenCalledWith(date, { action: DatabaseAction.DELETE, - ownerId: authStub.admin.user.id, + userIds: [authStub.admin.user.id], entityType: EntityType.ASSET, }); }); diff --git a/server/src/services/audit.service.ts b/server/src/services/audit.service.ts index d40167429f..b15ee9240a 100644 --- a/server/src/services/audit.service.ts +++ b/server/src/services/audit.service.ts @@ -53,7 +53,7 @@ export class AuditService { await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId); const audits = await this.repository.getAfter(dto.after, { - ownerId: userId, + userIds: [userId], entityType: dto.entityType, action: DatabaseAction.DELETE, }); @@ -62,7 +62,7 @@ export class AuditService { return { needsFullSync: duration > AUDIT_LOG_MAX_DURATION, - ids: audits.map(({ entityId }) => entityId), + ids: audits, }; } diff --git a/server/src/services/index.ts b/server/src/services/index.ts index 3c903c927d..6c40f8420a 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -22,6 +22,7 @@ import { SharedLinkService } from 'src/services/shared-link.service'; import { SmartInfoService } from 'src/services/smart-info.service'; import { StorageTemplateService } from 'src/services/storage-template.service'; import { StorageService } from 'src/services/storage.service'; +import { SyncService } from 'src/services/sync.service'; import { SystemConfigService } from 'src/services/system-config.service'; import { TagService } from 'src/services/tag.service'; import { TimelineService } from 'src/services/timeline.service'; @@ -53,6 +54,7 @@ export const services = [ SmartInfoService, StorageService, StorageTemplateService, + SyncService, SystemConfigService, TagService, TimelineService, diff --git a/server/src/services/sync.service.spec.ts b/server/src/services/sync.service.spec.ts new file mode 100644 index 0000000000..35ef4a6302 --- /dev/null +++ b/server/src/services/sync.service.spec.ts @@ -0,0 +1,101 @@ +import { mapAsset } from 'src/dtos/asset-response.dto'; +import { AssetEntity } from 'src/entities/asset.entity'; +import { IAccessRepository } from 'src/interfaces/access.interface'; +import { IAssetRepository } from 'src/interfaces/asset.interface'; +import { IAuditRepository } from 'src/interfaces/audit.interface'; +import { IPartnerRepository } from 'src/interfaces/partner.interface'; +import { SyncService } from 'src/services/sync.service'; +import { assetStub } from 'test/fixtures/asset.stub'; +import { authStub } from 'test/fixtures/auth.stub'; +import { partnerStub } from 'test/fixtures/partner.stub'; +import { newAccessRepositoryMock } from 'test/repositories/access.repository.mock'; +import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock'; +import { newAuditRepositoryMock } from 'test/repositories/audit.repository.mock'; +import { newPartnerRepositoryMock } from 'test/repositories/partner.repository.mock'; + +const untilDate = new Date(2024); +const mapAssetOpts = { auth: authStub.user1, stripMetadata: false, withStack: true }; + +describe(SyncService.name, () => { + let sut: SyncService; + let accessMock: jest.Mocked; + let assetMock: jest.Mocked; + let partnerMock: jest.Mocked; + let auditMock: jest.Mocked; + + beforeEach(() => { + partnerMock = newPartnerRepositoryMock(); + assetMock = newAssetRepositoryMock(); + accessMock = newAccessRepositoryMock(); + auditMock = newAuditRepositoryMock(); + sut = new SyncService(accessMock, assetMock, partnerMock, auditMock); + }); + + it('should exist', () => { + expect(sut).toBeDefined(); + }); + + describe('getAllAssetsForUserFullSync', () => { + it('should return a list of all assets owned by the user', async () => { + assetMock.getAllForUserFullSync.mockResolvedValue([assetStub.external, assetStub.hasEncodedVideo]); + await expect( + sut.getAllAssetsForUserFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate }), + ).resolves.toEqual([ + mapAsset(assetStub.external, mapAssetOpts), + mapAsset(assetStub.hasEncodedVideo, mapAssetOpts), + ]); + expect(assetMock.getAllForUserFullSync).toHaveBeenCalledWith({ + ownerId: authStub.user1.user.id, + updatedUntil: untilDate, + limit: 2, + }); + }); + }); + + describe('getChangesForDeltaSync', () => { + it('should return a response requiring a full sync when partners are out of sync', async () => { + partnerMock.getAll.mockResolvedValue([partnerStub.adminToUser1]); + await expect( + sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), + ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] }); + expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0); + expect(auditMock.getAfter).toHaveBeenCalledTimes(0); + }); + + it('should return a response requiring a full sync when last sync was too long ago', async () => { + partnerMock.getAll.mockResolvedValue([]); + await expect( + sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }), + ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] }); + expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0); + expect(auditMock.getAfter).toHaveBeenCalledTimes(0); + }); + + it('should return a response requiring a full sync when there are too many changes', async () => { + partnerMock.getAll.mockResolvedValue([]); + assetMock.getChangedDeltaSync.mockResolvedValue( + Array.from({ length: 10_000 }).fill(assetStub.image), + ); + await expect( + sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), + ).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] }); + expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1); + expect(auditMock.getAfter).toHaveBeenCalledTimes(0); + }); + + it('should return a response with changes and deletions', async () => { + partnerMock.getAll.mockResolvedValue([]); + assetMock.getChangedDeltaSync.mockResolvedValue([assetStub.image1]); + auditMock.getAfter.mockResolvedValue([assetStub.external.id]); + await expect( + sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }), + ).resolves.toEqual({ + needsFullSync: false, + upserted: [mapAsset(assetStub.image1, mapAssetOpts)], + deleted: [assetStub.external.id], + }); + expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1); + expect(auditMock.getAfter).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts new file mode 100644 index 0000000000..be11d36fa0 --- /dev/null +++ b/server/src/services/sync.service.ts @@ -0,0 +1,77 @@ +import { Inject } from '@nestjs/common'; +import _ from 'lodash'; +import { DateTime } from 'luxon'; +import { AUDIT_LOG_MAX_DURATION } from 'src/constants'; +import { AccessCore, Permission } from 'src/cores/access.core'; +import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto'; +import { DatabaseAction, EntityType } from 'src/entities/audit.entity'; +import { IAccessRepository } from 'src/interfaces/access.interface'; +import { IAssetRepository } from 'src/interfaces/asset.interface'; +import { IAuditRepository } from 'src/interfaces/audit.interface'; +import { IPartnerRepository } from 'src/interfaces/partner.interface'; + +export class SyncService { + private access: AccessCore; + + constructor( + @Inject(IAccessRepository) accessRepository: IAccessRepository, + @Inject(IAssetRepository) private assetRepository: IAssetRepository, + @Inject(IPartnerRepository) private partnerRepository: IPartnerRepository, + @Inject(IAuditRepository) private auditRepository: IAuditRepository, + ) { + this.access = AccessCore.create(accessRepository); + } + + async getAllAssetsForUserFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise { + const userId = dto.userId || auth.user.id; + await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId); + const assets = await this.assetRepository.getAllForUserFullSync({ + ownerId: userId, + lastCreationDate: dto.lastCreationDate, + updatedUntil: dto.updatedUntil, + lastId: dto.lastId, + limit: dto.limit, + }); + const options = { auth, stripMetadata: false, withStack: true }; + return assets.map((a) => mapAsset(a, options)); + } + + async getChangesForDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise { + await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds); + const partner = await this.partnerRepository.getAll(auth.user.id); + const userIds = [auth.user.id, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)]; + userIds.sort(); + dto.userIds.sort(); + const duration = DateTime.now().diff(DateTime.fromJSDate(dto.updatedAfter)); + + if (!_.isEqual(userIds, dto.userIds) || duration > AUDIT_LOG_MAX_DURATION) { + // app does not have the correct partners synced + // or app has not synced in the last 100 days + return { needsFullSync: true, deleted: [], upserted: [] }; + } + + const limit = 10_000; + const upserted = await this.assetRepository.getChangedDeltaSync({ limit, updatedAfter: dto.updatedAfter, userIds }); + + if (upserted.length === limit) { + // too many changes -> do a full sync (paginated) instead + return { needsFullSync: true, deleted: [], upserted: [] }; + } + + const deleted = await this.auditRepository.getAfter(dto.updatedAfter, { + userIds: userIds, + entityType: EntityType.ASSET, + action: DatabaseAction.DELETE, + }); + + const options = { auth, stripMetadata: false, withStack: true }; + const result = { + needsFullSync: false, + upserted: upserted.map((a) => mapAsset(a, options)), + deleted, + }; + return result; + } +} diff --git a/server/test/repositories/asset.repository.mock.ts b/server/test/repositories/asset.repository.mock.ts index 4133880d6f..0bd6d04e07 100644 --- a/server/test/repositories/asset.repository.mock.ts +++ b/server/test/repositories/asset.repository.mock.ts @@ -35,5 +35,7 @@ export const newAssetRepositoryMock = (): jest.Mocked => { getAssetIdByCity: jest.fn(), getAssetIdByTag: jest.fn(), searchMetadata: jest.fn(), + getAllForUserFullSync: jest.fn(), + getChangedDeltaSync: jest.fn(), }; };