mirror of
https://github.com/immich-app/immich.git
synced 2024-12-29 15:11:58 +00:00
feat(server): efficient full app sync (#8755)
* feat(server): efficient full app sync * add SQL, fix test compile issues * fix linter warning * new sync controller+service, add tests * enable new sync controller+service * Update server/src/services/sync.service.ts Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> --------- Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
This commit is contained in:
Fynn Petersen-Frey
GitHub
parent
58e516c766
commit
103cb60a57
26 changed files with 685 additions and 13 deletions
6
mobile/openapi/.openapi-generator/FILES
generated
6
mobile/openapi/.openapi-generator/FILES
generated
|
|
@ -24,6 +24,7 @@ doc/AssetBulkUploadCheckDto.md
|
|||
doc/AssetBulkUploadCheckItem.md
|
||||
doc/AssetBulkUploadCheckResponseDto.md
|
||||
doc/AssetBulkUploadCheckResult.md
|
||||
doc/AssetDeltaSyncResponseDto.md
|
||||
doc/AssetFaceResponseDto.md
|
||||
doc/AssetFaceUpdateDto.md
|
||||
doc/AssetFaceUpdateItem.md
|
||||
|
|
@ -149,6 +150,7 @@ doc/SharedLinkType.md
|
|||
doc/SignUpDto.md
|
||||
doc/SmartInfoResponseDto.md
|
||||
doc/SmartSearchDto.md
|
||||
doc/SyncApi.md
|
||||
doc/SystemConfigApi.md
|
||||
doc/SystemConfigDto.md
|
||||
doc/SystemConfigFFmpegDto.md
|
||||
|
|
@ -218,6 +220,7 @@ lib/api/person_api.dart
|
|||
lib/api/search_api.dart
|
||||
lib/api/server_info_api.dart
|
||||
lib/api/shared_link_api.dart
|
||||
lib/api/sync_api.dart
|
||||
lib/api/system_config_api.dart
|
||||
lib/api/tag_api.dart
|
||||
lib/api/timeline_api.dart
|
||||
|
|
@ -248,6 +251,7 @@ lib/model/asset_bulk_upload_check_dto.dart
|
|||
lib/model/asset_bulk_upload_check_item.dart
|
||||
lib/model/asset_bulk_upload_check_response_dto.dart
|
||||
lib/model/asset_bulk_upload_check_result.dart
|
||||
lib/model/asset_delta_sync_response_dto.dart
|
||||
lib/model/asset_face_response_dto.dart
|
||||
lib/model/asset_face_update_dto.dart
|
||||
lib/model/asset_face_update_item.dart
|
||||
|
|
@ -427,6 +431,7 @@ test/asset_bulk_upload_check_dto_test.dart
|
|||
test/asset_bulk_upload_check_item_test.dart
|
||||
test/asset_bulk_upload_check_response_dto_test.dart
|
||||
test/asset_bulk_upload_check_result_test.dart
|
||||
test/asset_delta_sync_response_dto_test.dart
|
||||
test/asset_face_response_dto_test.dart
|
||||
test/asset_face_update_dto_test.dart
|
||||
test/asset_face_update_item_test.dart
|
||||
|
|
@ -552,6 +557,7 @@ test/shared_link_type_test.dart
|
|||
test/sign_up_dto_test.dart
|
||||
test/smart_info_response_dto_test.dart
|
||||
test/smart_search_dto_test.dart
|
||||
test/sync_api_test.dart
|
||||
test/system_config_api_test.dart
|
||||
test/system_config_dto_test.dart
|
||||
test/system_config_f_fmpeg_dto_test.dart
|
||||
|
|
|
|||
BIN
mobile/openapi/README.md
generated
BIN
mobile/openapi/README.md
generated
Binary file not shown.
BIN
mobile/openapi/doc/AssetDeltaSyncResponseDto.md
generated
Normal file
BIN
mobile/openapi/doc/AssetDeltaSyncResponseDto.md
generated
Normal file
Binary file not shown.
BIN
mobile/openapi/doc/SyncApi.md
generated
Normal file
BIN
mobile/openapi/doc/SyncApi.md
generated
Normal file
Binary file not shown.
BIN
mobile/openapi/lib/api.dart
generated
BIN
mobile/openapi/lib/api.dart
generated
Binary file not shown.
BIN
mobile/openapi/lib/api/sync_api.dart
generated
Normal file
BIN
mobile/openapi/lib/api/sync_api.dart
generated
Normal file
Binary file not shown.
BIN
mobile/openapi/lib/api_client.dart
generated
BIN
mobile/openapi/lib/api_client.dart
generated
Binary file not shown.
BIN
mobile/openapi/lib/model/asset_delta_sync_response_dto.dart
generated
Normal file
BIN
mobile/openapi/lib/model/asset_delta_sync_response_dto.dart
generated
Normal file
Binary file not shown.
BIN
mobile/openapi/test/asset_delta_sync_response_dto_test.dart
generated
Normal file
BIN
mobile/openapi/test/asset_delta_sync_response_dto_test.dart
generated
Normal file
Binary file not shown.
BIN
mobile/openapi/test/sync_api_test.dart
generated
Normal file
BIN
mobile/openapi/test/sync_api_test.dart
generated
Normal file
Binary file not shown.
|
|
@ -5566,6 +5566,140 @@
|
|||
]
|
||||
}
|
||||
},
|
||||
"/sync/delta-sync": {
|
||||
"get": {
|
||||
"operationId": "getDeltaSync",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "updatedAfter",
|
||||
"required": true,
|
||||
"in": "query",
|
||||
"schema": {
|
||||
"format": "date-time",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "userIds",
|
||||
"required": true,
|
||||
"in": "query",
|
||||
"schema": {
|
||||
"format": "uuid",
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/AssetDeltaSyncResponseDto"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": ""
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"bearer": []
|
||||
},
|
||||
{
|
||||
"cookie": []
|
||||
},
|
||||
{
|
||||
"api_key": []
|
||||
}
|
||||
],
|
||||
"tags": [
|
||||
"Sync"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/sync/full-sync": {
|
||||
"get": {
|
||||
"operationId": "getAllForUserFullSync",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "lastCreationDate",
|
||||
"required": false,
|
||||
"in": "query",
|
||||
"schema": {
|
||||
"format": "date-time",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "lastId",
|
||||
"required": false,
|
||||
"in": "query",
|
||||
"schema": {
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "limit",
|
||||
"required": true,
|
||||
"in": "query",
|
||||
"schema": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "updatedUntil",
|
||||
"required": true,
|
||||
"in": "query",
|
||||
"schema": {
|
||||
"format": "date-time",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "userId",
|
||||
"required": false,
|
||||
"in": "query",
|
||||
"schema": {
|
||||
"format": "uuid",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/AssetResponseDto"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": ""
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"bearer": []
|
||||
},
|
||||
{
|
||||
"cookie": []
|
||||
},
|
||||
{
|
||||
"api_key": []
|
||||
}
|
||||
],
|
||||
"tags": [
|
||||
"Sync"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/system-config": {
|
||||
"get": {
|
||||
"operationId": "getConfig",
|
||||
|
|
@ -7335,6 +7469,31 @@
|
|||
],
|
||||
"type": "object"
|
||||
},
|
||||
"AssetDeltaSyncResponseDto": {
|
||||
"properties": {
|
||||
"deleted": {
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": "array"
|
||||
},
|
||||
"needsFullSync": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"upserted": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/AssetResponseDto"
|
||||
},
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"deleted",
|
||||
"needsFullSync",
|
||||
"upserted"
|
||||
],
|
||||
"type": "object"
|
||||
},
|
||||
"AssetFaceResponseDto": {
|
||||
"properties": {
|
||||
"boundingBoxX1": {
|
||||
|
|
|
|||
|
|
@ -835,6 +835,11 @@ export type AssetIdsResponseDto = {
|
|||
error?: Error2;
|
||||
success: boolean;
|
||||
};
|
||||
export type AssetDeltaSyncResponseDto = {
|
||||
deleted: string[];
|
||||
needsFullSync: boolean;
|
||||
upserted: AssetResponseDto[];
|
||||
};
|
||||
export type SystemConfigFFmpegDto = {
|
||||
accel: TranscodeHWAccel;
|
||||
acceptedAudioCodecs: AudioCodec[];
|
||||
|
|
@ -2507,6 +2512,40 @@ export function addSharedLinkAssets({ id, key, assetIdsDto }: {
|
|||
body: assetIdsDto
|
||||
})));
|
||||
}
|
||||
export function getDeltaSync({ updatedAfter, userIds }: {
|
||||
updatedAfter: string;
|
||||
userIds: string[];
|
||||
}, opts?: Oazapfts.RequestOpts) {
|
||||
return oazapfts.ok(oazapfts.fetchJson<{
|
||||
status: 200;
|
||||
data: AssetDeltaSyncResponseDto;
|
||||
}>(`/sync/delta-sync${QS.query(QS.explode({
|
||||
updatedAfter,
|
||||
userIds
|
||||
}))}`, {
|
||||
...opts
|
||||
}));
|
||||
}
|
||||
export function getAllForUserFullSync({ lastCreationDate, lastId, limit, updatedUntil, userId }: {
|
||||
lastCreationDate?: string;
|
||||
lastId?: string;
|
||||
limit: number;
|
||||
updatedUntil: string;
|
||||
userId?: string;
|
||||
}, opts?: Oazapfts.RequestOpts) {
|
||||
return oazapfts.ok(oazapfts.fetchJson<{
|
||||
status: 200;
|
||||
data: AssetResponseDto[];
|
||||
}>(`/sync/full-sync${QS.query(QS.explode({
|
||||
lastCreationDate,
|
||||
lastId,
|
||||
limit,
|
||||
updatedUntil,
|
||||
userId
|
||||
}))}`, {
|
||||
...opts
|
||||
}));
|
||||
}
|
||||
export function getConfig(opts?: Oazapfts.RequestOpts) {
|
||||
return oazapfts.ok(oazapfts.fetchJson<{
|
||||
status: 200;
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ import { PersonController } from 'src/controllers/person.controller';
|
|||
import { SearchController } from 'src/controllers/search.controller';
|
||||
import { ServerInfoController } from 'src/controllers/server-info.controller';
|
||||
import { SharedLinkController } from 'src/controllers/shared-link.controller';
|
||||
import { SyncController } from 'src/controllers/sync.controller';
|
||||
import { SystemConfigController } from 'src/controllers/system-config.controller';
|
||||
import { TagController } from 'src/controllers/tag.controller';
|
||||
import { TimelineController } from 'src/controllers/timeline.controller';
|
||||
|
|
@ -43,6 +44,7 @@ export const controllers = [
|
|||
SearchController,
|
||||
ServerInfoController,
|
||||
SharedLinkController,
|
||||
SyncController,
|
||||
SystemConfigController,
|
||||
TagController,
|
||||
TimelineController,
|
||||
|
|
|
|||
24
server/src/controllers/sync.controller.ts
Normal file
24
server/src/controllers/sync.controller.ts
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
import { Controller, Get, Query } from '@nestjs/common';
|
||||
import { ApiTags } from '@nestjs/swagger';
|
||||
import { AssetResponseDto } from 'src/dtos/asset-response.dto';
|
||||
import { AuthDto } from 'src/dtos/auth.dto';
|
||||
import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto';
|
||||
import { Auth, Authenticated } from 'src/middleware/auth.guard';
|
||||
import { SyncService } from 'src/services/sync.service';
|
||||
|
||||
@ApiTags('Sync')
|
||||
@Controller('sync')
|
||||
@Authenticated()
|
||||
export class SyncController {
|
||||
constructor(private service: SyncService) {}
|
||||
|
||||
@Get('full-sync')
|
||||
getAllForUserFullSync(@Auth() auth: AuthDto, @Query() dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
|
||||
return this.service.getAllAssetsForUserFullSync(auth, dto);
|
||||
}
|
||||
|
||||
@Get('delta-sync')
|
||||
getDeltaSync(@Auth() auth: AuthDto, @Query() dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
|
||||
return this.service.getChangesForDeltaSync(auth, dto);
|
||||
}
|
||||
}
|
||||
38
server/src/dtos/sync.dto.ts
Normal file
38
server/src/dtos/sync.dto.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import { ApiProperty } from '@nestjs/swagger';
|
||||
import { Type } from 'class-transformer';
|
||||
import { IsInt, IsPositive } from 'class-validator';
|
||||
import { AssetResponseDto } from 'src/dtos/asset-response.dto';
|
||||
import { ValidateDate, ValidateUUID } from 'src/validation';
|
||||
|
||||
export class AssetFullSyncDto {
|
||||
@ValidateUUID({ optional: true })
|
||||
lastId?: string;
|
||||
|
||||
@ValidateDate({ optional: true })
|
||||
lastCreationDate?: Date;
|
||||
|
||||
@ValidateDate()
|
||||
updatedUntil!: Date;
|
||||
|
||||
@IsInt()
|
||||
@IsPositive()
|
||||
@Type(() => Number)
|
||||
@ApiProperty({ type: 'integer' })
|
||||
limit!: number;
|
||||
|
||||
@ValidateUUID({ optional: true })
|
||||
userId?: string;
|
||||
}
|
||||
|
||||
export class AssetDeltaSyncDto {
|
||||
@ValidateDate()
|
||||
updatedAfter!: Date;
|
||||
@ValidateUUID({ each: true })
|
||||
userIds!: string[];
|
||||
}
|
||||
|
||||
export class AssetDeltaSyncResponseDto {
|
||||
needsFullSync!: boolean;
|
||||
upserted!: AssetResponseDto[];
|
||||
deleted!: string[];
|
||||
}
|
||||
|
|
@ -133,6 +133,20 @@ export interface MetadataSearchOptions {
|
|||
numResults: number;
|
||||
}
|
||||
|
||||
export interface AssetFullSyncOptions {
|
||||
ownerId: string;
|
||||
lastCreationDate?: Date;
|
||||
lastId?: string;
|
||||
updatedUntil: Date;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
export interface AssetDeltaSyncOptions {
|
||||
userIds: string[];
|
||||
updatedAfter: Date;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>;
|
||||
|
||||
export const IAssetRepository = 'IAssetRepository';
|
||||
|
|
@ -175,4 +189,6 @@ export interface IAssetRepository {
|
|||
getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
|
||||
getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
|
||||
searchMetadata(query: string, userIds: string[], options: MetadataSearchOptions): Promise<AssetEntity[]>;
|
||||
getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]>;
|
||||
getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
import { AuditEntity, DatabaseAction, EntityType } from 'src/entities/audit.entity';
|
||||
import { DatabaseAction, EntityType } from 'src/entities/audit.entity';
|
||||
|
||||
export const IAuditRepository = 'IAuditRepository';
|
||||
|
||||
export interface AuditSearch {
|
||||
action?: DatabaseAction;
|
||||
entityType?: EntityType;
|
||||
ownerId?: string;
|
||||
userIds: string[];
|
||||
}
|
||||
|
||||
export interface IAuditRepository {
|
||||
getAfter(since: Date, options: AuditSearch): Promise<AuditEntity[]>;
|
||||
getAfter(since: Date, options: AuditSearch): Promise<string[]>;
|
||||
removeBefore(before: Date): Promise<void>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -768,3 +768,151 @@ ORDER BY
|
|||
"asset"."fileCreatedAt" DESC
|
||||
LIMIT
|
||||
250
|
||||
|
||||
-- AssetRepository.getAllForUserFullSync
|
||||
SELECT
|
||||
"asset"."id" AS "asset_id",
|
||||
"asset"."deviceAssetId" AS "asset_deviceAssetId",
|
||||
"asset"."ownerId" AS "asset_ownerId",
|
||||
"asset"."libraryId" AS "asset_libraryId",
|
||||
"asset"."deviceId" AS "asset_deviceId",
|
||||
"asset"."type" AS "asset_type",
|
||||
"asset"."originalPath" AS "asset_originalPath",
|
||||
"asset"."previewPath" AS "asset_previewPath",
|
||||
"asset"."thumbnailPath" AS "asset_thumbnailPath",
|
||||
"asset"."thumbhash" AS "asset_thumbhash",
|
||||
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
|
||||
"asset"."createdAt" AS "asset_createdAt",
|
||||
"asset"."updatedAt" AS "asset_updatedAt",
|
||||
"asset"."deletedAt" AS "asset_deletedAt",
|
||||
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
|
||||
"asset"."localDateTime" AS "asset_localDateTime",
|
||||
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
|
||||
"asset"."isFavorite" AS "asset_isFavorite",
|
||||
"asset"."isArchived" AS "asset_isArchived",
|
||||
"asset"."isExternal" AS "asset_isExternal",
|
||||
"asset"."isReadOnly" AS "asset_isReadOnly",
|
||||
"asset"."isOffline" AS "asset_isOffline",
|
||||
"asset"."checksum" AS "asset_checksum",
|
||||
"asset"."duration" AS "asset_duration",
|
||||
"asset"."isVisible" AS "asset_isVisible",
|
||||
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
|
||||
"asset"."originalFileName" AS "asset_originalFileName",
|
||||
"asset"."sidecarPath" AS "asset_sidecarPath",
|
||||
"asset"."stackId" AS "asset_stackId",
|
||||
"exifInfo"."assetId" AS "exifInfo_assetId",
|
||||
"exifInfo"."description" AS "exifInfo_description",
|
||||
"exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
|
||||
"exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight",
|
||||
"exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte",
|
||||
"exifInfo"."orientation" AS "exifInfo_orientation",
|
||||
"exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal",
|
||||
"exifInfo"."modifyDate" AS "exifInfo_modifyDate",
|
||||
"exifInfo"."timeZone" AS "exifInfo_timeZone",
|
||||
"exifInfo"."latitude" AS "exifInfo_latitude",
|
||||
"exifInfo"."longitude" AS "exifInfo_longitude",
|
||||
"exifInfo"."projectionType" AS "exifInfo_projectionType",
|
||||
"exifInfo"."city" AS "exifInfo_city",
|
||||
"exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID",
|
||||
"exifInfo"."autoStackId" AS "exifInfo_autoStackId",
|
||||
"exifInfo"."state" AS "exifInfo_state",
|
||||
"exifInfo"."country" AS "exifInfo_country",
|
||||
"exifInfo"."make" AS "exifInfo_make",
|
||||
"exifInfo"."model" AS "exifInfo_model",
|
||||
"exifInfo"."lensModel" AS "exifInfo_lensModel",
|
||||
"exifInfo"."fNumber" AS "exifInfo_fNumber",
|
||||
"exifInfo"."focalLength" AS "exifInfo_focalLength",
|
||||
"exifInfo"."iso" AS "exifInfo_iso",
|
||||
"exifInfo"."exposureTime" AS "exifInfo_exposureTime",
|
||||
"exifInfo"."profileDescription" AS "exifInfo_profileDescription",
|
||||
"exifInfo"."colorspace" AS "exifInfo_colorspace",
|
||||
"exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
|
||||
"exifInfo"."fps" AS "exifInfo_fps",
|
||||
"stack"."id" AS "stack_id",
|
||||
"stack"."primaryAssetId" AS "stack_primaryAssetId"
|
||||
FROM
|
||||
"assets" "asset"
|
||||
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
|
||||
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
|
||||
WHERE
|
||||
"asset"."ownerId" = $1
|
||||
AND ("asset"."fileCreatedAt", "asset"."id") < ($2, $3)
|
||||
AND "asset"."updatedAt" <= $4
|
||||
AND "asset"."isVisible" = true
|
||||
ORDER BY
|
||||
"asset"."fileCreatedAt" DESC,
|
||||
"asset"."id" DESC
|
||||
LIMIT
|
||||
10
|
||||
|
||||
-- AssetRepository.getChangedDeltaSync
|
||||
SELECT
|
||||
"AssetEntity"."id" AS "AssetEntity_id",
|
||||
"AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId",
|
||||
"AssetEntity"."ownerId" AS "AssetEntity_ownerId",
|
||||
"AssetEntity"."libraryId" AS "AssetEntity_libraryId",
|
||||
"AssetEntity"."deviceId" AS "AssetEntity_deviceId",
|
||||
"AssetEntity"."type" AS "AssetEntity_type",
|
||||
"AssetEntity"."originalPath" AS "AssetEntity_originalPath",
|
||||
"AssetEntity"."previewPath" AS "AssetEntity_previewPath",
|
||||
"AssetEntity"."thumbnailPath" AS "AssetEntity_thumbnailPath",
|
||||
"AssetEntity"."thumbhash" AS "AssetEntity_thumbhash",
|
||||
"AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath",
|
||||
"AssetEntity"."createdAt" AS "AssetEntity_createdAt",
|
||||
"AssetEntity"."updatedAt" AS "AssetEntity_updatedAt",
|
||||
"AssetEntity"."deletedAt" AS "AssetEntity_deletedAt",
|
||||
"AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt",
|
||||
"AssetEntity"."localDateTime" AS "AssetEntity_localDateTime",
|
||||
"AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt",
|
||||
"AssetEntity"."isFavorite" AS "AssetEntity_isFavorite",
|
||||
"AssetEntity"."isArchived" AS "AssetEntity_isArchived",
|
||||
"AssetEntity"."isExternal" AS "AssetEntity_isExternal",
|
||||
"AssetEntity"."isReadOnly" AS "AssetEntity_isReadOnly",
|
||||
"AssetEntity"."isOffline" AS "AssetEntity_isOffline",
|
||||
"AssetEntity"."checksum" AS "AssetEntity_checksum",
|
||||
"AssetEntity"."duration" AS "AssetEntity_duration",
|
||||
"AssetEntity"."isVisible" AS "AssetEntity_isVisible",
|
||||
"AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
|
||||
"AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
|
||||
"AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
|
||||
"AssetEntity"."stackId" AS "AssetEntity_stackId",
|
||||
"AssetEntity__AssetEntity_exifInfo"."assetId" AS "AssetEntity__AssetEntity_exifInfo_assetId",
|
||||
"AssetEntity__AssetEntity_exifInfo"."description" AS "AssetEntity__AssetEntity_exifInfo_description",
|
||||
"AssetEntity__AssetEntity_exifInfo"."exifImageWidth" AS "AssetEntity__AssetEntity_exifInfo_exifImageWidth",
|
||||
"AssetEntity__AssetEntity_exifInfo"."exifImageHeight" AS "AssetEntity__AssetEntity_exifInfo_exifImageHeight",
|
||||
"AssetEntity__AssetEntity_exifInfo"."fileSizeInByte" AS "AssetEntity__AssetEntity_exifInfo_fileSizeInByte",
|
||||
"AssetEntity__AssetEntity_exifInfo"."orientation" AS "AssetEntity__AssetEntity_exifInfo_orientation",
|
||||
"AssetEntity__AssetEntity_exifInfo"."dateTimeOriginal" AS "AssetEntity__AssetEntity_exifInfo_dateTimeOriginal",
|
||||
"AssetEntity__AssetEntity_exifInfo"."modifyDate" AS "AssetEntity__AssetEntity_exifInfo_modifyDate",
|
||||
"AssetEntity__AssetEntity_exifInfo"."timeZone" AS "AssetEntity__AssetEntity_exifInfo_timeZone",
|
||||
"AssetEntity__AssetEntity_exifInfo"."latitude" AS "AssetEntity__AssetEntity_exifInfo_latitude",
|
||||
"AssetEntity__AssetEntity_exifInfo"."longitude" AS "AssetEntity__AssetEntity_exifInfo_longitude",
|
||||
"AssetEntity__AssetEntity_exifInfo"."projectionType" AS "AssetEntity__AssetEntity_exifInfo_projectionType",
|
||||
"AssetEntity__AssetEntity_exifInfo"."city" AS "AssetEntity__AssetEntity_exifInfo_city",
|
||||
"AssetEntity__AssetEntity_exifInfo"."livePhotoCID" AS "AssetEntity__AssetEntity_exifInfo_livePhotoCID",
|
||||
"AssetEntity__AssetEntity_exifInfo"."autoStackId" AS "AssetEntity__AssetEntity_exifInfo_autoStackId",
|
||||
"AssetEntity__AssetEntity_exifInfo"."state" AS "AssetEntity__AssetEntity_exifInfo_state",
|
||||
"AssetEntity__AssetEntity_exifInfo"."country" AS "AssetEntity__AssetEntity_exifInfo_country",
|
||||
"AssetEntity__AssetEntity_exifInfo"."make" AS "AssetEntity__AssetEntity_exifInfo_make",
|
||||
"AssetEntity__AssetEntity_exifInfo"."model" AS "AssetEntity__AssetEntity_exifInfo_model",
|
||||
"AssetEntity__AssetEntity_exifInfo"."lensModel" AS "AssetEntity__AssetEntity_exifInfo_lensModel",
|
||||
"AssetEntity__AssetEntity_exifInfo"."fNumber" AS "AssetEntity__AssetEntity_exifInfo_fNumber",
|
||||
"AssetEntity__AssetEntity_exifInfo"."focalLength" AS "AssetEntity__AssetEntity_exifInfo_focalLength",
|
||||
"AssetEntity__AssetEntity_exifInfo"."iso" AS "AssetEntity__AssetEntity_exifInfo_iso",
|
||||
"AssetEntity__AssetEntity_exifInfo"."exposureTime" AS "AssetEntity__AssetEntity_exifInfo_exposureTime",
|
||||
"AssetEntity__AssetEntity_exifInfo"."profileDescription" AS "AssetEntity__AssetEntity_exifInfo_profileDescription",
|
||||
"AssetEntity__AssetEntity_exifInfo"."colorspace" AS "AssetEntity__AssetEntity_exifInfo_colorspace",
|
||||
"AssetEntity__AssetEntity_exifInfo"."bitsPerSample" AS "AssetEntity__AssetEntity_exifInfo_bitsPerSample",
|
||||
"AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps",
|
||||
"AssetEntity__AssetEntity_stack"."id" AS "AssetEntity__AssetEntity_stack_id",
|
||||
"AssetEntity__AssetEntity_stack"."primaryAssetId" AS "AssetEntity__AssetEntity_stack_primaryAssetId"
|
||||
FROM
|
||||
"assets" "AssetEntity"
|
||||
LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id"
|
||||
LEFT JOIN "asset_stack" "AssetEntity__AssetEntity_stack" ON "AssetEntity__AssetEntity_stack"."id" = "AssetEntity"."stackId"
|
||||
WHERE
|
||||
(
|
||||
("AssetEntity"."ownerId" IN ($1))
|
||||
AND ("AssetEntity"."isVisible" = $2)
|
||||
AND ("AssetEntity"."updatedAt" > $3)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,15 +2,18 @@ import { Injectable } from '@nestjs/common';
|
|||
import { InjectRepository } from '@nestjs/typeorm';
|
||||
import path from 'node:path';
|
||||
import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
|
||||
import { AssetOrder } from 'src/entities/album.entity';
|
||||
import { AlbumEntity, AssetOrder } from 'src/entities/album.entity';
|
||||
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
|
||||
import { AssetEntity, AssetType } from 'src/entities/asset.entity';
|
||||
import { ExifEntity } from 'src/entities/exif.entity';
|
||||
import { PartnerEntity } from 'src/entities/partner.entity';
|
||||
import { SmartInfoEntity } from 'src/entities/smart-info.entity';
|
||||
import {
|
||||
AssetBuilderOptions,
|
||||
AssetCreate,
|
||||
AssetDeltaSyncOptions,
|
||||
AssetExploreFieldOptions,
|
||||
AssetFullSyncOptions,
|
||||
AssetPathEntity,
|
||||
AssetStats,
|
||||
AssetStatsOptions,
|
||||
|
|
@ -39,6 +42,7 @@ import {
|
|||
FindOptionsWhere,
|
||||
In,
|
||||
IsNull,
|
||||
MoreThan,
|
||||
Not,
|
||||
Repository,
|
||||
} from 'typeorm';
|
||||
|
|
@ -61,6 +65,8 @@ export class AssetRepository implements IAssetRepository {
|
|||
@InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
|
||||
@InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository<AssetJobStatusEntity>,
|
||||
@InjectRepository(SmartInfoEntity) private smartInfoRepository: Repository<SmartInfoEntity>,
|
||||
@InjectRepository(PartnerEntity) private partnerRepository: Repository<PartnerEntity>,
|
||||
@InjectRepository(AlbumEntity) private albumRepository: Repository<AlbumEntity>,
|
||||
) {}
|
||||
|
||||
async upsertExif(exif: Partial<ExifEntity>): Promise<void> {
|
||||
|
|
@ -781,4 +787,55 @@ export class AssetRepository implements IAssetRepository {
|
|||
}) as AssetEntity,
|
||||
);
|
||||
}
|
||||
|
||||
@GenerateSql({
|
||||
params: [
|
||||
{
|
||||
ownerId: DummyValue.UUID,
|
||||
lastCreationDate: DummyValue.DATE,
|
||||
lastId: DummyValue.STRING,
|
||||
updatedUntil: DummyValue.DATE,
|
||||
limit: 10,
|
||||
},
|
||||
],
|
||||
})
|
||||
getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]> {
|
||||
const { ownerId, lastCreationDate, lastId, updatedUntil, limit } = options;
|
||||
let builder = this.repository
|
||||
.createQueryBuilder('asset')
|
||||
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
|
||||
.leftJoinAndSelect('asset.stack', 'stack')
|
||||
.where('asset.ownerId = :ownerId', { ownerId });
|
||||
if (lastCreationDate !== undefined && lastId !== undefined) {
|
||||
builder = builder.andWhere('(asset.fileCreatedAt, asset.id) < (:lastCreationDate, :lastId)', {
|
||||
lastCreationDate,
|
||||
lastId,
|
||||
});
|
||||
}
|
||||
return builder
|
||||
.andWhere('asset.updatedAt <= :updatedUntil', { updatedUntil })
|
||||
.andWhere('asset.isVisible = true')
|
||||
.orderBy('asset.fileCreatedAt', 'DESC')
|
||||
.addOrderBy('asset.id', 'DESC')
|
||||
.limit(limit)
|
||||
.withDeleted()
|
||||
.getMany();
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE }] })
|
||||
getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]> {
|
||||
return this.repository.find({
|
||||
where: {
|
||||
ownerId: In(options.userIds),
|
||||
isVisible: true,
|
||||
updatedAt: MoreThan(options.updatedAfter),
|
||||
},
|
||||
relations: {
|
||||
exifInfo: true,
|
||||
stack: true,
|
||||
},
|
||||
take: options.limit,
|
||||
withDeleted: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,24 +2,25 @@ import { InjectRepository } from '@nestjs/typeorm';
|
|||
import { AuditEntity } from 'src/entities/audit.entity';
|
||||
import { AuditSearch, IAuditRepository } from 'src/interfaces/audit.interface';
|
||||
import { Instrumentation } from 'src/utils/instrumentation';
|
||||
import { LessThan, MoreThan, Repository } from 'typeorm';
|
||||
import { In, LessThan, MoreThan, Repository } from 'typeorm';
|
||||
|
||||
@Instrumentation()
|
||||
export class AuditRepository implements IAuditRepository {
|
||||
constructor(@InjectRepository(AuditEntity) private repository: Repository<AuditEntity>) {}
|
||||
|
||||
getAfter(since: Date, options: AuditSearch): Promise<AuditEntity[]> {
|
||||
getAfter(since: Date, options: AuditSearch): Promise<string[]> {
|
||||
return this.repository
|
||||
.createQueryBuilder('audit')
|
||||
.where({
|
||||
createdAt: MoreThan(since),
|
||||
action: options.action,
|
||||
entityType: options.entityType,
|
||||
ownerId: options.ownerId,
|
||||
ownerId: In(options.userIds),
|
||||
})
|
||||
.distinctOn(['audit.entityId', 'audit.entityType'])
|
||||
.orderBy('audit.entityId, audit.entityType, audit.createdAt', 'DESC')
|
||||
.getMany();
|
||||
.select('audit.entityId')
|
||||
.getRawMany();
|
||||
}
|
||||
|
||||
async removeBefore(before: Date): Promise<void> {
|
||||
|
|
|
|||
|
|
@ -61,13 +61,13 @@ describe(AuditService.name, () => {
|
|||
|
||||
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
|
||||
action: DatabaseAction.DELETE,
|
||||
ownerId: authStub.admin.user.id,
|
||||
userIds: [authStub.admin.user.id],
|
||||
entityType: EntityType.ASSET,
|
||||
});
|
||||
});
|
||||
|
||||
it('should get any new or updated assets and deleted ids', async () => {
|
||||
auditMock.getAfter.mockResolvedValue([auditStub.delete]);
|
||||
auditMock.getAfter.mockResolvedValue([auditStub.delete.entityId]);
|
||||
|
||||
const date = new Date();
|
||||
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
|
||||
|
|
@ -77,7 +77,7 @@ describe(AuditService.name, () => {
|
|||
|
||||
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
|
||||
action: DatabaseAction.DELETE,
|
||||
ownerId: authStub.admin.user.id,
|
||||
userIds: [authStub.admin.user.id],
|
||||
entityType: EntityType.ASSET,
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ export class AuditService {
|
|||
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
|
||||
|
||||
const audits = await this.repository.getAfter(dto.after, {
|
||||
ownerId: userId,
|
||||
userIds: [userId],
|
||||
entityType: dto.entityType,
|
||||
action: DatabaseAction.DELETE,
|
||||
});
|
||||
|
|
@ -62,7 +62,7 @@ export class AuditService {
|
|||
|
||||
return {
|
||||
needsFullSync: duration > AUDIT_LOG_MAX_DURATION,
|
||||
ids: audits.map(({ entityId }) => entityId),
|
||||
ids: audits,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import { SharedLinkService } from 'src/services/shared-link.service';
|
|||
import { SmartInfoService } from 'src/services/smart-info.service';
|
||||
import { StorageTemplateService } from 'src/services/storage-template.service';
|
||||
import { StorageService } from 'src/services/storage.service';
|
||||
import { SyncService } from 'src/services/sync.service';
|
||||
import { SystemConfigService } from 'src/services/system-config.service';
|
||||
import { TagService } from 'src/services/tag.service';
|
||||
import { TimelineService } from 'src/services/timeline.service';
|
||||
|
|
@ -53,6 +54,7 @@ export const services = [
|
|||
SmartInfoService,
|
||||
StorageService,
|
||||
StorageTemplateService,
|
||||
SyncService,
|
||||
SystemConfigService,
|
||||
TagService,
|
||||
TimelineService,
|
||||
|
|
|
|||
101
server/src/services/sync.service.spec.ts
Normal file
101
server/src/services/sync.service.spec.ts
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
import { mapAsset } from 'src/dtos/asset-response.dto';
|
||||
import { AssetEntity } from 'src/entities/asset.entity';
|
||||
import { IAccessRepository } from 'src/interfaces/access.interface';
|
||||
import { IAssetRepository } from 'src/interfaces/asset.interface';
|
||||
import { IAuditRepository } from 'src/interfaces/audit.interface';
|
||||
import { IPartnerRepository } from 'src/interfaces/partner.interface';
|
||||
import { SyncService } from 'src/services/sync.service';
|
||||
import { assetStub } from 'test/fixtures/asset.stub';
|
||||
import { authStub } from 'test/fixtures/auth.stub';
|
||||
import { partnerStub } from 'test/fixtures/partner.stub';
|
||||
import { newAccessRepositoryMock } from 'test/repositories/access.repository.mock';
|
||||
import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
|
||||
import { newAuditRepositoryMock } from 'test/repositories/audit.repository.mock';
|
||||
import { newPartnerRepositoryMock } from 'test/repositories/partner.repository.mock';
|
||||
|
||||
const untilDate = new Date(2024);
|
||||
const mapAssetOpts = { auth: authStub.user1, stripMetadata: false, withStack: true };
|
||||
|
||||
describe(SyncService.name, () => {
|
||||
let sut: SyncService;
|
||||
let accessMock: jest.Mocked<IAccessRepository>;
|
||||
let assetMock: jest.Mocked<IAssetRepository>;
|
||||
let partnerMock: jest.Mocked<IPartnerRepository>;
|
||||
let auditMock: jest.Mocked<IAuditRepository>;
|
||||
|
||||
beforeEach(() => {
|
||||
partnerMock = newPartnerRepositoryMock();
|
||||
assetMock = newAssetRepositoryMock();
|
||||
accessMock = newAccessRepositoryMock();
|
||||
auditMock = newAuditRepositoryMock();
|
||||
sut = new SyncService(accessMock, assetMock, partnerMock, auditMock);
|
||||
});
|
||||
|
||||
it('should exist', () => {
|
||||
expect(sut).toBeDefined();
|
||||
});
|
||||
|
||||
describe('getAllAssetsForUserFullSync', () => {
|
||||
it('should return a list of all assets owned by the user', async () => {
|
||||
assetMock.getAllForUserFullSync.mockResolvedValue([assetStub.external, assetStub.hasEncodedVideo]);
|
||||
await expect(
|
||||
sut.getAllAssetsForUserFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate }),
|
||||
).resolves.toEqual([
|
||||
mapAsset(assetStub.external, mapAssetOpts),
|
||||
mapAsset(assetStub.hasEncodedVideo, mapAssetOpts),
|
||||
]);
|
||||
expect(assetMock.getAllForUserFullSync).toHaveBeenCalledWith({
|
||||
ownerId: authStub.user1.user.id,
|
||||
updatedUntil: untilDate,
|
||||
limit: 2,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getChangesForDeltaSync', () => {
|
||||
it('should return a response requiring a full sync when partners are out of sync', async () => {
|
||||
partnerMock.getAll.mockResolvedValue([partnerStub.adminToUser1]);
|
||||
await expect(
|
||||
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
|
||||
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
|
||||
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
|
||||
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
it('should return a response requiring a full sync when last sync was too long ago', async () => {
|
||||
partnerMock.getAll.mockResolvedValue([]);
|
||||
await expect(
|
||||
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }),
|
||||
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
|
||||
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
|
||||
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
it('should return a response requiring a full sync when there are too many changes', async () => {
|
||||
partnerMock.getAll.mockResolvedValue([]);
|
||||
assetMock.getChangedDeltaSync.mockResolvedValue(
|
||||
Array.from<AssetEntity>({ length: 10_000 }).fill(assetStub.image),
|
||||
);
|
||||
await expect(
|
||||
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
|
||||
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
|
||||
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1);
|
||||
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
|
||||
});
|
||||
|
||||
it('should return a response with changes and deletions', async () => {
|
||||
partnerMock.getAll.mockResolvedValue([]);
|
||||
assetMock.getChangedDeltaSync.mockResolvedValue([assetStub.image1]);
|
||||
auditMock.getAfter.mockResolvedValue([assetStub.external.id]);
|
||||
await expect(
|
||||
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
|
||||
).resolves.toEqual({
|
||||
needsFullSync: false,
|
||||
upserted: [mapAsset(assetStub.image1, mapAssetOpts)],
|
||||
deleted: [assetStub.external.id],
|
||||
});
|
||||
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1);
|
||||
expect(auditMock.getAfter).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
77
server/src/services/sync.service.ts
Normal file
77
server/src/services/sync.service.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import { Inject } from '@nestjs/common';
|
||||
import _ from 'lodash';
|
||||
import { DateTime } from 'luxon';
|
||||
import { AUDIT_LOG_MAX_DURATION } from 'src/constants';
|
||||
import { AccessCore, Permission } from 'src/cores/access.core';
|
||||
import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
|
||||
import { AuthDto } from 'src/dtos/auth.dto';
|
||||
import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto';
|
||||
import { DatabaseAction, EntityType } from 'src/entities/audit.entity';
|
||||
import { IAccessRepository } from 'src/interfaces/access.interface';
|
||||
import { IAssetRepository } from 'src/interfaces/asset.interface';
|
||||
import { IAuditRepository } from 'src/interfaces/audit.interface';
|
||||
import { IPartnerRepository } from 'src/interfaces/partner.interface';
|
||||
|
||||
export class SyncService {
|
||||
private access: AccessCore;
|
||||
|
||||
constructor(
|
||||
@Inject(IAccessRepository) accessRepository: IAccessRepository,
|
||||
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
|
||||
@Inject(IPartnerRepository) private partnerRepository: IPartnerRepository,
|
||||
@Inject(IAuditRepository) private auditRepository: IAuditRepository,
|
||||
) {
|
||||
this.access = AccessCore.create(accessRepository);
|
||||
}
|
||||
|
||||
async getAllAssetsForUserFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
|
||||
const userId = dto.userId || auth.user.id;
|
||||
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
|
||||
const assets = await this.assetRepository.getAllForUserFullSync({
|
||||
ownerId: userId,
|
||||
lastCreationDate: dto.lastCreationDate,
|
||||
updatedUntil: dto.updatedUntil,
|
||||
lastId: dto.lastId,
|
||||
limit: dto.limit,
|
||||
});
|
||||
const options = { auth, stripMetadata: false, withStack: true };
|
||||
return assets.map((a) => mapAsset(a, options));
|
||||
}
|
||||
|
||||
async getChangesForDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
|
||||
await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds);
|
||||
const partner = await this.partnerRepository.getAll(auth.user.id);
|
||||
const userIds = [auth.user.id, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)];
|
||||
userIds.sort();
|
||||
dto.userIds.sort();
|
||||
const duration = DateTime.now().diff(DateTime.fromJSDate(dto.updatedAfter));
|
||||
|
||||
if (!_.isEqual(userIds, dto.userIds) || duration > AUDIT_LOG_MAX_DURATION) {
|
||||
// app does not have the correct partners synced
|
||||
// or app has not synced in the last 100 days
|
||||
return { needsFullSync: true, deleted: [], upserted: [] };
|
||||
}
|
||||
|
||||
const limit = 10_000;
|
||||
const upserted = await this.assetRepository.getChangedDeltaSync({ limit, updatedAfter: dto.updatedAfter, userIds });
|
||||
|
||||
if (upserted.length === limit) {
|
||||
// too many changes -> do a full sync (paginated) instead
|
||||
return { needsFullSync: true, deleted: [], upserted: [] };
|
||||
}
|
||||
|
||||
const deleted = await this.auditRepository.getAfter(dto.updatedAfter, {
|
||||
userIds: userIds,
|
||||
entityType: EntityType.ASSET,
|
||||
action: DatabaseAction.DELETE,
|
||||
});
|
||||
|
||||
const options = { auth, stripMetadata: false, withStack: true };
|
||||
const result = {
|
||||
needsFullSync: false,
|
||||
upserted: upserted.map((a) => mapAsset(a, options)),
|
||||
deleted,
|
||||
};
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -35,5 +35,7 @@ export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
|
|||
getAssetIdByCity: jest.fn(),
|
||||
getAssetIdByTag: jest.fn(),
|
||||
searchMetadata: jest.fn(),
|
||||
getAllForUserFullSync: jest.fn(),
|
||||
getChangedDeltaSync: jest.fn(),
|
||||
};
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue