c-l/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-28 22:51:59 +00:00
Code Releases Activity

fix(server): stacked assets for full sync, userIds as array for delta sync (#9100)

Browse source 
* fix(server): stacked assets for full sync, userIds as array for delta sync

* refactor(server): sync

* fix getDeltaSync after partner removal

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
This commit is contained in:
Fynn Petersen-Frey 2024-04-29 05:24:21 +02:00 committed by GitHub
parent fc2e709ad4
commit 32e7cfea3d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 306 additions and 218 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
mobile/openapi/.openapi-generator/FILES generated
View file

@ -28,12 +28,14 @@ doc/AssetBulkUploadCheckDto.md
doc/AssetBulkUploadCheckItem.md
doc/AssetBulkUploadCheckResponseDto.md
doc/AssetBulkUploadCheckResult.md
doc/AssetDeltaSyncDto.md
doc/AssetDeltaSyncResponseDto.md
doc/AssetFaceResponseDto.md
doc/AssetFaceUpdateDto.md
doc/AssetFaceUpdateItem.md
doc/AssetFaceWithoutPersonResponseDto.md
doc/AssetFileUploadResponseDto.md
doc/AssetFullSyncDto.md
doc/AssetIdsDto.md
doc/AssetIdsResponseDto.md
doc/AssetJobName.md
@ -265,12 +267,14 @@ lib/model/asset_bulk_upload_check_dto.dart
lib/model/asset_bulk_upload_check_item.dart
lib/model/asset_bulk_upload_check_response_dto.dart
lib/model/asset_bulk_upload_check_result.dart
lib/model/asset_delta_sync_dto.dart
lib/model/asset_delta_sync_response_dto.dart
lib/model/asset_face_response_dto.dart
lib/model/asset_face_update_dto.dart
lib/model/asset_face_update_item.dart
lib/model/asset_face_without_person_response_dto.dart
lib/model/asset_file_upload_response_dto.dart
lib/model/asset_full_sync_dto.dart
lib/model/asset_ids_dto.dart
lib/model/asset_ids_response_dto.dart
lib/model/asset_job_name.dart
@ -449,12 +453,14 @@ test/asset_bulk_upload_check_dto_test.dart
test/asset_bulk_upload_check_item_test.dart
test/asset_bulk_upload_check_response_dto_test.dart
test/asset_bulk_upload_check_result_test.dart
test/asset_delta_sync_dto_test.dart
test/asset_delta_sync_response_dto_test.dart
test/asset_face_response_dto_test.dart
test/asset_face_update_dto_test.dart
test/asset_face_update_item_test.dart
test/asset_face_without_person_response_dto_test.dart
test/asset_file_upload_response_dto_test.dart
test/asset_full_sync_dto_test.dart
test/asset_ids_dto_test.dart
test/asset_ids_response_dto_test.dart
test/asset_job_name_test.dart

BIN
mobile/openapi/README.md generated
View file

Binary file not shown.

BIN
mobile/openapi/doc/AssetDeltaSyncDto.md generated Normal file
View file

Binary file not shown.

BIN
mobile/openapi/doc/AssetFullSyncDto.md generated Normal file
View file

Binary file not shown.

BIN
mobile/openapi/doc/SyncApi.md generated
View file

Binary file not shown.

BIN
mobile/openapi/lib/api.dart generated
View file

Binary file not shown.

BIN
mobile/openapi/lib/api/sync_api.dart generated
View file

Binary file not shown.

BIN
mobile/openapi/lib/api_client.dart generated
View file

Binary file not shown.

BIN
mobile/openapi/lib/model/asset_delta_sync_dto.dart generated Normal file
View file

Binary file not shown.

BIN
mobile/openapi/lib/model/asset_full_sync_dto.dart generated Normal file
View file

Binary file not shown.

BIN
mobile/openapi/test/asset_delta_sync_dto_test.dart generated Normal file
View file

Binary file not shown.

BIN
mobile/openapi/test/asset_full_sync_dto_test.dart generated Normal file
View file

Binary file not shown.

BIN
mobile/openapi/test/sync_api_test.dart generated
View file

Binary file not shown.

139
open-api/immich-openapi-specs.json
View file

@ -4958,31 +4958,19 @@
}
},
"/sync/delta-sync": {
"get": {
"post": {
"operationId": "getDeltaSync",
"parameters": [
{
"name": "updatedAfter",
"required": true,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "userIds",
"required": true,
"in": "query",
"schema": {
"format": "uuid",
"type": "array",
"items": {
"type": "string"
"parameters": [],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/AssetDeltaSyncDto"
}
}
}
],
},
"required": true
},
"responses": {
"200": {
"content": {
@ -5012,55 +5000,19 @@
}
},
"/sync/full-sync": {
"get": {
"operationId": "getAllForUserFullSync",
"parameters": [
{
"name": "lastCreationDate",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
"post": {
"operationId": "getFullSyncForUser",
"parameters": [],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/AssetFullSyncDto"
}
}
},
{
"name": "lastId",
"required": false,
"in": "query",
"schema": {
"format": "uuid",
"type": "string"
}
},
{
"name": "limit",
"required": true,
"in": "query",
"schema": {
"minimum": 1,
"type": "integer"
}
},
{
"name": "updatedUntil",
"required": true,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "userId",
"required": false,
"in": "query",
"schema": {
"format": "uuid",
"type": "string"
}
}
],
"required": true
},
"responses": {
"200": {
"content": {
@ -7023,6 +6975,26 @@
],
"type": "object"
},
"AssetDeltaSyncDto": {
"properties": {
"updatedAfter": {
"format": "date-time",
"type": "string"
},
"userIds": {
"items": {
"format": "uuid",
"type": "string"
},
"type": "array"
}
},
"required": [
"updatedAfter",
"userIds"
],
"type": "object"
},
"AssetDeltaSyncResponseDto": {
"properties": {
"deleted": {
@ -7175,6 +7147,35 @@
],
"type": "object"
},
"AssetFullSyncDto": {
"properties": {
"lastCreationDate": {
"format": "date-time",
"type": "string"
},
"lastId": {
"format": "uuid",
"type": "string"
},
"limit": {
"minimum": 1,
"type": "integer"
},
"updatedUntil": {
"format": "date-time",
"type": "string"
},
"userId": {
"format": "uuid",
"type": "string"
}
},
"required": [
"limit",
"updatedUntil"
],
"type": "object"
},
"AssetIdsDto": {
"properties": {
"assetIds": {

49
open-api/typescript-sdk/src/fetch-client.ts
View file

@ -836,11 +836,22 @@ export type AssetIdsResponseDto = {
error?: Error2;
success: boolean;
};
export type AssetDeltaSyncDto = {
updatedAfter: string;
userIds: string[];
};
export type AssetDeltaSyncResponseDto = {
deleted: string[];
needsFullSync: boolean;
upserted: AssetResponseDto[];
};
export type AssetFullSyncDto = {
lastCreationDate?: string;
lastId?: string;
limit: number;
updatedUntil: string;
userId?: string;
};
export type SystemConfigFFmpegDto = {
accel: TranscodeHWAccel;
acceptedAudioCodecs: AudioCodec[];
@ -2372,39 +2383,29 @@ export function addSharedLinkAssets({ id, key, assetIdsDto }: {
body: assetIdsDto
})));
}
export function getDeltaSync({ updatedAfter, userIds }: {
updatedAfter: string;
userIds: string[];
export function getDeltaSync({ assetDeltaSyncDto }: {
assetDeltaSyncDto: AssetDeltaSyncDto;
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: AssetDeltaSyncResponseDto;
}>(`/sync/delta-sync${QS.query(QS.explode({
updatedAfter,
userIds
}))}`, {
...opts
}));
}>("/sync/delta-sync", oazapfts.json({
...opts,
method: "POST",
body: assetDeltaSyncDto
})));
}
export function getAllForUserFullSync({ lastCreationDate, lastId, limit, updatedUntil, userId }: {
lastCreationDate?: string;
lastId?: string;
limit: number;
updatedUntil: string;
userId?: string;
export function getFullSyncForUser({ assetFullSyncDto }: {
assetFullSyncDto: AssetFullSyncDto;
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: AssetResponseDto[];
}>(`/sync/full-sync${QS.query(QS.explode({
lastCreationDate,
lastId,
limit,
updatedUntil,
userId
}))}`, {
...opts
}));
}>("/sync/full-sync", oazapfts.json({
...opts,
method: "POST",
body: assetFullSyncDto
})));
}
export function getConfig(opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{

16
server/src/controllers/sync.controller.ts
View file

@ -1,4 +1,4 @@
import { Controller, Get, Query } from '@nestjs/common';
import { Body, Controller, HttpCode, HttpStatus, Post } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { AssetResponseDto } from 'src/dtos/asset-response.dto';
import { AuthDto } from 'src/dtos/auth.dto';
@ -12,13 +12,15 @@ import { SyncService } from 'src/services/sync.service';
export class SyncController {
constructor(private service: SyncService) {}
@Get('full-sync')
getAllForUserFullSync(@Auth() auth: AuthDto, @Query() dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
return this.service.getAllAssetsForUserFullSync(auth, dto);
@Post('full-sync')
@HttpCode(HttpStatus.OK)
getFullSyncForUser(@Auth() auth: AuthDto, @Body() dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
return this.service.getFullSync(auth, dto);
}
@Get('delta-sync')
getDeltaSync(@Auth() auth: AuthDto, @Query() dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
return this.service.getChangesForDeltaSync(auth, dto);
@Post('delta-sync')
@HttpCode(HttpStatus.OK)
getDeltaSync(@Auth() auth: AuthDto, @Body() dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
return this.service.getDeltaSync(auth, dto);
}
}

3
server/src/dtos/sync.dto.ts
View file

@ -1,5 +1,4 @@
import { ApiProperty } from '@nestjs/swagger';
import { Type } from 'class-transformer';
import { IsInt, IsPositive } from 'class-validator';
import { AssetResponseDto } from 'src/dtos/asset-response.dto';
import { ValidateDate, ValidateUUID } from 'src/validation';
@ -16,7 +15,6 @@ export class AssetFullSyncDto {
@IsInt()
@IsPositive()
@Type(() => Number)
@ApiProperty({ type: 'integer' })
limit!: number;
@ -27,6 +25,7 @@ export class AssetFullSyncDto {
export class AssetDeltaSyncDto {
@ValidateDate()
updatedAfter!: Date;
@ValidateUUID({ each: true })
userIds!: string[];
}

2
server/src/interfaces/asset.interface.ts
View file

@ -134,6 +134,8 @@ export interface AssetFullSyncOptions {
lastCreationDate?: Date;
lastId?: string;
updatedUntil: Date;
isArchived?: false;
withStacked?: true;
limit: number;
}

202
server/src/queries/asset.repository.sql
View file

@ -798,16 +798,47 @@ SELECT
"exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
"exifInfo"."fps" AS "exifInfo_fps",
"stack"."id" AS "stack_id",
"stack"."primaryAssetId" AS "stack_primaryAssetId"
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."previewPath" AS "stackedAssets_previewPath",
"stackedAssets"."thumbnailPath" AS "stackedAssets_thumbnailPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isReadOnly" AS "stackedAssets_isReadOnly",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE
"asset"."ownerId" = $1
"asset"."isVisible" = true
AND "asset"."ownerId" IN ($1)
AND ("asset"."fileCreatedAt", "asset"."id") < ($2, $3)
AND "asset"."updatedAt" <= $4
AND "asset"."isVisible" = true
ORDER BY
"asset"."fileCreatedAt" DESC,
"asset"."id" DESC
@ -816,72 +847,105 @@ LIMIT
-- AssetRepository.getChangedDeltaSync
SELECT
"AssetEntity"."id" AS "AssetEntity_id",
"AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId",
"AssetEntity"."ownerId" AS "AssetEntity_ownerId",
"AssetEntity"."libraryId" AS "AssetEntity_libraryId",
"AssetEntity"."deviceId" AS "AssetEntity_deviceId",
"AssetEntity"."type" AS "AssetEntity_type",
"AssetEntity"."originalPath" AS "AssetEntity_originalPath",
"AssetEntity"."previewPath" AS "AssetEntity_previewPath",
"AssetEntity"."thumbnailPath" AS "AssetEntity_thumbnailPath",
"AssetEntity"."thumbhash" AS "AssetEntity_thumbhash",
"AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath",
"AssetEntity"."createdAt" AS "AssetEntity_createdAt",
"AssetEntity"."updatedAt" AS "AssetEntity_updatedAt",
"AssetEntity"."deletedAt" AS "AssetEntity_deletedAt",
"AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt",
"AssetEntity"."localDateTime" AS "AssetEntity_localDateTime",
"AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt",
"AssetEntity"."isFavorite" AS "AssetEntity_isFavorite",
"AssetEntity"."isArchived" AS "AssetEntity_isArchived",
"AssetEntity"."isExternal" AS "AssetEntity_isExternal",
"AssetEntity"."isReadOnly" AS "AssetEntity_isReadOnly",
"AssetEntity"."isOffline" AS "AssetEntity_isOffline",
"AssetEntity"."checksum" AS "AssetEntity_checksum",
"AssetEntity"."duration" AS "AssetEntity_duration",
"AssetEntity"."isVisible" AS "AssetEntity_isVisible",
"AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
"AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
"AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
"AssetEntity"."stackId" AS "AssetEntity_stackId",
"AssetEntity__AssetEntity_exifInfo"."assetId" AS "AssetEntity__AssetEntity_exifInfo_assetId",
"AssetEntity__AssetEntity_exifInfo"."description" AS "AssetEntity__AssetEntity_exifInfo_description",
"AssetEntity__AssetEntity_exifInfo"."exifImageWidth" AS "AssetEntity__AssetEntity_exifInfo_exifImageWidth",
"AssetEntity__AssetEntity_exifInfo"."exifImageHeight" AS "AssetEntity__AssetEntity_exifInfo_exifImageHeight",
"AssetEntity__AssetEntity_exifInfo"."fileSizeInByte" AS "AssetEntity__AssetEntity_exifInfo_fileSizeInByte",
"AssetEntity__AssetEntity_exifInfo"."orientation" AS "AssetEntity__AssetEntity_exifInfo_orientation",
"AssetEntity__AssetEntity_exifInfo"."dateTimeOriginal" AS "AssetEntity__AssetEntity_exifInfo_dateTimeOriginal",
"AssetEntity__AssetEntity_exifInfo"."modifyDate" AS "AssetEntity__AssetEntity_exifInfo_modifyDate",
"AssetEntity__AssetEntity_exifInfo"."timeZone" AS "AssetEntity__AssetEntity_exifInfo_timeZone",
"AssetEntity__AssetEntity_exifInfo"."latitude" AS "AssetEntity__AssetEntity_exifInfo_latitude",
"AssetEntity__AssetEntity_exifInfo"."longitude" AS "AssetEntity__AssetEntity_exifInfo_longitude",
"AssetEntity__AssetEntity_exifInfo"."projectionType" AS "AssetEntity__AssetEntity_exifInfo_projectionType",
"AssetEntity__AssetEntity_exifInfo"."city" AS "AssetEntity__AssetEntity_exifInfo_city",
"AssetEntity__AssetEntity_exifInfo"."livePhotoCID" AS "AssetEntity__AssetEntity_exifInfo_livePhotoCID",
"AssetEntity__AssetEntity_exifInfo"."autoStackId" AS "AssetEntity__AssetEntity_exifInfo_autoStackId",
"AssetEntity__AssetEntity_exifInfo"."state" AS "AssetEntity__AssetEntity_exifInfo_state",
"AssetEntity__AssetEntity_exifInfo"."country" AS "AssetEntity__AssetEntity_exifInfo_country",
"AssetEntity__AssetEntity_exifInfo"."make" AS "AssetEntity__AssetEntity_exifInfo_make",
"AssetEntity__AssetEntity_exifInfo"."model" AS "AssetEntity__AssetEntity_exifInfo_model",
"AssetEntity__AssetEntity_exifInfo"."lensModel" AS "AssetEntity__AssetEntity_exifInfo_lensModel",
"AssetEntity__AssetEntity_exifInfo"."fNumber" AS "AssetEntity__AssetEntity_exifInfo_fNumber",
"AssetEntity__AssetEntity_exifInfo"."focalLength" AS "AssetEntity__AssetEntity_exifInfo_focalLength",
"AssetEntity__AssetEntity_exifInfo"."iso" AS "AssetEntity__AssetEntity_exifInfo_iso",
"AssetEntity__AssetEntity_exifInfo"."exposureTime" AS "AssetEntity__AssetEntity_exifInfo_exposureTime",
"AssetEntity__AssetEntity_exifInfo"."profileDescription" AS "AssetEntity__AssetEntity_exifInfo_profileDescription",
"AssetEntity__AssetEntity_exifInfo"."colorspace" AS "AssetEntity__AssetEntity_exifInfo_colorspace",
"AssetEntity__AssetEntity_exifInfo"."bitsPerSample" AS "AssetEntity__AssetEntity_exifInfo_bitsPerSample",
"AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps",
"AssetEntity__AssetEntity_stack"."id" AS "AssetEntity__AssetEntity_stack_id",
"AssetEntity__AssetEntity_stack"."primaryAssetId" AS "AssetEntity__AssetEntity_stack_primaryAssetId"
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."originalPath" AS "asset_originalPath",
"asset"."previewPath" AS "asset_previewPath",
"asset"."thumbnailPath" AS "asset_thumbnailPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isReadOnly" AS "asset_isReadOnly",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"exifInfo"."assetId" AS "exifInfo_assetId",
"exifInfo"."description" AS "exifInfo_description",
"exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
"exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight",
"exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte",
"exifInfo"."orientation" AS "exifInfo_orientation",
"exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal",
"exifInfo"."modifyDate" AS "exifInfo_modifyDate",
"exifInfo"."timeZone" AS "exifInfo_timeZone",
"exifInfo"."latitude" AS "exifInfo_latitude",
"exifInfo"."longitude" AS "exifInfo_longitude",
"exifInfo"."projectionType" AS "exifInfo_projectionType",
"exifInfo"."city" AS "exifInfo_city",
"exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID",
"exifInfo"."autoStackId" AS "exifInfo_autoStackId",
"exifInfo"."state" AS "exifInfo_state",
"exifInfo"."country" AS "exifInfo_country",
"exifInfo"."make" AS "exifInfo_make",
"exifInfo"."model" AS "exifInfo_model",
"exifInfo"."lensModel" AS "exifInfo_lensModel",
"exifInfo"."fNumber" AS "exifInfo_fNumber",
"exifInfo"."focalLength" AS "exifInfo_focalLength",
"exifInfo"."iso" AS "exifInfo_iso",
"exifInfo"."exposureTime" AS "exifInfo_exposureTime",
"exifInfo"."profileDescription" AS "exifInfo_profileDescription",
"exifInfo"."colorspace" AS "exifInfo_colorspace",
"exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
"exifInfo"."fps" AS "exifInfo_fps",
"stack"."id" AS "stack_id",
"stack"."primaryAssetId" AS "stack_primaryAssetId",
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."previewPath" AS "stackedAssets_previewPath",
"stackedAssets"."thumbnailPath" AS "stackedAssets_thumbnailPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isReadOnly" AS "stackedAssets_isReadOnly",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId"
FROM
"assets" "AssetEntity"
LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id"
LEFT JOIN "asset_stack" "AssetEntity__AssetEntity_stack" ON "AssetEntity__AssetEntity_stack"."id" = "AssetEntity"."stackId"
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE
(
("AssetEntity"."ownerId" IN ($1))
AND ("AssetEntity"."isVisible" = $2)
AND ("AssetEntity"."updatedAt" > $3)
"asset"."isVisible" = true
AND "asset"."ownerId" IN ($1)
AND (
"stack"."primaryAssetId" = "asset"."id"
OR "asset"."stackId" IS NULL
)
AND "asset"."updatedAt" > $2

35
server/src/repositories/asset.repository.ts
View file

@ -710,21 +710,23 @@ export class AssetRepository implements IAssetRepository {
],
})
getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]> {
const { ownerId, lastCreationDate, lastId, updatedUntil, limit } = options;
const builder = this.repository
.createQueryBuilder('asset')
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
.leftJoinAndSelect('asset.stack', 'stack')
.where('asset.ownerId = :ownerId', { ownerId });
const { ownerId, isArchived, withStacked, lastCreationDate, lastId, updatedUntil, limit } = options;
const builder = this.getBuilder({
userIds: [ownerId],
exifInfo: true,
withStacked,
isArchived,
});
if (lastCreationDate !== undefined && lastId !== undefined) {
builder.andWhere('(asset.fileCreatedAt, asset.id) < (:lastCreationDate, :lastId)', {
lastCreationDate,
lastId,
});
}
return builder
.andWhere('asset.updatedAt <= :updatedUntil', { updatedUntil })
.andWhere('asset.isVisible = true')
.orderBy('asset.fileCreatedAt', 'DESC')
.addOrderBy('asset.id', 'DESC')
.limit(limit)
@ -734,18 +736,11 @@ export class AssetRepository implements IAssetRepository {
@GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE }] })
getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]> {
return this.repository.find({
where: {
ownerId: In(options.userIds),
isVisible: true,
updatedAt: MoreThan(options.updatedAfter),
},
relations: {
exifInfo: true,
stack: true,
},
take: options.limit,
withDeleted: true,
});
const builder = this.getBuilder({ userIds: options.userIds, exifInfo: true, withStacked: true })
.andWhere({ updatedAt: MoreThan(options.updatedAfter) })
.take(options.limit)
.withDeleted();
return builder.getMany();
}
}

13
server/src/services/sync.service.spec.ts
View file

@ -39,13 +39,12 @@ describe(SyncService.name, () => {
describe('getAllAssetsForUserFullSync', () => {
it('should return a list of all assets owned by the user', async () => {
assetMock.getAllForUserFullSync.mockResolvedValue([assetStub.external, assetStub.hasEncodedVideo]);
await expect(
sut.getAllAssetsForUserFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate }),
).resolves.toEqual([
await expect(sut.getFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate })).resolves.toEqual([
mapAsset(assetStub.external, mapAssetOpts),
mapAsset(assetStub.hasEncodedVideo, mapAssetOpts),
]);
expect(assetMock.getAllForUserFullSync).toHaveBeenCalledWith({
withStacked: true,
ownerId: authStub.user1.user.id,
updatedUntil: untilDate,
limit: 2,
@ -57,7 +56,7 @@ describe(SyncService.name, () => {
it('should return a response requiring a full sync when partners are out of sync', async () => {
partnerMock.getAll.mockResolvedValue([partnerStub.adminToUser1]);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
@ -66,7 +65,7 @@ describe(SyncService.name, () => {
it('should return a response requiring a full sync when last sync was too long ago', async () => {
partnerMock.getAll.mockResolvedValue([]);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }),
sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
@ -78,7 +77,7 @@ describe(SyncService.name, () => {
Array.from<AssetEntity>({ length: 10_000 }).fill(assetStub.image),
);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1);
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
@ -89,7 +88,7 @@ describe(SyncService.name, () => {
assetMock.getChangedDeltaSync.mockResolvedValue([assetStub.image1]);
auditMock.getAfter.mockResolvedValue([assetStub.external.id]);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({
needsFullSync: false,
upserted: [mapAsset(assetStub.image1, mapAssetOpts)],

59
server/src/services/sync.service.ts
View file

@ -1,5 +1,4 @@
import { Inject } from '@nestjs/common';
import _ from 'lodash';
import { DateTime } from 'luxon';
import { AUDIT_LOG_MAX_DURATION } from 'src/constants';
import { AccessCore, Permission } from 'src/cores/access.core';
@ -11,6 +10,9 @@ import { IAccessRepository } from 'src/interfaces/access.interface';
import { IAssetRepository } from 'src/interfaces/asset.interface';
import { IAuditRepository } from 'src/interfaces/audit.interface';
import { IPartnerRepository } from 'src/interfaces/partner.interface';
import { setIsEqual } from 'src/utils/set';
const FULL_SYNC = { needsFullSync: true, deleted: [], upserted: [] };
export class SyncService {
private access: AccessCore;
@ -24,52 +26,69 @@ export class SyncService {
this.access = AccessCore.create(accessRepository);
}
async getAllAssetsForUserFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
async getFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
// mobile implementation is faster if this is a single id
const userId = dto.userId || auth.user.id;
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
const assets = await this.assetRepository.getAllForUserFullSync({
ownerId: userId,
// no archived assets for partner user
isArchived: userId === auth.user.id ? undefined : false,
// no stack for partner user
withStacked: userId === auth.user.id ? true : undefined,
lastCreationDate: dto.lastCreationDate,
updatedUntil: dto.updatedUntil,
lastId: dto.lastId,
limit: dto.limit,
});
const options = { auth, stripMetadata: false, withStack: true };
return assets.map((a) => mapAsset(a, options));
return assets.map((a) => mapAsset(a, { auth, stripMetadata: false, withStack: true }));
}
async getChangesForDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds);
const partner = await this.partnerRepository.getAll(auth.user.id);
const userIds = [auth.user.id, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)];
userIds.sort();
dto.userIds.sort();
async getDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
// app has not synced in the last 100 days
const duration = DateTime.now().diff(DateTime.fromJSDate(dto.updatedAfter));
if (!_.isEqual(userIds, dto.userIds) || duration > AUDIT_LOG_MAX_DURATION) {
// app does not have the correct partners synced
// or app has not synced in the last 100 days
return { needsFullSync: true, deleted: [], upserted: [] };
if (duration > AUDIT_LOG_MAX_DURATION) {
return FULL_SYNC;
}
const authUserId = auth.user.id;
// app does not have the correct partners synced
const partner = await this.partnerRepository.getAll(authUserId);
const userIds = [authUserId, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)];
if (!setIsEqual(new Set(userIds), new Set(dto.userIds))) {
return FULL_SYNC;
}
await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds);
const limit = 10_000;
const upserted = await this.assetRepository.getChangedDeltaSync({ limit, updatedAfter: dto.updatedAfter, userIds });
// too many changes, need to do a full sync
if (upserted.length === limit) {
// too many changes -> do a full sync (paginated) instead
return { needsFullSync: true, deleted: [], upserted: [] };
return FULL_SYNC;
}
const deleted = await this.auditRepository.getAfter(dto.updatedAfter, {
userIds: userIds,
userIds,
entityType: EntityType.ASSET,
action: DatabaseAction.DELETE,
});
const options = { auth, stripMetadata: false, withStack: true };
const result = {
needsFullSync: false,
upserted: upserted.map((a) => mapAsset(a, options)),
upserted: upserted
// do not return archived assets for partner users
.filter((a) => a.ownerId === auth.user.id || (a.ownerId !== auth.user.id && !a.isArchived))
.map((a) =>
mapAsset(a, {
auth,
stripMetadata: false,
// ignore stacks for non partner users
withStack: a.ownerId === authUserId,
}),
),
deleted,
};
return result;