c-l/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-28 14:41:59 +00:00
Code Releases Activity

feat(server): efficient full app sync (#8755)

Browse source 
* feat(server): efficient full app sync

* add SQL, fix test compile issues

* fix linter warning

* new sync controller+service, add tests

* enable new sync controller+service

* Update server/src/services/sync.service.ts

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
This commit is contained in:
Fynn Petersen-Frey 2024-04-16 07:26:37 +02:00 committed by GitHub
parent 58e516c766
commit 103cb60a57
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 685 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
mobile/openapi/.openapi-generator/FILES generated
View file

@ -24,6 +24,7 @@ doc/AssetBulkUploadCheckDto.md
doc/AssetBulkUploadCheckItem.md
doc/AssetBulkUploadCheckResponseDto.md
doc/AssetBulkUploadCheckResult.md
doc/AssetDeltaSyncResponseDto.md
doc/AssetFaceResponseDto.md
doc/AssetFaceUpdateDto.md
doc/AssetFaceUpdateItem.md
@ -149,6 +150,7 @@ doc/SharedLinkType.md
doc/SignUpDto.md
doc/SmartInfoResponseDto.md
doc/SmartSearchDto.md
doc/SyncApi.md
doc/SystemConfigApi.md
doc/SystemConfigDto.md
doc/SystemConfigFFmpegDto.md
@ -218,6 +220,7 @@ lib/api/person_api.dart
lib/api/search_api.dart
lib/api/server_info_api.dart
lib/api/shared_link_api.dart
lib/api/sync_api.dart
lib/api/system_config_api.dart
lib/api/tag_api.dart
lib/api/timeline_api.dart
@ -248,6 +251,7 @@ lib/model/asset_bulk_upload_check_dto.dart
lib/model/asset_bulk_upload_check_item.dart
lib/model/asset_bulk_upload_check_response_dto.dart
lib/model/asset_bulk_upload_check_result.dart
lib/model/asset_delta_sync_response_dto.dart
lib/model/asset_face_response_dto.dart
lib/model/asset_face_update_dto.dart
lib/model/asset_face_update_item.dart
@ -427,6 +431,7 @@ test/asset_bulk_upload_check_dto_test.dart
test/asset_bulk_upload_check_item_test.dart
test/asset_bulk_upload_check_response_dto_test.dart
test/asset_bulk_upload_check_result_test.dart
test/asset_delta_sync_response_dto_test.dart
test/asset_face_response_dto_test.dart
test/asset_face_update_dto_test.dart
test/asset_face_update_item_test.dart
@ -552,6 +557,7 @@ test/shared_link_type_test.dart
test/sign_up_dto_test.dart
test/smart_info_response_dto_test.dart
test/smart_search_dto_test.dart
test/sync_api_test.dart
test/system_config_api_test.dart
test/system_config_dto_test.dart
test/system_config_f_fmpeg_dto_test.dart

BIN
mobile/openapi/README.md generated
View file

Binary file not shown.

BIN
mobile/openapi/doc/AssetDeltaSyncResponseDto.md generated Normal file
View file

Binary file not shown.

BIN
mobile/openapi/doc/SyncApi.md generated Normal file
View file

Binary file not shown.

BIN
mobile/openapi/lib/api.dart generated
View file

Binary file not shown.

BIN
mobile/openapi/lib/api/sync_api.dart generated Normal file
View file

Binary file not shown.

BIN
mobile/openapi/lib/api_client.dart generated
View file

Binary file not shown.

BIN
mobile/openapi/lib/model/asset_delta_sync_response_dto.dart generated Normal file
View file

Binary file not shown.

BIN
mobile/openapi/test/asset_delta_sync_response_dto_test.dart generated Normal file
View file

Binary file not shown.

BIN
mobile/openapi/test/sync_api_test.dart generated Normal file
View file

Binary file not shown.

159
open-api/immich-openapi-specs.json
View file

@ -5566,6 +5566,140 @@
]
}
},
"/sync/delta-sync": {
"get": {
"operationId": "getDeltaSync",
"parameters": [
{
"name": "updatedAfter",
"required": true,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "userIds",
"required": true,
"in": "query",
"schema": {
"format": "uuid",
"type": "array",
"items": {
"type": "string"
}
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/AssetDeltaSyncResponseDto"
}
}
},
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"Sync"
]
}
},
"/sync/full-sync": {
"get": {
"operationId": "getAllForUserFullSync",
"parameters": [
{
"name": "lastCreationDate",
"required": false,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "lastId",
"required": false,
"in": "query",
"schema": {
"format": "uuid",
"type": "string"
}
},
{
"name": "limit",
"required": true,
"in": "query",
"schema": {
"type": "integer"
}
},
{
"name": "updatedUntil",
"required": true,
"in": "query",
"schema": {
"format": "date-time",
"type": "string"
}
},
{
"name": "userId",
"required": false,
"in": "query",
"schema": {
"format": "uuid",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"items": {
"$ref": "#/components/schemas/AssetResponseDto"
},
"type": "array"
}
}
},
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"Sync"
]
}
},
"/system-config": {
"get": {
"operationId": "getConfig",
@ -7335,6 +7469,31 @@
],
"type": "object"
},
"AssetDeltaSyncResponseDto": {
"properties": {
"deleted": {
"items": {
"type": "string"
},
"type": "array"
},
"needsFullSync": {
"type": "boolean"
},
"upserted": {
"items": {
"$ref": "#/components/schemas/AssetResponseDto"
},
"type": "array"
}
},
"required": [
"deleted",
"needsFullSync",
"upserted"
],
"type": "object"
},
"AssetFaceResponseDto": {
"properties": {
"boundingBoxX1": {

39
open-api/typescript-sdk/src/fetch-client.ts
View file

@ -835,6 +835,11 @@ export type AssetIdsResponseDto = {
error?: Error2;
success: boolean;
};
export type AssetDeltaSyncResponseDto = {
deleted: string[];
needsFullSync: boolean;
upserted: AssetResponseDto[];
};
export type SystemConfigFFmpegDto = {
accel: TranscodeHWAccel;
acceptedAudioCodecs: AudioCodec[];
@ -2507,6 +2512,40 @@ export function addSharedLinkAssets({ id, key, assetIdsDto }: {
body: assetIdsDto
})));
}
export function getDeltaSync({ updatedAfter, userIds }: {
updatedAfter: string;
userIds: string[];
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: AssetDeltaSyncResponseDto;
}>(`/sync/delta-sync${QS.query(QS.explode({
updatedAfter,
userIds
}))}`, {
...opts
}));
}
export function getAllForUserFullSync({ lastCreationDate, lastId, limit, updatedUntil, userId }: {
lastCreationDate?: string;
lastId?: string;
limit: number;
updatedUntil: string;
userId?: string;
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: AssetResponseDto[];
}>(`/sync/full-sync${QS.query(QS.explode({
lastCreationDate,
lastId,
limit,
updatedUntil,
userId
}))}`, {
...opts
}));
}
export function getConfig(opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;

2
server/src/controllers/index.ts
View file

@ -17,6 +17,7 @@ import { PersonController } from 'src/controllers/person.controller';
import { SearchController } from 'src/controllers/search.controller';
import { ServerInfoController } from 'src/controllers/server-info.controller';
import { SharedLinkController } from 'src/controllers/shared-link.controller';
import { SyncController } from 'src/controllers/sync.controller';
import { SystemConfigController } from 'src/controllers/system-config.controller';
import { TagController } from 'src/controllers/tag.controller';
import { TimelineController } from 'src/controllers/timeline.controller';
@ -43,6 +44,7 @@ export const controllers = [
SearchController,
ServerInfoController,
SharedLinkController,
SyncController,
SystemConfigController,
TagController,
TimelineController,

24
server/src/controllers/sync.controller.ts Normal file
View file

@ -0,0 +1,24 @@
import { Controller, Get, Query } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { AssetResponseDto } from 'src/dtos/asset-response.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto';
import { Auth, Authenticated } from 'src/middleware/auth.guard';
import { SyncService } from 'src/services/sync.service';
@ApiTags('Sync')
@Controller('sync')
@Authenticated()
export class SyncController {
constructor(private service: SyncService) {}
@Get('full-sync')
getAllForUserFullSync(@Auth() auth: AuthDto, @Query() dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
return this.service.getAllAssetsForUserFullSync(auth, dto);
}
@Get('delta-sync')
getDeltaSync(@Auth() auth: AuthDto, @Query() dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
return this.service.getChangesForDeltaSync(auth, dto);
}
}

38
server/src/dtos/sync.dto.ts Normal file
View file

@ -0,0 +1,38 @@
import { ApiProperty } from '@nestjs/swagger';
import { Type } from 'class-transformer';
import { IsInt, IsPositive } from 'class-validator';
import { AssetResponseDto } from 'src/dtos/asset-response.dto';
import { ValidateDate, ValidateUUID } from 'src/validation';
export class AssetFullSyncDto {
@ValidateUUID({ optional: true })
lastId?: string;
@ValidateDate({ optional: true })
lastCreationDate?: Date;
@ValidateDate()
updatedUntil!: Date;
@IsInt()
@IsPositive()
@Type(() => Number)
@ApiProperty({ type: 'integer' })
limit!: number;
@ValidateUUID({ optional: true })
userId?: string;
}
export class AssetDeltaSyncDto {
@ValidateDate()
updatedAfter!: Date;
@ValidateUUID({ each: true })
userIds!: string[];
}
export class AssetDeltaSyncResponseDto {
needsFullSync!: boolean;
upserted!: AssetResponseDto[];
deleted!: string[];
}

16
server/src/interfaces/asset.interface.ts
View file

@ -133,6 +133,20 @@ export interface MetadataSearchOptions {
numResults: number;
}
export interface AssetFullSyncOptions {
ownerId: string;
lastCreationDate?: Date;
lastId?: string;
updatedUntil: Date;
limit: number;
}
export interface AssetDeltaSyncOptions {
userIds: string[];
updatedAfter: Date;
limit: number;
}
export type AssetPathEntity = Pick<AssetEntity, 'id' | 'originalPath' | 'isOffline'>;
export const IAssetRepository = 'IAssetRepository';
@ -175,4 +189,6 @@ export interface IAssetRepository {
getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
searchMetadata(query: string, userIds: string[], options: MetadataSearchOptions): Promise<AssetEntity[]>;
getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]>;
getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]>;
}

6
server/src/interfaces/audit.interface.ts
View file

@ -1,14 +1,14 @@
import { AuditEntity, DatabaseAction, EntityType } from 'src/entities/audit.entity';
import { DatabaseAction, EntityType } from 'src/entities/audit.entity';
export const IAuditRepository = 'IAuditRepository';
export interface AuditSearch {
action?: DatabaseAction;
entityType?: EntityType;
ownerId?: string;
userIds: string[];
}
export interface IAuditRepository {
getAfter(since: Date, options: AuditSearch): Promise<AuditEntity[]>;
getAfter(since: Date, options: AuditSearch): Promise<string[]>;
removeBefore(before: Date): Promise<void>;
}

148
server/src/queries/asset.repository.sql
View file

@ -768,3 +768,151 @@ ORDER BY
"asset"."fileCreatedAt" DESC
LIMIT
250
-- AssetRepository.getAllForUserFullSync
SELECT
"asset"."id" AS "asset_id",
"asset"."deviceAssetId" AS "asset_deviceAssetId",
"asset"."ownerId" AS "asset_ownerId",
"asset"."libraryId" AS "asset_libraryId",
"asset"."deviceId" AS "asset_deviceId",
"asset"."type" AS "asset_type",
"asset"."originalPath" AS "asset_originalPath",
"asset"."previewPath" AS "asset_previewPath",
"asset"."thumbnailPath" AS "asset_thumbnailPath",
"asset"."thumbhash" AS "asset_thumbhash",
"asset"."encodedVideoPath" AS "asset_encodedVideoPath",
"asset"."createdAt" AS "asset_createdAt",
"asset"."updatedAt" AS "asset_updatedAt",
"asset"."deletedAt" AS "asset_deletedAt",
"asset"."fileCreatedAt" AS "asset_fileCreatedAt",
"asset"."localDateTime" AS "asset_localDateTime",
"asset"."fileModifiedAt" AS "asset_fileModifiedAt",
"asset"."isFavorite" AS "asset_isFavorite",
"asset"."isArchived" AS "asset_isArchived",
"asset"."isExternal" AS "asset_isExternal",
"asset"."isReadOnly" AS "asset_isReadOnly",
"asset"."isOffline" AS "asset_isOffline",
"asset"."checksum" AS "asset_checksum",
"asset"."duration" AS "asset_duration",
"asset"."isVisible" AS "asset_isVisible",
"asset"."livePhotoVideoId" AS "asset_livePhotoVideoId",
"asset"."originalFileName" AS "asset_originalFileName",
"asset"."sidecarPath" AS "asset_sidecarPath",
"asset"."stackId" AS "asset_stackId",
"exifInfo"."assetId" AS "exifInfo_assetId",
"exifInfo"."description" AS "exifInfo_description",
"exifInfo"."exifImageWidth" AS "exifInfo_exifImageWidth",
"exifInfo"."exifImageHeight" AS "exifInfo_exifImageHeight",
"exifInfo"."fileSizeInByte" AS "exifInfo_fileSizeInByte",
"exifInfo"."orientation" AS "exifInfo_orientation",
"exifInfo"."dateTimeOriginal" AS "exifInfo_dateTimeOriginal",
"exifInfo"."modifyDate" AS "exifInfo_modifyDate",
"exifInfo"."timeZone" AS "exifInfo_timeZone",
"exifInfo"."latitude" AS "exifInfo_latitude",
"exifInfo"."longitude" AS "exifInfo_longitude",
"exifInfo"."projectionType" AS "exifInfo_projectionType",
"exifInfo"."city" AS "exifInfo_city",
"exifInfo"."livePhotoCID" AS "exifInfo_livePhotoCID",
"exifInfo"."autoStackId" AS "exifInfo_autoStackId",
"exifInfo"."state" AS "exifInfo_state",
"exifInfo"."country" AS "exifInfo_country",
"exifInfo"."make" AS "exifInfo_make",
"exifInfo"."model" AS "exifInfo_model",
"exifInfo"."lensModel" AS "exifInfo_lensModel",
"exifInfo"."fNumber" AS "exifInfo_fNumber",
"exifInfo"."focalLength" AS "exifInfo_focalLength",
"exifInfo"."iso" AS "exifInfo_iso",
"exifInfo"."exposureTime" AS "exifInfo_exposureTime",
"exifInfo"."profileDescription" AS "exifInfo_profileDescription",
"exifInfo"."colorspace" AS "exifInfo_colorspace",
"exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
"exifInfo"."fps" AS "exifInfo_fps",
"stack"."id" AS "stack_id",
"stack"."primaryAssetId" AS "stack_primaryAssetId"
FROM
"assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
WHERE
"asset"."ownerId" = $1
AND ("asset"."fileCreatedAt", "asset"."id") < ($2, $3)
AND "asset"."updatedAt" <= $4
AND "asset"."isVisible" = true
ORDER BY
"asset"."fileCreatedAt" DESC,
"asset"."id" DESC
LIMIT
10
-- AssetRepository.getChangedDeltaSync
SELECT
"AssetEntity"."id" AS "AssetEntity_id",
"AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId",
"AssetEntity"."ownerId" AS "AssetEntity_ownerId",
"AssetEntity"."libraryId" AS "AssetEntity_libraryId",
"AssetEntity"."deviceId" AS "AssetEntity_deviceId",
"AssetEntity"."type" AS "AssetEntity_type",
"AssetEntity"."originalPath" AS "AssetEntity_originalPath",
"AssetEntity"."previewPath" AS "AssetEntity_previewPath",
"AssetEntity"."thumbnailPath" AS "AssetEntity_thumbnailPath",
"AssetEntity"."thumbhash" AS "AssetEntity_thumbhash",
"AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath",
"AssetEntity"."createdAt" AS "AssetEntity_createdAt",
"AssetEntity"."updatedAt" AS "AssetEntity_updatedAt",
"AssetEntity"."deletedAt" AS "AssetEntity_deletedAt",
"AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt",
"AssetEntity"."localDateTime" AS "AssetEntity_localDateTime",
"AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt",
"AssetEntity"."isFavorite" AS "AssetEntity_isFavorite",
"AssetEntity"."isArchived" AS "AssetEntity_isArchived",
"AssetEntity"."isExternal" AS "AssetEntity_isExternal",
"AssetEntity"."isReadOnly" AS "AssetEntity_isReadOnly",
"AssetEntity"."isOffline" AS "AssetEntity_isOffline",
"AssetEntity"."checksum" AS "AssetEntity_checksum",
"AssetEntity"."duration" AS "AssetEntity_duration",
"AssetEntity"."isVisible" AS "AssetEntity_isVisible",
"AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
"AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
"AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
"AssetEntity"."stackId" AS "AssetEntity_stackId",
"AssetEntity__AssetEntity_exifInfo"."assetId" AS "AssetEntity__AssetEntity_exifInfo_assetId",
"AssetEntity__AssetEntity_exifInfo"."description" AS "AssetEntity__AssetEntity_exifInfo_description",
"AssetEntity__AssetEntity_exifInfo"."exifImageWidth" AS "AssetEntity__AssetEntity_exifInfo_exifImageWidth",
"AssetEntity__AssetEntity_exifInfo"."exifImageHeight" AS "AssetEntity__AssetEntity_exifInfo_exifImageHeight",
"AssetEntity__AssetEntity_exifInfo"."fileSizeInByte" AS "AssetEntity__AssetEntity_exifInfo_fileSizeInByte",
"AssetEntity__AssetEntity_exifInfo"."orientation" AS "AssetEntity__AssetEntity_exifInfo_orientation",
"AssetEntity__AssetEntity_exifInfo"."dateTimeOriginal" AS "AssetEntity__AssetEntity_exifInfo_dateTimeOriginal",
"AssetEntity__AssetEntity_exifInfo"."modifyDate" AS "AssetEntity__AssetEntity_exifInfo_modifyDate",
"AssetEntity__AssetEntity_exifInfo"."timeZone" AS "AssetEntity__AssetEntity_exifInfo_timeZone",
"AssetEntity__AssetEntity_exifInfo"."latitude" AS "AssetEntity__AssetEntity_exifInfo_latitude",
"AssetEntity__AssetEntity_exifInfo"."longitude" AS "AssetEntity__AssetEntity_exifInfo_longitude",
"AssetEntity__AssetEntity_exifInfo"."projectionType" AS "AssetEntity__AssetEntity_exifInfo_projectionType",
"AssetEntity__AssetEntity_exifInfo"."city" AS "AssetEntity__AssetEntity_exifInfo_city",
"AssetEntity__AssetEntity_exifInfo"."livePhotoCID" AS "AssetEntity__AssetEntity_exifInfo_livePhotoCID",
"AssetEntity__AssetEntity_exifInfo"."autoStackId" AS "AssetEntity__AssetEntity_exifInfo_autoStackId",
"AssetEntity__AssetEntity_exifInfo"."state" AS "AssetEntity__AssetEntity_exifInfo_state",
"AssetEntity__AssetEntity_exifInfo"."country" AS "AssetEntity__AssetEntity_exifInfo_country",
"AssetEntity__AssetEntity_exifInfo"."make" AS "AssetEntity__AssetEntity_exifInfo_make",
"AssetEntity__AssetEntity_exifInfo"."model" AS "AssetEntity__AssetEntity_exifInfo_model",
"AssetEntity__AssetEntity_exifInfo"."lensModel" AS "AssetEntity__AssetEntity_exifInfo_lensModel",
"AssetEntity__AssetEntity_exifInfo"."fNumber" AS "AssetEntity__AssetEntity_exifInfo_fNumber",
"AssetEntity__AssetEntity_exifInfo"."focalLength" AS "AssetEntity__AssetEntity_exifInfo_focalLength",
"AssetEntity__AssetEntity_exifInfo"."iso" AS "AssetEntity__AssetEntity_exifInfo_iso",
"AssetEntity__AssetEntity_exifInfo"."exposureTime" AS "AssetEntity__AssetEntity_exifInfo_exposureTime",
"AssetEntity__AssetEntity_exifInfo"."profileDescription" AS "AssetEntity__AssetEntity_exifInfo_profileDescription",
"AssetEntity__AssetEntity_exifInfo"."colorspace" AS "AssetEntity__AssetEntity_exifInfo_colorspace",
"AssetEntity__AssetEntity_exifInfo"."bitsPerSample" AS "AssetEntity__AssetEntity_exifInfo_bitsPerSample",
"AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps",
"AssetEntity__AssetEntity_stack"."id" AS "AssetEntity__AssetEntity_stack_id",
"AssetEntity__AssetEntity_stack"."primaryAssetId" AS "AssetEntity__AssetEntity_stack_primaryAssetId"
FROM
"assets" "AssetEntity"
LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id"
LEFT JOIN "asset_stack" "AssetEntity__AssetEntity_stack" ON "AssetEntity__AssetEntity_stack"."id" = "AssetEntity"."stackId"
WHERE
(
("AssetEntity"."ownerId" IN ($1))
AND ("AssetEntity"."isVisible" = $2)
AND ("AssetEntity"."updatedAt" > $3)
)

59
server/src/repositories/asset.repository.ts
View file

@ -2,15 +2,18 @@ import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import path from 'node:path';
import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
import { AssetOrder } from 'src/entities/album.entity';
import { AlbumEntity, AssetOrder } from 'src/entities/album.entity';
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
import { AssetEntity, AssetType } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { PartnerEntity } from 'src/entities/partner.entity';
import { SmartInfoEntity } from 'src/entities/smart-info.entity';
import {
AssetBuilderOptions,
AssetCreate,
AssetDeltaSyncOptions,
AssetExploreFieldOptions,
AssetFullSyncOptions,
AssetPathEntity,
AssetStats,
AssetStatsOptions,
@ -39,6 +42,7 @@ import {
FindOptionsWhere,
In,
IsNull,
MoreThan,
Not,
Repository,
} from 'typeorm';
@ -61,6 +65,8 @@ export class AssetRepository implements IAssetRepository {
@InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
@InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository<AssetJobStatusEntity>,
@InjectRepository(SmartInfoEntity) private smartInfoRepository: Repository<SmartInfoEntity>,
@InjectRepository(PartnerEntity) private partnerRepository: Repository<PartnerEntity>,
@InjectRepository(AlbumEntity) private albumRepository: Repository<AlbumEntity>,
) {}
async upsertExif(exif: Partial<ExifEntity>): Promise<void> {
@ -781,4 +787,55 @@ export class AssetRepository implements IAssetRepository {
}) as AssetEntity,
);
}
@GenerateSql({
params: [
{
ownerId: DummyValue.UUID,
lastCreationDate: DummyValue.DATE,
lastId: DummyValue.STRING,
updatedUntil: DummyValue.DATE,
limit: 10,
},
],
})
getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]> {
const { ownerId, lastCreationDate, lastId, updatedUntil, limit } = options;
let builder = this.repository
.createQueryBuilder('asset')
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
.leftJoinAndSelect('asset.stack', 'stack')
.where('asset.ownerId = :ownerId', { ownerId });
if (lastCreationDate !== undefined && lastId !== undefined) {
builder = builder.andWhere('(asset.fileCreatedAt, asset.id) < (:lastCreationDate, :lastId)', {
lastCreationDate,
lastId,
});
}
return builder
.andWhere('asset.updatedAt <= :updatedUntil', { updatedUntil })
.andWhere('asset.isVisible = true')
.orderBy('asset.fileCreatedAt', 'DESC')
.addOrderBy('asset.id', 'DESC')
.limit(limit)
.withDeleted()
.getMany();
}
@GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE }] })
getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]> {
return this.repository.find({
where: {
ownerId: In(options.userIds),
isVisible: true,
updatedAt: MoreThan(options.updatedAfter),
},
relations: {
exifInfo: true,
stack: true,
},
take: options.limit,
withDeleted: true,
});
}
}

9
server/src/repositories/audit.repository.ts
View file

@ -2,24 +2,25 @@ import { InjectRepository } from '@nestjs/typeorm';
import { AuditEntity } from 'src/entities/audit.entity';
import { AuditSearch, IAuditRepository } from 'src/interfaces/audit.interface';
import { Instrumentation } from 'src/utils/instrumentation';
import { LessThan, MoreThan, Repository } from 'typeorm';
import { In, LessThan, MoreThan, Repository } from 'typeorm';
@Instrumentation()
export class AuditRepository implements IAuditRepository {
constructor(@InjectRepository(AuditEntity) private repository: Repository<AuditEntity>) {}
getAfter(since: Date, options: AuditSearch): Promise<AuditEntity[]> {
getAfter(since: Date, options: AuditSearch): Promise<string[]> {
return this.repository
.createQueryBuilder('audit')
.where({
createdAt: MoreThan(since),
action: options.action,
entityType: options.entityType,
ownerId: options.ownerId,
ownerId: In(options.userIds),
})
.distinctOn(['audit.entityId', 'audit.entityType'])
.orderBy('audit.entityId, audit.entityType, audit.createdAt', 'DESC')
.getMany();
.select('audit.entityId')
.getRawMany();
}
async removeBefore(before: Date): Promise<void> {

6
server/src/services/audit.service.spec.ts
View file

@ -61,13 +61,13 @@ describe(AuditService.name, () => {
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
action: DatabaseAction.DELETE,
ownerId: authStub.admin.user.id,
userIds: [authStub.admin.user.id],
entityType: EntityType.ASSET,
});
});
it('should get any new or updated assets and deleted ids', async () => {
auditMock.getAfter.mockResolvedValue([auditStub.delete]);
auditMock.getAfter.mockResolvedValue([auditStub.delete.entityId]);
const date = new Date();
await expect(sut.getDeletes(authStub.admin, { after: date, entityType: EntityType.ASSET })).resolves.toEqual({
@ -77,7 +77,7 @@ describe(AuditService.name, () => {
expect(auditMock.getAfter).toHaveBeenCalledWith(date, {
action: DatabaseAction.DELETE,
ownerId: authStub.admin.user.id,
userIds: [authStub.admin.user.id],
entityType: EntityType.ASSET,
});
});

4
server/src/services/audit.service.ts
View file

@ -53,7 +53,7 @@ export class AuditService {
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
const audits = await this.repository.getAfter(dto.after, {
ownerId: userId,
userIds: [userId],
entityType: dto.entityType,
action: DatabaseAction.DELETE,
});
@ -62,7 +62,7 @@ export class AuditService {
return {
needsFullSync: duration > AUDIT_LOG_MAX_DURATION,
ids: audits.map(({ entityId }) => entityId),
ids: audits,
};
}

2
server/src/services/index.ts
View file

@ -22,6 +22,7 @@ import { SharedLinkService } from 'src/services/shared-link.service';
import { SmartInfoService } from 'src/services/smart-info.service';
import { StorageTemplateService } from 'src/services/storage-template.service';
import { StorageService } from 'src/services/storage.service';
import { SyncService } from 'src/services/sync.service';
import { SystemConfigService } from 'src/services/system-config.service';
import { TagService } from 'src/services/tag.service';
import { TimelineService } from 'src/services/timeline.service';
@ -53,6 +54,7 @@ export const services = [
SmartInfoService,
StorageService,
StorageTemplateService,
SyncService,
SystemConfigService,
TagService,
TimelineService,

101
server/src/services/sync.service.spec.ts Normal file
View file

@ -0,0 +1,101 @@
import { mapAsset } from 'src/dtos/asset-response.dto';
import { AssetEntity } from 'src/entities/asset.entity';
import { IAccessRepository } from 'src/interfaces/access.interface';
import { IAssetRepository } from 'src/interfaces/asset.interface';
import { IAuditRepository } from 'src/interfaces/audit.interface';
import { IPartnerRepository } from 'src/interfaces/partner.interface';
import { SyncService } from 'src/services/sync.service';
import { assetStub } from 'test/fixtures/asset.stub';
import { authStub } from 'test/fixtures/auth.stub';
import { partnerStub } from 'test/fixtures/partner.stub';
import { newAccessRepositoryMock } from 'test/repositories/access.repository.mock';
import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
import { newAuditRepositoryMock } from 'test/repositories/audit.repository.mock';
import { newPartnerRepositoryMock } from 'test/repositories/partner.repository.mock';
const untilDate = new Date(2024);
const mapAssetOpts = { auth: authStub.user1, stripMetadata: false, withStack: true };
describe(SyncService.name, () => {
let sut: SyncService;
let accessMock: jest.Mocked<IAccessRepository>;
let assetMock: jest.Mocked<IAssetRepository>;
let partnerMock: jest.Mocked<IPartnerRepository>;
let auditMock: jest.Mocked<IAuditRepository>;
beforeEach(() => {
partnerMock = newPartnerRepositoryMock();
assetMock = newAssetRepositoryMock();
accessMock = newAccessRepositoryMock();
auditMock = newAuditRepositoryMock();
sut = new SyncService(accessMock, assetMock, partnerMock, auditMock);
});
it('should exist', () => {
expect(sut).toBeDefined();
});
describe('getAllAssetsForUserFullSync', () => {
it('should return a list of all assets owned by the user', async () => {
assetMock.getAllForUserFullSync.mockResolvedValue([assetStub.external, assetStub.hasEncodedVideo]);
await expect(
sut.getAllAssetsForUserFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate }),
).resolves.toEqual([
mapAsset(assetStub.external, mapAssetOpts),
mapAsset(assetStub.hasEncodedVideo, mapAssetOpts),
]);
expect(assetMock.getAllForUserFullSync).toHaveBeenCalledWith({
ownerId: authStub.user1.user.id,
updatedUntil: untilDate,
limit: 2,
});
});
});
describe('getChangesForDeltaSync', () => {
it('should return a response requiring a full sync when partners are out of sync', async () => {
partnerMock.getAll.mockResolvedValue([partnerStub.adminToUser1]);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
});
it('should return a response requiring a full sync when last sync was too long ago', async () => {
partnerMock.getAll.mockResolvedValue([]);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
});
it('should return a response requiring a full sync when there are too many changes', async () => {
partnerMock.getAll.mockResolvedValue([]);
assetMock.getChangedDeltaSync.mockResolvedValue(
Array.from<AssetEntity>({ length: 10_000 }).fill(assetStub.image),
);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1);
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
});
it('should return a response with changes and deletions', async () => {
partnerMock.getAll.mockResolvedValue([]);
assetMock.getChangedDeltaSync.mockResolvedValue([assetStub.image1]);
auditMock.getAfter.mockResolvedValue([assetStub.external.id]);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({
needsFullSync: false,
upserted: [mapAsset(assetStub.image1, mapAssetOpts)],
deleted: [assetStub.external.id],
});
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1);
expect(auditMock.getAfter).toHaveBeenCalledTimes(1);
});
});
});

77
server/src/services/sync.service.ts Normal file
View file

@ -0,0 +1,77 @@
import { Inject } from '@nestjs/common';
import _ from 'lodash';
import { DateTime } from 'luxon';
import { AUDIT_LOG_MAX_DURATION } from 'src/constants';
import { AccessCore, Permission } from 'src/cores/access.core';
import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto';
import { DatabaseAction, EntityType } from 'src/entities/audit.entity';
import { IAccessRepository } from 'src/interfaces/access.interface';
import { IAssetRepository } from 'src/interfaces/asset.interface';
import { IAuditRepository } from 'src/interfaces/audit.interface';
import { IPartnerRepository } from 'src/interfaces/partner.interface';
export class SyncService {
private access: AccessCore;
constructor(
@Inject(IAccessRepository) accessRepository: IAccessRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(IPartnerRepository) private partnerRepository: IPartnerRepository,
@Inject(IAuditRepository) private auditRepository: IAuditRepository,
) {
this.access = AccessCore.create(accessRepository);
}
async getAllAssetsForUserFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
const userId = dto.userId || auth.user.id;
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
const assets = await this.assetRepository.getAllForUserFullSync({
ownerId: userId,
lastCreationDate: dto.lastCreationDate,
updatedUntil: dto.updatedUntil,
lastId: dto.lastId,
limit: dto.limit,
});
const options = { auth, stripMetadata: false, withStack: true };
return assets.map((a) => mapAsset(a, options));
}
async getChangesForDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds);
const partner = await this.partnerRepository.getAll(auth.user.id);
const userIds = [auth.user.id, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)];
userIds.sort();
dto.userIds.sort();
const duration = DateTime.now().diff(DateTime.fromJSDate(dto.updatedAfter));
if (!_.isEqual(userIds, dto.userIds) || duration > AUDIT_LOG_MAX_DURATION) {
// app does not have the correct partners synced
// or app has not synced in the last 100 days
return { needsFullSync: true, deleted: [], upserted: [] };
}
const limit = 10_000;
const upserted = await this.assetRepository.getChangedDeltaSync({ limit, updatedAfter: dto.updatedAfter, userIds });
if (upserted.length === limit) {
// too many changes -> do a full sync (paginated) instead
return { needsFullSync: true, deleted: [], upserted: [] };
}
const deleted = await this.auditRepository.getAfter(dto.updatedAfter, {
userIds: userIds,
entityType: EntityType.ASSET,
action: DatabaseAction.DELETE,
});
const options = { auth, stripMetadata: false, withStack: true };
const result = {
needsFullSync: false,
upserted: upserted.map((a) => mapAsset(a, options)),
deleted,
};
return result;
}
}

2
server/test/repositories/asset.repository.mock.ts
View file

@ -35,5 +35,7 @@ export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
getAssetIdByCity: jest.fn(),
getAssetIdByTag: jest.fn(),
searchMetadata: jest.fn(),
getAllForUserFullSync: jest.fn(),
getChangedDeltaSync: jest.fn(),
};
};