1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-01-27 22:22:45 +01:00

fix(server): proper asset sync ()

* fix(server,mobile): proper asset sync

* fix CI issues

* only use id instead of createdAt+id

* remove createdAt index

* fix typo

* cleanup createdAt usage

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Fynn Petersen-Frey 2024-06-09 21:19:28 +02:00 committed by GitHub
parent 69795a3763
commit 972c66d467
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 37 additions and 103 deletions

View file

@ -101,7 +101,6 @@ class AssetService {
const int chunkSize = 10000; const int chunkSize = 10000;
try { try {
final List<Asset> allAssets = []; final List<Asset> allAssets = [];
DateTime? lastCreationDate;
String? lastId; String? lastId;
// will break on error or once all assets are loaded // will break on error or once all assets are loaded
while (true) { while (true) {
@ -109,15 +108,17 @@ class AssetService {
limit: chunkSize, limit: chunkSize,
updatedUntil: until, updatedUntil: until,
lastId: lastId, lastId: lastId,
lastCreationDate: lastCreationDate,
userId: user.id, userId: user.id,
); );
log.fine("Requesting $chunkSize assets from $lastId");
final List<AssetResponseDto>? assets = final List<AssetResponseDto>? assets =
await _apiService.syncApi.getFullSyncForUser(dto); await _apiService.syncApi.getFullSyncForUser(dto);
if (assets == null) return null; if (assets == null) return null;
log.fine(
"Received ${assets.length} assets from ${assets.firstOrNull?.id} to ${assets.lastOrNull?.id}",
);
allAssets.addAll(assets.map(Asset.remote)); allAssets.addAll(assets.map(Asset.remote));
if (assets.isEmpty) break; if (assets.length != chunkSize) break;
lastCreationDate = assets.last.fileCreatedAt;
lastId = assets.last.id; lastId = assets.last.id;
} }
return allAssets; return allAssets;

Binary file not shown.

View file

@ -7432,10 +7432,6 @@
}, },
"AssetFullSyncDto": { "AssetFullSyncDto": {
"properties": { "properties": {
"lastCreationDate": {
"format": "date-time",
"type": "string"
},
"lastId": { "lastId": {
"format": "uuid", "format": "uuid",
"type": "string" "type": "string"

View file

@ -904,7 +904,6 @@ export type AssetDeltaSyncResponseDto = {
upserted: AssetResponseDto[]; upserted: AssetResponseDto[];
}; };
export type AssetFullSyncDto = { export type AssetFullSyncDto = {
lastCreationDate?: string;
lastId?: string; lastId?: string;
limit: number; limit: number;
updatedUntil: string; updatedUntil: string;

View file

@ -127,10 +127,10 @@ export function mapAsset(entity: AssetEntity, options: AssetMapOptions = {}): As
stackParentId: withStack ? entity.stack?.primaryAssetId : undefined, stackParentId: withStack ? entity.stack?.primaryAssetId : undefined,
stack: withStack stack: withStack
? entity.stack?.assets ? entity.stack?.assets
.filter((a) => a.id !== entity.stack?.primaryAssetId) ?.filter((a) => a.id !== entity.stack?.primaryAssetId)
.map((a) => mapAsset(a, { stripMetadata, auth: options.auth })) ?.map((a) => mapAsset(a, { stripMetadata, auth: options.auth }))
: undefined, : undefined,
stackCount: entity.stack?.assets?.length ?? null, stackCount: entity.stack?.assetCount ?? entity.stack?.assets?.length ?? null,
isOffline: entity.isOffline, isOffline: entity.isOffline,
hasMetadata: true, hasMetadata: true,
duplicateId: entity.duplicateId, duplicateId: entity.duplicateId,

View file

@ -7,9 +7,6 @@ export class AssetFullSyncDto {
@ValidateUUID({ optional: true }) @ValidateUUID({ optional: true })
lastId?: string; lastId?: string;
@ValidateDate({ optional: true })
lastCreationDate?: Date;
@ValidateDate() @ValidateDate()
updatedUntil!: Date; updatedUntil!: Date;

View file

@ -16,4 +16,6 @@ export class AssetStackEntity {
@Column({ nullable: false }) @Column({ nullable: false })
primaryAssetId!: string; primaryAssetId!: string;
assetCount?: number;
} }

View file

@ -122,7 +122,6 @@ export interface AssetExploreOptions extends AssetExploreFieldOptions {
export interface AssetFullSyncOptions { export interface AssetFullSyncOptions {
ownerId: string; ownerId: string;
lastCreationDate?: Date;
lastId?: string; lastId?: string;
updatedUntil: Date; updatedUntil: Date;
limit: number; limit: number;

View file

@ -1049,50 +1049,18 @@ SELECT
"exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
"exifInfo"."fps" AS "exifInfo_fps", "exifInfo"."fps" AS "exifInfo_fps",
"stack"."id" AS "stack_id", "stack"."id" AS "stack_id",
"stack"."primaryAssetId" AS "stack_primaryAssetId", "stack"."primaryAssetId" AS "stack_primaryAssetId"
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."previewPath" AS "stackedAssets_previewPath",
"stackedAssets"."thumbnailPath" AS "stackedAssets_thumbnailPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM FROM
"assets" "asset" "assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE WHERE
"asset"."isVisible" = true "asset"."isVisible" = true
AND "asset"."ownerId" IN ($1) AND "asset"."ownerId" IN ($1)
AND ("asset"."fileCreatedAt", "asset"."id") < ($2, $3) AND "asset"."id" > $2
AND "asset"."updatedAt" <= $4 AND "asset"."updatedAt" <= $3
ORDER BY ORDER BY
"asset"."fileCreatedAt" DESC, "asset"."id" ASC
"asset"."id" DESC
LIMIT LIMIT
10 10
@ -1156,42 +1124,11 @@ SELECT
"exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample", "exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
"exifInfo"."fps" AS "exifInfo_fps", "exifInfo"."fps" AS "exifInfo_fps",
"stack"."id" AS "stack_id", "stack"."id" AS "stack_id",
"stack"."primaryAssetId" AS "stack_primaryAssetId", "stack"."primaryAssetId" AS "stack_primaryAssetId"
"stackedAssets"."id" AS "stackedAssets_id",
"stackedAssets"."deviceAssetId" AS "stackedAssets_deviceAssetId",
"stackedAssets"."ownerId" AS "stackedAssets_ownerId",
"stackedAssets"."libraryId" AS "stackedAssets_libraryId",
"stackedAssets"."deviceId" AS "stackedAssets_deviceId",
"stackedAssets"."type" AS "stackedAssets_type",
"stackedAssets"."originalPath" AS "stackedAssets_originalPath",
"stackedAssets"."previewPath" AS "stackedAssets_previewPath",
"stackedAssets"."thumbnailPath" AS "stackedAssets_thumbnailPath",
"stackedAssets"."thumbhash" AS "stackedAssets_thumbhash",
"stackedAssets"."encodedVideoPath" AS "stackedAssets_encodedVideoPath",
"stackedAssets"."createdAt" AS "stackedAssets_createdAt",
"stackedAssets"."updatedAt" AS "stackedAssets_updatedAt",
"stackedAssets"."deletedAt" AS "stackedAssets_deletedAt",
"stackedAssets"."fileCreatedAt" AS "stackedAssets_fileCreatedAt",
"stackedAssets"."localDateTime" AS "stackedAssets_localDateTime",
"stackedAssets"."fileModifiedAt" AS "stackedAssets_fileModifiedAt",
"stackedAssets"."isFavorite" AS "stackedAssets_isFavorite",
"stackedAssets"."isArchived" AS "stackedAssets_isArchived",
"stackedAssets"."isExternal" AS "stackedAssets_isExternal",
"stackedAssets"."isOffline" AS "stackedAssets_isOffline",
"stackedAssets"."checksum" AS "stackedAssets_checksum",
"stackedAssets"."duration" AS "stackedAssets_duration",
"stackedAssets"."isVisible" AS "stackedAssets_isVisible",
"stackedAssets"."livePhotoVideoId" AS "stackedAssets_livePhotoVideoId",
"stackedAssets"."originalFileName" AS "stackedAssets_originalFileName",
"stackedAssets"."sidecarPath" AS "stackedAssets_sidecarPath",
"stackedAssets"."stackId" AS "stackedAssets_stackId",
"stackedAssets"."duplicateId" AS "stackedAssets_duplicateId"
FROM FROM
"assets" "asset" "assets" "asset"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id" LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "asset"."id"
LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId" LEFT JOIN "asset_stack" "stack" ON "stack"."id" = "asset"."stackId"
LEFT JOIN "assets" "stackedAssets" ON "stackedAssets"."stackId" = "stack"."id"
AND ("stackedAssets"."deletedAt" IS NULL)
WHERE WHERE
"asset"."isVisible" = true "asset"."isVisible" = true
AND "asset"."ownerId" IN ($1) AND "asset"."ownerId" IN ($1)

View file

@ -763,36 +763,40 @@ export class AssetRepository implements IAssetRepository {
], ],
}) })
getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]> { getAllForUserFullSync(options: AssetFullSyncOptions): Promise<AssetEntity[]> {
const { ownerId, lastCreationDate, lastId, updatedUntil, limit } = options; const { ownerId, lastId, updatedUntil, limit } = options;
const builder = this.getBuilder({ const builder = this.getBuilder({
userIds: [ownerId], userIds: [ownerId],
exifInfo: true, // also joins stack information exifInfo: false, // need to do this manually because `exifInfo: true` also loads stacked assets messing with `limit`
withStacked: false, // return all assets individually as expected by the app withStacked: false, // return all assets individually as expected by the app
}); })
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
.leftJoinAndSelect('asset.stack', 'stack')
.loadRelationCountAndMap('stack.assetCount', 'stack.assets', 'stackedAssetsCount');
if (lastCreationDate !== undefined && lastId !== undefined) { if (lastId !== undefined) {
builder.andWhere('(asset.fileCreatedAt, asset.id) < (:lastCreationDate, :lastId)', { builder.andWhere('asset.id > :lastId', { lastId });
lastCreationDate,
lastId,
});
} }
builder
return builder
.andWhere('asset.updatedAt <= :updatedUntil', { updatedUntil }) .andWhere('asset.updatedAt <= :updatedUntil', { updatedUntil })
.orderBy('asset.fileCreatedAt', 'DESC') .orderBy('asset.id', 'ASC')
.addOrderBy('asset.id', 'DESC') .limit(limit) // cannot use `take` for performance reasons
.limit(limit) .withDeleted();
.withDeleted() return builder.getMany();
.getMany();
} }
@GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE }] }) @GenerateSql({ params: [{ userIds: [DummyValue.UUID], updatedAfter: DummyValue.DATE }] })
getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]> { getChangedDeltaSync(options: AssetDeltaSyncOptions): Promise<AssetEntity[]> {
const builder = this.getBuilder({ userIds: options.userIds, exifInfo: true, withStacked: false }) const builder = this.getBuilder({
userIds: options.userIds,
exifInfo: false, // need to do this manually because `exifInfo: true` also loads stacked assets messing with `limit`
withStacked: false, // return all assets individually as expected by the app
})
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
.leftJoinAndSelect('asset.stack', 'stack')
.loadRelationCountAndMap('stack.assetCount', 'stack.assets', 'stackedAssetsCount')
.andWhere({ updatedAt: MoreThan(options.updatedAfter) }) .andWhere({ updatedAt: MoreThan(options.updatedAfter) })
.limit(options.limit) .limit(options.limit) // cannot use `take` for performance reasons
.withDeleted(); .withDeleted();
return builder.getMany(); return builder.getMany();
} }
} }

View file

@ -32,7 +32,6 @@ export class SyncService {
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId); await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
const assets = await this.assetRepository.getAllForUserFullSync({ const assets = await this.assetRepository.getAllForUserFullSync({
ownerId: userId, ownerId: userId,
lastCreationDate: dto.lastCreationDate,
updatedUntil: dto.updatedUntil, updatedUntil: dto.updatedUntil,
lastId: dto.lastId, lastId: dto.lastId,
limit: dto.limit, limit: dto.limit,