diff --git a/cli/src/api/open-api/api.ts b/cli/src/api/open-api/api.ts index 5a2445846a..9265d439fc 100644 --- a/cli/src/api/open-api/api.ts +++ b/cli/src/api/open-api/api.ts @@ -6695,16 +6695,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration }, /** * Get all AssetEntity belong to the user + * @param {number} [skip] + * @param {number} [take] * @param {string} [userId] * @param {boolean} [isFavorite] * @param {boolean} [isArchived] - * @param {number} [skip] * @param {string} [updatedAfter] + * @param {string} [updatedBefore] * @param {string} [ifNoneMatch] ETag of data already cached on the client * @param {*} [options] Override http request option. * @throws {RequiredError} */ - getAllAssets: async (userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise => { + getAllAssets: async (skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise => { const localVarPath = `/asset`; // use dummy base URL string because the URL constructor only accepts absolute URLs. const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); @@ -6726,6 +6728,14 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration // http bearer authentication required await setBearerAuthToObject(localVarHeaderParameter, configuration) + if (skip !== undefined) { + localVarQueryParameter['skip'] = skip; + } + + if (take !== undefined) { + localVarQueryParameter['take'] = take; + } + if (userId !== undefined) { localVarQueryParameter['userId'] = userId; } @@ -6738,16 +6748,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration localVarQueryParameter['isArchived'] = isArchived; } - if (skip !== undefined) { - localVarQueryParameter['skip'] = skip; - } - if (updatedAfter !== undefined) { localVarQueryParameter['updatedAfter'] = (updatedAfter as any instanceof Date) ? (updatedAfter as any).toISOString() : updatedAfter; } + if (updatedBefore !== undefined) { + localVarQueryParameter['updatedBefore'] = (updatedBefore as any instanceof Date) ? + (updatedBefore as any).toISOString() : + updatedBefore; + } + if (ifNoneMatch != null) { localVarHeaderParameter['if-none-match'] = String(ifNoneMatch); } @@ -8066,17 +8078,19 @@ export const AssetApiFp = function(configuration?: Configuration) { }, /** * Get all AssetEntity belong to the user + * @param {number} [skip] + * @param {number} [take] * @param {string} [userId] * @param {boolean} [isFavorite] * @param {boolean} [isArchived] - * @param {number} [skip] * @param {string} [updatedAfter] + * @param {string} [updatedBefore] * @param {string} [ifNoneMatch] ETag of data already cached on the client * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async getAllAssets(userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { - const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(userId, isFavorite, isArchived, skip, updatedAfter, ifNoneMatch, options); + async getAllAssets(skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(skip, take, userId, isFavorite, isArchived, updatedAfter, updatedBefore, ifNoneMatch, options); return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); }, /** @@ -8421,7 +8435,7 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath * @throws {RequiredError} */ getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig): AxiosPromise> { - return localVarFp.getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath)); + return localVarFp.getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath)); }, /** * Get a single asset\'s information @@ -8719,6 +8733,20 @@ export interface AssetApiDownloadFileRequest { * @interface AssetApiGetAllAssetsRequest */ export interface AssetApiGetAllAssetsRequest { + /** + * + * @type {number} + * @memberof AssetApiGetAllAssets + */ + readonly skip?: number + + /** + * + * @type {number} + * @memberof AssetApiGetAllAssets + */ + readonly take?: number + /** * * @type {string} @@ -8742,17 +8770,17 @@ export interface AssetApiGetAllAssetsRequest { /** * - * @type {number} + * @type {string} * @memberof AssetApiGetAllAssets */ - readonly skip?: number + readonly updatedAfter?: string /** * * @type {string} * @memberof AssetApiGetAllAssets */ - readonly updatedAfter?: string + readonly updatedBefore?: string /** * ETag of data already cached on the client @@ -9430,7 +9458,7 @@ export class AssetApi extends BaseAPI { * @memberof AssetApi */ public getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig) { - return AssetApiFp(this.configuration).getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath)); + return AssetApiFp(this.configuration).getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath)); } /** diff --git a/mobile/lib/shared/services/asset.service.dart b/mobile/lib/shared/services/asset.service.dart index 488395b16a..8b1ee6a33f 100644 --- a/mobile/lib/shared/services/asset.service.dart +++ b/mobile/lib/shared/services/asset.service.dart @@ -62,20 +62,31 @@ class AssetService { /// Returns `null` if the server state did not change, else list of assets Future?> _getRemoteAssets(User user) async { + const int chunkSize = 5000; try { - final List? assets = - await _apiService.assetApi.getAllAssets( - userId: user.id, - ); - if (assets == null) { - return null; - } else if (assets.isNotEmpty && assets.first.ownerId != user.id) { - log.warning("Make sure that server and app versions match!" - " The server returned assets for user ${assets.first.ownerId}" - " while requesting assets of user ${user.id}"); - return null; + final DateTime now = DateTime.now().toUtc(); + final List allAssets = []; + for (int i = 0;; i += chunkSize) { + final List? assets = + await _apiService.assetApi.getAllAssets( + userId: user.id, + // updatedBefore is important! without it we could + // a) get the same Asset multiple times in different versions (when + // the asset is modified while the chunks are loaded from the server) + // b) miss assets when new assets are inserted in between the calls + updatedBefore: now, + skip: i, + take: chunkSize, + ); + if (assets == null) { + return null; + } + allAssets.addAll(assets.map(Asset.remote)); + if (assets.length < chunkSize) { + break; + } } - return assets.map(Asset.remote).toList(); + return allAssets; } catch (error, stack) { log.severe( 'Error while getting remote assets: ${error.toString()}', diff --git a/mobile/lib/shared/services/sync.service.dart b/mobile/lib/shared/services/sync.service.dart index 34d84401aa..19fc076e48 100644 --- a/mobile/lib/shared/services/sync.service.dart +++ b/mobile/lib/shared/services/sync.service.dart @@ -197,7 +197,7 @@ class SyncService { User user, FutureOr?> Function(User user) loadAssets, ) async { - final DateTime now = DateTime.now(); + final DateTime now = DateTime.now().toUtc(); final List? remote = await loadAssets(user); if (remote == null) { return false; @@ -210,6 +210,10 @@ class SyncService { assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); remote.sort(Asset.compareByChecksum); + + // filter our duplicates that might be introduced by the chunked retrieval + remote.uniqueConsecutive(compare: Asset.compareByChecksum); + final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true); if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) { await _updateUserAssetsETag(user, now); @@ -759,6 +763,12 @@ class SyncService { final List toAdd = []; final List toUpdate = []; final List toRemove = []; + if (assets.isEmpty || inDb.isEmpty) { + // fast path for trivial cases: halfes memory usage during initial sync + return assets.isEmpty + ? (toAdd, toUpdate, inDb) // remove all from DB + : (assets, toUpdate, toRemove); // add all assets + } diffSortedListsSync( inDb, assets, diff --git a/mobile/openapi/doc/AssetApi.md b/mobile/openapi/doc/AssetApi.md index e072c3ded9..8118419478 100644 Binary files a/mobile/openapi/doc/AssetApi.md and b/mobile/openapi/doc/AssetApi.md differ diff --git a/mobile/openapi/lib/api/asset_api.dart b/mobile/openapi/lib/api/asset_api.dart index 4b74d29333..0d3c2bfe89 100644 Binary files a/mobile/openapi/lib/api/asset_api.dart and b/mobile/openapi/lib/api/asset_api.dart differ diff --git a/mobile/openapi/test/asset_api_test.dart b/mobile/openapi/test/asset_api_test.dart index 179b23bbab..e241c34ca9 100644 Binary files a/mobile/openapi/test/asset_api_test.dart and b/mobile/openapi/test/asset_api_test.dart differ diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index 1fb05fbcf5..05aec878ad 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -914,6 +914,22 @@ "description": "Get all AssetEntity belong to the user", "operationId": "getAllAssets", "parameters": [ + { + "name": "skip", + "required": false, + "in": "query", + "schema": { + "type": "integer" + } + }, + { + "name": "take", + "required": false, + "in": "query", + "schema": { + "type": "integer" + } + }, { "name": "userId", "required": false, @@ -940,15 +956,16 @@ } }, { - "name": "skip", + "name": "updatedAfter", "required": false, "in": "query", "schema": { - "type": "number" + "format": "date-time", + "type": "string" } }, { - "name": "updatedAfter", + "name": "updatedBefore", "required": false, "in": "query", "schema": { diff --git a/server/src/immich/api-v1/asset/asset-repository.ts b/server/src/immich/api-v1/asset/asset-repository.ts index 9dac7e604e..13cf6bf17c 100644 --- a/server/src/immich/api-v1/asset/asset-repository.ts +++ b/server/src/immich/api-v1/asset/asset-repository.ts @@ -1,8 +1,8 @@ import { AssetCreate } from '@app/domain'; import { AssetEntity } from '@app/infra/entities'; +import OptionalBetween from '@app/infra/utils/optional-between.util'; import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { MoreThan } from 'typeorm'; import { In } from 'typeorm/find-options/operator/In'; import { Repository } from 'typeorm/repository/Repository'; import { AssetSearchDto } from './dto/asset-search.dto'; @@ -129,7 +129,7 @@ export class AssetRepository implements IAssetRepository { isVisible: true, isFavorite: dto.isFavorite, isArchived: dto.isArchived, - updatedAt: dto.updatedAfter ? MoreThan(dto.updatedAfter) : undefined, + updatedAt: OptionalBetween(dto.updatedAfter, dto.updatedBefore), }, relations: { exifInfo: true, @@ -137,6 +137,7 @@ export class AssetRepository implements IAssetRepository { stack: true, }, skip: dto.skip || 0, + take: dto.take, order: { fileCreatedAt: 'DESC', }, diff --git a/server/src/immich/api-v1/asset/dto/asset-search.dto.ts b/server/src/immich/api-v1/asset/dto/asset-search.dto.ts index 3067edebe1..d73856ab9a 100644 --- a/server/src/immich/api-v1/asset/dto/asset-search.dto.ts +++ b/server/src/immich/api-v1/asset/dto/asset-search.dto.ts @@ -1,7 +1,7 @@ import { Optional, toBoolean } from '@app/domain'; import { ApiProperty } from '@nestjs/swagger'; import { Transform, Type } from 'class-transformer'; -import { IsBoolean, IsDate, IsNotEmpty, IsNumber, IsUUID } from 'class-validator'; +import { IsBoolean, IsDate, IsInt, IsNotEmpty, IsUUID } from 'class-validator'; export class AssetSearchDto { @Optional() @@ -17,9 +17,17 @@ export class AssetSearchDto { isArchived?: boolean; @Optional() - @IsNumber() + @IsInt() + @Type(() => Number) + @ApiProperty({ type: 'integer' }) skip?: number; + @Optional() + @IsInt() + @Type(() => Number) + @ApiProperty({ type: 'integer' }) + take?: number; + @Optional() @IsUUID('4') @ApiProperty({ format: 'uuid' }) @@ -29,4 +37,9 @@ export class AssetSearchDto { @IsDate() @Type(() => Date) updatedAfter?: Date; + + @Optional() + @IsDate() + @Type(() => Date) + updatedBefore?: Date; } diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index 5a2445846a..9265d439fc 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -6695,16 +6695,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration }, /** * Get all AssetEntity belong to the user + * @param {number} [skip] + * @param {number} [take] * @param {string} [userId] * @param {boolean} [isFavorite] * @param {boolean} [isArchived] - * @param {number} [skip] * @param {string} [updatedAfter] + * @param {string} [updatedBefore] * @param {string} [ifNoneMatch] ETag of data already cached on the client * @param {*} [options] Override http request option. * @throws {RequiredError} */ - getAllAssets: async (userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise => { + getAllAssets: async (skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise => { const localVarPath = `/asset`; // use dummy base URL string because the URL constructor only accepts absolute URLs. const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); @@ -6726,6 +6728,14 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration // http bearer authentication required await setBearerAuthToObject(localVarHeaderParameter, configuration) + if (skip !== undefined) { + localVarQueryParameter['skip'] = skip; + } + + if (take !== undefined) { + localVarQueryParameter['take'] = take; + } + if (userId !== undefined) { localVarQueryParameter['userId'] = userId; } @@ -6738,16 +6748,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration localVarQueryParameter['isArchived'] = isArchived; } - if (skip !== undefined) { - localVarQueryParameter['skip'] = skip; - } - if (updatedAfter !== undefined) { localVarQueryParameter['updatedAfter'] = (updatedAfter as any instanceof Date) ? (updatedAfter as any).toISOString() : updatedAfter; } + if (updatedBefore !== undefined) { + localVarQueryParameter['updatedBefore'] = (updatedBefore as any instanceof Date) ? + (updatedBefore as any).toISOString() : + updatedBefore; + } + if (ifNoneMatch != null) { localVarHeaderParameter['if-none-match'] = String(ifNoneMatch); } @@ -8066,17 +8078,19 @@ export const AssetApiFp = function(configuration?: Configuration) { }, /** * Get all AssetEntity belong to the user + * @param {number} [skip] + * @param {number} [take] * @param {string} [userId] * @param {boolean} [isFavorite] * @param {boolean} [isArchived] - * @param {number} [skip] * @param {string} [updatedAfter] + * @param {string} [updatedBefore] * @param {string} [ifNoneMatch] ETag of data already cached on the client * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async getAllAssets(userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { - const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(userId, isFavorite, isArchived, skip, updatedAfter, ifNoneMatch, options); + async getAllAssets(skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(skip, take, userId, isFavorite, isArchived, updatedAfter, updatedBefore, ifNoneMatch, options); return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); }, /** @@ -8421,7 +8435,7 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath * @throws {RequiredError} */ getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig): AxiosPromise> { - return localVarFp.getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath)); + return localVarFp.getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath)); }, /** * Get a single asset\'s information @@ -8719,6 +8733,20 @@ export interface AssetApiDownloadFileRequest { * @interface AssetApiGetAllAssetsRequest */ export interface AssetApiGetAllAssetsRequest { + /** + * + * @type {number} + * @memberof AssetApiGetAllAssets + */ + readonly skip?: number + + /** + * + * @type {number} + * @memberof AssetApiGetAllAssets + */ + readonly take?: number + /** * * @type {string} @@ -8742,17 +8770,17 @@ export interface AssetApiGetAllAssetsRequest { /** * - * @type {number} + * @type {string} * @memberof AssetApiGetAllAssets */ - readonly skip?: number + readonly updatedAfter?: string /** * * @type {string} * @memberof AssetApiGetAllAssets */ - readonly updatedAfter?: string + readonly updatedBefore?: string /** * ETag of data already cached on the client @@ -9430,7 +9458,7 @@ export class AssetApi extends BaseAPI { * @memberof AssetApi */ public getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig) { - return AssetApiFp(this.configuration).getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath)); + return AssetApiFp(this.configuration).getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath)); } /**