diff --git a/server/src/interfaces/asset.interface.ts b/server/src/interfaces/asset.interface.ts index 35bb41aac0..b388a23392 100644 --- a/server/src/interfaces/asset.interface.ts +++ b/server/src/interfaces/asset.interface.ts @@ -178,9 +178,9 @@ export interface IAssetRepository { getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>; getRandom(userIds: string[], count: number): Promise<AssetEntity[]>; getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | null>; - getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>; deleteAll(ownerId: string): Promise<void>; getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>; + getAllInLibrary(pagination: PaginationOptions, libraryId: string): Paginated<AssetEntity>; getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>; getLivePhotoCount(motionId: string): Promise<number>; updateAll(ids: string[], options: Partial<AssetUpdateAllOptions>): Promise<void>; @@ -201,4 +201,5 @@ export interface IAssetRepository { upsertFiles(files: UpsertFileOptions[]): Promise<void>; updateOffline(library: LibraryEntity): Promise<UpdateResult>; getNewPaths(libraryId: string, paths: string[]): Promise<string[]>; + getAssetCount(id: string, options: AssetSearchOptions): Promise<number | undefined>; } diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index da619f0d35..6f8d81408e 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -197,14 +197,6 @@ export class AssetRepository implements IAssetRepository { return this.getAll(pagination, { ...options, userIds: [userId] }); } - @GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] }) - getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null> { - return this.repository.findOne({ - where: { library: { id: libraryId }, originalPath }, - withDeleted: true, - }); - } - @GenerateSql({ params: [DummyValue.UUID, [DummyValue.STRING]] }) @ChunkedArray({ paramIndex: 1 }) async getPathsNotInLibrary(libraryId: string, originalPaths: string[]): Promise<string[]> { @@ -231,6 +223,20 @@ export class AssetRepository implements IAssetRepository { }); } + getAllInLibrary(pagination: PaginationOptions, libraryId: string): Paginated<AssetEntity> { + const builder = this.repository + .createQueryBuilder('asset') + .select('asset.id') + .where('asset.libraryId = :libraryId', { libraryId }) + .withDeleted(); + + return paginatedBuilder<AssetEntity>(builder, { + mode: PaginationMode.SKIP_TAKE, + skip: pagination.skip, + take: pagination.take, + }); + } + /** * Get assets by device's Id on the database * @param ownerId @@ -779,4 +785,10 @@ export class AssetRepository implements IAssetRepository { .query(rawSql, [paths, libraryId]) .then((result) => result.map((row: { path: string }) => row.path)); } + + async getAssetCount(id: string, options: AssetSearchOptions = {}): Promise<number | undefined> { + let builder = this.repository.createQueryBuilder('asset').leftJoinAndSelect('asset.files', 'files'); + builder = searchAssetBuilder(builder, options); + return builder.getCount(); + } } diff --git a/server/src/repositories/trash.repository.ts b/server/src/repositories/trash.repository.ts index d24f4f709a..6ca6f4351f 100644 --- a/server/src/repositories/trash.repository.ts +++ b/server/src/repositories/trash.repository.ts @@ -14,6 +14,7 @@ export class TrashRepository implements ITrashRepository { .createQueryBuilder('asset') .select('asset.id') .where({ status: AssetStatus.DELETED }) + .orWhere({ isOffline: true }) .withDeleted(), pagination, ); @@ -34,10 +35,13 @@ export class TrashRepository implements ITrashRepository { } async empty(userId: string): Promise<number> { - const result = await this.assetRepository.update( - { ownerId: userId, status: AssetStatus.TRASHED }, - { status: AssetStatus.DELETED }, - ); + const result = await this.assetRepository + .createQueryBuilder() + .update(AssetEntity) + .set({ status: AssetStatus.DELETED }) + .where({ ownerId: userId, status: AssetStatus.TRASHED }) + .orWhere({ ownerId: userId, isOffline: true }) + .execute(); return result.affected || 0; } diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index dc391d241d..7fbaa40f6f 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -388,15 +388,14 @@ export class LibraryService extends BaseService { this.logger.log(`Starting to scan library ${id}`); - await this.jobRepository.queueAll([ - { - name: JobName.LIBRARY_QUEUE_SYNC_FILES, - data: { - id, - }, + await this.jobRepository.queue({ + name: JobName.LIBRARY_QUEUE_SYNC_FILES, + data: { + id, }, - { name: JobName.LIBRARY_QUEUE_SYNC_ASSETS, data: { id } }, - ]); + }); + + await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SYNC_ASSETS, data: { id } }); } @OnJob({ name: JobName.LIBRARY_QUEUE_SYNC_ALL, queue: QueueName.LIBRARY }) @@ -434,6 +433,8 @@ export class LibraryService extends BaseService { const assetIdsToOffline: string[] = []; const assetIdsToUpdate: string[] = []; + this.logger.debug(`Checking batch of ${assets.length} existing asset(s) in library ${job.libraryId}`); + for (const asset of assets) { const action = await this.handleSyncAsset(asset); switch (action) { @@ -447,30 +448,28 @@ export class LibraryService extends BaseService { } if (assetIdsToOffline.length) { - await this.assetRepository.updateAll(assetIdsToOffline, { isOffline: true, deletedAt: new Date() }); - this.logger.log( - `Originals are missing for ${assetIdsToOffline.length} asset(s) in library ${job.libraryId}, marked offline`, - ); + await this.assetRepository.updateAll(assetIdsToOffline, { + isOffline: true, + status: AssetStatus.TRASHED, + deletedAt: new Date(), + }); } if (assetIdsToUpdate.length) { //TODO: When we have asset status, we need to leave deletedAt as is when status is trashed await this.assetRepository.updateAll(assetIdsToUpdate, { isOffline: false, + status: AssetStatus.ACTIVE, deletedAt: null, }); - - this.logger.log( - `Found ${assetIdsToOffline.length} asset(s) with modified files for library ${job.libraryId}, queuing refresh...`, - ); - await this.queuePostSyncJobs(assetIdsToUpdate); } const remainingCount = assets.length - assetIdsToOffline.length - assetIdsToUpdate.length; - if (remainingCount > 0) { - this.logger.log(`${remainingCount} asset(s) are unchanged in library ${job.libraryId}, no action required`); - } + + this.logger.log( + `Checked existing asset(s): ${assetIdsToOffline.length} offlined, ${assetIdsToUpdate.length} updated, ${remainingCount} unchanged of batch of ${assets.length} in library ${job.libraryId}.`, + ); return JobStatus.SUCCESS; } @@ -606,7 +605,7 @@ export class LibraryService extends BaseService { `Finished crawling ${crawlCount} file(s) of which ${importCount} file(s) are queued for import for library ${library.id}`, ); } else { - this.logger.log(`All ${crawlCount} file(s) on disk are already in ${library.id}`); + this.logger.log(`All ${crawlCount} file(s) on disk are already in library ${library.id}`); } await this.libraryRepository.update({ id: job.id, refreshedAt: new Date() }); @@ -621,11 +620,17 @@ export class LibraryService extends BaseService { return JobStatus.SKIPPED; } - const assetCount = (await this.getStatistics(library.id)).total; + const assetCount = await this.assetRepository.getAssetCount(library.id, { withDeleted: true }); + + if (!assetCount) { + this.logger.log(`Library ${library.id} is empty, no need to check assets`); + return JobStatus.SUCCESS; + } this.logger.log( - `Scanning library ${library.id} for assets outside of import paths and/or matching an exclusion pattern...`, + `${assetCount} asset(s) in library ${library.id} will be checked against import paths and exclusion patterns...`, ); + const offlineResult = await this.assetRepository.updateOffline(library); const affectedAssetCount = offlineResult.affected; @@ -638,6 +643,8 @@ export class LibraryService extends BaseService { this.logger.log( `All ${assetCount} asset(s) in ${library.id} are outside of import paths and/or match an exclusion pattern, marked as offline`, ); + + return JobStatus.SUCCESS; } else if (affectedAssetCount !== assetCount && affectedAssetCount > 0) { this.logger.log( `${offlineResult.affected} asset(s) out of ${assetCount} were marked offline due to import paths and/or exclusion patterns for library ${library.id}`, @@ -651,7 +658,7 @@ export class LibraryService extends BaseService { this.logger.log(`Scanning library ${library.id} for assets missing from disk...`); const existingAssets = usePagination(JOBS_LIBRARY_PAGINATION_SIZE, (pagination) => - this.assetRepository.getAll(pagination, { libraryId: job.id, withDeleted: true }), + this.assetRepository.getAllInLibrary(pagination, job.id), ); let currentAssetCount = 0; @@ -667,12 +674,12 @@ export class LibraryService extends BaseService { }); this.logger.log( - `Queued check of ${assets.length} existing asset(s) in library ${library.id}, ${currentAssetCount} of ${assetCount} queued in total`, + `Queued check of ${currentAssetCount} of ${assetCount} existing asset(s) so far in library ${library.id}`, ); } if (currentAssetCount) { - this.logger.log(`Finished queuing ${currentAssetCount} file checks for library ${library.id}`); + this.logger.log(`Finished queuing ${currentAssetCount} asset check(s) for library ${library.id}`); } return JobStatus.SUCCESS;