Merge branch 'feat/inline-offline-check' of https://github.com/immich-app/immich into feat/inline-offline-check

server/src/interfaces/asset.interface.ts

@@ -178,9 +178,9 @@ export interface IAssetRepository {
   getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>;
   getRandom(userIds: string[], count: number): Promise<AssetEntity[]>;
   getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
-  getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>;
   deleteAll(ownerId: string): Promise<void>;
   getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
+  getAllInLibrary(pagination: PaginationOptions, libraryId: string): Paginated<AssetEntity>;
   getAllByDeviceId(userId: string, deviceId: string): Promise<string[]>;
   getLivePhotoCount(motionId: string): Promise<number>;
   updateAll(ids: string[], options: Partial<AssetUpdateAllOptions>): Promise<void>;
@@ -201,4 +201,5 @@ export interface IAssetRepository {
   upsertFiles(files: UpsertFileOptions[]): Promise<void>;
   updateOffline(library: LibraryEntity): Promise<UpdateResult>;
   getNewPaths(libraryId: string, paths: string[]): Promise<string[]>;
+  getAssetCount(id: string, options: AssetSearchOptions): Promise<number | undefined>;
 }

server/src/repositories/asset.repository.ts

@@ -197,14 +197,6 @@ export class AssetRepository implements IAssetRepository {
     return this.getAll(pagination, { ...options, userIds: [userId] });
   }
 
-  @GenerateSql({ params: [DummyValue.UUID, DummyValue.STRING] })
-  getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null> {
-    return this.repository.findOne({
-      where: { library: { id: libraryId }, originalPath },
-      withDeleted: true,
-    });
-  }
-
   @GenerateSql({ params: [DummyValue.UUID, [DummyValue.STRING]] })
   @ChunkedArray({ paramIndex: 1 })
   async getPathsNotInLibrary(libraryId: string, originalPaths: string[]): Promise<string[]> {
@@ -231,6 +223,20 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
+  getAllInLibrary(pagination: PaginationOptions, libraryId: string): Paginated<AssetEntity> {
+    const builder = this.repository
+      .createQueryBuilder('asset')
+      .select('asset.id')
+      .where('asset.libraryId = :libraryId', { libraryId })
+      .withDeleted();
+
+    return paginatedBuilder<AssetEntity>(builder, {
+      mode: PaginationMode.SKIP_TAKE,
+      skip: pagination.skip,
+      take: pagination.take,
+    });
+  }
+
   /**
    * Get assets by device's Id on the database
    * @param ownerId
@@ -779,4 +785,10 @@ export class AssetRepository implements IAssetRepository {
       .query(rawSql, [paths, libraryId])
       .then((result) => result.map((row: { path: string }) => row.path));
   }
+
+  async getAssetCount(id: string, options: AssetSearchOptions = {}): Promise<number | undefined> {
+    let builder = this.repository.createQueryBuilder('asset').leftJoinAndSelect('asset.files', 'files');
+    builder = searchAssetBuilder(builder, options);
+    return builder.getCount();
+  }
 }

server/src/repositories/trash.repository.ts

@@ -14,6 +14,7 @@ export class TrashRepository implements ITrashRepository {
         .createQueryBuilder('asset')
         .select('asset.id')
         .where({ status: AssetStatus.DELETED })
+        .orWhere({ isOffline: true })
         .withDeleted(),
       pagination,
     );
@@ -34,10 +35,13 @@ export class TrashRepository implements ITrashRepository {
   }
 
   async empty(userId: string): Promise<number> {
-    const result = await this.assetRepository.update(
-      { ownerId: userId, status: AssetStatus.TRASHED },
-      { status: AssetStatus.DELETED },
-    );
+    const result = await this.assetRepository
+      .createQueryBuilder()
+      .update(AssetEntity)
+      .set({ status: AssetStatus.DELETED })
+      .where({ ownerId: userId, status: AssetStatus.TRASHED })
+      .orWhere({ ownerId: userId, isOffline: true })
+      .execute();
 
     return result.affected || 0;
   }

server/src/services/library.service.ts

@@ -388,15 +388,14 @@ export class LibraryService extends BaseService {
 
     this.logger.log(`Starting to scan library ${id}`);
 
-    await this.jobRepository.queueAll([
-      {
-        name: JobName.LIBRARY_QUEUE_SYNC_FILES,
-        data: {
-          id,
-        },
+    await this.jobRepository.queue({
+      name: JobName.LIBRARY_QUEUE_SYNC_FILES,
+      data: {
+        id,
       },
-      { name: JobName.LIBRARY_QUEUE_SYNC_ASSETS, data: { id } },
-    ]);
+    });
+
+    await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SYNC_ASSETS, data: { id } });
   }
 
   @OnJob({ name: JobName.LIBRARY_QUEUE_SYNC_ALL, queue: QueueName.LIBRARY })
@@ -434,6 +433,8 @@ export class LibraryService extends BaseService {
     const assetIdsToOffline: string[] = [];
     const assetIdsToUpdate: string[] = [];
 
+    this.logger.debug(`Checking batch of ${assets.length} existing asset(s) in library ${job.libraryId}`);
+
     for (const asset of assets) {
       const action = await this.handleSyncAsset(asset);
       switch (action) {
@@ -447,30 +448,28 @@ export class LibraryService extends BaseService {
     }
 
     if (assetIdsToOffline.length) {
-      await this.assetRepository.updateAll(assetIdsToOffline, { isOffline: true, deletedAt: new Date() });
-      this.logger.log(
-        `Originals are missing for ${assetIdsToOffline.length} asset(s) in library ${job.libraryId}, marked offline`,
-      );
+      await this.assetRepository.updateAll(assetIdsToOffline, {
+        isOffline: true,
+        status: AssetStatus.TRASHED,
+        deletedAt: new Date(),
+      });
     }
 
     if (assetIdsToUpdate.length) {
       //TODO: When we have asset status, we need to leave deletedAt as is when status is trashed
       await this.assetRepository.updateAll(assetIdsToUpdate, {
         isOffline: false,
+        status: AssetStatus.ACTIVE,
         deletedAt: null,
       });
-
-      this.logger.log(
-        `Found ${assetIdsToOffline.length} asset(s) with modified files for library ${job.libraryId}, queuing refresh...`,
-      );
-
       await this.queuePostSyncJobs(assetIdsToUpdate);
     }
 
     const remainingCount = assets.length - assetIdsToOffline.length - assetIdsToUpdate.length;
-    if (remainingCount > 0) {
-      this.logger.log(`${remainingCount} asset(s) are unchanged in library ${job.libraryId}, no action required`);
-    }
+
+    this.logger.log(
+      `Checked existing asset(s): ${assetIdsToOffline.length} offlined, ${assetIdsToUpdate.length} updated, ${remainingCount} unchanged of batch of ${assets.length} in library ${job.libraryId}.`,
+    );
 
     return JobStatus.SUCCESS;
   }
@@ -606,7 +605,7 @@ export class LibraryService extends BaseService {
         `Finished crawling ${crawlCount} file(s) of which ${importCount} file(s) are queued for import for library ${library.id}`,
       );
     } else {
-      this.logger.log(`All ${crawlCount} file(s) on disk are already in ${library.id}`);
+      this.logger.log(`All ${crawlCount} file(s) on disk are already in library ${library.id}`);
     }
 
     await this.libraryRepository.update({ id: job.id, refreshedAt: new Date() });
@@ -621,11 +620,17 @@ export class LibraryService extends BaseService {
       return JobStatus.SKIPPED;
     }
 
-    const assetCount = (await this.getStatistics(library.id)).total;
+    const assetCount = await this.assetRepository.getAssetCount(library.id, { withDeleted: true });
+
+    if (!assetCount) {
+      this.logger.log(`Library ${library.id} is empty, no need to check assets`);
+      return JobStatus.SUCCESS;
+    }
 
     this.logger.log(
-      `Scanning library ${library.id} for assets outside of import paths and/or matching an exclusion pattern...`,
+      `${assetCount} asset(s) in library ${library.id} will be checked against import paths and exclusion patterns...`,
     );
+
     const offlineResult = await this.assetRepository.updateOffline(library);
 
     const affectedAssetCount = offlineResult.affected;
@@ -638,6 +643,8 @@ export class LibraryService extends BaseService {
       this.logger.log(
         `All ${assetCount} asset(s) in ${library.id} are outside of import paths and/or match an exclusion pattern, marked as offline`,
       );
+
+      return JobStatus.SUCCESS;
     } else if (affectedAssetCount !== assetCount && affectedAssetCount > 0) {
       this.logger.log(
         `${offlineResult.affected} asset(s) out of ${assetCount} were marked offline due to import paths and/or exclusion patterns for library ${library.id}`,
@@ -651,7 +658,7 @@ export class LibraryService extends BaseService {
     this.logger.log(`Scanning library ${library.id} for assets missing from disk...`);
 
     const existingAssets = usePagination(JOBS_LIBRARY_PAGINATION_SIZE, (pagination) =>
-      this.assetRepository.getAll(pagination, { libraryId: job.id, withDeleted: true }),
+      this.assetRepository.getAllInLibrary(pagination, job.id),
     );
 
     let currentAssetCount = 0;
@@ -667,12 +674,12 @@ export class LibraryService extends BaseService {
       });
 
       this.logger.log(
-        `Queued check of ${assets.length} existing asset(s) in library ${library.id}, ${currentAssetCount} of ${assetCount} queued in total`,
+        `Queued check of ${currentAssetCount} of ${assetCount} existing asset(s) so far in library ${library.id}`,
       );
     }
 
     if (currentAssetCount) {
-      this.logger.log(`Finished queuing ${currentAssetCount} file checks for library ${library.id}`);
+      this.logger.log(`Finished queuing ${currentAssetCount} asset check(s) for library ${library.id}`);
     }
 
     return JobStatus.SUCCESS;