perf(server): use queries to refresh library assets (#7685)

* use queries instead of js

* missing await

* add mock methods

* fix test

* update sql

* linting

server/src/domain/library/library.service.spec.ts

@@ -144,6 +144,7 @@ describe(LibraryService.name, () => {
 
       libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
       storageMock.crawl.mockResolvedValue(['/data/user1/photo.jpg']);
+      assetMock.getPathsNotInLibrary.mockResolvedValue(['/data/user1/photo.jpg']);
       assetMock.getByLibraryId.mockResolvedValue([]);
       userMock.get.mockResolvedValue(userStub.admin);
 

server/src/domain/library/library.service.ts

@@ -621,29 +621,18 @@ export class LibraryService extends EventEmitter {
       pathsToCrawl: validImportPaths,
       exclusionPatterns: library.exclusionPatterns,
     });
-
     const crawledAssetPaths = rawPaths.map((filePath) => path.normalize(filePath));
 
     this.logger.debug(`Found ${crawledAssetPaths.length} asset(s) when crawling import paths ${library.importPaths}`);
-    const assetsInLibrary = await this.assetRepository.getByLibraryId([job.id]);
-    const onlineFiles = new Set(crawledAssetPaths);
-    const offlineAssetIds = assetsInLibrary
-      .filter((asset) => !onlineFiles.has(asset.originalPath))
-      .filter((asset) => !asset.isOffline)
-      .map((asset) => asset.id);
-    this.logger.debug(`Marking ${offlineAssetIds.length} assets as offline`);
 
-    await this.assetRepository.updateAll(offlineAssetIds, { isOffline: true });
+    await this.assetRepository.updateOfflineLibraryAssets(library.id, crawledAssetPaths);
 
     if (crawledAssetPaths.length > 0) {
       let filteredPaths: string[] = [];
       if (job.refreshAllFiles || job.refreshModifiedFiles) {
         filteredPaths = crawledAssetPaths;
       } else {
-        const onlinePathsInLibrary = new Set(
+        filteredPaths = await this.assetRepository.getPathsNotInLibrary(library.id, crawledAssetPaths);
-          assetsInLibrary.filter((asset) => !asset.isOffline).map((asset) => asset.originalPath),
-        );
-        filteredPaths = crawledAssetPaths.filter((assetPath) => !onlinePathsInLibrary.has(assetPath));
 
         this.logger.debug(`Will import ${filteredPaths.length} new asset(s)`);
       }

server/src/domain/repositories/asset.repository.ts

@@ -136,6 +136,8 @@ export interface IAssetRepository {
   getLastUpdatedAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;
   getByLibraryId(libraryIds: string[]): Promise<AssetEntity[]>;
   getByLibraryIdAndOriginalPath(libraryId: string, originalPath: string): Promise<AssetEntity | null>;
+  getPathsNotInLibrary(libraryId: string, originalPaths: string[]): Promise<string[]>;
+  updateOfflineLibraryAssets(libraryId: string, originalPaths: string[]): Promise<void>;
   deleteAll(ownerId: string): Promise<void>;
   getAll(pagination: PaginationOptions, options?: AssetSearchOptions): Paginated<AssetEntity>;
   getAllByFileCreationDate(

server/src/infra/repositories/asset.repository.ts

@@ -199,6 +199,29 @@ export class AssetRepository implements IAssetRepository {
     });
   }
 
+  @GenerateSql({ params: [DummyValue.UUID, [DummyValue.STRING]] })
+  @ChunkedArray({ paramIndex: 1 })
+  async getPathsNotInLibrary(libraryId: string, originalPaths: string[]): Promise<string[]> {
+    const result = await this.repository.query(
+      `
+      WITH paths AS (SELECT unnest($2::text[]) AS path)
+      SELECT path FROM paths
+      WHERE NOT EXISTS (SELECT 1 FROM assets WHERE "libraryId" = $1 AND "originalPath" = path);
+    `,
+      [libraryId, originalPaths],
+    );
+    return result.map((row: { path: string }) => row.path);
+  }
+
+  @GenerateSql({ params: [DummyValue.UUID, [DummyValue.STRING]] })
+  @ChunkedArray({ paramIndex: 1 })
+  async updateOfflineLibraryAssets(libraryId: string, originalPaths: string[]): Promise<void> {
+    await this.repository.update(
+      { library: { id: libraryId }, originalPath: Not(In(originalPaths)), isOffline: false },
+      { isOffline: true },
+    );
+  }
+
   getAll(pagination: PaginationOptions, options: AssetSearchOptions = {}): Paginated<AssetEntity> {
     let builder = this.repository.createQueryBuilder('asset');
     builder = searchAssetBuilder(builder, options);

server/src/infra/sql/asset.repository.sql

@@ -395,6 +395,39 @@ ORDER BY
 LIMIT
   1
 
+-- AssetRepository.getPathsNotInLibrary
+WITH
+  paths AS (
+    SELECT
+      unnest($2::text []) AS path
+  )
+SELECT
+  path
+FROM
+  paths
+WHERE
+  NOT EXISTS (
+    SELECT
+      1
+    FROM
+      assets
+    WHERE
+      "libraryId" = $1
+      AND "originalPath" = path
+  );
+
+-- AssetRepository.updateOfflineLibraryAssets
+UPDATE "assets"
+SET
+  "isOffline" = $1,
+  "updatedAt" = CURRENT_TIMESTAMP
+WHERE
+  (
+    "libraryId" = $2
+    AND NOT ("originalPath" IN ($3))
+    AND "isOffline" = $4
+  )
+
 -- AssetRepository.getAllByFileCreationDate
 SELECT
   "asset"."id" AS "asset_id",

server/test/repositories/asset.repository.mock.ts

@@ -23,6 +23,8 @@ export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
     updateAll: jest.fn(),
     getByLibraryId: jest.fn(),
     getByLibraryIdAndOriginalPath: jest.fn(),
+    updateOfflineLibraryAssets: jest.fn(),
+    getPathsNotInLibrary: jest.fn(),
     deleteAll: jest.fn(),
     save: jest.fn(),
     remove: jest.fn(),