refactor: library e2e (#7969)

e2e/src/api/specs/library.e2e-spec.ts

@@ -1,20 +1,48 @@
-import { LibraryResponseDto, LibraryType, LoginResponseDto, getAllLibraries } from '@immich/sdk';
+import {
+  LibraryResponseDto,
+  LibraryType,
+  LoginResponseDto,
+  ScanLibraryDto,
+  getAllLibraries,
+  scanLibrary,
+} from '@immich/sdk';
+import { existsSync, rmdirSync } from 'node:fs';
+import { Socket } from 'socket.io-client';
 import { userDto, uuidDto } from 'src/fixtures';
 import { errorDto } from 'src/responses';
-import { app, asBearerAuth, testAssetDirInternal, utils } from 'src/utils';
+import { app, asBearerAuth, testAssetDir, testAssetDirInternal, utils } from 'src/utils';
 import request from 'supertest';
-import { beforeAll, describe, expect, it } from 'vitest';
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+
+const scan = async (accessToken: string, id: string, dto: ScanLibraryDto = {}) =>
+  scanLibrary({ id, scanLibraryDto: dto }, { headers: asBearerAuth(accessToken) });
 
 describe('/library', () => {
   let admin: LoginResponseDto;
   let user: LoginResponseDto;
   let library: LibraryResponseDto;
+  let websocket: Socket;
 
   beforeAll(async () => {
     await utils.resetDatabase();
     admin = await utils.adminSetup();
     user = await utils.userSetup(admin.accessToken, userDto.user1);
     library = await utils.createLibrary(admin.accessToken, { type: LibraryType.External });
+    websocket = await utils.connectWebsocket(admin.accessToken);
+  });
+
+  afterAll(() => {
+    utils.disconnectWebsocket(websocket);
+  });
+
+  beforeEach(() => {
+    utils.resetEvents();
+    const tempDir = `${testAssetDir}/temp`;
+    if (existsSync(tempDir)) {
+      rmdirSync(tempDir, { recursive: true });
+    }
+    utils.createImageFile(`${testAssetDir}/temp/directoryA/assetA.png`);
+    utils.createImageFile(`${testAssetDir}/temp/directoryB/assetB.png`);
   });
 
   describe('GET /library', () => {
@@ -376,6 +404,36 @@ describe('/library', () => {
         ]),
       );
     });
+
+    it('should delete an external library with assets', async () => {
+      const library = await utils.createLibrary(admin.accessToken, {
+        type: LibraryType.External,
+        importPaths: [`${testAssetDirInternal}/temp`],
+      });
+
+      await scan(admin.accessToken, library.id);
+      await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 2 });
+
+      const { status, body } = await request(app)
+        .delete(`/library/${library.id}`)
+        .set('Authorization', `Bearer ${admin.accessToken}`);
+
+      expect(status).toBe(204);
+      expect(body).toEqual({});
+
+      const libraries = await getAllLibraries({}, { headers: asBearerAuth(admin.accessToken) });
+      expect(libraries).not.toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: library.id,
+          }),
+        ]),
+      );
+
+      // ensure no files get deleted
+      expect(existsSync(`${testAssetDir}/temp/directoryA/assetA.png`)).toBe(true);
+      expect(existsSync(`${testAssetDir}/temp/directoryB/assetB.png`)).toBe(true);
+    });
   });
 
   describe('GET /library/:id/statistics', () => {
@@ -394,6 +452,89 @@ describe('/library', () => {
       expect(status).toBe(401);
       expect(body).toEqual(errorDto.unauthorized);
     });
+
+    it('should not scan an upload library', async () => {
+      const library = await utils.createLibrary(admin.accessToken, {
+        type: LibraryType.Upload,
+      });
+
+      const { status, body } = await request(app)
+        .post(`/library/${library.id}/scan`)
+        .set('Authorization', `Bearer ${admin.accessToken}`);
+
+      expect(status).toBe(400);
+      expect(body).toEqual(errorDto.badRequest('Can only refresh external libraries'));
+    });
+
+    it('should scan external library', async () => {
+      const library = await utils.createLibrary(admin.accessToken, {
+        type: LibraryType.External,
+        importPaths: [`${testAssetDirInternal}/temp/directoryA`],
+      });
+
+      await scan(admin.accessToken, library.id);
+      await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 1 });
+
+      const { assets } = await utils.metadataSearch(admin.accessToken, {
+        originalPath: `${testAssetDirInternal}/temp/directoryA/assetA.png`,
+      });
+      expect(assets.count).toBe(1);
+    });
+
+    it('should scan external library with exclusion pattern', async () => {
+      const library = await utils.createLibrary(admin.accessToken, {
+        type: LibraryType.External,
+        importPaths: [`${testAssetDirInternal}/temp`],
+        exclusionPatterns: ['**/directoryA'],
+      });
+
+      await scan(admin.accessToken, library.id);
+      await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 1 });
+
+      const { assets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id });
+
+      expect(assets.count).toBe(1);
+      expect(assets.items[0].originalPath.includes('directoryB'));
+    });
+
+    it('should scan multiple import paths', async () => {
+      const library = await utils.createLibrary(admin.accessToken, {
+        type: LibraryType.External,
+        importPaths: [`${testAssetDirInternal}/temp/directoryA`, `${testAssetDirInternal}/temp/directoryB`],
+      });
+
+      await scan(admin.accessToken, library.id);
+      await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 2 });
+
+      const { assets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id });
+
+      expect(assets.count).toBe(2);
+      expect(assets.items.find((asset) => asset.originalPath.includes('directoryA'))).toBeDefined();
+      expect(assets.items.find((asset) => asset.originalPath.includes('directoryB'))).toBeDefined();
+    });
+
+    it('should pick up new files', async () => {
+      const library = await utils.createLibrary(admin.accessToken, {
+        type: LibraryType.External,
+        importPaths: [`${testAssetDirInternal}/temp`],
+      });
+
+      await scan(admin.accessToken, library.id);
+      await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 2 });
+
+      const { assets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id });
+
+      expect(assets.count).toBe(2);
+
+      utils.createImageFile(`${testAssetDir}/temp/directoryA/assetB.png`);
+
+      await scan(admin.accessToken, library.id);
+      await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 3 });
+
+      const { assets: newAssets } = await utils.metadataSearch(admin.accessToken, { libraryId: library.id });
+
+      expect(newAssets.count).toBe(3);
+    });
   });
 
   describe('POST /library/:id/removeOffline', () => {

e2e/src/utils.ts

@@ -5,6 +5,7 @@ import {
   CreateAssetDto,
   CreateLibraryDto,
   CreateUserDto,
+  MetadataSearchDto,
   PersonCreateDto,
   SharedLinkCreateDto,
   ValidateLibraryDto,
@@ -16,8 +17,10 @@ import {
   createUser,
   defaults,
   deleteAssets,
+  getAllAssets,
   getAssetInfo,
   login,
+  searchMetadata,
   setAdminOnboarding,
   signUpAdmin,
   validate,
@@ -25,9 +28,9 @@ import {
 import { BrowserContext } from '@playwright/test';
 import { exec, spawn } from 'node:child_process';
 import { createHash } from 'node:crypto';
-import { existsSync } from 'node:fs';
+import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
-import path from 'node:path';
+import path, { dirname } from 'node:path';
 import { promisify } from 'node:util';
 import pg from 'pg';
 import { io, type Socket } from 'socket.io-client';
@@ -37,7 +40,7 @@ import request from 'supertest';
 
 type CliResponse = { stdout: string; stderr: string; exitCode: number | null };
 type EventType = 'assetUpload' | 'assetDelete' | 'userDelete';
-type WaitOptions = { event: EventType; id: string; timeout?: number };
+type WaitOptions = { event: EventType; id?: string; total?: number; timeout?: number };
 type AdminSetupOptions = { onboarding?: boolean };
 type AssetData = { bytes?: Buffer; filename: string };
 
@@ -83,16 +86,30 @@ const events: Record<EventType, Set<string>> = {
   userDelete: new Set<string>(),
 };
 
-const callbacks: Record<string, () => void> = {};
+const idCallbacks: Record<string, () => void> = {};
+const countCallbacks: Record<string, { count: number; callback: () => void }> = {};
 
 const execPromise = promisify(exec);
 
 const onEvent = ({ event, id }: { event: EventType; id: string }) => {
-  events[event].add(id);
+  // console.log(`Received event: ${event} [id=${id}]`);
-  const callback = callbacks[id];
+  const set = events[event];
-  if (callback) {
+  set.add(id);
-    callback();
+
-    delete callbacks[id];
+  const idCallback = idCallbacks[id];
+  if (idCallback) {
+    idCallback();
+    delete idCallbacks[id];
+  }
+
+  const item = countCallbacks[event];
+  if (item) {
+    const { count, callback: countCallback } = item;
+
+    if (set.size >= count) {
+      countCallback();
+      delete countCallbacks[event];
+    }
   }
 };
 
@@ -184,20 +201,43 @@ export const utils = {
     }
   },
 
-  waitForWebsocketEvent: async ({ event, id, timeout: ms }: WaitOptions): Promise<void> => {
+  resetEvents: () => {
-    console.log(`Waiting for ${event} [${id}]`);
+    for (const set of Object.values(events)) {
+      set.clear();
+    }
+  },
+
+  waitForWebsocketEvent: async ({ event, id, total: count, timeout: ms }: WaitOptions): Promise<void> => {
+    if (!id && !count) {
+      throw new Error('id or count must be provided for waitForWebsocketEvent');
+    }
+
+    const type = id ? `id=${id}` : `count=${count}`;
+    console.log(`Waiting for ${event} [${type}]`);
     const set = events[event];
-    if (set.has(id)) {
+    if ((id && set.has(id)) || (count && set.size >= count)) {
       return;
     }
 
     return new Promise<void>((resolve, reject) => {
       const timeout = setTimeout(() => reject(new Error(`Timed out waiting for ${event} event`)), ms || 10_000);
 
-      callbacks[id] = () => {
+      if (id) {
-        clearTimeout(timeout);
+        idCallbacks[id] = () => {
-        resolve();
+          clearTimeout(timeout);
-      };
+          resolve();
+        };
+      }
+
+      if (count) {
+        countCallbacks[event] = {
+          count,
+          callback: () => {
+            clearTimeout(timeout);
+            resolve();
+          },
+        };
+      }
     });
   },
 
@@ -263,8 +303,31 @@ export const utils = {
     return body as AssetFileUploadResponseDto;
   },
 
+  createImageFile: (path: string) => {
+    if (!existsSync(dirname(path))) {
+      mkdirSync(dirname(path), { recursive: true });
+    }
+    if (!existsSync(path)) {
+      writeFileSync(path, makeRandomImage());
+    }
+  },
+
+  removeImageFile: (path: string) => {
+    if (!existsSync(path)) {
+      return;
+    }
+
+    rmSync(path);
+  },
+
   getAssetInfo: (accessToken: string, id: string) => getAssetInfo({ id }, { headers: asBearerAuth(accessToken) }),
 
+  getAllAssets: (accessToken: string) => getAllAssets({}, { headers: asBearerAuth(accessToken) }),
+
+  metadataSearch: async (accessToken: string, dto: MetadataSearchDto) => {
+    return searchMetadata({ metadataSearchDto: dto }, { headers: asBearerAuth(accessToken) });
+  },
+
   deleteAssets: (accessToken: string, ids: string[]) =>
     deleteAssets({ assetBulkDeleteDto: { ids } }, { headers: asBearerAuth(accessToken) }),
 

server/e2e/jobs/specs/library.e2e-spec.ts

@@ -34,117 +34,7 @@ describe(`${LibraryController.name} (e2e)`, () => {
     await restoreTempFolder();
   });
 
-  describe('DELETE /library/:id', () => {
-    it('should delete an external library with assets', async () => {
-      const library = await api.libraryApi.create(server, admin.accessToken, {
-        type: LibraryType.EXTERNAL,
-        importPaths: [`${IMMICH_TEST_ASSET_PATH}/albums/nature`],
-      });
-      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id);
-
-      const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
-      expect(assets.length).toBeGreaterThan(2);
-
-      const { status } = await request(server)
-        .delete(`/library/${library.id}`)
-        .set('Authorization', `Bearer ${admin.accessToken}`);
-
-      expect(status).toBe(204);
-
-      const libraries = await api.libraryApi.getAll(server, admin.accessToken);
-      expect(libraries).toHaveLength(1);
-      expect(libraries).not.toEqual(
-        expect.arrayContaining([
-          expect.objectContaining({
-            id: library.id,
-          }),
-        ]),
-      );
-    });
-  });
-
   describe('POST /library/:id/scan', () => {
-    it('should require authentication', async () => {
-      const { status, body } = await request(server).post(`/library/${uuidStub.notFound}/scan`).send({});
-
-      expect(status).toBe(401);
-      expect(body).toEqual(errorStub.unauthorized);
-    });
-
-    it('should scan external library with import paths', async () => {
-      const library = await api.libraryApi.create(server, admin.accessToken, {
-        type: LibraryType.EXTERNAL,
-        importPaths: [`${IMMICH_TEST_ASSET_PATH}/albums/nature`],
-      });
-      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id);
-
-      const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
-
-      expect(assets).toEqual(
-        expect.arrayContaining([
-          expect.objectContaining({
-            type: AssetType.IMAGE,
-            originalFileName: 'el_torcal_rocks',
-            libraryId: library.id,
-            resized: true,
-            thumbhash: expect.any(String),
-            exifInfo: expect.objectContaining({
-              exifImageWidth: 512,
-              exifImageHeight: 341,
-              latitude: null,
-              longitude: null,
-            }),
-          }),
-          expect.objectContaining({
-            type: AssetType.IMAGE,
-            originalFileName: 'silver_fir',
-            libraryId: library.id,
-            resized: true,
-            thumbhash: expect.any(String),
-            exifInfo: expect.objectContaining({
-              exifImageWidth: 511,
-              exifImageHeight: 323,
-              latitude: null,
-              longitude: null,
-            }),
-          }),
-        ]),
-      );
-    });
-
-    it('should scan external library with exclusion pattern', async () => {
-      const library = await api.libraryApi.create(server, admin.accessToken, {
-        type: LibraryType.EXTERNAL,
-        importPaths: [`${IMMICH_TEST_ASSET_PATH}/albums/nature`],
-        exclusionPatterns: ['**/el_corcal*'],
-      });
-
-      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id);
-
-      const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
-
-      expect(assets).toEqual(
-        expect.arrayContaining([
-          expect.not.objectContaining({
-            // Excluded by exclusion pattern
-            originalFileName: 'el_torcal_rocks',
-          }),
-          expect.objectContaining({
-            type: AssetType.IMAGE,
-            originalFileName: 'silver_fir',
-            libraryId: library.id,
-            resized: true,
-            exifInfo: expect.objectContaining({
-              exifImageWidth: 511,
-              exifImageHeight: 323,
-              latitude: null,
-              longitude: null,
-            }),
-          }),
-        ]),
-      );
-    });
-
     it('should offline missing files', async () => {
       await fs.promises.cp(`${IMMICH_TEST_ASSET_PATH}/albums/nature`, `${IMMICH_TEST_ASSET_TEMP_PATH}/albums/nature`, {
         recursive: true,
@@ -345,19 +235,6 @@ describe(`${LibraryController.name} (e2e)`, () => {
         );
       });
     });
-
-    it('should not scan an upload library', async () => {
-      const library = await api.libraryApi.create(server, admin.accessToken, {
-        type: LibraryType.UPLOAD,
-      });
-
-      const { status, body } = await request(server)
-        .post(`/library/${library.id}/scan`)
-        .set('Authorization', `Bearer ${admin.accessToken}`);
-
-      expect(status).toBe(400);
-      expect(body).toEqual(errorStub.badRequest('Can only refresh external libraries'));
-    });
   });
 
   describe('POST /library/:id/removeOffline', () => {