1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-01-08 12:56:48 +01:00
immich/server/e2e/jobs/specs/library-watcher.e2e-spec.ts

233 lines
7.8 KiB
TypeScript
Raw Normal View History

feat(server): Automatic watching of library folders (#6192) * feat: initial watch support * allow offline files * chore: ignore query errors when resetting e2e db * revert db query * add savepoint * guard the user query * chore: openapi and db migration * wip * support multiple libraries * fix tests * wip * can now cleanup chokidar watchers * fix unit tests * add library watch queue * add missing init from merge * wip * can now filter file extensions * remove watch api from non job client * Fix e2e test * watch library with updated import path and exclusion pattern * add library watch frontend ui * case sensitive watching extensions * can auto watch libraries * move watcher e2e tests to separate file * don't watch libraries from a queue * use event emitters * shorten e2e test timeout * refactor chokidar code to filesystem provider * expose chokidar parameters to config file * fix storage mock * set default config for library watching * add fs provider mocks * cleanup * add more unit tests for watcher * chore: fix format + sql * add more tests * move unwatch feature back to library service * add file event unit tests * chore: formatting * add documentation * fix e2e tests * chore: fix e2e tests * fix library updating * test cleanup * fix typo * cleanup * fixing as per pr comments * reduce library watch config file * update storage config and mocks * move negative event tests to unit tests * fix library watcher e2e * make watch configuration global * remove the feature flag * refactor watcher teardown * fix microservices init * centralize asset scan job queue * improve docs * add more tests * chore: open api * initialize app service * fix docs * fix library watch feature flag * Update docs/docs/features/libraries.md Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> * fix: import right app service * don't be truthy * fix test speling * stricter library update tests * move fs watcher mock to external file * subscribe to config changes * docker does not need polling * make library watch() private * feat: add configuration ui --------- Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-31 09:15:54 +01:00
import { LibraryResponseDto, LibraryService, LoginResponseDto } from '@app/domain';
import { AssetType, LibraryType } from '@app/infra/entities';
2024-01-31 17:26:51 +01:00
import fs from 'fs/promises';
feat(server): Automatic watching of library folders (#6192) * feat: initial watch support * allow offline files * chore: ignore query errors when resetting e2e db * revert db query * add savepoint * guard the user query * chore: openapi and db migration * wip * support multiple libraries * fix tests * wip * can now cleanup chokidar watchers * fix unit tests * add library watch queue * add missing init from merge * wip * can now filter file extensions * remove watch api from non job client * Fix e2e test * watch library with updated import path and exclusion pattern * add library watch frontend ui * case sensitive watching extensions * can auto watch libraries * move watcher e2e tests to separate file * don't watch libraries from a queue * use event emitters * shorten e2e test timeout * refactor chokidar code to filesystem provider * expose chokidar parameters to config file * fix storage mock * set default config for library watching * add fs provider mocks * cleanup * add more unit tests for watcher * chore: fix format + sql * add more tests * move unwatch feature back to library service * add file event unit tests * chore: formatting * add documentation * fix e2e tests * chore: fix e2e tests * fix library updating * test cleanup * fix typo * cleanup * fixing as per pr comments * reduce library watch config file * update storage config and mocks * move negative event tests to unit tests * fix library watcher e2e * make watch configuration global * remove the feature flag * refactor watcher teardown * fix microservices init * centralize asset scan job queue * improve docs * add more tests * chore: open api * initialize app service * fix docs * fix library watch feature flag * Update docs/docs/features/libraries.md Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> * fix: import right app service * don't be truthy * fix test speling * stricter library update tests * move fs watcher mock to external file * subscribe to config changes * docker does not need polling * make library watch() private * feat: add configuration ui --------- Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-31 09:15:54 +01:00
import path from 'path';
import {
IMMICH_TEST_ASSET_PATH,
IMMICH_TEST_ASSET_TEMP_PATH,
restoreTempFolder,
testApp,
waitForEvent,
} from '../../../src/test-utils/utils';
2024-01-31 17:26:51 +01:00
import { api } from '../../client';
feat(server): Automatic watching of library folders (#6192) * feat: initial watch support * allow offline files * chore: ignore query errors when resetting e2e db * revert db query * add savepoint * guard the user query * chore: openapi and db migration * wip * support multiple libraries * fix tests * wip * can now cleanup chokidar watchers * fix unit tests * add library watch queue * add missing init from merge * wip * can now filter file extensions * remove watch api from non job client * Fix e2e test * watch library with updated import path and exclusion pattern * add library watch frontend ui * case sensitive watching extensions * can auto watch libraries * move watcher e2e tests to separate file * don't watch libraries from a queue * use event emitters * shorten e2e test timeout * refactor chokidar code to filesystem provider * expose chokidar parameters to config file * fix storage mock * set default config for library watching * add fs provider mocks * cleanup * add more unit tests for watcher * chore: fix format + sql * add more tests * move unwatch feature back to library service * add file event unit tests * chore: formatting * add documentation * fix e2e tests * chore: fix e2e tests * fix library updating * test cleanup * fix typo * cleanup * fixing as per pr comments * reduce library watch config file * update storage config and mocks * move negative event tests to unit tests * fix library watcher e2e * make watch configuration global * remove the feature flag * refactor watcher teardown * fix microservices init * centralize asset scan job queue * improve docs * add more tests * chore: open api * initialize app service * fix docs * fix library watch feature flag * Update docs/docs/features/libraries.md Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> * fix: import right app service * don't be truthy * fix test speling * stricter library update tests * move fs watcher mock to external file * subscribe to config changes * docker does not need polling * make library watch() private * feat: add configuration ui --------- Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-31 09:15:54 +01:00
describe(`Library watcher (e2e)`, () => {
let server: any;
let admin: LoginResponseDto;
let libraryService: LibraryService;
const configFilePath = process.env.IMMICH_CONFIG_FILE;
beforeAll(async () => {
process.env.IMMICH_CONFIG_FILE = path.normalize(`${__dirname}/../config/library-watcher-e2e-config.json`);
server = (await testApp.create()).getHttpServer();
libraryService = testApp.get(LibraryService);
});
beforeEach(async () => {
await testApp.reset();
await restoreTempFolder();
await api.authApi.adminSignUp(server);
admin = await api.authApi.adminLogin(server);
await api.userApi.setExternalPath(server, admin.accessToken, admin.userId, '/');
});
afterEach(async () => {
await libraryService.unwatchAll();
});
afterAll(async () => {
await testApp.teardown();
await restoreTempFolder();
process.env.IMMICH_CONFIG_FILE = configFilePath;
});
describe('Event handling', () => {
describe('Single import path', () => {
beforeEach(async () => {
2024-01-31 17:26:51 +01:00
await api.libraryApi.create(server, admin.accessToken, {
feat(server): Automatic watching of library folders (#6192) * feat: initial watch support * allow offline files * chore: ignore query errors when resetting e2e db * revert db query * add savepoint * guard the user query * chore: openapi and db migration * wip * support multiple libraries * fix tests * wip * can now cleanup chokidar watchers * fix unit tests * add library watch queue * add missing init from merge * wip * can now filter file extensions * remove watch api from non job client * Fix e2e test * watch library with updated import path and exclusion pattern * add library watch frontend ui * case sensitive watching extensions * can auto watch libraries * move watcher e2e tests to separate file * don't watch libraries from a queue * use event emitters * shorten e2e test timeout * refactor chokidar code to filesystem provider * expose chokidar parameters to config file * fix storage mock * set default config for library watching * add fs provider mocks * cleanup * add more unit tests for watcher * chore: fix format + sql * add more tests * move unwatch feature back to library service * add file event unit tests * chore: formatting * add documentation * fix e2e tests * chore: fix e2e tests * fix library updating * test cleanup * fix typo * cleanup * fixing as per pr comments * reduce library watch config file * update storage config and mocks * move negative event tests to unit tests * fix library watcher e2e * make watch configuration global * remove the feature flag * refactor watcher teardown * fix microservices init * centralize asset scan job queue * improve docs * add more tests * chore: open api * initialize app service * fix docs * fix library watch feature flag * Update docs/docs/features/libraries.md Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> * fix: import right app service * don't be truthy * fix test speling * stricter library update tests * move fs watcher mock to external file * subscribe to config changes * docker does not need polling * make library watch() private * feat: add configuration ui --------- Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-31 09:15:54 +01:00
type: LibraryType.EXTERNAL,
importPaths: [`${IMMICH_TEST_ASSET_TEMP_PATH}`],
});
});
it('should import a new file', async () => {
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file.jpg`,
);
await waitForEvent(libraryService, 'add');
const afterAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(afterAssets.length).toEqual(1);
});
it('should import new files with case insensitive extensions', async () => {
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file2.JPG`,
);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file3.Jpg`,
);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file4.jpG`,
);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file5.jPg`,
);
await waitForEvent(libraryService, 'add');
await waitForEvent(libraryService, 'add');
await waitForEvent(libraryService, 'add');
await waitForEvent(libraryService, 'add');
const afterAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(afterAssets.length).toEqual(4);
});
it('should update a changed file', async () => {
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file.jpg`,
);
await waitForEvent(libraryService, 'add');
const originalAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(originalAssets.length).toEqual(1);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/prairie_falcon.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/file.jpg`,
);
await waitForEvent(libraryService, 'change');
const afterAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(afterAssets).toEqual([
expect.objectContaining({
// Make sure we keep the original asset id
id: originalAssets[0].id,
type: AssetType.IMAGE,
exifInfo: expect.objectContaining({
make: 'Canon',
model: 'Canon EOS R5',
exifImageWidth: 800,
exifImageHeight: 533,
exposureTime: '1/4000',
}),
}),
]);
});
});
describe('Multiple import paths', () => {
beforeEach(async () => {
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1`, { recursive: true });
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2`, { recursive: true });
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3`, { recursive: true });
2024-01-31 17:26:51 +01:00
await api.libraryApi.create(server, admin.accessToken, {
feat(server): Automatic watching of library folders (#6192) * feat: initial watch support * allow offline files * chore: ignore query errors when resetting e2e db * revert db query * add savepoint * guard the user query * chore: openapi and db migration * wip * support multiple libraries * fix tests * wip * can now cleanup chokidar watchers * fix unit tests * add library watch queue * add missing init from merge * wip * can now filter file extensions * remove watch api from non job client * Fix e2e test * watch library with updated import path and exclusion pattern * add library watch frontend ui * case sensitive watching extensions * can auto watch libraries * move watcher e2e tests to separate file * don't watch libraries from a queue * use event emitters * shorten e2e test timeout * refactor chokidar code to filesystem provider * expose chokidar parameters to config file * fix storage mock * set default config for library watching * add fs provider mocks * cleanup * add more unit tests for watcher * chore: fix format + sql * add more tests * move unwatch feature back to library service * add file event unit tests * chore: formatting * add documentation * fix e2e tests * chore: fix e2e tests * fix library updating * test cleanup * fix typo * cleanup * fixing as per pr comments * reduce library watch config file * update storage config and mocks * move negative event tests to unit tests * fix library watcher e2e * make watch configuration global * remove the feature flag * refactor watcher teardown * fix microservices init * centralize asset scan job queue * improve docs * add more tests * chore: open api * initialize app service * fix docs * fix library watch feature flag * Update docs/docs/features/libraries.md Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> * fix: import right app service * don't be truthy * fix test speling * stricter library update tests * move fs watcher mock to external file * subscribe to config changes * docker does not need polling * make library watch() private * feat: add configuration ui --------- Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
2024-01-31 09:15:54 +01:00
type: LibraryType.EXTERNAL,
importPaths: [
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3`,
],
});
});
it('should add new files in multiple import paths', async () => {
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/el_torcal_rocks.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1/file2.jpg`,
);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/polemonium_reptans.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2/file3.jpg`,
);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/tanners_ridge.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3/file4.jpg`,
);
await waitForEvent(libraryService, 'add');
await waitForEvent(libraryService, 'add');
await waitForEvent(libraryService, 'add');
const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(assets.length).toEqual(3);
});
it('should offline a removed file', async () => {
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/polemonium_reptans.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1/file.jpg`,
);
await waitForEvent(libraryService, 'add');
const addedAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(addedAssets.length).toEqual(1);
await fs.unlink(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1/file.jpg`);
await waitForEvent(libraryService, 'unlink');
const afterAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(afterAssets[0].isOffline).toEqual(true);
});
});
});
describe('Configuration', () => {
let library: LibraryResponseDto;
beforeEach(async () => {
library = await api.libraryApi.create(server, admin.accessToken, {
type: LibraryType.EXTERNAL,
importPaths: [
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3`,
],
});
await api.userApi.setExternalPath(server, admin.accessToken, admin.userId, '/');
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1`, { recursive: true });
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2`, { recursive: true });
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3`, { recursive: true });
});
it('should use an updated import paths', async () => {
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir4`, { recursive: true });
await api.libraryApi.setImportPaths(server, admin.accessToken, library.id, [
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir4`,
]);
await fs.copyFile(
`${IMMICH_TEST_ASSET_PATH}/albums/nature/polemonium_reptans.jpg`,
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir4/file.jpg`,
);
await waitForEvent(libraryService, 'add');
const afterAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
expect(afterAssets.length).toEqual(1);
});
});
});