c-l/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-29 15:11:58 +00:00
Code Releases Activity

fix: library watching (#6802)

Browse source
This commit is contained in:
Jason Rasmussen 2024-01-31 11:26:51 -05:00 committed by GitHub
parent c083636c51
commit b0d7434e7b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 30 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
server/e2e/jobs/specs/library-watcher.e2e-spec.ts
View file

@ -1,8 +1,6 @@
import { LibraryResponseDto, LibraryService, LoginResponseDto } from '@app/domain';
import { AssetType, LibraryType } from '@app/infra/entities';
import * as fs from 'fs/promises';
import { api } from '../../client';
import fs from 'fs/promises';
import path from 'path';
import {
IMMICH_TEST_ASSET_PATH,
@ -11,6 +9,7 @@ import {
testApp,
waitForEvent,
} from '../../../src/test-utils/utils';
import { api } from '../../client';
describe(`Library watcher (e2e)`, () => {
let server: any;
@ -45,11 +44,9 @@ describe(`Library watcher (e2e)`, () => {
});
describe('Event handling', () => {
let library: LibraryResponseDto;
describe('Single import path', () => {
beforeEach(async () => {
library = await api.libraryApi.create(server, admin.accessToken, {
await api.libraryApi.create(server, admin.accessToken, {
type: LibraryType.EXTERNAL,
importPaths: [`${IMMICH_TEST_ASSET_TEMP_PATH}`],
});
@ -139,7 +136,7 @@ describe(`Library watcher (e2e)`, () => {
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2`, { recursive: true });
await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3`, { recursive: true });
library = await api.libraryApi.create(server, admin.accessToken, {
await api.libraryApi.create(server, admin.accessToken, {
type: LibraryType.EXTERNAL,
importPaths: [
`${IMMICH_TEST_ASSET_TEMP_PATH}/dir1`,

42
server/package-lock.json generated
View file

@ -28,6 +28,7 @@
"axios": "^1.5.0",
"bcrypt": "^5.1.1",
"bullmq": "^4.8.0",
"chokidar": "^3.5.3",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.6",
@ -78,7 +79,6 @@
"@types/ua-parser-js": "^0.7.36",
"@typescript-eslint/eslint-plugin": "^6.4.1",
"@typescript-eslint/parser": "^6.4.1",
"chokidar": "^3.5.3",
"dotenv": "^16.3.1",
"eslint": "^8.48.0",
"eslint-config-prettier": "^9.0.0",
@ -3981,7 +3981,6 @@
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
"integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==",
"dev": true,
"dependencies": {
"normalize-path": "^3.0.0",
"picomatch": "^2.0.4"
@ -3994,7 +3993,6 @@
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true,
"engines": {
"node": ">=8.6"
},
@ -4370,7 +4368,6 @@
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
"integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==",
"dev": true,
"engines": {
"node": ">=8"
}
@ -4434,7 +4431,6 @@
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
"integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
"dev": true,
"dependencies": {
"fill-range": "^7.0.1"
},
@ -4720,7 +4716,6 @@
"version": "3.5.3",
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz",
"integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==",
"dev": true,
"funding": [
{
"type": "individual",
@ -6230,7 +6225,6 @@
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
"integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
"dev": true,
"dependencies": {
"to-regex-range": "^5.0.1"
},
@ -6530,7 +6524,6 @@
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
"dev": true,
"hasInstallScript": true,
"optional": true,
"os": [
@ -6702,7 +6695,6 @@
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
"integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
"dev": true,
"dependencies": {
"is-glob": "^4.0.1"
},
@ -7115,7 +7107,6 @@
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
"integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
"dev": true,
"dependencies": {
"binary-extensions": "^2.0.0"
},
@ -7139,7 +7130,6 @@
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
"integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
"dev": true,
"engines": {
"node": ">=0.10.0"
}
@ -7165,7 +7155,6 @@
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
"integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
"dev": true,
"dependencies": {
"is-extglob": "^2.1.1"
},
@ -7185,7 +7174,6 @@
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
"dev": true,
"engines": {
"node": ">=0.12.0"
}
@ -9791,7 +9779,6 @@
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
"integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==",
"dev": true,
"dependencies": {
"picomatch": "^2.2.1"
},
@ -9803,7 +9790,6 @@
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true,
"engines": {
"node": ">=8.6"
},
@ -11283,7 +11269,6 @@
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
"integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
"dev": true,
"dependencies": {
"is-number": "^7.0.0"
},
@ -15043,7 +15028,6 @@
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
"integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==",
"dev": true,
"requires": {
"normalize-path": "^3.0.0",
"picomatch": "^2.0.4"
@ -15052,8 +15036,7 @@
"picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="
}
}
},
@ -15355,8 +15338,7 @@
"binary-extensions": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
"integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==",
"dev": true
"integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA=="
},
"bl": {
"version": "4.1.0",
@ -15415,7 +15397,6 @@
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
"integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
"dev": true,
"requires": {
"fill-range": "^7.0.1"
}
@ -15601,7 +15582,6 @@
"version": "3.5.3",
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz",
"integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==",
"dev": true,
"requires": {
"anymatch": "~3.1.2",
"braces": "~3.0.2",
@ -16743,7 +16723,6 @@
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
"integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
"dev": true,
"requires": {
"to-regex-range": "^5.0.1"
}
@ -16969,7 +16948,6 @@
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
"dev": true,
"optional": true
},
"function-bind": {
@ -17101,7 +17079,6 @@
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
"integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
"dev": true,
"requires": {
"is-glob": "^4.0.1"
}
@ -17372,7 +17349,6 @@
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
"integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
"dev": true,
"requires": {
"binary-extensions": "^2.0.0"
}
@ -17389,8 +17365,7 @@
"is-extglob": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
"integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
"dev": true
"integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="
},
"is-fullwidth-code-point": {
"version": "3.0.0",
@ -17407,7 +17382,6 @@
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
"integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
"dev": true,
"requires": {
"is-extglob": "^2.1.1"
}
@ -17420,8 +17394,7 @@
"is-number": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
"dev": true
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="
},
"is-path-inside": {
"version": "3.0.3",
@ -19405,7 +19378,6 @@
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
"integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==",
"dev": true,
"requires": {
"picomatch": "^2.2.1"
},
@ -19413,8 +19385,7 @@
"picomatch": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="
}
}
},
@ -20565,7 +20536,6 @@
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
"integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
"dev": true,
"requires": {
"is-number": "^7.0.0"
}

2
server/package.json
View file

@ -53,6 +53,7 @@
"axios": "^1.5.0",
"bcrypt": "^5.1.1",
"bullmq": "^4.8.0",
"chokidar": "^3.5.3",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.6",
@ -103,7 +104,6 @@
"@types/ua-parser-js": "^0.7.36",
"@typescript-eslint/eslint-plugin": "^6.4.1",
"@typescript-eslint/parser": "^6.4.1",
"chokidar": "^3.5.3",
"dotenv": "^16.3.1",
"eslint": "^8.48.0",
"eslint-config-prettier": "^9.0.0",

39
server/src/domain/library/library.service.ts
View file

@ -1,19 +1,17 @@
import { AssetType, LibraryType } from '@app/infra/entities';
import { ImmichLogger } from '@app/infra/logger';
import { BadRequestException, Inject, Injectable } from '@nestjs/common';
import picomatch from 'picomatch';
import { EventEmitter } from 'events';
import { R_OK } from 'node:constants';
import { Stats } from 'node:fs';
import path from 'node:path';
import { basename, parse } from 'path';
import picomatch from 'picomatch';
import { AccessCore, Permission } from '../access';
import { AuthDto } from '../auth';
import { mimeTypes } from '../domain.constant';
import { usePagination, validateCronExpression } from '../domain.util';
import { IBaseJob, IEntityJob, ILibraryFileJob, ILibraryRefreshJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { ImmichLogger } from '@app/infra/logger';
import { EventEmitter } from 'events';
import {
IAccessRepository,
IAssetRepository,
@ -59,31 +57,33 @@ export class LibraryService extends EventEmitter {
this.access = AccessCore.create(accessRepository);
this.configCore = SystemConfigCore.create(configRepository);
this.configCore.addValidator((config) => {
if (!validateCronExpression(config.library.scan.cronExpression)) {
throw new Error(`Invalid cron expression ${config.library.scan.cronExpression}`);
const { scan } = config.library;
if (!validateCronExpression(scan.cronExpression)) {
throw new Error(`Invalid cron expression ${scan.cronExpression}`);
}
});
}
async init() {
const config = await this.configCore.getConfig();
this.watchLibraries = config.library.watch.enabled;
const { watch, scan } = config.library;
this.watchLibraries = watch.enabled;
this.jobRepository.addCronJob(
'libraryScan',
config.library.scan.cronExpression,
scan.cronExpression,
() => this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }),
config.library.scan.enabled,
scan.enabled,
);
if (this.watchLibraries) {
await this.watchAll();
}
this.configCore.config$.subscribe(async (config) => {
this.jobRepository.updateCronJob('libraryScan', config.library.scan.cronExpression, config.library.scan.enabled);
this.configCore.config$.subscribe(async ({ library }) => {
this.jobRepository.updateCronJob('libraryScan', library.scan.cronExpression, library.scan.enabled);
if (config.library.watch.enabled !== this.watchLibraries) {
this.watchLibraries = config.library.watch.enabled;
if (library.watch.enabled !== this.watchLibraries) {
this.watchLibraries = library.watch.enabled;
if (this.watchLibraries) {
await this.watchAll();
} else {
@ -116,15 +116,14 @@ export class LibraryService extends EventEmitter {
});
const config = await this.configCore.getConfig();
const { usePolling, interval } = config.library.watch;
this.logger.debug(
`Settings for watcher: usePolling: ${config.library.watch.usePolling}, interval: ${config.library.watch.interval}`,
);
this.logger.debug(`Settings for watcher: usePolling: ${usePolling}, interval: ${interval}`);
const watcher = this.storageRepository.watch(library.importPaths, {
usePolling: config.library.watch.usePolling,
interval: config.library.watch.interval,
binaryInterval: config.library.watch.interval,
usePolling,
interval,
binaryInterval: interval,
ignoreInitial: true,
});

3
server/src/infra/entities/system-config.entity.ts
View file

@ -53,9 +53,6 @@ export enum SystemConfigKey {
LIBRARY_WATCH_ENABLED = 'library.watch.enabled',
LIBRARY_WATCH_USE_POLLING = 'library.watch.usePolling',
LIBRARY_WATCH_INTERVAL = 'library.watch.interval',
LIBRARY_WATCH_BINARY_INTERVAL = 'library.watch.binaryInterval',
LIBRARY_WATCH_WRITE_STABILITY_THRESHOLD = 'library.watch.awaitWriteFinish.stabilityThreshold',
LIBRARY_WATCH_WRITE_POLL_INTERVAL = 'library.watch.awaitWriteFinish.pollInterval',
LOGGING_ENABLED = 'logging.enabled',
LOGGING_LEVEL = 'logging.level',