feat(server): custom library scanning interval (#4390)

* add automatic library scan config options

* add validation

* open api

* use CronJob instead of cron-validator

* fix tests

* catch potential error of the library scan initialization

* better description for input field

* move library scan job initialization to server app service

* fix tests

* add comments to all parameters of cronjob contructor

* make scan a child of a more general library object

* open api

* chore: cleanup

* move cronjob handling to job repoistory

* web: select for common cron expressions

* fix open api

* fix tests

* put scanning settings in nested accordion

* fix system config validation

* refactor, tests

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

cli/src/api/open-api/api.ts

@@ -3283,6 +3283,12 @@ export interface SystemConfigDto {
      * @memberof SystemConfigDto
      */
     'job': SystemConfigJobDto;
+    /**
+     * 
+     * @type {SystemConfigLibraryDto}
+     * @memberof SystemConfigDto
+     */
+    'library': SystemConfigLibraryDto;
     /**
      * 
      * @type {SystemConfigMachineLearningDto}
@@ -3534,6 +3540,38 @@ export interface SystemConfigJobDto {
      */
     'videoConversion': JobSettingsDto;
 }
+/**
+ * 
+ * @export
+ * @interface SystemConfigLibraryDto
+ */
+export interface SystemConfigLibraryDto {
+    /**
+     * 
+     * @type {SystemConfigLibraryScanDto}
+     * @memberof SystemConfigLibraryDto
+     */
+    'scan': SystemConfigLibraryScanDto;
+}
+/**
+ * 
+ * @export
+ * @interface SystemConfigLibraryScanDto
+ */
+export interface SystemConfigLibraryScanDto {
+    /**
+     * 
+     * @type {string}
+     * @memberof SystemConfigLibraryScanDto
+     */
+    'cronExpression': string;
+    /**
+     * 
+     * @type {boolean}
+     * @memberof SystemConfigLibraryScanDto
+     */
+    'enabled': boolean;
+}
 /**
  * 
  * @export

mobile/openapi/.openapi-generator/FILES

@@ -128,6 +128,8 @@ doc/SystemConfigApi.md
 doc/SystemConfigDto.md
 doc/SystemConfigFFmpegDto.md
 doc/SystemConfigJobDto.md
+doc/SystemConfigLibraryDto.md
+doc/SystemConfigLibraryScanDto.md
 doc/SystemConfigMachineLearningDto.md
 doc/SystemConfigMapDto.md
 doc/SystemConfigNewVersionCheckDto.md
@@ -296,6 +298,8 @@ lib/model/smart_info_response_dto.dart
 lib/model/system_config_dto.dart
 lib/model/system_config_f_fmpeg_dto.dart
 lib/model/system_config_job_dto.dart
+lib/model/system_config_library_dto.dart
+lib/model/system_config_library_scan_dto.dart
 lib/model/system_config_machine_learning_dto.dart
 lib/model/system_config_map_dto.dart
 lib/model/system_config_new_version_check_dto.dart
@@ -451,6 +455,8 @@ test/system_config_api_test.dart
 test/system_config_dto_test.dart
 test/system_config_f_fmpeg_dto_test.dart
 test/system_config_job_dto_test.dart
+test/system_config_library_dto_test.dart
+test/system_config_library_scan_dto_test.dart
 test/system_config_machine_learning_dto_test.dart
 test/system_config_map_dto_test.dart
 test/system_config_new_version_check_dto_test.dart

server/immich-openapi-specs.json

@@ -8061,6 +8061,9 @@
           "job": {
             "$ref": "#/components/schemas/SystemConfigJobDto"
           },
+          "library": {
+            "$ref": "#/components/schemas/SystemConfigLibraryDto"
+          },
           "machineLearning": {
             "$ref": "#/components/schemas/SystemConfigMachineLearningDto"
           },
@@ -8104,7 +8107,8 @@
           "job",
           "thumbnail",
           "trash",
-          "theme"
+          "theme",
+          "library"
         ],
         "type": "object"
       },
@@ -8238,6 +8242,32 @@
         ],
         "type": "object"
       },
+      "SystemConfigLibraryDto": {
+        "properties": {
+          "scan": {
+            "$ref": "#/components/schemas/SystemConfigLibraryScanDto"
+          }
+        },
+        "required": [
+          "scan"
+        ],
+        "type": "object"
+      },
+      "SystemConfigLibraryScanDto": {
+        "properties": {
+          "cronExpression": {
+            "type": "string"
+          },
+          "enabled": {
+            "type": "boolean"
+          }
+        },
+        "required": [
+          "enabled",
+          "cronExpression"
+        ],
+        "type": "object"
+      },
       "SystemConfigMachineLearningDto": {
         "properties": {
           "classification": {

server/package-lock.json

@@ -1683,66 +1683,6 @@
         "darwin"
       ]
     },
-    "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz",
-      "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz",
-      "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==",
-      "cpu": [
-        "arm"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz",
-      "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz",
-      "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz",
-      "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
     "node_modules/@nestjs/bull-shared": {
       "version": "10.0.1",
       "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.0.1.tgz",
@@ -6118,15 +6058,6 @@
         "exiftool-vendored.pl": "12.67.0"
       }
     },
-    "node_modules/exiftool-vendored.exe": {
-      "version": "12.67.0",
-      "resolved": "https://registry.npmjs.org/exiftool-vendored.exe/-/exiftool-vendored.exe-12.67.0.tgz",
-      "integrity": "sha512-wzgMDoL/VWH34l38g22cVUn43mVFtTSVj0HRjfjR46+4fGwpSvSueeYbwLCZ5NvBAVINCS5Rz9Rl2DVmqoIjsw==",
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
     "node_modules/exiftool-vendored.pl": {
       "version": "12.67.0",
       "resolved": "https://registry.npmjs.org/exiftool-vendored.pl/-/exiftool-vendored.pl-12.67.0.tgz",
@@ -14300,36 +14231,6 @@
       "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==",
       "optional": true
     },
-    "@msgpackr-extract/msgpackr-extract-darwin-x64": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz",
-      "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==",
-      "optional": true
-    },
-    "@msgpackr-extract/msgpackr-extract-linux-arm": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz",
-      "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==",
-      "optional": true
-    },
-    "@msgpackr-extract/msgpackr-extract-linux-arm64": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz",
-      "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==",
-      "optional": true
-    },
-    "@msgpackr-extract/msgpackr-extract-linux-x64": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz",
-      "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==",
-      "optional": true
-    },
-    "@msgpackr-extract/msgpackr-extract-win32-x64": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz",
-      "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==",
-      "optional": true
-    },
     "@nestjs/bull-shared": {
       "version": "10.0.1",
       "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.0.1.tgz",
@@ -16944,6 +16845,11 @@
         "luxon": "^3.2.1"
       }
     },
+    "cron-validator": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/cron-validator/-/cron-validator-1.3.1.tgz",
+      "integrity": "sha512-C1HsxuPCY/5opR55G5/WNzyEGDWFVG+6GLrA+fW/sCTcP6A6NTjUP2AK7B8n2PyFs90kDG2qzwm8LMheADku6A=="
+    },
     "cross-spawn": {
       "version": "7.0.3",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
@@ -17608,12 +17514,6 @@
         }
       }
     },
-    "exiftool-vendored.exe": {
-      "version": "12.67.0",
-      "resolved": "https://registry.npmjs.org/exiftool-vendored.exe/-/exiftool-vendored.exe-12.67.0.tgz",
-      "integrity": "sha512-wzgMDoL/VWH34l38g22cVUn43mVFtTSVj0HRjfjR46+4fGwpSvSueeYbwLCZ5NvBAVINCS5Rz9Rl2DVmqoIjsw==",
-      "optional": true
-    },
     "exiftool-vendored.pl": {
       "version": "12.67.0",
       "resolved": "https://registry.npmjs.org/exiftool-vendored.pl/-/exiftool-vendored.pl-12.67.0.tgz",

server/src/domain/domain.util.ts

@@ -1,6 +1,7 @@
 import { applyDecorators } from '@nestjs/common';
 import { ApiProperty } from '@nestjs/swagger';
 import { IsArray, IsNotEmpty, IsOptional, IsString, IsUUID, ValidateIf, ValidationOptions } from 'class-validator';
+import { CronJob } from 'cron';
 import { basename, extname } from 'node:path';
 import sanitize from 'sanitize-filename';
 
@@ -18,6 +19,16 @@ export function ValidateUUID({ optional, each }: Options = { optional: false, ea
   );
 }
 
+export function validateCronExpression(expression: string) {
+  try {
+    new CronJob(expression, () => {});
+  } catch (error) {
+    return false;
+  }
+
+  return true;
+}
+
 interface IValue {
   value?: string;
 }

server/src/domain/job/job.service.spec.ts

@@ -61,7 +61,6 @@ describe(JobService.name, () => {
         [{ name: JobName.PERSON_CLEANUP }],
         [{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }],
         [{ name: JobName.CLEAN_OLD_AUDIT_LOGS }],
-        [{ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }],
       ]);
     });
   });

server/src/domain/job/job.service.ts

@@ -153,7 +153,6 @@ export class JobService {
     await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
     await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
     await this.jobRepository.queue({ name: JobName.CLEAN_OLD_AUDIT_LOGS });
-    await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } });
   }
 
   /**

server/src/domain/library/library.service.spec.ts

@@ -1,4 +1,4 @@
-import { AssetType, LibraryType, UserEntity } from '@app/infra/entities';
+import { AssetType, LibraryType, SystemConfig, SystemConfigKey, UserEntity } from '@app/infra/entities';
 import { BadRequestException } from '@nestjs/common';
 
 import {
@@ -12,6 +12,7 @@ import {
   newJobRepositoryMock,
   newLibraryRepositoryMock,
   newStorageRepositoryMock,
+  newSystemConfigRepositoryMock,
   newUserRepositoryMock,
   userStub,
 } from '@test';
@@ -23,8 +24,10 @@ import {
   IJobRepository,
   ILibraryRepository,
   IStorageRepository,
+  ISystemConfigRepository,
   IUserRepository,
 } from '../repositories';
+import { SystemConfigCore } from '../system-config/system-config.core';
 import { LibraryService } from './library.service';
 
 describe(LibraryService.name, () => {
@@ -32,6 +35,7 @@ describe(LibraryService.name, () => {
 
   let accessMock: IAccessRepositoryMock;
   let assetMock: jest.Mocked<IAssetRepository>;
+  let configMock: jest.Mocked<ISystemConfigRepository>;
   let cryptoMock: jest.Mocked<ICryptoRepository>;
   let userMock: jest.Mocked<IUserRepository>;
   let jobMock: jest.Mocked<IJobRepository>;
@@ -40,6 +44,7 @@ describe(LibraryService.name, () => {
 
   beforeEach(() => {
     accessMock = newAccessRepositoryMock();
+    configMock = newSystemConfigRepositoryMock();
     libraryMock = newLibraryRepositoryMock();
     userMock = newUserRepositoryMock();
     assetMock = newAssetRepositoryMock();
@@ -55,13 +60,46 @@ describe(LibraryService.name, () => {
 
     accessMock.library.hasOwnerAccess.mockResolvedValue(true);
 
-    sut = new LibraryService(accessMock, assetMock, cryptoMock, jobMock, libraryMock, storageMock, userMock);
+    sut = new LibraryService(
+      accessMock,
+      assetMock,
+      configMock,
+      cryptoMock,
+      jobMock,
+      libraryMock,
+      storageMock,
+      userMock,
+    );
   });
 
   it('should work', () => {
     expect(sut).toBeDefined();
   });
 
+  describe('init', () => {
+    it('should init cron job and subscribe to config changes', async () => {
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.LIBRARY_SCAN_ENABLED, value: true },
+        { key: SystemConfigKey.LIBRARY_SCAN_CRON_EXPRESSION, value: '0 0 * * *' },
+      ]);
+
+      await sut.init();
+      expect(configMock.load).toHaveBeenCalled();
+      expect(jobMock.addCronJob).toHaveBeenCalled();
+
+      SystemConfigCore.create(newSystemConfigRepositoryMock(false)).config$.next({
+        library: {
+          scan: {
+            enabled: true,
+            cronExpression: '0 1 * * *',
+          },
+        },
+      } as SystemConfig);
+
+      expect(jobMock.updateCronJob).toHaveBeenCalledWith('libraryScan', '0 1 * * *', true);
+    });
+  });
+
   describe('handleQueueAssetRefresh', () => {
     it("should not queue assets outside of user's external path", async () => {
       const mockLibraryJob: ILibraryRefreshJob = {

server/src/domain/library/library.service.ts

@@ -7,7 +7,7 @@ import { basename, parse } from 'path';
 import { AccessCore, Permission } from '../access';
 import { AuthUserDto } from '../auth';
 import { mimeTypes } from '../domain.constant';
-import { usePagination } from '../domain.util';
+import { usePagination, validateCronExpression } from '../domain.util';
 import { IBaseJob, IEntityJob, ILibraryFileJob, ILibraryRefreshJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
 
 import {
@@ -17,9 +17,11 @@ import {
   IJobRepository,
   ILibraryRepository,
   IStorageRepository,
+  ISystemConfigRepository,
   IUserRepository,
   WithProperty,
 } from '../repositories';
+import { SystemConfigCore } from '../system-config';
 import {
   CreateLibraryDto,
   LibraryResponseDto,
@@ -33,10 +35,12 @@ import {
 export class LibraryService {
   readonly logger = new Logger(LibraryService.name);
   private access: AccessCore;
+  private configCore: SystemConfigCore;
 
   constructor(
     @Inject(IAccessRepository) accessRepository: IAccessRepository,
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
     @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
     @Inject(IJobRepository) private jobRepository: IJobRepository,
     @Inject(ILibraryRepository) private repository: ILibraryRepository,
@@ -44,6 +48,26 @@ export class LibraryService {
     @Inject(IUserRepository) private userRepository: IUserRepository,
   ) {
     this.access = AccessCore.create(accessRepository);
+    this.configCore = SystemConfigCore.create(configRepository);
+    this.configCore.addValidator((config) => {
+      if (!validateCronExpression(config.library.scan.cronExpression)) {
+        throw new Error(`Invalid cron expression ${config.library.scan.cronExpression}`);
+      }
+    });
+  }
+
+  async init() {
+    const config = await this.configCore.getConfig();
+    this.jobRepository.addCronJob(
+      'libraryScan',
+      config.library.scan.cronExpression,
+      () => this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }),
+      config.library.scan.enabled,
+    );
+
+    this.configCore.config$.subscribe((config) => {
+      this.jobRepository.updateCronJob('libraryScan', config.library.scan.cronExpression, config.library.scan.enabled);
+    });
   }
 
   async getStatistics(authUser: AuthUserDto, id: string): Promise<LibraryStatsResponseDto> {

server/src/domain/repositories/job.repository.ts

@@ -111,6 +111,9 @@ export const IJobRepository = 'IJobRepository';
 
 export interface IJobRepository {
   addHandler(queueName: QueueName, concurrency: number, handler: JobItemHandler): void;
+  addCronJob(name: string, expression: string, onTick: () => void, start?: boolean): void;
+  updateCronJob(name: string, expression?: string, start?: boolean): void;
+  deleteCronJob(name: string): void;
   setConcurrency(queueName: QueueName, concurrency: number): void;
   queue(item: JobItem): Promise<void>;
   pause(name: QueueName): Promise<void>;

server/src/domain/system-config/dto/index.ts

@@ -1,4 +1,5 @@
 export * from './system-config-ffmpeg.dto';
+export * from './system-config-library.dto';
 export * from './system-config-oauth.dto';
 export * from './system-config-password-login.dto';
 export * from './system-config-storage-template.dto';

server/src/domain/system-config/dto/system-config-library.dto.ts

@@ -0,0 +1,40 @@
+import { validateCronExpression } from '@app/domain';
+import { Type } from 'class-transformer';
+import {
+  IsBoolean,
+  IsNotEmpty,
+  IsObject,
+  IsString,
+  Validate,
+  ValidateIf,
+  ValidateNested,
+  ValidatorConstraint,
+  ValidatorConstraintInterface,
+} from 'class-validator';
+
+const isEnabled = (config: SystemConfigLibraryScanDto) => config.enabled;
+
+@ValidatorConstraint({ name: 'cronValidator' })
+class CronValidator implements ValidatorConstraintInterface {
+  validate(expression: string): boolean {
+    return validateCronExpression(expression);
+  }
+}
+
+export class SystemConfigLibraryScanDto {
+  @IsBoolean()
+  enabled!: boolean;
+
+  @ValidateIf(isEnabled)
+  @IsNotEmpty()
+  @Validate(CronValidator, { message: 'Invalid cron expression' })
+  @IsString()
+  cronExpression!: string;
+}
+
+export class SystemConfigLibraryDto {
+  @Type(() => SystemConfigLibraryScanDto)
+  @ValidateNested()
+  @IsObject()
+  scan!: SystemConfigLibraryScanDto;
+}

server/src/domain/system-config/dto/system-config.dto.ts

@@ -3,6 +3,7 @@ import { Type } from 'class-transformer';
 import { IsObject, ValidateNested } from 'class-validator';
 import { SystemConfigFFmpegDto } from './system-config-ffmpeg.dto';
 import { SystemConfigJobDto } from './system-config-job.dto';
+import { SystemConfigLibraryDto } from './system-config-library.dto';
 import { SystemConfigMachineLearningDto } from './system-config-machine-learning.dto';
 import { SystemConfigMapDto } from './system-config-map.dto';
 import { SystemConfigNewVersionCheckDto } from './system-config-new-version-check.dto';
@@ -74,6 +75,11 @@ export class SystemConfigDto implements SystemConfig {
   @ValidateNested()
   @IsObject()
   theme!: SystemConfigThemeDto;
+
+  @Type(() => SystemConfigLibraryDto)
+  @ValidateNested()
+  @IsObject()
+  library!: SystemConfigLibraryDto;
 }
 
 export function mapConfig(config: SystemConfig): SystemConfigDto {

server/src/domain/system-config/system-config.core.ts

@@ -13,6 +13,7 @@ import {
   VideoCodec,
 } from '@app/infra/entities';
 import { BadRequestException, ForbiddenException, Injectable, Logger } from '@nestjs/common';
+import { CronExpression } from '@nestjs/schedule';
 import { plainToInstance } from 'class-transformer';
 import { validate } from 'class-validator';
 import * as _ from 'lodash';
@@ -120,6 +121,12 @@ export const defaults = Object.freeze<SystemConfig>({
   theme: {
     customCss: '',
   },
+  library: {
+    scan: {
+      enabled: true,
+      cronExpression: CronExpression.EVERY_DAY_AT_MIDNIGHT,
+    },
+  },
 });
 
 export enum FeatureFlag {

server/src/domain/system-config/system-config.service.spec.ts

@@ -121,6 +121,12 @@ const updatedConfig = Object.freeze<SystemConfig>({
   theme: {
     customCss: '',
   },
+  library: {
+    scan: {
+      enabled: true,
+      cronExpression: '0 0 * * *',
+    },
+  },
 });
 
 describe(SystemConfigService.name, () => {

server/src/immich/app.service.ts

@@ -1,4 +1,4 @@
-import { JobService, ONE_HOUR, SearchService, ServerInfoService, StorageService } from '@app/domain';
+import { JobService, LibraryService, ONE_HOUR, SearchService, ServerInfoService, StorageService } from '@app/domain';
 import { Injectable, Logger } from '@nestjs/common';
 import { Cron, CronExpression, Interval } from '@nestjs/schedule';
 
@@ -8,6 +8,7 @@ export class AppService {
 
   constructor(
     private jobService: JobService,
+    private libraryService: LibraryService,
     private searchService: SearchService,
     private storageService: StorageService,
     private serverService: ServerInfoService,
@@ -28,6 +29,7 @@ export class AppService {
     await this.searchService.init();
     await this.serverService.handleVersionCheck();
     this.logger.log(`Feature Flags: ${JSON.stringify(await this.serverService.getFeatures(), null, 2)}`);
+    await this.libraryService.init();
   }
 
   async destroy() {

server/src/infra/entities/system-config.entity.ts

@@ -94,6 +94,9 @@ export enum SystemConfigKey {
   TRASH_DAYS = 'trash.days',
 
   THEME_CUSTOM_CSS = 'theme.customCss',
+
+  LIBRARY_SCAN_ENABLED = 'library.scan.enabled',
+  LIBRARY_SCAN_CRON_EXPRESSION = 'library.scan.cronExpression',
 }
 
 export enum TranscodePolicy {
@@ -232,4 +235,10 @@ export interface SystemConfig {
   theme: {
     customCss: string;
   };
+  library: {
+    scan: {
+      enabled: boolean;
+      cronExpression: string;
+    };
+  };
 }

server/src/infra/repositories/job.repository.ts

@@ -2,7 +2,9 @@ import { IJobRepository, JobCounts, JobItem, JobName, JOBS_TO_QUEUE, QueueName,
 import { getQueueToken } from '@nestjs/bullmq';
 import { Injectable, Logger } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
+import { SchedulerRegistry } from '@nestjs/schedule';
 import { Job, JobsOptions, Processor, Queue, Worker, WorkerOptions } from 'bullmq';
+import { CronJob, CronTime } from 'cron';
 import { bullConfig } from '../infra.config';
 
 @Injectable()
@@ -10,7 +12,10 @@ export class JobRepository implements IJobRepository {
   private workers: Partial<Record<QueueName, Worker>> = {};
   private logger = new Logger(JobRepository.name);
 
-  constructor(private moduleRef: ModuleRef) {}
+  constructor(
+    private moduleRef: ModuleRef,
+    private schedulerReqistry: SchedulerRegistry,
+  ) {}
 
   addHandler(queueName: QueueName, concurrency: number, handler: (item: JobItem) => Promise<void>) {
     const workerHandler: Processor = async (job: Job) => handler(job as JobItem);
@@ -18,6 +23,43 @@ export class JobRepository implements IJobRepository {
     this.workers[queueName] = new Worker(queueName, workerHandler, workerOptions);
   }
 
+  addCronJob(name: string, expression: string, onTick: () => void, start = true): void {
+    const job = new CronJob(
+      expression,
+      onTick,
+      // function to run onComplete
+      undefined,
+      // whether it should start directly
+      start,
+      // timezone
+      undefined,
+      // context
+      undefined,
+      // runOnInit
+      undefined,
+      // utcOffset
+      undefined,
+      // prevents memory leaking by automatically stopping when the node process finishes
+      true,
+    );
+
+    this.schedulerReqistry.addCronJob(name, job);
+  }
+
+  updateCronJob(name: string, expression?: string, start?: boolean): void {
+    const job = this.schedulerReqistry.getCronJob(name);
+    if (expression) {
+      job.setTime(new CronTime(expression));
+    }
+    if (start !== undefined) {
+      start ? job.start() : job.stop();
+    }
+  }
+
+  deleteCronJob(name: string): void {
+    this.schedulerReqistry.deleteCronJob(name);
+  }
+
   setConcurrency(queueName: QueueName, concurrency: number) {
     const worker = this.workers[queueName];
     if (!worker) {

server/test/repositories/job.repository.mock.ts

@@ -3,6 +3,9 @@ import { IJobRepository } from '@app/domain';
 export const newJobRepositoryMock = (): jest.Mocked<IJobRepository> => {
   return {
     addHandler: jest.fn(),
+    addCronJob: jest.fn(),
+    deleteCronJob: jest.fn(),
+    updateCronJob: jest.fn(),
     setConcurrency: jest.fn(),
     empty: jest.fn(),
     pause: jest.fn(),

server/test/test-utils.ts

@@ -49,6 +49,10 @@ export const testApp = {
       .overrideProvider(IJobRepository)
       .useValue({
         addHandler: (_queueName: QueueName, _concurrency: number, handler: JobItemHandler) => (_handler = handler),
+        addCronJob: jest.fn(),
+        updateCronJob: jest.fn(),
+        deleteCronJob: jest.fn(),
+        validateCronExpression: jest.fn(),
         queue: (item: JobItem) => jobs && _handler(item),
         resume: jest.fn(),
         empty: jest.fn(),

web/src/api/open-api/api.ts

@@ -3283,6 +3283,12 @@ export interface SystemConfigDto {
      * @memberof SystemConfigDto
      */
     'job': SystemConfigJobDto;
+    /**
+     * 
+     * @type {SystemConfigLibraryDto}
+     * @memberof SystemConfigDto
+     */
+    'library': SystemConfigLibraryDto;
     /**
      * 
      * @type {SystemConfigMachineLearningDto}
@@ -3534,6 +3540,38 @@ export interface SystemConfigJobDto {
      */
     'videoConversion': JobSettingsDto;
 }
+/**
+ * 
+ * @export
+ * @interface SystemConfigLibraryDto
+ */
+export interface SystemConfigLibraryDto {
+    /**
+     * 
+     * @type {SystemConfigLibraryScanDto}
+     * @memberof SystemConfigLibraryDto
+     */
+    'scan': SystemConfigLibraryScanDto;
+}
+/**
+ * 
+ * @export
+ * @interface SystemConfigLibraryScanDto
+ */
+export interface SystemConfigLibraryScanDto {
+    /**
+     * 
+     * @type {string}
+     * @memberof SystemConfigLibraryScanDto
+     */
+    'cronExpression': string;
+    /**
+     * 
+     * @type {boolean}
+     * @memberof SystemConfigLibraryScanDto
+     */
+    'enabled': boolean;
+}
 /**
  * 
  * @export

web/src/lib/components/admin-page/settings/library-settings/library-settings.svelte

@@ -0,0 +1,145 @@
+<script lang="ts">
+  import {
+    notificationController,
+    NotificationType,
+  } from '$lib/components/shared-components/notification/notification';
+  import { api, SystemConfigLibraryDto } from '@api';
+  import SettingButtonsRow from '../setting-buttons-row.svelte';
+  import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
+  import SettingSwitch from '../setting-switch.svelte';
+  import { isEqual } from 'lodash-es';
+  import { fade } from 'svelte/transition';
+  import { handleError } from '../../../../utils/handle-error';
+  import SettingAccordion from '../setting-accordion.svelte';
+
+  export let libraryConfig: SystemConfigLibraryDto; // this is the config that is being edited
+  export let disabled = false;
+
+  const cronExpressionOptions = [
+    { title: 'Every night at midnight', expression: '0 0 * * *' },
+    { title: 'Every night at 2am', expression: '0 2 * * *' },
+    { title: 'Every day at 1pm', expression: '0 13 * * *' },
+    { title: 'Every 6 hours', expression: '0 */6 * * *' },
+  ];
+
+  let savedConfig: SystemConfigLibraryDto;
+  let defaultConfig: SystemConfigLibraryDto;
+
+  async function getConfigs() {
+    [savedConfig, defaultConfig] = await Promise.all([
+      api.systemConfigApi.getConfig().then((res) => res.data.library),
+      api.systemConfigApi.getDefaults().then((res) => res.data.library),
+    ]);
+  }
+
+  async function saveSetting() {
+    try {
+      const { data: configs } = await api.systemConfigApi.getConfig();
+
+      const result = await api.systemConfigApi.updateConfig({
+        systemConfigDto: {
+          ...configs,
+          library: libraryConfig,
+        },
+      });
+
+      libraryConfig = { ...result.data.library };
+      savedConfig = { ...result.data.library };
+
+      notificationController.show({
+        message: 'Library settings saved',
+        type: NotificationType.Info,
+      });
+    } catch (e) {
+      handleError(e, 'Unable to save settings');
+    }
+  }
+
+  async function reset() {
+    const { data: resetConfig } = await api.systemConfigApi.getConfig();
+
+    libraryConfig = { ...resetConfig.library };
+    savedConfig = { ...resetConfig.library };
+
+    notificationController.show({
+      message: 'Reset library settings to the recent saved settings',
+      type: NotificationType.Info,
+    });
+  }
+
+  async function resetToDefault() {
+    const { data: configs } = await api.systemConfigApi.getDefaults();
+
+    libraryConfig = { ...configs.library };
+    defaultConfig = { ...configs.library };
+
+    notificationController.show({
+      message: 'Reset library settings to default',
+      type: NotificationType.Info,
+    });
+  }
+</script>
+
+<div>
+  {#await getConfigs() then}
+    <div in:fade={{ duration: 500 }}>
+      <SettingAccordion title="Scanning" subtitle="Settings for library scanning" isOpen>
+        <form autocomplete="off" on:submit|preventDefault>
+          <div class="ml-4 mt-4 flex flex-col gap-4">
+            <SettingSwitch
+              title="ENABLED"
+              {disabled}
+              subtitle="Enable automatic library scanning"
+              bind:checked={libraryConfig.scan.enabled}
+            />
+
+            <div class="flex flex-col my-2 dark:text-immich-dark-fg">
+              <label class="text-sm" for="expression-select">Cron Expression Presets</label>
+              <select
+                class="p-2 mt-2 text-sm rounded-lg bg-slate-200 hover:cursor-pointer dark:bg-gray-600"
+                disabled={disabled || !libraryConfig.scan.enabled}
+                name="expression"
+                id="expression-select"
+                bind:value={libraryConfig.scan.cronExpression}
+              >
+                {#each cronExpressionOptions as { title, expression }}
+                  <option value={expression}>{title}</option>
+                {/each}
+              </select>
+            </div>
+
+            <SettingInputField
+              inputType={SettingInputFieldType.TEXT}
+              required={true}
+              disabled={disabled || !libraryConfig.scan.enabled}
+              label="Cron Expression"
+              bind:value={libraryConfig.scan.cronExpression}
+              isEdited={libraryConfig.scan.cronExpression !== savedConfig.scan.cronExpression}
+            >
+              <svelte:fragment slot="desc">
+                <p class="text-sm dark:text-immich-dark-fg">
+                  Set the scanning interval using the cron format. For more information please refer to e.g. <a
+                    href="https://crontab.guru"
+                    class="underline"
+                    target="_blank"
+                    rel="noreferrer">Crontab Guru</a
+                  >
+                </p>
+              </svelte:fragment>
+            </SettingInputField>
+          </div>
+
+          <div class="ml-4">
+            <SettingButtonsRow
+              on:reset={reset}
+              on:save={saveSetting}
+              on:reset-to-default={resetToDefault}
+              showResetToDefault={!isEqual(savedConfig, defaultConfig)}
+              {disabled}
+            />
+          </div>
+        </form>
+      </SettingAccordion>
+    </div>
+  {/await}
+</div>

web/src/routes/admin/system-settings/+page.svelte

@@ -20,6 +20,7 @@
   import Icon from '$lib/components/elements/icon.svelte';
   import type { PageData } from './$types';
   import NewVersionCheckSettings from '$lib/components/admin-page/settings/new-version-check-settings/new-version-check-settings.svelte';
+  import LibrarySettings from '$lib/components/admin-page/settings/library-settings/library-settings.svelte';
   import { mdiAlert, mdiContentCopy, mdiDownload } from '@mdi/js';
 
   export let data: PageData;
@@ -69,6 +70,10 @@
         <JobSettings disabled={$featureFlags.configFile} jobConfig={configs.job} />
       </SettingAccordion>
 
+      <SettingAccordion title="Library" subtitle="Manage library settings">
+        <LibrarySettings disabled={$featureFlags.configFile} libraryConfig={configs.library} />
+      </SettingAccordion>
+
       <SettingAccordion title="Machine Learning Settings" subtitle="Manage machine learning features and settings">
         <MachineLearningSettings disabled={$featureFlags.configFile} machineLearningConfig={configs.machineLearning} />
       </SettingAccordion>