c-l/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-04-10 10:06:25 +02:00
Code Releases Activity

feat(server): dynamic job concurrency ()

Browse source 
* feat(server): dynamic job concurrency

* styling and add setting info to top of the job list

* regenerate api

* remove DETECT_OBJECT job

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Jason Rasmussen 2023-06-01 06:32:51 -04:00 • committed by GitHub
parent 656dc08406
commit 2493dfaba3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
48 changed files with 1454 additions and 490 deletions

6
mobile/openapi/.openapi-generator/FILES generated
View file

@ -57,6 +57,7 @@ doc/JobCommand.md
doc/JobCommandDto.md
doc/JobCountsDto.md
doc/JobName.md
doc/JobSettingsDto.md
doc/JobStatusDto.md
doc/LoginCredentialDto.md
doc/LoginResponseDto.md
@ -95,6 +96,7 @@ doc/SmartInfoResponseDto.md
doc/SystemConfigApi.md
doc/SystemConfigDto.md
doc/SystemConfigFFmpegDto.md
doc/SystemConfigJobDto.md
doc/SystemConfigOAuthDto.md
doc/SystemConfigPasswordLoginDto.md
doc/SystemConfigStorageTemplateDto.md
@ -186,6 +188,7 @@ lib/model/job_command.dart
lib/model/job_command_dto.dart
lib/model/job_counts_dto.dart
lib/model/job_name.dart
lib/model/job_settings_dto.dart
lib/model/job_status_dto.dart
lib/model/login_credential_dto.dart
lib/model/login_response_dto.dart
@ -217,6 +220,7 @@ lib/model/sign_up_dto.dart
lib/model/smart_info_response_dto.dart
lib/model/system_config_dto.dart
lib/model/system_config_f_fmpeg_dto.dart
lib/model/system_config_job_dto.dart
lib/model/system_config_o_auth_dto.dart
lib/model/system_config_password_login_dto.dart
lib/model/system_config_storage_template_dto.dart
@ -288,6 +292,7 @@ test/job_command_dto_test.dart
test/job_command_test.dart
test/job_counts_dto_test.dart
test/job_name_test.dart
test/job_settings_dto_test.dart
test/job_status_dto_test.dart
test/login_credential_dto_test.dart
test/login_response_dto_test.dart
@ -326,6 +331,7 @@ test/smart_info_response_dto_test.dart
test/system_config_api_test.dart
test/system_config_dto_test.dart
test/system_config_f_fmpeg_dto_test.dart
test/system_config_job_dto_test.dart
test/system_config_o_auth_dto_test.dart
test/system_config_password_login_dto_test.dart
test/system_config_storage_template_dto_test.dart

2
mobile/openapi/README.md generated
View file

@ -225,6 +225,7 @@ Class | Method | HTTP request | Description
- [JobCommandDto](doc//JobCommandDto.md)
- [JobCountsDto](doc//JobCountsDto.md)
- [JobName](doc//JobName.md)
- [JobSettingsDto](doc//JobSettingsDto.md)
- [JobStatusDto](doc//JobStatusDto.md)
- [LoginCredentialDto](doc//LoginCredentialDto.md)
- [LoginResponseDto](doc//LoginResponseDto.md)
@ -256,6 +257,7 @@ Class | Method | HTTP request | Description
- [SmartInfoResponseDto](doc//SmartInfoResponseDto.md)
- [SystemConfigDto](doc//SystemConfigDto.md)
- [SystemConfigFFmpegDto](doc//SystemConfigFFmpegDto.md)
- [SystemConfigJobDto](doc//SystemConfigJobDto.md)
- [SystemConfigOAuthDto](doc//SystemConfigOAuthDto.md)
- [SystemConfigPasswordLoginDto](doc//SystemConfigPasswordLoginDto.md)
- [SystemConfigStorageTemplateDto](doc//SystemConfigStorageTemplateDto.md)

20
mobile/openapi/doc/AllJobStatusResponseDto.md generated
View file

@ -8,16 +8,16 @@ import 'package:openapi/api.dart';
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**thumbnailGenerationQueue** | [**JobStatusDto**](JobStatusDto.md) | |
**metadataExtractionQueue** | [**JobStatusDto**](JobStatusDto.md) | |
**videoConversionQueue** | [**JobStatusDto**](JobStatusDto.md) | |
**objectTaggingQueue** | [**JobStatusDto**](JobStatusDto.md) | |
**clipEncodingQueue** | [**JobStatusDto**](JobStatusDto.md) | |
**storageTemplateMigrationQueue** | [**JobStatusDto**](JobStatusDto.md) | |
**backgroundTaskQueue** | [**JobStatusDto**](JobStatusDto.md) | |
**searchQueue** | [**JobStatusDto**](JobStatusDto.md) | |
**recognizeFacesQueue** | [**JobStatusDto**](JobStatusDto.md) | |
**sidecarQueue** | [**JobStatusDto**](JobStatusDto.md) | |
**thumbnailGeneration** | [**JobStatusDto**](JobStatusDto.md) | |
**metadataExtraction** | [**JobStatusDto**](JobStatusDto.md) | |
**videoConversion** | [**JobStatusDto**](JobStatusDto.md) | |
**objectTagging** | [**JobStatusDto**](JobStatusDto.md) | |
**clipEncoding** | [**JobStatusDto**](JobStatusDto.md) | |
**storageTemplateMigration** | [**JobStatusDto**](JobStatusDto.md) | |
**backgroundTask** | [**JobStatusDto**](JobStatusDto.md) | |
**search** | [**JobStatusDto**](JobStatusDto.md) | |
**recognizeFaces** | [**JobStatusDto**](JobStatusDto.md) | |
**sidecar** | [**JobStatusDto**](JobStatusDto.md) | |
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

15
mobile/openapi/doc/JobSettingsDto.md generated Normal file
View file

@ -0,0 +1,15 @@
# openapi.model.JobSettingsDto
## Load the model package
```dart
import 'package:openapi/api.dart';
```
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**concurrency** | **int** | |
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

1
mobile/openapi/doc/SystemConfigDto.md generated
View file

@ -12,6 +12,7 @@ Name | Type | Description | Notes
**oauth** | [**SystemConfigOAuthDto**](SystemConfigOAuthDto.md) | |
**passwordLogin** | [**SystemConfigPasswordLoginDto**](SystemConfigPasswordLoginDto.md) | |
**storageTemplate** | [**SystemConfigStorageTemplateDto**](SystemConfigStorageTemplateDto.md) | |
**job** | [**SystemConfigJobDto**](SystemConfigJobDto.md) | |
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

24
mobile/openapi/doc/SystemConfigJobDto.md generated Normal file
View file

@ -0,0 +1,24 @@
# openapi.model.SystemConfigJobDto
## Load the model package
```dart
import 'package:openapi/api.dart';
```
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**thumbnailGeneration** | [**JobSettingsDto**](JobSettingsDto.md) | |
**metadataExtraction** | [**JobSettingsDto**](JobSettingsDto.md) | |
**videoConversion** | [**JobSettingsDto**](JobSettingsDto.md) | |
**objectTagging** | [**JobSettingsDto**](JobSettingsDto.md) | |
**clipEncoding** | [**JobSettingsDto**](JobSettingsDto.md) | |
**storageTemplateMigration** | [**JobSettingsDto**](JobSettingsDto.md) | |
**backgroundTask** | [**JobSettingsDto**](JobSettingsDto.md) | |
**search** | [**JobSettingsDto**](JobSettingsDto.md) | |
**recognizeFaces** | [**JobSettingsDto**](JobSettingsDto.md) | |
**sidecar** | [**JobSettingsDto**](JobSettingsDto.md) | |
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

2
mobile/openapi/lib/api.dart generated
View file

@ -92,6 +92,7 @@ part 'model/job_command.dart';
part 'model/job_command_dto.dart';
part 'model/job_counts_dto.dart';
part 'model/job_name.dart';
part 'model/job_settings_dto.dart';
part 'model/job_status_dto.dart';
part 'model/login_credential_dto.dart';
part 'model/login_response_dto.dart';
@ -123,6 +124,7 @@ part 'model/sign_up_dto.dart';
part 'model/smart_info_response_dto.dart';
part 'model/system_config_dto.dart';
part 'model/system_config_f_fmpeg_dto.dart';
part 'model/system_config_job_dto.dart';
part 'model/system_config_o_auth_dto.dart';
part 'model/system_config_password_login_dto.dart';
part 'model/system_config_storage_template_dto.dart';

4
mobile/openapi/lib/api_client.dart generated
View file

@ -279,6 +279,8 @@ class ApiClient {
return JobCountsDto.fromJson(value);
case 'JobName':
return JobNameTypeTransformer().decode(value);
case 'JobSettingsDto':
return JobSettingsDto.fromJson(value);
case 'JobStatusDto':
return JobStatusDto.fromJson(value);
case 'LoginCredentialDto':
@ -341,6 +343,8 @@ class ApiClient {
return SystemConfigDto.fromJson(value);
case 'SystemConfigFFmpegDto':
return SystemConfigFFmpegDto.fromJson(value);
case 'SystemConfigJobDto':
return SystemConfigJobDto.fromJson(value);
case 'SystemConfigOAuthDto':
return SystemConfigOAuthDto.fromJson(value);
case 'SystemConfigPasswordLoginDto':

142
mobile/openapi/lib/model/all_job_status_response_dto.dart generated
View file

@ -13,80 +13,80 @@ part of openapi.api;
class AllJobStatusResponseDto {
/// Returns a new [AllJobStatusResponseDto] instance.
AllJobStatusResponseDto({
required this.thumbnailGenerationQueue,
required this.metadataExtractionQueue,
required this.videoConversionQueue,
required this.objectTaggingQueue,
required this.clipEncodingQueue,
required this.storageTemplateMigrationQueue,
required this.backgroundTaskQueue,
required this.searchQueue,
required this.recognizeFacesQueue,
required this.sidecarQueue,
required this.thumbnailGeneration,
required this.metadataExtraction,
required this.videoConversion,
required this.objectTagging,
required this.clipEncoding,
required this.storageTemplateMigration,
required this.backgroundTask,
required this.search,
required this.recognizeFaces,
required this.sidecar,
});
JobStatusDto thumbnailGenerationQueue;
JobStatusDto thumbnailGeneration;
JobStatusDto metadataExtractionQueue;
JobStatusDto metadataExtraction;
JobStatusDto videoConversionQueue;
JobStatusDto videoConversion;
JobStatusDto objectTaggingQueue;
JobStatusDto objectTagging;
JobStatusDto clipEncodingQueue;
JobStatusDto clipEncoding;
JobStatusDto storageTemplateMigrationQueue;
JobStatusDto storageTemplateMigration;
JobStatusDto backgroundTaskQueue;
JobStatusDto backgroundTask;
JobStatusDto searchQueue;
JobStatusDto search;
JobStatusDto recognizeFacesQueue;
JobStatusDto recognizeFaces;
JobStatusDto sidecarQueue;
JobStatusDto sidecar;
@override
bool operator ==(Object other) => identical(this, other) || other is AllJobStatusResponseDto &&
other.thumbnailGenerationQueue == thumbnailGenerationQueue &&
other.metadataExtractionQueue == metadataExtractionQueue &&
other.videoConversionQueue == videoConversionQueue &&
other.objectTaggingQueue == objectTaggingQueue &&
other.clipEncodingQueue == clipEncodingQueue &&
other.storageTemplateMigrationQueue == storageTemplateMigrationQueue &&
other.backgroundTaskQueue == backgroundTaskQueue &&
other.searchQueue == searchQueue &&
other.recognizeFacesQueue == recognizeFacesQueue &&
other.sidecarQueue == sidecarQueue;
other.thumbnailGeneration == thumbnailGeneration &&
other.metadataExtraction == metadataExtraction &&
other.videoConversion == videoConversion &&
other.objectTagging == objectTagging &&
other.clipEncoding == clipEncoding &&
other.storageTemplateMigration == storageTemplateMigration &&
other.backgroundTask == backgroundTask &&
other.search == search &&
other.recognizeFaces == recognizeFaces &&
other.sidecar == sidecar;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(thumbnailGenerationQueue.hashCode) +
(metadataExtractionQueue.hashCode) +
(videoConversionQueue.hashCode) +
(objectTaggingQueue.hashCode) +
(clipEncodingQueue.hashCode) +
(storageTemplateMigrationQueue.hashCode) +
(backgroundTaskQueue.hashCode) +
(searchQueue.hashCode) +
(recognizeFacesQueue.hashCode) +
(sidecarQueue.hashCode);
(thumbnailGeneration.hashCode) +
(metadataExtraction.hashCode) +
(videoConversion.hashCode) +
(objectTagging.hashCode) +
(clipEncoding.hashCode) +
(storageTemplateMigration.hashCode) +
(backgroundTask.hashCode) +
(search.hashCode) +
(recognizeFaces.hashCode) +
(sidecar.hashCode);
@override
String toString() => 'AllJobStatusResponseDto[thumbnailGenerationQueue=$thumbnailGenerationQueue, metadataExtractionQueue=$metadataExtractionQueue, videoConversionQueue=$videoConversionQueue, objectTaggingQueue=$objectTaggingQueue, clipEncodingQueue=$clipEncodingQueue, storageTemplateMigrationQueue=$storageTemplateMigrationQueue, backgroundTaskQueue=$backgroundTaskQueue, searchQueue=$searchQueue, recognizeFacesQueue=$recognizeFacesQueue, sidecarQueue=$sidecarQueue]';
String toString() => 'AllJobStatusResponseDto[thumbnailGeneration=$thumbnailGeneration, metadataExtraction=$metadataExtraction, videoConversion=$videoConversion, objectTagging=$objectTagging, clipEncoding=$clipEncoding, storageTemplateMigration=$storageTemplateMigration, backgroundTask=$backgroundTask, search=$search, recognizeFaces=$recognizeFaces, sidecar=$sidecar]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'thumbnail-generation-queue'] = this.thumbnailGenerationQueue;
json[r'metadata-extraction-queue'] = this.metadataExtractionQueue;
json[r'video-conversion-queue'] = this.videoConversionQueue;
json[r'object-tagging-queue'] = this.objectTaggingQueue;
json[r'clip-encoding-queue'] = this.clipEncodingQueue;
json[r'storage-template-migration-queue'] = this.storageTemplateMigrationQueue;
json[r'background-task-queue'] = this.backgroundTaskQueue;
json[r'search-queue'] = this.searchQueue;
json[r'recognize-faces-queue'] = this.recognizeFacesQueue;
json[r'sidecar-queue'] = this.sidecarQueue;
json[r'thumbnailGeneration'] = this.thumbnailGeneration;
json[r'metadataExtraction'] = this.metadataExtraction;
json[r'videoConversion'] = this.videoConversion;
json[r'objectTagging'] = this.objectTagging;
json[r'clipEncoding'] = this.clipEncoding;
json[r'storageTemplateMigration'] = this.storageTemplateMigration;
json[r'backgroundTask'] = this.backgroundTask;
json[r'search'] = this.search;
json[r'recognizeFaces'] = this.recognizeFaces;
json[r'sidecar'] = this.sidecar;
return json;
}
@ -109,16 +109,16 @@ class AllJobStatusResponseDto {
}());
return AllJobStatusResponseDto(
thumbnailGenerationQueue: JobStatusDto.fromJson(json[r'thumbnail-generation-queue'])!,
metadataExtractionQueue: JobStatusDto.fromJson(json[r'metadata-extraction-queue'])!,
videoConversionQueue: JobStatusDto.fromJson(json[r'video-conversion-queue'])!,
objectTaggingQueue: JobStatusDto.fromJson(json[r'object-tagging-queue'])!,
clipEncodingQueue: JobStatusDto.fromJson(json[r'clip-encoding-queue'])!,
storageTemplateMigrationQueue: JobStatusDto.fromJson(json[r'storage-template-migration-queue'])!,
backgroundTaskQueue: JobStatusDto.fromJson(json[r'background-task-queue'])!,
searchQueue: JobStatusDto.fromJson(json[r'search-queue'])!,
recognizeFacesQueue: JobStatusDto.fromJson(json[r'recognize-faces-queue'])!,
sidecarQueue: JobStatusDto.fromJson(json[r'sidecar-queue'])!,
thumbnailGeneration: JobStatusDto.fromJson(json[r'thumbnailGeneration'])!,
metadataExtraction: JobStatusDto.fromJson(json[r'metadataExtraction'])!,
videoConversion: JobStatusDto.fromJson(json[r'videoConversion'])!,
objectTagging: JobStatusDto.fromJson(json[r'objectTagging'])!,
clipEncoding: JobStatusDto.fromJson(json[r'clipEncoding'])!,
storageTemplateMigration: JobStatusDto.fromJson(json[r'storageTemplateMigration'])!,
backgroundTask: JobStatusDto.fromJson(json[r'backgroundTask'])!,
search: JobStatusDto.fromJson(json[r'search'])!,
recognizeFaces: JobStatusDto.fromJson(json[r'recognizeFaces'])!,
sidecar: JobStatusDto.fromJson(json[r'sidecar'])!,
);
}
return null;
@ -166,16 +166,16 @@ class AllJobStatusResponseDto {
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'thumbnail-generation-queue',
'metadata-extraction-queue',
'video-conversion-queue',
'object-tagging-queue',
'clip-encoding-queue',
'storage-template-migration-queue',
'background-task-queue',
'search-queue',
'recognize-faces-queue',
'sidecar-queue',
'thumbnailGeneration',
'metadataExtraction',
'videoConversion',
'objectTagging',
'clipEncoding',
'storageTemplateMigration',
'backgroundTask',
'search',
'recognizeFaces',
'sidecar',
};
}

60
mobile/openapi/lib/model/job_name.dart generated
View file

@ -23,29 +23,29 @@ class JobName {
String toJson() => value;
static const thumbnailGenerationQueue = JobName._(r'thumbnail-generation-queue');
static const metadataExtractionQueue = JobName._(r'metadata-extraction-queue');
static const videoConversionQueue = JobName._(r'video-conversion-queue');
static const objectTaggingQueue = JobName._(r'object-tagging-queue');
static const recognizeFacesQueue = JobName._(r'recognize-faces-queue');
static const clipEncodingQueue = JobName._(r'clip-encoding-queue');
static const backgroundTaskQueue = JobName._(r'background-task-queue');
static const storageTemplateMigrationQueue = JobName._(r'storage-template-migration-queue');
static const searchQueue = JobName._(r'search-queue');
static const sidecarQueue = JobName._(r'sidecar-queue');
static const thumbnailGeneration = JobName._(r'thumbnailGeneration');
static const metadataExtraction = JobName._(r'metadataExtraction');
static const videoConversion = JobName._(r'videoConversion');
static const objectTagging = JobName._(r'objectTagging');
static const recognizeFaces = JobName._(r'recognizeFaces');
static const clipEncoding = JobName._(r'clipEncoding');
static const backgroundTask = JobName._(r'backgroundTask');
static const storageTemplateMigration = JobName._(r'storageTemplateMigration');
static const search = JobName._(r'search');
static const sidecar = JobName._(r'sidecar');
/// List of all possible values in this [enum][JobName].
static const values = <JobName>[
thumbnailGenerationQueue,
metadataExtractionQueue,
videoConversionQueue,
objectTaggingQueue,
recognizeFacesQueue,
clipEncodingQueue,
backgroundTaskQueue,
storageTemplateMigrationQueue,
searchQueue,
sidecarQueue,
thumbnailGeneration,
metadataExtraction,
videoConversion,
objectTagging,
recognizeFaces,
clipEncoding,
backgroundTask,
storageTemplateMigration,
search,
sidecar,
];
static JobName? fromJson(dynamic value) => JobNameTypeTransformer().decode(value);
@ -84,16 +84,16 @@ class JobNameTypeTransformer {
JobName? decode(dynamic data, {bool allowNull = true}) {
if (data != null) {
switch (data) {
case r'thumbnail-generation-queue': return JobName.thumbnailGenerationQueue;
case r'metadata-extraction-queue': return JobName.metadataExtractionQueue;
case r'video-conversion-queue': return JobName.videoConversionQueue;
case r'object-tagging-queue': return JobName.objectTaggingQueue;
case r'recognize-faces-queue': return JobName.recognizeFacesQueue;
case r'clip-encoding-queue': return JobName.clipEncodingQueue;
case r'background-task-queue': return JobName.backgroundTaskQueue;
case r'storage-template-migration-queue': return JobName.storageTemplateMigrationQueue;
case r'search-queue': return JobName.searchQueue;
case r'sidecar-queue': return JobName.sidecarQueue;
case r'thumbnailGeneration': return JobName.thumbnailGeneration;
case r'metadataExtraction': return JobName.metadataExtraction;
case r'videoConversion': return JobName.videoConversion;
case r'objectTagging': return JobName.objectTagging;
case r'recognizeFaces': return JobName.recognizeFaces;
case r'clipEncoding': return JobName.clipEncoding;
case r'backgroundTask': return JobName.backgroundTask;
case r'storageTemplateMigration': return JobName.storageTemplateMigration;
case r'search': return JobName.search;
case r'sidecar': return JobName.sidecar;
default:
if (!allowNull) {
throw ArgumentError('Unknown enum value to decode: $data');

109
mobile/openapi/lib/model/job_settings_dto.dart generated Normal file
View file

@ -0,0 +1,109 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class JobSettingsDto {
/// Returns a new [JobSettingsDto] instance.
JobSettingsDto({
required this.concurrency,
});
int concurrency;
@override
bool operator ==(Object other) => identical(this, other) || other is JobSettingsDto &&
other.concurrency == concurrency;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(concurrency.hashCode);
@override
String toString() => 'JobSettingsDto[concurrency=$concurrency]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'concurrency'] = this.concurrency;
return json;
}
/// Returns a new [JobSettingsDto] instance and imports its values from
/// [value] if it's a [Map], null otherwise.
// ignore: prefer_constructors_over_static_methods
static JobSettingsDto? fromJson(dynamic value) {
if (value is Map) {
final json = value.cast<String, dynamic>();
// Ensure that the map contains the required keys.
// Note 1: the values aren't checked for validity beyond being non-null.
// Note 2: this code is stripped in release mode!
assert(() {
requiredKeys.forEach((key) {
assert(json.containsKey(key), 'Required key "JobSettingsDto[$key]" is missing from JSON.');
assert(json[key] != null, 'Required key "JobSettingsDto[$key]" has a null value in JSON.');
});
return true;
}());
return JobSettingsDto(
concurrency: mapValueOfType<int>(json, r'concurrency')!,
);
}
return null;
}
static List<JobSettingsDto> listFromJson(dynamic json, {bool growable = false,}) {
final result = <JobSettingsDto>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = JobSettingsDto.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
static Map<String, JobSettingsDto> mapFromJson(dynamic json) {
final map = <String, JobSettingsDto>{};
if (json is Map && json.isNotEmpty) {
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
for (final entry in json.entries) {
final value = JobSettingsDto.fromJson(entry.value);
if (value != null) {
map[entry.key] = value;
}
}
}
return map;
}
// maps a json object with a list of JobSettingsDto-objects as value to a dart map
static Map<String, List<JobSettingsDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
final map = <String, List<JobSettingsDto>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = JobSettingsDto.listFromJson(entry.value, growable: growable,);
}
}
return map;
}
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'concurrency',
};
}

14
mobile/openapi/lib/model/system_config_dto.dart generated
View file

@ -17,6 +17,7 @@ class SystemConfigDto {
required this.oauth,
required this.passwordLogin,
required this.storageTemplate,
required this.job,
});
SystemConfigFFmpegDto ffmpeg;
@ -27,12 +28,15 @@ class SystemConfigDto {
SystemConfigStorageTemplateDto storageTemplate;
SystemConfigJobDto job;
@override
bool operator ==(Object other) => identical(this, other) || other is SystemConfigDto &&
other.ffmpeg == ffmpeg &&
other.oauth == oauth &&
other.passwordLogin == passwordLogin &&
other.storageTemplate == storageTemplate;
other.storageTemplate == storageTemplate &&
other.job == job;
@override
int get hashCode =>
@ -40,10 +44,11 @@ class SystemConfigDto {
(ffmpeg.hashCode) +
(oauth.hashCode) +
(passwordLogin.hashCode) +
(storageTemplate.hashCode);
(storageTemplate.hashCode) +
(job.hashCode);
@override
String toString() => 'SystemConfigDto[ffmpeg=$ffmpeg, oauth=$oauth, passwordLogin=$passwordLogin, storageTemplate=$storageTemplate]';
String toString() => 'SystemConfigDto[ffmpeg=$ffmpeg, oauth=$oauth, passwordLogin=$passwordLogin, storageTemplate=$storageTemplate, job=$job]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
@ -51,6 +56,7 @@ class SystemConfigDto {
json[r'oauth'] = this.oauth;
json[r'passwordLogin'] = this.passwordLogin;
json[r'storageTemplate'] = this.storageTemplate;
json[r'job'] = this.job;
return json;
}
@ -77,6 +83,7 @@ class SystemConfigDto {
oauth: SystemConfigOAuthDto.fromJson(json[r'oauth'])!,
passwordLogin: SystemConfigPasswordLoginDto.fromJson(json[r'passwordLogin'])!,
storageTemplate: SystemConfigStorageTemplateDto.fromJson(json[r'storageTemplate'])!,
job: SystemConfigJobDto.fromJson(json[r'job'])!,
);
}
return null;
@ -128,6 +135,7 @@ class SystemConfigDto {
'oauth',
'passwordLogin',
'storageTemplate',
'job',
};
}

181
mobile/openapi/lib/model/system_config_job_dto.dart generated Normal file
View file

@ -0,0 +1,181 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class SystemConfigJobDto {
/// Returns a new [SystemConfigJobDto] instance.
SystemConfigJobDto({
required this.thumbnailGeneration,
required this.metadataExtraction,
required this.videoConversion,
required this.objectTagging,
required this.clipEncoding,
required this.storageTemplateMigration,
required this.backgroundTask,
required this.search,
required this.recognizeFaces,
required this.sidecar,
});
JobSettingsDto thumbnailGeneration;
JobSettingsDto metadataExtraction;
JobSettingsDto videoConversion;
JobSettingsDto objectTagging;
JobSettingsDto clipEncoding;
JobSettingsDto storageTemplateMigration;
JobSettingsDto backgroundTask;
JobSettingsDto search;
JobSettingsDto recognizeFaces;
JobSettingsDto sidecar;
@override
bool operator ==(Object other) => identical(this, other) || other is SystemConfigJobDto &&
other.thumbnailGeneration == thumbnailGeneration &&
other.metadataExtraction == metadataExtraction &&
other.videoConversion == videoConversion &&
other.objectTagging == objectTagging &&
other.clipEncoding == clipEncoding &&
other.storageTemplateMigration == storageTemplateMigration &&
other.backgroundTask == backgroundTask &&
other.search == search &&
other.recognizeFaces == recognizeFaces &&
other.sidecar == sidecar;
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(thumbnailGeneration.hashCode) +
(metadataExtraction.hashCode) +
(videoConversion.hashCode) +
(objectTagging.hashCode) +
(clipEncoding.hashCode) +
(storageTemplateMigration.hashCode) +
(backgroundTask.hashCode) +
(search.hashCode) +
(recognizeFaces.hashCode) +
(sidecar.hashCode);
@override
String toString() => 'SystemConfigJobDto[thumbnailGeneration=$thumbnailGeneration, metadataExtraction=$metadataExtraction, videoConversion=$videoConversion, objectTagging=$objectTagging, clipEncoding=$clipEncoding, storageTemplateMigration=$storageTemplateMigration, backgroundTask=$backgroundTask, search=$search, recognizeFaces=$recognizeFaces, sidecar=$sidecar]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'thumbnailGeneration'] = this.thumbnailGeneration;
json[r'metadataExtraction'] = this.metadataExtraction;
json[r'videoConversion'] = this.videoConversion;
json[r'objectTagging'] = this.objectTagging;
json[r'clipEncoding'] = this.clipEncoding;
json[r'storageTemplateMigration'] = this.storageTemplateMigration;
json[r'backgroundTask'] = this.backgroundTask;
json[r'search'] = this.search;
json[r'recognizeFaces'] = this.recognizeFaces;
json[r'sidecar'] = this.sidecar;
return json;
}
/// Returns a new [SystemConfigJobDto] instance and imports its values from
/// [value] if it's a [Map], null otherwise.
// ignore: prefer_constructors_over_static_methods
static SystemConfigJobDto? fromJson(dynamic value) {
if (value is Map) {
final json = value.cast<String, dynamic>();
// Ensure that the map contains the required keys.
// Note 1: the values aren't checked for validity beyond being non-null.
// Note 2: this code is stripped in release mode!
assert(() {
requiredKeys.forEach((key) {
assert(json.containsKey(key), 'Required key "SystemConfigJobDto[$key]" is missing from JSON.');
assert(json[key] != null, 'Required key "SystemConfigJobDto[$key]" has a null value in JSON.');
});
return true;
}());
return SystemConfigJobDto(
thumbnailGeneration: JobSettingsDto.fromJson(json[r'thumbnailGeneration'])!,
metadataExtraction: JobSettingsDto.fromJson(json[r'metadataExtraction'])!,
videoConversion: JobSettingsDto.fromJson(json[r'videoConversion'])!,
objectTagging: JobSettingsDto.fromJson(json[r'objectTagging'])!,
clipEncoding: JobSettingsDto.fromJson(json[r'clipEncoding'])!,
storageTemplateMigration: JobSettingsDto.fromJson(json[r'storageTemplateMigration'])!,
backgroundTask: JobSettingsDto.fromJson(json[r'backgroundTask'])!,
search: JobSettingsDto.fromJson(json[r'search'])!,
recognizeFaces: JobSettingsDto.fromJson(json[r'recognizeFaces'])!,
sidecar: JobSettingsDto.fromJson(json[r'sidecar'])!,
);
}
return null;
}
static List<SystemConfigJobDto> listFromJson(dynamic json, {bool growable = false,}) {
final result = <SystemConfigJobDto>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = SystemConfigJobDto.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
static Map<String, SystemConfigJobDto> mapFromJson(dynamic json) {
final map = <String, SystemConfigJobDto>{};
if (json is Map && json.isNotEmpty) {
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
for (final entry in json.entries) {
final value = SystemConfigJobDto.fromJson(entry.value);
if (value != null) {
map[entry.key] = value;
}
}
}
return map;
}
// maps a json object with a list of SystemConfigJobDto-objects as value to a dart map
static Map<String, List<SystemConfigJobDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
final map = <String, List<SystemConfigJobDto>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = SystemConfigJobDto.listFromJson(entry.value, growable: growable,);
}
}
return map;
}
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'thumbnailGeneration',
'metadataExtraction',
'videoConversion',
'objectTagging',
'clipEncoding',
'storageTemplateMigration',
'backgroundTask',
'search',
'recognizeFaces',
'sidecar',
};
}

40
mobile/openapi/test/all_job_status_response_dto_test.dart generated
View file

@ -16,53 +16,53 @@ void main() {
// final instance = AllJobStatusResponseDto();
group('test AllJobStatusResponseDto', () {
// JobStatusDto thumbnailGenerationQueue
test('to test the property `thumbnailGenerationQueue`', () async {
// JobStatusDto thumbnailGeneration
test('to test the property `thumbnailGeneration`', () async {
// TODO
});
// JobStatusDto metadataExtractionQueue
test('to test the property `metadataExtractionQueue`', () async {
// JobStatusDto metadataExtraction
test('to test the property `metadataExtraction`', () async {
// TODO
});
// JobStatusDto videoConversionQueue
test('to test the property `videoConversionQueue`', () async {
// JobStatusDto videoConversion
test('to test the property `videoConversion`', () async {
// TODO
});
// JobStatusDto objectTaggingQueue
test('to test the property `objectTaggingQueue`', () async {
// JobStatusDto objectTagging
test('to test the property `objectTagging`', () async {
// TODO
});
// JobStatusDto clipEncodingQueue
test('to test the property `clipEncodingQueue`', () async {
// JobStatusDto clipEncoding
test('to test the property `clipEncoding`', () async {
// TODO
});
// JobStatusDto storageTemplateMigrationQueue
test('to test the property `storageTemplateMigrationQueue`', () async {
// JobStatusDto storageTemplateMigration
test('to test the property `storageTemplateMigration`', () async {
// TODO
});
// JobStatusDto backgroundTaskQueue
test('to test the property `backgroundTaskQueue`', () async {
// JobStatusDto backgroundTask
test('to test the property `backgroundTask`', () async {
// TODO
});
// JobStatusDto searchQueue
test('to test the property `searchQueue`', () async {
// JobStatusDto search
test('to test the property `search`', () async {
// TODO
});
// JobStatusDto recognizeFacesQueue
test('to test the property `recognizeFacesQueue`', () async {
// JobStatusDto recognizeFaces
test('to test the property `recognizeFaces`', () async {
// TODO
});
// JobStatusDto sidecarQueue
test('to test the property `sidecarQueue`', () async {
// JobStatusDto sidecar
test('to test the property `sidecar`', () async {
// TODO
});

27
mobile/openapi/test/job_settings_dto_test.dart generated Normal file
View file

@ -0,0 +1,27 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
import 'package:openapi/api.dart';
import 'package:test/test.dart';
// tests for JobSettingsDto
void main() {
// final instance = JobSettingsDto();
group('test JobSettingsDto', () {
// int concurrency
test('to test the property `concurrency`', () async {
// TODO
});
});
}

5
mobile/openapi/test/system_config_dto_test.dart generated
View file

@ -36,6 +36,11 @@ void main() {
// TODO
});
// SystemConfigJobDto job
test('to test the property `job`', () async {
// TODO
});
});

72
mobile/openapi/test/system_config_job_dto_test.dart generated Normal file
View file

@ -0,0 +1,72 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
import 'package:openapi/api.dart';
import 'package:test/test.dart';
// tests for SystemConfigJobDto
void main() {
// final instance = SystemConfigJobDto();
group('test SystemConfigJobDto', () {
// JobSettingsDto thumbnailGeneration
test('to test the property `thumbnailGeneration`', () async {
// TODO
});
// JobSettingsDto metadataExtraction
test('to test the property `metadataExtraction`', () async {
// TODO
});
// JobSettingsDto videoConversion
test('to test the property `videoConversion`', () async {
// TODO
});
// JobSettingsDto objectTagging
test('to test the property `objectTagging`', () async {
// TODO
});
// JobSettingsDto clipEncoding
test('to test the property `clipEncoding`', () async {
// TODO
});
// JobSettingsDto storageTemplateMigration
test('to test the property `storageTemplateMigration`', () async {
// TODO
});
// JobSettingsDto backgroundTask
test('to test the property `backgroundTask`', () async {
// TODO
});
// JobSettingsDto search
test('to test the property `search`', () async {
// TODO
});
// JobSettingsDto recognizeFaces
test('to test the property `recognizeFaces`', () async {
// TODO
});
// JobSettingsDto sidecar
test('to test the property `sidecar`', () async {
// TODO
});
});
}

2
server/apps/immich/src/controllers/job.controller.ts
View file

@ -19,6 +19,6 @@ export class JobController {
@Put('/:jobId')
async sendJobCommand(@Param() { jobId }: JobIdDto, @Body() dto: JobCommandDto): Promise<JobStatusDto> {
await this.service.handleCommand(jobId, dto);
return await this.service.getJobStatus(jobId);
return this.service.getJobStatus(jobId);
}
}

75
server/apps/microservices/src/app.service.ts Normal file
View file

@ -0,0 +1,75 @@
import {
FacialRecognitionService,
IDeleteFilesJob,
JobName,
JobService,
MediaService,
MetadataService,
PersonService,
SearchService,
SmartInfoService,
StorageService,
StorageTemplateService,
SystemConfigService,
UserService,
} from '@app/domain';
import { Injectable } from '@nestjs/common';
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
@Injectable()
export class AppService {
constructor(
// TODO refactor to domain
private metadataProcessor: MetadataExtractionProcessor,
private facialRecognitionService: FacialRecognitionService,
private jobService: JobService,
private mediaService: MediaService,
private metadataService: MetadataService,
private personService: PersonService,
private searchService: SearchService,
private smartInfoService: SmartInfoService,
private storageTemplateService: StorageTemplateService,
private storageService: StorageService,
private systemConfigService: SystemConfigService,
private userService: UserService,
) {}
async init() {
await this.jobService.registerHandlers({
[JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data),
[JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(),
[JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data),
[JobName.QUEUE_OBJECT_TAGGING]: (data) => this.smartInfoService.handleQueueObjectTagging(data),
[JobName.CLASSIFY_IMAGE]: (data) => this.smartInfoService.handleClassifyImage(data),
[JobName.QUEUE_ENCODE_CLIP]: (data) => this.smartInfoService.handleQueueEncodeClip(data),
[JobName.ENCODE_CLIP]: (data) => this.smartInfoService.handleEncodeClip(data),
[JobName.SEARCH_INDEX_ALBUMS]: () => this.searchService.handleIndexAlbums(),
[JobName.SEARCH_INDEX_ASSETS]: () => this.searchService.handleIndexAssets(),
[JobName.SEARCH_INDEX_FACES]: () => this.searchService.handleIndexFaces(),
[JobName.SEARCH_INDEX_ALBUM]: (data) => this.searchService.handleIndexAlbum(data),
[JobName.SEARCH_INDEX_ASSET]: (data) => this.searchService.handleIndexAsset(data),
[JobName.SEARCH_INDEX_FACE]: (data) => this.searchService.handleIndexFace(data),
[JobName.SEARCH_REMOVE_ALBUM]: (data) => this.searchService.handleRemoveAlbum(data),
[JobName.SEARCH_REMOVE_ASSET]: (data) => this.searchService.handleRemoveAsset(data),
[JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data),
[JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(),
[JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data),
[JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(),
[JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data),
[JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data),
[JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data),
[JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data),
[JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data),
[JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data),
[JobName.METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleMetadataExtraction(data),
[JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleQueueRecognizeFaces(data),
[JobName.RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleRecognizeFaces(data),
[JobName.GENERATE_FACE_THUMBNAIL]: (data) => this.facialRecognitionService.handleGenerateFaceThumbnail(data),
[JobName.PERSON_CLEANUP]: () => this.personService.handlePersonCleanup(),
[JobName.QUEUE_SIDECAR]: (data) => this.metadataService.handleQueueSidecar(data),
[JobName.SIDECAR_DISCOVERY]: (data) => this.metadataService.handleSidecarDiscovery(data),
[JobName.SIDECAR_SYNC]: () => this.metadataService.handleSidecarSync(),
});
}
}

4
server/apps/microservices/src/main.ts
View file

@ -2,8 +2,8 @@ import { getLogLevels, SERVER_VERSION } from '@app/domain';
import { RedisIoAdapter } from '@app/infra';
import { Logger } from '@nestjs/common';
import { NestFactory } from '@nestjs/core';
import { AppService } from './app.service';
import { MicroservicesModule } from './microservices.module';
import { ProcessorService } from './processor.service';
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
const logger = new Logger('ImmichMicroservice');
@ -15,7 +15,7 @@ async function bootstrap() {
const listeningPort = Number(process.env.MICROSERVICES_PORT) || 3002;
await app.get(ProcessorService).init();
await app.get(AppService).init();
app.useWebSocketAdapter(new RedisIoAdapter(app));

4
server/apps/microservices/src/microservices.module.ts
View file

@ -3,7 +3,7 @@ import { InfraModule } from '@app/infra';
import { ExifEntity } from '@app/infra/entities';
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { ProcessorService } from './processor.service';
import { AppService } from './app.service';
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
@Module({
@ -12,6 +12,6 @@ import { MetadataExtractionProcessor } from './processors/metadata-extraction.pr
DomainModule.register({ imports: [InfraModule] }),
TypeOrmModule.forFeature([ExifEntity]),
],
providers: [MetadataExtractionProcessor, ProcessorService],
providers: [MetadataExtractionProcessor, AppService],
})
export class MicroservicesModule {}

113
server/apps/microservices/src/processor.service.ts
View file

@ -1,113 +0,0 @@
import {
FacialRecognitionService,
IDeleteFilesJob,
JobItem,
JobName,
JobService,
JOBS_TO_QUEUE,
MediaService,
MetadataService,
PersonService,
QueueName,
QUEUE_TO_CONCURRENCY,
SearchService,
SmartInfoService,
StorageService,
StorageTemplateService,
SystemConfigService,
UserService,
} from '@app/domain';
import { getQueueToken } from '@nestjs/bull';
import { Injectable, Logger } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { Queue } from 'bull';
import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
type JobHandler<T = any> = (data: T) => boolean | Promise<boolean>;
@Injectable()
export class ProcessorService {
constructor(
private moduleRef: ModuleRef,
// TODO refactor to domain
private metadataProcessor: MetadataExtractionProcessor,
private facialRecognitionService: FacialRecognitionService,
private jobService: JobService,
private mediaService: MediaService,
private metadataService: MetadataService,
private personService: PersonService,
private searchService: SearchService,
private smartInfoService: SmartInfoService,
private storageTemplateService: StorageTemplateService,
private storageService: StorageService,
private systemConfigService: SystemConfigService,
private userService: UserService,
) {}
private logger = new Logger(ProcessorService.name);
private handlers: Record<JobName, JobHandler> = {
[JobName.DELETE_FILES]: (data: IDeleteFilesJob) => this.storageService.handleDeleteFiles(data),
[JobName.USER_DELETE_CHECK]: () => this.userService.handleUserDeleteCheck(),
[JobName.USER_DELETION]: (data) => this.userService.handleUserDelete(data),
[JobName.QUEUE_OBJECT_TAGGING]: (data) => this.smartInfoService.handleQueueObjectTagging(data),
[JobName.CLASSIFY_IMAGE]: (data) => this.smartInfoService.handleClassifyImage(data),
[JobName.QUEUE_ENCODE_CLIP]: (data) => this.smartInfoService.handleQueueEncodeClip(data),
[JobName.ENCODE_CLIP]: (data) => this.smartInfoService.handleEncodeClip(data),
[JobName.SEARCH_INDEX_ALBUMS]: () => this.searchService.handleIndexAlbums(),
[JobName.SEARCH_INDEX_ASSETS]: () => this.searchService.handleIndexAssets(),
[JobName.SEARCH_INDEX_FACES]: () => this.searchService.handleIndexFaces(),
[JobName.SEARCH_INDEX_ALBUM]: (data) => this.searchService.handleIndexAlbum(data),
[JobName.SEARCH_INDEX_ASSET]: (data) => this.searchService.handleIndexAsset(data),
[JobName.SEARCH_INDEX_FACE]: (data) => this.searchService.handleIndexFace(data),
[JobName.SEARCH_REMOVE_ALBUM]: (data) => this.searchService.handleRemoveAlbum(data),
[JobName.SEARCH_REMOVE_ASSET]: (data) => this.searchService.handleRemoveAsset(data),
[JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data),
[JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(),
[JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data),
[JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(),
[JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data),
[JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data),
[JobName.GENERATE_WEBP_THUMBNAIL]: (data) => this.mediaService.handleGenerateWepbThumbnail(data),
[JobName.QUEUE_VIDEO_CONVERSION]: (data) => this.mediaService.handleQueueVideoConversion(data),
[JobName.VIDEO_CONVERSION]: (data) => this.mediaService.handleVideoConversion(data),
[JobName.QUEUE_METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleQueueMetadataExtraction(data),
[JobName.METADATA_EXTRACTION]: (data) => this.metadataProcessor.handleMetadataExtraction(data),
[JobName.QUEUE_RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleQueueRecognizeFaces(data),
[JobName.RECOGNIZE_FACES]: (data) => this.facialRecognitionService.handleRecognizeFaces(data),
[JobName.GENERATE_FACE_THUMBNAIL]: (data) => this.facialRecognitionService.handleGenerateFaceThumbnail(data),
[JobName.PERSON_CLEANUP]: () => this.personService.handlePersonCleanup(),
[JobName.QUEUE_SIDECAR]: (data) => this.metadataService.handleQueueSidecar(data),
[JobName.SIDECAR_DISCOVERY]: (data) => this.metadataService.handleSidecarDiscovery(data),
[JobName.SIDECAR_SYNC]: () => this.metadataService.handleSidecarSync(),
};
async init() {
const queueSeen: Partial<Record<QueueName, boolean>> = {};
for (const jobName of Object.values(JobName)) {
const handler = this.handlers[jobName];
const queueName = JOBS_TO_QUEUE[jobName];
const queue = this.moduleRef.get<Queue>(getQueueToken(queueName), { strict: false });
// only set concurrency on the first job for a queue, since concurrency stacks
const seen = queueSeen[queueName];
const concurrency = seen ? 0 : QUEUE_TO_CONCURRENCY[queueName];
queueSeen[queueName] = true;
await queue.isReady();
queue.process(jobName, concurrency, async (job): Promise<void> => {
try {
const success = await handler(job.data);
if (success) {
await this.jobService.onDone({ name: jobName, data: job.data } as JobItem);
}
} catch (error: Error | any) {
this.logger.error(`Unable to run job handler: ${error}`, error?.stack, job.data);
}
});
}
}
}

124
server/immich-openapi-specs.json
View file

@ -5106,63 +5106,63 @@
"AllJobStatusResponseDto": {
"type": "object",
"properties": {
"thumbnail-generation-queue": {
"thumbnailGeneration": {
"$ref": "#/components/schemas/JobStatusDto"
},
"metadata-extraction-queue": {
"metadataExtraction": {
"$ref": "#/components/schemas/JobStatusDto"
},
"video-conversion-queue": {
"videoConversion": {
"$ref": "#/components/schemas/JobStatusDto"
},
"object-tagging-queue": {
"objectTagging": {
"$ref": "#/components/schemas/JobStatusDto"
},
"clip-encoding-queue": {
"clipEncoding": {
"$ref": "#/components/schemas/JobStatusDto"
},
"storage-template-migration-queue": {
"storageTemplateMigration": {
"$ref": "#/components/schemas/JobStatusDto"
},
"background-task-queue": {
"backgroundTask": {
"$ref": "#/components/schemas/JobStatusDto"
},
"search-queue": {
"search": {
"$ref": "#/components/schemas/JobStatusDto"
},
"recognize-faces-queue": {
"recognizeFaces": {
"$ref": "#/components/schemas/JobStatusDto"
},
"sidecar-queue": {
"sidecar": {
"$ref": "#/components/schemas/JobStatusDto"
}
},
"required": [
"thumbnail-generation-queue",
"metadata-extraction-queue",
"video-conversion-queue",
"object-tagging-queue",
"clip-encoding-queue",
"storage-template-migration-queue",
"background-task-queue",
"search-queue",
"recognize-faces-queue",
"sidecar-queue"
"thumbnailGeneration",
"metadataExtraction",
"videoConversion",
"objectTagging",
"clipEncoding",
"storageTemplateMigration",
"backgroundTask",
"search",
"recognizeFaces",
"sidecar"
]
},
"JobName": {
"type": "string",
"enum": [
"thumbnail-generation-queue",
"metadata-extraction-queue",
"video-conversion-queue",
"object-tagging-queue",
"recognize-faces-queue",
"clip-encoding-queue",
"background-task-queue",
"storage-template-migration-queue",
"search-queue",
"sidecar-queue"
"thumbnailGeneration",
"metadataExtraction",
"videoConversion",
"objectTagging",
"recognizeFaces",
"clipEncoding",
"backgroundTask",
"storageTemplateMigration",
"search",
"sidecar"
]
},
"JobCommand": {
@ -5733,6 +5733,64 @@
"template"
]
},
"JobSettingsDto": {
"type": "object",
"properties": {
"concurrency": {
"type": "integer"
}
},
"required": [
"concurrency"
]
},
"SystemConfigJobDto": {
"type": "object",
"properties": {
"thumbnailGeneration": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"metadataExtraction": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"videoConversion": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"objectTagging": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"clipEncoding": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"storageTemplateMigration": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"backgroundTask": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"search": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"recognizeFaces": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"sidecar": {
"$ref": "#/components/schemas/JobSettingsDto"
}
},
"required": [
"thumbnailGeneration",
"metadataExtraction",
"videoConversion",
"objectTagging",
"clipEncoding",
"storageTemplateMigration",
"backgroundTask",
"search",
"recognizeFaces",
"sidecar"
]
},
"SystemConfigDto": {
"type": "object",
"properties": {
@ -5747,13 +5805,17 @@
},
"storageTemplate": {
"$ref": "#/components/schemas/SystemConfigStorageTemplateDto"
},
"job": {
"$ref": "#/components/schemas/SystemConfigJobDto"
}
},
"required": [
"ffmpeg",
"oauth",
"passwordLogin",
"storageTemplate"
"storageTemplate",
"job"
]
},
"SystemConfigTemplateStorageOptionDto": {

34
server/libs/domain/src/job/job.constants.ts
View file

@ -1,14 +1,14 @@
export enum QueueName {
THUMBNAIL_GENERATION = 'thumbnail-generation-queue',
METADATA_EXTRACTION = 'metadata-extraction-queue',
VIDEO_CONVERSION = 'video-conversion-queue',
OBJECT_TAGGING = 'object-tagging-queue',
RECOGNIZE_FACES = 'recognize-faces-queue',
CLIP_ENCODING = 'clip-encoding-queue',
BACKGROUND_TASK = 'background-task-queue',
STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration-queue',
SEARCH = 'search-queue',
SIDECAR = 'sidecar-queue',
THUMBNAIL_GENERATION = 'thumbnailGeneration',
METADATA_EXTRACTION = 'metadataExtraction',
VIDEO_CONVERSION = 'videoConversion',
OBJECT_TAGGING = 'objectTagging',
RECOGNIZE_FACES = 'recognizeFaces',
CLIP_ENCODING = 'clipEncoding',
BACKGROUND_TASK = 'backgroundTask',
STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration',
SEARCH = 'search',
SIDECAR = 'sidecar',
}
export enum JobCommand {
@ -135,17 +135,3 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.SIDECAR_DISCOVERY]: QueueName.SIDECAR,
[JobName.SIDECAR_SYNC]: QueueName.SIDECAR,
};
// max concurrency for each queue (total concurrency across all jobs)
export const QUEUE_TO_CONCURRENCY: Record<QueueName, number> = {
[QueueName.BACKGROUND_TASK]: 5,
[QueueName.CLIP_ENCODING]: 2,
[QueueName.METADATA_EXTRACTION]: 5,
[QueueName.OBJECT_TAGGING]: 2,
[QueueName.RECOGNIZE_FACES]: 2,
[QueueName.SEARCH]: 5,
[QueueName.SIDECAR]: 5,
[QueueName.STORAGE_TEMPLATE_MIGRATION]: 5,
[QueueName.THUMBNAIL_GENERATION]: 5,
[QueueName.VIDEO_CONVERSION]: 1,
};

18
server/libs/domain/src/job/job.repository.ts
View file

@ -33,13 +33,13 @@ export type JobItem =
| { name: JobName.GENERATE_WEBP_THUMBNAIL; data: IEntityJob }
// User Deletion
| { name: JobName.USER_DELETE_CHECK }
| { name: JobName.USER_DELETE_CHECK; data?: IBaseJob }
| { name: JobName.USER_DELETION; data: IEntityJob }
// Storage Template
| { name: JobName.STORAGE_TEMPLATE_MIGRATION }
| { name: JobName.STORAGE_TEMPLATE_MIGRATION; data?: IBaseJob }
| { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE; data: IEntityJob }
| { name: JobName.SYSTEM_CONFIG_CHANGE }
| { name: JobName.SYSTEM_CONFIG_CHANGE; data?: IBaseJob }
// Metadata Extraction
| { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob }
@ -67,22 +67,26 @@ export type JobItem =
| { name: JobName.DELETE_FILES; data: IDeleteFilesJob }
// Asset Deletion
| { name: JobName.PERSON_CLEANUP }
| { name: JobName.PERSON_CLEANUP; data?: IBaseJob }
// Search
| { name: JobName.SEARCH_INDEX_ASSETS }
| { name: JobName.SEARCH_INDEX_ASSETS; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_ASSET; data: IBulkEntityJob }
| { name: JobName.SEARCH_INDEX_FACES }
| { name: JobName.SEARCH_INDEX_FACES; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_FACE; data: IAssetFaceJob }
| { name: JobName.SEARCH_INDEX_ALBUMS }
| { name: JobName.SEARCH_INDEX_ALBUMS; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_ALBUM; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_ASSET; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_ALBUM; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_FACE; data: IAssetFaceJob };
export type JobHandler<T = any> = (data: T) => boolean | Promise<boolean>;
export const IJobRepository = 'IJobRepository';
export interface IJobRepository {
addHandler(queueName: QueueName, concurrency: number, handler: (job: JobItem) => Promise<void>): void;
setConcurrency(queueName: QueueName, concurrency: number): void;
queue(item: JobItem): Promise<void>;
pause(name: QueueName): Promise<void>;
resume(name: QueueName): Promise<void>;

65
server/libs/domain/src/job/job.service.spec.ts
View file

@ -1,20 +1,28 @@
import { BadRequestException } from '@nestjs/common';
import { newAssetRepositoryMock, newCommunicationRepositoryMock, newJobRepositoryMock } from '../../test';
import {
newAssetRepositoryMock,
newCommunicationRepositoryMock,
newJobRepositoryMock,
newSystemConfigRepositoryMock,
} from '../../test';
import { IAssetRepository } from '../asset';
import { ICommunicationRepository } from '../communication';
import { IJobRepository, JobCommand, JobName, JobService, QueueName } from '../job';
import { IJobRepository, JobCommand, JobHandler, JobName, JobService, QueueName } from '../job';
import { ISystemConfigRepository } from '../system-config';
describe(JobService.name, () => {
let sut: JobService;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let communicationMock: jest.Mocked<ICommunicationRepository>;
let jobMock: jest.Mocked<IJobRepository>;
beforeEach(async () => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
communicationMock = newCommunicationRepositoryMock();
jobMock = newJobRepositoryMock();
sut = new JobService(assetMock, communicationMock, jobMock);
sut = new JobService(assetMock, communicationMock, jobMock, configMock);
});
it('should work', () => {
@ -64,16 +72,16 @@ describe(JobService.name, () => {
};
await expect(sut.getAllJobsStatus()).resolves.toEqual({
'background-task-queue': expectedJobStatus,
'clip-encoding-queue': expectedJobStatus,
'metadata-extraction-queue': expectedJobStatus,
'object-tagging-queue': expectedJobStatus,
'search-queue': expectedJobStatus,
'storage-template-migration-queue': expectedJobStatus,
'thumbnail-generation-queue': expectedJobStatus,
'video-conversion-queue': expectedJobStatus,
'recognize-faces-queue': expectedJobStatus,
'sidecar-queue': expectedJobStatus,
[QueueName.BACKGROUND_TASK]: expectedJobStatus,
[QueueName.CLIP_ENCODING]: expectedJobStatus,
[QueueName.METADATA_EXTRACTION]: expectedJobStatus,
[QueueName.OBJECT_TAGGING]: expectedJobStatus,
[QueueName.SEARCH]: expectedJobStatus,
[QueueName.STORAGE_TEMPLATE_MIGRATION]: expectedJobStatus,
[QueueName.THUMBNAIL_GENERATION]: expectedJobStatus,
[QueueName.VIDEO_CONVERSION]: expectedJobStatus,
[QueueName.RECOGNIZE_FACES]: expectedJobStatus,
[QueueName.SIDECAR]: expectedJobStatus,
});
});
});
@ -147,6 +155,14 @@ describe(JobService.name, () => {
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_METADATA_EXTRACTION, data: { force: false } });
});
it('should handle a start sidecar command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.SIDECAR, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_SIDECAR, data: { force: false } });
});
it('should handle a start thumbnail generation command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
@ -155,6 +171,14 @@ describe(JobService.name, () => {
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
});
it('should handle a start recognize faces command', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await sut.handleCommand(QueueName.RECOGNIZE_FACES, { command: JobCommand.START, force: false });
expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.QUEUE_RECOGNIZE_FACES, data: { force: false } });
});
it('should throw a bad request when an invalid queue is used', async () => {
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
@ -165,4 +189,19 @@ describe(JobService.name, () => {
expect(jobMock.queue).not.toHaveBeenCalled();
});
});
describe('registerHandlers', () => {
it('should register a handler for each queue', async () => {
const mock = jest.fn();
const handlers = Object.values(JobName).reduce((map, jobName) => ({ ...map, [jobName]: mock }), {}) as Record<
JobName,
JobHandler
>;
await sut.registerHandlers(handlers);
expect(configMock.load).toHaveBeenCalled();
expect(jobMock.addHandler).toHaveBeenCalledTimes(Object.keys(QueueName).length);
});
});
});

40
server/libs/domain/src/job/job.service.ts
View file

@ -2,20 +2,26 @@ import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common'
import { IAssetRepository, mapAsset } from '../asset';
import { CommunicationEvent, ICommunicationRepository } from '../communication';
import { assertMachineLearningEnabled } from '../domain.constant';
import { ISystemConfigRepository } from '../system-config';
import { SystemConfigCore } from '../system-config/system-config.core';
import { JobCommandDto } from './dto';
import { JobCommand, JobName, QueueName } from './job.constants';
import { IJobRepository, JobItem } from './job.repository';
import { IJobRepository, JobHandler, JobItem } from './job.repository';
import { AllJobStatusResponseDto, JobStatusDto } from './response-dto';
@Injectable()
export class JobService {
private logger = new Logger(JobService.name);
private configCore: SystemConfigCore;
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(ICommunicationRepository) private communicationRepository: ICommunicationRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
) {}
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
) {
this.configCore = new SystemConfigCore(configRepository);
}
handleCommand(queueName: QueueName, dto: JobCommandDto): Promise<void> {
this.logger.debug(`Handling command: queue=${queueName},force=${dto.force}`);
@ -90,6 +96,36 @@ export class JobService {
}
}
async registerHandlers(jobHandlers: Record<JobName, JobHandler>) {
const config = await this.configCore.getConfig();
for (const queueName of Object.values(QueueName)) {
const concurrency = config.job[queueName].concurrency;
this.logger.debug(`Registering ${queueName} with a concurrency of ${concurrency}`);
this.jobRepository.addHandler(queueName, concurrency, async (item: JobItem): Promise<void> => {
const { name, data } = item;
try {
const handler = jobHandlers[name];
const success = await handler(data);
if (success) {
await this.onDone(item);
}
} catch (error: Error | any) {
this.logger.error(`Unable to run job handler: ${error}`, error?.stack, data);
}
});
}
this.configCore.config$.subscribe((config) => {
this.logger.log(`Updating queue concurrency settings`);
for (const queueName of Object.values(QueueName)) {
const concurrency = config.job[queueName].concurrency;
this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);
this.jobRepository.setConcurrency(queueName, concurrency);
}
});
}
async handleNightlyJobs() {
await this.jobRepository.queue({ name: JobName.USER_DELETE_CHECK });
await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });

73
server/libs/domain/src/system-config/dto/system-config-job.dto.ts Normal file
View file

@ -0,0 +1,73 @@
import { ApiProperty } from '@nestjs/swagger';
import { Type } from 'class-transformer';
import { IsInt, IsObject, IsPositive, ValidateNested } from 'class-validator';
import { QueueName } from '../../job';
export class JobSettingsDto {
@IsInt()
@IsPositive()
@ApiProperty({ type: 'integer' })
concurrency!: number;
}
export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.THUMBNAIL_GENERATION]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.METADATA_EXTRACTION]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.VIDEO_CONVERSION]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.OBJECT_TAGGING]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.CLIP_ENCODING]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.BACKGROUND_TASK]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.SEARCH]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.RECOGNIZE_FACES]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.SIDECAR]!: JobSettingsDto;
}

6
server/libs/domain/src/system-config/dto/system-config.dto.ts
View file

@ -1,6 +1,7 @@
import { SystemConfig } from '@app/infra/entities';
import { Type } from 'class-transformer';
import { IsObject, ValidateNested } from 'class-validator';
import { SystemConfigJobDto } from './system-config-job.dto';
import { SystemConfigFFmpegDto } from './system-config-ffmpeg.dto';
import { SystemConfigOAuthDto } from './system-config-oauth.dto';
import { SystemConfigPasswordLoginDto } from './system-config-password-login.dto';
@ -26,6 +27,11 @@ export class SystemConfigDto {
@ValidateNested()
@IsObject()
storageTemplate!: SystemConfigStorageTemplateDto;
@Type(() => SystemConfigJobDto)
@ValidateNested()
@IsObject()
job!: SystemConfigJobDto;
}
export function mapConfig(config: SystemConfig): SystemConfigDto {

25
server/libs/domain/src/system-config/system-config.core.ts
View file

@ -1,13 +1,20 @@
import { SystemConfig, SystemConfigEntity, SystemConfigKey, TranscodePreset } from '@app/infra/entities';
import {
SystemConfig,
SystemConfigEntity,
SystemConfigKey,
SystemConfigValue,
TranscodePreset,
} from '@app/infra/entities';
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
import * as _ from 'lodash';
import { Subject } from 'rxjs';
import { DeepPartial } from 'typeorm';
import { QueueName } from '../job/job.constants';
import { ISystemConfigRepository } from './system-config.repository';
export type SystemConfigValidator = (config: SystemConfig) => void | Promise<void>;
const defaults: SystemConfig = Object.freeze({
const defaults = Object.freeze<SystemConfig>({
ffmpeg: {
crf: 23,
threads: 0,
@ -19,6 +26,18 @@ const defaults: SystemConfig = Object.freeze({
twoPass: false,
transcode: TranscodePreset.REQUIRED,
},
job: {
[QueueName.BACKGROUND_TASK]: { concurrency: 5 },
[QueueName.CLIP_ENCODING]: { concurrency: 2 },
[QueueName.METADATA_EXTRACTION]: { concurrency: 5 },
[QueueName.OBJECT_TAGGING]: { concurrency: 2 },
[QueueName.RECOGNIZE_FACES]: { concurrency: 2 },
[QueueName.SEARCH]: { concurrency: 5 },
[QueueName.SIDECAR]: { concurrency: 5 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
},
oauth: {
enabled: false,
issuerUrl: '',
@ -85,7 +104,7 @@ export class SystemConfigCore {
for (const key of Object.values(SystemConfigKey)) {
// get via dot notation
const item = { key, value: _.get(config, key) };
const item = { key, value: _.get(config, key) as SystemConfigValue };
const defaultValue = _.get(defaults, key);
const isMissing = !_.has(config, key);

18
server/libs/domain/src/system-config/system-config.service.spec.ts
View file

@ -1,7 +1,7 @@
import { SystemConfigEntity, SystemConfigKey, TranscodePreset } from '@app/infra/entities';
import { SystemConfig, SystemConfigEntity, SystemConfigKey, TranscodePreset } from '@app/infra/entities';
import { BadRequestException } from '@nestjs/common';
import { newJobRepositoryMock, newSystemConfigRepositoryMock, systemConfigStub } from '../../test';
import { IJobRepository, JobName } from '../job';
import { IJobRepository, JobName, QueueName } from '../job';
import { SystemConfigValidator } from './system-config.core';
import { ISystemConfigRepository } from './system-config.repository';
import { SystemConfigService } from './system-config.service';
@ -11,7 +11,19 @@ const updates: SystemConfigEntity[] = [
{ key: SystemConfigKey.OAUTH_AUTO_LAUNCH, value: true },
];
const updatedConfig = Object.freeze({
const updatedConfig = Object.freeze<SystemConfig>({
job: {
[QueueName.BACKGROUND_TASK]: { concurrency: 5 },
[QueueName.CLIP_ENCODING]: { concurrency: 2 },
[QueueName.METADATA_EXTRACTION]: { concurrency: 5 },
[QueueName.OBJECT_TAGGING]: { concurrency: 2 },
[QueueName.RECOGNIZE_FACES]: { concurrency: 2 },
[QueueName.SEARCH]: { concurrency: 5 },
[QueueName.SIDECAR]: { concurrency: 5 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
},
ffmpeg: {
crf: 30,
threads: 0,

13
server/libs/domain/test/fixtures.ts
View file

@ -23,6 +23,7 @@ import {
AuthUserDto,
ExifResponseDto,
mapUser,
QueueName,
SearchResult,
SharedLinkResponseDto,
TagResponseDto,
@ -531,6 +532,18 @@ export const systemConfigStub = {
twoPass: false,
transcode: TranscodePreset.REQUIRED,
},
job: {
[QueueName.BACKGROUND_TASK]: { concurrency: 5 },
[QueueName.CLIP_ENCODING]: { concurrency: 2 },
[QueueName.METADATA_EXTRACTION]: { concurrency: 5 },
[QueueName.OBJECT_TAGGING]: { concurrency: 2 },
[QueueName.RECOGNIZE_FACES]: { concurrency: 2 },
[QueueName.SEARCH]: { concurrency: 5 },
[QueueName.SIDECAR]: { concurrency: 5 },
[QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
[QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
[QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
},
oauth: {
autoLaunch: false,
autoRegister: true,

2
server/libs/domain/test/job.repository.mock.ts
View file

@ -2,6 +2,8 @@ import { IJobRepository } from '../src';
export const newJobRepositoryMock = (): jest.Mocked<IJobRepository> => {
return {
addHandler: jest.fn(),
setConcurrency: jest.fn(),
empty: jest.fn(),
pause: jest.fn(),
resume: jest.fn(),

20
server/libs/infra/src/entities/system-config.entity.ts
View file

@ -1,7 +1,8 @@
import { Column, Entity, PrimaryColumn } from 'typeorm';
import { QueueName } from '../../../domain/src';
@Entity('system_config')
export class SystemConfigEntity<T = string | boolean | number> {
export class SystemConfigEntity<T = SystemConfigValue> {
@PrimaryColumn()
key!: SystemConfigKey;
@ -9,7 +10,7 @@ export class SystemConfigEntity<T = string | boolean | number> {
value!: T;
}
export type SystemConfigValue = any;
export type SystemConfigValue = string | number | boolean;
// dot notation matches path in `SystemConfig`
export enum SystemConfigKey {
@ -22,6 +23,18 @@ export enum SystemConfigKey {
FFMPEG_MAX_BITRATE = 'ffmpeg.maxBitrate',
FFMPEG_TWO_PASS = 'ffmpeg.twoPass',
FFMPEG_TRANSCODE = 'ffmpeg.transcode',
JOB_THUMBNAIL_GENERATION_CONCURRENCY = 'job.thumbnailGeneration.concurrency',
JOB_METADATA_EXTRACTION_CONCURRENCY = 'job.metadataExtraction.concurrency',
JOB_VIDEO_CONVERSION_CONCURRENCY = 'job.videoConversion.concurrency',
JOB_OBJECT_TAGGING_CONCURRENCY = 'job.objectTagging.concurrency',
JOB_RECOGNIZE_FACES_CONCURRENCY = 'job.recognizeFaces.concurrency',
JOB_CLIP_ENCODING_CONCURRENCY = 'job.clipEncoding.concurrency',
JOB_BACKGROUND_TASK_CONCURRENCY = 'job.backgroundTask.concurrency',
JOB_STORAGE_TEMPLATE_MIGRATION_CONCURRENCY = 'job.storageTemplateMigration.concurrency',
JOB_SEARCH_CONCURRENCY = 'job.search.concurrency',
JOB_SIDECAR_CONCURRENCY = 'job.sidecar.concurrency',
OAUTH_ENABLED = 'oauth.enabled',
OAUTH_ISSUER_URL = 'oauth.issuerUrl',
OAUTH_CLIENT_ID = 'oauth.clientId',
@ -32,7 +45,9 @@ export enum SystemConfigKey {
OAUTH_AUTO_REGISTER = 'oauth.autoRegister',
OAUTH_MOBILE_OVERRIDE_ENABLED = 'oauth.mobileOverrideEnabled',
OAUTH_MOBILE_REDIRECT_URI = 'oauth.mobileRedirectUri',
PASSWORD_LOGIN_ENABLED = 'passwordLogin.enabled',
STORAGE_TEMPLATE = 'storageTemplate.template',
}
@ -55,6 +70,7 @@ export interface SystemConfig {
twoPass: boolean;
transcode: TranscodePreset;
};
job: Record<QueueName, { concurrency: number }>;
oauth: {
enabled: boolean;
issuerUrl: string;

9
server/libs/infra/src/infra.config.ts
View file

@ -1,5 +1,6 @@
import { QueueName } from '@app/domain';
import { BullModuleOptions } from '@nestjs/bull';
import { RegisterQueueOptions } from '@nestjs/bullmq';
import { QueueOptions } from 'bullmq';
import { RedisOptions } from 'ioredis';
import { InitOptions } from 'local-reverse-geocoder';
import { ConfigurationOptions } from 'typesense/lib/Typesense/Configuration';
@ -26,9 +27,9 @@ function parseRedisConfig(): RedisOptions {
export const redisConfig: RedisOptions = parseRedisConfig();
export const bullConfig: BullModuleOptions = {
export const bullConfig: QueueOptions = {
prefix: 'immich_bull',
redis: redisConfig,
connection: redisConfig,
defaultJobOptions: {
attempts: 3,
removeOnComplete: true,
@ -36,7 +37,7 @@ export const bullConfig: BullModuleOptions = {
},
};
export const bullQueues: BullModuleOptions[] = Object.values(QueueName).map((name) => ({ name }));
export const bullQueues: RegisterQueueOptions[] = Object.values(QueueName).map((name) => ({ name }));
function parseTypeSenseConfig(): ConfigurationOptions {
const typesenseURL = process.env.TYPESENSE_URL;

2
server/libs/infra/src/infra.module.ts
View file

@ -21,7 +21,7 @@ import {
IUserRepository,
IUserTokenRepository,
} from '@app/domain';
import { BullModule } from '@nestjs/bull';
import { BullModule } from '@nestjs/bullmq';
import { Global, Module, Provider } from '@nestjs/common';
import { ConfigModule } from '@nestjs/config';
import { TypeOrmModule } from '@nestjs/typeorm';

41
server/libs/infra/src/repositories/job.repository.ts
View file

@ -1,13 +1,33 @@
import { IJobRepository, JobCounts, JobItem, JobName, JOBS_TO_QUEUE, QueueName, QueueStatus } from '@app/domain';
import { getQueueToken } from '@nestjs/bull';
import { Injectable } from '@nestjs/common';
import { getQueueToken } from '@nestjs/bullmq';
import { Injectable, Logger } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { JobOptions, Queue, type JobCounts as BullJobCounts } from 'bull';
import { Job, JobsOptions, Processor, Queue, Worker, WorkerOptions } from 'bullmq';
import { bullConfig } from '../infra.config';
@Injectable()
export class JobRepository implements IJobRepository {
private workers: Partial<Record<QueueName, Worker>> = {};
private logger = new Logger(JobRepository.name);
constructor(private moduleRef: ModuleRef) {}
addHandler(queueName: QueueName, concurrency: number, handler: (item: JobItem) => Promise<void>) {
const workerHandler: Processor = async (job: Job) => handler(job as JobItem);
const workerOptions: WorkerOptions = { ...bullConfig, concurrency };
this.workers[queueName] = new Worker(queueName, workerHandler, workerOptions);
}
setConcurrency(queueName: QueueName, concurrency: number) {
const worker = this.workers[queueName];
if (!worker) {
this.logger.warn(`Unable to set queue concurrency, worker not found: '${queueName}'`);
return;
}
worker.concurrency = concurrency;
}
async getQueueStatus(name: QueueName): Promise<QueueStatus> {
const queue = this.getQueue(name);
@ -26,13 +46,18 @@ export class JobRepository implements IJobRepository {
}
empty(name: QueueName) {
return this.getQueue(name).empty();
return this.getQueue(name).drain();
}
getJobCounts(name: QueueName): Promise<JobCounts> {
// Typecast needed because the `paused` key is missing from Bull's
// type definition. Can be removed once fixed upstream.
return this.getQueue(name).getJobCounts() as Promise<BullJobCounts & { paused: number }>;
return this.getQueue(name).getJobCounts(
'active',
'completed',
'failed',
'delayed',
'waiting',
'paused',
) as unknown as Promise<JobCounts>;
}
async queue(item: JobItem): Promise<void> {
@ -43,7 +68,7 @@ export class JobRepository implements IJobRepository {
await this.getQueue(JOBS_TO_QUEUE[jobName]).add(jobName, jobData, jobOptions);
}
private getJobOptions(item: JobItem): JobOptions | null {
private getJobOptions(item: JobItem): JobsOptions | null {
switch (item.name) {
case JobName.GENERATE_FACE_THUMBNAIL:
return { priority: 1 };

2
server/libs/infra/src/repositories/system-config.repository.ts
View file

@ -9,7 +9,7 @@ export class SystemConfigRepository implements ISystemConfigRepository {
private repository: Repository<SystemConfigEntity>,
) {}
load(): Promise<SystemConfigEntity<string | boolean | number>[]> {
load(): Promise<SystemConfigEntity[]> {
return this.repository.find();
}

204
server/package-lock.json generated
View file

@ -10,7 +10,7 @@
"license": "UNLICENSED",
"dependencies": {
"@babel/runtime": "^7.20.13",
"@nestjs/bull": "^0.6.2",
"@nestjs/bullmq": "^1.1.0",
"@nestjs/common": "^9.2.1",
"@nestjs/config": "^2.2.0",
"@nestjs/core": "^9.2.1",
@ -24,7 +24,7 @@
"archiver": "^5.3.1",
"axios": "^0.26.0",
"bcrypt": "^5.0.1",
"bull": "^4.10.2",
"bullmq": "^3.14.1",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.6",
@ -1507,20 +1507,6 @@
"win32"
]
},
"node_modules/@nestjs/bull": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/@nestjs/bull/-/bull-0.6.3.tgz",
"integrity": "sha512-CckH9O3t9qSiO4RCzdYvtFSaaMfIhTXMYagV/rtmVvI1SX5XNnxEaQXvtjxDBXF9DB1JE/5AejIl6ICym+MJIw==",
"dependencies": {
"@nestjs/bull-shared": "^0.1.3",
"tslib": "2.5.0"
},
"peerDependencies": {
"@nestjs/common": "^6.10.11 || ^7.0.0 || ^8.0.0 || ^9.0.0",
"@nestjs/core": "^6.10.11 || ^7.0.0 || ^8.0.0 || ^9.0.0",
"bull": "^3.3 || ^4.0.0"
}
},
"node_modules/@nestjs/bull-shared": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-0.1.3.tgz",
@ -1533,6 +1519,20 @@
"@nestjs/core": "^6.10.11 || ^7.0.0 || ^8.0.0 || ^9.0.0"
}
},
"node_modules/@nestjs/bullmq": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-1.1.0.tgz",
"integrity": "sha512-XloO39ACm9TuB8XOX53iMUaCW5BTQAnZADtX4a9kczJZU/5/cQVBfSCyfzq9L6dfi5EX6w/1Ayyv+5qBQ5yrzw==",
"dependencies": {
"@nestjs/bull-shared": "^0.1.3",
"tslib": "2.5.0"
},
"peerDependencies": {
"@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0",
"@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0",
"bullmq": "^3.0.0"
}
},
"node_modules/@nestjs/cli": {
"version": "9.4.2",
"resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-9.4.2.tgz",
@ -4232,30 +4232,56 @@
"node": ">=0.2.0"
}
},
"node_modules/bull": {
"version": "4.10.4",
"resolved": "https://registry.npmjs.org/bull/-/bull-4.10.4.tgz",
"integrity": "sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA==",
"node_modules/bullmq": {
"version": "3.14.1",
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.14.1.tgz",
"integrity": "sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==",
"dependencies": {
"cron-parser": "^4.2.1",
"debuglog": "^1.0.0",
"get-port": "^5.1.1",
"ioredis": "^5.0.0",
"cron-parser": "^4.6.0",
"glob": "^8.0.3",
"ioredis": "^5.3.2",
"lodash": "^4.17.21",
"msgpackr": "^1.5.2",
"semver": "^7.3.2",
"uuid": "^8.3.0"
"msgpackr": "^1.6.2",
"semver": "^7.3.7",
"tslib": "^2.0.0",
"uuid": "^9.0.0"
}
},
"node_modules/bullmq/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"node_modules/bullmq/node_modules/glob": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
"integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^5.0.1",
"once": "^1.3.0"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/bull/node_modules/uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
"bin": {
"uuid": "dist/bin/uuid"
"node_modules/bullmq/node_modules/minimatch": {
"version": "5.1.6",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
"integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
"dependencies": {
"brace-expansion": "^2.0.1"
},
"engines": {
"node": ">=10"
}
},
"node_modules/busboy": {
@ -5013,14 +5039,6 @@
}
}
},
"node_modules/debuglog": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz",
"integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==",
"engines": {
"node": "*"
}
},
"node_modules/decimal.js": {
"version": "10.4.3",
"resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz",
@ -6422,17 +6440,6 @@
"node": ">=8.0.0"
}
},
"node_modules/get-port": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz",
"integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==",
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/get-stream": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
@ -8429,9 +8436,9 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/msgpackr": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.1.tgz",
"integrity": "sha512-jJdrNH8tzfCtT0rjPFryBXjRDQE7rqfLkah4/8B4gYa7NNZYFBcGxqWBtfQpGC+oYyBwlkj3fARk4aooKNPHxg==",
"version": "1.9.2",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.2.tgz",
"integrity": "sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==",
"optionalDependencies": {
"msgpackr-extract": "^3.0.2"
}
@ -13122,15 +13129,6 @@
"integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==",
"optional": true
},
"@nestjs/bull": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/@nestjs/bull/-/bull-0.6.3.tgz",
"integrity": "sha512-CckH9O3t9qSiO4RCzdYvtFSaaMfIhTXMYagV/rtmVvI1SX5XNnxEaQXvtjxDBXF9DB1JE/5AejIl6ICym+MJIw==",
"requires": {
"@nestjs/bull-shared": "^0.1.3",
"tslib": "2.5.0"
}
},
"@nestjs/bull-shared": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-0.1.3.tgz",
@ -13139,6 +13137,15 @@
"tslib": "2.5.0"
}
},
"@nestjs/bullmq": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-1.1.0.tgz",
"integrity": "sha512-XloO39ACm9TuB8XOX53iMUaCW5BTQAnZADtX4a9kczJZU/5/cQVBfSCyfzq9L6dfi5EX6w/1Ayyv+5qBQ5yrzw==",
"requires": {
"@nestjs/bull-shared": "^0.1.3",
"tslib": "2.5.0"
}
},
"@nestjs/cli": {
"version": "9.4.2",
"resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-9.4.2.tgz",
@ -15212,25 +15219,48 @@
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="
},
"bull": {
"version": "4.10.4",
"resolved": "https://registry.npmjs.org/bull/-/bull-4.10.4.tgz",
"integrity": "sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA==",
"bullmq": {
"version": "3.14.1",
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.14.1.tgz",
"integrity": "sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==",
"requires": {
"cron-parser": "^4.2.1",
"debuglog": "^1.0.0",
"get-port": "^5.1.1",
"ioredis": "^5.0.0",
"cron-parser": "^4.6.0",
"glob": "^8.0.3",
"ioredis": "^5.3.2",
"lodash": "^4.17.21",
"msgpackr": "^1.5.2",
"semver": "^7.3.2",
"uuid": "^8.3.0"
"msgpackr": "^1.6.2",
"semver": "^7.3.7",
"tslib": "^2.0.0",
"uuid": "^9.0.0"
},
"dependencies": {
"uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
"brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"requires": {
"balanced-match": "^1.0.0"
}
},
"glob": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
"integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^5.0.1",
"once": "^1.3.0"
}
},
"minimatch": {
"version": "5.1.6",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
"integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
"requires": {
"brace-expansion": "^2.0.1"
}
}
}
},
@ -15800,11 +15830,6 @@
"ms": "2.1.2"
}
},
"debuglog": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz",
"integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw=="
},
"decimal.js": {
"version": "10.4.3",
"resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz",
@ -16867,11 +16892,6 @@
"integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==",
"dev": true
},
"get-port": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz",
"integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ=="
},
"get-stream": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
@ -18386,9 +18406,9 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"msgpackr": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.1.tgz",
"integrity": "sha512-jJdrNH8tzfCtT0rjPFryBXjRDQE7rqfLkah4/8B4gYa7NNZYFBcGxqWBtfQpGC+oYyBwlkj3fARk4aooKNPHxg==",
"version": "1.9.2",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.2.tgz",
"integrity": "sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==",
"requires": {
"msgpackr-extract": "^3.0.2"
}

10
server/package.json
View file

@ -41,7 +41,7 @@
},
"dependencies": {
"@babel/runtime": "^7.20.13",
"@nestjs/bull": "^0.6.2",
"@nestjs/bullmq": "^1.1.0",
"@nestjs/common": "^9.2.1",
"@nestjs/config": "^2.2.0",
"@nestjs/core": "^9.2.1",
@ -55,7 +55,7 @@
"archiver": "^5.3.1",
"axios": "^0.26.0",
"bcrypt": "^5.0.1",
"bull": "^4.10.2",
"bullmq": "^3.14.1",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.6",
@ -140,9 +140,9 @@
"coverageThreshold": {
"./libs/domain/": {
"branches": 80,
"functions": 85,
"lines": 93,
"statements": 93
"functions": 80,
"lines": 90,
"statements": 90
}
},
"setupFilesAfterEnv": [

20
web/src/api/api.ts
View file

@ -15,7 +15,8 @@ import {
ShareApi,
SystemConfigApi,
UserApi,
UserApiFp
UserApiFp,
JobName
} from './open-api';
import { BASE_PATH } from './open-api/base';
import { DUMMY_BASE_URL, toPathString } from './open-api/common';
@ -106,6 +107,23 @@ export class ImmichApi {
const path = `/person/${personId}/thumbnail`;
return this.createUrl(path);
}
public getJobName(jobName: JobName) {
const names: Record<JobName, string> = {
[JobName.ThumbnailGeneration]: 'Generate Thumbnails',
[JobName.MetadataExtraction]: 'Extract Metadata',
[JobName.Sidecar]: 'Sidecar Metadata',
[JobName.ObjectTagging]: 'Tag Objects',
[JobName.ClipEncoding]: 'Encode Clip',
[JobName.RecognizeFaces]: 'Recognize Faces',
[JobName.VideoConversion]: 'Transcode Videos',
[JobName.StorageTemplateMigration]: 'Storage Template Migration',
[JobName.BackgroundTask]: 'Background Tasks',
[JobName.Search]: 'Search'
};
return names[jobName];
}
}
export const api = new ImmichApi({ basePath: '/api' });

126
web/src/api/open-api/api.ts generated
View file

@ -296,61 +296,61 @@ export interface AllJobStatusResponseDto {
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'thumbnail-generation-queue': JobStatusDto;
'thumbnailGeneration': JobStatusDto;
/**
*
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'metadata-extraction-queue': JobStatusDto;
'metadataExtraction': JobStatusDto;
/**
*
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'video-conversion-queue': JobStatusDto;
'videoConversion': JobStatusDto;
/**
*
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'object-tagging-queue': JobStatusDto;
'objectTagging': JobStatusDto;
/**
*
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'clip-encoding-queue': JobStatusDto;
'clipEncoding': JobStatusDto;
/**
*
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'storage-template-migration-queue': JobStatusDto;
'storageTemplateMigration': JobStatusDto;
/**
*
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'background-task-queue': JobStatusDto;
'backgroundTask': JobStatusDto;
/**
*
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'search-queue': JobStatusDto;
'search': JobStatusDto;
/**
*
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'recognize-faces-queue': JobStatusDto;
'recognizeFaces': JobStatusDto;
/**
*
* @type {JobStatusDto}
* @memberof AllJobStatusResponseDto
*/
'sidecar-queue': JobStatusDto;
'sidecar': JobStatusDto;
}
/**
*
@ -1486,21 +1486,34 @@ export interface JobCountsDto {
*/
export const JobName = {
ThumbnailGenerationQueue: 'thumbnail-generation-queue',
MetadataExtractionQueue: 'metadata-extraction-queue',
VideoConversionQueue: 'video-conversion-queue',
ObjectTaggingQueue: 'object-tagging-queue',
RecognizeFacesQueue: 'recognize-faces-queue',
ClipEncodingQueue: 'clip-encoding-queue',
BackgroundTaskQueue: 'background-task-queue',
StorageTemplateMigrationQueue: 'storage-template-migration-queue',
SearchQueue: 'search-queue',
SidecarQueue: 'sidecar-queue'
ThumbnailGeneration: 'thumbnailGeneration',
MetadataExtraction: 'metadataExtraction',
VideoConversion: 'videoConversion',
ObjectTagging: 'objectTagging',
RecognizeFaces: 'recognizeFaces',
ClipEncoding: 'clipEncoding',
BackgroundTask: 'backgroundTask',
StorageTemplateMigration: 'storageTemplateMigration',
Search: 'search',
Sidecar: 'sidecar'
} as const;
export type JobName = typeof JobName[keyof typeof JobName];
/**
*
* @export
* @interface JobSettingsDto
*/
export interface JobSettingsDto {
/**
*
* @type {number}
* @memberof JobSettingsDto
*/
'concurrency': number;
}
/**
*
* @export
@ -2247,6 +2260,12 @@ export interface SystemConfigDto {
* @memberof SystemConfigDto
*/
'storageTemplate': SystemConfigStorageTemplateDto;
/**
*
* @type {SystemConfigJobDto}
* @memberof SystemConfigDto
*/
'job': SystemConfigJobDto;
}
/**
*
@ -2319,6 +2338,73 @@ export const SystemConfigFFmpegDtoTranscodeEnum = {
export type SystemConfigFFmpegDtoTranscodeEnum = typeof SystemConfigFFmpegDtoTranscodeEnum[keyof typeof SystemConfigFFmpegDtoTranscodeEnum];
/**
*
* @export
* @interface SystemConfigJobDto
*/
export interface SystemConfigJobDto {
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'thumbnailGeneration': JobSettingsDto;
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'metadataExtraction': JobSettingsDto;
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'videoConversion': JobSettingsDto;
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'objectTagging': JobSettingsDto;
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'clipEncoding': JobSettingsDto;
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'storageTemplateMigration': JobSettingsDto;
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'backgroundTask': JobSettingsDto;
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'search': JobSettingsDto;
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'recognizeFaces': JobSettingsDto;
/**
*
* @type {JobSettingsDto}
* @memberof SystemConfigJobDto
*/
'sidecar': JobSettingsDto;
}
/**
*
* @export

2
web/src/lib/components/admin-page/jobs/job-tile.svelte
View file

@ -30,7 +30,7 @@
</script>
<div
class="flex sm:flex-row flex-col bg-gray-100 dark:bg-immich-dark-gray rounded-3xl overflow-hidden"
class="flex sm:flex-row flex-col bg-gray-100 dark:bg-immich-dark-gray rounded-[35px] overflow-hidden"
>
<div class="flex flex-col w-full">
{#if queueStatus.isPaused}

51
web/src/lib/components/admin-page/jobs/jobs-panel.svelte
View file

@ -9,15 +9,17 @@
import Icon from 'svelte-material-icons/DotsVertical.svelte';
import FaceRecognition from 'svelte-material-icons/FaceRecognition.svelte';
import FileJpgBox from 'svelte-material-icons/FileJpgBox.svelte';
import FolderMove from 'svelte-material-icons/FolderMove.svelte';
import Table from 'svelte-material-icons/Table.svelte';
import FileXmlBox from 'svelte-material-icons/FileXmlBox.svelte';
import FolderMove from 'svelte-material-icons/FolderMove.svelte';
import Information from 'svelte-material-icons/Information.svelte';
import Table from 'svelte-material-icons/Table.svelte';
import TagMultiple from 'svelte-material-icons/TagMultiple.svelte';
import VectorCircle from 'svelte-material-icons/VectorCircle.svelte';
import Video from 'svelte-material-icons/Video.svelte';
import ConfirmDialogue from '../../shared-components/confirm-dialogue.svelte';
import JobTile from './job-tile.svelte';
import StorageMigrationDescription from './storage-migration-description.svelte';
import { AppRoute } from '$lib/constants';
export let jobs: AllJobStatusResponseDto;
@ -45,52 +47,52 @@
const onFaceConfirm = () => {
faceConfirm = false;
handleCommand(JobName.RecognizeFacesQueue, { command: JobCommand.Start, force: true });
handleCommand(JobName.RecognizeFaces, { command: JobCommand.Start, force: true });
};
const jobDetails: Partial<Record<JobName, JobDetails>> = {
[JobName.ThumbnailGenerationQueue]: {
[JobName.ThumbnailGeneration]: {
icon: FileJpgBox,
title: 'Generate Thumbnails',
title: api.getJobName(JobName.ThumbnailGeneration),
subtitle: 'Regenerate JPEG and WebP thumbnails'
},
[JobName.MetadataExtractionQueue]: {
[JobName.MetadataExtraction]: {
icon: Table,
title: 'Extract Metadata',
title: api.getJobName(JobName.MetadataExtraction),
subtitle: 'Extract metadata information i.e. GPS, resolution...etc'
},
[JobName.SidecarQueue]: {
title: 'Sidecar Metadata',
[JobName.Sidecar]: {
title: api.getJobName(JobName.Sidecar),
icon: FileXmlBox,
subtitle: 'Discover or synchronize sidecar metadata from the filesystem',
allText: 'SYNC',
missingText: 'DISCOVER'
},
[JobName.ObjectTaggingQueue]: {
[JobName.ObjectTagging]: {
icon: TagMultiple,
title: 'Tag Objects',
title: api.getJobName(JobName.ObjectTagging),
subtitle:
'Run machine learning to tag objects\nNote that some assets may not have any objects detected'
},
[JobName.ClipEncodingQueue]: {
[JobName.ClipEncoding]: {
icon: VectorCircle,
title: 'Encode Clip',
title: api.getJobName(JobName.ClipEncoding),
subtitle: 'Run machine learning to generate clip embeddings'
},
[JobName.RecognizeFacesQueue]: {
[JobName.RecognizeFaces]: {
icon: FaceRecognition,
title: 'Recognize Faces',
title: api.getJobName(JobName.RecognizeFaces),
subtitle: 'Run machine learning to recognize faces',
handleCommand: handleFaceCommand
},
[JobName.VideoConversionQueue]: {
[JobName.VideoConversion]: {
icon: Video,
title: 'Transcode Videos',
title: api.getJobName(JobName.VideoConversion),
subtitle: 'Transcode videos not in the desired format'
},
[JobName.StorageTemplateMigrationQueue]: {
[JobName.StorageTemplateMigration]: {
icon: FolderMove,
title: 'Storage Template Migration',
title: api.getJobName(JobName.StorageTemplateMigration),
allowForceCommand: false,
component: StorageMigrationDescription
}
@ -128,6 +130,17 @@
{/if}
<div class="flex flex-col gap-7">
<div class="flex dark:text-white text-black gap-2 bg-gray-200 dark:bg-gray-700 p-6 rounded-full">
<Information />
<p class="text-xs">
MANAGE JOB CURRENCENCY LEVEL IN
<a
href={`${AppRoute.ADMIN_SETTINGS}?open=job-settings`}
class="text-immich-primary dark:text-immich-dark-primary font-medium">JOB SETTINGS</a
>
</p>
</div>
{#each jobDetailsArray as [jobName, { title, subtitle, allText, missingText, allowForceCommand, icon, component, handleCommand: handleCommandOverride }]}
{@const { jobCounts, queueStatus } = jobs[jobName]}
<JobTile

103
web/src/lib/components/admin-page/settings/job-settings/job-settings.svelte Normal file
View file

@ -0,0 +1,103 @@
<script lang="ts">
import {
notificationController,
NotificationType
} from '$lib/components/shared-components/notification/notification';
import { api, JobName, SystemConfigJobDto } from '@api';
import { isEqual } from 'lodash-es';
import { fade } from 'svelte/transition';
import { handleError } from '../../../../utils/handle-error';
import SettingButtonsRow from '../setting-buttons-row.svelte';
import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
export let jobConfig: SystemConfigJobDto; // this is the config that is being edited
let savedConfig: SystemConfigJobDto;
let defaultConfig: SystemConfigJobDto;
const ignoredJobs = [JobName.BackgroundTask, JobName.Search] as JobName[];
const jobNames = Object.values(JobName).filter(
(jobName) => !ignoredJobs.includes(jobName as JobName)
);
async function getConfigs() {
[savedConfig, defaultConfig] = await Promise.all([
api.systemConfigApi.getConfig().then((res) => res.data.job),
api.systemConfigApi.getDefaults().then((res) => res.data.job)
]);
}
async function saveSetting() {
try {
const { data: configs } = await api.systemConfigApi.getConfig();
const result = await api.systemConfigApi.updateConfig({
systemConfigDto: {
...configs,
job: jobConfig
}
});
jobConfig = { ...result.data.job };
savedConfig = { ...result.data.job };
notificationController.show({ message: 'Job settings saved', type: NotificationType.Info });
} catch (error) {
handleError(error, 'Unable to save settings');
}
}
async function reset() {
const { data: resetConfig } = await api.systemConfigApi.getConfig();
jobConfig = { ...resetConfig.job };
savedConfig = { ...resetConfig.job };
notificationController.show({
message: 'Reset Job settings to the recent saved settings',
type: NotificationType.Info
});
}
async function resetToDefault() {
const { data: configs } = await api.systemConfigApi.getDefaults();
jobConfig = { ...configs.job };
defaultConfig = { ...configs.job };
notificationController.show({
message: 'Reset Job settings to default',
type: NotificationType.Info
});
}
</script>
<div>
{#await getConfigs() then}
<div in:fade={{ duration: 500 }}>
<form autocomplete="off" on:submit|preventDefault>
{#each jobNames as jobName}
<div class="flex flex-col gap-4 ml-4 mt-4">
<SettingInputField
inputType={SettingInputFieldType.NUMBER}
label="{api.getJobName(jobName)} Concurrency"
desc=""
bind:value={jobConfig[jobName].concurrency}
required={true}
isEdited={!(jobConfig[jobName].concurrency == savedConfig[jobName].concurrency)}
/>
</div>
{/each}
<div class="ml-4">
<SettingButtonsRow
on:reset={reset}
on:save={saveSetting}
on:reset-to-default={resetToDefault}
showResetToDefault={!isEqual(savedConfig, defaultConfig)}
/>
</div>
</form>
</div>
{/await}
</div>

3
web/src/lib/components/admin-page/settings/setting-input-field.svelte
View file

@ -21,6 +21,9 @@
const handleInput = (e: Event) => {
value = (e.target as HTMLInputElement).value;
if (inputType === SettingInputFieldType.NUMBER) {
value = Number(value) || 0;
}
};
</script>

2
web/src/lib/utils/handle-error.ts
View file

@ -9,7 +9,7 @@ export function handleError(error: unknown, message: string) {
let serverMessage = (error as ApiError)?.response?.data?.message;
if (serverMessage) {
serverMessage = `${String(serverMessage).slice(0, 50)}\n<i>(Immich Server Error)<i>`;
serverMessage = `${String(serverMessage).slice(0, 75)}\n(Immich Server Error)`;
}
notificationController.show({

9
web/src/routes/admin/system-settings/+page.svelte
View file

@ -1,6 +1,7 @@
<script lang="ts">
import { page } from '$app/stores';
import FFmpegSettings from '$lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte';
import JobSettings from '$lib/components/admin-page/settings/job-settings/job-settings.svelte';
import OAuthSettings from '$lib/components/admin-page/settings/oauth/oauth-settings.svelte';
import PasswordLoginSettings from '$lib/components/admin-page/settings/password-login/password-login-settings.svelte';
import SettingAccordion from '$lib/components/admin-page/settings/setting-accordion.svelte';
@ -28,6 +29,14 @@
<FFmpegSettings ffmpegConfig={configs.ffmpeg} />
</SettingAccordion>
<SettingAccordion
title="Job Settings"
subtitle="Manage job concurrency"
isOpen={$page.url.searchParams.get('open') === 'job-settings'}
>
<JobSettings jobConfig={configs.job} />
</SettingAccordion>
<SettingAccordion
title="Password Authentication"
subtitle="Manage login with password settings"