From cd375a976e14c059225f2e8c0d69cac6182a5fff Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Tue, 31 Oct 2023 21:19:12 +0100 Subject: [PATCH] feat(server): custom library scanning interval (#4390) * add automatic library scan config options * add validation * open api * use CronJob instead of cron-validator * fix tests * catch potential error of the library scan initialization * better description for input field * move library scan job initialization to server app service * fix tests * add comments to all parameters of cronjob contructor * make scan a child of a more general library object * open api * chore: cleanup * move cronjob handling to job repoistory * web: select for common cron expressions * fix open api * fix tests * put scanning settings in nested accordion * fix system config validation * refactor, tests --------- Co-authored-by: Jason Rasmussen --- cli/src/api/open-api/api.ts | 38 +++++ mobile/openapi/.openapi-generator/FILES | 6 + mobile/openapi/README.md | Bin 21936 -> 22064 bytes mobile/openapi/doc/SystemConfigDto.md | Bin 1371 -> 1448 bytes mobile/openapi/doc/SystemConfigLibraryDto.md | Bin 0 -> 469 bytes .../openapi/doc/SystemConfigLibraryScanDto.md | Bin 0 -> 459 bytes mobile/openapi/lib/api.dart | Bin 7158 -> 7253 bytes mobile/openapi/lib/api_client.dart | Bin 21368 -> 21568 bytes .../openapi/lib/model/system_config_dto.dart | Bin 6044 -> 6314 bytes .../lib/model/system_config_library_dto.dart | Bin 0 -> 2907 bytes .../model/system_config_library_scan_dto.dart | Bin 0 -> 3298 bytes .../openapi/test/system_config_dto_test.dart | Bin 1926 -> 2043 bytes .../test/system_config_library_dto_test.dart | Bin 0 -> 602 bytes .../system_config_library_scan_dto_test.dart | Bin 0 -> 711 bytes server/immich-openapi-specs.json | 32 +++- server/package-lock.json | 110 +------------ server/src/domain/domain.util.ts | 11 ++ server/src/domain/job/job.service.spec.ts | 1 - server/src/domain/job/job.service.ts | 1 - .../domain/library/library.service.spec.ts | 42 ++++- server/src/domain/library/library.service.ts | 26 +++- .../src/domain/repositories/job.repository.ts | 3 + server/src/domain/system-config/dto/index.ts | 1 + .../dto/system-config-library.dto.ts | 40 +++++ .../system-config/dto/system-config.dto.ts | 6 + .../system-config/system-config.core.ts | 7 + .../system-config.service.spec.ts | 6 + server/src/immich/app.service.ts | 4 +- .../infra/entities/system-config.entity.ts | 9 ++ .../src/infra/repositories/job.repository.ts | 44 +++++- .../test/repositories/job.repository.mock.ts | 3 + server/test/test-utils.ts | 4 + web/src/api/open-api/api.ts | 38 +++++ .../library-settings/library-settings.svelte | 145 ++++++++++++++++++ .../routes/admin/system-settings/+page.svelte | 5 + 35 files changed, 469 insertions(+), 113 deletions(-) create mode 100644 mobile/openapi/doc/SystemConfigLibraryDto.md create mode 100644 mobile/openapi/doc/SystemConfigLibraryScanDto.md create mode 100644 mobile/openapi/lib/model/system_config_library_dto.dart create mode 100644 mobile/openapi/lib/model/system_config_library_scan_dto.dart create mode 100644 mobile/openapi/test/system_config_library_dto_test.dart create mode 100644 mobile/openapi/test/system_config_library_scan_dto_test.dart create mode 100644 server/src/domain/system-config/dto/system-config-library.dto.ts create mode 100644 web/src/lib/components/admin-page/settings/library-settings/library-settings.svelte diff --git a/cli/src/api/open-api/api.ts b/cli/src/api/open-api/api.ts index e79388602d..97dc8523c3 100644 --- a/cli/src/api/open-api/api.ts +++ b/cli/src/api/open-api/api.ts @@ -3283,6 +3283,12 @@ export interface SystemConfigDto { * @memberof SystemConfigDto */ 'job': SystemConfigJobDto; + /** + * + * @type {SystemConfigLibraryDto} + * @memberof SystemConfigDto + */ + 'library': SystemConfigLibraryDto; /** * * @type {SystemConfigMachineLearningDto} @@ -3534,6 +3540,38 @@ export interface SystemConfigJobDto { */ 'videoConversion': JobSettingsDto; } +/** + * + * @export + * @interface SystemConfigLibraryDto + */ +export interface SystemConfigLibraryDto { + /** + * + * @type {SystemConfigLibraryScanDto} + * @memberof SystemConfigLibraryDto + */ + 'scan': SystemConfigLibraryScanDto; +} +/** + * + * @export + * @interface SystemConfigLibraryScanDto + */ +export interface SystemConfigLibraryScanDto { + /** + * + * @type {string} + * @memberof SystemConfigLibraryScanDto + */ + 'cronExpression': string; + /** + * + * @type {boolean} + * @memberof SystemConfigLibraryScanDto + */ + 'enabled': boolean; +} /** * * @export diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 6350677f1e..c73dcfd065 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -128,6 +128,8 @@ doc/SystemConfigApi.md doc/SystemConfigDto.md doc/SystemConfigFFmpegDto.md doc/SystemConfigJobDto.md +doc/SystemConfigLibraryDto.md +doc/SystemConfigLibraryScanDto.md doc/SystemConfigMachineLearningDto.md doc/SystemConfigMapDto.md doc/SystemConfigNewVersionCheckDto.md @@ -296,6 +298,8 @@ lib/model/smart_info_response_dto.dart lib/model/system_config_dto.dart lib/model/system_config_f_fmpeg_dto.dart lib/model/system_config_job_dto.dart +lib/model/system_config_library_dto.dart +lib/model/system_config_library_scan_dto.dart lib/model/system_config_machine_learning_dto.dart lib/model/system_config_map_dto.dart lib/model/system_config_new_version_check_dto.dart @@ -451,6 +455,8 @@ test/system_config_api_test.dart test/system_config_dto_test.dart test/system_config_f_fmpeg_dto_test.dart test/system_config_job_dto_test.dart +test/system_config_library_dto_test.dart +test/system_config_library_scan_dto_test.dart test/system_config_machine_learning_dto_test.dart test/system_config_map_dto_test.dart test/system_config_new_version_check_dto_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 6ec60396206a866dfe9b85a84e7d05010d1bd834..9e5462b084bcffa46fe97165485d5b9ddb1ff686 100644 GIT binary patch delta 69 zcmdn6nsLJ##tkuk>^_-EMTtd~lMncc!Px?S65LSn;N-+SkQ^6+Ke;hLa&wGd2s;3t C=o!%f delta 14 Wcmdn6hH=Ac#tkukoA3CAumb=y9R^GQ diff --git a/mobile/openapi/doc/SystemConfigDto.md b/mobile/openapi/doc/SystemConfigDto.md index 98a6266402860fda08c1b95c7bafa289fbca4e3d..73c5b70dcfee85759f9a9444897fd13356161dbc 100644 GIT binary patch delta 44 pcmcc3wSs#?G9yP$W>QgNQDywZ2O{h~5ccGUjKVPX=3qt+765q#58(g+ delta 12 TcmZ3%eVc1TGUMhAj6awGA-x3* diff --git a/mobile/openapi/doc/SystemConfigLibraryDto.md b/mobile/openapi/doc/SystemConfigLibraryDto.md new file mode 100644 index 0000000000000000000000000000000000000000..22c8ddf34d56cfa2cc328cb4d30b4810bac684a5 GIT binary patch literal 469 zcma)2!D_=W488j+1Ua-ZIK6MDZs}prma^^^0`X$EHLES-XotW)esWe=H#RmSFi-F4 ziKhZE+MuJWJQ{nh_Txa--(xtq6EJo87m6DL8UoefKO`fRrjKLV5iebt3b?H zkAe7Np5M;dtDK(&i@PkHlJ+5yfeb{pQx^C;e;$3lDu`V$o(#H#c1TGfuTC0XAu0dD zOo%R^tKWQp7*nbc$s$BU!MJ_`2`5#mp&dZabc;BAk?e4|PhaJPe4A_HgNbxJ-jwC) ysa{vxy0p5`*u7eCr1I0{8H1ckic9|OUhg0O=}nbMaN47B^dsVP;jiUO#@Gk)TaGON literal 0 HcmV?d00001 diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 7a621b7f473323b087ed88968de8ff8588a0b621..d72aafe58b0bae87a0dbdd2fc3f7373fe56b1e56 100644 GIT binary patch delta 37 tcmexne$`?FquAvAf(n!Oi}Uj2WF{3Q7FEUlNHtFCi6S+Otv!Dp1i=Bo7*QdsVK3iGB`OgZ*rrJ#AE>nR53?$ M6_C2kz7E>L0C>h2D*ylh delta 18 acmX@Gg7L>P#tps>lieMKHmf^I3IhO5I|jM{ diff --git a/mobile/openapi/lib/model/system_config_dto.dart b/mobile/openapi/lib/model/system_config_dto.dart index 89c7e5f7d2cb7e45118e7e9ac2c466aed73ebc9e..c8407c2ce29acb6d2f1bd5fea840869565698035 100644 GIT binary patch delta 204 zcmbQEzshjK0Y;9T%%q~kqRRNm2be@QA7IR36!w9LxRm58z$7TFeDtj)VPCNN4udFrtWwhAQ~nZ-$!ftDzXtxRf3s)mvJ-#bG} zij2x>Py@ClYHmE|%utibXflDf|K87E{4=|oeSUjCyM~+hx3dtg=WsW_hfnkC_c#BY zp&41e$(XkB%jmC{Bf1qEsWeY=rITEg{1R$e8=fV+Frml=*UsGxCM$*HCMlKNpqVY0OkVw& zqy^JvG{STiR1T`*l2szY@6~9OmdqGf?2YBycS`1Bb1T-G>HW1;@Gt_HAPk?mt+h13 zK=K_dhpks2T&XFK&~4Zzpojo15MT>oB9k&1;`-qN>lyT1pas=Kmd@sl`o^_JNYH+* zR7soC%qk6W9L(2Wc#2!K1=kl4Pr*Z%vNE82Vf*C!cYg)qBmn2MQS0oH=Usm^ak2##aYq9fhheA4FQuG__i`2tbuQ=ths?)t8Ei;>6TD-IU0;folD&3Dg;@V zs0gBeCyY~LkAzC|9Gk=Sm9A6Nj7dmM3AULPX@VVDsEkPQ3n!HP+EwBJYykBYwrFBe zfvbF4B^4`>PD*_TbI}&kh9Y8@X&(e zq0!vbrF9}4PO)71ItgP6Ob}B^aDMJ|7YTEx>}*@veH$5J$Ken(3Dgc}l{0;O-V)+z(tTf(tQ>?{q+OWz@sfiEX%wodY&Ha7W5wL_s#1lI5*500?d^K?r0I53){_Y%5OGQooi&tp1w z{m4Ax6j$}WWi_HZ!A-j-V#x37x$L@dk+be`^$V5}w550a%J(!Hoxj&AmGA`B=-M?I zZXR!Go;j?PKhAc-7Pv&+MW^^MBQLx$`Zfa-hdesbOAW=uz8+8tsq0iXPsxo81~v2C zj>Z^$H9eW|cQo}(xjaXD%_Qtii(e`WcXv$tFgFoV*TL8mSlBi6V{3<<>bz)<<^m?{ ziDd%!1@Cmxy;`9J4W7*miLX&IYYuqw8B9w}!r;SVen;KUp3`|e0d zmg1LW4G`NB?|8@OdG2_FL2odC_kT<$Z+;u!jX%Gij<4YQj4rwnvX*A(iHFrgWT%f?q%>OT*Kc7ktU34eMvITq$i2daz>6HYQ$` zHrD)~QfPFSY>A&Mrt#ZyZE$mK4!b8x8q1`OMT!AMCb)L)=3uZwNN(an$pxC(oXPm- zALArv+VpxD&VtH7t+-?>5#j%3ua^|e7`WdU%a5NrV1Q73 z=C;(*00YU_Fgxxs2jNmqc!aLU?E=~vpaunOE=-hYCCC4+G&e>l86L2}z)duV1_Dz3r2OVYnyxLf5=Xn(U6{FQkP9w~#ZFe^4pMj9tJ+wxm|c z6ND1!B(v6+}*++f)(&q`5%D;t$&>HSvTbHRrfipk2-u+ z#h&+F%v-kl#fJzX-C*&EWCT`yGD>1hFQZIIR=^j01w+`ge%xMqGMvRs_q&n%b96eO zzb8&rRy*s48ZTrj5Ep>_wj|B(EF54m1>aK)2y5UgD@tx4(`s4mI&-hE)-p8cQ(4Xs zrg;dmEDEo=wJ@#(TY|vC%CIxIrlQLPg>Paii5Po~6-kWkC|4;7#J@OEzQ4L`90wIA zKFgXAOe}CYQkh_C^?QI0Rn8R}ja;%;r8{B8)aCz9=>9NKVX=Fn7w1 zmn*xecSjDfWd>CcwZ@L`On2)w%U%J)6FQ1W)b50MmOKLp@XlzIb9ocw1}mt zS6R!`178!+p`r4!iVhgm@LfRt%kAPSx8kVXdQMS=Dm%zjowSDvt=iq`#D;alYLkMw zoaDrjH}sqxD=6$p)lhQ*&HNGL2!N9gO*H;)=Sxq9?&keJ*0kpY8uJ?Y4}Z zM~k;-7)S7C-tlj~p@!}Jy>=;$6SSjiQ+x;jK2pPUcqxDBuZ7L=6;&L~NXMCZ;~mhy z>dJT#p(BlrwC}ES-nK3+QroFYACW5=9n??=FROOkqphxIgMN?3UMQcJNU`eJtxoc+ z((r^`2^{AsA}Sj=TY?LlV&9d0*r?EpsM9krXiGc;nC86ENi)2nBn?gw3`J0)gdRB1 w42xl@a2R~LpWM+jTy+(9|2C8F@8sJIEu%*V%+m?QUrYV^wsGU{We;1`f3n3uO8@`> literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/system_config_dto_test.dart b/mobile/openapi/test/system_config_dto_test.dart index 75e604539736284125a4d352a1409cb5dee90479..c8b5c0d9c1bc8885a6db747abf33606c49b558df 100644 GIT binary patch delta 49 tcmZqU|INQ)3zM)QgNQKd^szCsR!89&*8U6BLMDPXhQyoKo!3jogO5>WsE delta 12 Tcmey(-^RaT3)5yz=1VL9B7_9H diff --git a/mobile/openapi/test/system_config_library_dto_test.dart b/mobile/openapi/test/system_config_library_dto_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..f7051c82eda2c8bbd86daaddb09eb57a8f478574 GIT binary patch literal 602 zcmZ`#%}>HG6u}y!BeSQHyXFLww-1%{O`UNIXMn}AIb0Q zHI7Lf!}2xHuAkCv`m)T^1Xk-!I)NmEZI;7xmaJEAR|xaShmsk0Kbzjph+jo5mEp8f zhE}5CbLeDec}aQ0TQ1$?yo**V;|3Y5`DVSPx^vX<&rTSmOV(o5%wTuiSQIyQm|iGp z9g~iV5)DNqxCwMSXx0eHEp3#n5#|m|(z_d49GJ0$oH9bBS@FSYK3AGcrbUcz>aUI# z{~(TA7yEJscvHVyz%PMaZyj$Rm8?X)5$}eXK5>eCDN#bpgdC0GhX5d_5|T9_!YKuZ z`wP_3ctMa*8`bGiqY39`!Cke1FI91K_3%z*v+`48C0?8! zpPXiMNgX>u(>gIN^>_v&j{zHlJr2lm^-|N|oy3C<+yAsvFn)|i@cT%jcD93_-irPs zO3-i#hwZiF5G_YTjRL79lw=PFVLdSh(>v}|7I&7-#}B67SqWLT?-3T>*57Q0XU-#A zPq}`_^VEy;za$Yu%;)1pFhl1$fUX4AQ;0ZxcHB~TOIxurt!v9gblDF_4Gflec?7>R z0QI%Rwg*b%)Cd=E*q2A=nVLFrqhBs&;2Yw_3Z5YevyMXCK`oBUUtTZ-a;#yH8rL+J zE)aJJUUUf(CwP1<=2S=-L1P literal 0 HcmV?d00001 diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index 49567f7f60..6f8d639e9c 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -8061,6 +8061,9 @@ "job": { "$ref": "#/components/schemas/SystemConfigJobDto" }, + "library": { + "$ref": "#/components/schemas/SystemConfigLibraryDto" + }, "machineLearning": { "$ref": "#/components/schemas/SystemConfigMachineLearningDto" }, @@ -8104,7 +8107,8 @@ "job", "thumbnail", "trash", - "theme" + "theme", + "library" ], "type": "object" }, @@ -8238,6 +8242,32 @@ ], "type": "object" }, + "SystemConfigLibraryDto": { + "properties": { + "scan": { + "$ref": "#/components/schemas/SystemConfigLibraryScanDto" + } + }, + "required": [ + "scan" + ], + "type": "object" + }, + "SystemConfigLibraryScanDto": { + "properties": { + "cronExpression": { + "type": "string" + }, + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled", + "cronExpression" + ], + "type": "object" + }, "SystemConfigMachineLearningDto": { "properties": { "classification": { diff --git a/server/package-lock.json b/server/package-lock.json index 0842da0921..7cebecaf88 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -1683,66 +1683,6 @@ "darwin" ] }, - "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz", - "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz", - "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz", - "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz", - "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz", - "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ] - }, "node_modules/@nestjs/bull-shared": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.0.1.tgz", @@ -6118,15 +6058,6 @@ "exiftool-vendored.pl": "12.67.0" } }, - "node_modules/exiftool-vendored.exe": { - "version": "12.67.0", - "resolved": "https://registry.npmjs.org/exiftool-vendored.exe/-/exiftool-vendored.exe-12.67.0.tgz", - "integrity": "sha512-wzgMDoL/VWH34l38g22cVUn43mVFtTSVj0HRjfjR46+4fGwpSvSueeYbwLCZ5NvBAVINCS5Rz9Rl2DVmqoIjsw==", - "optional": true, - "os": [ - "win32" - ] - }, "node_modules/exiftool-vendored.pl": { "version": "12.67.0", "resolved": "https://registry.npmjs.org/exiftool-vendored.pl/-/exiftool-vendored.pl-12.67.0.tgz", @@ -14300,36 +14231,6 @@ "integrity": "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==", "optional": true }, - "@msgpackr-extract/msgpackr-extract-darwin-x64": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.2.tgz", - "integrity": "sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==", - "optional": true - }, - "@msgpackr-extract/msgpackr-extract-linux-arm": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.2.tgz", - "integrity": "sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==", - "optional": true - }, - "@msgpackr-extract/msgpackr-extract-linux-arm64": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.2.tgz", - "integrity": "sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==", - "optional": true - }, - "@msgpackr-extract/msgpackr-extract-linux-x64": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.2.tgz", - "integrity": "sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==", - "optional": true - }, - "@msgpackr-extract/msgpackr-extract-win32-x64": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.2.tgz", - "integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==", - "optional": true - }, "@nestjs/bull-shared": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-10.0.1.tgz", @@ -16944,6 +16845,11 @@ "luxon": "^3.2.1" } }, + "cron-validator": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/cron-validator/-/cron-validator-1.3.1.tgz", + "integrity": "sha512-C1HsxuPCY/5opR55G5/WNzyEGDWFVG+6GLrA+fW/sCTcP6A6NTjUP2AK7B8n2PyFs90kDG2qzwm8LMheADku6A==" + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -17608,12 +17514,6 @@ } } }, - "exiftool-vendored.exe": { - "version": "12.67.0", - "resolved": "https://registry.npmjs.org/exiftool-vendored.exe/-/exiftool-vendored.exe-12.67.0.tgz", - "integrity": "sha512-wzgMDoL/VWH34l38g22cVUn43mVFtTSVj0HRjfjR46+4fGwpSvSueeYbwLCZ5NvBAVINCS5Rz9Rl2DVmqoIjsw==", - "optional": true - }, "exiftool-vendored.pl": { "version": "12.67.0", "resolved": "https://registry.npmjs.org/exiftool-vendored.pl/-/exiftool-vendored.pl-12.67.0.tgz", diff --git a/server/src/domain/domain.util.ts b/server/src/domain/domain.util.ts index 9b7ee75219..04ec4f430d 100644 --- a/server/src/domain/domain.util.ts +++ b/server/src/domain/domain.util.ts @@ -1,6 +1,7 @@ import { applyDecorators } from '@nestjs/common'; import { ApiProperty } from '@nestjs/swagger'; import { IsArray, IsNotEmpty, IsOptional, IsString, IsUUID, ValidateIf, ValidationOptions } from 'class-validator'; +import { CronJob } from 'cron'; import { basename, extname } from 'node:path'; import sanitize from 'sanitize-filename'; @@ -18,6 +19,16 @@ export function ValidateUUID({ optional, each }: Options = { optional: false, ea ); } +export function validateCronExpression(expression: string) { + try { + new CronJob(expression, () => {}); + } catch (error) { + return false; + } + + return true; +} + interface IValue { value?: string; } diff --git a/server/src/domain/job/job.service.spec.ts b/server/src/domain/job/job.service.spec.ts index dac22a3ec2..fa909d1ae8 100644 --- a/server/src/domain/job/job.service.spec.ts +++ b/server/src/domain/job/job.service.spec.ts @@ -61,7 +61,6 @@ describe(JobService.name, () => { [{ name: JobName.PERSON_CLEANUP }], [{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }], [{ name: JobName.CLEAN_OLD_AUDIT_LOGS }], - [{ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }], ]); }); }); diff --git a/server/src/domain/job/job.service.ts b/server/src/domain/job/job.service.ts index 7b65467af6..7ebffcc693 100644 --- a/server/src/domain/job/job.service.ts +++ b/server/src/domain/job/job.service.ts @@ -153,7 +153,6 @@ export class JobService { await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP }); await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }); await this.jobRepository.queue({ name: JobName.CLEAN_OLD_AUDIT_LOGS }); - await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }); } /** diff --git a/server/src/domain/library/library.service.spec.ts b/server/src/domain/library/library.service.spec.ts index b13675a357..3d7d68736f 100644 --- a/server/src/domain/library/library.service.spec.ts +++ b/server/src/domain/library/library.service.spec.ts @@ -1,4 +1,4 @@ -import { AssetType, LibraryType, UserEntity } from '@app/infra/entities'; +import { AssetType, LibraryType, SystemConfig, SystemConfigKey, UserEntity } from '@app/infra/entities'; import { BadRequestException } from '@nestjs/common'; import { @@ -12,6 +12,7 @@ import { newJobRepositoryMock, newLibraryRepositoryMock, newStorageRepositoryMock, + newSystemConfigRepositoryMock, newUserRepositoryMock, userStub, } from '@test'; @@ -23,8 +24,10 @@ import { IJobRepository, ILibraryRepository, IStorageRepository, + ISystemConfigRepository, IUserRepository, } from '../repositories'; +import { SystemConfigCore } from '../system-config/system-config.core'; import { LibraryService } from './library.service'; describe(LibraryService.name, () => { @@ -32,6 +35,7 @@ describe(LibraryService.name, () => { let accessMock: IAccessRepositoryMock; let assetMock: jest.Mocked; + let configMock: jest.Mocked; let cryptoMock: jest.Mocked; let userMock: jest.Mocked; let jobMock: jest.Mocked; @@ -40,6 +44,7 @@ describe(LibraryService.name, () => { beforeEach(() => { accessMock = newAccessRepositoryMock(); + configMock = newSystemConfigRepositoryMock(); libraryMock = newLibraryRepositoryMock(); userMock = newUserRepositoryMock(); assetMock = newAssetRepositoryMock(); @@ -55,13 +60,46 @@ describe(LibraryService.name, () => { accessMock.library.hasOwnerAccess.mockResolvedValue(true); - sut = new LibraryService(accessMock, assetMock, cryptoMock, jobMock, libraryMock, storageMock, userMock); + sut = new LibraryService( + accessMock, + assetMock, + configMock, + cryptoMock, + jobMock, + libraryMock, + storageMock, + userMock, + ); }); it('should work', () => { expect(sut).toBeDefined(); }); + describe('init', () => { + it('should init cron job and subscribe to config changes', async () => { + configMock.load.mockResolvedValue([ + { key: SystemConfigKey.LIBRARY_SCAN_ENABLED, value: true }, + { key: SystemConfigKey.LIBRARY_SCAN_CRON_EXPRESSION, value: '0 0 * * *' }, + ]); + + await sut.init(); + expect(configMock.load).toHaveBeenCalled(); + expect(jobMock.addCronJob).toHaveBeenCalled(); + + SystemConfigCore.create(newSystemConfigRepositoryMock(false)).config$.next({ + library: { + scan: { + enabled: true, + cronExpression: '0 1 * * *', + }, + }, + } as SystemConfig); + + expect(jobMock.updateCronJob).toHaveBeenCalledWith('libraryScan', '0 1 * * *', true); + }); + }); + describe('handleQueueAssetRefresh', () => { it("should not queue assets outside of user's external path", async () => { const mockLibraryJob: ILibraryRefreshJob = { diff --git a/server/src/domain/library/library.service.ts b/server/src/domain/library/library.service.ts index 4943fc2004..6bec17c6b0 100644 --- a/server/src/domain/library/library.service.ts +++ b/server/src/domain/library/library.service.ts @@ -7,7 +7,7 @@ import { basename, parse } from 'path'; import { AccessCore, Permission } from '../access'; import { AuthUserDto } from '../auth'; import { mimeTypes } from '../domain.constant'; -import { usePagination } from '../domain.util'; +import { usePagination, validateCronExpression } from '../domain.util'; import { IBaseJob, IEntityJob, ILibraryFileJob, ILibraryRefreshJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job'; import { @@ -17,9 +17,11 @@ import { IJobRepository, ILibraryRepository, IStorageRepository, + ISystemConfigRepository, IUserRepository, WithProperty, } from '../repositories'; +import { SystemConfigCore } from '../system-config'; import { CreateLibraryDto, LibraryResponseDto, @@ -33,10 +35,12 @@ import { export class LibraryService { readonly logger = new Logger(LibraryService.name); private access: AccessCore; + private configCore: SystemConfigCore; constructor( @Inject(IAccessRepository) accessRepository: IAccessRepository, @Inject(IAssetRepository) private assetRepository: IAssetRepository, + @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(ILibraryRepository) private repository: ILibraryRepository, @@ -44,6 +48,26 @@ export class LibraryService { @Inject(IUserRepository) private userRepository: IUserRepository, ) { this.access = AccessCore.create(accessRepository); + this.configCore = SystemConfigCore.create(configRepository); + this.configCore.addValidator((config) => { + if (!validateCronExpression(config.library.scan.cronExpression)) { + throw new Error(`Invalid cron expression ${config.library.scan.cronExpression}`); + } + }); + } + + async init() { + const config = await this.configCore.getConfig(); + this.jobRepository.addCronJob( + 'libraryScan', + config.library.scan.cronExpression, + () => this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }), + config.library.scan.enabled, + ); + + this.configCore.config$.subscribe((config) => { + this.jobRepository.updateCronJob('libraryScan', config.library.scan.cronExpression, config.library.scan.enabled); + }); } async getStatistics(authUser: AuthUserDto, id: string): Promise { diff --git a/server/src/domain/repositories/job.repository.ts b/server/src/domain/repositories/job.repository.ts index 3527c9ea62..4b426062f2 100644 --- a/server/src/domain/repositories/job.repository.ts +++ b/server/src/domain/repositories/job.repository.ts @@ -111,6 +111,9 @@ export const IJobRepository = 'IJobRepository'; export interface IJobRepository { addHandler(queueName: QueueName, concurrency: number, handler: JobItemHandler): void; + addCronJob(name: string, expression: string, onTick: () => void, start?: boolean): void; + updateCronJob(name: string, expression?: string, start?: boolean): void; + deleteCronJob(name: string): void; setConcurrency(queueName: QueueName, concurrency: number): void; queue(item: JobItem): Promise; pause(name: QueueName): Promise; diff --git a/server/src/domain/system-config/dto/index.ts b/server/src/domain/system-config/dto/index.ts index 4a94b4cc8b..652e34cc50 100644 --- a/server/src/domain/system-config/dto/index.ts +++ b/server/src/domain/system-config/dto/index.ts @@ -1,4 +1,5 @@ export * from './system-config-ffmpeg.dto'; +export * from './system-config-library.dto'; export * from './system-config-oauth.dto'; export * from './system-config-password-login.dto'; export * from './system-config-storage-template.dto'; diff --git a/server/src/domain/system-config/dto/system-config-library.dto.ts b/server/src/domain/system-config/dto/system-config-library.dto.ts new file mode 100644 index 0000000000..2280e70938 --- /dev/null +++ b/server/src/domain/system-config/dto/system-config-library.dto.ts @@ -0,0 +1,40 @@ +import { validateCronExpression } from '@app/domain'; +import { Type } from 'class-transformer'; +import { + IsBoolean, + IsNotEmpty, + IsObject, + IsString, + Validate, + ValidateIf, + ValidateNested, + ValidatorConstraint, + ValidatorConstraintInterface, +} from 'class-validator'; + +const isEnabled = (config: SystemConfigLibraryScanDto) => config.enabled; + +@ValidatorConstraint({ name: 'cronValidator' }) +class CronValidator implements ValidatorConstraintInterface { + validate(expression: string): boolean { + return validateCronExpression(expression); + } +} + +export class SystemConfigLibraryScanDto { + @IsBoolean() + enabled!: boolean; + + @ValidateIf(isEnabled) + @IsNotEmpty() + @Validate(CronValidator, { message: 'Invalid cron expression' }) + @IsString() + cronExpression!: string; +} + +export class SystemConfigLibraryDto { + @Type(() => SystemConfigLibraryScanDto) + @ValidateNested() + @IsObject() + scan!: SystemConfigLibraryScanDto; +} diff --git a/server/src/domain/system-config/dto/system-config.dto.ts b/server/src/domain/system-config/dto/system-config.dto.ts index 975f5df893..dbd45855ca 100644 --- a/server/src/domain/system-config/dto/system-config.dto.ts +++ b/server/src/domain/system-config/dto/system-config.dto.ts @@ -3,6 +3,7 @@ import { Type } from 'class-transformer'; import { IsObject, ValidateNested } from 'class-validator'; import { SystemConfigFFmpegDto } from './system-config-ffmpeg.dto'; import { SystemConfigJobDto } from './system-config-job.dto'; +import { SystemConfigLibraryDto } from './system-config-library.dto'; import { SystemConfigMachineLearningDto } from './system-config-machine-learning.dto'; import { SystemConfigMapDto } from './system-config-map.dto'; import { SystemConfigNewVersionCheckDto } from './system-config-new-version-check.dto'; @@ -74,6 +75,11 @@ export class SystemConfigDto implements SystemConfig { @ValidateNested() @IsObject() theme!: SystemConfigThemeDto; + + @Type(() => SystemConfigLibraryDto) + @ValidateNested() + @IsObject() + library!: SystemConfigLibraryDto; } export function mapConfig(config: SystemConfig): SystemConfigDto { diff --git a/server/src/domain/system-config/system-config.core.ts b/server/src/domain/system-config/system-config.core.ts index df4ef374b1..4596370a4f 100644 --- a/server/src/domain/system-config/system-config.core.ts +++ b/server/src/domain/system-config/system-config.core.ts @@ -13,6 +13,7 @@ import { VideoCodec, } from '@app/infra/entities'; import { BadRequestException, ForbiddenException, Injectable, Logger } from '@nestjs/common'; +import { CronExpression } from '@nestjs/schedule'; import { plainToInstance } from 'class-transformer'; import { validate } from 'class-validator'; import * as _ from 'lodash'; @@ -120,6 +121,12 @@ export const defaults = Object.freeze({ theme: { customCss: '', }, + library: { + scan: { + enabled: true, + cronExpression: CronExpression.EVERY_DAY_AT_MIDNIGHT, + }, + }, }); export enum FeatureFlag { diff --git a/server/src/domain/system-config/system-config.service.spec.ts b/server/src/domain/system-config/system-config.service.spec.ts index c3808a8cdf..29ed44e915 100644 --- a/server/src/domain/system-config/system-config.service.spec.ts +++ b/server/src/domain/system-config/system-config.service.spec.ts @@ -121,6 +121,12 @@ const updatedConfig = Object.freeze({ theme: { customCss: '', }, + library: { + scan: { + enabled: true, + cronExpression: '0 0 * * *', + }, + }, }); describe(SystemConfigService.name, () => { diff --git a/server/src/immich/app.service.ts b/server/src/immich/app.service.ts index 110380753e..ef9975d8c4 100644 --- a/server/src/immich/app.service.ts +++ b/server/src/immich/app.service.ts @@ -1,4 +1,4 @@ -import { JobService, ONE_HOUR, SearchService, ServerInfoService, StorageService } from '@app/domain'; +import { JobService, LibraryService, ONE_HOUR, SearchService, ServerInfoService, StorageService } from '@app/domain'; import { Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression, Interval } from '@nestjs/schedule'; @@ -8,6 +8,7 @@ export class AppService { constructor( private jobService: JobService, + private libraryService: LibraryService, private searchService: SearchService, private storageService: StorageService, private serverService: ServerInfoService, @@ -28,6 +29,7 @@ export class AppService { await this.searchService.init(); await this.serverService.handleVersionCheck(); this.logger.log(`Feature Flags: ${JSON.stringify(await this.serverService.getFeatures(), null, 2)}`); + await this.libraryService.init(); } async destroy() { diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index de31bad32e..b71a44c0a2 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -94,6 +94,9 @@ export enum SystemConfigKey { TRASH_DAYS = 'trash.days', THEME_CUSTOM_CSS = 'theme.customCss', + + LIBRARY_SCAN_ENABLED = 'library.scan.enabled', + LIBRARY_SCAN_CRON_EXPRESSION = 'library.scan.cronExpression', } export enum TranscodePolicy { @@ -232,4 +235,10 @@ export interface SystemConfig { theme: { customCss: string; }; + library: { + scan: { + enabled: boolean; + cronExpression: string; + }; + }; } diff --git a/server/src/infra/repositories/job.repository.ts b/server/src/infra/repositories/job.repository.ts index d34ee6819a..067ba9bbf0 100644 --- a/server/src/infra/repositories/job.repository.ts +++ b/server/src/infra/repositories/job.repository.ts @@ -2,7 +2,9 @@ import { IJobRepository, JobCounts, JobItem, JobName, JOBS_TO_QUEUE, QueueName, import { getQueueToken } from '@nestjs/bullmq'; import { Injectable, Logger } from '@nestjs/common'; import { ModuleRef } from '@nestjs/core'; +import { SchedulerRegistry } from '@nestjs/schedule'; import { Job, JobsOptions, Processor, Queue, Worker, WorkerOptions } from 'bullmq'; +import { CronJob, CronTime } from 'cron'; import { bullConfig } from '../infra.config'; @Injectable() @@ -10,7 +12,10 @@ export class JobRepository implements IJobRepository { private workers: Partial> = {}; private logger = new Logger(JobRepository.name); - constructor(private moduleRef: ModuleRef) {} + constructor( + private moduleRef: ModuleRef, + private schedulerReqistry: SchedulerRegistry, + ) {} addHandler(queueName: QueueName, concurrency: number, handler: (item: JobItem) => Promise) { const workerHandler: Processor = async (job: Job) => handler(job as JobItem); @@ -18,6 +23,43 @@ export class JobRepository implements IJobRepository { this.workers[queueName] = new Worker(queueName, workerHandler, workerOptions); } + addCronJob(name: string, expression: string, onTick: () => void, start = true): void { + const job = new CronJob( + expression, + onTick, + // function to run onComplete + undefined, + // whether it should start directly + start, + // timezone + undefined, + // context + undefined, + // runOnInit + undefined, + // utcOffset + undefined, + // prevents memory leaking by automatically stopping when the node process finishes + true, + ); + + this.schedulerReqistry.addCronJob(name, job); + } + + updateCronJob(name: string, expression?: string, start?: boolean): void { + const job = this.schedulerReqistry.getCronJob(name); + if (expression) { + job.setTime(new CronTime(expression)); + } + if (start !== undefined) { + start ? job.start() : job.stop(); + } + } + + deleteCronJob(name: string): void { + this.schedulerReqistry.deleteCronJob(name); + } + setConcurrency(queueName: QueueName, concurrency: number) { const worker = this.workers[queueName]; if (!worker) { diff --git a/server/test/repositories/job.repository.mock.ts b/server/test/repositories/job.repository.mock.ts index 16db4ca692..fe794d1dc5 100644 --- a/server/test/repositories/job.repository.mock.ts +++ b/server/test/repositories/job.repository.mock.ts @@ -3,6 +3,9 @@ import { IJobRepository } from '@app/domain'; export const newJobRepositoryMock = (): jest.Mocked => { return { addHandler: jest.fn(), + addCronJob: jest.fn(), + deleteCronJob: jest.fn(), + updateCronJob: jest.fn(), setConcurrency: jest.fn(), empty: jest.fn(), pause: jest.fn(), diff --git a/server/test/test-utils.ts b/server/test/test-utils.ts index 6b45c6ee6c..4ac0cf0bfa 100644 --- a/server/test/test-utils.ts +++ b/server/test/test-utils.ts @@ -49,6 +49,10 @@ export const testApp = { .overrideProvider(IJobRepository) .useValue({ addHandler: (_queueName: QueueName, _concurrency: number, handler: JobItemHandler) => (_handler = handler), + addCronJob: jest.fn(), + updateCronJob: jest.fn(), + deleteCronJob: jest.fn(), + validateCronExpression: jest.fn(), queue: (item: JobItem) => jobs && _handler(item), resume: jest.fn(), empty: jest.fn(), diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index e79388602d..97dc8523c3 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -3283,6 +3283,12 @@ export interface SystemConfigDto { * @memberof SystemConfigDto */ 'job': SystemConfigJobDto; + /** + * + * @type {SystemConfigLibraryDto} + * @memberof SystemConfigDto + */ + 'library': SystemConfigLibraryDto; /** * * @type {SystemConfigMachineLearningDto} @@ -3534,6 +3540,38 @@ export interface SystemConfigJobDto { */ 'videoConversion': JobSettingsDto; } +/** + * + * @export + * @interface SystemConfigLibraryDto + */ +export interface SystemConfigLibraryDto { + /** + * + * @type {SystemConfigLibraryScanDto} + * @memberof SystemConfigLibraryDto + */ + 'scan': SystemConfigLibraryScanDto; +} +/** + * + * @export + * @interface SystemConfigLibraryScanDto + */ +export interface SystemConfigLibraryScanDto { + /** + * + * @type {string} + * @memberof SystemConfigLibraryScanDto + */ + 'cronExpression': string; + /** + * + * @type {boolean} + * @memberof SystemConfigLibraryScanDto + */ + 'enabled': boolean; +} /** * * @export diff --git a/web/src/lib/components/admin-page/settings/library-settings/library-settings.svelte b/web/src/lib/components/admin-page/settings/library-settings/library-settings.svelte new file mode 100644 index 0000000000..2330507e0e --- /dev/null +++ b/web/src/lib/components/admin-page/settings/library-settings/library-settings.svelte @@ -0,0 +1,145 @@ + + +
+ {#await getConfigs() then} +
+ +
+
+ + +
+ + +
+ + + +

+ Set the scanning interval using the cron format. For more information please refer to e.g. Crontab Guru +

+ + +
+ +
+ +
+
+ +
+ {/await} +
diff --git a/web/src/routes/admin/system-settings/+page.svelte b/web/src/routes/admin/system-settings/+page.svelte index a86c912739..8dd4954b0c 100644 --- a/web/src/routes/admin/system-settings/+page.svelte +++ b/web/src/routes/admin/system-settings/+page.svelte @@ -20,6 +20,7 @@ import Icon from '$lib/components/elements/icon.svelte'; import type { PageData } from './$types'; import NewVersionCheckSettings from '$lib/components/admin-page/settings/new-version-check-settings/new-version-check-settings.svelte'; + import LibrarySettings from '$lib/components/admin-page/settings/library-settings/library-settings.svelte'; import { mdiAlert, mdiContentCopy, mdiDownload } from '@mdi/js'; export let data: PageData; @@ -69,6 +70,10 @@ + + + +