From de69d0031e881642289cc5876bd3873858d49173 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 19 Dec 2022 12:13:10 -0600 Subject: [PATCH] chore(server) Add job for storage migration (#1117) --- mobile/openapi/doc/AllJobStatusResponseDto.md | Bin 855 -> 970 bytes .../model/all_job_status_response_dto.dart | Bin 6910 -> 7801 bytes mobile/openapi/lib/model/job_id.dart | Bin 2870 -> 3065 bytes .../all_job_status_response_dto_test.dart | Bin 1588 -> 1859 bytes .../src/api-v1/asset/asset-repository.ts | 17 ++-- .../immich/src/api-v1/asset/asset.module.ts | 19 +--- .../immich/src/api-v1/job/dto/get-job.dto.ts | 1 + .../apps/immich/src/api-v1/job/job.module.ts | 58 ++----------- .../apps/immich/src/api-v1/job/job.service.ts | 34 ++++++++ .../all-job-status-response.dto.ts | 6 ++ .../system-config/system-config.module.ts | 9 +- .../system-config/system-config.service.ts | 11 ++- .../immich/src/api-v1/user/user.service.ts | 5 +- server/apps/immich/src/app.module.ts | 15 +--- .../background-task/background-task.module.ts | 5 -- .../schedule-tasks/schedule-tasks.module.ts | 36 +------- .../microservices/src/microservices.module.ts | 82 ++---------------- .../processors/storage-migration.processor.ts | 61 +++++++++++++ .../src/processors/thumbnail.processor.ts | 27 ++---- server/immich-openapi-specs.json | 13 ++- .../common/src/config/bull-queue.config.ts | 19 ++++ server/libs/common/src/config/index.ts | 1 + .../src/immich-config.service.ts | 6 ++ .../bull-queue-registration.constant.ts | 32 +++++++ .../job/src/constants/job-name.constant.ts | 6 ++ .../job/src/constants/queue-name.constant.ts | 1 + server/libs/storage/src/storage.service.ts | 63 ++++++++++++-- web/src/api/open-api/api.ts | 15 +++- .../admin-page/jobs/jobs-panel.svelte | 44 ++++++++++ .../settings/setting-accordion.svelte | 2 +- .../storage-template-settings.svelte | 10 +++ web/src/routes/admin/+layout.svelte | 2 +- .../routes/admin/system-settings/+page.svelte | 2 + 33 files changed, 364 insertions(+), 238 deletions(-) create mode 100644 server/apps/microservices/src/processors/storage-migration.processor.ts create mode 100644 server/libs/common/src/config/bull-queue.config.ts create mode 100644 server/libs/job/src/constants/bull-queue-registration.constant.ts diff --git a/mobile/openapi/doc/AllJobStatusResponseDto.md b/mobile/openapi/doc/AllJobStatusResponseDto.md index 3fa53791dfe7bc92a7e1a12e77c33be79917ae3b..606375b8a6826886de39aa742b34f812637c3aed 100644 GIT binary patch delta 48 xcmcc4c8Yz&Pe!idlKi5?^i^oj4SC;w#J$)}~2SsV-(oy@_cFnJTxQULH06HfpD delta 15 XcmX@bew}T@PsYi1Ogks5GA{)HHYNr| diff --git a/mobile/openapi/lib/model/all_job_status_response_dto.dart b/mobile/openapi/lib/model/all_job_status_response_dto.dart index 7be7166a77ee9491c708eb93d37ebb66c040ae62..88865d79931fe48dcf00476e5121ec8d5aaef91e 100644 GIT binary patch delta 616 zcmexo`qO4Z5))T(Nq$jcda7?``s6K)DwC6#7Eiv#BqR_F7cNRH$;{82{F`aVWE(H+CFUd$P(!(^`F}WnOEEQw`rrhQ$Y+Q`U9CSWBk15m delta 82 zcmV-Y0ImP|JpMJXVgi$p0;7{(1Fn-c1dFrd1epP|!v+HZvk3?U0h2iiqq7DIB>|IT o3)Hh~42A)-It~y4qmK^-v+Wdg0kgdp^ahjD9e@UXI|_XY3j1Rm`v3p{ diff --git a/mobile/openapi/lib/model/job_id.dart b/mobile/openapi/lib/model/job_id.dart index 308d9c06c1af60051f9501fa8ccc6bb5fd28e946..19919f6d72d93c192afe59a345e900cefb362a32 100644 GIT binary patch delta 187 zcmdlc_EUU=JhP&LLUBoANoKM_a(-TM36PUtl$f3xlA2qPlUS1Ko0&e@kV#kyDy&-q z71zzpOfLdz%g>uE&AdknWC*%Gi1g;m%o%K+$%)0O3PtL;O;NW}C`v6UEy`2y%1`o4 M(Zgw+Ar~tn0DY51X8-^I delta 20 ccmew~Jj>*2CN8exlKi5?^i^hr#HNW95*OtzCZu(EGfV_{*M+{erg;_w8+ XRTm{r-piuPg%FsW&tf!Li;W8au4^8P delta 27 jcmX@iw}oeeJj>+0Z0wsiv9K_4aVaR&YFcw~)p7v;^F diff --git a/server/apps/immich/src/api-v1/asset/asset-repository.ts b/server/apps/immich/src/api-v1/asset/asset-repository.ts index bd8d1ab6ce..cc2cc0937a 100644 --- a/server/apps/immich/src/api-v1/asset/asset-repository.ts +++ b/server/apps/immich/src/api-v1/asset/asset-repository.ts @@ -15,6 +15,7 @@ import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-as import { In } from 'typeorm/find-options/operator/In'; import { UpdateAssetDto } from './dto/update-asset.dto'; import { ITagRepository, TAG_REPOSITORY } from '../tag/tag.repository'; +import { IsNull } from 'typeorm'; export interface IAssetRepository { create( @@ -69,14 +70,14 @@ export class AssetRepository implements IAssetRepository { } async getAssetWithNoThumbnail(): Promise { - return await this.assetRepository - .createQueryBuilder('asset') - .where('asset.resizePath IS NULL') - .andWhere('asset.isVisible = true') - .orWhere('asset.resizePath = :resizePath', { resizePath: '' }) - .orWhere('asset.webpPath IS NULL') - .orWhere('asset.webpPath = :webpPath', { webpPath: '' }) - .getMany(); + return await this.assetRepository.find({ + where: [ + { resizePath: IsNull(), isVisible: true }, + { resizePath: '', isVisible: true }, + { webpPath: IsNull(), isVisible: true }, + { webpPath: '', isVisible: true }, + ], + }); } async getAssetWithNoEXIF(): Promise { diff --git a/server/apps/immich/src/api-v1/asset/asset.module.ts b/server/apps/immich/src/api-v1/asset/asset.module.ts index bdd489223d..b0d6d1775d 100644 --- a/server/apps/immich/src/api-v1/asset/asset.module.ts +++ b/server/apps/immich/src/api-v1/asset/asset.module.ts @@ -7,13 +7,13 @@ import { BullModule } from '@nestjs/bull'; import { BackgroundTaskModule } from '../../modules/background-task/background-task.module'; import { BackgroundTaskService } from '../../modules/background-task/background-task.service'; import { CommunicationModule } from '../communication/communication.module'; -import { QueueNameEnum } from '@app/job/constants/queue-name.constant'; import { AssetRepository, ASSET_REPOSITORY } from './asset-repository'; import { DownloadModule } from '../../modules/download/download.module'; import { TagModule } from '../tag/tag.module'; import { AlbumModule } from '../album/album.module'; import { UserModule } from '../user/user.module'; import { StorageModule } from '@app/storage'; +import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant'; const ASSET_REPOSITORY_PROVIDER = { provide: ASSET_REPOSITORY, @@ -31,22 +31,7 @@ const ASSET_REPOSITORY_PROVIDER = { TagModule, StorageModule, forwardRef(() => AlbumModule), - BullModule.registerQueue({ - name: QueueNameEnum.ASSET_UPLOADED, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }), - BullModule.registerQueue({ - name: QueueNameEnum.VIDEO_CONVERSION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }), + BullModule.registerQueue(...immichSharedQueues), ], controllers: [AssetController], providers: [AssetService, BackgroundTaskService, ASSET_REPOSITORY_PROVIDER], diff --git a/server/apps/immich/src/api-v1/job/dto/get-job.dto.ts b/server/apps/immich/src/api-v1/job/dto/get-job.dto.ts index 8280c66940..f67b48e0fd 100644 --- a/server/apps/immich/src/api-v1/job/dto/get-job.dto.ts +++ b/server/apps/immich/src/api-v1/job/dto/get-job.dto.ts @@ -6,6 +6,7 @@ export enum JobId { METADATA_EXTRACTION = 'metadata-extraction', VIDEO_CONVERSION = 'video-conversion', MACHINE_LEARNING = 'machine-learning', + STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration', } export class GetJobDto { diff --git a/server/apps/immich/src/api-v1/job/job.module.ts b/server/apps/immich/src/api-v1/job/job.module.ts index 09ac3885ad..41a0b9d7d3 100644 --- a/server/apps/immich/src/api-v1/job/job.module.ts +++ b/server/apps/immich/src/api-v1/job/job.module.ts @@ -6,13 +6,15 @@ import { ImmichJwtModule } from '../../modules/immich-jwt/immich-jwt.module'; import { JwtModule } from '@nestjs/jwt'; import { jwtConfig } from '../../config/jwt.config'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { BullModule } from '@nestjs/bull'; -import { QueueNameEnum } from '@app/job'; import { ExifEntity } from '@app/database/entities/exif.entity'; import { TagModule } from '../tag/tag.module'; import { AssetModule } from '../asset/asset.module'; import { UserModule } from '../user/user.module'; +import { StorageModule } from '@app/storage'; +import { BullModule } from '@nestjs/bull'; +import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant'; + @Module({ imports: [ TypeOrmModule.forFeature([ExifEntity]), @@ -21,56 +23,8 @@ import { UserModule } from '../user/user.module'; AssetModule, UserModule, JwtModule.register(jwtConfig), - BullModule.registerQueue( - { - name: QueueNameEnum.THUMBNAIL_GENERATION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - { - name: QueueNameEnum.ASSET_UPLOADED, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - { - name: QueueNameEnum.METADATA_EXTRACTION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - { - name: QueueNameEnum.VIDEO_CONVERSION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - { - name: QueueNameEnum.CHECKSUM_GENERATION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - { - name: QueueNameEnum.MACHINE_LEARNING, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - ), + StorageModule, + BullModule.registerQueue(...immichSharedQueues), ], controllers: [JobController], providers: [JobService, ImmichJwtService], diff --git a/server/apps/immich/src/api-v1/job/job.service.ts b/server/apps/immich/src/api-v1/job/job.service.ts index 761a70906f..7b61d34a9e 100644 --- a/server/apps/immich/src/api-v1/job/job.service.ts +++ b/server/apps/immich/src/api-v1/job/job.service.ts @@ -6,6 +6,7 @@ import { IVideoTranscodeJob, MachineLearningJobNameEnum, QueueNameEnum, + templateMigrationProcessorName, videoMetadataExtractionProcessorName, } from '@app/job'; import { InjectQueue } from '@nestjs/bull'; @@ -18,6 +19,7 @@ import { AssetType } from '@app/database/entities/asset.entity'; import { GetJobDto, JobId } from './dto/get-job.dto'; import { JobStatusResponseDto } from './response-dto/job-status-response.dto'; import { IMachineLearningJob } from '@app/job/interfaces/machine-learning.interface'; +import { StorageService } from '@app/storage'; @Injectable() export class JobService { @@ -34,12 +36,18 @@ export class JobService { @InjectQueue(QueueNameEnum.MACHINE_LEARNING) private machineLearningQueue: Queue, + @InjectQueue(QueueNameEnum.STORAGE_MIGRATION) + private storageMigrationQueue: Queue, + @Inject(ASSET_REPOSITORY) private _assetRepository: IAssetRepository, + + private storageService: StorageService, ) { this.thumbnailGeneratorQueue.empty(); this.metadataExtractionQueue.empty(); this.videoConversionQueue.empty(); + this.storageMigrationQueue.empty(); } async startJob(jobDto: GetJobDto): Promise { @@ -52,6 +60,8 @@ export class JobService { return 0; case JobId.MACHINE_LEARNING: return this.runMachineLearningPipeline(); + case JobId.STORAGE_TEMPLATE_MIGRATION: + return this.runStorageMigration(); default: throw new BadRequestException('Invalid job id'); } @@ -62,6 +72,7 @@ export class JobService { const metadataExtractionJobCount = await this.metadataExtractionQueue.getJobCounts(); const videoConversionJobCount = await this.videoConversionQueue.getJobCounts(); const machineLearningJobCount = await this.machineLearningQueue.getJobCounts(); + const storageMigrationJobCount = await this.storageMigrationQueue.getJobCounts(); const response = new AllJobStatusResponseDto(); response.isThumbnailGenerationActive = Boolean(thumbnailGeneratorJobCount.waiting); @@ -73,6 +84,9 @@ export class JobService { response.isMachineLearningActive = Boolean(machineLearningJobCount.waiting); response.machineLearningQueueCount = machineLearningJobCount; + response.isStorageMigrationActive = Boolean(storageMigrationJobCount.active); + response.storageMigrationQueueCount = storageMigrationJobCount; + return response; } @@ -93,6 +107,11 @@ export class JobService { response.queueCount = await this.videoConversionQueue.getJobCounts(); } + if (query.jobId === JobId.STORAGE_TEMPLATE_MIGRATION) { + response.isActive = Boolean((await this.storageMigrationQueue.getJobCounts()).waiting); + response.queueCount = await this.storageMigrationQueue.getJobCounts(); + } + return response; } @@ -110,6 +129,9 @@ export class JobService { case JobId.MACHINE_LEARNING: this.machineLearningQueue.empty(); return 0; + case JobId.STORAGE_TEMPLATE_MIGRATION: + this.storageMigrationQueue.empty(); + return 0; default: throw new BadRequestException('Invalid job id'); } @@ -177,4 +199,16 @@ export class JobService { return assetWithNoSmartInfo.length; } + + async runStorageMigration() { + const jobCount = await this.storageMigrationQueue.getJobCounts(); + + if (jobCount.active > 0) { + throw new BadRequestException('Storage migration job is already running'); + } + + await this.storageMigrationQueue.add(templateMigrationProcessorName, {}, { jobId: randomUUID() }); + + return 1; + } } diff --git a/server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts b/server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts index 884982bb5e..06cbbdf2a5 100644 --- a/server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts +++ b/server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts @@ -17,6 +17,7 @@ export class AllJobStatusResponseDto { isMetadataExtractionActive!: boolean; isVideoConversionActive!: boolean; isMachineLearningActive!: boolean; + isStorageMigrationActive!: boolean; @ApiProperty({ type: JobCounts, @@ -37,4 +38,9 @@ export class AllJobStatusResponseDto { type: JobCounts, }) machineLearningQueueCount!: JobCounts; + + @ApiProperty({ + type: JobCounts, + }) + storageMigrationQueueCount!: JobCounts; } diff --git a/server/apps/immich/src/api-v1/system-config/system-config.module.ts b/server/apps/immich/src/api-v1/system-config/system-config.module.ts index ac54404031..3fd920156e 100644 --- a/server/apps/immich/src/api-v1/system-config/system-config.module.ts +++ b/server/apps/immich/src/api-v1/system-config/system-config.module.ts @@ -1,4 +1,6 @@ import { SystemConfigEntity } from '@app/database/entities/system-config.entity'; +import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant'; +import { BullModule } from '@nestjs/bull'; import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { ImmichConfigModule } from 'libs/immich-config/src'; @@ -7,7 +9,12 @@ import { SystemConfigController } from './system-config.controller'; import { SystemConfigService } from './system-config.service'; @Module({ - imports: [ImmichJwtModule, ImmichConfigModule, TypeOrmModule.forFeature([SystemConfigEntity])], + imports: [ + ImmichJwtModule, + ImmichConfigModule, + TypeOrmModule.forFeature([SystemConfigEntity]), + BullModule.registerQueue(...immichSharedQueues), + ], controllers: [SystemConfigController], providers: [SystemConfigService], }) diff --git a/server/apps/immich/src/api-v1/system-config/system-config.service.ts b/server/apps/immich/src/api-v1/system-config/system-config.service.ts index db02cb5f20..38fcf95938 100644 --- a/server/apps/immich/src/api-v1/system-config/system-config.service.ts +++ b/server/apps/immich/src/api-v1/system-config/system-config.service.ts @@ -1,3 +1,4 @@ +import { QueueNameEnum, updateTemplateProcessorName } from '@app/job'; import { supportedDayTokens, supportedHourTokens, @@ -7,14 +8,21 @@ import { supportedSecondTokens, supportedYearTokens, } from '@app/storage/constants/supported-datetime-template'; +import { InjectQueue } from '@nestjs/bull'; import { Injectable } from '@nestjs/common'; +import { Queue } from 'bull'; +import { randomUUID } from 'crypto'; import { ImmichConfigService } from 'libs/immich-config/src'; import { mapConfig, SystemConfigDto } from './dto/system-config.dto'; import { SystemConfigTemplateStorageOptionDto } from './response-dto/system-config-template-storage-option.dto'; @Injectable() export class SystemConfigService { - constructor(private immichConfigService: ImmichConfigService) {} + constructor( + private immichConfigService: ImmichConfigService, + @InjectQueue(QueueNameEnum.STORAGE_MIGRATION) + private storageMigrationQueue: Queue, + ) {} public async getConfig(): Promise { const config = await this.immichConfigService.getConfig(); @@ -28,6 +36,7 @@ export class SystemConfigService { public async updateConfig(dto: SystemConfigDto): Promise { const config = await this.immichConfigService.updateConfig(dto); + this.storageMigrationQueue.add(updateTemplateProcessorName, {}, { jobId: randomUUID() }); return mapConfig(config); } diff --git a/server/apps/immich/src/api-v1/user/user.service.ts b/server/apps/immich/src/api-v1/user/user.service.ts index 1b629c690f..d3ab094a21 100644 --- a/server/apps/immich/src/api-v1/user/user.service.ts +++ b/server/apps/immich/src/api-v1/user/user.service.ts @@ -10,7 +10,7 @@ import { UnauthorizedException, } from '@nestjs/common'; import { Response as Res } from 'express'; -import { createReadStream } from 'fs'; +import { constants, createReadStream } from 'fs'; import { AuthUserDto } from '../../decorators/auth-user.decorator'; import { CreateUserDto } from './dto/create-user.dto'; import { UpdateUserDto } from './dto/update-user.dto'; @@ -22,6 +22,7 @@ import { import { mapUserCountResponse, UserCountResponseDto } from './response-dto/user-count-response.dto'; import { mapUser, UserResponseDto } from './response-dto/user-response.dto'; import { IUserRepository, USER_REPOSITORY } from './user-repository'; +import fs from 'fs/promises'; @Injectable() export class UserService { @@ -196,6 +197,8 @@ export class UserService { throw new NotFoundException('User does not have a profile image'); } + await fs.access(user.profileImagePath, constants.R_OK | constants.W_OK); + res.set({ 'Content-Type': 'image/jpeg', }); diff --git a/server/apps/immich/src/app.module.ts b/server/apps/immich/src/app.module.ts index 25ed0ab358..504ddfbe17 100644 --- a/server/apps/immich/src/app.module.ts +++ b/server/apps/immich/src/app.module.ts @@ -1,4 +1,4 @@ -import { immichAppConfig } from '@app/common/config'; +import { immichAppConfig, immichBullAsyncConfig } from '@app/common/config'; import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common'; import { UserModule } from './api-v1/user/user.module'; import { AssetModule } from './api-v1/asset/asset.module'; @@ -36,18 +36,7 @@ import { TagModule } from './api-v1/tag/tag.module'; DeviceInfoModule, - BullModule.forRootAsync({ - useFactory: async () => ({ - prefix: 'immich_bull', - redis: { - host: process.env.REDIS_HOSTNAME || 'immich_redis', - port: parseInt(process.env.REDIS_PORT || '6379'), - db: parseInt(process.env.REDIS_DBINDEX || '0'), - password: process.env.REDIS_PASSWORD || undefined, - path: process.env.REDIS_SOCKET || undefined, - }, - }), - }), + BullModule.forRootAsync(immichBullAsyncConfig), ServerInfoModule, diff --git a/server/apps/immich/src/modules/background-task/background-task.module.ts b/server/apps/immich/src/modules/background-task/background-task.module.ts index 11e1e7a69e..6ebeead9f3 100644 --- a/server/apps/immich/src/modules/background-task/background-task.module.ts +++ b/server/apps/immich/src/modules/background-task/background-task.module.ts @@ -11,11 +11,6 @@ import { BackgroundTaskService } from './background-task.service'; imports: [ BullModule.registerQueue({ name: 'background-task', - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, }), TypeOrmModule.forFeature([AssetEntity, ExifEntity, SmartInfoEntity]), ], diff --git a/server/apps/immich/src/modules/schedule-tasks/schedule-tasks.module.ts b/server/apps/immich/src/modules/schedule-tasks/schedule-tasks.module.ts index 0e30e9ac66..4139276179 100644 --- a/server/apps/immich/src/modules/schedule-tasks/schedule-tasks.module.ts +++ b/server/apps/immich/src/modules/schedule-tasks/schedule-tasks.module.ts @@ -3,46 +3,14 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { AssetEntity } from '@app/database/entities/asset.entity'; import { ScheduleTasksService } from './schedule-tasks.service'; -import { QueueNameEnum } from '@app/job/constants/queue-name.constant'; import { ExifEntity } from '@app/database/entities/exif.entity'; import { UserEntity } from '@app/database/entities/user.entity'; +import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant'; @Module({ imports: [ TypeOrmModule.forFeature([AssetEntity, ExifEntity, UserEntity]), - BullModule.registerQueue({ - name: QueueNameEnum.USER_DELETION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }), - BullModule.registerQueue({ - name: QueueNameEnum.VIDEO_CONVERSION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }), - BullModule.registerQueue({ - name: QueueNameEnum.THUMBNAIL_GENERATION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }), - - BullModule.registerQueue({ - name: QueueNameEnum.METADATA_EXTRACTION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }), + BullModule.registerQueue(...immichSharedQueues), ], providers: [ScheduleTasksService], }) diff --git a/server/apps/microservices/src/microservices.module.ts b/server/apps/microservices/src/microservices.module.ts index 7fa81b6dc2..e57c7672a9 100644 --- a/server/apps/microservices/src/microservices.module.ts +++ b/server/apps/microservices/src/microservices.module.ts @@ -1,10 +1,10 @@ -import { immichAppConfig } from '@app/common/config'; +import { immichAppConfig, immichBullAsyncConfig } from '@app/common/config'; import { DatabaseModule } from '@app/database'; import { AssetEntity } from '@app/database/entities/asset.entity'; import { ExifEntity } from '@app/database/entities/exif.entity'; import { SmartInfoEntity } from '@app/database/entities/smart-info.entity'; import { UserEntity } from '@app/database/entities/user.entity'; -import { QueueNameEnum } from '@app/job/constants/queue-name.constant'; +import { StorageModule } from '@app/storage'; import { BullModule } from '@nestjs/bull'; import { Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; @@ -16,9 +16,11 @@ import { AssetUploadedProcessor } from './processors/asset-uploaded.processor'; import { GenerateChecksumProcessor } from './processors/generate-checksum.processor'; import { MachineLearningProcessor } from './processors/machine-learning.processor'; import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor'; +import { StorageMigrationProcessor } from './processors/storage-migration.processor'; import { ThumbnailGeneratorProcessor } from './processors/thumbnail.processor'; import { UserDeletionProcessor } from './processors/user-deletion.processor'; import { VideoTranscodeProcessor } from './processors/video-transcode.processor'; +import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant'; @Module({ imports: [ @@ -26,76 +28,9 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor' DatabaseModule, ImmichConfigModule, TypeOrmModule.forFeature([UserEntity, ExifEntity, AssetEntity, SmartInfoEntity]), - BullModule.forRootAsync({ - useFactory: async () => ({ - prefix: 'immich_bull', - redis: { - host: process.env.REDIS_HOSTNAME || 'immich_redis', - port: parseInt(process.env.REDIS_PORT || '6379'), - db: parseInt(process.env.REDIS_DBINDEX || '0'), - password: process.env.REDIS_PASSWORD || undefined, - path: process.env.REDIS_SOCKET || undefined, - }, - }), - }), - BullModule.registerQueue( - { - name: QueueNameEnum.USER_DELETION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - { - name: QueueNameEnum.THUMBNAIL_GENERATION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - { - name: QueueNameEnum.ASSET_UPLOADED, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - { - name: QueueNameEnum.METADATA_EXTRACTION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - { - name: QueueNameEnum.VIDEO_CONVERSION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - { - name: QueueNameEnum.CHECKSUM_GENERATION, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - { - name: QueueNameEnum.MACHINE_LEARNING, - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, - }, - ), + StorageModule, + BullModule.forRootAsync(immichBullAsyncConfig), + BullModule.registerQueue(...immichSharedQueues), CommunicationModule, ], controllers: [], @@ -108,7 +43,8 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor' GenerateChecksumProcessor, MachineLearningProcessor, UserDeletionProcessor, + StorageMigrationProcessor, ], - exports: [], + exports: [BullModule], }) export class MicroservicesModule {} diff --git a/server/apps/microservices/src/processors/storage-migration.processor.ts b/server/apps/microservices/src/processors/storage-migration.processor.ts new file mode 100644 index 0000000000..8c913776fa --- /dev/null +++ b/server/apps/microservices/src/processors/storage-migration.processor.ts @@ -0,0 +1,61 @@ +import { APP_UPLOAD_LOCATION } from '@app/common'; +import { AssetEntity } from '@app/database/entities/asset.entity'; +import { ImmichConfigService } from '@app/immich-config'; +import { QueueNameEnum, templateMigrationProcessorName, updateTemplateProcessorName } from '@app/job'; +import { StorageService } from '@app/storage'; +import { Process, Processor } from '@nestjs/bull'; +import { Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; + +@Processor(QueueNameEnum.STORAGE_MIGRATION) +export class StorageMigrationProcessor { + readonly logger: Logger = new Logger(StorageMigrationProcessor.name); + + constructor( + private storageService: StorageService, + private immichConfigService: ImmichConfigService, + + @InjectRepository(AssetEntity) + private assetRepository: Repository, + ) {} + + /** + * Migration process when a new user set a new storage template. + * @param job + */ + @Process({ name: templateMigrationProcessorName, concurrency: 100 }) + async templateMigration() { + console.time('migrating-time'); + const assets = await this.assetRepository.find({ + relations: ['exifInfo'], + }); + + const livePhotoMap: Record = {}; + + for (const asset of assets) { + if (asset.livePhotoVideoId) { + livePhotoMap[asset.livePhotoVideoId] = asset; + } + } + + for (const asset of assets) { + const livePhotoParentAsset = livePhotoMap[asset.id]; + const filename = asset.exifInfo?.imageName || livePhotoParentAsset?.exifInfo?.imageName || asset.id; + await this.storageService.moveAsset(asset, filename); + } + + await this.storageService.removeEmptyDirectories(APP_UPLOAD_LOCATION); + console.timeEnd('migrating-time'); + } + + /** + * Update config when a new storage template is set. + * This is to ensure the synchronization between processes. + * @param job + */ + @Process({ name: updateTemplateProcessorName, concurrency: 1 }) + async updateTemplate() { + await this.immichConfigService.refreshConfig(); + } +} diff --git a/server/apps/microservices/src/processors/thumbnail.processor.ts b/server/apps/microservices/src/processors/thumbnail.processor.ts index c01e3da2aa..3c9236736d 100644 --- a/server/apps/microservices/src/processors/thumbnail.processor.ts +++ b/server/apps/microservices/src/processors/thumbnail.processor.ts @@ -1,5 +1,4 @@ import { APP_UPLOAD_LOCATION } from '@app/common'; -import { ImmichLogLevel } from '@app/common/constants/log-level.constant'; import { AssetEntity, AssetType } from '@app/database/entities/asset.entity'; import { WebpGeneratorProcessor, @@ -11,7 +10,6 @@ import { } from '@app/job'; import { InjectQueue, Process, Processor } from '@nestjs/bull'; import { Logger } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { InjectRepository } from '@nestjs/typeorm'; import { mapAsset } from 'apps/immich/src/api-v1/asset/response-dto/asset-response.dto'; import { Job, Queue } from 'bull'; @@ -27,7 +25,7 @@ import { IMachineLearningJob } from '@app/job/interfaces/machine-learning.interf @Processor(QueueNameEnum.THUMBNAIL_GENERATION) export class ThumbnailGeneratorProcessor { - private logLevel: ImmichLogLevel; + readonly logger: Logger = new Logger(ThumbnailGeneratorProcessor.name); constructor( @InjectRepository(AssetEntity) @@ -40,12 +38,7 @@ export class ThumbnailGeneratorProcessor { @InjectQueue(QueueNameEnum.MACHINE_LEARNING) private machineLearningQueue: Queue, - - private configService: ConfigService, - ) { - this.logLevel = this.configService.get('LOG_LEVEL') || ImmichLogLevel.SIMPLE; - // TODO - Add observable paterrn to listen to the config change - } + ) {} @Process({ name: generateJPEGThumbnailProcessorName, concurrency: 3 }) async generateJPEGThumbnail(job: Job) { @@ -70,12 +63,8 @@ export class ThumbnailGeneratorProcessor { .rotate() .toFile(jpegThumbnailPath); await this.assetRepository.update({ id: asset.id }, { resizePath: jpegThumbnailPath }); - } catch (error) { - Logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id); - - if (this.logLevel == ImmichLogLevel.VERBOSE) { - console.trace('Failed to generate jpeg thumbnail for asset', error); - } + } catch (error: any) { + this.logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id, error.stack); } // Update resize path to send to generate webp queue @@ -140,12 +129,8 @@ export class ThumbnailGeneratorProcessor { try { await sharp(asset.resizePath, { failOnError: false }).resize(250).webp().rotate().toFile(webpPath); await this.assetRepository.update({ id: asset.id }, { webpPath: webpPath }); - } catch (error) { - Logger.error('Failed to generate webp thumbnail for asset: ' + asset.id); - - if (this.logLevel == ImmichLogLevel.VERBOSE) { - console.trace('Failed to generate webp thumbnail for asset', error); - } + } catch (error: any) { + this.logger.error('Failed to generate webp thumbnail for asset: ' + asset.id, error.stack); } } } diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index f306868cf6..94a6499837 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -3562,6 +3562,9 @@ "machineLearningQueueCount": { "$ref": "#/components/schemas/JobCounts" }, + "storageMigrationQueueCount": { + "$ref": "#/components/schemas/JobCounts" + }, "isThumbnailGenerationActive": { "type": "boolean" }, @@ -3573,6 +3576,9 @@ }, "isMachineLearningActive": { "type": "boolean" + }, + "isStorageMigrationActive": { + "type": "boolean" } }, "required": [ @@ -3580,10 +3586,12 @@ "metadataExtractionQueueCount", "videoConversionQueueCount", "machineLearningQueueCount", + "storageMigrationQueueCount", "isThumbnailGenerationActive", "isMetadataExtractionActive", "isVideoConversionActive", - "isMachineLearningActive" + "isMachineLearningActive", + "isStorageMigrationActive" ] }, "JobId": { @@ -3592,7 +3600,8 @@ "thumbnail-generation", "metadata-extraction", "video-conversion", - "machine-learning" + "machine-learning", + "storage-template-migration" ] }, "JobStatusResponseDto": { diff --git a/server/libs/common/src/config/bull-queue.config.ts b/server/libs/common/src/config/bull-queue.config.ts new file mode 100644 index 0000000000..0917ecbe9f --- /dev/null +++ b/server/libs/common/src/config/bull-queue.config.ts @@ -0,0 +1,19 @@ +import { SharedBullAsyncConfiguration } from '@nestjs/bull'; + +export const immichBullAsyncConfig: SharedBullAsyncConfiguration = { + useFactory: async () => ({ + prefix: 'immich_bull', + redis: { + host: process.env.REDIS_HOSTNAME || 'immich_redis', + port: parseInt(process.env.REDIS_PORT || '6379'), + db: parseInt(process.env.REDIS_DBINDEX || '0'), + password: process.env.REDIS_PASSWORD || undefined, + path: process.env.REDIS_SOCKET || undefined, + }, + defaultJobOptions: { + attempts: 3, + removeOnComplete: true, + removeOnFail: false, + }, + }), +}; diff --git a/server/libs/common/src/config/index.ts b/server/libs/common/src/config/index.ts index 41e7ed15eb..4ae6b9100b 100644 --- a/server/libs/common/src/config/index.ts +++ b/server/libs/common/src/config/index.ts @@ -1 +1,2 @@ export * from './app.config'; +export * from './bull-queue.config'; diff --git a/server/libs/immich-config/src/immich-config.service.ts b/server/libs/immich-config/src/immich-config.service.ts index 157b2d7a62..ce30ce91c8 100644 --- a/server/libs/immich-config/src/immich-config.service.ts +++ b/server/libs/immich-config/src/immich-config.service.ts @@ -102,4 +102,10 @@ export class ImmichConfigService { return newConfig; } + + public async refreshConfig() { + const newConfig = await this.getConfig(); + + this.config$.next(newConfig); + } } diff --git a/server/libs/job/src/constants/bull-queue-registration.constant.ts b/server/libs/job/src/constants/bull-queue-registration.constant.ts new file mode 100644 index 0000000000..240536cb11 --- /dev/null +++ b/server/libs/job/src/constants/bull-queue-registration.constant.ts @@ -0,0 +1,32 @@ +import { BullModuleOptions } from '@nestjs/bull'; +import { QueueNameEnum } from './queue-name.constant'; + +/** + * Shared queues between apps and microservices + */ +export const immichSharedQueues: BullModuleOptions[] = [ + { + name: QueueNameEnum.USER_DELETION, + }, + { + name: QueueNameEnum.THUMBNAIL_GENERATION, + }, + { + name: QueueNameEnum.ASSET_UPLOADED, + }, + { + name: QueueNameEnum.METADATA_EXTRACTION, + }, + { + name: QueueNameEnum.VIDEO_CONVERSION, + }, + { + name: QueueNameEnum.CHECKSUM_GENERATION, + }, + { + name: QueueNameEnum.MACHINE_LEARNING, + }, + { + name: QueueNameEnum.STORAGE_MIGRATION, + }, +]; diff --git a/server/libs/job/src/constants/job-name.constant.ts b/server/libs/job/src/constants/job-name.constant.ts index 9daea50082..5979e36833 100644 --- a/server/libs/job/src/constants/job-name.constant.ts +++ b/server/libs/job/src/constants/job-name.constant.ts @@ -34,3 +34,9 @@ export enum MachineLearningJobNameEnum { * User deletion Queue Jobs */ export const userDeletionProcessorName = 'user-deletion'; + +/** + * Storage Template Migration Queue Jobs + */ +export const templateMigrationProcessorName = 'template-migration'; +export const updateTemplateProcessorName = 'update-template'; diff --git a/server/libs/job/src/constants/queue-name.constant.ts b/server/libs/job/src/constants/queue-name.constant.ts index 3ddf72ea79..2741b6507e 100644 --- a/server/libs/job/src/constants/queue-name.constant.ts +++ b/server/libs/job/src/constants/queue-name.constant.ts @@ -6,4 +6,5 @@ export enum QueueNameEnum { ASSET_UPLOADED = 'asset-uploaded-queue', MACHINE_LEARNING = 'machine-learning-queue', USER_DELETION = 'user-deletion-queue', + STORAGE_MIGRATION = 'storage-template-migration', } diff --git a/server/libs/storage/src/storage.service.ts b/server/libs/storage/src/storage.service.ts index e714a10390..fab6e751f6 100644 --- a/server/libs/storage/src/storage.service.ts +++ b/server/libs/storage/src/storage.service.ts @@ -26,7 +26,7 @@ const moveFile = promisify(mv); @Injectable() export class StorageService { - readonly log = new Logger(StorageService.name); + readonly logger = new Logger(StorageService.name); private storageTemplate: HandlebarsTemplateDelegate; @@ -41,7 +41,7 @@ export class StorageService { this.immichConfigService.addValidator((config) => this.validateConfig(config)); this.immichConfigService.config$.subscribe((config) => { - this.log.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`); + this.logger.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`); this.storageTemplate = this.compile(config.storageTemplate.template); }); } @@ -54,14 +54,40 @@ export class StorageService { const rootPath = path.join(APP_UPLOAD_LOCATION, asset.userId); const storagePath = this.render(this.storageTemplate, asset, sanitized, ext); const fullPath = path.normalize(path.join(rootPath, storagePath)); + let destination = `${fullPath}.${ext}`; if (!fullPath.startsWith(rootPath)) { - this.log.warn(`Skipped attempt to access an invalid path: ${fullPath}. Path should start with ${rootPath}`); + this.logger.warn(`Skipped attempt to access an invalid path: ${fullPath}. Path should start with ${rootPath}`); return asset; } + if (source === destination) { + return asset; + } + + /** + * In case of migrating duplicate filename to a new path, we need to check if it is already migrated + * Due to the mechanism of appending +1, +2, +3, etc to the filename + * + * Example: + * Source = upload/abc/def/FullSizeRender+7.heic + * Expected Destination = upload/abc/def/FullSizeRender.heic + * + * The file is already at the correct location, but since there are other FullSizeRender.heic files in the + * destination, it was renamed to FullSizeRender+7.heic. + * + * The lines below will be used to check if the differences between the source and destination is only the + * +7 suffix, and if so, it will be considered as already migrated. + */ + if (source.startsWith(fullPath) && source.endsWith(`.${ext}`)) { + const diff = source.replace(fullPath, '').replace(`.${ext}`, ''); + const hasDuplicationAnnotation = /^\+\d+$/.test(diff); + if (hasDuplicationAnnotation) { + return asset; + } + } + let duplicateCount = 0; - let destination = `${fullPath}.${ext}`; while (true) { const exists = await this.checkFileExist(destination); @@ -70,7 +96,7 @@ export class StorageService { } duplicateCount++; - destination = `${fullPath}_${duplicateCount}.${ext}`; + destination = `${fullPath}+${duplicateCount}.${ext}`; } await this.safeMove(source, destination); @@ -78,7 +104,7 @@ export class StorageService { asset.originalPath = destination; return await this.assetRepository.save(asset); } catch (error: any) { - this.log.error(error, error.stack); + this.logger.error(error); return asset; } } @@ -115,7 +141,7 @@ export class StorageService { 'jpg', ); } catch (e) { - this.log.warn(`Storage template validation failed: ${e}`); + this.logger.warn(`Storage template validation failed: ${e}`); throw new Error(`Invalid storage template: ${e}`); } } @@ -150,4 +176,27 @@ export class StorageService { return template(substitutions); } + + public async removeEmptyDirectories(directory: string) { + // lstat does not follow symlinks (in contrast to stat) + const fileStats = await fsPromise.lstat(directory); + if (!fileStats.isDirectory()) { + return; + } + let fileNames = await fsPromise.readdir(directory); + if (fileNames.length > 0) { + const recursiveRemovalPromises = fileNames.map((fileName) => + this.removeEmptyDirectories(path.join(directory, fileName)), + ); + await Promise.all(recursiveRemovalPromises); + + // re-evaluate fileNames; after deleting subdirectory + // we may have parent directory empty now + fileNames = await fsPromise.readdir(directory); + } + + if (fileNames.length === 0) { + await fsPromise.rmdir(directory); + } + } } diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index 7392b8fb14..156fd87f6c 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -225,6 +225,12 @@ export interface AllJobStatusResponseDto { * @memberof AllJobStatusResponseDto */ 'machineLearningQueueCount': JobCounts; + /** + * + * @type {JobCounts} + * @memberof AllJobStatusResponseDto + */ + 'storageMigrationQueueCount': JobCounts; /** * * @type {boolean} @@ -249,6 +255,12 @@ export interface AllJobStatusResponseDto { * @memberof AllJobStatusResponseDto */ 'isMachineLearningActive': boolean; + /** + * + * @type {boolean} + * @memberof AllJobStatusResponseDto + */ + 'isStorageMigrationActive': boolean; } /** * @@ -1038,7 +1050,8 @@ export const JobId = { ThumbnailGeneration: 'thumbnail-generation', MetadataExtraction: 'metadata-extraction', VideoConversion: 'video-conversion', - MachineLearning: 'machine-learning' + MachineLearning: 'machine-learning', + StorageTemplateMigration: 'storage-template-migration' } as const; export type JobId = typeof JobId[keyof typeof JobId]; diff --git a/web/src/lib/components/admin-page/jobs/jobs-panel.svelte b/web/src/lib/components/admin-page/jobs/jobs-panel.svelte index 0fd019061f..f327a1e649 100644 --- a/web/src/lib/components/admin-page/jobs/jobs-panel.svelte +++ b/web/src/lib/components/admin-page/jobs/jobs-panel.svelte @@ -9,6 +9,7 @@ let allJobsStatus: AllJobStatusResponseDto; let setIntervalHandler: NodeJS.Timer; + onMount(async () => { const { data } = await api.jobApi.getAllJobsStatus(); allJobsStatus = data; @@ -104,6 +105,33 @@ }); } }; + + const runTemplateMigration = async () => { + try { + const { data } = await api.jobApi.sendJobCommand(JobId.StorageTemplateMigration, { + command: JobCommand.Start + }); + + if (data) { + notificationController.show({ + message: `Storage migration started`, + type: NotificationType.Info + }); + } else { + notificationController.show({ + message: `All files have been migrated to the new storage template`, + type: NotificationType.Info + }); + } + } catch (e) { + console.log('[ERROR] runTemplateMigration', e); + + notificationController.show({ + message: `Error running template migration job, check console for more detail`, + type: NotificationType.Error + }); + } + };
@@ -135,4 +163,20 @@ > Note that some asset does not have any object detected, this is normal. + + + Apply the current + Storage template + to previously uploaded assets +
diff --git a/web/src/lib/components/admin-page/settings/setting-accordion.svelte b/web/src/lib/components/admin-page/settings/setting-accordion.svelte index ddbb9e8c84..c63b89353b 100644 --- a/web/src/lib/components/admin-page/settings/setting-accordion.svelte +++ b/web/src/lib/components/admin-page/settings/setting-accordion.svelte @@ -3,7 +3,7 @@ export let title: string; export let subtitle = ''; - let isOpen = false; + export let isOpen = false; const toggle = () => (isOpen = !isOpen); diff --git a/web/src/lib/components/admin-page/settings/storate-template/storage-template-settings.svelte b/web/src/lib/components/admin-page/settings/storate-template/storage-template-settings.svelte index b21924a876..02e2f13ca7 100644 --- a/web/src/lib/components/admin-page/settings/storate-template/storage-template-settings.svelte +++ b/web/src/lib/components/admin-page/settings/storate-template/storage-template-settings.svelte @@ -214,6 +214,16 @@ +
+

+ Template changes will only apply to new assets. To retroactively apply the template to + previously uploaded assets, run the Storage Migration Job +

+
+
-
+
diff --git a/web/src/routes/admin/system-settings/+page.svelte b/web/src/routes/admin/system-settings/+page.svelte index 741c882e5e..7a7e1a72d6 100644 --- a/web/src/routes/admin/system-settings/+page.svelte +++ b/web/src/routes/admin/system-settings/+page.svelte @@ -6,6 +6,7 @@ import LoadingSpinner from '$lib/components/shared-components/loading-spinner.svelte'; import { api, SystemConfigDto } from '@api'; import type { PageData } from './$types'; + import { page } from '$app/stores'; let systemConfig: SystemConfigDto; export let data: PageData; @@ -39,6 +40,7 @@