chore(server) Add job for storage migration (#1117)

server/apps/immich/src/api-v1/asset/asset-repository.ts

@@ -15,6 +15,7 @@ import { CheckExistingAssetsResponseDto } from './response-dto/check-existing-as
 import { In } from 'typeorm/find-options/operator/In';
 import { UpdateAssetDto } from './dto/update-asset.dto';
 import { ITagRepository, TAG_REPOSITORY } from '../tag/tag.repository';
+import { IsNull } from 'typeorm';
 
 export interface IAssetRepository {
   create(
@@ -69,14 +70,14 @@ export class AssetRepository implements IAssetRepository {
   }
 
   async getAssetWithNoThumbnail(): Promise<AssetEntity[]> {
-    return await this.assetRepository
+    return await this.assetRepository.find({
-      .createQueryBuilder('asset')
+      where: [
-      .where('asset.resizePath IS NULL')
+        { resizePath: IsNull(), isVisible: true },
-      .andWhere('asset.isVisible = true')
+        { resizePath: '', isVisible: true },
-      .orWhere('asset.resizePath = :resizePath', { resizePath: '' })
+        { webpPath: IsNull(), isVisible: true },
-      .orWhere('asset.webpPath IS NULL')
+        { webpPath: '', isVisible: true },
-      .orWhere('asset.webpPath = :webpPath', { webpPath: '' })
+      ],
-      .getMany();
+    });
   }
 
   async getAssetWithNoEXIF(): Promise<AssetEntity[]> {

server/apps/immich/src/api-v1/asset/asset.module.ts

@@ -7,13 +7,13 @@ import { BullModule } from '@nestjs/bull';
 import { BackgroundTaskModule } from '../../modules/background-task/background-task.module';
 import { BackgroundTaskService } from '../../modules/background-task/background-task.service';
 import { CommunicationModule } from '../communication/communication.module';
-import { QueueNameEnum } from '@app/job/constants/queue-name.constant';
 import { AssetRepository, ASSET_REPOSITORY } from './asset-repository';
 import { DownloadModule } from '../../modules/download/download.module';
 import { TagModule } from '../tag/tag.module';
 import { AlbumModule } from '../album/album.module';
 import { UserModule } from '../user/user.module';
 import { StorageModule } from '@app/storage';
+import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant';
 
 const ASSET_REPOSITORY_PROVIDER = {
   provide: ASSET_REPOSITORY,
@@ -31,22 +31,7 @@ const ASSET_REPOSITORY_PROVIDER = {
     TagModule,
     StorageModule,
     forwardRef(() => AlbumModule),
-    BullModule.registerQueue({
+    BullModule.registerQueue(...immichSharedQueues),
-      name: QueueNameEnum.ASSET_UPLOADED,
-      defaultJobOptions: {
-        attempts: 3,
-        removeOnComplete: true,
-        removeOnFail: false,
-      },
-    }),
-    BullModule.registerQueue({
-      name: QueueNameEnum.VIDEO_CONVERSION,
-      defaultJobOptions: {
-        attempts: 3,
-        removeOnComplete: true,
-        removeOnFail: false,
-      },
-    }),
   ],
   controllers: [AssetController],
   providers: [AssetService, BackgroundTaskService, ASSET_REPOSITORY_PROVIDER],

server/apps/immich/src/api-v1/job/dto/get-job.dto.ts

@@ -6,6 +6,7 @@ export enum JobId {
   METADATA_EXTRACTION = 'metadata-extraction',
   VIDEO_CONVERSION = 'video-conversion',
   MACHINE_LEARNING = 'machine-learning',
+  STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration',
 }
 
 export class GetJobDto {

server/apps/immich/src/api-v1/job/job.module.ts

@@ -6,13 +6,15 @@ import { ImmichJwtModule } from '../../modules/immich-jwt/immich-jwt.module';
 import { JwtModule } from '@nestjs/jwt';
 import { jwtConfig } from '../../config/jwt.config';
 import { TypeOrmModule } from '@nestjs/typeorm';
-import { BullModule } from '@nestjs/bull';
-import { QueueNameEnum } from '@app/job';
 import { ExifEntity } from '@app/database/entities/exif.entity';
 import { TagModule } from '../tag/tag.module';
 import { AssetModule } from '../asset/asset.module';
 import { UserModule } from '../user/user.module';
 
+import { StorageModule } from '@app/storage';
+import { BullModule } from '@nestjs/bull';
+import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant';
+
 @Module({
   imports: [
     TypeOrmModule.forFeature([ExifEntity]),
@@ -21,56 +23,8 @@ import { UserModule } from '../user/user.module';
     AssetModule,
     UserModule,
     JwtModule.register(jwtConfig),
-    BullModule.registerQueue(
+    StorageModule,
-      {
+    BullModule.registerQueue(...immichSharedQueues),
-        name: QueueNameEnum.THUMBNAIL_GENERATION,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      {
-        name: QueueNameEnum.ASSET_UPLOADED,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      {
-        name: QueueNameEnum.METADATA_EXTRACTION,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      {
-        name: QueueNameEnum.VIDEO_CONVERSION,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      {
-        name: QueueNameEnum.CHECKSUM_GENERATION,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      {
-        name: QueueNameEnum.MACHINE_LEARNING,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-    ),
   ],
   controllers: [JobController],
   providers: [JobService, ImmichJwtService],

server/apps/immich/src/api-v1/job/job.service.ts

@@ -6,6 +6,7 @@ import {
   IVideoTranscodeJob,
   MachineLearningJobNameEnum,
   QueueNameEnum,
+  templateMigrationProcessorName,
   videoMetadataExtractionProcessorName,
 } from '@app/job';
 import { InjectQueue } from '@nestjs/bull';
@@ -18,6 +19,7 @@ import { AssetType } from '@app/database/entities/asset.entity';
 import { GetJobDto, JobId } from './dto/get-job.dto';
 import { JobStatusResponseDto } from './response-dto/job-status-response.dto';
 import { IMachineLearningJob } from '@app/job/interfaces/machine-learning.interface';
+import { StorageService } from '@app/storage';
 
 @Injectable()
 export class JobService {
@@ -34,12 +36,18 @@ export class JobService {
     @InjectQueue(QueueNameEnum.MACHINE_LEARNING)
     private machineLearningQueue: Queue<IMachineLearningJob>,
 
+    @InjectQueue(QueueNameEnum.STORAGE_MIGRATION)
+    private storageMigrationQueue: Queue,
+
     @Inject(ASSET_REPOSITORY)
     private _assetRepository: IAssetRepository,
+
+    private storageService: StorageService,
   ) {
     this.thumbnailGeneratorQueue.empty();
     this.metadataExtractionQueue.empty();
     this.videoConversionQueue.empty();
+    this.storageMigrationQueue.empty();
   }
 
   async startJob(jobDto: GetJobDto): Promise<number> {
@@ -52,6 +60,8 @@ export class JobService {
         return 0;
       case JobId.MACHINE_LEARNING:
         return this.runMachineLearningPipeline();
+      case JobId.STORAGE_TEMPLATE_MIGRATION:
+        return this.runStorageMigration();
       default:
         throw new BadRequestException('Invalid job id');
     }
@@ -62,6 +72,7 @@ export class JobService {
     const metadataExtractionJobCount = await this.metadataExtractionQueue.getJobCounts();
     const videoConversionJobCount = await this.videoConversionQueue.getJobCounts();
     const machineLearningJobCount = await this.machineLearningQueue.getJobCounts();
+    const storageMigrationJobCount = await this.storageMigrationQueue.getJobCounts();
 
     const response = new AllJobStatusResponseDto();
     response.isThumbnailGenerationActive = Boolean(thumbnailGeneratorJobCount.waiting);
@@ -73,6 +84,9 @@ export class JobService {
     response.isMachineLearningActive = Boolean(machineLearningJobCount.waiting);
     response.machineLearningQueueCount = machineLearningJobCount;
 
+    response.isStorageMigrationActive = Boolean(storageMigrationJobCount.active);
+    response.storageMigrationQueueCount = storageMigrationJobCount;
+
     return response;
   }
 
@@ -93,6 +107,11 @@ export class JobService {
       response.queueCount = await this.videoConversionQueue.getJobCounts();
     }
 
+    if (query.jobId === JobId.STORAGE_TEMPLATE_MIGRATION) {
+      response.isActive = Boolean((await this.storageMigrationQueue.getJobCounts()).waiting);
+      response.queueCount = await this.storageMigrationQueue.getJobCounts();
+    }
+
     return response;
   }
 
@@ -110,6 +129,9 @@ export class JobService {
       case JobId.MACHINE_LEARNING:
         this.machineLearningQueue.empty();
         return 0;
+      case JobId.STORAGE_TEMPLATE_MIGRATION:
+        this.storageMigrationQueue.empty();
+        return 0;
       default:
         throw new BadRequestException('Invalid job id');
     }
@@ -177,4 +199,16 @@ export class JobService {
 
     return assetWithNoSmartInfo.length;
   }
+
+  async runStorageMigration() {
+    const jobCount = await this.storageMigrationQueue.getJobCounts();
+
+    if (jobCount.active > 0) {
+      throw new BadRequestException('Storage migration job is already running');
+    }
+
+    await this.storageMigrationQueue.add(templateMigrationProcessorName, {}, { jobId: randomUUID() });
+
+    return 1;
+  }
 }

server/apps/immich/src/api-v1/job/response-dto/all-job-status-response.dto.ts

@@ -17,6 +17,7 @@ export class AllJobStatusResponseDto {
   isMetadataExtractionActive!: boolean;
   isVideoConversionActive!: boolean;
   isMachineLearningActive!: boolean;
+  isStorageMigrationActive!: boolean;
 
   @ApiProperty({
     type: JobCounts,
@@ -37,4 +38,9 @@ export class AllJobStatusResponseDto {
     type: JobCounts,
   })
   machineLearningQueueCount!: JobCounts;
+
+  @ApiProperty({
+    type: JobCounts,
+  })
+  storageMigrationQueueCount!: JobCounts;
 }

server/apps/immich/src/api-v1/system-config/system-config.module.ts

@@ -1,4 +1,6 @@
 import { SystemConfigEntity } from '@app/database/entities/system-config.entity';
+import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant';
+import { BullModule } from '@nestjs/bull';
 import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { ImmichConfigModule } from 'libs/immich-config/src';
@@ -7,7 +9,12 @@ import { SystemConfigController } from './system-config.controller';
 import { SystemConfigService } from './system-config.service';
 
 @Module({
-  imports: [ImmichJwtModule, ImmichConfigModule, TypeOrmModule.forFeature([SystemConfigEntity])],
+  imports: [
+    ImmichJwtModule,
+    ImmichConfigModule,
+    TypeOrmModule.forFeature([SystemConfigEntity]),
+    BullModule.registerQueue(...immichSharedQueues),
+  ],
   controllers: [SystemConfigController],
   providers: [SystemConfigService],
 })

server/apps/immich/src/api-v1/system-config/system-config.service.ts

@@ -1,3 +1,4 @@
+import { QueueNameEnum, updateTemplateProcessorName } from '@app/job';
 import {
   supportedDayTokens,
   supportedHourTokens,
@@ -7,14 +8,21 @@ import {
   supportedSecondTokens,
   supportedYearTokens,
 } from '@app/storage/constants/supported-datetime-template';
+import { InjectQueue } from '@nestjs/bull';
 import { Injectable } from '@nestjs/common';
+import { Queue } from 'bull';
+import { randomUUID } from 'crypto';
 import { ImmichConfigService } from 'libs/immich-config/src';
 import { mapConfig, SystemConfigDto } from './dto/system-config.dto';
 import { SystemConfigTemplateStorageOptionDto } from './response-dto/system-config-template-storage-option.dto';
 
 @Injectable()
 export class SystemConfigService {
-  constructor(private immichConfigService: ImmichConfigService) {}
+  constructor(
+    private immichConfigService: ImmichConfigService,
+    @InjectQueue(QueueNameEnum.STORAGE_MIGRATION)
+    private storageMigrationQueue: Queue,
+  ) {}
 
   public async getConfig(): Promise<SystemConfigDto> {
     const config = await this.immichConfigService.getConfig();
@@ -28,6 +36,7 @@ export class SystemConfigService {
 
   public async updateConfig(dto: SystemConfigDto): Promise<SystemConfigDto> {
     const config = await this.immichConfigService.updateConfig(dto);
+    this.storageMigrationQueue.add(updateTemplateProcessorName, {}, { jobId: randomUUID() });
     return mapConfig(config);
   }
 

server/apps/immich/src/api-v1/user/user.service.ts

@@ -10,7 +10,7 @@ import {
   UnauthorizedException,
 } from '@nestjs/common';
 import { Response as Res } from 'express';
-import { createReadStream } from 'fs';
+import { constants, createReadStream } from 'fs';
 import { AuthUserDto } from '../../decorators/auth-user.decorator';
 import { CreateUserDto } from './dto/create-user.dto';
 import { UpdateUserDto } from './dto/update-user.dto';
@@ -22,6 +22,7 @@ import {
 import { mapUserCountResponse, UserCountResponseDto } from './response-dto/user-count-response.dto';
 import { mapUser, UserResponseDto } from './response-dto/user-response.dto';
 import { IUserRepository, USER_REPOSITORY } from './user-repository';
+import fs from 'fs/promises';
 
 @Injectable()
 export class UserService {
@@ -196,6 +197,8 @@ export class UserService {
         throw new NotFoundException('User does not have a profile image');
       }
 
+      await fs.access(user.profileImagePath, constants.R_OK | constants.W_OK);
+
       res.set({
         'Content-Type': 'image/jpeg',
       });

server/apps/immich/src/app.module.ts

@@ -1,4 +1,4 @@
-import { immichAppConfig } from '@app/common/config';
+import { immichAppConfig, immichBullAsyncConfig } from '@app/common/config';
 import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
 import { UserModule } from './api-v1/user/user.module';
 import { AssetModule } from './api-v1/asset/asset.module';
@@ -36,18 +36,7 @@ import { TagModule } from './api-v1/tag/tag.module';
 
     DeviceInfoModule,
 
-    BullModule.forRootAsync({
+    BullModule.forRootAsync(immichBullAsyncConfig),
-      useFactory: async () => ({
-        prefix: 'immich_bull',
-        redis: {
-          host: process.env.REDIS_HOSTNAME || 'immich_redis',
-          port: parseInt(process.env.REDIS_PORT || '6379'),
-          db: parseInt(process.env.REDIS_DBINDEX || '0'),
-          password: process.env.REDIS_PASSWORD || undefined,
-          path: process.env.REDIS_SOCKET || undefined,
-        },
-      }),
-    }),
 
     ServerInfoModule,
 

server/apps/immich/src/modules/background-task/background-task.module.ts

@@ -11,11 +11,6 @@ import { BackgroundTaskService } from './background-task.service';
   imports: [
     BullModule.registerQueue({
       name: 'background-task',
-      defaultJobOptions: {
-        attempts: 3,
-        removeOnComplete: true,
-        removeOnFail: false,
-      },
     }),
     TypeOrmModule.forFeature([AssetEntity, ExifEntity, SmartInfoEntity]),
   ],

server/apps/immich/src/modules/schedule-tasks/schedule-tasks.module.ts

@@ -3,46 +3,14 @@ import { Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { AssetEntity } from '@app/database/entities/asset.entity';
 import { ScheduleTasksService } from './schedule-tasks.service';
-import { QueueNameEnum } from '@app/job/constants/queue-name.constant';
 import { ExifEntity } from '@app/database/entities/exif.entity';
 import { UserEntity } from '@app/database/entities/user.entity';
+import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant';
 
 @Module({
   imports: [
     TypeOrmModule.forFeature([AssetEntity, ExifEntity, UserEntity]),
-    BullModule.registerQueue({
+    BullModule.registerQueue(...immichSharedQueues),
-      name: QueueNameEnum.USER_DELETION,
-      defaultJobOptions: {
-        attempts: 3,
-        removeOnComplete: true,
-        removeOnFail: false,
-      },
-    }),
-    BullModule.registerQueue({
-      name: QueueNameEnum.VIDEO_CONVERSION,
-      defaultJobOptions: {
-        attempts: 3,
-        removeOnComplete: true,
-        removeOnFail: false,
-      },
-    }),
-    BullModule.registerQueue({
-      name: QueueNameEnum.THUMBNAIL_GENERATION,
-      defaultJobOptions: {
-        attempts: 3,
-        removeOnComplete: true,
-        removeOnFail: false,
-      },
-    }),
-
-    BullModule.registerQueue({
-      name: QueueNameEnum.METADATA_EXTRACTION,
-      defaultJobOptions: {
-        attempts: 3,
-        removeOnComplete: true,
-        removeOnFail: false,
-      },
-    }),
   ],
   providers: [ScheduleTasksService],
 })

server/apps/microservices/src/microservices.module.ts

@@ -1,10 +1,10 @@
-import { immichAppConfig } from '@app/common/config';
+import { immichAppConfig, immichBullAsyncConfig } from '@app/common/config';
 import { DatabaseModule } from '@app/database';
 import { AssetEntity } from '@app/database/entities/asset.entity';
 import { ExifEntity } from '@app/database/entities/exif.entity';
 import { SmartInfoEntity } from '@app/database/entities/smart-info.entity';
 import { UserEntity } from '@app/database/entities/user.entity';
-import { QueueNameEnum } from '@app/job/constants/queue-name.constant';
+import { StorageModule } from '@app/storage';
 import { BullModule } from '@nestjs/bull';
 import { Module } from '@nestjs/common';
 import { ConfigModule } from '@nestjs/config';
@@ -16,9 +16,11 @@ import { AssetUploadedProcessor } from './processors/asset-uploaded.processor';
 import { GenerateChecksumProcessor } from './processors/generate-checksum.processor';
 import { MachineLearningProcessor } from './processors/machine-learning.processor';
 import { MetadataExtractionProcessor } from './processors/metadata-extraction.processor';
+import { StorageMigrationProcessor } from './processors/storage-migration.processor';
 import { ThumbnailGeneratorProcessor } from './processors/thumbnail.processor';
 import { UserDeletionProcessor } from './processors/user-deletion.processor';
 import { VideoTranscodeProcessor } from './processors/video-transcode.processor';
+import { immichSharedQueues } from '@app/job/constants/bull-queue-registration.constant';
 
 @Module({
   imports: [
@@ -26,76 +28,9 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor'
     DatabaseModule,
     ImmichConfigModule,
     TypeOrmModule.forFeature([UserEntity, ExifEntity, AssetEntity, SmartInfoEntity]),
-    BullModule.forRootAsync({
+    StorageModule,
-      useFactory: async () => ({
+    BullModule.forRootAsync(immichBullAsyncConfig),
-        prefix: 'immich_bull',
+    BullModule.registerQueue(...immichSharedQueues),
-        redis: {
-          host: process.env.REDIS_HOSTNAME || 'immich_redis',
-          port: parseInt(process.env.REDIS_PORT || '6379'),
-          db: parseInt(process.env.REDIS_DBINDEX || '0'),
-          password: process.env.REDIS_PASSWORD || undefined,
-          path: process.env.REDIS_SOCKET || undefined,
-        },
-      }),
-    }),
-    BullModule.registerQueue(
-      {
-        name: QueueNameEnum.USER_DELETION,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      {
-        name: QueueNameEnum.THUMBNAIL_GENERATION,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      {
-        name: QueueNameEnum.ASSET_UPLOADED,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      {
-        name: QueueNameEnum.METADATA_EXTRACTION,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      {
-        name: QueueNameEnum.VIDEO_CONVERSION,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      {
-        name: QueueNameEnum.CHECKSUM_GENERATION,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      {
-        name: QueueNameEnum.MACHINE_LEARNING,
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-    ),
     CommunicationModule,
   ],
   controllers: [],
@@ -108,7 +43,8 @@ import { VideoTranscodeProcessor } from './processors/video-transcode.processor'
     GenerateChecksumProcessor,
     MachineLearningProcessor,
     UserDeletionProcessor,
+    StorageMigrationProcessor,
   ],
-  exports: [],
+  exports: [BullModule],
 })
 export class MicroservicesModule {}

server/apps/microservices/src/processors/storage-migration.processor.ts

@@ -0,0 +1,61 @@
+import { APP_UPLOAD_LOCATION } from '@app/common';
+import { AssetEntity } from '@app/database/entities/asset.entity';
+import { ImmichConfigService } from '@app/immich-config';
+import { QueueNameEnum, templateMigrationProcessorName, updateTemplateProcessorName } from '@app/job';
+import { StorageService } from '@app/storage';
+import { Process, Processor } from '@nestjs/bull';
+import { Logger } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Repository } from 'typeorm';
+
+@Processor(QueueNameEnum.STORAGE_MIGRATION)
+export class StorageMigrationProcessor {
+  readonly logger: Logger = new Logger(StorageMigrationProcessor.name);
+
+  constructor(
+    private storageService: StorageService,
+    private immichConfigService: ImmichConfigService,
+
+    @InjectRepository(AssetEntity)
+    private assetRepository: Repository<AssetEntity>,
+  ) {}
+
+  /**
+   * Migration process when a new user set a new storage template.
+   * @param job
+   */
+  @Process({ name: templateMigrationProcessorName, concurrency: 100 })
+  async templateMigration() {
+    console.time('migrating-time');
+    const assets = await this.assetRepository.find({
+      relations: ['exifInfo'],
+    });
+
+    const livePhotoMap: Record<string, AssetEntity> = {};
+
+    for (const asset of assets) {
+      if (asset.livePhotoVideoId) {
+        livePhotoMap[asset.livePhotoVideoId] = asset;
+      }
+    }
+
+    for (const asset of assets) {
+      const livePhotoParentAsset = livePhotoMap[asset.id];
+      const filename = asset.exifInfo?.imageName || livePhotoParentAsset?.exifInfo?.imageName || asset.id;
+      await this.storageService.moveAsset(asset, filename);
+    }
+
+    await this.storageService.removeEmptyDirectories(APP_UPLOAD_LOCATION);
+    console.timeEnd('migrating-time');
+  }
+
+  /**
+   * Update config when a new storage template is set.
+   * This is to ensure the synchronization between processes.
+   * @param job
+   */
+  @Process({ name: updateTemplateProcessorName, concurrency: 1 })
+  async updateTemplate() {
+    await this.immichConfigService.refreshConfig();
+  }
+}

server/apps/microservices/src/processors/thumbnail.processor.ts

@@ -1,5 +1,4 @@
 import { APP_UPLOAD_LOCATION } from '@app/common';
-import { ImmichLogLevel } from '@app/common/constants/log-level.constant';
 import { AssetEntity, AssetType } from '@app/database/entities/asset.entity';
 import {
   WebpGeneratorProcessor,
@@ -11,7 +10,6 @@ import {
 } from '@app/job';
 import { InjectQueue, Process, Processor } from '@nestjs/bull';
 import { Logger } from '@nestjs/common';
-import { ConfigService } from '@nestjs/config';
 import { InjectRepository } from '@nestjs/typeorm';
 import { mapAsset } from 'apps/immich/src/api-v1/asset/response-dto/asset-response.dto';
 import { Job, Queue } from 'bull';
@@ -27,7 +25,7 @@ import { IMachineLearningJob } from '@app/job/interfaces/machine-learning.interf
 
 @Processor(QueueNameEnum.THUMBNAIL_GENERATION)
 export class ThumbnailGeneratorProcessor {
-  private logLevel: ImmichLogLevel;
+  readonly logger: Logger = new Logger(ThumbnailGeneratorProcessor.name);
 
   constructor(
     @InjectRepository(AssetEntity)
@@ -40,12 +38,7 @@ export class ThumbnailGeneratorProcessor {
 
     @InjectQueue(QueueNameEnum.MACHINE_LEARNING)
     private machineLearningQueue: Queue<IMachineLearningJob>,
-
+  ) {}
-    private configService: ConfigService,
-  ) {
-    this.logLevel = this.configService.get('LOG_LEVEL') || ImmichLogLevel.SIMPLE;
-    // TODO - Add observable paterrn to listen to the config change
-  }
 
   @Process({ name: generateJPEGThumbnailProcessorName, concurrency: 3 })
   async generateJPEGThumbnail(job: Job<JpegGeneratorProcessor>) {
@@ -70,12 +63,8 @@ export class ThumbnailGeneratorProcessor {
           .rotate()
           .toFile(jpegThumbnailPath);
         await this.assetRepository.update({ id: asset.id }, { resizePath: jpegThumbnailPath });
-      } catch (error) {
+      } catch (error: any) {
-        Logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id);
+        this.logger.error('Failed to generate jpeg thumbnail for asset: ' + asset.id, error.stack);
-
-        if (this.logLevel == ImmichLogLevel.VERBOSE) {
-          console.trace('Failed to generate jpeg thumbnail for asset', error);
-        }
       }
 
       // Update resize path to send to generate webp queue
@@ -140,12 +129,8 @@ export class ThumbnailGeneratorProcessor {
     try {
       await sharp(asset.resizePath, { failOnError: false }).resize(250).webp().rotate().toFile(webpPath);
       await this.assetRepository.update({ id: asset.id }, { webpPath: webpPath });
-    } catch (error) {
+    } catch (error: any) {
-      Logger.error('Failed to generate webp thumbnail for asset: ' + asset.id);
+      this.logger.error('Failed to generate webp thumbnail for asset: ' + asset.id, error.stack);
-
-      if (this.logLevel == ImmichLogLevel.VERBOSE) {
-        console.trace('Failed to generate webp thumbnail for asset', error);
-      }
     }
   }
 }

server/immich-openapi-specs.json

@@ -3562,6 +3562,9 @@
           "machineLearningQueueCount": {
             "$ref": "#/components/schemas/JobCounts"
           },
+          "storageMigrationQueueCount": {
+            "$ref": "#/components/schemas/JobCounts"
+          },
           "isThumbnailGenerationActive": {
             "type": "boolean"
           },
@@ -3573,6 +3576,9 @@
           },
           "isMachineLearningActive": {
             "type": "boolean"
+          },
+          "isStorageMigrationActive": {
+            "type": "boolean"
           }
         },
         "required": [
@@ -3580,10 +3586,12 @@
           "metadataExtractionQueueCount",
           "videoConversionQueueCount",
           "machineLearningQueueCount",
+          "storageMigrationQueueCount",
           "isThumbnailGenerationActive",
           "isMetadataExtractionActive",
           "isVideoConversionActive",
-          "isMachineLearningActive"
+          "isMachineLearningActive",
+          "isStorageMigrationActive"
         ]
       },
       "JobId": {
@@ -3592,7 +3600,8 @@
           "thumbnail-generation",
           "metadata-extraction",
           "video-conversion",
-          "machine-learning"
+          "machine-learning",
+          "storage-template-migration"
         ]
       },
       "JobStatusResponseDto": {

server/libs/common/src/config/bull-queue.config.ts

@@ -0,0 +1,19 @@
+import { SharedBullAsyncConfiguration } from '@nestjs/bull';
+
+export const immichBullAsyncConfig: SharedBullAsyncConfiguration = {
+  useFactory: async () => ({
+    prefix: 'immich_bull',
+    redis: {
+      host: process.env.REDIS_HOSTNAME || 'immich_redis',
+      port: parseInt(process.env.REDIS_PORT || '6379'),
+      db: parseInt(process.env.REDIS_DBINDEX || '0'),
+      password: process.env.REDIS_PASSWORD || undefined,
+      path: process.env.REDIS_SOCKET || undefined,
+    },
+    defaultJobOptions: {
+      attempts: 3,
+      removeOnComplete: true,
+      removeOnFail: false,
+    },
+  }),
+};

server/libs/common/src/config/index.ts

@@ -1 +1,2 @@
 export * from './app.config';
+export * from './bull-queue.config';

server/libs/immich-config/src/immich-config.service.ts

@@ -102,4 +102,10 @@ export class ImmichConfigService {
 
     return newConfig;
   }
+
+  public async refreshConfig() {
+    const newConfig = await this.getConfig();
+
+    this.config$.next(newConfig);
+  }
 }

server/libs/job/src/constants/bull-queue-registration.constant.ts

@@ -0,0 +1,32 @@
+import { BullModuleOptions } from '@nestjs/bull';
+import { QueueNameEnum } from './queue-name.constant';
+
+/**
+ * Shared queues between apps and microservices
+ */
+export const immichSharedQueues: BullModuleOptions[] = [
+  {
+    name: QueueNameEnum.USER_DELETION,
+  },
+  {
+    name: QueueNameEnum.THUMBNAIL_GENERATION,
+  },
+  {
+    name: QueueNameEnum.ASSET_UPLOADED,
+  },
+  {
+    name: QueueNameEnum.METADATA_EXTRACTION,
+  },
+  {
+    name: QueueNameEnum.VIDEO_CONVERSION,
+  },
+  {
+    name: QueueNameEnum.CHECKSUM_GENERATION,
+  },
+  {
+    name: QueueNameEnum.MACHINE_LEARNING,
+  },
+  {
+    name: QueueNameEnum.STORAGE_MIGRATION,
+  },
+];

server/libs/job/src/constants/job-name.constant.ts

@@ -34,3 +34,9 @@ export enum MachineLearningJobNameEnum {
  * User deletion Queue Jobs
  */
 export const userDeletionProcessorName = 'user-deletion';
+
+/**
+ * Storage Template Migration Queue Jobs
+ */
+export const templateMigrationProcessorName = 'template-migration';
+export const updateTemplateProcessorName = 'update-template';

server/libs/job/src/constants/queue-name.constant.ts

@@ -6,4 +6,5 @@ export enum QueueNameEnum {
   ASSET_UPLOADED = 'asset-uploaded-queue',
   MACHINE_LEARNING = 'machine-learning-queue',
   USER_DELETION = 'user-deletion-queue',
+  STORAGE_MIGRATION = 'storage-template-migration',
 }

server/libs/storage/src/storage.service.ts

@@ -26,7 +26,7 @@ const moveFile = promisify<string, string, mv.Options>(mv);
 
 @Injectable()
 export class StorageService {
-  readonly log = new Logger(StorageService.name);
+  readonly logger = new Logger(StorageService.name);
 
   private storageTemplate: HandlebarsTemplateDelegate<any>;
 
@@ -41,7 +41,7 @@ export class StorageService {
     this.immichConfigService.addValidator((config) => this.validateConfig(config));
 
     this.immichConfigService.config$.subscribe((config) => {
-      this.log.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`);
+      this.logger.debug(`Received new config, recompiling storage template: ${config.storageTemplate.template}`);
       this.storageTemplate = this.compile(config.storageTemplate.template);
     });
   }
@@ -54,14 +54,40 @@ export class StorageService {
       const rootPath = path.join(APP_UPLOAD_LOCATION, asset.userId);
       const storagePath = this.render(this.storageTemplate, asset, sanitized, ext);
       const fullPath = path.normalize(path.join(rootPath, storagePath));
+      let destination = `${fullPath}.${ext}`;
 
       if (!fullPath.startsWith(rootPath)) {
-        this.log.warn(`Skipped attempt to access an invalid path: ${fullPath}. Path should start with ${rootPath}`);
+        this.logger.warn(`Skipped attempt to access an invalid path: ${fullPath}. Path should start with ${rootPath}`);
         return asset;
       }
 
+      if (source === destination) {
+        return asset;
+      }
+
+      /**
+       * In case of migrating duplicate filename to a new path, we need to check if it is already migrated
+       * Due to the mechanism of appending +1, +2, +3, etc to the filename
+       *
+       * Example:
+       * Source = upload/abc/def/FullSizeRender+7.heic
+       * Expected Destination = upload/abc/def/FullSizeRender.heic
+       *
+       * The file is already at the correct location, but since there are other FullSizeRender.heic files in the
+       * destination, it was renamed to FullSizeRender+7.heic.
+       *
+       * The lines below will be used to check if the differences between the source and destination is only the
+       * +7 suffix, and if so, it will be considered as already migrated.
+       */
+      if (source.startsWith(fullPath) && source.endsWith(`.${ext}`)) {
+        const diff = source.replace(fullPath, '').replace(`.${ext}`, '');
+        const hasDuplicationAnnotation = /^\+\d+$/.test(diff);
+        if (hasDuplicationAnnotation) {
+          return asset;
+        }
+      }
+
       let duplicateCount = 0;
-      let destination = `${fullPath}.${ext}`;
 
       while (true) {
         const exists = await this.checkFileExist(destination);
@@ -70,7 +96,7 @@ export class StorageService {
         }
 
         duplicateCount++;
-        destination = `${fullPath}_${duplicateCount}.${ext}`;
+        destination = `${fullPath}+${duplicateCount}.${ext}`;
       }
 
       await this.safeMove(source, destination);
@@ -78,7 +104,7 @@ export class StorageService {
       asset.originalPath = destination;
       return await this.assetRepository.save(asset);
     } catch (error: any) {
-      this.log.error(error, error.stack);
+      this.logger.error(error);
       return asset;
     }
   }
@@ -115,7 +141,7 @@ export class StorageService {
         'jpg',
       );
     } catch (e) {
-      this.log.warn(`Storage template validation failed: ${e}`);
+      this.logger.warn(`Storage template validation failed: ${e}`);
       throw new Error(`Invalid storage template: ${e}`);
     }
   }
@@ -150,4 +176,27 @@ export class StorageService {
 
     return template(substitutions);
   }
+
+  public async removeEmptyDirectories(directory: string) {
+    // lstat does not follow symlinks (in contrast to stat)
+    const fileStats = await fsPromise.lstat(directory);
+    if (!fileStats.isDirectory()) {
+      return;
+    }
+    let fileNames = await fsPromise.readdir(directory);
+    if (fileNames.length > 0) {
+      const recursiveRemovalPromises = fileNames.map((fileName) =>
+        this.removeEmptyDirectories(path.join(directory, fileName)),
+      );
+      await Promise.all(recursiveRemovalPromises);
+
+      // re-evaluate fileNames; after deleting subdirectory
+      // we may have parent directory empty now
+      fileNames = await fsPromise.readdir(directory);
+    }
+
+    if (fileNames.length === 0) {
+      await fsPromise.rmdir(directory);
+    }
+  }
 }

web/src/api/open-api/api.ts

@@ -225,6 +225,12 @@ export interface AllJobStatusResponseDto {
      * @memberof AllJobStatusResponseDto
      */
     'machineLearningQueueCount': JobCounts;
+    /**
+     * 
+     * @type {JobCounts}
+     * @memberof AllJobStatusResponseDto
+     */
+    'storageMigrationQueueCount': JobCounts;
     /**
      * 
      * @type {boolean}
@@ -249,6 +255,12 @@ export interface AllJobStatusResponseDto {
      * @memberof AllJobStatusResponseDto
      */
     'isMachineLearningActive': boolean;
+    /**
+     * 
+     * @type {boolean}
+     * @memberof AllJobStatusResponseDto
+     */
+    'isStorageMigrationActive': boolean;
 }
 /**
  * 
@@ -1038,7 +1050,8 @@ export const JobId = {
     ThumbnailGeneration: 'thumbnail-generation',
     MetadataExtraction: 'metadata-extraction',
     VideoConversion: 'video-conversion',
-    MachineLearning: 'machine-learning'
+    MachineLearning: 'machine-learning',
+    StorageTemplateMigration: 'storage-template-migration'
 } as const;
 
 export type JobId = typeof JobId[keyof typeof JobId];

web/src/lib/components/admin-page/jobs/jobs-panel.svelte

@@ -9,6 +9,7 @@
 
 	let allJobsStatus: AllJobStatusResponseDto;
 	let setIntervalHandler: NodeJS.Timer;
+
 	onMount(async () => {
 		const { data } = await api.jobApi.getAllJobsStatus();
 		allJobsStatus = data;
@@ -104,6 +105,33 @@
 			});
 		}
 	};
+
+	const runTemplateMigration = async () => {
+		try {
+			const { data } = await api.jobApi.sendJobCommand(JobId.StorageTemplateMigration, {
+				command: JobCommand.Start
+			});
+
+			if (data) {
+				notificationController.show({
+					message: `Storage migration started`,
+					type: NotificationType.Info
+				});
+			} else {
+				notificationController.show({
+					message: `All files have been migrated to the new storage template`,
+					type: NotificationType.Info
+				});
+			}
+		} catch (e) {
+			console.log('[ERROR] runTemplateMigration', e);
+
+			notificationController.show({
+				message: `Error running template migration job, check console for more detail`,
+				type: NotificationType.Error
+			});
+		}
+	};
 </script>
 
 <div class="flex flex-col gap-10">
@@ -135,4 +163,20 @@
 	>
 		Note that some asset does not have any object detected, this is normal.
 	</JobTile>
+
+	<JobTile
+		title={'Storage migration'}
+		subtitle={''}
+		on:click={runTemplateMigration}
+		jobStatus={allJobsStatus?.isStorageMigrationActive}
+		waitingJobCount={allJobsStatus?.storageMigrationQueueCount.waiting}
+		activeJobCount={allJobsStatus?.storageMigrationQueueCount.active}
+	>
+		Apply the current
+		<a
+			href="/admin/system-settings?open=storage-template"
+			class="text-immich-primary dark:text-immich-dark-primary">Storage template</a
+		>
+		to previously uploaded assets
+	</JobTile>
 </div>

web/src/lib/components/admin-page/settings/setting-accordion.svelte

@@ -3,7 +3,7 @@
 	export let title: string;
 	export let subtitle = '';
 
-	let isOpen = false;
+	export let isOpen = false;
 	const toggle = () => (isOpen = !isOpen);
 </script>
 

web/src/lib/components/admin-page/settings/storate-template/storage-template-settings.svelte

@@ -214,6 +214,16 @@
 						</div>
 					</div>
 
+					<div id="migration-info" class="text-sm mt-4">
+						<p>
+							Template changes will only apply to new assets. To retroactively apply the template to
+							previously uploaded assets, run the <a
+								href="/admin/jobs-status"
+								class="text-immich-primary dark:text-immich-dark-primary">Storage Migration Job</a
+							>
+						</p>
+					</div>
+
 					<SettingButtonsRow
 						on:reset={reset}
 						on:save={saveSetting}

web/src/routes/admin/+layout.svelte

@@ -73,7 +73,7 @@
 			</div>
 
 			<section id="setting-content" class="pt-[85px] flex place-content-center">
-				<section class="w-[800px] pt-5">
+				<section class="w-[800px] pt-5 pb-28">
 					<slot />
 				</section>
 			</section>

web/src/routes/admin/system-settings/+page.svelte

@@ -6,6 +6,7 @@
 	import LoadingSpinner from '$lib/components/shared-components/loading-spinner.svelte';
 	import { api, SystemConfigDto } from '@api';
 	import type { PageData } from './$types';
+	import { page } from '$app/stores';
 
 	let systemConfig: SystemConfigDto;
 	export let data: PageData;
@@ -39,6 +40,7 @@
 		<SettingAccordion
 			title="Storage Template"
 			subtitle="Manage the folder structure and file name of the upload asset"
+			isOpen={$page.url.searchParams.get('open') === 'storage-template'}
 		>
 			<StorageTemplateSettings storageConfig={configs.storageTemplate} user={data.user} />
 		</SettingAccordion>