chore(server): Store generated files (thumbnails, encoded video) in subdirectories (#4112)

* save thumbnails in subdirectories

* migration job, migrate assets and face thumbnails

* fix tests

* directory depth of two instead of three

* cleanup empty dirs after migration

* clean up empty dirs after migration, migrate people without assetId

* add job card for new migration job

* fix removeEmptyDirs race condition because of missing await

* cleanup empty directories after asset deletion

* move ensurePath to storage core

* rename jobs

* remove unnecessary property of IEntityJob

* use updated person getById, minor refactoring

* ensure that directory cleanup doesn't interfere with migration

* better description for job in ui

* fix remove directories when migration is done

* cleanup empty folders at start of migration

* fix: actually persist concurrency setting

* add comment explaining regex

* chore: cleanup

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

cli/src/api/open-api/api.ts

@@ -307,6 +307,12 @@ export interface AllJobStatusResponseDto {
      * @memberof AllJobStatusResponseDto
      */
     'metadataExtraction': JobStatusDto;
+    /**
+     * 
+     * @type {JobStatusDto}
+     * @memberof AllJobStatusResponseDto
+     */
+    'migration': JobStatusDto;
     /**
      * 
      * @type {JobStatusDto}
@@ -1779,6 +1785,7 @@ export const JobName = {
     ClipEncoding: 'clipEncoding',
     BackgroundTask: 'backgroundTask',
     StorageTemplateMigration: 'storageTemplateMigration',
+    Migration: 'migration',
     Search: 'search',
     Sidecar: 'sidecar',
     Library: 'library'
@@ -3240,6 +3247,12 @@ export interface SystemConfigJobDto {
      * @memberof SystemConfigJobDto
      */
     'metadataExtraction': JobSettingsDto;
+    /**
+     * 
+     * @type {JobSettingsDto}
+     * @memberof SystemConfigJobDto
+     */
+    'migration': JobSettingsDto;
     /**
      * 
      * @type {JobSettingsDto}

mobile/openapi/doc/AllJobStatusResponseDto.md

@@ -12,6 +12,7 @@ Name | Type | Description | Notes
 **clipEncoding** | [**JobStatusDto**](JobStatusDto.md) |  | 
 **library_** | [**JobStatusDto**](JobStatusDto.md) |  | 
 **metadataExtraction** | [**JobStatusDto**](JobStatusDto.md) |  | 
+**migration** | [**JobStatusDto**](JobStatusDto.md) |  | 
 **objectTagging** | [**JobStatusDto**](JobStatusDto.md) |  | 
 **recognizeFaces** | [**JobStatusDto**](JobStatusDto.md) |  | 
 **search** | [**JobStatusDto**](JobStatusDto.md) |  | 

mobile/openapi/doc/SystemConfigJobDto.md

@@ -12,6 +12,7 @@ Name | Type | Description | Notes
 **clipEncoding** | [**JobSettingsDto**](JobSettingsDto.md) |  | 
 **library_** | [**JobSettingsDto**](JobSettingsDto.md) |  | 
 **metadataExtraction** | [**JobSettingsDto**](JobSettingsDto.md) |  | 
+**migration** | [**JobSettingsDto**](JobSettingsDto.md) |  | 
 **objectTagging** | [**JobSettingsDto**](JobSettingsDto.md) |  | 
 **recognizeFaces** | [**JobSettingsDto**](JobSettingsDto.md) |  | 
 **search** | [**JobSettingsDto**](JobSettingsDto.md) |  | 

mobile/openapi/lib/model/all_job_status_response_dto.dart

@@ -17,6 +17,7 @@ class AllJobStatusResponseDto {
     required this.clipEncoding,
     required this.library_,
     required this.metadataExtraction,
+    required this.migration,
     required this.objectTagging,
     required this.recognizeFaces,
     required this.search,
@@ -34,6 +35,8 @@ class AllJobStatusResponseDto {
 
   JobStatusDto metadataExtraction;
 
+  JobStatusDto migration;
+
   JobStatusDto objectTagging;
 
   JobStatusDto recognizeFaces;
@@ -54,6 +57,7 @@ class AllJobStatusResponseDto {
      other.clipEncoding == clipEncoding &&
      other.library_ == library_ &&
      other.metadataExtraction == metadataExtraction &&
+     other.migration == migration &&
      other.objectTagging == objectTagging &&
      other.recognizeFaces == recognizeFaces &&
      other.search == search &&
@@ -69,6 +73,7 @@ class AllJobStatusResponseDto {
     (clipEncoding.hashCode) +
     (library_.hashCode) +
     (metadataExtraction.hashCode) +
+    (migration.hashCode) +
     (objectTagging.hashCode) +
     (recognizeFaces.hashCode) +
     (search.hashCode) +
@@ -78,7 +83,7 @@ class AllJobStatusResponseDto {
     (videoConversion.hashCode);
 
   @override
-  String toString() => 'AllJobStatusResponseDto[backgroundTask=$backgroundTask, clipEncoding=$clipEncoding, library_=$library_, metadataExtraction=$metadataExtraction, objectTagging=$objectTagging, recognizeFaces=$recognizeFaces, search=$search, sidecar=$sidecar, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
+  String toString() => 'AllJobStatusResponseDto[backgroundTask=$backgroundTask, clipEncoding=$clipEncoding, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, objectTagging=$objectTagging, recognizeFaces=$recognizeFaces, search=$search, sidecar=$sidecar, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
@@ -86,6 +91,7 @@ class AllJobStatusResponseDto {
       json[r'clipEncoding'] = this.clipEncoding;
       json[r'library'] = this.library_;
       json[r'metadataExtraction'] = this.metadataExtraction;
+      json[r'migration'] = this.migration;
       json[r'objectTagging'] = this.objectTagging;
       json[r'recognizeFaces'] = this.recognizeFaces;
       json[r'search'] = this.search;
@@ -108,6 +114,7 @@ class AllJobStatusResponseDto {
         clipEncoding: JobStatusDto.fromJson(json[r'clipEncoding'])!,
         library_: JobStatusDto.fromJson(json[r'library'])!,
         metadataExtraction: JobStatusDto.fromJson(json[r'metadataExtraction'])!,
+        migration: JobStatusDto.fromJson(json[r'migration'])!,
         objectTagging: JobStatusDto.fromJson(json[r'objectTagging'])!,
         recognizeFaces: JobStatusDto.fromJson(json[r'recognizeFaces'])!,
         search: JobStatusDto.fromJson(json[r'search'])!,
@@ -166,6 +173,7 @@ class AllJobStatusResponseDto {
     'clipEncoding',
     'library',
     'metadataExtraction',
+    'migration',
     'objectTagging',
     'recognizeFaces',
     'search',

mobile/openapi/lib/model/job_name.dart

@@ -31,6 +31,7 @@ class JobName {
   static const clipEncoding = JobName._(r'clipEncoding');
   static const backgroundTask = JobName._(r'backgroundTask');
   static const storageTemplateMigration = JobName._(r'storageTemplateMigration');
+  static const migration = JobName._(r'migration');
   static const search = JobName._(r'search');
   static const sidecar = JobName._(r'sidecar');
   static const library_ = JobName._(r'library');
@@ -45,6 +46,7 @@ class JobName {
     clipEncoding,
     backgroundTask,
     storageTemplateMigration,
+    migration,
     search,
     sidecar,
     library_,
@@ -94,6 +96,7 @@ class JobNameTypeTransformer {
         case r'clipEncoding': return JobName.clipEncoding;
         case r'backgroundTask': return JobName.backgroundTask;
         case r'storageTemplateMigration': return JobName.storageTemplateMigration;
+        case r'migration': return JobName.migration;
         case r'search': return JobName.search;
         case r'sidecar': return JobName.sidecar;
         case r'library': return JobName.library_;

mobile/openapi/lib/model/system_config_job_dto.dart

@@ -17,6 +17,7 @@ class SystemConfigJobDto {
     required this.clipEncoding,
     required this.library_,
     required this.metadataExtraction,
+    required this.migration,
     required this.objectTagging,
     required this.recognizeFaces,
     required this.search,
@@ -34,6 +35,8 @@ class SystemConfigJobDto {
 
   JobSettingsDto metadataExtraction;
 
+  JobSettingsDto migration;
+
   JobSettingsDto objectTagging;
 
   JobSettingsDto recognizeFaces;
@@ -54,6 +57,7 @@ class SystemConfigJobDto {
      other.clipEncoding == clipEncoding &&
      other.library_ == library_ &&
      other.metadataExtraction == metadataExtraction &&
+     other.migration == migration &&
      other.objectTagging == objectTagging &&
      other.recognizeFaces == recognizeFaces &&
      other.search == search &&
@@ -69,6 +73,7 @@ class SystemConfigJobDto {
     (clipEncoding.hashCode) +
     (library_.hashCode) +
     (metadataExtraction.hashCode) +
+    (migration.hashCode) +
     (objectTagging.hashCode) +
     (recognizeFaces.hashCode) +
     (search.hashCode) +
@@ -78,7 +83,7 @@ class SystemConfigJobDto {
     (videoConversion.hashCode);
 
   @override
-  String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, clipEncoding=$clipEncoding, library_=$library_, metadataExtraction=$metadataExtraction, objectTagging=$objectTagging, recognizeFaces=$recognizeFaces, search=$search, sidecar=$sidecar, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
+  String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, clipEncoding=$clipEncoding, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, objectTagging=$objectTagging, recognizeFaces=$recognizeFaces, search=$search, sidecar=$sidecar, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
@@ -86,6 +91,7 @@ class SystemConfigJobDto {
       json[r'clipEncoding'] = this.clipEncoding;
       json[r'library'] = this.library_;
       json[r'metadataExtraction'] = this.metadataExtraction;
+      json[r'migration'] = this.migration;
       json[r'objectTagging'] = this.objectTagging;
       json[r'recognizeFaces'] = this.recognizeFaces;
       json[r'search'] = this.search;
@@ -108,6 +114,7 @@ class SystemConfigJobDto {
         clipEncoding: JobSettingsDto.fromJson(json[r'clipEncoding'])!,
         library_: JobSettingsDto.fromJson(json[r'library'])!,
         metadataExtraction: JobSettingsDto.fromJson(json[r'metadataExtraction'])!,
+        migration: JobSettingsDto.fromJson(json[r'migration'])!,
         objectTagging: JobSettingsDto.fromJson(json[r'objectTagging'])!,
         recognizeFaces: JobSettingsDto.fromJson(json[r'recognizeFaces'])!,
         search: JobSettingsDto.fromJson(json[r'search'])!,
@@ -166,6 +173,7 @@ class SystemConfigJobDto {
     'clipEncoding',
     'library',
     'metadataExtraction',
+    'migration',
     'objectTagging',
     'recognizeFaces',
     'search',

mobile/openapi/test/all_job_status_response_dto_test.dart

@@ -36,6 +36,11 @@ void main() {
       // TODO
     });
 
+    // JobStatusDto migration
+    test('to test the property `migration`', () async {
+      // TODO
+    });
+
     // JobStatusDto objectTagging
     test('to test the property `objectTagging`', () async {
       // TODO

mobile/openapi/test/system_config_job_dto_test.dart

@@ -36,6 +36,11 @@ void main() {
       // TODO
     });
 
+    // JobSettingsDto migration
+    test('to test the property `migration`', () async {
+      // TODO
+    });
+
     // JobSettingsDto objectTagging
     test('to test the property `objectTagging`', () async {
       // TODO

server/immich-openapi-specs.json

@@ -5343,6 +5343,9 @@
           "metadataExtraction": {
             "$ref": "#/components/schemas/JobStatusDto"
           },
+          "migration": {
+            "$ref": "#/components/schemas/JobStatusDto"
+          },
           "objectTagging": {
             "$ref": "#/components/schemas/JobStatusDto"
           },
@@ -5372,6 +5375,7 @@
           "objectTagging",
           "clipEncoding",
           "storageTemplateMigration",
+          "migration",
           "backgroundTask",
           "search",
           "recognizeFaces",
@@ -6535,6 +6539,7 @@
           "clipEncoding",
           "backgroundTask",
           "storageTemplateMigration",
+          "migration",
           "search",
           "sidecar",
           "library"
@@ -7693,6 +7698,9 @@
           "metadataExtraction": {
             "$ref": "#/components/schemas/JobSettingsDto"
           },
+          "migration": {
+            "$ref": "#/components/schemas/JobSettingsDto"
+          },
           "objectTagging": {
             "$ref": "#/components/schemas/JobSettingsDto"
           },
@@ -7722,6 +7730,7 @@
           "objectTagging",
           "clipEncoding",
           "storageTemplateMigration",
+          "migration",
           "backgroundTask",
           "search",
           "recognizeFaces",

server/src/domain/asset/asset.service.ts

@@ -57,7 +57,7 @@ export interface UploadFile {
 export class AssetService {
   private logger = new Logger(AssetService.name);
   private access: AccessCore;
-  private storageCore = new StorageCore();
+  private storageCore: StorageCore;
 
   constructor(
     @Inject(IAccessRepository) accessRepository: IAccessRepository,
@@ -67,6 +67,7 @@ export class AssetService {
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
   ) {
     this.access = new AccessCore(accessRepository);
+    this.storageCore = new StorageCore(storageRepository);
   }
 
   canUploadFile({ authUser, fieldName, file }: UploadRequest): true {

server/src/domain/facial-recognition/facial-recognition.service.spec.ts

@@ -307,14 +307,14 @@ describe(FacialRecognitionService.name, () => {
       await sut.handleGenerateFaceThumbnail(face.middle);
 
       expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1']);
-      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/pe/rs');
       expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', {
         left: 95,
         top: 95,
         width: 110,
         height: 110,
       });
-      expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
+      expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', {
         format: 'jpeg',
         size: 250,
         quality: 80,
@@ -323,7 +323,7 @@ describe(FacialRecognitionService.name, () => {
       expect(personMock.update).toHaveBeenCalledWith({
         faceAssetId: 'asset-1',
         id: 'person-1',
-        thumbnailPath: 'upload/thumbs/user-id/person-1.jpeg',
+        thumbnailPath: 'upload/thumbs/user-id/pe/rs/person-1.jpeg',
       });
     });
 
@@ -338,7 +338,7 @@ describe(FacialRecognitionService.name, () => {
         width: 510,
         height: 510,
       });
-      expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
+      expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', {
         format: 'jpeg',
         size: 250,
         quality: 80,
@@ -357,7 +357,7 @@ describe(FacialRecognitionService.name, () => {
         width: 202,
         height: 202,
       });
-      expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
+      expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', {
         format: 'jpeg',
         size: 250,
         quality: 80,

server/src/domain/facial-recognition/facial-recognition.services.ts

@@ -1,5 +1,4 @@
 import { Inject, Logger } from '@nestjs/common';
-import { join } from 'path';
 import { IAssetRepository, WithoutProperty } from '../asset';
 import { usePagination } from '../domain.util';
 import { IBaseJob, IEntityJob, IFaceThumbnailJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
@@ -13,8 +12,8 @@ import { AssetFaceId, IFaceRepository } from './face.repository';
 
 export class FacialRecognitionService {
   private logger = new Logger(FacialRecognitionService.name);
-  private storageCore = new StorageCore();
   private configCore: SystemConfigCore;
+  private storageCore: StorageCore;
 
   constructor(
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
@@ -28,6 +27,7 @@ export class FacialRecognitionService {
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
   ) {
     this.configCore = new SystemConfigCore(configRepository);
+    this.storageCore = new StorageCore(storageRepository);
   }
 
   async handleQueueRecognizeFaces({ force }: IBaseJob) {
@@ -117,6 +117,21 @@ export class FacialRecognitionService {
     return true;
   }
 
+  async handlePersonMigration({ id }: IEntityJob) {
+    const person = await this.personRepository.getById(id);
+    if (!person) {
+      return false;
+    }
+
+    const path = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, person.ownerId, `${id}.jpeg`);
+    if (person.thumbnailPath && person.thumbnailPath !== path) {
+      await this.storageRepository.moveFile(person.thumbnailPath, path);
+      await this.personRepository.update({ id, thumbnailPath: path });
+    }
+
+    return true;
+  }
+
   async handleGenerateFaceThumbnail(data: IFaceThumbnailJob) {
     const { machineLearning } = await this.configCore.getConfig();
     if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) {
@@ -132,9 +147,7 @@ export class FacialRecognitionService {
 
     this.logger.verbose(`Cropping face for person: ${personId}`);
 
-    const outputFolder = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId);
+    const output = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${personId}.jpeg`);
-    const output = join(outputFolder, `${personId}.jpeg`);
-    this.storageRepository.mkdirSync(outputFolder);
 
     const { x1, y1, x2, y2 } = boundingBox;
 

server/src/domain/job/job.constants.ts

@@ -7,6 +7,7 @@ export enum QueueName {
   CLIP_ENCODING = 'clipEncoding',
   BACKGROUND_TASK = 'backgroundTask',
   STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration',
+  MIGRATION = 'migration',
   SEARCH = 'search',
   SIDECAR = 'sidecar',
   LIBRARY = 'library',
@@ -45,6 +46,11 @@ export enum JobName {
   STORAGE_TEMPLATE_MIGRATION_SINGLE = 'storage-template-migration-single',
   SYSTEM_CONFIG_CHANGE = 'system-config-change',
 
+  // migration
+  QUEUE_MIGRATION = 'queue-migration',
+  MIGRATE_ASSET = 'migrate-asset',
+  MIGRATE_PERSON = 'migrate-person',
+
   // object tagging
   QUEUE_OBJECT_TAGGING = 'queue-object-tagging',
   CLASSIFY_IMAGE = 'classify-image',
@@ -119,6 +125,11 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
   [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: QueueName.STORAGE_TEMPLATE_MIGRATION,
   [JobName.SYSTEM_CONFIG_CHANGE]: QueueName.STORAGE_TEMPLATE_MIGRATION,
 
+  // migration
+  [JobName.QUEUE_MIGRATION]: QueueName.MIGRATION,
+  [JobName.MIGRATE_ASSET]: QueueName.MIGRATION,
+  [JobName.MIGRATE_PERSON]: QueueName.MIGRATION,
+
   // object tagging
   [JobName.QUEUE_OBJECT_TAGGING]: QueueName.OBJECT_TAGGING,
   [JobName.CLASSIFY_IMAGE]: QueueName.OBJECT_TAGGING,

server/src/domain/job/job.dto.ts

@@ -68,6 +68,9 @@ export class AllJobStatusResponseDto implements Record<QueueName, JobStatusDto>
   @ApiProperty({ type: JobStatusDto })
   [QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobStatusDto;
 
+  @ApiProperty({ type: JobStatusDto })
+  [QueueName.MIGRATION]!: JobStatusDto;
+
   @ApiProperty({ type: JobStatusDto })
   [QueueName.BACKGROUND_TASK]!: JobStatusDto;
 

server/src/domain/job/job.repository.ts

@@ -46,6 +46,11 @@ export type JobItem =
   | { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE; data: IEntityJob }
   | { name: JobName.SYSTEM_CONFIG_CHANGE; data?: IBaseJob }
 
+  // Migration
+  | { name: JobName.QUEUE_MIGRATION; data?: IBaseJob }
+  | { name: JobName.MIGRATE_ASSET; data?: IEntityJob }
+  | { name: JobName.MIGRATE_PERSON; data?: IEntityJob }
+
   // Metadata Extraction
   | { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob }
   | { name: JobName.METADATA_EXTRACTION; data: IEntityJob }

server/src/domain/job/job.service.spec.ts

@@ -94,6 +94,7 @@ describe(JobService.name, () => {
         [QueueName.OBJECT_TAGGING]: expectedJobStatus,
         [QueueName.SEARCH]: expectedJobStatus,
         [QueueName.STORAGE_TEMPLATE_MIGRATION]: expectedJobStatus,
+        [QueueName.MIGRATION]: expectedJobStatus,
         [QueueName.THUMBNAIL_GENERATION]: expectedJobStatus,
         [QueueName.VIDEO_CONVERSION]: expectedJobStatus,
         [QueueName.RECOGNIZE_FACES]: expectedJobStatus,
@@ -229,6 +230,7 @@ describe(JobService.name, () => {
           [QueueName.SIDECAR]: { concurrency: 10 },
           [QueueName.LIBRARY]: { concurrency: 10 },
           [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 },
+          [QueueName.MIGRATION]: { concurrency: 10 },
           [QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 },
           [QueueName.VIDEO_CONVERSION]: { concurrency: 10 },
         },
@@ -242,6 +244,7 @@ describe(JobService.name, () => {
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10);
+      expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.MIGRATION, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10);
       expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10);
     });

server/src/domain/job/job.service.ts

@@ -76,6 +76,9 @@ export class JobService {
       case QueueName.STORAGE_TEMPLATE_MIGRATION:
         return this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION });
 
+      case QueueName.MIGRATION:
+        return this.jobRepository.queue({ name: JobName.QUEUE_MIGRATION });
+
       case QueueName.OBJECT_TAGGING:
         await this.configCore.requireFeature(FeatureFlag.TAG_IMAGE);
         return this.jobRepository.queue({ name: JobName.QUEUE_OBJECT_TAGGING, data: { force } });

server/src/domain/media/media.service.spec.ts

@@ -202,8 +202,8 @@ describe(MediaService.name, () => {
       assetMock.getByIds.mockResolvedValue([assetStub.image]);
       await sut.handleGenerateJpegThumbnail({ id: assetStub.image.id });
 
-      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se');
-      expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.jpeg', {
+      expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/as/se/asset-id.jpeg', {
         size: 1440,
         format: 'jpeg',
         quality: 80,
@@ -211,7 +211,7 @@ describe(MediaService.name, () => {
       });
       expect(assetMock.save).toHaveBeenCalledWith({
         id: 'asset-id',
-        resizePath: 'upload/thumbs/user-id/asset-id.jpeg',
+        resizePath: 'upload/thumbs/user-id/as/se/asset-id.jpeg',
       });
     });
 
@@ -220,19 +220,23 @@ describe(MediaService.name, () => {
       assetMock.getByIds.mockResolvedValue([assetStub.video]);
       await sut.handleGenerateJpegThumbnail({ id: assetStub.video.id });
 
-      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se');
-      expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
-        inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
+        '/original/path.ext',
-        outputOptions: [
+        'upload/thumbs/user-id/as/se/asset-id.jpeg',
-          '-frames:v 1',
+        {
-          '-v verbose',
+          inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
-          '-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p',
+          outputOptions: [
-        ],
+            '-frames:v 1',
-        twoPass: false,
+            '-v verbose',
-      });
+            '-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p',
+          ],
+          twoPass: false,
+        },
+      );
       expect(assetMock.save).toHaveBeenCalledWith({
         id: 'asset-id',
-        resizePath: 'upload/thumbs/user-id/asset-id.jpeg',
+        resizePath: 'upload/thumbs/user-id/as/se/asset-id.jpeg',
       });
     });
 
@@ -241,19 +245,23 @@ describe(MediaService.name, () => {
       assetMock.getByIds.mockResolvedValue([assetStub.video]);
       await sut.handleGenerateJpegThumbnail({ id: assetStub.video.id });
 
-      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se');
-      expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
-        inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
+        '/original/path.ext',
-        outputOptions: [
+        'upload/thumbs/user-id/as/se/asset-id.jpeg',
-          '-frames:v 1',
+        {
-          '-v verbose',
+          inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
-          '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p',
+          outputOptions: [
-        ],
+            '-frames:v 1',
-        twoPass: false,
+            '-v verbose',
-      });
+            '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p',
+          ],
+          twoPass: false,
+        },
+      );
       expect(assetMock.save).toHaveBeenCalledWith({
         id: 'asset-id',
-        resizePath: 'upload/thumbs/user-id/asset-id.jpeg',
+        resizePath: 'upload/thumbs/user-id/as/se/asset-id.jpeg',
       });
     });
 
@@ -275,13 +283,16 @@ describe(MediaService.name, () => {
       assetMock.getByIds.mockResolvedValue([assetStub.image]);
       await sut.handleGenerateWebpThumbnail({ id: assetStub.image.id });
 
-      expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.webp', {
+      expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/as/se/asset-id.webp', {
         format: 'webp',
         size: 250,
         quality: 80,
         colorspace: Colorspace.P3,
       });
-      expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: 'upload/thumbs/user-id/asset-id.webp' });
+      expect(assetMock.save).toHaveBeenCalledWith({
+        id: 'asset-id',
+        webpPath: 'upload/thumbs/user-id/as/se/asset-id.webp',
+      });
     });
   });
 
@@ -375,7 +386,7 @@ describe(MediaService.name, () => {
       expect(storageMock.mkdirSync).toHaveBeenCalled();
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -416,7 +427,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -442,7 +453,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -471,7 +482,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -498,7 +509,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -525,7 +536,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -552,7 +563,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -603,7 +614,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -635,7 +646,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -664,7 +675,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -695,7 +706,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -728,7 +739,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -760,7 +771,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -791,7 +802,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -821,7 +832,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -851,7 +862,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -881,7 +892,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -914,7 +925,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -976,7 +987,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
           outputOptions: [
@@ -1014,7 +1025,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
           outputOptions: [
@@ -1048,7 +1059,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
           outputOptions: [
@@ -1083,7 +1094,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
           outputOptions: [
@@ -1114,7 +1125,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
           outputOptions: [
@@ -1150,7 +1161,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
           outputOptions: [
@@ -1186,7 +1197,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
           outputOptions: [
@@ -1219,7 +1230,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
           outputOptions: [
@@ -1263,7 +1274,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
           outputOptions: [
@@ -1295,7 +1306,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
           outputOptions: [
@@ -1329,7 +1340,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
           outputOptions: [
@@ -1359,7 +1370,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/card1', '-filter_hw_device accel'],
           outputOptions: [
@@ -1385,7 +1396,7 @@ describe(MediaService.name, () => {
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD129', '-filter_hw_device accel'],
           outputOptions: [
@@ -1418,7 +1429,7 @@ describe(MediaService.name, () => {
       expect(mediaMock.transcode).toHaveBeenCalledTimes(2);
       expect(mediaMock.transcode).toHaveBeenLastCalledWith(
         '/original/path.ext',
-        'upload/encoded-video/user-id/asset-id.mp4',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
         {
           inputOptions: [],
           outputOptions: [
@@ -1455,7 +1466,7 @@ describe(MediaService.name, () => {
     await sut.handleVideoConversion({ id: assetStub.video.id });
     expect(mediaMock.transcode).toHaveBeenCalledWith(
       '/original/path.ext',
-      'upload/encoded-video/user-id/asset-id.mp4',
+      'upload/encoded-video/user-id/as/se/asset-id.mp4',
       {
         inputOptions: [],
         outputOptions: [
@@ -1482,7 +1493,7 @@ describe(MediaService.name, () => {
     await sut.handleVideoConversion({ id: assetStub.video.id });
     expect(mediaMock.transcode).toHaveBeenCalledWith(
       '/original/path.ext',
-      'upload/encoded-video/user-id/asset-id.mp4',
+      'upload/encoded-video/user-id/as/se/asset-id.mp4',
       {
         inputOptions: [],
         outputOptions: [
@@ -1509,7 +1520,7 @@ describe(MediaService.name, () => {
     await sut.handleVideoConversion({ id: assetStub.video.id });
     expect(mediaMock.transcode).toHaveBeenCalledWith(
       '/original/path.ext',
-      'upload/encoded-video/user-id/asset-id.mp4',
+      'upload/encoded-video/user-id/as/se/asset-id.mp4',
       {
         inputOptions: [],
         outputOptions: [

server/src/domain/media/media.service.ts

@@ -1,9 +1,8 @@
 import { AssetEntity, AssetType, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities';
 import { Inject, Injectable, Logger, UnsupportedMediaTypeException } from '@nestjs/common';
-import { join } from 'path';
 import { IAssetRepository, WithoutProperty } from '../asset';
 import { usePagination } from '../domain.util';
-import { IBaseJob, IEntityJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
+import { IBaseJob, IEntityJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job';
 import { IPersonRepository } from '../person';
 import { IStorageRepository, StorageCore, StorageFolder } from '../storage';
 import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config';
@@ -14,8 +13,8 @@ import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIC
 @Injectable()
 export class MediaService {
   private logger = new Logger(MediaService.name);
-  private storageCore = new StorageCore();
   private configCore: SystemConfigCore;
+  private storageCore: StorageCore;
 
   constructor(
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
@@ -26,11 +25,10 @@ export class MediaService {
     @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
   ) {
     this.configCore = new SystemConfigCore(configRepository);
+    this.storageCore = new StorageCore(this.storageRepository);
   }
 
-  async handleQueueGenerateThumbnails(job: IBaseJob) {
+  async handleQueueGenerateThumbnails({ force }: IBaseJob) {
-    const { force } = job;
-
     const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
       return force
         ? this.assetRepository.getAll(pagination)
@@ -81,6 +79,58 @@ export class MediaService {
     return true;
   }
 
+  async handleQueueMigration() {
+    const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
+      this.assetRepository.getAll(pagination),
+    );
+
+    const { active, waiting } = await this.jobRepository.getJobCounts(QueueName.MIGRATION);
+    if (active === 1 && waiting === 0) {
+      await this.storageCore.removeEmptyDirs(StorageFolder.THUMBNAILS);
+      await this.storageCore.removeEmptyDirs(StorageFolder.ENCODED_VIDEO);
+    }
+
+    for await (const assets of assetPagination) {
+      for (const asset of assets) {
+        await this.jobRepository.queue({ name: JobName.MIGRATE_ASSET, data: { id: asset.id } });
+      }
+    }
+
+    const people = await this.personRepository.getAll();
+    for (const person of people) {
+      await this.jobRepository.queue({ name: JobName.MIGRATE_PERSON, data: { id: person.id } });
+    }
+
+    return true;
+  }
+
+  async handleAssetMigration({ id }: IEntityJob) {
+    const [asset] = await this.assetRepository.getByIds([id]);
+    if (!asset) {
+      return false;
+    }
+    const resizePath = this.ensureThumbnailPath(asset, 'jpeg');
+    const webpPath = this.ensureThumbnailPath(asset, 'webp');
+    const encodedVideoPath = this.ensureEncodedVideoPath(asset, 'mp4');
+
+    if (asset.resizePath && asset.resizePath !== resizePath) {
+      await this.storageRepository.moveFile(asset.resizePath, resizePath);
+      await this.assetRepository.save({ id: asset.id, resizePath });
+    }
+
+    if (asset.webpPath && asset.webpPath !== webpPath) {
+      await this.storageRepository.moveFile(asset.webpPath, webpPath);
+      await this.assetRepository.save({ id: asset.id, webpPath });
+    }
+
+    if (asset.encodedVideoPath && asset.encodedVideoPath !== encodedVideoPath) {
+      await this.storageRepository.moveFile(asset.encodedVideoPath, encodedVideoPath);
+      await this.assetRepository.save({ id: asset.id, encodedVideoPath });
+    }
+
+    return true;
+  }
+
   async handleGenerateJpegThumbnail({ id }: IEntityJob) {
     const [asset] = await this.assetRepository.getByIds([id]);
     if (!asset) {
@@ -184,9 +234,7 @@ export class MediaService {
     }
 
     const input = asset.originalPath;
-    const outputFolder = this.storageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, asset.ownerId);
+    const output = this.ensureEncodedVideoPath(asset, 'mp4');
-    const output = join(outputFolder, `${asset.id}.mp4`);
-    this.storageRepository.mkdirSync(outputFolder);
 
     const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input);
     const mainVideoStream = this.getMainStream(videoStreams);
@@ -330,8 +378,10 @@ export class MediaService {
   }
 
   ensureThumbnailPath(asset: AssetEntity, extension: string): string {
-    const folderPath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId);
+    return this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.${extension}`);
-    this.storageRepository.mkdirSync(folderPath);
+  }
-    return join(folderPath, `${asset.id}.${extension}`);
+
+  ensureEncodedVideoPath(asset: AssetEntity, extension: string): string {
+    return this.storageCore.ensurePath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}.${extension}`);
   }
 }

server/src/domain/person/person.service.spec.ts

@@ -37,10 +37,10 @@ describe(PersonService.name, () => {
 
   beforeEach(async () => {
     accessMock = newAccessRepositoryMock();
-    personMock = newPersonRepositoryMock();
-    storageMock = newStorageRepositoryMock();
     configMock = newSystemConfigRepositoryMock();
     jobMock = newJobRepositoryMock();
+    personMock = newPersonRepositoryMock();
+    storageMock = newStorageRepositoryMock();
     sut = new PersonService(accessMock, personMock, configMock, storageMock, jobMock);
   });
 

server/src/domain/server-info/server-info.service.ts

@@ -16,8 +16,8 @@ import {
 
 @Injectable()
 export class ServerInfoService {
-  private storageCore = new StorageCore();
   private configCore: SystemConfigCore;
+  private storageCore: StorageCore;
 
   constructor(
     @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@@ -25,6 +25,7 @@ export class ServerInfoService {
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
   ) {
     this.configCore = new SystemConfigCore(configRepository);
+    this.storageCore = new StorageCore(storageRepository);
   }
 
   async getInfo(): Promise<ServerInfoResponseDto> {

server/src/domain/storage-template/storage-template.service.ts

@@ -30,7 +30,7 @@ export interface MoveAssetMetadata {
 export class StorageTemplateService {
   private logger = new Logger(StorageTemplateService.name);
   private configCore: SystemConfigCore;
-  private storageCore = new StorageCore();
+  private storageCore: StorageCore;
   private storageTemplate: HandlebarsTemplateDelegate<any>;
 
   constructor(
@@ -44,6 +44,7 @@ export class StorageTemplateService {
     this.configCore = new SystemConfigCore(configRepository);
     this.configCore.addValidator((config) => this.validate(config));
     this.configCore.config$.subscribe((config) => this.onConfig(config));
+    this.storageCore = new StorageCore(storageRepository);
   }
 
   async handleMigrationSingle({ id }: IEntityJob) {

server/src/domain/storage/storage.core.ts

@@ -1,5 +1,6 @@
 import { join } from 'node:path';
 import { APP_MEDIA_LOCATION } from '../domain.constant';
+import { IStorageRepository } from './storage.repository';
 
 export enum StorageFolder {
   ENCODED_VIDEO = 'encoded-video',
@@ -10,6 +11,8 @@ export enum StorageFolder {
 }
 
 export class StorageCore {
+  constructor(private repository: IStorageRepository) {}
+
   getFolderLocation(
     folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS,
     userId: string,
@@ -24,4 +27,22 @@ export class StorageCore {
   getBaseFolder(folder: StorageFolder) {
     return join(APP_MEDIA_LOCATION, folder);
   }
+
+  ensurePath(
+    folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS,
+    ownerId: string,
+    fileName: string,
+  ): string {
+    const folderPath = join(
+      this.getFolderLocation(folder, ownerId),
+      fileName.substring(0, 2),
+      fileName.substring(2, 4),
+    );
+    this.repository.mkdirSync(folderPath);
+    return join(folderPath, fileName);
+  }
+
+  removeEmptyDirs(folder: StorageFolder) {
+    return this.repository.removeEmptyDirs(this.getBaseFolder(folder));
+  }
 }

server/src/domain/storage/storage.repository.ts

@@ -26,7 +26,7 @@ export interface IStorageRepository {
   createReadStream(filepath: string, mimeType?: string | null): Promise<ImmichReadStream>;
   unlink(filepath: string): Promise<void>;
   unlinkDir(folder: string, options?: { recursive?: boolean; force?: boolean }): Promise<void>;
-  removeEmptyDirs(folder: string): Promise<void>;
+  removeEmptyDirs(folder: string, self?: boolean): Promise<void>;
   moveFile(source: string, target: string): Promise<void>;
   checkFileExists(filepath: string, mode?: number): Promise<boolean>;
   mkdirSync(filepath: string): void;

server/src/domain/storage/storage.service.ts

@@ -6,9 +6,11 @@ import { IStorageRepository } from './storage.repository';
 @Injectable()
 export class StorageService {
   private logger = new Logger(StorageService.name);
-  private storageCore = new StorageCore();
+  private storageCore: StorageCore;
 
-  constructor(@Inject(IStorageRepository) private storageRepository: IStorageRepository) {}
+  constructor(@Inject(IStorageRepository) private storageRepository: IStorageRepository) {
+    this.storageCore = new StorageCore(storageRepository);
+  }
 
   init() {
     const libraryBase = this.storageCore.getBaseFolder(StorageFolder.LIBRARY);

server/src/domain/system-config/dto/system-config-job.dto.ts

@@ -47,6 +47,12 @@ export class SystemConfigJobDto implements Record<QueueName, JobSettingsDto> {
   @Type(() => JobSettingsDto)
   [QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobSettingsDto;
 
+  @ApiProperty({ type: JobSettingsDto })
+  @ValidateNested()
+  @IsObject()
+  @Type(() => JobSettingsDto)
+  [QueueName.MIGRATION]!: JobSettingsDto;
+
   @ApiProperty({ type: JobSettingsDto })
   @ValidateNested()
   @IsObject()

server/src/domain/system-config/system-config.core.ts

@@ -53,6 +53,7 @@ export const defaults = Object.freeze<SystemConfig>({
     [QueueName.SIDECAR]: { concurrency: 5 },
     [QueueName.LIBRARY]: { concurrency: 1 },
     [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
+    [QueueName.MIGRATION]: { concurrency: 5 },
     [QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
     [QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
   },

server/src/domain/system-config/system-config.service.spec.ts

@@ -33,6 +33,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
     [QueueName.SIDECAR]: { concurrency: 5 },
     [QueueName.LIBRARY]: { concurrency: 1 },
     [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 },
+    [QueueName.MIGRATION]: { concurrency: 5 },
     [QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 },
     [QueueName.VIDEO_CONVERSION]: { concurrency: 1 },
   },

server/src/domain/user/user.service.ts

@@ -25,8 +25,8 @@ import { IUserRepository } from './user.repository';
 @Injectable()
 export class UserService {
   private logger = new Logger(UserService.name);
+  private storageCore: StorageCore;
   private userCore: UserCore;
-  private storageCore = new StorageCore();
 
   constructor(
     @Inject(IUserRepository) private userRepository: IUserRepository,
@@ -37,6 +37,7 @@ export class UserService {
     @Inject(IJobRepository) private jobRepository: IJobRepository,
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
   ) {
+    this.storageCore = new StorageCore(storageRepository);
     this.userCore = new UserCore(userRepository, libraryRepository, cryptoRepository);
   }
 

server/src/infra/entities/system-config.entity.ts

@@ -43,6 +43,7 @@ export enum SystemConfigKey {
   JOB_SEARCH_CONCURRENCY = 'job.search.concurrency',
   JOB_SIDECAR_CONCURRENCY = 'job.sidecar.concurrency',
   JOB_LIBRARY_CONCURRENCY = 'job.library.concurrency',
+  JOB_MIGRATION_CONCURRENCY = 'job.migration.concurrency',
 
   MACHINE_LEARNING_ENABLED = 'machineLearning.enabled',
   MACHINE_LEARNING_URL = 'machineLearning.url',

server/src/infra/repositories/filesystem.provider.ts

@@ -66,11 +66,7 @@ export class FilesystemProvider implements IStorageRepository {
     await fs.rm(folder, options);
   }
 
-  async removeEmptyDirs(directory: string) {
+  async removeEmptyDirs(directory: string, self: boolean = false) {
-    this._removeEmptyDirs(directory, false);
-  }
-
-  private async _removeEmptyDirs(directory: string, self: boolean) {
     // lstat does not follow symlinks (in contrast to stat)
     const stats = await fs.lstat(directory);
     if (!stats.isDirectory()) {
@@ -78,7 +74,7 @@ export class FilesystemProvider implements IStorageRepository {
     }
 
     const files = await fs.readdir(directory);
-    await Promise.all(files.map((file) => this._removeEmptyDirs(path.join(directory, file), true)));
+    await Promise.all(files.map((file) => this.removeEmptyDirs(path.join(directory, file), true)));
 
     if (self) {
       const updated = await fs.readdir(directory);

server/src/microservices/app.service.ts

@@ -63,6 +63,9 @@ export class AppService {
       [JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data),
       [JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(),
       [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data),
+      [JobName.QUEUE_MIGRATION]: () => this.mediaService.handleQueueMigration(),
+      [JobName.MIGRATE_ASSET]: (data) => this.mediaService.handleAssetMigration(data),
+      [JobName.MIGRATE_PERSON]: (data) => this.facialRecognitionService.handlePersonMigration(data),
       [JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(),
       [JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data),
       [JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data),

server/src/microservices/processors/metadata-extraction.processor.ts

@@ -50,7 +50,7 @@ const validate = <T>(value: T): T | null => (typeof value === 'string' ? null :
 export class MetadataExtractionProcessor {
   private logger = new Logger(MetadataExtractionProcessor.name);
   private reverseGeocodingEnabled: boolean;
-  private storageCore = new StorageCore();
+  private storageCore: StorageCore;
 
   constructor(
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
@@ -63,6 +63,7 @@ export class MetadataExtractionProcessor {
     configService: ConfigService,
   ) {
     this.reverseGeocodingEnabled = !configService.get('DISABLE_REVERSE_GEOCODING');
+    this.storageCore = new StorageCore(storageRepository);
   }
 
   async init(deleteCache = false) {

web/src/api/api.ts

@@ -131,6 +131,7 @@ export class ImmichApi {
       [JobName.RecognizeFaces]: 'Recognize Faces',
       [JobName.VideoConversion]: 'Transcode Videos',
       [JobName.StorageTemplateMigration]: 'Storage Template Migration',
+      [JobName.Migration]: 'Migration',
       [JobName.BackgroundTask]: 'Background Tasks',
       [JobName.Search]: 'Search',
       [JobName.Library]: 'Library',

web/src/api/open-api/api.ts

@@ -307,6 +307,12 @@ export interface AllJobStatusResponseDto {
      * @memberof AllJobStatusResponseDto
      */
     'metadataExtraction': JobStatusDto;
+    /**
+     * 
+     * @type {JobStatusDto}
+     * @memberof AllJobStatusResponseDto
+     */
+    'migration': JobStatusDto;
     /**
      * 
      * @type {JobStatusDto}
@@ -1779,6 +1785,7 @@ export const JobName = {
     ClipEncoding: 'clipEncoding',
     BackgroundTask: 'backgroundTask',
     StorageTemplateMigration: 'storageTemplateMigration',
+    Migration: 'migration',
     Search: 'search',
     Sidecar: 'sidecar',
     Library: 'library'
@@ -3240,6 +3247,12 @@ export interface SystemConfigJobDto {
      * @memberof SystemConfigJobDto
      */
     'metadataExtraction': JobSettingsDto;
+    /**
+     * 
+     * @type {JobSettingsDto}
+     * @memberof SystemConfigJobDto
+     */
+    'migration': JobSettingsDto;
     /**
      * 
      * @type {JobSettingsDto}

web/src/lib/components/admin-page/jobs/jobs-panel.svelte

@@ -110,6 +110,12 @@
       allowForceCommand: false,
       component: StorageMigrationDescription,
     },
+    [JobName.Migration]: {
+      icon: FolderMove,
+      title: api.getJobName(JobName.Migration),
+      subtitle: 'Migrate thumbnails for assets and faces to the latest folder structure',
+      allowForceCommand: false,
+    },
   };
   $: jobList = Object.entries(jobDetails) as [JobName, JobDetails][];