diff --git a/cli/src/api/open-api/api.ts b/cli/src/api/open-api/api.ts index 66085a52f0..e39b6e4f12 100644 --- a/cli/src/api/open-api/api.ts +++ b/cli/src/api/open-api/api.ts @@ -307,6 +307,12 @@ export interface AllJobStatusResponseDto { * @memberof AllJobStatusResponseDto */ 'metadataExtraction': JobStatusDto; + /** + * + * @type {JobStatusDto} + * @memberof AllJobStatusResponseDto + */ + 'migration': JobStatusDto; /** * * @type {JobStatusDto} @@ -1779,6 +1785,7 @@ export const JobName = { ClipEncoding: 'clipEncoding', BackgroundTask: 'backgroundTask', StorageTemplateMigration: 'storageTemplateMigration', + Migration: 'migration', Search: 'search', Sidecar: 'sidecar', Library: 'library' @@ -3240,6 +3247,12 @@ export interface SystemConfigJobDto { * @memberof SystemConfigJobDto */ 'metadataExtraction': JobSettingsDto; + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'migration': JobSettingsDto; /** * * @type {JobSettingsDto} diff --git a/mobile/openapi/doc/AllJobStatusResponseDto.md b/mobile/openapi/doc/AllJobStatusResponseDto.md index d2e64304fc..71b50efbe0 100644 Binary files a/mobile/openapi/doc/AllJobStatusResponseDto.md and b/mobile/openapi/doc/AllJobStatusResponseDto.md differ diff --git a/mobile/openapi/doc/SystemConfigJobDto.md b/mobile/openapi/doc/SystemConfigJobDto.md index d2738d19d5..5660b245ee 100644 Binary files a/mobile/openapi/doc/SystemConfigJobDto.md and b/mobile/openapi/doc/SystemConfigJobDto.md differ diff --git a/mobile/openapi/lib/model/all_job_status_response_dto.dart b/mobile/openapi/lib/model/all_job_status_response_dto.dart index 65736edcf5..4d3008a3bc 100644 Binary files a/mobile/openapi/lib/model/all_job_status_response_dto.dart and b/mobile/openapi/lib/model/all_job_status_response_dto.dart differ diff --git a/mobile/openapi/lib/model/job_name.dart b/mobile/openapi/lib/model/job_name.dart index 27369551a0..c1008d0791 100644 Binary files a/mobile/openapi/lib/model/job_name.dart and b/mobile/openapi/lib/model/job_name.dart differ diff --git a/mobile/openapi/lib/model/system_config_job_dto.dart b/mobile/openapi/lib/model/system_config_job_dto.dart index a2f1695713..d7ff7ea6fb 100644 Binary files a/mobile/openapi/lib/model/system_config_job_dto.dart and b/mobile/openapi/lib/model/system_config_job_dto.dart differ diff --git a/mobile/openapi/test/all_job_status_response_dto_test.dart b/mobile/openapi/test/all_job_status_response_dto_test.dart index 68a5983e70..4f6d964cbe 100644 Binary files a/mobile/openapi/test/all_job_status_response_dto_test.dart and b/mobile/openapi/test/all_job_status_response_dto_test.dart differ diff --git a/mobile/openapi/test/system_config_job_dto_test.dart b/mobile/openapi/test/system_config_job_dto_test.dart index a5557662f9..4900e722f7 100644 Binary files a/mobile/openapi/test/system_config_job_dto_test.dart and b/mobile/openapi/test/system_config_job_dto_test.dart differ diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index 825b707c60..f5f58868ad 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -5343,6 +5343,9 @@ "metadataExtraction": { "$ref": "#/components/schemas/JobStatusDto" }, + "migration": { + "$ref": "#/components/schemas/JobStatusDto" + }, "objectTagging": { "$ref": "#/components/schemas/JobStatusDto" }, @@ -5372,6 +5375,7 @@ "objectTagging", "clipEncoding", "storageTemplateMigration", + "migration", "backgroundTask", "search", "recognizeFaces", @@ -6535,6 +6539,7 @@ "clipEncoding", "backgroundTask", "storageTemplateMigration", + "migration", "search", "sidecar", "library" @@ -7693,6 +7698,9 @@ "metadataExtraction": { "$ref": "#/components/schemas/JobSettingsDto" }, + "migration": { + "$ref": "#/components/schemas/JobSettingsDto" + }, "objectTagging": { "$ref": "#/components/schemas/JobSettingsDto" }, @@ -7722,6 +7730,7 @@ "objectTagging", "clipEncoding", "storageTemplateMigration", + "migration", "backgroundTask", "search", "recognizeFaces", diff --git a/server/src/domain/asset/asset.service.ts b/server/src/domain/asset/asset.service.ts index c70cbe006b..dea3e28a95 100644 --- a/server/src/domain/asset/asset.service.ts +++ b/server/src/domain/asset/asset.service.ts @@ -57,7 +57,7 @@ export interface UploadFile { export class AssetService { private logger = new Logger(AssetService.name); private access: AccessCore; - private storageCore = new StorageCore(); + private storageCore: StorageCore; constructor( @Inject(IAccessRepository) accessRepository: IAccessRepository, @@ -67,6 +67,7 @@ export class AssetService { @Inject(IStorageRepository) private storageRepository: IStorageRepository, ) { this.access = new AccessCore(accessRepository); + this.storageCore = new StorageCore(storageRepository); } canUploadFile({ authUser, fieldName, file }: UploadRequest): true { diff --git a/server/src/domain/facial-recognition/facial-recognition.service.spec.ts b/server/src/domain/facial-recognition/facial-recognition.service.spec.ts index 064d942dee..ea7b3d08dc 100644 --- a/server/src/domain/facial-recognition/facial-recognition.service.spec.ts +++ b/server/src/domain/facial-recognition/facial-recognition.service.spec.ts @@ -307,14 +307,14 @@ describe(FacialRecognitionService.name, () => { await sut.handleGenerateFaceThumbnail(face.middle); expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1']); - expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/pe/rs'); expect(mediaMock.crop).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg', { left: 95, top: 95, width: 110, height: 110, }); - expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { + expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', { format: 'jpeg', size: 250, quality: 80, @@ -323,7 +323,7 @@ describe(FacialRecognitionService.name, () => { expect(personMock.update).toHaveBeenCalledWith({ faceAssetId: 'asset-1', id: 'person-1', - thumbnailPath: 'upload/thumbs/user-id/person-1.jpeg', + thumbnailPath: 'upload/thumbs/user-id/pe/rs/person-1.jpeg', }); }); @@ -338,7 +338,7 @@ describe(FacialRecognitionService.name, () => { width: 510, height: 510, }); - expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { + expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', { format: 'jpeg', size: 250, quality: 80, @@ -357,7 +357,7 @@ describe(FacialRecognitionService.name, () => { width: 202, height: 202, }); - expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { + expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/pe/rs/person-1.jpeg', { format: 'jpeg', size: 250, quality: 80, diff --git a/server/src/domain/facial-recognition/facial-recognition.services.ts b/server/src/domain/facial-recognition/facial-recognition.services.ts index 2e94273ce0..9d173a06ff 100644 --- a/server/src/domain/facial-recognition/facial-recognition.services.ts +++ b/server/src/domain/facial-recognition/facial-recognition.services.ts @@ -1,5 +1,4 @@ import { Inject, Logger } from '@nestjs/common'; -import { join } from 'path'; import { IAssetRepository, WithoutProperty } from '../asset'; import { usePagination } from '../domain.util'; import { IBaseJob, IEntityJob, IFaceThumbnailJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job'; @@ -13,8 +12,8 @@ import { AssetFaceId, IFaceRepository } from './face.repository'; export class FacialRecognitionService { private logger = new Logger(FacialRecognitionService.name); - private storageCore = new StorageCore(); private configCore: SystemConfigCore; + private storageCore: StorageCore; constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, @@ -28,6 +27,7 @@ export class FacialRecognitionService { @Inject(IStorageRepository) private storageRepository: IStorageRepository, ) { this.configCore = new SystemConfigCore(configRepository); + this.storageCore = new StorageCore(storageRepository); } async handleQueueRecognizeFaces({ force }: IBaseJob) { @@ -117,6 +117,21 @@ export class FacialRecognitionService { return true; } + async handlePersonMigration({ id }: IEntityJob) { + const person = await this.personRepository.getById(id); + if (!person) { + return false; + } + + const path = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, person.ownerId, `${id}.jpeg`); + if (person.thumbnailPath && person.thumbnailPath !== path) { + await this.storageRepository.moveFile(person.thumbnailPath, path); + await this.personRepository.update({ id, thumbnailPath: path }); + } + + return true; + } + async handleGenerateFaceThumbnail(data: IFaceThumbnailJob) { const { machineLearning } = await this.configCore.getConfig(); if (!machineLearning.enabled || !machineLearning.facialRecognition.enabled) { @@ -132,9 +147,7 @@ export class FacialRecognitionService { this.logger.verbose(`Cropping face for person: ${personId}`); - const outputFolder = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId); - const output = join(outputFolder, `${personId}.jpeg`); - this.storageRepository.mkdirSync(outputFolder); + const output = this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${personId}.jpeg`); const { x1, y1, x2, y2 } = boundingBox; diff --git a/server/src/domain/job/job.constants.ts b/server/src/domain/job/job.constants.ts index 0f3bc76470..1d5a474ffb 100644 --- a/server/src/domain/job/job.constants.ts +++ b/server/src/domain/job/job.constants.ts @@ -7,6 +7,7 @@ export enum QueueName { CLIP_ENCODING = 'clipEncoding', BACKGROUND_TASK = 'backgroundTask', STORAGE_TEMPLATE_MIGRATION = 'storageTemplateMigration', + MIGRATION = 'migration', SEARCH = 'search', SIDECAR = 'sidecar', LIBRARY = 'library', @@ -45,6 +46,11 @@ export enum JobName { STORAGE_TEMPLATE_MIGRATION_SINGLE = 'storage-template-migration-single', SYSTEM_CONFIG_CHANGE = 'system-config-change', + // migration + QUEUE_MIGRATION = 'queue-migration', + MIGRATE_ASSET = 'migrate-asset', + MIGRATE_PERSON = 'migrate-person', + // object tagging QUEUE_OBJECT_TAGGING = 'queue-object-tagging', CLASSIFY_IMAGE = 'classify-image', @@ -119,6 +125,11 @@ export const JOBS_TO_QUEUE: Record = { [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: QueueName.STORAGE_TEMPLATE_MIGRATION, [JobName.SYSTEM_CONFIG_CHANGE]: QueueName.STORAGE_TEMPLATE_MIGRATION, + // migration + [JobName.QUEUE_MIGRATION]: QueueName.MIGRATION, + [JobName.MIGRATE_ASSET]: QueueName.MIGRATION, + [JobName.MIGRATE_PERSON]: QueueName.MIGRATION, + // object tagging [JobName.QUEUE_OBJECT_TAGGING]: QueueName.OBJECT_TAGGING, [JobName.CLASSIFY_IMAGE]: QueueName.OBJECT_TAGGING, diff --git a/server/src/domain/job/job.dto.ts b/server/src/domain/job/job.dto.ts index c98821d7e9..ec4cf7bb6e 100644 --- a/server/src/domain/job/job.dto.ts +++ b/server/src/domain/job/job.dto.ts @@ -68,6 +68,9 @@ export class AllJobStatusResponseDto implements Record @ApiProperty({ type: JobStatusDto }) [QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobStatusDto; + @ApiProperty({ type: JobStatusDto }) + [QueueName.MIGRATION]!: JobStatusDto; + @ApiProperty({ type: JobStatusDto }) [QueueName.BACKGROUND_TASK]!: JobStatusDto; diff --git a/server/src/domain/job/job.repository.ts b/server/src/domain/job/job.repository.ts index 1ae0144275..f52eda8916 100644 --- a/server/src/domain/job/job.repository.ts +++ b/server/src/domain/job/job.repository.ts @@ -46,6 +46,11 @@ export type JobItem = | { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE; data: IEntityJob } | { name: JobName.SYSTEM_CONFIG_CHANGE; data?: IBaseJob } + // Migration + | { name: JobName.QUEUE_MIGRATION; data?: IBaseJob } + | { name: JobName.MIGRATE_ASSET; data?: IEntityJob } + | { name: JobName.MIGRATE_PERSON; data?: IEntityJob } + // Metadata Extraction | { name: JobName.QUEUE_METADATA_EXTRACTION; data: IBaseJob } | { name: JobName.METADATA_EXTRACTION; data: IEntityJob } diff --git a/server/src/domain/job/job.service.spec.ts b/server/src/domain/job/job.service.spec.ts index a45958a6ea..224f78836f 100644 --- a/server/src/domain/job/job.service.spec.ts +++ b/server/src/domain/job/job.service.spec.ts @@ -94,6 +94,7 @@ describe(JobService.name, () => { [QueueName.OBJECT_TAGGING]: expectedJobStatus, [QueueName.SEARCH]: expectedJobStatus, [QueueName.STORAGE_TEMPLATE_MIGRATION]: expectedJobStatus, + [QueueName.MIGRATION]: expectedJobStatus, [QueueName.THUMBNAIL_GENERATION]: expectedJobStatus, [QueueName.VIDEO_CONVERSION]: expectedJobStatus, [QueueName.RECOGNIZE_FACES]: expectedJobStatus, @@ -229,6 +230,7 @@ describe(JobService.name, () => { [QueueName.SIDECAR]: { concurrency: 10 }, [QueueName.LIBRARY]: { concurrency: 10 }, [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 10 }, + [QueueName.MIGRATION]: { concurrency: 10 }, [QueueName.THUMBNAIL_GENERATION]: { concurrency: 10 }, [QueueName.VIDEO_CONVERSION]: { concurrency: 10 }, }, @@ -242,6 +244,7 @@ describe(JobService.name, () => { expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.SIDECAR, 10); expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.LIBRARY, 10); expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.STORAGE_TEMPLATE_MIGRATION, 10); + expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.MIGRATION, 10); expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.THUMBNAIL_GENERATION, 10); expect(jobMock.setConcurrency).toHaveBeenCalledWith(QueueName.VIDEO_CONVERSION, 10); }); diff --git a/server/src/domain/job/job.service.ts b/server/src/domain/job/job.service.ts index a910f7381c..3c152ee064 100644 --- a/server/src/domain/job/job.service.ts +++ b/server/src/domain/job/job.service.ts @@ -76,6 +76,9 @@ export class JobService { case QueueName.STORAGE_TEMPLATE_MIGRATION: return this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION }); + case QueueName.MIGRATION: + return this.jobRepository.queue({ name: JobName.QUEUE_MIGRATION }); + case QueueName.OBJECT_TAGGING: await this.configCore.requireFeature(FeatureFlag.TAG_IMAGE); return this.jobRepository.queue({ name: JobName.QUEUE_OBJECT_TAGGING, data: { force } }); diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index fe280cad22..ceaf741d5c 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -202,8 +202,8 @@ describe(MediaService.name, () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); await sut.handleGenerateJpegThumbnail({ id: assetStub.image.id }); - expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); - expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.jpeg', { + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se'); + expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/as/se/asset-id.jpeg', { size: 1440, format: 'jpeg', quality: 80, @@ -211,7 +211,7 @@ describe(MediaService.name, () => { }); expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', - resizePath: 'upload/thumbs/user-id/asset-id.jpeg', + resizePath: 'upload/thumbs/user-id/as/se/asset-id.jpeg', }); }); @@ -220,19 +220,23 @@ describe(MediaService.name, () => { assetMock.getByIds.mockResolvedValue([assetStub.video]); await sut.handleGenerateJpegThumbnail({ id: assetStub.video.id }); - expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); - expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', { - inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'], - outputOptions: [ - '-frames:v 1', - '-v verbose', - '-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p', - ], - twoPass: false, - }); + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se'); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/thumbs/user-id/as/se/asset-id.jpeg', + { + inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'], + outputOptions: [ + '-frames:v 1', + '-v verbose', + '-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p', + ], + twoPass: false, + }, + ); expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', - resizePath: 'upload/thumbs/user-id/asset-id.jpeg', + resizePath: 'upload/thumbs/user-id/as/se/asset-id.jpeg', }); }); @@ -241,19 +245,23 @@ describe(MediaService.name, () => { assetMock.getByIds.mockResolvedValue([assetStub.video]); await sut.handleGenerateJpegThumbnail({ id: assetStub.video.id }); - expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); - expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', { - inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'], - outputOptions: [ - '-frames:v 1', - '-v verbose', - '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p', - ], - twoPass: false, - }); + expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id/as/se'); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/thumbs/user-id/as/se/asset-id.jpeg', + { + inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'], + outputOptions: [ + '-frames:v 1', + '-v verbose', + '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p', + ], + twoPass: false, + }, + ); expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', - resizePath: 'upload/thumbs/user-id/asset-id.jpeg', + resizePath: 'upload/thumbs/user-id/as/se/asset-id.jpeg', }); }); @@ -275,13 +283,16 @@ describe(MediaService.name, () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); await sut.handleGenerateWebpThumbnail({ id: assetStub.image.id }); - expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.webp', { + expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/as/se/asset-id.webp', { format: 'webp', size: 250, quality: 80, colorspace: Colorspace.P3, }); - expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: 'upload/thumbs/user-id/asset-id.webp' }); + expect(assetMock.save).toHaveBeenCalledWith({ + id: 'asset-id', + webpPath: 'upload/thumbs/user-id/as/se/asset-id.webp', + }); }); }); @@ -375,7 +386,7 @@ describe(MediaService.name, () => { expect(storageMock.mkdirSync).toHaveBeenCalled(); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -416,7 +427,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -442,7 +453,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -471,7 +482,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -498,7 +509,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -525,7 +536,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -552,7 +563,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -603,7 +614,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -635,7 +646,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -664,7 +675,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -695,7 +706,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -728,7 +739,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -760,7 +771,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -791,7 +802,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -821,7 +832,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -851,7 +862,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -881,7 +892,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -914,7 +925,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -976,7 +987,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], outputOptions: [ @@ -1014,7 +1025,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], outputOptions: [ @@ -1048,7 +1059,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], outputOptions: [ @@ -1083,7 +1094,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], outputOptions: [ @@ -1114,7 +1125,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], outputOptions: [ @@ -1150,7 +1161,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], outputOptions: [ @@ -1186,7 +1197,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], outputOptions: [ @@ -1219,7 +1230,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], outputOptions: [ @@ -1263,7 +1274,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], outputOptions: [ @@ -1295,7 +1306,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], outputOptions: [ @@ -1329,7 +1340,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], outputOptions: [ @@ -1359,7 +1370,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/card1', '-filter_hw_device accel'], outputOptions: [ @@ -1385,7 +1396,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD129', '-filter_hw_device accel'], outputOptions: [ @@ -1418,7 +1429,7 @@ describe(MediaService.name, () => { expect(mediaMock.transcode).toHaveBeenCalledTimes(2); expect(mediaMock.transcode).toHaveBeenLastCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -1455,7 +1466,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -1482,7 +1493,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ @@ -1509,7 +1520,7 @@ describe(MediaService.name, () => { await sut.handleVideoConversion({ id: assetStub.video.id }); expect(mediaMock.transcode).toHaveBeenCalledWith( '/original/path.ext', - 'upload/encoded-video/user-id/asset-id.mp4', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', { inputOptions: [], outputOptions: [ diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index 4ff9f0cc50..413057cf4e 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -1,9 +1,8 @@ import { AssetEntity, AssetType, TranscodeHWAccel, TranscodePolicy, VideoCodec } from '@app/infra/entities'; import { Inject, Injectable, Logger, UnsupportedMediaTypeException } from '@nestjs/common'; -import { join } from 'path'; import { IAssetRepository, WithoutProperty } from '../asset'; import { usePagination } from '../domain.util'; -import { IBaseJob, IEntityJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job'; +import { IBaseJob, IEntityJob, IJobRepository, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job'; import { IPersonRepository } from '../person'; import { IStorageRepository, StorageCore, StorageFolder } from '../storage'; import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config'; @@ -14,8 +13,8 @@ import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIC @Injectable() export class MediaService { private logger = new Logger(MediaService.name); - private storageCore = new StorageCore(); private configCore: SystemConfigCore; + private storageCore: StorageCore; constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, @@ -26,11 +25,10 @@ export class MediaService { @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, ) { this.configCore = new SystemConfigCore(configRepository); + this.storageCore = new StorageCore(this.storageRepository); } - async handleQueueGenerateThumbnails(job: IBaseJob) { - const { force } = job; - + async handleQueueGenerateThumbnails({ force }: IBaseJob) { const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => { return force ? this.assetRepository.getAll(pagination) @@ -81,6 +79,58 @@ export class MediaService { return true; } + async handleQueueMigration() { + const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => + this.assetRepository.getAll(pagination), + ); + + const { active, waiting } = await this.jobRepository.getJobCounts(QueueName.MIGRATION); + if (active === 1 && waiting === 0) { + await this.storageCore.removeEmptyDirs(StorageFolder.THUMBNAILS); + await this.storageCore.removeEmptyDirs(StorageFolder.ENCODED_VIDEO); + } + + for await (const assets of assetPagination) { + for (const asset of assets) { + await this.jobRepository.queue({ name: JobName.MIGRATE_ASSET, data: { id: asset.id } }); + } + } + + const people = await this.personRepository.getAll(); + for (const person of people) { + await this.jobRepository.queue({ name: JobName.MIGRATE_PERSON, data: { id: person.id } }); + } + + return true; + } + + async handleAssetMigration({ id }: IEntityJob) { + const [asset] = await this.assetRepository.getByIds([id]); + if (!asset) { + return false; + } + const resizePath = this.ensureThumbnailPath(asset, 'jpeg'); + const webpPath = this.ensureThumbnailPath(asset, 'webp'); + const encodedVideoPath = this.ensureEncodedVideoPath(asset, 'mp4'); + + if (asset.resizePath && asset.resizePath !== resizePath) { + await this.storageRepository.moveFile(asset.resizePath, resizePath); + await this.assetRepository.save({ id: asset.id, resizePath }); + } + + if (asset.webpPath && asset.webpPath !== webpPath) { + await this.storageRepository.moveFile(asset.webpPath, webpPath); + await this.assetRepository.save({ id: asset.id, webpPath }); + } + + if (asset.encodedVideoPath && asset.encodedVideoPath !== encodedVideoPath) { + await this.storageRepository.moveFile(asset.encodedVideoPath, encodedVideoPath); + await this.assetRepository.save({ id: asset.id, encodedVideoPath }); + } + + return true; + } + async handleGenerateJpegThumbnail({ id }: IEntityJob) { const [asset] = await this.assetRepository.getByIds([id]); if (!asset) { @@ -184,9 +234,7 @@ export class MediaService { } const input = asset.originalPath; - const outputFolder = this.storageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, asset.ownerId); - const output = join(outputFolder, `${asset.id}.mp4`); - this.storageRepository.mkdirSync(outputFolder); + const output = this.ensureEncodedVideoPath(asset, 'mp4'); const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input); const mainVideoStream = this.getMainStream(videoStreams); @@ -330,8 +378,10 @@ export class MediaService { } ensureThumbnailPath(asset: AssetEntity, extension: string): string { - const folderPath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId); - this.storageRepository.mkdirSync(folderPath); - return join(folderPath, `${asset.id}.${extension}`); + return this.storageCore.ensurePath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}.${extension}`); + } + + ensureEncodedVideoPath(asset: AssetEntity, extension: string): string { + return this.storageCore.ensurePath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}.${extension}`); } } diff --git a/server/src/domain/person/person.service.spec.ts b/server/src/domain/person/person.service.spec.ts index 175b302dd1..5c0b25a5ec 100644 --- a/server/src/domain/person/person.service.spec.ts +++ b/server/src/domain/person/person.service.spec.ts @@ -37,10 +37,10 @@ describe(PersonService.name, () => { beforeEach(async () => { accessMock = newAccessRepositoryMock(); - personMock = newPersonRepositoryMock(); - storageMock = newStorageRepositoryMock(); configMock = newSystemConfigRepositoryMock(); jobMock = newJobRepositoryMock(); + personMock = newPersonRepositoryMock(); + storageMock = newStorageRepositoryMock(); sut = new PersonService(accessMock, personMock, configMock, storageMock, jobMock); }); diff --git a/server/src/domain/server-info/server-info.service.ts b/server/src/domain/server-info/server-info.service.ts index 86dc4b8d06..0ef1e568f3 100644 --- a/server/src/domain/server-info/server-info.service.ts +++ b/server/src/domain/server-info/server-info.service.ts @@ -16,8 +16,8 @@ import { @Injectable() export class ServerInfoService { - private storageCore = new StorageCore(); private configCore: SystemConfigCore; + private storageCore: StorageCore; constructor( @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, @@ -25,6 +25,7 @@ export class ServerInfoService { @Inject(IStorageRepository) private storageRepository: IStorageRepository, ) { this.configCore = new SystemConfigCore(configRepository); + this.storageCore = new StorageCore(storageRepository); } async getInfo(): Promise { diff --git a/server/src/domain/storage-template/storage-template.service.ts b/server/src/domain/storage-template/storage-template.service.ts index 9be6664453..5191bf557d 100644 --- a/server/src/domain/storage-template/storage-template.service.ts +++ b/server/src/domain/storage-template/storage-template.service.ts @@ -30,7 +30,7 @@ export interface MoveAssetMetadata { export class StorageTemplateService { private logger = new Logger(StorageTemplateService.name); private configCore: SystemConfigCore; - private storageCore = new StorageCore(); + private storageCore: StorageCore; private storageTemplate: HandlebarsTemplateDelegate; constructor( @@ -44,6 +44,7 @@ export class StorageTemplateService { this.configCore = new SystemConfigCore(configRepository); this.configCore.addValidator((config) => this.validate(config)); this.configCore.config$.subscribe((config) => this.onConfig(config)); + this.storageCore = new StorageCore(storageRepository); } async handleMigrationSingle({ id }: IEntityJob) { diff --git a/server/src/domain/storage/storage.core.ts b/server/src/domain/storage/storage.core.ts index ef90d433c0..40f05a1867 100644 --- a/server/src/domain/storage/storage.core.ts +++ b/server/src/domain/storage/storage.core.ts @@ -1,5 +1,6 @@ import { join } from 'node:path'; import { APP_MEDIA_LOCATION } from '../domain.constant'; +import { IStorageRepository } from './storage.repository'; export enum StorageFolder { ENCODED_VIDEO = 'encoded-video', @@ -10,6 +11,8 @@ export enum StorageFolder { } export class StorageCore { + constructor(private repository: IStorageRepository) {} + getFolderLocation( folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS, userId: string, @@ -24,4 +27,22 @@ export class StorageCore { getBaseFolder(folder: StorageFolder) { return join(APP_MEDIA_LOCATION, folder); } + + ensurePath( + folder: StorageFolder.ENCODED_VIDEO | StorageFolder.UPLOAD | StorageFolder.PROFILE | StorageFolder.THUMBNAILS, + ownerId: string, + fileName: string, + ): string { + const folderPath = join( + this.getFolderLocation(folder, ownerId), + fileName.substring(0, 2), + fileName.substring(2, 4), + ); + this.repository.mkdirSync(folderPath); + return join(folderPath, fileName); + } + + removeEmptyDirs(folder: StorageFolder) { + return this.repository.removeEmptyDirs(this.getBaseFolder(folder)); + } } diff --git a/server/src/domain/storage/storage.repository.ts b/server/src/domain/storage/storage.repository.ts index 659fe68237..f41136a133 100644 --- a/server/src/domain/storage/storage.repository.ts +++ b/server/src/domain/storage/storage.repository.ts @@ -26,7 +26,7 @@ export interface IStorageRepository { createReadStream(filepath: string, mimeType?: string | null): Promise; unlink(filepath: string): Promise; unlinkDir(folder: string, options?: { recursive?: boolean; force?: boolean }): Promise; - removeEmptyDirs(folder: string): Promise; + removeEmptyDirs(folder: string, self?: boolean): Promise; moveFile(source: string, target: string): Promise; checkFileExists(filepath: string, mode?: number): Promise; mkdirSync(filepath: string): void; diff --git a/server/src/domain/storage/storage.service.ts b/server/src/domain/storage/storage.service.ts index 23e8aa11e5..d53449b08a 100644 --- a/server/src/domain/storage/storage.service.ts +++ b/server/src/domain/storage/storage.service.ts @@ -6,9 +6,11 @@ import { IStorageRepository } from './storage.repository'; @Injectable() export class StorageService { private logger = new Logger(StorageService.name); - private storageCore = new StorageCore(); + private storageCore: StorageCore; - constructor(@Inject(IStorageRepository) private storageRepository: IStorageRepository) {} + constructor(@Inject(IStorageRepository) private storageRepository: IStorageRepository) { + this.storageCore = new StorageCore(storageRepository); + } init() { const libraryBase = this.storageCore.getBaseFolder(StorageFolder.LIBRARY); diff --git a/server/src/domain/system-config/dto/system-config-job.dto.ts b/server/src/domain/system-config/dto/system-config-job.dto.ts index b8064a5b37..e3d19418f6 100644 --- a/server/src/domain/system-config/dto/system-config-job.dto.ts +++ b/server/src/domain/system-config/dto/system-config-job.dto.ts @@ -47,6 +47,12 @@ export class SystemConfigJobDto implements Record { @Type(() => JobSettingsDto) [QueueName.STORAGE_TEMPLATE_MIGRATION]!: JobSettingsDto; + @ApiProperty({ type: JobSettingsDto }) + @ValidateNested() + @IsObject() + @Type(() => JobSettingsDto) + [QueueName.MIGRATION]!: JobSettingsDto; + @ApiProperty({ type: JobSettingsDto }) @ValidateNested() @IsObject() diff --git a/server/src/domain/system-config/system-config.core.ts b/server/src/domain/system-config/system-config.core.ts index 013813edbd..57a2f71f96 100644 --- a/server/src/domain/system-config/system-config.core.ts +++ b/server/src/domain/system-config/system-config.core.ts @@ -53,6 +53,7 @@ export const defaults = Object.freeze({ [QueueName.SIDECAR]: { concurrency: 5 }, [QueueName.LIBRARY]: { concurrency: 1 }, [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 }, + [QueueName.MIGRATION]: { concurrency: 5 }, [QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 }, [QueueName.VIDEO_CONVERSION]: { concurrency: 1 }, }, diff --git a/server/src/domain/system-config/system-config.service.spec.ts b/server/src/domain/system-config/system-config.service.spec.ts index 328651e2cf..67484e06d4 100644 --- a/server/src/domain/system-config/system-config.service.spec.ts +++ b/server/src/domain/system-config/system-config.service.spec.ts @@ -33,6 +33,7 @@ const updatedConfig = Object.freeze({ [QueueName.SIDECAR]: { concurrency: 5 }, [QueueName.LIBRARY]: { concurrency: 1 }, [QueueName.STORAGE_TEMPLATE_MIGRATION]: { concurrency: 5 }, + [QueueName.MIGRATION]: { concurrency: 5 }, [QueueName.THUMBNAIL_GENERATION]: { concurrency: 5 }, [QueueName.VIDEO_CONVERSION]: { concurrency: 1 }, }, diff --git a/server/src/domain/user/user.service.ts b/server/src/domain/user/user.service.ts index 3f35e2c358..dac75044a3 100644 --- a/server/src/domain/user/user.service.ts +++ b/server/src/domain/user/user.service.ts @@ -25,8 +25,8 @@ import { IUserRepository } from './user.repository'; @Injectable() export class UserService { private logger = new Logger(UserService.name); + private storageCore: StorageCore; private userCore: UserCore; - private storageCore = new StorageCore(); constructor( @Inject(IUserRepository) private userRepository: IUserRepository, @@ -37,6 +37,7 @@ export class UserService { @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, ) { + this.storageCore = new StorageCore(storageRepository); this.userCore = new UserCore(userRepository, libraryRepository, cryptoRepository); } diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index 38aa3a046a..825e22dbee 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -43,6 +43,7 @@ export enum SystemConfigKey { JOB_SEARCH_CONCURRENCY = 'job.search.concurrency', JOB_SIDECAR_CONCURRENCY = 'job.sidecar.concurrency', JOB_LIBRARY_CONCURRENCY = 'job.library.concurrency', + JOB_MIGRATION_CONCURRENCY = 'job.migration.concurrency', MACHINE_LEARNING_ENABLED = 'machineLearning.enabled', MACHINE_LEARNING_URL = 'machineLearning.url', diff --git a/server/src/infra/repositories/filesystem.provider.ts b/server/src/infra/repositories/filesystem.provider.ts index 9dcb17425c..c4d9c5b953 100644 --- a/server/src/infra/repositories/filesystem.provider.ts +++ b/server/src/infra/repositories/filesystem.provider.ts @@ -66,11 +66,7 @@ export class FilesystemProvider implements IStorageRepository { await fs.rm(folder, options); } - async removeEmptyDirs(directory: string) { - this._removeEmptyDirs(directory, false); - } - - private async _removeEmptyDirs(directory: string, self: boolean) { + async removeEmptyDirs(directory: string, self: boolean = false) { // lstat does not follow symlinks (in contrast to stat) const stats = await fs.lstat(directory); if (!stats.isDirectory()) { @@ -78,7 +74,7 @@ export class FilesystemProvider implements IStorageRepository { } const files = await fs.readdir(directory); - await Promise.all(files.map((file) => this._removeEmptyDirs(path.join(directory, file), true))); + await Promise.all(files.map((file) => this.removeEmptyDirs(path.join(directory, file), true))); if (self) { const updated = await fs.readdir(directory); diff --git a/server/src/microservices/app.service.ts b/server/src/microservices/app.service.ts index 023181f5b5..b283a7e94b 100644 --- a/server/src/microservices/app.service.ts +++ b/server/src/microservices/app.service.ts @@ -63,6 +63,9 @@ export class AppService { [JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data), [JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(), [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data), + [JobName.QUEUE_MIGRATION]: () => this.mediaService.handleQueueMigration(), + [JobName.MIGRATE_ASSET]: (data) => this.mediaService.handleAssetMigration(data), + [JobName.MIGRATE_PERSON]: (data) => this.facialRecognitionService.handlePersonMigration(data), [JobName.SYSTEM_CONFIG_CHANGE]: () => this.systemConfigService.refreshConfig(), [JobName.QUEUE_GENERATE_THUMBNAILS]: (data) => this.mediaService.handleQueueGenerateThumbnails(data), [JobName.GENERATE_JPEG_THUMBNAIL]: (data) => this.mediaService.handleGenerateJpegThumbnail(data), diff --git a/server/src/microservices/processors/metadata-extraction.processor.ts b/server/src/microservices/processors/metadata-extraction.processor.ts index 5c8ad06302..1cfbb1d44c 100644 --- a/server/src/microservices/processors/metadata-extraction.processor.ts +++ b/server/src/microservices/processors/metadata-extraction.processor.ts @@ -50,7 +50,7 @@ const validate = (value: T): T | null => (typeof value === 'string' ? null : export class MetadataExtractionProcessor { private logger = new Logger(MetadataExtractionProcessor.name); private reverseGeocodingEnabled: boolean; - private storageCore = new StorageCore(); + private storageCore: StorageCore; constructor( @Inject(IAssetRepository) private assetRepository: IAssetRepository, @@ -63,6 +63,7 @@ export class MetadataExtractionProcessor { configService: ConfigService, ) { this.reverseGeocodingEnabled = !configService.get('DISABLE_REVERSE_GEOCODING'); + this.storageCore = new StorageCore(storageRepository); } async init(deleteCache = false) { diff --git a/web/src/api/api.ts b/web/src/api/api.ts index 70fa5d33ab..00b60dfca8 100644 --- a/web/src/api/api.ts +++ b/web/src/api/api.ts @@ -131,6 +131,7 @@ export class ImmichApi { [JobName.RecognizeFaces]: 'Recognize Faces', [JobName.VideoConversion]: 'Transcode Videos', [JobName.StorageTemplateMigration]: 'Storage Template Migration', + [JobName.Migration]: 'Migration', [JobName.BackgroundTask]: 'Background Tasks', [JobName.Search]: 'Search', [JobName.Library]: 'Library', diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index 66085a52f0..e39b6e4f12 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -307,6 +307,12 @@ export interface AllJobStatusResponseDto { * @memberof AllJobStatusResponseDto */ 'metadataExtraction': JobStatusDto; + /** + * + * @type {JobStatusDto} + * @memberof AllJobStatusResponseDto + */ + 'migration': JobStatusDto; /** * * @type {JobStatusDto} @@ -1779,6 +1785,7 @@ export const JobName = { ClipEncoding: 'clipEncoding', BackgroundTask: 'backgroundTask', StorageTemplateMigration: 'storageTemplateMigration', + Migration: 'migration', Search: 'search', Sidecar: 'sidecar', Library: 'library' @@ -3240,6 +3247,12 @@ export interface SystemConfigJobDto { * @memberof SystemConfigJobDto */ 'metadataExtraction': JobSettingsDto; + /** + * + * @type {JobSettingsDto} + * @memberof SystemConfigJobDto + */ + 'migration': JobSettingsDto; /** * * @type {JobSettingsDto} diff --git a/web/src/lib/components/admin-page/jobs/jobs-panel.svelte b/web/src/lib/components/admin-page/jobs/jobs-panel.svelte index 8b4cb3d896..2d2fcc7e1b 100644 --- a/web/src/lib/components/admin-page/jobs/jobs-panel.svelte +++ b/web/src/lib/components/admin-page/jobs/jobs-panel.svelte @@ -110,6 +110,12 @@ allowForceCommand: false, component: StorageMigrationDescription, }, + [JobName.Migration]: { + icon: FolderMove, + title: api.getJobName(JobName.Migration), + subtitle: 'Migrate thumbnails for assets and faces to the latest folder structure', + allowForceCommand: false, + }, }; $: jobList = Object.entries(jobDetails) as [JobName, JobDetails][];