From b0d5c7035b9ab991a61d7e173ae65689c0b9c016 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Tue, 28 Mar 2023 16:04:11 -0400 Subject: [PATCH] feat(server): apply storage migration after exif completes (#2093) * feat(server): apply storage migraiton after exif completes * feat: same for videos * fix: migration for live photos --- .../immich/src/api-v1/asset/asset.core.ts | 50 ++----------------- .../src/api-v1/asset/asset.service.spec.ts | 19 +------ .../immich/src/api-v1/asset/asset.service.ts | 13 ++--- server/apps/microservices/src/processors.ts | 5 ++ .../metadata-extraction.processor.ts | 15 ++++-- server/libs/domain/src/domain.util.ts | 4 ++ server/libs/domain/src/job/job.constants.ts | 1 + server/libs/domain/src/job/job.repository.ts | 1 + .../storage-template.service.ts | 20 ++++++++ server/libs/infra/src/job/job.repository.ts | 4 ++ 10 files changed, 54 insertions(+), 78 deletions(-) diff --git a/server/apps/immich/src/api-v1/asset/asset.core.ts b/server/apps/immich/src/api-v1/asset/asset.core.ts index 2481721519..3ca0dd0bac 100644 --- a/server/apps/immich/src/api-v1/asset/asset.core.ts +++ b/server/apps/immich/src/api-v1/asset/asset.core.ts @@ -1,29 +1,10 @@ -import { - AuthUserDto, - IJobRepository, - IStorageRepository, - ISystemConfigRepository, - JobName, - StorageTemplateCore, -} from '@app/domain'; -import { AssetEntity, SystemConfig, UserEntity } from '@app/infra/db/entities'; -import { Logger } from '@nestjs/common'; +import { AuthUserDto, IJobRepository, JobName } from '@app/domain'; +import { AssetEntity, UserEntity } from '@app/infra/db/entities'; import { IAssetRepository } from './asset-repository'; import { CreateAssetDto, UploadFile } from './dto/create-asset.dto'; export class AssetCore { - private templateCore: StorageTemplateCore; - private logger = new Logger(AssetCore.name); - - constructor( - private repository: IAssetRepository, - private jobRepository: IJobRepository, - configRepository: ISystemConfigRepository, - config: SystemConfig, - private storageRepository: IStorageRepository, - ) { - this.templateCore = new StorageTemplateCore(configRepository, config, storageRepository); - } + constructor(private repository: IAssetRepository, private jobRepository: IJobRepository) {} async create( authUser: AuthUserDto, @@ -31,7 +12,7 @@ export class AssetCore { file: UploadFile, livePhotoAssetId?: string, ): Promise { - let asset = await this.repository.create({ + const asset = await this.repository.create({ owner: { id: authUser.id } as UserEntity, mimeType: file.mimeType, @@ -56,31 +37,8 @@ export class AssetCore { sharedLinks: [], }); - asset = await this.moveAsset(asset, file.originalName); - await this.jobRepository.queue({ name: JobName.ASSET_UPLOADED, data: { asset, fileName: file.originalName } }); return asset; } - - async moveAsset(asset: AssetEntity, originalName: string) { - const destination = await this.templateCore.getTemplatePath(asset, originalName); - if (asset.originalPath !== destination) { - const source = asset.originalPath; - - try { - await this.storageRepository.moveFile(asset.originalPath, destination); - try { - await this.repository.save({ id: asset.id, originalPath: destination }); - asset.originalPath = destination; - } catch (error: any) { - this.logger.warn('Unable to save new originalPath to database, undoing move', error?.stack); - await this.storageRepository.moveFile(destination, source); - } - } catch (error: any) { - this.logger.error(`Problem applying storage template`, error?.stack, { id: asset.id, source, destination }); - } - } - return asset; - } } diff --git a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts index 5eefb06202..5883b30ca4 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.spec.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.spec.ts @@ -8,14 +8,7 @@ import { TimeGroupEnum } from './dto/get-asset-count-by-time-bucket.dto'; import { AssetCountByUserIdResponseDto } from './response-dto/asset-count-by-user-id-response.dto'; import { DownloadService } from '../../modules/download/download.service'; import { AlbumRepository, IAlbumRepository } from '../album/album-repository'; -import { - ICryptoRepository, - IJobRepository, - ISharedLinkRepository, - IStorageRepository, - ISystemConfigRepository, - JobName, -} from '@app/domain'; +import { ICryptoRepository, IJobRepository, ISharedLinkRepository, IStorageRepository, JobName } from '@app/domain'; import { assetEntityStub, authStub, @@ -24,10 +17,8 @@ import { newJobRepositoryMock, newSharedLinkRepositoryMock, newStorageRepositoryMock, - newSystemConfigRepositoryMock, sharedLinkResponseStub, sharedLinkStub, - systemConfigStub, } from '@app/domain/../test'; import { CreateAssetsShareLinkDto } from './dto/create-asset-shared-link.dto'; import { BadRequestException, ForbiddenException } from '@nestjs/common'; @@ -121,7 +112,6 @@ describe('AssetService', () => { let albumRepositoryMock: jest.Mocked; let downloadServiceMock: jest.Mocked>; let sharedLinkRepositoryMock: jest.Mocked; - let configMock: jest.Mocked; let cryptoMock: jest.Mocked; let jobMock: jest.Mocked; let storageMock: jest.Mocked; @@ -160,7 +150,6 @@ describe('AssetService', () => { sharedLinkRepositoryMock = newSharedLinkRepositoryMock(); jobMock = newJobRepositoryMock(); - configMock = newSystemConfigRepositoryMock(); cryptoMock = newCryptoRepositoryMock(); storageMock = newStorageRepositoryMock(); @@ -171,8 +160,6 @@ describe('AssetService', () => { downloadServiceMock as DownloadService, sharedLinkRepositoryMock, jobMock, - configMock, - systemConfigStub.defaults, cryptoMock, storageMock, ); @@ -273,10 +260,6 @@ describe('AssetService', () => { await expect(sut.uploadFile(authStub.user1, dto, file)).resolves.toEqual({ duplicate: false, id: 'id_1' }); expect(assetRepositoryMock.create).toHaveBeenCalled(); - expect(assetRepositoryMock.save).toHaveBeenCalledWith({ - id: 'id_1', - originalPath: 'upload/library/user_id_1/2022/2022-06-19/asset_1.jpeg', - }); }); it('should handle a duplicate', async () => { diff --git a/server/apps/immich/src/api-v1/asset/asset.service.ts b/server/apps/immich/src/api-v1/asset/asset.service.ts index 5927315647..f7b2cf0431 100644 --- a/server/apps/immich/src/api-v1/asset/asset.service.ts +++ b/server/apps/immich/src/api-v1/asset/asset.service.ts @@ -12,7 +12,7 @@ import { import { InjectRepository } from '@nestjs/typeorm'; import { QueryFailedError, Repository } from 'typeorm'; import { AuthUserDto } from '../../decorators/auth-user.decorator'; -import { AssetEntity, AssetType, SharedLinkType, SystemConfig } from '@app/infra/db/entities'; +import { AssetEntity, AssetType, SharedLinkType } from '@app/infra/db/entities'; import { constants, createReadStream, stat } from 'fs'; import { ServeFileDto } from './dto/serve-file.dto'; import { Response as Res } from 'express'; @@ -24,10 +24,9 @@ import { CheckDuplicateAssetDto } from './dto/check-duplicate-asset.dto'; import { CuratedObjectsResponseDto } from './response-dto/curated-objects-response.dto'; import { AssetResponseDto, + getLivePhotoMotionFilename, ImmichReadStream, - INITIAL_SYSTEM_CONFIG, IStorageRepository, - ISystemConfigRepository, JobName, mapAsset, mapAssetWithoutExif, @@ -62,8 +61,6 @@ import { mapSharedLink, SharedLinkResponseDto } from '@app/domain'; import { AssetSearchDto } from './dto/asset-search.dto'; import { AddAssetsDto } from '../album/dto/add-assets.dto'; import { RemoveAssetsDto } from '../album/dto/remove-assets.dto'; -import path from 'path'; -import { getFileNameWithoutExtension } from '@app/domain'; const fileInfo = promisify(stat); @@ -86,12 +83,10 @@ export class AssetService { private downloadService: DownloadService, @Inject(ISharedLinkRepository) sharedLinkRepository: ISharedLinkRepository, @Inject(IJobRepository) private jobRepository: IJobRepository, - @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, - @Inject(INITIAL_SYSTEM_CONFIG) config: SystemConfig, @Inject(ICryptoRepository) cryptoRepository: ICryptoRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, ) { - this.assetCore = new AssetCore(_assetRepository, jobRepository, configRepository, config, storageRepository); + this.assetCore = new AssetCore(_assetRepository, jobRepository); this.shareCore = new ShareCore(sharedLinkRepository, cryptoRepository); } @@ -104,7 +99,7 @@ export class AssetService { if (livePhotoFile) { livePhotoFile = { ...livePhotoFile, - originalName: getFileNameWithoutExtension(file.originalName) + path.extname(livePhotoFile.originalName), + originalName: getLivePhotoMotionFilename(file.originalName, livePhotoFile.originalName), }; } diff --git a/server/apps/microservices/src/processors.ts b/server/apps/microservices/src/processors.ts index 61dfbdddae..384bb76ca9 100644 --- a/server/apps/microservices/src/processors.ts +++ b/server/apps/microservices/src/processors.ts @@ -127,6 +127,11 @@ export class StorageTemplateMigrationProcessor { async onTemplateMigration() { await this.storageTemplateService.handleTemplateMigration(); } + + @Process({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE }) + async onTemplateMigrationSingle(job: Job) { + await this.storageTemplateService.handleTemplateMigrationSingle(job.data); + } } @Processor(QueueName.THUMBNAIL_GENERATION) diff --git a/server/apps/microservices/src/processors/metadata-extraction.processor.ts b/server/apps/microservices/src/processors/metadata-extraction.processor.ts index 4373ec08b5..aae60773d3 100644 --- a/server/apps/microservices/src/processors/metadata-extraction.processor.ts +++ b/server/apps/microservices/src/processors/metadata-extraction.processor.ts @@ -10,7 +10,7 @@ import { QueueName, WithoutProperty, } from '@app/domain'; -import { AssetEntity, AssetType, ExifEntity } from '@app/infra/db/entities'; +import { AssetType, ExifEntity } from '@app/infra/db/entities'; import { Process, Processor } from '@nestjs/bull'; import { Inject, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; @@ -173,7 +173,8 @@ export class MetadataExtractionProcessor { @Process(JobName.EXIF_EXTRACTION) async extractExifInfo(job: Job) { try { - const { asset, fileName }: { asset: AssetEntity; fileName: string } = job.data; + let asset = job.data.asset; + const fileName = job.data.fileName; const exifData = await exiftool.read(asset.originalPath).catch((e) => { this.logger.warn(`The exifData parsing failed due to: ${e} on file ${asset.originalPath}`); return null; @@ -256,7 +257,8 @@ export class MetadataExtractionProcessor { } await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] }); - await this.assetCore.save({ id: asset.id, fileCreatedAt: fileCreatedAt?.toISOString() }); + asset = await this.assetCore.save({ id: asset.id, fileCreatedAt: fileCreatedAt?.toISOString() }); + await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { asset } }); } catch (error: any) { this.logger.error(`Error extracting EXIF ${error}`, error?.stack); } @@ -273,7 +275,8 @@ export class MetadataExtractionProcessor { @Process({ name: JobName.EXTRACT_VIDEO_METADATA, concurrency: 2 }) async extractVideoMetadata(job: Job) { - const { asset, fileName } = job.data; + let asset = job.data.asset; + const fileName = job.data.fileName; if (!asset.isVisible) { return; @@ -318,6 +321,7 @@ export class MetadataExtractionProcessor { if (photoAsset) { await this.assetCore.save({ id: photoAsset.id, livePhotoVideoId: asset.id }); await this.assetCore.save({ id: asset.id, isVisible: false }); + newExif.imageName = (photoAsset.exifInfo as ExifEntity).imageName; } } @@ -373,7 +377,8 @@ export class MetadataExtractionProcessor { } await this.exifRepository.upsert(newExif, { conflictPaths: ['assetId'] }); - await this.assetCore.save({ id: asset.id, duration: durationString, fileCreatedAt }); + asset = await this.assetCore.save({ id: asset.id, duration: durationString, fileCreatedAt }); + await this.jobRepository.queue({ name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { asset } }); } catch (err) { ``; // do nothing diff --git a/server/libs/domain/src/domain.util.ts b/server/libs/domain/src/domain.util.ts index 5c69912ab5..c38dc7bb19 100644 --- a/server/libs/domain/src/domain.util.ts +++ b/server/libs/domain/src/domain.util.ts @@ -4,6 +4,10 @@ export function getFileNameWithoutExtension(path: string): string { return basename(path, extname(path)); } +export function getLivePhotoMotionFilename(stillName: string, motionName: string) { + return getFileNameWithoutExtension(stillName) + extname(motionName); +} + const KiB = Math.pow(1024, 1); const MiB = Math.pow(1024, 2); const GiB = Math.pow(1024, 3); diff --git a/server/libs/domain/src/job/job.constants.ts b/server/libs/domain/src/job/job.constants.ts index d9d9f99725..9e08f7015a 100644 --- a/server/libs/domain/src/job/job.constants.ts +++ b/server/libs/domain/src/job/job.constants.ts @@ -41,6 +41,7 @@ export enum JobName { // storage template STORAGE_TEMPLATE_MIGRATION = 'storage-template-migration', + STORAGE_TEMPLATE_MIGRATION_SINGLE = 'storage-template-migration-single', SYSTEM_CONFIG_CHANGE = 'system-config-change', // object tagging diff --git a/server/libs/domain/src/job/job.repository.ts b/server/libs/domain/src/job/job.repository.ts index 1c5594ec83..0adea7289f 100644 --- a/server/libs/domain/src/job/job.repository.ts +++ b/server/libs/domain/src/job/job.repository.ts @@ -36,6 +36,7 @@ export type JobItem = // Storage Template | { name: JobName.STORAGE_TEMPLATE_MIGRATION } + | { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE; data: IAssetJob } | { name: JobName.SYSTEM_CONFIG_CHANGE } // Metadata Extraction diff --git a/server/libs/domain/src/storage-template/storage-template.service.ts b/server/libs/domain/src/storage-template/storage-template.service.ts index 947e1b8070..028114d771 100644 --- a/server/libs/domain/src/storage-template/storage-template.service.ts +++ b/server/libs/domain/src/storage-template/storage-template.service.ts @@ -2,6 +2,8 @@ import { AssetEntity, SystemConfig } from '@app/infra/db/entities'; import { Inject, Injectable, Logger } from '@nestjs/common'; import { IAssetRepository } from '../asset/asset.repository'; import { APP_MEDIA_LOCATION } from '../domain.constant'; +import { getLivePhotoMotionFilename } from '../domain.util'; +import { IAssetJob } from '../job'; import { IStorageRepository } from '../storage/storage.repository'; import { INITIAL_SYSTEM_CONFIG, ISystemConfigRepository } from '../system-config'; import { StorageTemplateCore } from './storage-template.core'; @@ -20,6 +22,24 @@ export class StorageTemplateService { this.core = new StorageTemplateCore(configRepository, config, storageRepository); } + async handleTemplateMigrationSingle(data: IAssetJob) { + const { asset } = data; + + try { + const filename = asset.exifInfo?.imageName || asset.id; + await this.moveAsset(asset, filename); + + // move motion part of live photo + if (asset.livePhotoVideoId) { + const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId]); + const motionFilename = getLivePhotoMotionFilename(filename, livePhotoVideo.originalPath); + await this.moveAsset(livePhotoVideo, motionFilename); + } + } catch (error: any) { + this.logger.error('Error running single template migration', error); + } + } + async handleTemplateMigration() { try { console.time('migrating-time'); diff --git a/server/libs/infra/src/job/job.repository.ts b/server/libs/infra/src/job/job.repository.ts index 0371ab50df..b3117d878d 100644 --- a/server/libs/infra/src/job/job.repository.ts +++ b/server/libs/infra/src/job/job.repository.ts @@ -99,6 +99,10 @@ export class JobRepository implements IJobRepository { await this.storageTemplateMigration.add(item.name); break; + case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE: + await this.storageTemplateMigration.add(item.name, item.data); + break; + case JobName.SYSTEM_CONFIG_CHANGE: await this.backgroundTask.add(item.name, {}); break;