From fa30120bc46609ac9bee4f52f28ea9630a8eafc8 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 10 Oct 2024 17:24:48 -0400 Subject: [PATCH] refactor(server): media paths --- server/src/cores/storage.core.ts | 60 +------------------ server/src/enum.ts | 8 --- server/src/interfaces/config.interface.ts | 10 ++++ server/src/repositories/config.repository.ts | 11 ++++ server/src/services/audit.service.ts | 31 ++++------ server/src/services/base.service.ts | 39 ++++++++++++ server/src/services/download.service.ts | 5 +- server/src/services/media.service.ts | 17 +++--- server/src/services/person.service.ts | 3 +- server/src/services/server.service.ts | 7 +-- .../src/services/storage-template.service.ts | 14 ++--- server/src/services/storage.service.ts | 42 ++++++------- server/src/services/user.service.ts | 20 ++++--- server/src/utils/file.ts | 4 ++ server/test/fixtures/library.stub.ts | 2 +- .../repositories/config.repository.mock.ts | 8 +++ .../repositories/storage.repository.mock.ts | 6 +- 17 files changed, 145 insertions(+), 142 deletions(-) diff --git a/server/src/cores/storage.core.ts b/server/src/cores/storage.core.ts index 8e42cd1076..fc8095c469 100644 --- a/server/src/cores/storage.core.ts +++ b/server/src/cores/storage.core.ts @@ -3,7 +3,7 @@ import { dirname, join, resolve } from 'node:path'; import { APP_MEDIA_LOCATION } from 'src/constants'; import { AssetEntity } from 'src/entities/asset.entity'; import { PersonEntity } from 'src/entities/person.entity'; -import { AssetFileType, AssetPathType, ImageFormat, PathType, PersonPathType, StorageFolder } from 'src/enum'; +import { AssetFileType, AssetPathType, ImageFormat, PathType, PersonPathType } from 'src/enum'; import { IAssetRepository } from 'src/interfaces/asset.interface'; import { IConfigRepository } from 'src/interfaces/config.interface'; import { ICryptoRepository } from 'src/interfaces/crypto.interface'; @@ -15,9 +15,6 @@ import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interf import { getAssetFiles } from 'src/utils/asset.util'; import { getConfig } from 'src/utils/config'; -export const THUMBNAIL_DIR = resolve(join(APP_MEDIA_LOCATION, StorageFolder.THUMBNAILS)); -export const ENCODED_VIDEO_DIR = resolve(join(APP_MEDIA_LOCATION, StorageFolder.ENCODED_VIDEO)); - export interface MoveRequest { entityId: string; pathType: PathType; @@ -72,42 +69,6 @@ export class StorageCore { return instance; } - static reset() { - instance = null; - } - - static getFolderLocation(folder: StorageFolder, userId: string) { - return join(StorageCore.getBaseFolder(folder), userId); - } - - static getLibraryFolder(user: { storageLabel: string | null; id: string }) { - return join(StorageCore.getBaseFolder(StorageFolder.LIBRARY), user.storageLabel || user.id); - } - - static getBaseFolder(folder: StorageFolder) { - return join(APP_MEDIA_LOCATION, folder); - } - - static getPersonThumbnailPath(person: PersonEntity) { - return StorageCore.getNestedPath(StorageFolder.THUMBNAILS, person.ownerId, `${person.id}.jpeg`); - } - - static getImagePath(asset: AssetEntity, type: GeneratedImageType, format: ImageFormat) { - return StorageCore.getNestedPath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}-${type}.${format}`); - } - - static getEncodedVideoPath(asset: AssetEntity) { - return StorageCore.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}.mp4`); - } - - static getAndroidMotionPath(asset: AssetEntity, uuid: string) { - return StorageCore.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${uuid}-MP.mp4`); - } - - static isAndroidMotionPath(originalPath: string) { - return originalPath.startsWith(StorageCore.getBaseFolder(StorageFolder.ENCODED_VIDEO)); - } - static isImmichPath(path: string) { const resolvedPath = resolve(path); const resolvedAppMediaLocation = resolve(APP_MEDIA_LOCATION); @@ -118,10 +79,6 @@ export class StorageCore { return normalizedPath.startsWith(normalizedAppMediaLocation); } - static isGeneratedAsset(path: string) { - return path.startsWith(THUMBNAIL_DIR) || path.startsWith(ENCODED_VIDEO_DIR); - } - async moveAssetImage(asset: AssetEntity, pathType: GeneratedImageType, format: ImageFormat) { const { id: entityId, files } = asset; const { thumbnailFile, previewFile } = getAssetFiles(files); @@ -144,6 +101,7 @@ export class StorageCore { } async movePersonFile(person: PersonEntity, pathType: PersonPathType) { + const { mediaPaths } = this.configRepository.getEnv(); const { id: entityId, thumbnailPath } = person; switch (pathType) { case PersonPathType.FACE: { @@ -151,7 +109,7 @@ export class StorageCore { entityId, pathType, oldPath: thumbnailPath, - newPath: StorageCore.getPersonThumbnailPath(person), + newPath: buildPath({ mediaPaths, personThumbnail: person }), }); } } @@ -275,10 +233,6 @@ export class StorageCore { this.storageRepository.mkdirSync(dirname(input)); } - removeEmptyDirs(folder: StorageFolder) { - return this.storageRepository.removeEmptyDirs(StorageCore.getBaseFolder(folder)); - } - private savePath(pathType: PathType, id: string, newPath: string) { switch (pathType) { case AssetPathType.ORIGINAL: { @@ -302,14 +256,6 @@ export class StorageCore { } } - static getNestedFolder(folder: StorageFolder, ownerId: string, filename: string): string { - return join(StorageCore.getFolderLocation(folder, ownerId), filename.slice(0, 2), filename.slice(2, 4)); - } - - static getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string { - return join(this.getNestedFolder(folder, ownerId, filename), filename); - } - static getTempPathInDir(dir: string): string { return join(dir, `${randomUUID()}.tmp`); } diff --git a/server/src/enum.ts b/server/src/enum.ts index 109e9a90b7..18d26e04f9 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -153,14 +153,6 @@ export enum SharedLinkType { INDIVIDUAL = 'INDIVIDUAL', } -export enum StorageFolder { - ENCODED_VIDEO = 'encoded-video', - LIBRARY = 'library', - UPLOAD = 'upload', - PROFILE = 'profile', - THUMBNAILS = 'thumbs', -} - export enum SystemMetadataKey { REVERSE_GEOCODING_STATE = 'reverse-geocoding-state', FACIAL_RECOGNITION_STATE = 'facial-recognition-state', diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.ts index d105e40cf9..40c04a3bbc 100644 --- a/server/src/interfaces/config.interface.ts +++ b/server/src/interfaces/config.interface.ts @@ -3,6 +3,14 @@ import { VectorExtension } from 'src/interfaces/database.interface'; export const IConfigRepository = 'IConfigRepository'; +export type MediaPaths = { + uploads: string; + library: string; + profile: string; + thumbnails: string; + encodedVideos: string; +}; + export interface EnvData { port: number; environment: ImmichEnvironment; @@ -41,6 +49,8 @@ export interface EnvData { server: string; }; + mediaPaths: MediaPaths; + resourcePaths: { lockFile: string; geodata: { diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index d9b7c36384..fe1bc276a2 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -44,6 +44,9 @@ export class ConfigRepository implements IConfigRepository { const environment = process.env.IMMICH_ENV as ImmichEnvironment; const isProd = environment === ImmichEnvironment.PRODUCTION; const buildFolder = process.env.IMMICH_BUILD_DATA || '/build'; + // TODO change default to /data or similar + const uploadFolder = process.env.IMMICH_MEDIA_LOCATION || './upload'; + const folders = { geodata: join(buildFolder, 'geodata'), web: join(buildFolder, 'www'), @@ -86,6 +89,14 @@ export class ConfigRepository implements IConfigRepository { licensePublicKey: isProd ? productionKeys : stagingKeys, + mediaPaths: { + uploads: join(uploadFolder, 'upload'), + library: join(uploadFolder, 'library'), + profile: join(uploadFolder, 'profile'), + thumbnails: join(uploadFolder, 'thumbs'), + encodedVideos: join(uploadFolder, 'encoded-video'), + }, + resourcePaths: { lockFile: join(buildFolder, 'build-lock.json'), geodata: { diff --git a/server/src/services/audit.service.ts b/server/src/services/audit.service.ts index d891c88b39..30a73da2d9 100644 --- a/server/src/services/audit.service.ts +++ b/server/src/services/audit.service.ts @@ -12,15 +12,7 @@ import { PathEntityType, } from 'src/dtos/audit.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { - AssetFileType, - AssetPathType, - DatabaseAction, - Permission, - PersonPathType, - StorageFolder, - UserPathType, -} from 'src/enum'; +import { AssetFileType, AssetPathType, DatabaseAction, Permission, PersonPathType, UserPathType } from 'src/enum'; import { JOBS_ASSET_PAGINATION_SIZE, JobStatus } from 'src/interfaces/job.interface'; import { BaseService } from 'src/services/base.service'; import { getAssetFiles } from 'src/utils/asset.util'; @@ -119,19 +111,16 @@ export class AuditService extends BaseService { async getFileReport() { const hasFile = (items: Set, filename: string) => items.has(filename) || items.has(this.fullPath(filename)); - const crawl = async (folder: StorageFolder) => - new Set( - await this.storageRepository.crawl({ - includeHidden: true, - pathsToCrawl: [StorageCore.getBaseFolder(folder)], - }), - ); + const crawl = async (folder: string) => + new Set(await this.storageRepository.crawl({ includeHidden: true, pathsToCrawl: [folder] })); - const uploadFiles = await crawl(StorageFolder.UPLOAD); - const libraryFiles = await crawl(StorageFolder.LIBRARY); - const thumbFiles = await crawl(StorageFolder.THUMBNAILS); - const videoFiles = await crawl(StorageFolder.ENCODED_VIDEO); - const profileFiles = await crawl(StorageFolder.PROFILE); + const { mediaPaths } = this.configRepository.getEnv(); + + const uploadFiles = await crawl(mediaPaths.uploads); + const libraryFiles = await crawl(mediaPaths.library); + const thumbFiles = await crawl(mediaPaths.thumbnails); + const videoFiles = await crawl(mediaPaths.encodedVideos); + const profileFiles = await crawl(mediaPaths.profile); const allFiles = new Set(); for (const list of [libraryFiles, thumbFiles, videoFiles, profileFiles, uploadFiles]) { for (const item of list) { diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index 2bb717b45b..78a9bcf801 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -1,6 +1,8 @@ import { Inject } from '@nestjs/common'; +import { join } from 'node:path'; import { SystemConfig } from 'src/config'; import { StorageCore } from 'src/cores/storage.core'; +import { AssetPathType } from 'src/enum'; import { IAccessRepository } from 'src/interfaces/access.interface'; import { IActivityRepository } from 'src/interfaces/activity.interface'; import { IAlbumUserRepository } from 'src/interfaces/album-user.interface'; @@ -41,6 +43,11 @@ import { IViewRepository } from 'src/interfaces/view.interface'; import { AccessRequest, checkAccess, requireAccess } from 'src/utils/access'; import { getConfig, updateConfig } from 'src/utils/config'; +type PathRequest = { id: string; ownerId: string }; +type ThumbnailPathRequest = PathRequest & { image: SystemConfig['image'] }; + +const asNestedPath = (filename: string) => join(filename.slice(0, 2), filename.slice(2, 4), filename); + export class BaseService { protected storageCore: StorageCore; @@ -119,4 +126,36 @@ export class BaseService { checkAccess(request: AccessRequest) { return checkAccess(this.accessRepository, request); } + + buildAssetPreviewPath({ image, id, ownerId }: ThumbnailPathRequest) { + const { mediaPaths } = this.configRepository.getEnv(); + const filename = `${id}-${AssetPathType.PREVIEW}.${image.preview.format}`; + return join(mediaPaths.thumbnails, ownerId, asNestedPath(filename)); + } + + buildAssetThumbnailPath({ image, id, ownerId }: ThumbnailPathRequest) { + const { mediaPaths } = this.configRepository.getEnv(); + const filename = `${id}-${AssetPathType.THUMBNAIL}.${image.thumbnail.format}`; + return join(mediaPaths.thumbnails, ownerId, asNestedPath(filename)); + } + + buildPersonThumbnailPath({ id, ownerId }: PathRequest) { + const { mediaPaths } = this.configRepository.getEnv(); + return join(mediaPaths.thumbnails, ownerId, asNestedPath(`${id}.jpeg`)); + } + + buildEncodedVideoPath({ id, ownerId }: PathRequest) { + const { mediaPaths } = this.configRepository.getEnv(); + return join(mediaPaths.encodedVideos, ownerId, asNestedPath(`${id}.mp4`)); + } + + buildAndroidMotionPath({ id, ownerId }: PathRequest) { + const { mediaPaths } = this.configRepository.getEnv(); + return join(mediaPaths.encodedVideos, ownerId, asNestedPath(`${id}-MP.mp4`)); + } + + buildLibraryFolder({ id, storageLabel }: { id: string; storageLabel: string | null }) { + const { mediaPaths } = this.configRepository.getEnv(); + return join(mediaPaths.library, storageLabel || id); + } } diff --git a/server/src/services/download.service.ts b/server/src/services/download.service.ts index 3d66f009cf..369b844431 100644 --- a/server/src/services/download.service.ts +++ b/server/src/services/download.service.ts @@ -1,6 +1,5 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import { parse } from 'node:path'; -import { StorageCore } from 'src/cores/storage.core'; import { AssetIdsDto } from 'src/dtos/asset.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { DownloadArchiveInfo, DownloadInfoDto, DownloadResponseDto } from 'src/dtos/download.dto'; @@ -9,6 +8,7 @@ import { Permission } from 'src/enum'; import { ImmichReadStream } from 'src/interfaces/storage.interface'; import { BaseService } from 'src/services/base.service'; import { HumanReadableSize } from 'src/utils/bytes'; +import { isAndroidMotionPath } from 'src/utils/file'; import { usePagination } from 'src/utils/pagination'; import { getPreferences } from 'src/utils/preferences'; @@ -20,6 +20,7 @@ export class DownloadService extends BaseService { let archive: DownloadArchiveInfo = { size: 0, assetIds: [] }; const preferences = getPreferences(auth.user); + const { mediaPaths } = this.configRepository.getEnv(); const assetPagination = await this.getDownloadAssets(auth, dto); for await (const assets of assetPagination) { @@ -29,7 +30,7 @@ export class DownloadService extends BaseService { const motionAssets = await this.assetRepository.getByIds(motionIds, { exifInfo: true }); for (const motionAsset of motionAssets) { if ( - !StorageCore.isAndroidMotionPath(motionAsset.originalPath) || + !isAndroidMotionPath(mediaPaths, motionAsset.originalPath) || preferences.download.includeEmbeddedVideos ) { assets.push(motionAsset); diff --git a/server/src/services/media.service.ts b/server/src/services/media.service.ts index e4fd91f363..d25db9f335 100644 --- a/server/src/services/media.service.ts +++ b/server/src/services/media.service.ts @@ -10,7 +10,6 @@ import { AudioCodec, Colorspace, LogLevel, - StorageFolder, TranscodeHWAccel, TranscodePolicy, TranscodeTarget, @@ -97,8 +96,9 @@ export class MediaService extends BaseService { const { active, waiting } = await this.jobRepository.getJobCounts(QueueName.MIGRATION); if (active === 1 && waiting === 0) { - await this.storageCore.removeEmptyDirs(StorageFolder.THUMBNAILS); - await this.storageCore.removeEmptyDirs(StorageFolder.ENCODED_VIDEO); + const { mediaPaths } = this.configRepository.getEnv(); + await this.storageRepository.removeEmptyDirs(mediaPaths.thumbnails); + await this.storageRepository.removeEmptyDirs(mediaPaths.encodedVideos); } for await (const assets of assetPagination) { @@ -196,8 +196,9 @@ export class MediaService extends BaseService { private async generateImageThumbnails(asset: AssetEntity) { const { image } = await this.getConfig({ withCache: true }); - const previewPath = StorageCore.getImagePath(asset, AssetPathType.PREVIEW, image.preview.format); - const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.THUMBNAIL, image.thumbnail.format); + + const previewPath = this.buildAssetPreviewPath({ image, id: asset.id, ownerId: asset.ownerId }); + const thumbnailPath = this.buildAssetThumbnailPath({ image, id: asset.id, ownerId: asset.ownerId }); this.storageCore.ensureFolders(previewPath); const shouldExtract = image.extractEmbedded && mimeTypes.isRaw(asset.originalPath); @@ -230,8 +231,8 @@ export class MediaService extends BaseService { private async generateVideoThumbnails(asset: AssetEntity) { const { image, ffmpeg } = await this.getConfig({ withCache: true }); - const previewPath = StorageCore.getImagePath(asset, AssetPathType.PREVIEW, image.preview.format); - const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.THUMBNAIL, image.thumbnail.format); + const thumbnailPath = this.buildAssetThumbnailPath({ image, id: asset.id, ownerId: asset.ownerId }); + const previewPath = this.buildAssetPreviewPath({ image, id: asset.id, ownerId: asset.ownerId }); this.storageCore.ensureFolders(previewPath); const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); @@ -282,7 +283,7 @@ export class MediaService extends BaseService { } const input = asset.originalPath; - const output = StorageCore.getEncodedVideoPath(asset); + const output = this.buildEncodedVideoPath({ id, ownerId: asset.ownerId }); this.storageCore.ensureFolders(output); const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input, { diff --git a/server/src/services/person.service.ts b/server/src/services/person.service.ts index e5f016d8ef..10ed93f821 100644 --- a/server/src/services/person.service.ts +++ b/server/src/services/person.service.ts @@ -1,6 +1,5 @@ import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common'; import { FACE_THUMBNAIL_SIZE } from 'src/constants'; -import { StorageCore } from 'src/cores/storage.core'; import { BulkIdErrorReason, BulkIdResponseDto } from 'src/dtos/asset-ids.response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { @@ -559,7 +558,7 @@ export class PersonService extends BaseService { const { width, height, inputPath } = await this.getInputDimensions(asset, { width: oldWidth, height: oldHeight }); - const thumbnailPath = StorageCore.getPersonThumbnailPath(person); + const thumbnailPath = this.buildPersonThumbnailPath(person); this.storageCore.ensureFolders(thumbnailPath); const thumbnailOptions = { diff --git a/server/src/services/server.service.ts b/server/src/services/server.service.ts index 3fc319a2fd..d842ee17f7 100644 --- a/server/src/services/server.service.ts +++ b/server/src/services/server.service.ts @@ -1,6 +1,5 @@ import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common'; import { serverVersion } from 'src/constants'; -import { StorageCore } from 'src/cores/storage.core'; import { OnEvent } from 'src/decorators'; import { LicenseKeyDto, LicenseResponseDto } from 'src/dtos/license.dto'; import { @@ -13,7 +12,7 @@ import { ServerStorageResponseDto, UsageByUserDto, } from 'src/dtos/server.dto'; -import { StorageFolder, SystemMetadataKey } from 'src/enum'; +import { SystemMetadataKey } from 'src/enum'; import { UserStatsQueryResponse } from 'src/interfaces/user.interface'; import { BaseService } from 'src/services/base.service'; import { asHumanReadable } from 'src/utils/bytes'; @@ -49,8 +48,8 @@ export class ServerService extends BaseService { } async getStorage(): Promise { - const libraryBase = StorageCore.getBaseFolder(StorageFolder.LIBRARY); - const diskInfo = await this.storageRepository.checkDiskUsage(libraryBase); + const { mediaPaths } = this.configRepository.getEnv(); + const diskInfo = await this.storageRepository.checkDiskUsage(mediaPaths.library); const usagePercentage = (((diskInfo.total - diskInfo.free) / diskInfo.total) * 100).toFixed(2); diff --git a/server/src/services/storage-template.service.ts b/server/src/services/storage-template.service.ts index e400981f54..2875d833da 100644 --- a/server/src/services/storage-template.service.ts +++ b/server/src/services/storage-template.service.ts @@ -12,15 +12,14 @@ import { supportedWeekTokens, supportedYearTokens, } from 'src/constants'; -import { StorageCore } from 'src/cores/storage.core'; import { OnEvent } from 'src/decorators'; import { AssetEntity } from 'src/entities/asset.entity'; -import { AssetPathType, AssetType, StorageFolder } from 'src/enum'; +import { AssetPathType, AssetType } from 'src/enum'; import { DatabaseLock } from 'src/interfaces/database.interface'; import { ArgOf } from 'src/interfaces/event.interface'; import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobStatus } from 'src/interfaces/job.interface'; import { BaseService } from 'src/services/base.service'; -import { getLivePhotoMotionFilename } from 'src/utils/file'; +import { getLivePhotoMotionFilename, isAndroidMotionPath } from 'src/utils/file'; import { usePagination } from 'src/utils/pagination'; export interface MoveAssetMetadata { @@ -132,8 +131,8 @@ export class StorageTemplateService extends BaseService { } this.logger.debug('Cleaning up empty directories...'); - const libraryFolder = StorageCore.getBaseFolder(StorageFolder.LIBRARY); - await this.storageRepository.removeEmptyDirs(libraryFolder); + const { mediaPaths } = this.configRepository.getEnv(); + await this.storageRepository.removeEmptyDirs(mediaPaths.library); this.logger.log('Finished storage template migration'); @@ -141,7 +140,8 @@ export class StorageTemplateService extends BaseService { } async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) { - if (asset.isExternal || StorageCore.isAndroidMotionPath(asset.originalPath)) { + const { mediaPaths } = this.configRepository.getEnv(); + if (asset.isExternal || isAndroidMotionPath(mediaPaths, asset.originalPath)) { // External assets are not affected by storage template // TODO: shouldn't this only apply to external assets? return; @@ -186,7 +186,7 @@ export class StorageTemplateService extends BaseService { const source = asset.originalPath; const extension = path.extname(source).split('.').pop() as string; const sanitized = sanitize(path.basename(filename, `.${extension}`)); - const rootPath = StorageCore.getLibraryFolder({ id: asset.ownerId, storageLabel }); + const rootPath = this.buildLibraryFolder({ id: asset.ownerId, storageLabel }); let albumName = null; if (this.template.needsAlbum) { diff --git a/server/src/services/storage.service.ts b/server/src/services/storage.service.ts index e8620b4371..82e640618f 100644 --- a/server/src/services/storage.service.ts +++ b/server/src/services/storage.service.ts @@ -1,8 +1,7 @@ import { Injectable } from '@nestjs/common'; import { join } from 'node:path'; -import { StorageCore } from 'src/cores/storage.core'; import { OnEvent } from 'src/decorators'; -import { StorageFolder, SystemMetadataKey } from 'src/enum'; +import { SystemMetadataKey } from 'src/enum'; import { DatabaseLock } from 'src/interfaces/database.interface'; import { IDeleteFilesJob, JobStatus } from 'src/interfaces/job.interface'; import { BaseService } from 'src/services/base.service'; @@ -10,6 +9,8 @@ import { BaseService } from 'src/services/base.service'; export class ImmichStartupError extends Error {} export const isStartUpError = (error: unknown): error is ImmichStartupError => error instanceof ImmichStartupError; +type MountPaths = { folderPath: string; internalPath: string; externalPath: string }; + const docsMessage = `Please see https://immich.app/docs/administration/system-integrity#folder-checks for more information.`; @Injectable() @@ -17,6 +18,14 @@ export class StorageService extends BaseService { @OnEvent({ name: 'app.bootstrap' }) async onBootstrap() { const envData = this.configRepository.getEnv(); + const { mediaPaths } = this.configRepository.getEnv(); + const folders = [ + mediaPaths.encodedVideos, + mediaPaths.library, + mediaPaths.profile, + mediaPaths.thumbnails, + mediaPaths.uploads, + ]; await this.databaseRepository.withLock(DatabaseLock.SystemFileMounts, async () => { const flags = (await this.systemMetadataRepository.get(SystemMetadataKey.SYSTEM_FLAGS)) || { mountFiles: false }; @@ -26,14 +35,18 @@ export class StorageService extends BaseService { try { // check each folder exists and is writable - for (const folder of Object.values(StorageFolder)) { + for (const folder of folders) { + const internalPath = join(folder, '.immich'); + const externalPath = `/${folder.split('/').pop()}/.immich`; + const paths = { internalPath, externalPath, folderPath: folder }; + if (!enabled) { this.logger.log(`Writing initial mount file for the ${folder} folder`); - await this.createMountFile(folder); + await this.createMountFile(paths); } - await this.verifyReadAccess(folder); - await this.verifyWriteAccess(folder); + await this.verifyReadAccess(paths); + await this.verifyWriteAccess(paths); } if (!flags.mountFiles) { @@ -73,8 +86,7 @@ export class StorageService extends BaseService { return JobStatus.SUCCESS; } - private async verifyReadAccess(folder: StorageFolder) { - const { internalPath, externalPath } = this.getMountFilePaths(folder); + private async verifyReadAccess({ internalPath, externalPath }: MountPaths) { try { await this.storageRepository.readFile(internalPath); } catch (error) { @@ -83,8 +95,7 @@ export class StorageService extends BaseService { } } - private async createMountFile(folder: StorageFolder) { - const { folderPath, internalPath, externalPath } = this.getMountFilePaths(folder); + private async createMountFile({ folderPath, internalPath, externalPath }: MountPaths) { try { this.storageRepository.mkdirSync(folderPath); await this.storageRepository.createFile(internalPath, Buffer.from(`${Date.now()}`)); @@ -98,8 +109,7 @@ export class StorageService extends BaseService { } } - private async verifyWriteAccess(folder: StorageFolder) { - const { internalPath, externalPath } = this.getMountFilePaths(folder); + private async verifyWriteAccess({ internalPath, externalPath }: MountPaths) { try { await this.storageRepository.overwriteFile(internalPath, Buffer.from(`${Date.now()}`)); } catch (error) { @@ -107,12 +117,4 @@ export class StorageService extends BaseService { throw new ImmichStartupError(`Failed to write "${externalPath} - ${docsMessage}"`); } } - - private getMountFilePaths(folder: StorageFolder) { - const folderPath = StorageCore.getBaseFolder(folder); - const internalPath = join(folderPath, '.immich'); - const externalPath = `/${folder}/.immich`; - - return { folderPath, internalPath, externalPath }; - } } diff --git a/server/src/services/user.service.ts b/server/src/services/user.service.ts index f67d04cbd3..b798d06835 100644 --- a/server/src/services/user.service.ts +++ b/server/src/services/user.service.ts @@ -1,7 +1,7 @@ import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common'; import { DateTime } from 'luxon'; +import { join } from 'node:path'; import { SALT_ROUNDS } from 'src/constants'; -import { StorageCore } from 'src/cores/storage.core'; import { AuthDto } from 'src/dtos/auth.dto'; import { LicenseKeyDto, LicenseResponseDto } from 'src/dtos/license.dto'; import { UserPreferencesResponseDto, UserPreferencesUpdateDto, mapPreferences } from 'src/dtos/user-preferences.dto'; @@ -9,7 +9,7 @@ import { CreateProfileImageResponseDto } from 'src/dtos/user-profile.dto'; import { UserAdminResponseDto, UserResponseDto, UserUpdateMeDto, mapUser, mapUserAdmin } from 'src/dtos/user.dto'; import { UserMetadataEntity } from 'src/entities/user-metadata.entity'; import { UserEntity } from 'src/entities/user.entity'; -import { CacheControl, StorageFolder, UserMetadataKey } from 'src/enum'; +import { CacheControl, UserMetadataKey } from 'src/enum'; import { IEntityJob, JobName, JobStatus } from 'src/interfaces/job.interface'; import { UserFindOptions } from 'src/interfaces/user.interface'; import { BaseService } from 'src/services/base.service'; @@ -196,14 +196,20 @@ export class UserService extends BaseService { this.logger.log(`Deleting user: ${user.id}`); + const { mediaPaths } = this.configRepository.getEnv(); + const folders = [ - StorageCore.getLibraryFolder(user), - StorageCore.getFolderLocation(StorageFolder.UPLOAD, user.id), - StorageCore.getFolderLocation(StorageFolder.PROFILE, user.id), - StorageCore.getFolderLocation(StorageFolder.THUMBNAILS, user.id), - StorageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, user.id), + join(mediaPaths.library, user.id), + join(mediaPaths.uploads, user.id), + join(mediaPaths.profile, user.id), + join(mediaPaths.thumbnails, user.id), + join(mediaPaths.encodedVideos, user.id), ]; + if (user.storageLabel) { + folders.push(join(mediaPaths.library, user.storageLabel)); + } + for (const folder of folders) { this.logger.warn(`Removing user from filesystem: ${folder}`); await this.storageRepository.unlinkDir(folder, { recursive: true, force: true }); diff --git a/server/src/utils/file.ts b/server/src/utils/file.ts index 3b26c3e1ba..1cee8e188e 100644 --- a/server/src/utils/file.ts +++ b/server/src/utils/file.ts @@ -4,6 +4,7 @@ import { access, constants } from 'node:fs/promises'; import { basename, extname, isAbsolute } from 'node:path'; import { promisify } from 'node:util'; import { CacheControl } from 'src/enum'; +import { MediaPaths } from 'src/interfaces/config.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { ImmichReadStream } from 'src/interfaces/storage.interface'; import { isConnectionAborted } from 'src/utils/misc'; @@ -20,6 +21,9 @@ export function getLivePhotoMotionFilename(stillName: string, motionName: string return getFileNameWithoutExtension(stillName) + extname(motionName); } +export const isAndroidMotionPath = (mediaPaths: MediaPaths, filePath: string) => + filePath.startsWith(mediaPaths.encodedVideos); + export class ImmichFileResponse { public readonly path!: string; public readonly contentType!: string; diff --git a/server/test/fixtures/library.stub.ts b/server/test/fixtures/library.stub.ts index 1a83ffe5d7..9978795c52 100644 --- a/server/test/fixtures/library.stub.ts +++ b/server/test/fixtures/library.stub.ts @@ -83,7 +83,7 @@ export const libraryStub = { assets: [], owner: userStub.admin, ownerId: 'user-id', - importPaths: [join(THUMBNAIL_DIR, 'library'), '/xyz', join(APP_MEDIA_LOCATION, 'library')], + importPaths: ['upload/thumbs/library', '/xyz', 'upload/library'], createdAt: new Date('2023-01-01'), updatedAt: new Date('2023-01-01'), refreshedAt: null, diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts index 852868ee31..050a4bc6a2 100644 --- a/server/test/repositories/config.repository.mock.ts +++ b/server/test/repositories/config.repository.mock.ts @@ -25,6 +25,14 @@ const envData: EnvData = { server: 'server-public-key', }, + mediaPaths: { + uploads: 'upload/upload', + library: 'upload/library', + profile: 'upload/profile', + thumbnails: 'upload/thumbs', + encodedVideos: 'upload/encoded-video', + }, + resourcePaths: { lockFile: 'build-lock.json', geodata: { diff --git a/server/test/repositories/storage.repository.mock.ts b/server/test/repositories/storage.repository.mock.ts index 5226e0bb1e..aa14a3c692 100644 --- a/server/test/repositories/storage.repository.mock.ts +++ b/server/test/repositories/storage.repository.mock.ts @@ -39,11 +39,7 @@ export const makeMockWatcher = return () => Promise.resolve(); }; -export const newStorageRepositoryMock = (reset = true): Mocked => { - if (reset) { - StorageCore.reset(); - } - +export const newStorageRepositoryMock = (): Mocked => { return { createZipStream: vitest.fn(), createReadStream: vitest.fn(),