1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2024-12-28 22:51:59 +00:00

refactor(server): media paths

This commit is contained in:
Jason Rasmussen 2024-10-10 17:24:48 -04:00
parent 8daa8073ae
commit fa30120bc4
No known key found for this signature in database
GPG key ID: 2EF24B77EAFA4A41
17 changed files with 145 additions and 142 deletions

View file

@ -3,7 +3,7 @@ import { dirname, join, resolve } from 'node:path';
import { APP_MEDIA_LOCATION } from 'src/constants'; import { APP_MEDIA_LOCATION } from 'src/constants';
import { AssetEntity } from 'src/entities/asset.entity'; import { AssetEntity } from 'src/entities/asset.entity';
import { PersonEntity } from 'src/entities/person.entity'; import { PersonEntity } from 'src/entities/person.entity';
import { AssetFileType, AssetPathType, ImageFormat, PathType, PersonPathType, StorageFolder } from 'src/enum'; import { AssetFileType, AssetPathType, ImageFormat, PathType, PersonPathType } from 'src/enum';
import { IAssetRepository } from 'src/interfaces/asset.interface'; import { IAssetRepository } from 'src/interfaces/asset.interface';
import { IConfigRepository } from 'src/interfaces/config.interface'; import { IConfigRepository } from 'src/interfaces/config.interface';
import { ICryptoRepository } from 'src/interfaces/crypto.interface'; import { ICryptoRepository } from 'src/interfaces/crypto.interface';
@ -15,9 +15,6 @@ import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interf
import { getAssetFiles } from 'src/utils/asset.util'; import { getAssetFiles } from 'src/utils/asset.util';
import { getConfig } from 'src/utils/config'; import { getConfig } from 'src/utils/config';
export const THUMBNAIL_DIR = resolve(join(APP_MEDIA_LOCATION, StorageFolder.THUMBNAILS));
export const ENCODED_VIDEO_DIR = resolve(join(APP_MEDIA_LOCATION, StorageFolder.ENCODED_VIDEO));
export interface MoveRequest { export interface MoveRequest {
entityId: string; entityId: string;
pathType: PathType; pathType: PathType;
@ -72,42 +69,6 @@ export class StorageCore {
return instance; return instance;
} }
static reset() {
instance = null;
}
static getFolderLocation(folder: StorageFolder, userId: string) {
return join(StorageCore.getBaseFolder(folder), userId);
}
static getLibraryFolder(user: { storageLabel: string | null; id: string }) {
return join(StorageCore.getBaseFolder(StorageFolder.LIBRARY), user.storageLabel || user.id);
}
static getBaseFolder(folder: StorageFolder) {
return join(APP_MEDIA_LOCATION, folder);
}
static getPersonThumbnailPath(person: PersonEntity) {
return StorageCore.getNestedPath(StorageFolder.THUMBNAILS, person.ownerId, `${person.id}.jpeg`);
}
static getImagePath(asset: AssetEntity, type: GeneratedImageType, format: ImageFormat) {
return StorageCore.getNestedPath(StorageFolder.THUMBNAILS, asset.ownerId, `${asset.id}-${type}.${format}`);
}
static getEncodedVideoPath(asset: AssetEntity) {
return StorageCore.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${asset.id}.mp4`);
}
static getAndroidMotionPath(asset: AssetEntity, uuid: string) {
return StorageCore.getNestedPath(StorageFolder.ENCODED_VIDEO, asset.ownerId, `${uuid}-MP.mp4`);
}
static isAndroidMotionPath(originalPath: string) {
return originalPath.startsWith(StorageCore.getBaseFolder(StorageFolder.ENCODED_VIDEO));
}
static isImmichPath(path: string) { static isImmichPath(path: string) {
const resolvedPath = resolve(path); const resolvedPath = resolve(path);
const resolvedAppMediaLocation = resolve(APP_MEDIA_LOCATION); const resolvedAppMediaLocation = resolve(APP_MEDIA_LOCATION);
@ -118,10 +79,6 @@ export class StorageCore {
return normalizedPath.startsWith(normalizedAppMediaLocation); return normalizedPath.startsWith(normalizedAppMediaLocation);
} }
static isGeneratedAsset(path: string) {
return path.startsWith(THUMBNAIL_DIR) || path.startsWith(ENCODED_VIDEO_DIR);
}
async moveAssetImage(asset: AssetEntity, pathType: GeneratedImageType, format: ImageFormat) { async moveAssetImage(asset: AssetEntity, pathType: GeneratedImageType, format: ImageFormat) {
const { id: entityId, files } = asset; const { id: entityId, files } = asset;
const { thumbnailFile, previewFile } = getAssetFiles(files); const { thumbnailFile, previewFile } = getAssetFiles(files);
@ -144,6 +101,7 @@ export class StorageCore {
} }
async movePersonFile(person: PersonEntity, pathType: PersonPathType) { async movePersonFile(person: PersonEntity, pathType: PersonPathType) {
const { mediaPaths } = this.configRepository.getEnv();
const { id: entityId, thumbnailPath } = person; const { id: entityId, thumbnailPath } = person;
switch (pathType) { switch (pathType) {
case PersonPathType.FACE: { case PersonPathType.FACE: {
@ -151,7 +109,7 @@ export class StorageCore {
entityId, entityId,
pathType, pathType,
oldPath: thumbnailPath, oldPath: thumbnailPath,
newPath: StorageCore.getPersonThumbnailPath(person), newPath: buildPath({ mediaPaths, personThumbnail: person }),
}); });
} }
} }
@ -275,10 +233,6 @@ export class StorageCore {
this.storageRepository.mkdirSync(dirname(input)); this.storageRepository.mkdirSync(dirname(input));
} }
removeEmptyDirs(folder: StorageFolder) {
return this.storageRepository.removeEmptyDirs(StorageCore.getBaseFolder(folder));
}
private savePath(pathType: PathType, id: string, newPath: string) { private savePath(pathType: PathType, id: string, newPath: string) {
switch (pathType) { switch (pathType) {
case AssetPathType.ORIGINAL: { case AssetPathType.ORIGINAL: {
@ -302,14 +256,6 @@ export class StorageCore {
} }
} }
static getNestedFolder(folder: StorageFolder, ownerId: string, filename: string): string {
return join(StorageCore.getFolderLocation(folder, ownerId), filename.slice(0, 2), filename.slice(2, 4));
}
static getNestedPath(folder: StorageFolder, ownerId: string, filename: string): string {
return join(this.getNestedFolder(folder, ownerId, filename), filename);
}
static getTempPathInDir(dir: string): string { static getTempPathInDir(dir: string): string {
return join(dir, `${randomUUID()}.tmp`); return join(dir, `${randomUUID()}.tmp`);
} }

View file

@ -153,14 +153,6 @@ export enum SharedLinkType {
INDIVIDUAL = 'INDIVIDUAL', INDIVIDUAL = 'INDIVIDUAL',
} }
export enum StorageFolder {
ENCODED_VIDEO = 'encoded-video',
LIBRARY = 'library',
UPLOAD = 'upload',
PROFILE = 'profile',
THUMBNAILS = 'thumbs',
}
export enum SystemMetadataKey { export enum SystemMetadataKey {
REVERSE_GEOCODING_STATE = 'reverse-geocoding-state', REVERSE_GEOCODING_STATE = 'reverse-geocoding-state',
FACIAL_RECOGNITION_STATE = 'facial-recognition-state', FACIAL_RECOGNITION_STATE = 'facial-recognition-state',

View file

@ -3,6 +3,14 @@ import { VectorExtension } from 'src/interfaces/database.interface';
export const IConfigRepository = 'IConfigRepository'; export const IConfigRepository = 'IConfigRepository';
export type MediaPaths = {
uploads: string;
library: string;
profile: string;
thumbnails: string;
encodedVideos: string;
};
export interface EnvData { export interface EnvData {
port: number; port: number;
environment: ImmichEnvironment; environment: ImmichEnvironment;
@ -41,6 +49,8 @@ export interface EnvData {
server: string; server: string;
}; };
mediaPaths: MediaPaths;
resourcePaths: { resourcePaths: {
lockFile: string; lockFile: string;
geodata: { geodata: {

View file

@ -44,6 +44,9 @@ export class ConfigRepository implements IConfigRepository {
const environment = process.env.IMMICH_ENV as ImmichEnvironment; const environment = process.env.IMMICH_ENV as ImmichEnvironment;
const isProd = environment === ImmichEnvironment.PRODUCTION; const isProd = environment === ImmichEnvironment.PRODUCTION;
const buildFolder = process.env.IMMICH_BUILD_DATA || '/build'; const buildFolder = process.env.IMMICH_BUILD_DATA || '/build';
// TODO change default to /data or similar
const uploadFolder = process.env.IMMICH_MEDIA_LOCATION || './upload';
const folders = { const folders = {
geodata: join(buildFolder, 'geodata'), geodata: join(buildFolder, 'geodata'),
web: join(buildFolder, 'www'), web: join(buildFolder, 'www'),
@ -86,6 +89,14 @@ export class ConfigRepository implements IConfigRepository {
licensePublicKey: isProd ? productionKeys : stagingKeys, licensePublicKey: isProd ? productionKeys : stagingKeys,
mediaPaths: {
uploads: join(uploadFolder, 'upload'),
library: join(uploadFolder, 'library'),
profile: join(uploadFolder, 'profile'),
thumbnails: join(uploadFolder, 'thumbs'),
encodedVideos: join(uploadFolder, 'encoded-video'),
},
resourcePaths: { resourcePaths: {
lockFile: join(buildFolder, 'build-lock.json'), lockFile: join(buildFolder, 'build-lock.json'),
geodata: { geodata: {

View file

@ -12,15 +12,7 @@ import {
PathEntityType, PathEntityType,
} from 'src/dtos/audit.dto'; } from 'src/dtos/audit.dto';
import { AuthDto } from 'src/dtos/auth.dto'; import { AuthDto } from 'src/dtos/auth.dto';
import { import { AssetFileType, AssetPathType, DatabaseAction, Permission, PersonPathType, UserPathType } from 'src/enum';
AssetFileType,
AssetPathType,
DatabaseAction,
Permission,
PersonPathType,
StorageFolder,
UserPathType,
} from 'src/enum';
import { JOBS_ASSET_PAGINATION_SIZE, JobStatus } from 'src/interfaces/job.interface'; import { JOBS_ASSET_PAGINATION_SIZE, JobStatus } from 'src/interfaces/job.interface';
import { BaseService } from 'src/services/base.service'; import { BaseService } from 'src/services/base.service';
import { getAssetFiles } from 'src/utils/asset.util'; import { getAssetFiles } from 'src/utils/asset.util';
@ -119,19 +111,16 @@ export class AuditService extends BaseService {
async getFileReport() { async getFileReport() {
const hasFile = (items: Set<string>, filename: string) => items.has(filename) || items.has(this.fullPath(filename)); const hasFile = (items: Set<string>, filename: string) => items.has(filename) || items.has(this.fullPath(filename));
const crawl = async (folder: StorageFolder) => const crawl = async (folder: string) =>
new Set( new Set(await this.storageRepository.crawl({ includeHidden: true, pathsToCrawl: [folder] }));
await this.storageRepository.crawl({
includeHidden: true,
pathsToCrawl: [StorageCore.getBaseFolder(folder)],
}),
);
const uploadFiles = await crawl(StorageFolder.UPLOAD); const { mediaPaths } = this.configRepository.getEnv();
const libraryFiles = await crawl(StorageFolder.LIBRARY);
const thumbFiles = await crawl(StorageFolder.THUMBNAILS); const uploadFiles = await crawl(mediaPaths.uploads);
const videoFiles = await crawl(StorageFolder.ENCODED_VIDEO); const libraryFiles = await crawl(mediaPaths.library);
const profileFiles = await crawl(StorageFolder.PROFILE); const thumbFiles = await crawl(mediaPaths.thumbnails);
const videoFiles = await crawl(mediaPaths.encodedVideos);
const profileFiles = await crawl(mediaPaths.profile);
const allFiles = new Set<string>(); const allFiles = new Set<string>();
for (const list of [libraryFiles, thumbFiles, videoFiles, profileFiles, uploadFiles]) { for (const list of [libraryFiles, thumbFiles, videoFiles, profileFiles, uploadFiles]) {
for (const item of list) { for (const item of list) {

View file

@ -1,6 +1,8 @@
import { Inject } from '@nestjs/common'; import { Inject } from '@nestjs/common';
import { join } from 'node:path';
import { SystemConfig } from 'src/config'; import { SystemConfig } from 'src/config';
import { StorageCore } from 'src/cores/storage.core'; import { StorageCore } from 'src/cores/storage.core';
import { AssetPathType } from 'src/enum';
import { IAccessRepository } from 'src/interfaces/access.interface'; import { IAccessRepository } from 'src/interfaces/access.interface';
import { IActivityRepository } from 'src/interfaces/activity.interface'; import { IActivityRepository } from 'src/interfaces/activity.interface';
import { IAlbumUserRepository } from 'src/interfaces/album-user.interface'; import { IAlbumUserRepository } from 'src/interfaces/album-user.interface';
@ -41,6 +43,11 @@ import { IViewRepository } from 'src/interfaces/view.interface';
import { AccessRequest, checkAccess, requireAccess } from 'src/utils/access'; import { AccessRequest, checkAccess, requireAccess } from 'src/utils/access';
import { getConfig, updateConfig } from 'src/utils/config'; import { getConfig, updateConfig } from 'src/utils/config';
type PathRequest = { id: string; ownerId: string };
type ThumbnailPathRequest = PathRequest & { image: SystemConfig['image'] };
const asNestedPath = (filename: string) => join(filename.slice(0, 2), filename.slice(2, 4), filename);
export class BaseService { export class BaseService {
protected storageCore: StorageCore; protected storageCore: StorageCore;
@ -119,4 +126,36 @@ export class BaseService {
checkAccess(request: AccessRequest) { checkAccess(request: AccessRequest) {
return checkAccess(this.accessRepository, request); return checkAccess(this.accessRepository, request);
} }
buildAssetPreviewPath({ image, id, ownerId }: ThumbnailPathRequest) {
const { mediaPaths } = this.configRepository.getEnv();
const filename = `${id}-${AssetPathType.PREVIEW}.${image.preview.format}`;
return join(mediaPaths.thumbnails, ownerId, asNestedPath(filename));
}
buildAssetThumbnailPath({ image, id, ownerId }: ThumbnailPathRequest) {
const { mediaPaths } = this.configRepository.getEnv();
const filename = `${id}-${AssetPathType.THUMBNAIL}.${image.thumbnail.format}`;
return join(mediaPaths.thumbnails, ownerId, asNestedPath(filename));
}
buildPersonThumbnailPath({ id, ownerId }: PathRequest) {
const { mediaPaths } = this.configRepository.getEnv();
return join(mediaPaths.thumbnails, ownerId, asNestedPath(`${id}.jpeg`));
}
buildEncodedVideoPath({ id, ownerId }: PathRequest) {
const { mediaPaths } = this.configRepository.getEnv();
return join(mediaPaths.encodedVideos, ownerId, asNestedPath(`${id}.mp4`));
}
buildAndroidMotionPath({ id, ownerId }: PathRequest) {
const { mediaPaths } = this.configRepository.getEnv();
return join(mediaPaths.encodedVideos, ownerId, asNestedPath(`${id}-MP.mp4`));
}
buildLibraryFolder({ id, storageLabel }: { id: string; storageLabel: string | null }) {
const { mediaPaths } = this.configRepository.getEnv();
return join(mediaPaths.library, storageLabel || id);
}
} }

View file

@ -1,6 +1,5 @@
import { BadRequestException, Injectable } from '@nestjs/common'; import { BadRequestException, Injectable } from '@nestjs/common';
import { parse } from 'node:path'; import { parse } from 'node:path';
import { StorageCore } from 'src/cores/storage.core';
import { AssetIdsDto } from 'src/dtos/asset.dto'; import { AssetIdsDto } from 'src/dtos/asset.dto';
import { AuthDto } from 'src/dtos/auth.dto'; import { AuthDto } from 'src/dtos/auth.dto';
import { DownloadArchiveInfo, DownloadInfoDto, DownloadResponseDto } from 'src/dtos/download.dto'; import { DownloadArchiveInfo, DownloadInfoDto, DownloadResponseDto } from 'src/dtos/download.dto';
@ -9,6 +8,7 @@ import { Permission } from 'src/enum';
import { ImmichReadStream } from 'src/interfaces/storage.interface'; import { ImmichReadStream } from 'src/interfaces/storage.interface';
import { BaseService } from 'src/services/base.service'; import { BaseService } from 'src/services/base.service';
import { HumanReadableSize } from 'src/utils/bytes'; import { HumanReadableSize } from 'src/utils/bytes';
import { isAndroidMotionPath } from 'src/utils/file';
import { usePagination } from 'src/utils/pagination'; import { usePagination } from 'src/utils/pagination';
import { getPreferences } from 'src/utils/preferences'; import { getPreferences } from 'src/utils/preferences';
@ -20,6 +20,7 @@ export class DownloadService extends BaseService {
let archive: DownloadArchiveInfo = { size: 0, assetIds: [] }; let archive: DownloadArchiveInfo = { size: 0, assetIds: [] };
const preferences = getPreferences(auth.user); const preferences = getPreferences(auth.user);
const { mediaPaths } = this.configRepository.getEnv();
const assetPagination = await this.getDownloadAssets(auth, dto); const assetPagination = await this.getDownloadAssets(auth, dto);
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
@ -29,7 +30,7 @@ export class DownloadService extends BaseService {
const motionAssets = await this.assetRepository.getByIds(motionIds, { exifInfo: true }); const motionAssets = await this.assetRepository.getByIds(motionIds, { exifInfo: true });
for (const motionAsset of motionAssets) { for (const motionAsset of motionAssets) {
if ( if (
!StorageCore.isAndroidMotionPath(motionAsset.originalPath) || !isAndroidMotionPath(mediaPaths, motionAsset.originalPath) ||
preferences.download.includeEmbeddedVideos preferences.download.includeEmbeddedVideos
) { ) {
assets.push(motionAsset); assets.push(motionAsset);

View file

@ -10,7 +10,6 @@ import {
AudioCodec, AudioCodec,
Colorspace, Colorspace,
LogLevel, LogLevel,
StorageFolder,
TranscodeHWAccel, TranscodeHWAccel,
TranscodePolicy, TranscodePolicy,
TranscodeTarget, TranscodeTarget,
@ -97,8 +96,9 @@ export class MediaService extends BaseService {
const { active, waiting } = await this.jobRepository.getJobCounts(QueueName.MIGRATION); const { active, waiting } = await this.jobRepository.getJobCounts(QueueName.MIGRATION);
if (active === 1 && waiting === 0) { if (active === 1 && waiting === 0) {
await this.storageCore.removeEmptyDirs(StorageFolder.THUMBNAILS); const { mediaPaths } = this.configRepository.getEnv();
await this.storageCore.removeEmptyDirs(StorageFolder.ENCODED_VIDEO); await this.storageRepository.removeEmptyDirs(mediaPaths.thumbnails);
await this.storageRepository.removeEmptyDirs(mediaPaths.encodedVideos);
} }
for await (const assets of assetPagination) { for await (const assets of assetPagination) {
@ -196,8 +196,9 @@ export class MediaService extends BaseService {
private async generateImageThumbnails(asset: AssetEntity) { private async generateImageThumbnails(asset: AssetEntity) {
const { image } = await this.getConfig({ withCache: true }); const { image } = await this.getConfig({ withCache: true });
const previewPath = StorageCore.getImagePath(asset, AssetPathType.PREVIEW, image.preview.format);
const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.THUMBNAIL, image.thumbnail.format); const previewPath = this.buildAssetPreviewPath({ image, id: asset.id, ownerId: asset.ownerId });
const thumbnailPath = this.buildAssetThumbnailPath({ image, id: asset.id, ownerId: asset.ownerId });
this.storageCore.ensureFolders(previewPath); this.storageCore.ensureFolders(previewPath);
const shouldExtract = image.extractEmbedded && mimeTypes.isRaw(asset.originalPath); const shouldExtract = image.extractEmbedded && mimeTypes.isRaw(asset.originalPath);
@ -230,8 +231,8 @@ export class MediaService extends BaseService {
private async generateVideoThumbnails(asset: AssetEntity) { private async generateVideoThumbnails(asset: AssetEntity) {
const { image, ffmpeg } = await this.getConfig({ withCache: true }); const { image, ffmpeg } = await this.getConfig({ withCache: true });
const previewPath = StorageCore.getImagePath(asset, AssetPathType.PREVIEW, image.preview.format); const thumbnailPath = this.buildAssetThumbnailPath({ image, id: asset.id, ownerId: asset.ownerId });
const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.THUMBNAIL, image.thumbnail.format); const previewPath = this.buildAssetPreviewPath({ image, id: asset.id, ownerId: asset.ownerId });
this.storageCore.ensureFolders(previewPath); this.storageCore.ensureFolders(previewPath);
const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
@ -282,7 +283,7 @@ export class MediaService extends BaseService {
} }
const input = asset.originalPath; const input = asset.originalPath;
const output = StorageCore.getEncodedVideoPath(asset); const output = this.buildEncodedVideoPath({ id, ownerId: asset.ownerId });
this.storageCore.ensureFolders(output); this.storageCore.ensureFolders(output);
const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input, { const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input, {

View file

@ -1,6 +1,5 @@
import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common'; import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common';
import { FACE_THUMBNAIL_SIZE } from 'src/constants'; import { FACE_THUMBNAIL_SIZE } from 'src/constants';
import { StorageCore } from 'src/cores/storage.core';
import { BulkIdErrorReason, BulkIdResponseDto } from 'src/dtos/asset-ids.response.dto'; import { BulkIdErrorReason, BulkIdResponseDto } from 'src/dtos/asset-ids.response.dto';
import { AuthDto } from 'src/dtos/auth.dto'; import { AuthDto } from 'src/dtos/auth.dto';
import { import {
@ -559,7 +558,7 @@ export class PersonService extends BaseService {
const { width, height, inputPath } = await this.getInputDimensions(asset, { width: oldWidth, height: oldHeight }); const { width, height, inputPath } = await this.getInputDimensions(asset, { width: oldWidth, height: oldHeight });
const thumbnailPath = StorageCore.getPersonThumbnailPath(person); const thumbnailPath = this.buildPersonThumbnailPath(person);
this.storageCore.ensureFolders(thumbnailPath); this.storageCore.ensureFolders(thumbnailPath);
const thumbnailOptions = { const thumbnailOptions = {

View file

@ -1,6 +1,5 @@
import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common'; import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common';
import { serverVersion } from 'src/constants'; import { serverVersion } from 'src/constants';
import { StorageCore } from 'src/cores/storage.core';
import { OnEvent } from 'src/decorators'; import { OnEvent } from 'src/decorators';
import { LicenseKeyDto, LicenseResponseDto } from 'src/dtos/license.dto'; import { LicenseKeyDto, LicenseResponseDto } from 'src/dtos/license.dto';
import { import {
@ -13,7 +12,7 @@ import {
ServerStorageResponseDto, ServerStorageResponseDto,
UsageByUserDto, UsageByUserDto,
} from 'src/dtos/server.dto'; } from 'src/dtos/server.dto';
import { StorageFolder, SystemMetadataKey } from 'src/enum'; import { SystemMetadataKey } from 'src/enum';
import { UserStatsQueryResponse } from 'src/interfaces/user.interface'; import { UserStatsQueryResponse } from 'src/interfaces/user.interface';
import { BaseService } from 'src/services/base.service'; import { BaseService } from 'src/services/base.service';
import { asHumanReadable } from 'src/utils/bytes'; import { asHumanReadable } from 'src/utils/bytes';
@ -49,8 +48,8 @@ export class ServerService extends BaseService {
} }
async getStorage(): Promise<ServerStorageResponseDto> { async getStorage(): Promise<ServerStorageResponseDto> {
const libraryBase = StorageCore.getBaseFolder(StorageFolder.LIBRARY); const { mediaPaths } = this.configRepository.getEnv();
const diskInfo = await this.storageRepository.checkDiskUsage(libraryBase); const diskInfo = await this.storageRepository.checkDiskUsage(mediaPaths.library);
const usagePercentage = (((diskInfo.total - diskInfo.free) / diskInfo.total) * 100).toFixed(2); const usagePercentage = (((diskInfo.total - diskInfo.free) / diskInfo.total) * 100).toFixed(2);

View file

@ -12,15 +12,14 @@ import {
supportedWeekTokens, supportedWeekTokens,
supportedYearTokens, supportedYearTokens,
} from 'src/constants'; } from 'src/constants';
import { StorageCore } from 'src/cores/storage.core';
import { OnEvent } from 'src/decorators'; import { OnEvent } from 'src/decorators';
import { AssetEntity } from 'src/entities/asset.entity'; import { AssetEntity } from 'src/entities/asset.entity';
import { AssetPathType, AssetType, StorageFolder } from 'src/enum'; import { AssetPathType, AssetType } from 'src/enum';
import { DatabaseLock } from 'src/interfaces/database.interface'; import { DatabaseLock } from 'src/interfaces/database.interface';
import { ArgOf } from 'src/interfaces/event.interface'; import { ArgOf } from 'src/interfaces/event.interface';
import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobStatus } from 'src/interfaces/job.interface'; import { IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobStatus } from 'src/interfaces/job.interface';
import { BaseService } from 'src/services/base.service'; import { BaseService } from 'src/services/base.service';
import { getLivePhotoMotionFilename } from 'src/utils/file'; import { getLivePhotoMotionFilename, isAndroidMotionPath } from 'src/utils/file';
import { usePagination } from 'src/utils/pagination'; import { usePagination } from 'src/utils/pagination';
export interface MoveAssetMetadata { export interface MoveAssetMetadata {
@ -132,8 +131,8 @@ export class StorageTemplateService extends BaseService {
} }
this.logger.debug('Cleaning up empty directories...'); this.logger.debug('Cleaning up empty directories...');
const libraryFolder = StorageCore.getBaseFolder(StorageFolder.LIBRARY); const { mediaPaths } = this.configRepository.getEnv();
await this.storageRepository.removeEmptyDirs(libraryFolder); await this.storageRepository.removeEmptyDirs(mediaPaths.library);
this.logger.log('Finished storage template migration'); this.logger.log('Finished storage template migration');
@ -141,7 +140,8 @@ export class StorageTemplateService extends BaseService {
} }
async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) { async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) {
if (asset.isExternal || StorageCore.isAndroidMotionPath(asset.originalPath)) { const { mediaPaths } = this.configRepository.getEnv();
if (asset.isExternal || isAndroidMotionPath(mediaPaths, asset.originalPath)) {
// External assets are not affected by storage template // External assets are not affected by storage template
// TODO: shouldn't this only apply to external assets? // TODO: shouldn't this only apply to external assets?
return; return;
@ -186,7 +186,7 @@ export class StorageTemplateService extends BaseService {
const source = asset.originalPath; const source = asset.originalPath;
const extension = path.extname(source).split('.').pop() as string; const extension = path.extname(source).split('.').pop() as string;
const sanitized = sanitize(path.basename(filename, `.${extension}`)); const sanitized = sanitize(path.basename(filename, `.${extension}`));
const rootPath = StorageCore.getLibraryFolder({ id: asset.ownerId, storageLabel }); const rootPath = this.buildLibraryFolder({ id: asset.ownerId, storageLabel });
let albumName = null; let albumName = null;
if (this.template.needsAlbum) { if (this.template.needsAlbum) {

View file

@ -1,8 +1,7 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { join } from 'node:path'; import { join } from 'node:path';
import { StorageCore } from 'src/cores/storage.core';
import { OnEvent } from 'src/decorators'; import { OnEvent } from 'src/decorators';
import { StorageFolder, SystemMetadataKey } from 'src/enum'; import { SystemMetadataKey } from 'src/enum';
import { DatabaseLock } from 'src/interfaces/database.interface'; import { DatabaseLock } from 'src/interfaces/database.interface';
import { IDeleteFilesJob, JobStatus } from 'src/interfaces/job.interface'; import { IDeleteFilesJob, JobStatus } from 'src/interfaces/job.interface';
import { BaseService } from 'src/services/base.service'; import { BaseService } from 'src/services/base.service';
@ -10,6 +9,8 @@ import { BaseService } from 'src/services/base.service';
export class ImmichStartupError extends Error {} export class ImmichStartupError extends Error {}
export const isStartUpError = (error: unknown): error is ImmichStartupError => error instanceof ImmichStartupError; export const isStartUpError = (error: unknown): error is ImmichStartupError => error instanceof ImmichStartupError;
type MountPaths = { folderPath: string; internalPath: string; externalPath: string };
const docsMessage = `Please see https://immich.app/docs/administration/system-integrity#folder-checks for more information.`; const docsMessage = `Please see https://immich.app/docs/administration/system-integrity#folder-checks for more information.`;
@Injectable() @Injectable()
@ -17,6 +18,14 @@ export class StorageService extends BaseService {
@OnEvent({ name: 'app.bootstrap' }) @OnEvent({ name: 'app.bootstrap' })
async onBootstrap() { async onBootstrap() {
const envData = this.configRepository.getEnv(); const envData = this.configRepository.getEnv();
const { mediaPaths } = this.configRepository.getEnv();
const folders = [
mediaPaths.encodedVideos,
mediaPaths.library,
mediaPaths.profile,
mediaPaths.thumbnails,
mediaPaths.uploads,
];
await this.databaseRepository.withLock(DatabaseLock.SystemFileMounts, async () => { await this.databaseRepository.withLock(DatabaseLock.SystemFileMounts, async () => {
const flags = (await this.systemMetadataRepository.get(SystemMetadataKey.SYSTEM_FLAGS)) || { mountFiles: false }; const flags = (await this.systemMetadataRepository.get(SystemMetadataKey.SYSTEM_FLAGS)) || { mountFiles: false };
@ -26,14 +35,18 @@ export class StorageService extends BaseService {
try { try {
// check each folder exists and is writable // check each folder exists and is writable
for (const folder of Object.values(StorageFolder)) { for (const folder of folders) {
const internalPath = join(folder, '.immich');
const externalPath = `<UPLOAD_LOCATION>/${folder.split('/').pop()}/.immich`;
const paths = { internalPath, externalPath, folderPath: folder };
if (!enabled) { if (!enabled) {
this.logger.log(`Writing initial mount file for the ${folder} folder`); this.logger.log(`Writing initial mount file for the ${folder} folder`);
await this.createMountFile(folder); await this.createMountFile(paths);
} }
await this.verifyReadAccess(folder); await this.verifyReadAccess(paths);
await this.verifyWriteAccess(folder); await this.verifyWriteAccess(paths);
} }
if (!flags.mountFiles) { if (!flags.mountFiles) {
@ -73,8 +86,7 @@ export class StorageService extends BaseService {
return JobStatus.SUCCESS; return JobStatus.SUCCESS;
} }
private async verifyReadAccess(folder: StorageFolder) { private async verifyReadAccess({ internalPath, externalPath }: MountPaths) {
const { internalPath, externalPath } = this.getMountFilePaths(folder);
try { try {
await this.storageRepository.readFile(internalPath); await this.storageRepository.readFile(internalPath);
} catch (error) { } catch (error) {
@ -83,8 +95,7 @@ export class StorageService extends BaseService {
} }
} }
private async createMountFile(folder: StorageFolder) { private async createMountFile({ folderPath, internalPath, externalPath }: MountPaths) {
const { folderPath, internalPath, externalPath } = this.getMountFilePaths(folder);
try { try {
this.storageRepository.mkdirSync(folderPath); this.storageRepository.mkdirSync(folderPath);
await this.storageRepository.createFile(internalPath, Buffer.from(`${Date.now()}`)); await this.storageRepository.createFile(internalPath, Buffer.from(`${Date.now()}`));
@ -98,8 +109,7 @@ export class StorageService extends BaseService {
} }
} }
private async verifyWriteAccess(folder: StorageFolder) { private async verifyWriteAccess({ internalPath, externalPath }: MountPaths) {
const { internalPath, externalPath } = this.getMountFilePaths(folder);
try { try {
await this.storageRepository.overwriteFile(internalPath, Buffer.from(`${Date.now()}`)); await this.storageRepository.overwriteFile(internalPath, Buffer.from(`${Date.now()}`));
} catch (error) { } catch (error) {
@ -107,12 +117,4 @@ export class StorageService extends BaseService {
throw new ImmichStartupError(`Failed to write "${externalPath} - ${docsMessage}"`); throw new ImmichStartupError(`Failed to write "${externalPath} - ${docsMessage}"`);
} }
} }
private getMountFilePaths(folder: StorageFolder) {
const folderPath = StorageCore.getBaseFolder(folder);
const internalPath = join(folderPath, '.immich');
const externalPath = `<UPLOAD_LOCATION>/${folder}/.immich`;
return { folderPath, internalPath, externalPath };
}
} }

View file

@ -1,7 +1,7 @@
import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common'; import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common';
import { DateTime } from 'luxon'; import { DateTime } from 'luxon';
import { join } from 'node:path';
import { SALT_ROUNDS } from 'src/constants'; import { SALT_ROUNDS } from 'src/constants';
import { StorageCore } from 'src/cores/storage.core';
import { AuthDto } from 'src/dtos/auth.dto'; import { AuthDto } from 'src/dtos/auth.dto';
import { LicenseKeyDto, LicenseResponseDto } from 'src/dtos/license.dto'; import { LicenseKeyDto, LicenseResponseDto } from 'src/dtos/license.dto';
import { UserPreferencesResponseDto, UserPreferencesUpdateDto, mapPreferences } from 'src/dtos/user-preferences.dto'; import { UserPreferencesResponseDto, UserPreferencesUpdateDto, mapPreferences } from 'src/dtos/user-preferences.dto';
@ -9,7 +9,7 @@ import { CreateProfileImageResponseDto } from 'src/dtos/user-profile.dto';
import { UserAdminResponseDto, UserResponseDto, UserUpdateMeDto, mapUser, mapUserAdmin } from 'src/dtos/user.dto'; import { UserAdminResponseDto, UserResponseDto, UserUpdateMeDto, mapUser, mapUserAdmin } from 'src/dtos/user.dto';
import { UserMetadataEntity } from 'src/entities/user-metadata.entity'; import { UserMetadataEntity } from 'src/entities/user-metadata.entity';
import { UserEntity } from 'src/entities/user.entity'; import { UserEntity } from 'src/entities/user.entity';
import { CacheControl, StorageFolder, UserMetadataKey } from 'src/enum'; import { CacheControl, UserMetadataKey } from 'src/enum';
import { IEntityJob, JobName, JobStatus } from 'src/interfaces/job.interface'; import { IEntityJob, JobName, JobStatus } from 'src/interfaces/job.interface';
import { UserFindOptions } from 'src/interfaces/user.interface'; import { UserFindOptions } from 'src/interfaces/user.interface';
import { BaseService } from 'src/services/base.service'; import { BaseService } from 'src/services/base.service';
@ -196,14 +196,20 @@ export class UserService extends BaseService {
this.logger.log(`Deleting user: ${user.id}`); this.logger.log(`Deleting user: ${user.id}`);
const { mediaPaths } = this.configRepository.getEnv();
const folders = [ const folders = [
StorageCore.getLibraryFolder(user), join(mediaPaths.library, user.id),
StorageCore.getFolderLocation(StorageFolder.UPLOAD, user.id), join(mediaPaths.uploads, user.id),
StorageCore.getFolderLocation(StorageFolder.PROFILE, user.id), join(mediaPaths.profile, user.id),
StorageCore.getFolderLocation(StorageFolder.THUMBNAILS, user.id), join(mediaPaths.thumbnails, user.id),
StorageCore.getFolderLocation(StorageFolder.ENCODED_VIDEO, user.id), join(mediaPaths.encodedVideos, user.id),
]; ];
if (user.storageLabel) {
folders.push(join(mediaPaths.library, user.storageLabel));
}
for (const folder of folders) { for (const folder of folders) {
this.logger.warn(`Removing user from filesystem: ${folder}`); this.logger.warn(`Removing user from filesystem: ${folder}`);
await this.storageRepository.unlinkDir(folder, { recursive: true, force: true }); await this.storageRepository.unlinkDir(folder, { recursive: true, force: true });

View file

@ -4,6 +4,7 @@ import { access, constants } from 'node:fs/promises';
import { basename, extname, isAbsolute } from 'node:path'; import { basename, extname, isAbsolute } from 'node:path';
import { promisify } from 'node:util'; import { promisify } from 'node:util';
import { CacheControl } from 'src/enum'; import { CacheControl } from 'src/enum';
import { MediaPaths } from 'src/interfaces/config.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { ImmichReadStream } from 'src/interfaces/storage.interface'; import { ImmichReadStream } from 'src/interfaces/storage.interface';
import { isConnectionAborted } from 'src/utils/misc'; import { isConnectionAborted } from 'src/utils/misc';
@ -20,6 +21,9 @@ export function getLivePhotoMotionFilename(stillName: string, motionName: string
return getFileNameWithoutExtension(stillName) + extname(motionName); return getFileNameWithoutExtension(stillName) + extname(motionName);
} }
export const isAndroidMotionPath = (mediaPaths: MediaPaths, filePath: string) =>
filePath.startsWith(mediaPaths.encodedVideos);
export class ImmichFileResponse { export class ImmichFileResponse {
public readonly path!: string; public readonly path!: string;
public readonly contentType!: string; public readonly contentType!: string;

View file

@ -83,7 +83,7 @@ export const libraryStub = {
assets: [], assets: [],
owner: userStub.admin, owner: userStub.admin,
ownerId: 'user-id', ownerId: 'user-id',
importPaths: [join(THUMBNAIL_DIR, 'library'), '/xyz', join(APP_MEDIA_LOCATION, 'library')], importPaths: ['upload/thumbs/library', '/xyz', 'upload/library'],
createdAt: new Date('2023-01-01'), createdAt: new Date('2023-01-01'),
updatedAt: new Date('2023-01-01'), updatedAt: new Date('2023-01-01'),
refreshedAt: null, refreshedAt: null,

View file

@ -25,6 +25,14 @@ const envData: EnvData = {
server: 'server-public-key', server: 'server-public-key',
}, },
mediaPaths: {
uploads: 'upload/upload',
library: 'upload/library',
profile: 'upload/profile',
thumbnails: 'upload/thumbs',
encodedVideos: 'upload/encoded-video',
},
resourcePaths: { resourcePaths: {
lockFile: 'build-lock.json', lockFile: 'build-lock.json',
geodata: { geodata: {

View file

@ -39,11 +39,7 @@ export const makeMockWatcher =
return () => Promise.resolve(); return () => Promise.resolve();
}; };
export const newStorageRepositoryMock = (reset = true): Mocked<IStorageRepository> => { export const newStorageRepositoryMock = (): Mocked<IStorageRepository> => {
if (reset) {
StorageCore.reset();
}
return { return {
createZipStream: vitest.fn(), createZipStream: vitest.fn(),
createReadStream: vitest.fn(), createReadStream: vitest.fn(),