feat(server): start up folder checks (#12401)

server/src/entities/system-metadata.entity.ts

@@ -12,12 +12,14 @@ export class SystemMetadataEntity<T extends keyof SystemMetadata = SystemMetadat
 }
 
 export type VersionCheckMetadata = { checkedAt: string; releaseVersion: string };
+export type SystemFlags = { mountFiles: boolean };
 
 export interface SystemMetadata extends Record<SystemMetadataKey, Record<string, any>> {
-  [SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string };
-  [SystemMetadataKey.FACIAL_RECOGNITION_STATE]: { lastRun?: string };
   [SystemMetadataKey.ADMIN_ONBOARDING]: { isOnboarded: boolean };
-  [SystemMetadataKey.SYSTEM_CONFIG]: DeepPartial<SystemConfig>;
-  [SystemMetadataKey.VERSION_CHECK_STATE]: VersionCheckMetadata;
+  [SystemMetadataKey.FACIAL_RECOGNITION_STATE]: { lastRun?: string };
   [SystemMetadataKey.LICENSE]: { licenseKey: string; activationKey: string; activatedAt: Date };
+  [SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string };
+  [SystemMetadataKey.SYSTEM_CONFIG]: DeepPartial<SystemConfig>;
+  [SystemMetadataKey.SYSTEM_FLAGS]: SystemFlags;
+  [SystemMetadataKey.VERSION_CHECK_STATE]: VersionCheckMetadata;
 }

server/src/enum.ts

@@ -153,6 +153,7 @@ export enum SystemMetadataKey {
   FACIAL_RECOGNITION_STATE = 'facial-recognition-state',
   ADMIN_ONBOARDING = 'admin-onboarding',
   SYSTEM_CONFIG = 'system-config',
+  SYSTEM_FLAGS = 'system-flags',
   VERSION_CHECK_STATE = 'version-check-state',
   LICENSE = 'license',
 }

server/src/interfaces/database.interface.ts

@@ -15,6 +15,7 @@ export enum VectorIndex {
 export enum DatabaseLock {
   GeodataImport = 100,
   Migrations = 200,
+  SystemFileMounts = 300,
   StorageTemplateMigration = 420,
   CLIPDimSize = 512,
   LibraryWatch = 1337,

server/src/main.ts

@@ -17,7 +17,13 @@ async function bootstrapImmichAdmin() {
 
 function bootstrapWorker(name: string) {
   console.log(`Starting ${name} worker`);
+
   const worker = name === 'api' ? fork(`./dist/workers/${name}.js`) : new Worker(`./dist/workers/${name}.js`);
+
+  worker.on('error', (error) => {
+    console.error(`${name} worker error: ${error}`);
+  });
+
   worker.on('exit', (exitCode) => {
     if (exitCode !== 0) {
       console.error(`${name} worker exited with code ${exitCode}`);

server/src/services/storage.service.spec.ts

@@ -1,19 +1,29 @@
+import { SystemMetadataKey } from 'src/enum';
+import { IDatabaseRepository } from 'src/interfaces/database.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { IStorageRepository } from 'src/interfaces/storage.interface';
+import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
 import { StorageService } from 'src/services/storage.service';
+import { newDatabaseRepositoryMock } from 'test/repositories/database.repository.mock';
 import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock';
 import { newStorageRepositoryMock } from 'test/repositories/storage.repository.mock';
+import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock';
 import { Mocked } from 'vitest';
 
 describe(StorageService.name, () => {
   let sut: StorageService;
+  let databaseMock: Mocked<IDatabaseRepository>;
   let storageMock: Mocked<IStorageRepository>;
   let loggerMock: Mocked<ILoggerRepository>;
+  let systemMock: Mocked<ISystemMetadataRepository>;
 
   beforeEach(() => {
+    databaseMock = newDatabaseRepositoryMock();
     storageMock = newStorageRepositoryMock();
     loggerMock = newLoggerRepositoryMock();
-    sut = new StorageService(storageMock, loggerMock);
+    systemMock = newSystemMetadataRepositoryMock();
+
+    sut = new StorageService(databaseMock, storageMock, loggerMock, systemMock);
   });
 
   it('should work', () => {
@@ -21,9 +31,35 @@ describe(StorageService.name, () => {
   });
 
   describe('onBootstrap', () => {
-    it('should create the library folder on initialization', () => {
-      sut.onBootstrap();
+    it('should enable mount folder checking', async () => {
+      systemMock.get.mockResolvedValue(null);
+
+      await expect(sut.onBootstrap()).resolves.toBeUndefined();
+
+      expect(systemMock.set).toHaveBeenCalledWith(SystemMetadataKey.SYSTEM_FLAGS, { mountFiles: true });
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/encoded-video');
       expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/library');
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/profile');
+      expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs');
+    });
+
+    it('should throw an error if .immich is missing', async () => {
+      systemMock.get.mockResolvedValue({ mountFiles: true });
+      storageMock.readFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'"));
+
+      await expect(sut.onBootstrap()).rejects.toThrow('Failed to validate folder mount');
+
+      expect(storageMock.writeFile).not.toHaveBeenCalled();
+      expect(systemMock.set).not.toHaveBeenCalled();
+    });
+
+    it('should throw an error if .immich is present but read-only', async () => {
+      systemMock.get.mockResolvedValue({ mountFiles: true });
+      storageMock.writeFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'"));
+
+      await expect(sut.onBootstrap()).rejects.toThrow('Failed to validate folder mount');
+
+      expect(systemMock.set).not.toHaveBeenCalled();
     });
   });
 

server/src/services/storage.service.ts

@@ -1,23 +1,52 @@
 import { Inject, Injectable } from '@nestjs/common';
+import { join } from 'node:path';
 import { StorageCore, StorageFolder } from 'src/cores/storage.core';
 import { OnEmit } from 'src/decorators';
+import { SystemMetadataKey } from 'src/enum';
+import { DatabaseLock, IDatabaseRepository } from 'src/interfaces/database.interface';
 import { IDeleteFilesJob, JobStatus } from 'src/interfaces/job.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { IStorageRepository } from 'src/interfaces/storage.interface';
+import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
+import { ImmichStartupError } from 'src/utils/events';
 
 @Injectable()
 export class StorageService {
   constructor(
+    @Inject(IDatabaseRepository) private databaseRepository: IDatabaseRepository,
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
     @Inject(ILoggerRepository) private logger: ILoggerRepository,
+    @Inject(ISystemMetadataRepository) private systemMetadata: ISystemMetadataRepository,
   ) {
     this.logger.setContext(StorageService.name);
   }
 
   @OnEmit({ event: 'app.bootstrap' })
-  onBootstrap() {
-    const libraryBase = StorageCore.getBaseFolder(StorageFolder.LIBRARY);
-    this.storageRepository.mkdirSync(libraryBase);
+  async onBootstrap() {
+    await this.databaseRepository.withLock(DatabaseLock.SystemFileMounts, async () => {
+      const flags = (await this.systemMetadata.get(SystemMetadataKey.SYSTEM_FLAGS)) || { mountFiles: false };
+
+      this.logger.log('Verifying system mount folder checks');
+
+      // check each folder exists and is writable
+      for (const folder of Object.values(StorageFolder)) {
+        if (!flags.mountFiles) {
+          this.logger.log(`Writing initial mount file for the ${folder} folder`);
+          await this.verifyWriteAccess(folder);
+        }
+
+        await this.verifyReadAccess(folder);
+        await this.verifyWriteAccess(folder);
+      }
+
+      if (!flags.mountFiles) {
+        flags.mountFiles = true;
+        await this.systemMetadata.set(SystemMetadataKey.SYSTEM_FLAGS, flags);
+        this.logger.log('Successfully enabled system mount folders checks');
+      }
+
+      this.logger.log('Successfully verified system mount folder checks');
+    });
   }
 
   async handleDeleteFiles(job: IDeleteFilesJob) {
@@ -38,4 +67,38 @@ export class StorageService {
 
     return JobStatus.SUCCESS;
   }
+
+  private async verifyReadAccess(folder: StorageFolder) {
+    const { filePath } = this.getMountFilePaths(folder);
+    try {
+      await this.storageRepository.readFile(filePath);
+    } catch (error) {
+      this.logger.error(`Failed to read ${filePath}: ${error}`);
+      this.logger.error(
+        `The "${folder}" folder appears to be offline/missing, please make sure the volume is mounted with the correct permissions`,
+      );
+      throw new ImmichStartupError(`Failed to validate folder mount (read from "<MEDIA_LOCATION>/${folder}")`);
+    }
+  }
+
+  private async verifyWriteAccess(folder: StorageFolder) {
+    const { folderPath, filePath } = this.getMountFilePaths(folder);
+    try {
+      this.storageRepository.mkdirSync(folderPath);
+      await this.storageRepository.writeFile(filePath, Buffer.from(`${Date.now()}`));
+    } catch (error) {
+      this.logger.error(`Failed to write ${filePath}: ${error}`);
+      this.logger.error(
+        `The "${folder}" folder cannot be written to, please make sure the volume is mounted with the correct permissions`,
+      );
+      throw new ImmichStartupError(`Failed to validate folder mount (write to "<MEDIA_LOCATION>/${folder}")`);
+    }
+  }
+
+  private getMountFilePaths(folder: StorageFolder) {
+    const folderPath = StorageCore.getBaseFolder(folder);
+    const filePath = join(folderPath, '.immich');
+
+    return { folderPath, filePath };
+  }
 }

server/src/utils/events.ts

@@ -12,6 +12,9 @@ type Item<T extends EmitEvent> = {
   label: string;
 };
 
+export class ImmichStartupError extends Error {}
+export const isStartUpError = (error: unknown): error is ImmichStartupError => error instanceof ImmichStartupError;
+
 export const setupEventHandlers = (moduleRef: ModuleRef) => {
   const reflector = moduleRef.get(Reflector, { strict: false });
   const repository = moduleRef.get<IEventRepository>(IEventRepository);

server/src/workers/api.ts

@@ -9,6 +9,7 @@ import { envName, excludePaths, isDev, resourcePaths, serverVersion } from 'src/
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
 import { ApiService } from 'src/services/api.service';
+import { isStartUpError } from 'src/utils/events';
 import { otelStart } from 'src/utils/instrumentation';
 import { useSwagger } from 'src/utils/misc';
 
@@ -73,6 +74,9 @@ async function bootstrap() {
 }
 
 bootstrap().catch((error) => {
-  console.error(error);
-  throw error;
+  if (!isStartUpError(error)) {
+    console.error(error);
+  }
+  // eslint-disable-next-line unicorn/no-process-exit
+  process.exit(1);
 });

server/src/workers/microservices.ts

@@ -4,6 +4,7 @@ import { MicroservicesModule } from 'src/app.module';
 import { envName, serverVersion } from 'src/constants';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
+import { isStartUpError } from 'src/utils/events';
 import { otelStart } from 'src/utils/instrumentation';
 
 export async function bootstrap() {
@@ -25,7 +26,9 @@ export async function bootstrap() {
 
 if (!isMainThread) {
   bootstrap().catch((error) => {
-    console.error(error);
-    process.exit(1);
+    if (!isStartUpError(error)) {
+      console.error(error);
+    }
+    throw error;
   });
 }