From 51d4899cd1348d4c8093d63ae3a4a04dae290c71 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Wed, 16 Oct 2024 18:12:13 -0400 Subject: [PATCH 01/72] refactor(server): move host env to config repo (#13507) * refactor(server): access host through repository * refactor(server): access host through repository --- server/src/interfaces/config.interface.ts | 1 + server/src/repositories/config.repository.ts | 1 + server/src/workers/api.ts | 4 +--- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.ts index d105e40cf9..0e07116350 100644 --- a/server/src/interfaces/config.interface.ts +++ b/server/src/interfaces/config.interface.ts @@ -4,6 +4,7 @@ import { VectorExtension } from 'src/interfaces/database.interface'; export const IConfigRepository = 'IConfigRepository'; export interface EnvData { + host?: string; port: number; environment: ImmichEnvironment; configFile?: string; diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index d9b7c36384..883c65846a 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -50,6 +50,7 @@ export class ConfigRepository implements IConfigRepository { }; return { + host: process.env.IMMICH_HOST, port: Number(process.env.IMMICH_PORT) || 2283, environment, configFile: process.env.IMMICH_CONFIG_FILE, diff --git a/server/src/workers/api.ts b/server/src/workers/api.ts index 7535a902b8..4a7755fa1e 100644 --- a/server/src/workers/api.ts +++ b/server/src/workers/api.ts @@ -15,8 +15,6 @@ import { isStartUpError } from 'src/services/storage.service'; import { otelStart } from 'src/utils/instrumentation'; import { useSwagger } from 'src/utils/misc'; -const host = process.env.HOST; - function parseTrustedProxy(input?: string) { if (!input) { return []; @@ -36,7 +34,7 @@ async function bootstrap() { const logger = await app.resolve(ILoggerRepository); const configRepository = app.get(IConfigRepository); - const { environment, port, resourcePaths } = configRepository.getEnv(); + const { environment, host, port, resourcePaths } = configRepository.getEnv(); const isDev = environment === ImmichEnvironment.DEVELOPMENT; logger.setContext('Bootstrap'); From 8ac40a933a220a929e5e436215d30aa500d0fd92 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Wed, 16 Oct 2024 18:13:12 -0400 Subject: [PATCH 02/72] refactor(server): external domain fallback (#13506) --- server/src/constants.ts | 2 -- server/src/services/notification.service.ts | 14 +++++++++----- server/src/services/shared-link.service.spec.ts | 5 ++--- server/src/services/shared-link.service.ts | 10 +++++----- server/src/utils/misc.ts | 3 +++ server/test/repositories/config.repository.mock.ts | 5 ++--- 6 files changed, 21 insertions(+), 18 deletions(-) diff --git a/server/src/constants.ts b/server/src/constants.ts index 5317d5e13c..eef9ffab05 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -20,8 +20,6 @@ export const AUDIT_LOG_MAX_DURATION = Duration.fromObject({ days: 100 }); export const ONE_HOUR = Duration.fromObject({ hours: 1 }); export const APP_MEDIA_LOCATION = process.env.IMMICH_MEDIA_LOCATION || './upload'; -const HOST_SERVER_PORT = process.env.IMMICH_PORT || '2283'; -export const DEFAULT_EXTERNAL_DOMAIN = 'http://localhost:' + HOST_SERVER_PORT; export const citiesFile = 'cities500.txt'; diff --git a/server/src/services/notification.service.ts b/server/src/services/notification.service.ts index f6b338d79e..122a09ee2e 100644 --- a/server/src/services/notification.service.ts +++ b/server/src/services/notification.service.ts @@ -1,5 +1,4 @@ import { BadRequestException, Injectable } from '@nestjs/common'; -import { DEFAULT_EXTERNAL_DOMAIN } from 'src/constants'; import { OnEvent } from 'src/decorators'; import { SystemConfigSmtpDto } from 'src/dtos/system-config.dto'; import { AlbumEntity } from 'src/entities/album.entity'; @@ -16,6 +15,7 @@ import { EmailImageAttachment, EmailTemplate } from 'src/interfaces/notification import { BaseService } from 'src/services/base.service'; import { getAssetFiles } from 'src/utils/asset.util'; import { getFilenameExtension } from 'src/utils/file'; +import { getExternalDomain } from 'src/utils/misc'; import { isEqualObject } from 'src/utils/object'; import { getPreferences } from 'src/utils/preferences'; @@ -128,10 +128,11 @@ export class NotificationService extends BaseService { } const { server } = await this.getConfig({ withCache: false }); + const { port } = this.configRepository.getEnv(); const { html, text } = await this.notificationRepository.renderEmail({ template: EmailTemplate.TEST_EMAIL, data: { - baseUrl: server.externalDomain || DEFAULT_EXTERNAL_DOMAIN, + baseUrl: getExternalDomain(server, port), displayName: user.name, }, }); @@ -156,10 +157,11 @@ export class NotificationService extends BaseService { } const { server } = await this.getConfig({ withCache: true }); + const { port } = this.configRepository.getEnv(); const { html, text } = await this.notificationRepository.renderEmail({ template: EmailTemplate.WELCOME, data: { - baseUrl: server.externalDomain || DEFAULT_EXTERNAL_DOMAIN, + baseUrl: getExternalDomain(server, port), displayName: user.name, username: user.email, password: tempPassword, @@ -199,10 +201,11 @@ export class NotificationService extends BaseService { const attachment = await this.getAlbumThumbnailAttachment(album); const { server } = await this.getConfig({ withCache: false }); + const { port } = this.configRepository.getEnv(); const { html, text } = await this.notificationRepository.renderEmail({ template: EmailTemplate.ALBUM_INVITE, data: { - baseUrl: server.externalDomain || DEFAULT_EXTERNAL_DOMAIN, + baseUrl: getExternalDomain(server, port), albumId: album.id, albumName: album.albumName, senderName: album.owner.name, @@ -241,6 +244,7 @@ export class NotificationService extends BaseService { const attachment = await this.getAlbumThumbnailAttachment(album); const { server } = await this.getConfig({ withCache: false }); + const { port } = this.configRepository.getEnv(); for (const recipient of recipients) { const user = await this.userRepository.get(recipient.id, { withDeleted: false }); @@ -257,7 +261,7 @@ export class NotificationService extends BaseService { const { html, text } = await this.notificationRepository.renderEmail({ template: EmailTemplate.ALBUM_UPDATE, data: { - baseUrl: server.externalDomain || DEFAULT_EXTERNAL_DOMAIN, + baseUrl: getExternalDomain(server, port), albumId: album.id, albumName: album.albumName, recipientName: recipient.name, diff --git a/server/src/services/shared-link.service.spec.ts b/server/src/services/shared-link.service.spec.ts index d0959f31b8..6554421418 100644 --- a/server/src/services/shared-link.service.spec.ts +++ b/server/src/services/shared-link.service.spec.ts @@ -1,6 +1,5 @@ import { BadRequestException, ForbiddenException, UnauthorizedException } from '@nestjs/common'; import _ from 'lodash'; -import { DEFAULT_EXTERNAL_DOMAIN } from 'src/constants'; import { AssetIdErrorReason } from 'src/dtos/asset-ids.response.dto'; import { SharedLinkType } from 'src/enum'; import { ISharedLinkRepository } from 'src/interfaces/shared-link.interface'; @@ -304,7 +303,7 @@ describe(SharedLinkService.name, () => { sharedLinkMock.get.mockResolvedValue(sharedLinkStub.individual); await expect(sut.getMetadataTags(authStub.adminSharedLink)).resolves.toEqual({ description: '1 shared photos & videos', - imageUrl: `${DEFAULT_EXTERNAL_DOMAIN}/api/assets/asset-id/thumbnail?key=LCtkaJX4R1O_9D-2lq0STzsPryoL1UdAbyb6Sna1xxmQCSuqU2J1ZUsqt6GR-yGm1s0`, + imageUrl: `http://localhost:2283/api/assets/asset-id/thumbnail?key=LCtkaJX4R1O_9D-2lq0STzsPryoL1UdAbyb6Sna1xxmQCSuqU2J1ZUsqt6GR-yGm1s0`, title: 'Public Share', }); expect(sharedLinkMock.get).toHaveBeenCalled(); @@ -314,7 +313,7 @@ describe(SharedLinkService.name, () => { sharedLinkMock.get.mockResolvedValue({ ...sharedLinkStub.individual, album: undefined, assets: [] }); await expect(sut.getMetadataTags(authStub.adminSharedLink)).resolves.toEqual({ description: '0 shared photos & videos', - imageUrl: `${DEFAULT_EXTERNAL_DOMAIN}/feature-panel.png`, + imageUrl: `http://localhost:2283/feature-panel.png`, title: 'Public Share', }); expect(sharedLinkMock.get).toHaveBeenCalled(); diff --git a/server/src/services/shared-link.service.ts b/server/src/services/shared-link.service.ts index a01a2f45a3..5ef140d26d 100644 --- a/server/src/services/shared-link.service.ts +++ b/server/src/services/shared-link.service.ts @@ -1,21 +1,20 @@ import { BadRequestException, ForbiddenException, Injectable, UnauthorizedException } from '@nestjs/common'; -import { DEFAULT_EXTERNAL_DOMAIN } from 'src/constants'; import { AssetIdErrorReason, AssetIdsResponseDto } from 'src/dtos/asset-ids.response.dto'; import { AssetIdsDto } from 'src/dtos/asset.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { + mapSharedLink, + mapSharedLinkWithoutMetadata, SharedLinkCreateDto, SharedLinkEditDto, SharedLinkPasswordDto, SharedLinkResponseDto, - mapSharedLink, - mapSharedLinkWithoutMetadata, } from 'src/dtos/shared-link.dto'; import { AssetEntity } from 'src/entities/asset.entity'; import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { Permission, SharedLinkType } from 'src/enum'; import { BaseService } from 'src/services/base.service'; -import { OpenGraphTags } from 'src/utils/misc'; +import { getExternalDomain, OpenGraphTags } from 'src/utils/misc'; @Injectable() export class SharedLinkService extends BaseService { @@ -177,6 +176,7 @@ export class SharedLinkService extends BaseService { } const config = await this.getConfig({ withCache: true }); + const { port } = this.configRepository.getEnv(); const sharedLink = await this.findOrFail(auth.sharedLink.userId, auth.sharedLink.id); const assetId = sharedLink.album?.albumThumbnailAssetId || sharedLink.assets[0]?.id; const assetCount = sharedLink.assets.length > 0 ? sharedLink.assets.length : sharedLink.album?.assets.length || 0; @@ -187,7 +187,7 @@ export class SharedLinkService extends BaseService { return { title: sharedLink.album ? sharedLink.album.albumName : 'Public Share', description: sharedLink.description || `${assetCount} shared photos & videos`, - imageUrl: new URL(imagePath, config.server.externalDomain || DEFAULT_EXTERNAL_DOMAIN).href, + imageUrl: new URL(imagePath, getExternalDomain(config.server, port)).href, }; } diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index 9bea4e9585..e191170e0d 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -16,6 +16,9 @@ import { ImmichCookie, ImmichHeader } from 'src/dtos/auth.dto'; import { MetadataKey } from 'src/enum'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; +export const getExternalDomain = (server: SystemConfig['server'], port: number) => + server.externalDomain || `http://localhost:${port}`; + /** * @returns a list of strings representing the keys of the object in dot notation */ diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts index 852868ee31..8e71ba2dca 100644 --- a/server/test/repositories/config.repository.mock.ts +++ b/server/test/repositories/config.repository.mock.ts @@ -49,10 +49,9 @@ const envData: EnvData = { noColor: false, }; +export const mockEnvData = (config: Partial) => ({ ...envData, ...config }); export const newConfigRepositoryMock = (): Mocked => { return { - getEnv: vitest.fn().mockReturnValue(envData), + getEnv: vitest.fn().mockReturnValue(mockEnvData({})), }; }; - -export const mockEnvData = (config: Partial) => ({ ...envData, ...config }); From 01a9cda15dcc5440f0f9ce647c8a42d38ef6e5dc Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Wed, 16 Oct 2024 18:20:44 -0400 Subject: [PATCH 03/72] fix(server): never try to parse Duration from exif data (#13497) --- server/src/services/metadata.service.spec.ts | 114 ++++++++++--------- server/src/services/metadata.service.ts | 6 +- 2 files changed, 66 insertions(+), 54 deletions(-) diff --git a/server/src/services/metadata.service.spec.ts b/server/src/services/metadata.service.spec.ts index cd7f68ab1d..cc6eae6e3b 100644 --- a/server/src/services/metadata.service.spec.ts +++ b/server/src/services/metadata.service.spec.ts @@ -44,6 +44,12 @@ describe(MetadataService.name, () => { let tagMock: Mocked; let userMock: Mocked; + const mockReadTags = (exifData?: Partial, sidecarData?: Partial) => { + metadataMock.readTags.mockReset(); + metadataMock.readTags.mockResolvedValueOnce(exifData ?? {}); + metadataMock.readTags.mockResolvedValueOnce(sidecarData ?? {}); + }; + beforeEach(() => { ({ sut, @@ -62,6 +68,8 @@ describe(MetadataService.name, () => { userMock, } = newTestService(MetadataService)); + mockReadTags(); + delete process.env.TZ; }); @@ -258,13 +266,7 @@ describe(MetadataService.name, () => { const originalDate = new Date('2023-11-21T16:13:17.517Z'); const sidecarDate = new Date('2022-01-01T00:00:00.000Z'); assetMock.getByIds.mockResolvedValue([assetStub.sidecar]); - metadataMock.readTags.mockImplementation((path) => { - const map = { - [assetStub.sidecar.originalPath]: originalDate.toISOString(), - [assetStub.sidecar.sidecarPath as string]: sidecarDate.toISOString(), - }; - return Promise.resolve({ CreationDate: map[path] ?? new Date().toISOString() }); - }); + mockReadTags({ CreationDate: originalDate.toISOString() }, { CreationDate: sidecarDate.toISOString() }); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.sidecar.id], { faces: { person: false } }); @@ -280,9 +282,7 @@ describe(MetadataService.name, () => { it('should account for the server being in a non-UTC timezone', async () => { process.env.TZ = 'America/Los_Angeles'; assetMock.getByIds.mockResolvedValue([assetStub.sidecar]); - metadataMock.readTags.mockResolvedValueOnce({ - DateTimeOriginal: '2022:01:01 00:00:00', - }); + mockReadTags({ DateTimeOriginal: '2022:01:01 00:00:00' }); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(assetMock.upsertExif).toHaveBeenCalledWith( @@ -300,7 +300,7 @@ describe(MetadataService.name, () => { it('should handle lists of numbers', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ ISO: [160] }); + mockReadTags({ ISO: [160] }); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } }); @@ -317,7 +317,7 @@ describe(MetadataService.name, () => { assetMock.getByIds.mockResolvedValue([assetStub.withLocation]); systemMock.get.mockResolvedValue({ reverseGeocoding: { enabled: true } }); mapMock.reverseGeocode.mockResolvedValue({ city: 'City', state: 'State', country: 'Country' }); - metadataMock.readTags.mockResolvedValue({ + mockReadTags({ GPSLatitude: assetStub.withLocation.exifInfo!.latitude!, GPSLongitude: assetStub.withLocation.exifInfo!.longitude!, }); @@ -337,7 +337,7 @@ describe(MetadataService.name, () => { it('should discard latitude and longitude on null island', async () => { assetMock.getByIds.mockResolvedValue([assetStub.withLocation]); - metadataMock.readTags.mockResolvedValue({ + mockReadTags({ GPSLatitude: 0, GPSLongitude: 0, }); @@ -349,7 +349,7 @@ describe(MetadataService.name, () => { it('should extract tags from TagsList', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ TagsList: ['Parent'] }); + mockReadTags({ TagsList: ['Parent'] }); tagMock.upsertValue.mockResolvedValue(tagStub.parent); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -359,7 +359,7 @@ describe(MetadataService.name, () => { it('should extract hierarchy from TagsList', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ TagsList: ['Parent/Child'] }); + mockReadTags({ TagsList: ['Parent/Child'] }); tagMock.upsertValue.mockResolvedValueOnce(tagStub.parent); tagMock.upsertValue.mockResolvedValueOnce(tagStub.child); @@ -375,7 +375,7 @@ describe(MetadataService.name, () => { it('should extract tags from Keywords as a string', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ Keywords: 'Parent' }); + mockReadTags({ Keywords: 'Parent' }); tagMock.upsertValue.mockResolvedValue(tagStub.parent); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -385,7 +385,7 @@ describe(MetadataService.name, () => { it('should extract tags from Keywords as a list', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ Keywords: ['Parent'] }); + mockReadTags({ Keywords: ['Parent'] }); tagMock.upsertValue.mockResolvedValue(tagStub.parent); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -395,7 +395,7 @@ describe(MetadataService.name, () => { it('should extract tags from Keywords as a list with a number', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ Keywords: ['Parent', 2024] }); + mockReadTags({ Keywords: ['Parent', 2024] }); tagMock.upsertValue.mockResolvedValue(tagStub.parent); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -406,7 +406,7 @@ describe(MetadataService.name, () => { it('should extract hierarchal tags from Keywords', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ Keywords: 'Parent/Child' }); + mockReadTags({ Keywords: 'Parent/Child' }); tagMock.upsertValue.mockResolvedValue(tagStub.parent); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -421,7 +421,7 @@ describe(MetadataService.name, () => { it('should ignore Keywords when TagsList is present', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ Keywords: 'Child', TagsList: ['Parent/Child'] }); + mockReadTags({ Keywords: 'Child', TagsList: ['Parent/Child'] }); tagMock.upsertValue.mockResolvedValue(tagStub.parent); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -436,7 +436,7 @@ describe(MetadataService.name, () => { it('should extract hierarchy from HierarchicalSubject', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ HierarchicalSubject: ['Parent|Child', 'TagA'] }); + mockReadTags({ HierarchicalSubject: ['Parent|Child', 'TagA'] }); tagMock.upsertValue.mockResolvedValueOnce(tagStub.parent); tagMock.upsertValue.mockResolvedValueOnce(tagStub.child); @@ -453,7 +453,7 @@ describe(MetadataService.name, () => { it('should extract tags from HierarchicalSubject as a list with a number', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ HierarchicalSubject: ['Parent', 2024] }); + mockReadTags({ HierarchicalSubject: ['Parent', 2024] }); tagMock.upsertValue.mockResolvedValue(tagStub.parent); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -464,7 +464,7 @@ describe(MetadataService.name, () => { it('should extract ignore / characters in a HierarchicalSubject tag', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ HierarchicalSubject: ['Mom/Dad'] }); + mockReadTags({ HierarchicalSubject: ['Mom/Dad'] }); tagMock.upsertValue.mockResolvedValueOnce(tagStub.parent); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -478,7 +478,7 @@ describe(MetadataService.name, () => { it('should ignore HierarchicalSubject when TagsList is present', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ HierarchicalSubject: ['Parent2|Child2'], TagsList: ['Parent/Child'] }); + mockReadTags({ HierarchicalSubject: ['Parent2|Child2'], TagsList: ['Parent/Child'] }); tagMock.upsertValue.mockResolvedValue(tagStub.parent); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -493,7 +493,7 @@ describe(MetadataService.name, () => { it('should remove existing tags', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({}); + mockReadTags({}); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -518,7 +518,7 @@ describe(MetadataService.name, () => { it('should handle an invalid Directory Item', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ + mockReadTags({ MotionPhoto: 1, ContainerDirectory: [{ Foo: 100 }], }); @@ -529,7 +529,7 @@ describe(MetadataService.name, () => { it('should extract the correct video orientation', async () => { assetMock.getByIds.mockResolvedValue([assetStub.video]); mediaMock.probe.mockResolvedValue(probeStub.videoStreamVertical2160p); - metadataMock.readTags.mockResolvedValue({}); + mockReadTags({}); await sut.handleMetadataExtraction({ id: assetStub.video.id }); @@ -541,7 +541,7 @@ describe(MetadataService.name, () => { it('should extract the MotionPhotoVideo tag from Samsung HEIC motion photos', async () => { assetMock.getByIds.mockResolvedValue([{ ...assetStub.livePhotoWithOriginalFileName, livePhotoVideoId: null }]); - metadataMock.readTags.mockResolvedValue({ + mockReadTags({ Directory: 'foo/bar/', MotionPhotoVideo: new BinaryField(0, ''), // The below two are included to ensure that the MotionPhotoVideo tag is extracted @@ -589,7 +589,7 @@ describe(MetadataService.name, () => { it('should extract the EmbeddedVideo tag from Samsung JPEG motion photos', async () => { assetMock.getByIds.mockResolvedValue([{ ...assetStub.livePhotoWithOriginalFileName, livePhotoVideoId: null }]); - metadataMock.readTags.mockResolvedValue({ + mockReadTags({ Directory: 'foo/bar/', EmbeddedVideoFile: new BinaryField(0, ''), EmbeddedVideoType: 'MotionPhoto_Data', @@ -634,7 +634,7 @@ describe(MetadataService.name, () => { it('should extract the motion photo video from the XMP directory entry ', async () => { assetMock.getByIds.mockResolvedValue([{ ...assetStub.livePhotoWithOriginalFileName, livePhotoVideoId: null }]); - metadataMock.readTags.mockResolvedValue({ + mockReadTags({ Directory: 'foo/bar/', MotionPhoto: 1, MicroVideo: 1, @@ -680,7 +680,7 @@ describe(MetadataService.name, () => { it('should delete old motion photo video assets if they do not match what is extracted', async () => { assetMock.getByIds.mockResolvedValue([assetStub.livePhotoWithOriginalFileName]); - metadataMock.readTags.mockResolvedValue({ + mockReadTags({ Directory: 'foo/bar/', MotionPhoto: 1, MicroVideo: 1, @@ -705,7 +705,7 @@ describe(MetadataService.name, () => { it('should not create a new motion photo video asset if the hash of the extracted video matches an existing asset', async () => { assetMock.getByIds.mockResolvedValue([assetStub.livePhotoStillAsset]); - metadataMock.readTags.mockResolvedValue({ + mockReadTags({ Directory: 'foo/bar/', MotionPhoto: 1, MicroVideo: 1, @@ -727,7 +727,7 @@ describe(MetadataService.name, () => { it('should link and hide motion video asset to still asset if the hash of the extracted video matches an existing asset', async () => { assetMock.getByIds.mockResolvedValue([{ ...assetStub.livePhotoStillAsset, livePhotoVideoId: null }]); - metadataMock.readTags.mockResolvedValue({ + mockReadTags({ Directory: 'foo/bar/', MotionPhoto: 1, MicroVideo: 1, @@ -753,7 +753,7 @@ describe(MetadataService.name, () => { assetMock.getByIds.mockResolvedValue([ { ...assetStub.livePhotoStillAsset, livePhotoVideoId: null, isExternal: true }, ]); - metadataMock.readTags.mockResolvedValue({ + mockReadTags({ Directory: 'foo/bar/', MotionPhoto: 1, MicroVideo: 1, @@ -796,7 +796,7 @@ describe(MetadataService.name, () => { Rating: 3, }; assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue(tags); + mockReadTags(tags); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } }); @@ -854,7 +854,7 @@ describe(MetadataService.name, () => { tz: undefined, }; assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue(tags); + mockReadTags(tags); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { faces: { person: false } }); @@ -887,7 +887,7 @@ describe(MetadataService.name, () => { ); }); - it('only extracts duration for videos', async () => { + it('should only extract duration for videos', async () => { assetMock.getByIds.mockResolvedValue([{ ...assetStub.image }]); mediaMock.probe.mockResolvedValue({ ...probeStub.videoStreamH264, @@ -908,7 +908,7 @@ describe(MetadataService.name, () => { ); }); - it('omits duration of zero', async () => { + it('should omit duration of zero', async () => { assetMock.getByIds.mockResolvedValue([{ ...assetStub.video }]); mediaMock.probe.mockResolvedValue({ ...probeStub.videoStreamH264, @@ -930,7 +930,7 @@ describe(MetadataService.name, () => { ); }); - it('handles duration of 1 week', async () => { + it('should a handle duration of 1 week', async () => { assetMock.getByIds.mockResolvedValue([{ ...assetStub.video }]); mediaMock.probe.mockResolvedValue({ ...probeStub.videoStreamH264, @@ -952,9 +952,17 @@ describe(MetadataService.name, () => { ); }); - it('trims whitespace from description', async () => { + it('should ignore duration from exif data', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ Description: '\t \v \f \n \r' }); + mockReadTags({}, { Duration: { Value: 123 } }); + + await sut.handleMetadataExtraction({ id: assetStub.image.id }); + expect(assetMock.update).toHaveBeenCalledWith(expect.objectContaining({ duration: null })); + }); + + it('should trim whitespace from description', async () => { + assetMock.getByIds.mockResolvedValue([assetStub.image]); + mockReadTags({ Description: '\t \v \f \n \r' }); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(assetMock.upsertExif).toHaveBeenCalledWith( @@ -963,7 +971,7 @@ describe(MetadataService.name, () => { }), ); - metadataMock.readTags.mockResolvedValue({ ImageDescription: ' my\n description' }); + mockReadTags({ ImageDescription: ' my\n description' }); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(assetMock.upsertExif).toHaveBeenCalledWith( expect.objectContaining({ @@ -972,9 +980,9 @@ describe(MetadataService.name, () => { ); }); - it('handles a numeric description', async () => { + it('should handle a numeric description', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ Description: 1000 }); + mockReadTags({ Description: 1000 }); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(assetMock.upsertExif).toHaveBeenCalledWith( @@ -987,7 +995,7 @@ describe(MetadataService.name, () => { it('should skip importing metadata when the feature is disabled', async () => { assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); systemMock.get.mockResolvedValue({ metadata: { faces: { import: false } } }); - metadataMock.readTags.mockResolvedValue(metadataStub.withFace); + mockReadTags(metadataStub.withFace); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(personMock.getDistinctNames).not.toHaveBeenCalled(); }); @@ -995,7 +1003,7 @@ describe(MetadataService.name, () => { it('should skip importing metadata face for assets without tags.RegionInfo', async () => { assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } }); - metadataMock.readTags.mockResolvedValue(metadataStub.empty); + mockReadTags(metadataStub.empty); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(personMock.getDistinctNames).not.toHaveBeenCalled(); }); @@ -1003,7 +1011,7 @@ describe(MetadataService.name, () => { it('should skip importing faces without name', async () => { assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } }); - metadataMock.readTags.mockResolvedValue(metadataStub.withFaceNoName); + mockReadTags(metadataStub.withFaceNoName); personMock.getDistinctNames.mockResolvedValue([]); personMock.createAll.mockResolvedValue([]); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -1015,7 +1023,7 @@ describe(MetadataService.name, () => { it('should skip importing faces with empty name', async () => { assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } }); - metadataMock.readTags.mockResolvedValue(metadataStub.withFaceEmptyName); + mockReadTags(metadataStub.withFaceEmptyName); personMock.getDistinctNames.mockResolvedValue([]); personMock.createAll.mockResolvedValue([]); await sut.handleMetadataExtraction({ id: assetStub.image.id }); @@ -1027,7 +1035,7 @@ describe(MetadataService.name, () => { it('should apply metadata face tags creating new persons', async () => { assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } }); - metadataMock.readTags.mockResolvedValue(metadataStub.withFace); + mockReadTags(metadataStub.withFace); personMock.getDistinctNames.mockResolvedValue([]); personMock.createAll.mockResolvedValue([personStub.withName.id]); personMock.update.mockResolvedValue(personStub.withName); @@ -1064,7 +1072,7 @@ describe(MetadataService.name, () => { it('should assign metadata face tags to existing persons', async () => { assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]); systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } }); - metadataMock.readTags.mockResolvedValue(metadataStub.withFace); + mockReadTags(metadataStub.withFace); personMock.getDistinctNames.mockResolvedValue([{ id: personStub.withName.id, name: personStub.withName.name }]); personMock.createAll.mockResolvedValue([]); personMock.update.mockResolvedValue(personStub.withName); @@ -1095,7 +1103,7 @@ describe(MetadataService.name, () => { it('should handle invalid modify date', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ ModifyDate: '00:00:00.000' }); + mockReadTags({ ModifyDate: '00:00:00.000' }); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(assetMock.upsertExif).toHaveBeenCalledWith( @@ -1107,7 +1115,7 @@ describe(MetadataService.name, () => { it('should handle invalid rating value', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ Rating: 6 }); + mockReadTags({ Rating: 6 }); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(assetMock.upsertExif).toHaveBeenCalledWith( @@ -1119,7 +1127,7 @@ describe(MetadataService.name, () => { it('should handle valid rating value', async () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); - metadataMock.readTags.mockResolvedValue({ Rating: 5 }); + mockReadTags({ Rating: 5 }); await sut.handleMetadataExtraction({ id: assetStub.image.id }); expect(assetMock.upsertExif).toHaveBeenCalledWith( diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index a81d1b4904..38c86bcdb1 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -339,7 +339,7 @@ export class MetadataService extends BaseService { const sidecarTags = asset.sidecarPath ? await this.metadataRepository.readTags(asset.sidecarPath) : {}; const videoTags = asset.type === AssetType.VIDEO ? await this.getVideoTags(asset.originalPath) : {}; - // make sure dates comes from sidecar + // prefer dates from sidecar tags const sidecarDate = firstDateTime(sidecarTags as Tags, EXIF_DATE_TAGS); if (sidecarDate) { for (const tag of EXIF_DATE_TAGS) { @@ -347,6 +347,10 @@ export class MetadataService extends BaseService { } } + // prefer duration from video tags + delete mediaTags.Duration; + delete sidecarTags.Duration; + return { ...mediaTags, ...videoTags, ...sidecarTags }; } From 79acbc1d7ba27656df3b00af5f3e36e698d7dff1 Mon Sep 17 00:00:00 2001 From: bo0tzz Date: Thu, 17 Oct 2024 14:36:52 +0200 Subject: [PATCH 04/72] feat: show warning when running main branch build (#13462) * feat: show warning when running main branch build * fix: emoji weirdness * fix: use icon instead of emoji * fix: missing conditional --- .../shared-components/server-about-modal.svelte | 11 +++++++++++ .../shared-components/side-bar/server-status.svelte | 11 ++++++++++- web/src/lib/i18n/en.json | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/web/src/lib/components/shared-components/server-about-modal.svelte b/web/src/lib/components/shared-components/server-about-modal.svelte index 6a524331c2..1373a98d3f 100644 --- a/web/src/lib/components/shared-components/server-about-modal.svelte +++ b/web/src/lib/components/shared-components/server-about-modal.svelte @@ -4,6 +4,8 @@ import { type ServerAboutResponseDto, type ServerVersionHistoryResponseDto } from '@immich/sdk'; import { DateTime } from 'luxon'; import { t } from 'svelte-i18n'; + import { mdiAlert } from '@mdi/js'; + import Icon from '$lib/components/elements/icon.svelte'; export let onClose: () => void; @@ -152,6 +154,15 @@ {/if} + {#if info.sourceRef === 'main' && info.repository === 'immich-app/immich'} +
+ +

+ {$t('main_branch_warning')} +

+
+ {/if} +
{#if $connected && version} - + {:else}

{$t('unknown')}

{/if} diff --git a/web/src/lib/i18n/en.json b/web/src/lib/i18n/en.json index 155be38bcf..7b60fcbd2e 100644 --- a/web/src/lib/i18n/en.json +++ b/web/src/lib/i18n/en.json @@ -816,6 +816,7 @@ "look": "Look", "loop_videos": "Loop videos", "loop_videos_description": "Enable to automatically loop a video in the detail viewer.", + "main_branch_warning": "You're running a build from the main branch. We strongly recommend using a release version!", "make": "Make", "manage_shared_links": "Manage shared links", "manage_sharing_with_partners": "Manage sharing with partners", From 3f663106e8093824617128d7558423284c4741cd Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 17 Oct 2024 10:50:54 -0400 Subject: [PATCH 05/72] refactor(server): redis config (#13538) * refactor(server): redis config * refactor: cache parsed env data * chore: add database and redis tests --- server/src/app.module.ts | 10 +- server/src/config.ts | 35 --- server/src/interfaces/config.interface.ts | 10 + server/src/middleware/websocket.adapter.ts | 5 +- .../repositories/config.repository.spec.ts | 215 +++++++++++++----- server/src/repositories/config.repository.ts | 198 +++++++++------- server/src/repositories/job.repository.ts | 6 +- .../repositories/config.repository.mock.ts | 12 + 8 files changed, 318 insertions(+), 173 deletions(-) diff --git a/server/src/app.module.ts b/server/src/app.module.ts index 3f1e2ba08d..43aefbd0f0 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -7,7 +7,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { ClsModule } from 'nestjs-cls'; import { OpenTelemetryModule } from 'nestjs-otel'; import { commands } from 'src/commands'; -import { bullConfig, bullQueues, clsConfig, immichAppConfig } from 'src/config'; +import { clsConfig, immichAppConfig } from 'src/config'; import { controllers } from 'src/controllers'; import { databaseConfig } from 'src/database.config'; import { entities } from 'src/entities'; @@ -20,6 +20,7 @@ import { FileUploadInterceptor } from 'src/middleware/file-upload.interceptor'; import { GlobalExceptionFilter } from 'src/middleware/global-exception.filter'; import { LoggingInterceptor } from 'src/middleware/logging.interceptor'; import { repositories } from 'src/repositories'; +import { ConfigRepository } from 'src/repositories/config.repository'; import { services } from 'src/services'; import { DatabaseService } from 'src/services/database.service'; import { otelConfig } from 'src/utils/instrumentation'; @@ -35,9 +36,12 @@ const middleware = [ { provide: APP_GUARD, useClass: AuthGuard }, ]; +const configRepository = new ConfigRepository(); +const { bull } = configRepository.getEnv(); + const imports = [ - BullModule.forRoot(bullConfig), - BullModule.registerQueue(...bullQueues), + BullModule.forRoot(bull.config), + BullModule.registerQueue(...bull.queues), ClsModule.forRoot(clsConfig), ConfigModule.forRoot(immichAppConfig), OpenTelemetryModule.forRoot(otelConfig), diff --git a/server/src/config.ts b/server/src/config.ts index 4fdf23ecc2..ce97e0c2d0 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -1,9 +1,6 @@ -import { RegisterQueueOptions } from '@nestjs/bullmq'; import { ConfigModuleOptions } from '@nestjs/config'; import { CronExpression } from '@nestjs/schedule'; -import { QueueOptions } from 'bullmq'; import { Request, Response } from 'express'; -import { RedisOptions } from 'ioredis'; import Joi, { Root } from 'joi'; import { CLS_ID, ClsModuleOptions } from 'nestjs-cls'; import { ImmichHeader } from 'src/dtos/auth.dto'; @@ -363,38 +360,6 @@ export const immichAppConfig: ConfigModuleOptions = { }), }; -export function parseRedisConfig(): RedisOptions { - const redisUrl = process.env.REDIS_URL; - if (redisUrl && redisUrl.startsWith('ioredis://')) { - try { - const decodedString = Buffer.from(redisUrl.slice(10), 'base64').toString(); - return JSON.parse(decodedString); - } catch (error) { - throw new Error(`Failed to decode redis options: ${error}`); - } - } - return { - host: process.env.REDIS_HOSTNAME || 'redis', - port: Number.parseInt(process.env.REDIS_PORT || '6379'), - db: Number.parseInt(process.env.REDIS_DBINDEX || '0'), - username: process.env.REDIS_USERNAME || undefined, - password: process.env.REDIS_PASSWORD || undefined, - path: process.env.REDIS_SOCKET || undefined, - }; -} - -export const bullConfig: QueueOptions = { - prefix: 'immich_bull', - connection: parseRedisConfig(), - defaultJobOptions: { - attempts: 3, - removeOnComplete: true, - removeOnFail: false, - }, -}; - -export const bullQueues: RegisterQueueOptions[] = Object.values(QueueName).map((name) => ({ name })); - export const clsConfig: ClsModuleOptions = { middleware: { mount: true, diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.ts index 0e07116350..10e9a86aef 100644 --- a/server/src/interfaces/config.interface.ts +++ b/server/src/interfaces/config.interface.ts @@ -1,3 +1,6 @@ +import { RegisterQueueOptions } from '@nestjs/bullmq'; +import { QueueOptions } from 'bullmq'; +import { RedisOptions } from 'ioredis'; import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum'; import { VectorExtension } from 'src/interfaces/database.interface'; @@ -57,6 +60,13 @@ export interface EnvData { }; }; + redis: RedisOptions; + + bull: { + config: QueueOptions; + queues: RegisterQueueOptions[]; + }; + storage: { ignoreMountCheckErrors: boolean; }; diff --git a/server/src/middleware/websocket.adapter.ts b/server/src/middleware/websocket.adapter.ts index 4978b16102..da5e5e9816 100644 --- a/server/src/middleware/websocket.adapter.ts +++ b/server/src/middleware/websocket.adapter.ts @@ -3,7 +3,7 @@ import { IoAdapter } from '@nestjs/platform-socket.io'; import { createAdapter } from '@socket.io/redis-adapter'; import { Redis } from 'ioredis'; import { ServerOptions } from 'socket.io'; -import { parseRedisConfig } from 'src/config'; +import { IConfigRepository } from 'src/interfaces/config.interface'; export class WebSocketAdapter extends IoAdapter { constructor(private app: INestApplicationContext) { @@ -11,8 +11,9 @@ export class WebSocketAdapter extends IoAdapter { } createIOServer(port: number, options?: ServerOptions): any { + const { redis } = this.app.get(IConfigRepository).getEnv(); const server = super.createIOServer(port, options); - const pubClient = new Redis(parseRedisConfig()); + const pubClient = new Redis(redis); const subClient = pubClient.duplicate(); server.adapter(createAdapter(pubClient, subClient)); return server; diff --git a/server/src/repositories/config.repository.spec.ts b/server/src/repositories/config.repository.spec.ts index 83d89c6e01..78b512b2fd 100644 --- a/server/src/repositories/config.repository.spec.ts +++ b/server/src/repositories/config.repository.spec.ts @@ -1,76 +1,181 @@ -import { ConfigRepository } from 'src/repositories/config.repository'; +import { clearEnvCache, ConfigRepository } from 'src/repositories/config.repository'; -const getEnv = () => new ConfigRepository().getEnv(); +const getEnv = () => { + clearEnvCache(); + return new ConfigRepository().getEnv(); +}; + +const resetEnv = () => { + for (const env of [ + 'IMMICH_WORKERS_INCLUDE', + 'IMMICH_WORKERS_EXCLUDE', + + 'DB_URL', + 'DB_HOSTNAME', + 'DB_PORT', + 'DB_USERNAME', + 'DB_PASSWORD', + 'DB_DATABASE_NAME', + 'DB_SKIP_MIGRATIONS', + 'DB_VECTOR_EXTENSION', + + 'REDIS_HOSTNAME', + 'REDIS_PORT', + 'REDIS_DBINDEX', + 'REDIS_USERNAME', + 'REDIS_PASSWORD', + 'REDIS_SOCKET', + 'REDIS_URL', + + 'NO_COLOR', + ]) { + delete process.env[env]; + } +}; + +const sentinelConfig = { + sentinels: [ + { + host: 'redis-sentinel-node-0', + port: 26_379, + }, + { + host: 'redis-sentinel-node-1', + port: 26_379, + }, + { + host: 'redis-sentinel-node-2', + port: 26_379, + }, + ], + name: 'redis-sentinel', +}; describe('getEnv', () => { beforeEach(() => { - delete process.env.IMMICH_WORKERS_INCLUDE; - delete process.env.IMMICH_WORKERS_EXCLUDE; - delete process.env.NO_COLOR; + resetEnv(); }); - it('should return default workers', () => { - const { workers } = getEnv(); - expect(workers).toEqual(['api', 'microservices']); + describe('database', () => { + it('should use defaults', () => { + const { database } = getEnv(); + expect(database).toEqual({ + url: undefined, + host: 'database', + port: 5432, + name: 'immich', + username: 'postgres', + password: 'postgres', + skipMigrations: false, + vectorExtension: 'vectors', + }); + }); + + it('should allow skipping migrations', () => { + process.env.DB_SKIP_MIGRATIONS = 'true'; + const { database } = getEnv(); + expect(database).toMatchObject({ skipMigrations: true }); + }); }); - it('should return included workers', () => { - process.env.IMMICH_WORKERS_INCLUDE = 'api'; - const { workers } = getEnv(); - expect(workers).toEqual(['api']); + describe('redis', () => { + it('should use defaults', () => { + const { redis } = getEnv(); + expect(redis).toEqual({ + host: 'redis', + port: 6379, + db: 0, + username: undefined, + password: undefined, + path: undefined, + }); + }); + + it('should parse base64 encoded config, ignore other env', () => { + process.env.REDIS_URL = `ioredis://${Buffer.from(JSON.stringify(sentinelConfig)).toString('base64')}`; + process.env.REDIS_HOSTNAME = 'redis-host'; + process.env.REDIS_USERNAME = 'redis-user'; + process.env.REDIS_PASSWORD = 'redis-password'; + const { redis } = getEnv(); + expect(redis).toEqual(sentinelConfig); + }); + + it('should reject invalid json', () => { + process.env.REDIS_URL = `ioredis://${Buffer.from('{ "invalid json"').toString('base64')}`; + expect(() => getEnv()).toThrowError('Failed to decode redis options'); + }); }); - it('should excluded workers from defaults', () => { - process.env.IMMICH_WORKERS_EXCLUDE = 'api'; - const { workers } = getEnv(); - expect(workers).toEqual(['microservices']); + describe('noColor', () => { + beforeEach(() => { + delete process.env.NO_COLOR; + }); + + it('should default noColor to false', () => { + const { noColor } = getEnv(); + expect(noColor).toBe(false); + }); + + it('should map NO_COLOR=1 to true', () => { + process.env.NO_COLOR = '1'; + const { noColor } = getEnv(); + expect(noColor).toBe(true); + }); + + it('should map NO_COLOR=true to true', () => { + process.env.NO_COLOR = 'true'; + const { noColor } = getEnv(); + expect(noColor).toBe(true); + }); }); - it('should exclude workers from include list', () => { - process.env.IMMICH_WORKERS_INCLUDE = 'api,microservices,randomservice'; - process.env.IMMICH_WORKERS_EXCLUDE = 'randomservice,microservices'; - const { workers } = getEnv(); - expect(workers).toEqual(['api']); - }); + describe('workers', () => { + it('should return default workers', () => { + const { workers } = getEnv(); + expect(workers).toEqual(['api', 'microservices']); + }); - it('should remove whitespace from included workers before parsing', () => { - process.env.IMMICH_WORKERS_INCLUDE = 'api, microservices'; - const { workers } = getEnv(); - expect(workers).toEqual(['api', 'microservices']); - }); + it('should return included workers', () => { + process.env.IMMICH_WORKERS_INCLUDE = 'api'; + const { workers } = getEnv(); + expect(workers).toEqual(['api']); + }); - it('should remove whitespace from excluded workers before parsing', () => { - process.env.IMMICH_WORKERS_EXCLUDE = 'api, microservices'; - const { workers } = getEnv(); - expect(workers).toEqual([]); - }); + it('should excluded workers from defaults', () => { + process.env.IMMICH_WORKERS_EXCLUDE = 'api'; + const { workers } = getEnv(); + expect(workers).toEqual(['microservices']); + }); - it('should remove whitespace from included and excluded workers before parsing', () => { - process.env.IMMICH_WORKERS_INCLUDE = 'api, microservices, randomservice,randomservice2'; - process.env.IMMICH_WORKERS_EXCLUDE = 'randomservice,microservices, randomservice2'; - const { workers } = getEnv(); - expect(workers).toEqual(['api']); - }); + it('should exclude workers from include list', () => { + process.env.IMMICH_WORKERS_INCLUDE = 'api,microservices,randomservice'; + process.env.IMMICH_WORKERS_EXCLUDE = 'randomservice,microservices'; + const { workers } = getEnv(); + expect(workers).toEqual(['api']); + }); - it('should throw error for invalid workers', () => { - process.env.IMMICH_WORKERS_INCLUDE = 'api,microservices,randomservice'; - expect(getEnv).toThrowError('Invalid worker(s) found: api,microservices,randomservice'); - }); + it('should remove whitespace from included workers before parsing', () => { + process.env.IMMICH_WORKERS_INCLUDE = 'api, microservices'; + const { workers } = getEnv(); + expect(workers).toEqual(['api', 'microservices']); + }); - it('should default noColor to false', () => { - const { noColor } = getEnv(); - expect(noColor).toBe(false); - }); + it('should remove whitespace from excluded workers before parsing', () => { + process.env.IMMICH_WORKERS_EXCLUDE = 'api, microservices'; + const { workers } = getEnv(); + expect(workers).toEqual([]); + }); - it('should map NO_COLOR=1 to true', () => { - process.env.NO_COLOR = '1'; - const { noColor } = getEnv(); - expect(noColor).toBe(true); - }); + it('should remove whitespace from included and excluded workers before parsing', () => { + process.env.IMMICH_WORKERS_INCLUDE = 'api, microservices, randomservice,randomservice2'; + process.env.IMMICH_WORKERS_EXCLUDE = 'randomservice,microservices, randomservice2'; + const { workers } = getEnv(); + expect(workers).toEqual(['api']); + }); - it('should map NO_COLOR=true to true', () => { - process.env.NO_COLOR = 'true'; - const { noColor } = getEnv(); - expect(noColor).toBe(true); + it('should throw error for invalid workers', () => { + process.env.IMMICH_WORKERS_INCLUDE = 'api,microservices,randomservice'; + expect(getEnv).toThrowError('Invalid worker(s) found: api,microservices,randomservice'); + }); }); }); diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index 883c65846a..585e719e3a 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -4,6 +4,7 @@ import { citiesFile } from 'src/constants'; import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum'; import { EnvData, IConfigRepository } from 'src/interfaces/config.interface'; import { DatabaseExtension } from 'src/interfaces/database.interface'; +import { QueueName } from 'src/interfaces/job.interface'; import { setDifference } from 'src/utils/set'; // TODO replace src/config validation with class-validator, here @@ -29,86 +30,131 @@ const asSet = (value: string | undefined, defaults: ImmichWorker[]) => { return new Set(values.length === 0 ? defaults : (values as ImmichWorker[])); }; +const getEnv = (): EnvData => { + const included = asSet(process.env.IMMICH_WORKERS_INCLUDE, [ImmichWorker.API, ImmichWorker.MICROSERVICES]); + const excluded = asSet(process.env.IMMICH_WORKERS_EXCLUDE, []); + const workers = [...setDifference(included, excluded)]; + for (const worker of workers) { + if (!WORKER_TYPES.has(worker)) { + throw new Error(`Invalid worker(s) found: ${workers.join(',')}`); + } + } + + const environment = process.env.IMMICH_ENV as ImmichEnvironment; + const isProd = environment === ImmichEnvironment.PRODUCTION; + const buildFolder = process.env.IMMICH_BUILD_DATA || '/build'; + const folders = { + geodata: join(buildFolder, 'geodata'), + web: join(buildFolder, 'www'), + }; + + let redisConfig = { + host: process.env.REDIS_HOSTNAME || 'redis', + port: Number.parseInt(process.env.REDIS_PORT || '') || 6379, + db: Number.parseInt(process.env.REDIS_DBINDEX || '') || 0, + username: process.env.REDIS_USERNAME || undefined, + password: process.env.REDIS_PASSWORD || undefined, + path: process.env.REDIS_SOCKET || undefined, + }; + + const redisUrl = process.env.REDIS_URL; + if (redisUrl && redisUrl.startsWith('ioredis://')) { + try { + redisConfig = JSON.parse(Buffer.from(redisUrl.slice(10), 'base64').toString()); + } catch (error) { + throw new Error(`Failed to decode redis options: ${error}`); + } + } + + return { + host: process.env.IMMICH_HOST, + port: Number(process.env.IMMICH_PORT) || 2283, + environment, + configFile: process.env.IMMICH_CONFIG_FILE, + logLevel: process.env.IMMICH_LOG_LEVEL as LogLevel, + + buildMetadata: { + build: process.env.IMMICH_BUILD, + buildUrl: process.env.IMMICH_BUILD_URL, + buildImage: process.env.IMMICH_BUILD_IMAGE, + buildImageUrl: process.env.IMMICH_BUILD_IMAGE_URL, + repository: process.env.IMMICH_REPOSITORY, + repositoryUrl: process.env.IMMICH_REPOSITORY_URL, + sourceRef: process.env.IMMICH_SOURCE_REF, + sourceCommit: process.env.IMMICH_SOURCE_COMMIT, + sourceUrl: process.env.IMMICH_SOURCE_URL, + thirdPartySourceUrl: process.env.IMMICH_THIRD_PARTY_SOURCE_URL, + thirdPartyBugFeatureUrl: process.env.IMMICH_THIRD_PARTY_BUG_FEATURE_URL, + thirdPartyDocumentationUrl: process.env.IMMICH_THIRD_PARTY_DOCUMENTATION_URL, + thirdPartySupportUrl: process.env.IMMICH_THIRD_PARTY_SUPPORT_URL, + }, + + bull: { + config: { + prefix: 'immich_bull', + connection: { ...redisConfig }, + defaultJobOptions: { + attempts: 3, + removeOnComplete: true, + removeOnFail: false, + }, + }, + queues: Object.values(QueueName).map((name) => ({ name })), + }, + + database: { + url: process.env.DB_URL, + host: process.env.DB_HOSTNAME || 'database', + port: Number(process.env.DB_PORT) || 5432, + username: process.env.DB_USERNAME || 'postgres', + password: process.env.DB_PASSWORD || 'postgres', + name: process.env.DB_DATABASE_NAME || 'immich', + + skipMigrations: process.env.DB_SKIP_MIGRATIONS === 'true', + vectorExtension: + process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS, + }, + + licensePublicKey: isProd ? productionKeys : stagingKeys, + + redis: redisConfig, + + resourcePaths: { + lockFile: join(buildFolder, 'build-lock.json'), + geodata: { + dateFile: join(folders.geodata, 'geodata-date.txt'), + admin1: join(folders.geodata, 'admin1CodesASCII.txt'), + admin2: join(folders.geodata, 'admin2Codes.txt'), + cities500: join(folders.geodata, citiesFile), + naturalEarthCountriesPath: join(folders.geodata, 'ne_10m_admin_0_countries.geojson'), + }, + web: { + root: folders.web, + indexHtml: join(folders.web, 'index.html'), + }, + }, + + storage: { + ignoreMountCheckErrors: process.env.IMMICH_IGNORE_MOUNT_CHECK_ERRORS === 'true', + }, + + workers, + + noColor: !!process.env.NO_COLOR, + }; +}; + +let cached: EnvData | undefined; + @Injectable() export class ConfigRepository implements IConfigRepository { getEnv(): EnvData { - const included = asSet(process.env.IMMICH_WORKERS_INCLUDE, [ImmichWorker.API, ImmichWorker.MICROSERVICES]); - const excluded = asSet(process.env.IMMICH_WORKERS_EXCLUDE, []); - const workers = [...setDifference(included, excluded)]; - for (const worker of workers) { - if (!WORKER_TYPES.has(worker)) { - throw new Error(`Invalid worker(s) found: ${workers.join(',')}`); - } + if (!cached) { + cached = getEnv(); } - const environment = process.env.IMMICH_ENV as ImmichEnvironment; - const isProd = environment === ImmichEnvironment.PRODUCTION; - const buildFolder = process.env.IMMICH_BUILD_DATA || '/build'; - const folders = { - geodata: join(buildFolder, 'geodata'), - web: join(buildFolder, 'www'), - }; - - return { - host: process.env.IMMICH_HOST, - port: Number(process.env.IMMICH_PORT) || 2283, - environment, - configFile: process.env.IMMICH_CONFIG_FILE, - logLevel: process.env.IMMICH_LOG_LEVEL as LogLevel, - - buildMetadata: { - build: process.env.IMMICH_BUILD, - buildUrl: process.env.IMMICH_BUILD_URL, - buildImage: process.env.IMMICH_BUILD_IMAGE, - buildImageUrl: process.env.IMMICH_BUILD_IMAGE_URL, - repository: process.env.IMMICH_REPOSITORY, - repositoryUrl: process.env.IMMICH_REPOSITORY_URL, - sourceRef: process.env.IMMICH_SOURCE_REF, - sourceCommit: process.env.IMMICH_SOURCE_COMMIT, - sourceUrl: process.env.IMMICH_SOURCE_URL, - thirdPartySourceUrl: process.env.IMMICH_THIRD_PARTY_SOURCE_URL, - thirdPartyBugFeatureUrl: process.env.IMMICH_THIRD_PARTY_BUG_FEATURE_URL, - thirdPartyDocumentationUrl: process.env.IMMICH_THIRD_PARTY_DOCUMENTATION_URL, - thirdPartySupportUrl: process.env.IMMICH_THIRD_PARTY_SUPPORT_URL, - }, - - database: { - url: process.env.DB_URL, - host: process.env.DB_HOSTNAME || 'database', - port: Number(process.env.DB_PORT) || 5432, - username: process.env.DB_USERNAME || 'postgres', - password: process.env.DB_PASSWORD || 'postgres', - name: process.env.DB_DATABASE_NAME || 'immich', - - skipMigrations: process.env.DB_SKIP_MIGRATIONS === 'true', - vectorExtension: - process.env.DB_VECTOR_EXTENSION === 'pgvector' ? DatabaseExtension.VECTOR : DatabaseExtension.VECTORS, - }, - - licensePublicKey: isProd ? productionKeys : stagingKeys, - - resourcePaths: { - lockFile: join(buildFolder, 'build-lock.json'), - geodata: { - dateFile: join(folders.geodata, 'geodata-date.txt'), - admin1: join(folders.geodata, 'admin1CodesASCII.txt'), - admin2: join(folders.geodata, 'admin2Codes.txt'), - cities500: join(folders.geodata, citiesFile), - naturalEarthCountriesPath: join(folders.geodata, 'ne_10m_admin_0_countries.geojson'), - }, - web: { - root: folders.web, - indexHtml: join(folders.web, 'index.html'), - }, - }, - - storage: { - ignoreMountCheckErrors: process.env.IMMICH_IGNORE_MOUNT_CHECK_ERRORS === 'true', - }, - - workers, - - noColor: !!process.env.NO_COLOR, - }; + return cached; } } + +export const clearEnvCache = () => (cached = undefined); diff --git a/server/src/repositories/job.repository.ts b/server/src/repositories/job.repository.ts index 3f154ee016..3ff26f1ba4 100644 --- a/server/src/repositories/job.repository.ts +++ b/server/src/repositories/job.repository.ts @@ -5,7 +5,7 @@ import { SchedulerRegistry } from '@nestjs/schedule'; import { Job, JobsOptions, Processor, Queue, Worker, WorkerOptions } from 'bullmq'; import { CronJob, CronTime } from 'cron'; import { setTimeout } from 'node:timers/promises'; -import { bullConfig } from 'src/config'; +import { IConfigRepository } from 'src/interfaces/config.interface'; import { IJobRepository, JobCounts, @@ -106,14 +106,16 @@ export class JobRepository implements IJobRepository { constructor( private moduleReference: ModuleRef, private schedulerReqistry: SchedulerRegistry, + @Inject(IConfigRepository) private configRepository: IConfigRepository, @Inject(ILoggerRepository) private logger: ILoggerRepository, ) { this.logger.setContext(JobRepository.name); } addHandler(queueName: QueueName, concurrency: number, handler: (item: JobItem) => Promise) { + const { bull } = this.configRepository.getEnv(); const workerHandler: Processor = async (job: Job) => handler(job as JobItem); - const workerOptions: WorkerOptions = { ...bullConfig, concurrency }; + const workerOptions: WorkerOptions = { ...bull.config, concurrency }; this.workers[queueName] = new Worker(queueName, workerHandler, workerOptions); } diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts index 8e71ba2dca..65e419fe36 100644 --- a/server/test/repositories/config.repository.mock.ts +++ b/server/test/repositories/config.repository.mock.ts @@ -8,6 +8,12 @@ const envData: EnvData = { environment: ImmichEnvironment.PRODUCTION, buildMetadata: {}, + bull: { + config: { + prefix: 'immich_bull', + }, + queues: [{ name: 'queue-1' }], + }, database: { host: 'database', @@ -25,6 +31,12 @@ const envData: EnvData = { server: 'server-public-key', }, + redis: { + host: 'redis', + port: 6379, + db: 0, + }, + resourcePaths: { lockFile: 'build-lock.json', geodata: { From bb694aeeeba3b7e00ff00b5d76fd7a4b2b3565d1 Mon Sep 17 00:00:00 2001 From: jedi04 <78037206+jedi04@users.noreply.github.com> Date: Thu, 17 Oct 2024 21:54:50 +0530 Subject: [PATCH 06/72] fix(server): Allow passwordless users when oauth enabled (#13517) * fix(server): Allow passwordless users when oauth enabled * fix(web): Use features flags for checking oauth --- server/src/dtos/user.dto.ts | 1 - server/src/services/user-admin.service.ts | 4 ++++ web/src/lib/components/forms/create-user-form.svelte | 10 ++++++++-- web/src/routes/admin/user-management/+page.svelte | 3 ++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/server/src/dtos/user.dto.ts b/server/src/dtos/user.dto.ts index 36f0b6386f..593a7934bc 100644 --- a/server/src/dtos/user.dto.ts +++ b/server/src/dtos/user.dto.ts @@ -62,7 +62,6 @@ export class UserAdminCreateDto { @Transform(toEmail) email!: string; - @IsNotEmpty() @IsString() password!: string; diff --git a/server/src/services/user-admin.service.ts b/server/src/services/user-admin.service.ts index 94608a24ac..84a5b5842d 100644 --- a/server/src/services/user-admin.service.ts +++ b/server/src/services/user-admin.service.ts @@ -26,6 +26,10 @@ export class UserAdminService extends BaseService { async create(dto: UserAdminCreateDto): Promise { const { notify, ...rest } = dto; + const config = await this.getConfig({ withCache: false }); + if (!config.oauth.enabled && !rest.password) { + throw new BadRequestException('password is required'); + } const user = await createUser({ userRepo: this.userRepository, cryptoRepo: this.cryptoRepository }, rest); await this.eventRepository.emit('user.signup', { diff --git a/web/src/lib/components/forms/create-user-form.svelte b/web/src/lib/components/forms/create-user-form.svelte index 9c4b83002b..0687912542 100644 --- a/web/src/lib/components/forms/create-user-form.svelte +++ b/web/src/lib/components/forms/create-user-form.svelte @@ -13,6 +13,7 @@ export let onClose: () => void; export let onSubmit: () => void; export let onCancel: () => void; + export let oauthEnabled = false; let error: string; let success: string; @@ -90,12 +91,17 @@
- +
- +
diff --git a/web/src/routes/admin/user-management/+page.svelte b/web/src/routes/admin/user-management/+page.svelte index 2313b17cb1..80c0169176 100644 --- a/web/src/routes/admin/user-management/+page.svelte +++ b/web/src/routes/admin/user-management/+page.svelte @@ -15,7 +15,7 @@ notificationController, } from '$lib/components/shared-components/notification/notification'; import { locale } from '$lib/stores/preferences.store'; - import { serverConfig } from '$lib/stores/server-config.store'; + import { serverConfig, featureFlags } from '$lib/stores/server-config.store'; import { user } from '$lib/stores/user.store'; import { websocketEvents } from '$lib/stores/websocket'; import { copyToClipboard } from '$lib/utils'; @@ -113,6 +113,7 @@ onSubmit={onUserCreated} onCancel={() => (shouldShowCreateUserForm = false)} onClose={() => (shouldShowCreateUserForm = false)} + oauthEnabled={$featureFlags.oauth} /> {/if} From 3d971f69dc46998308f1db252c5d09c55f37c0b9 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 17 Oct 2024 13:11:51 -0400 Subject: [PATCH 07/72] refactor(server): storage template options (#13553) --- server/src/constants.ts | 29 ---------- .../controllers/system-config.controller.ts | 8 ++- .../services/storage-template.service.spec.ts | 35 +++++++++++ .../src/services/storage-template.service.ts | 58 ++++++++++++------- .../services/system-config.service.spec.ts | 35 ----------- server/src/services/system-config.service.ts | 27 +-------- 6 files changed, 80 insertions(+), 112 deletions(-) diff --git a/server/src/constants.ts b/server/src/constants.ts index eef9ffab05..e99970723a 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -30,35 +30,6 @@ export const excludePaths = ['/.well-known/immich', '/custom.css', '/favicon.ico export const FACE_THUMBNAIL_SIZE = 250; -export const supportedYearTokens = ['y', 'yy']; -export const supportedMonthTokens = ['M', 'MM', 'MMM', 'MMMM']; -export const supportedWeekTokens = ['W', 'WW']; -export const supportedDayTokens = ['d', 'dd']; -export const supportedHourTokens = ['h', 'hh', 'H', 'HH']; -export const supportedMinuteTokens = ['m', 'mm']; -export const supportedSecondTokens = ['s', 'ss', 'SSS']; -export const supportedPresetTokens = [ - '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}', - '{{y}}/{{MM}}-{{dd}}/{{filename}}', - '{{y}}/{{MMMM}}-{{dd}}/{{filename}}', - '{{y}}/{{MM}}/{{filename}}', - '{{y}}/{{#if album}}{{album}}{{else}}Other/{{MM}}{{/if}}/{{filename}}', - '{{y}}/{{MMM}}/{{filename}}', - '{{y}}/{{MMMM}}/{{filename}}', - '{{y}}/{{MM}}/{{dd}}/{{filename}}', - '{{y}}/{{MMMM}}/{{dd}}/{{filename}}', - '{{y}}/{{y}}-{{MM}}/{{y}}-{{MM}}-{{dd}}/{{filename}}', - '{{y}}-{{MM}}-{{dd}}/{{filename}}', - '{{y}}-{{MMM}}-{{dd}}/{{filename}}', - '{{y}}-{{MMMM}}-{{dd}}/{{filename}}', - '{{y}}/{{y}}-{{MM}}/{{filename}}', - '{{y}}/{{y}}-{{WW}}/{{filename}}', - '{{y}}/{{y}}-{{MM}}-{{dd}}/{{assetId}}', - '{{y}}/{{y}}-{{MM}}/{{assetId}}', - '{{y}}/{{y}}-{{WW}}/{{assetId}}', - '{{album}}/{{filename}}', -]; - type ModelInfo = { dimSize: number }; export const CLIP_MODEL_INFO: Record = { RN101__openai: { dimSize: 512 }, diff --git a/server/src/controllers/system-config.controller.ts b/server/src/controllers/system-config.controller.ts index f59c8ad66c..58e8bde87b 100644 --- a/server/src/controllers/system-config.controller.ts +++ b/server/src/controllers/system-config.controller.ts @@ -3,12 +3,16 @@ import { ApiTags } from '@nestjs/swagger'; import { SystemConfigDto, SystemConfigTemplateStorageOptionDto } from 'src/dtos/system-config.dto'; import { Permission } from 'src/enum'; import { Authenticated } from 'src/middleware/auth.guard'; +import { StorageTemplateService } from 'src/services/storage-template.service'; import { SystemConfigService } from 'src/services/system-config.service'; @ApiTags('System Config') @Controller('system-config') export class SystemConfigController { - constructor(private service: SystemConfigService) {} + constructor( + private service: SystemConfigService, + private storageTemplateService: StorageTemplateService, + ) {} @Get() @Authenticated({ permission: Permission.SYSTEM_CONFIG_READ, admin: true }) @@ -31,6 +35,6 @@ export class SystemConfigController { @Get('storage-template-options') @Authenticated({ permission: Permission.SYSTEM_CONFIG_READ, admin: true }) getStorageTemplateOptions(): SystemConfigTemplateStorageOptionDto { - return this.service.getStorageTemplateOptions(); + return this.storageTemplateService.getStorageTemplateOptions(); } } diff --git a/server/src/services/storage-template.service.spec.ts b/server/src/services/storage-template.service.spec.ts index 6e5af3baf9..fd063bd50d 100644 --- a/server/src/services/storage-template.service.spec.ts +++ b/server/src/services/storage-template.service.spec.ts @@ -70,6 +70,41 @@ describe(StorageTemplateService.name, () => { }); }); + describe('getStorageTemplateOptions', () => { + it('should send back the datetime variables', () => { + expect(sut.getStorageTemplateOptions()).toEqual({ + dayOptions: ['d', 'dd'], + hourOptions: ['h', 'hh', 'H', 'HH'], + minuteOptions: ['m', 'mm'], + monthOptions: ['M', 'MM', 'MMM', 'MMMM'], + presetOptions: [ + '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}', + '{{y}}/{{MM}}-{{dd}}/{{filename}}', + '{{y}}/{{MMMM}}-{{dd}}/{{filename}}', + '{{y}}/{{MM}}/{{filename}}', + '{{y}}/{{#if album}}{{album}}{{else}}Other/{{MM}}{{/if}}/{{filename}}', + '{{y}}/{{MMM}}/{{filename}}', + '{{y}}/{{MMMM}}/{{filename}}', + '{{y}}/{{MM}}/{{dd}}/{{filename}}', + '{{y}}/{{MMMM}}/{{dd}}/{{filename}}', + '{{y}}/{{y}}-{{MM}}/{{y}}-{{MM}}-{{dd}}/{{filename}}', + '{{y}}-{{MM}}-{{dd}}/{{filename}}', + '{{y}}-{{MMM}}-{{dd}}/{{filename}}', + '{{y}}-{{MMMM}}-{{dd}}/{{filename}}', + '{{y}}/{{y}}-{{MM}}/{{filename}}', + '{{y}}/{{y}}-{{WW}}/{{filename}}', + '{{y}}/{{y}}-{{MM}}-{{dd}}/{{assetId}}', + '{{y}}/{{y}}-{{MM}}/{{assetId}}', + '{{y}}/{{y}}-{{WW}}/{{assetId}}', + '{{album}}/{{filename}}', + ], + secondOptions: ['s', 'ss', 'SSS'], + weekOptions: ['W', 'WW'], + yearOptions: ['y', 'yy'], + }); + }); + }); + describe('handleMigrationSingle', () => { it('should skip when storage template is disabled', async () => { systemMock.get.mockResolvedValue({ storageTemplate: { enabled: false } }); diff --git a/server/src/services/storage-template.service.ts b/server/src/services/storage-template.service.ts index e400981f54..d239435660 100644 --- a/server/src/services/storage-template.service.ts +++ b/server/src/services/storage-template.service.ts @@ -3,17 +3,9 @@ import handlebar from 'handlebars'; import { DateTime } from 'luxon'; import path from 'node:path'; import sanitize from 'sanitize-filename'; -import { - supportedDayTokens, - supportedHourTokens, - supportedMinuteTokens, - supportedMonthTokens, - supportedSecondTokens, - supportedWeekTokens, - supportedYearTokens, -} from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; import { OnEvent } from 'src/decorators'; +import { SystemConfigTemplateStorageOptionDto } from 'src/dtos/system-config.dto'; import { AssetEntity } from 'src/entities/asset.entity'; import { AssetPathType, AssetType, StorageFolder } from 'src/enum'; import { DatabaseLock } from 'src/interfaces/database.interface'; @@ -23,6 +15,38 @@ import { BaseService } from 'src/services/base.service'; import { getLivePhotoMotionFilename } from 'src/utils/file'; import { usePagination } from 'src/utils/pagination'; +const storageTokens = { + secondOptions: ['s', 'ss', 'SSS'], + minuteOptions: ['m', 'mm'], + dayOptions: ['d', 'dd'], + weekOptions: ['W', 'WW'], + hourOptions: ['h', 'hh', 'H', 'HH'], + yearOptions: ['y', 'yy'], + monthOptions: ['M', 'MM', 'MMM', 'MMMM'], +}; + +const storagePresets = [ + '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}', + '{{y}}/{{MM}}-{{dd}}/{{filename}}', + '{{y}}/{{MMMM}}-{{dd}}/{{filename}}', + '{{y}}/{{MM}}/{{filename}}', + '{{y}}/{{#if album}}{{album}}{{else}}Other/{{MM}}{{/if}}/{{filename}}', + '{{y}}/{{MMM}}/{{filename}}', + '{{y}}/{{MMMM}}/{{filename}}', + '{{y}}/{{MM}}/{{dd}}/{{filename}}', + '{{y}}/{{MMMM}}/{{dd}}/{{filename}}', + '{{y}}/{{y}}-{{MM}}/{{y}}-{{MM}}-{{dd}}/{{filename}}', + '{{y}}-{{MM}}-{{dd}}/{{filename}}', + '{{y}}-{{MMM}}-{{dd}}/{{filename}}', + '{{y}}-{{MMMM}}-{{dd}}/{{filename}}', + '{{y}}/{{y}}-{{MM}}/{{filename}}', + '{{y}}/{{y}}-{{WW}}/{{filename}}', + '{{y}}/{{y}}-{{MM}}-{{dd}}/{{assetId}}', + '{{y}}/{{y}}-{{MM}}/{{assetId}}', + '{{y}}/{{y}}-{{WW}}/{{assetId}}', + '{{album}}/{{filename}}', +]; + export interface MoveAssetMetadata { storageLabel: string | null; filename: string; @@ -80,6 +104,10 @@ export class StorageTemplateService extends BaseService { } } + getStorageTemplateOptions(): SystemConfigTemplateStorageOptionDto { + return { ...storageTokens, presetOptions: storagePresets }; + } + async handleMigrationSingle({ id }: IEntityJob): Promise { const config = await this.getConfig({ withCache: true }); const storageTemplateEnabled = config.storageTemplate.enabled; @@ -277,17 +305,7 @@ export class StorageTemplateService extends BaseService { const zone = asset.exifInfo?.timeZone || systemTimeZone; const dt = DateTime.fromJSDate(asset.fileCreatedAt, { zone }); - const dateTokens = [ - ...supportedYearTokens, - ...supportedMonthTokens, - ...supportedWeekTokens, - ...supportedDayTokens, - ...supportedHourTokens, - ...supportedMinuteTokens, - ...supportedSecondTokens, - ]; - - for (const token of dateTokens) { + for (const token of Object.values(storageTokens).flat()) { substitutions[token] = dt.toFormat(token); } diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index 52a5b1dcd8..807d8299b8 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -341,41 +341,6 @@ describe(SystemConfigService.name, () => { } }); - describe('getStorageTemplateOptions', () => { - it('should send back the datetime variables', () => { - expect(sut.getStorageTemplateOptions()).toEqual({ - dayOptions: ['d', 'dd'], - hourOptions: ['h', 'hh', 'H', 'HH'], - minuteOptions: ['m', 'mm'], - monthOptions: ['M', 'MM', 'MMM', 'MMMM'], - presetOptions: [ - '{{y}}/{{y}}-{{MM}}-{{dd}}/{{filename}}', - '{{y}}/{{MM}}-{{dd}}/{{filename}}', - '{{y}}/{{MMMM}}-{{dd}}/{{filename}}', - '{{y}}/{{MM}}/{{filename}}', - '{{y}}/{{#if album}}{{album}}{{else}}Other/{{MM}}{{/if}}/{{filename}}', - '{{y}}/{{MMM}}/{{filename}}', - '{{y}}/{{MMMM}}/{{filename}}', - '{{y}}/{{MM}}/{{dd}}/{{filename}}', - '{{y}}/{{MMMM}}/{{dd}}/{{filename}}', - '{{y}}/{{y}}-{{MM}}/{{y}}-{{MM}}-{{dd}}/{{filename}}', - '{{y}}-{{MM}}-{{dd}}/{{filename}}', - '{{y}}-{{MMM}}-{{dd}}/{{filename}}', - '{{y}}-{{MMMM}}-{{dd}}/{{filename}}', - '{{y}}/{{y}}-{{MM}}/{{filename}}', - '{{y}}/{{y}}-{{WW}}/{{filename}}', - '{{y}}/{{y}}-{{MM}}-{{dd}}/{{assetId}}', - '{{y}}/{{y}}-{{MM}}/{{assetId}}', - '{{y}}/{{y}}-{{WW}}/{{assetId}}', - '{{album}}/{{filename}}', - ], - secondOptions: ['s', 'ss', 'SSS'], - weekOptions: ['W', 'WW'], - yearOptions: ['y', 'yy'], - }); - }); - }); - describe('updateConfig', () => { it('should update the config and emit an event', async () => { systemMock.get.mockResolvedValue(partialConfig); diff --git a/server/src/services/system-config.service.ts b/server/src/services/system-config.service.ts index 96a1f0897b..8f19b22173 100644 --- a/server/src/services/system-config.service.ts +++ b/server/src/services/system-config.service.ts @@ -2,18 +2,8 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import { instanceToPlain } from 'class-transformer'; import _ from 'lodash'; import { defaults } from 'src/config'; -import { - supportedDayTokens, - supportedHourTokens, - supportedMinuteTokens, - supportedMonthTokens, - supportedPresetTokens, - supportedSecondTokens, - supportedWeekTokens, - supportedYearTokens, -} from 'src/constants'; import { OnEvent } from 'src/decorators'; -import { SystemConfigDto, SystemConfigTemplateStorageOptionDto, mapConfig } from 'src/dtos/system-config.dto'; +import { SystemConfigDto, mapConfig } from 'src/dtos/system-config.dto'; import { ArgOf } from 'src/interfaces/event.interface'; import { BaseService } from 'src/services/base.service'; import { clearConfigCache } from 'src/utils/config'; @@ -77,21 +67,6 @@ export class SystemConfigService extends BaseService { return mapConfig(newConfig); } - getStorageTemplateOptions(): SystemConfigTemplateStorageOptionDto { - const options = new SystemConfigTemplateStorageOptionDto(); - - options.dayOptions = supportedDayTokens; - options.weekOptions = supportedWeekTokens; - options.monthOptions = supportedMonthTokens; - options.yearOptions = supportedYearTokens; - options.hourOptions = supportedHourTokens; - options.secondOptions = supportedSecondTokens; - options.minuteOptions = supportedMinuteTokens; - options.presetOptions = supportedPresetTokens; - - return options; - } - async getCustomCss(): Promise { const { theme } = await this.getConfig({ withCache: false }); return theme.customCss; From d9949434f60c07e01a6ffbd5a833ad90e749396d Mon Sep 17 00:00:00 2001 From: CrushedAsian255 <155333241+CrushedAsian255@users.noreply.github.com> Date: Fri, 18 Oct 2024 04:13:35 +1100 Subject: [PATCH 08/72] feat(web): Change relink person icon from minus to pencil (#13536) The relink person icon is currently a minus symbol. This can be confusing as it looks like a "remove person" button. Changing it to a pencil makes it clear it is an editing operation, not a removing operation. I don't know how to write Dart code, so I cannot help with the Mobile app. --- web/src/lib/components/faces-page/person-side-panel.svelte | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/web/src/lib/components/faces-page/person-side-panel.svelte b/web/src/lib/components/faces-page/person-side-panel.svelte index 23682f65f4..13f356dfc0 100644 --- a/web/src/lib/components/faces-page/person-side-panel.svelte +++ b/web/src/lib/components/faces-page/person-side-panel.svelte @@ -15,9 +15,8 @@ type AssetFaceResponseDto, type PersonResponseDto, } from '@immich/sdk'; - import { mdiAccountOff } from '@mdi/js'; import Icon from '$lib/components/elements/icon.svelte'; - import { mdiArrowLeftThin, mdiMinus, mdiRestart } from '@mdi/js'; + import { mdiAccountOff, mdiArrowLeftThin, mdiPencil, mdiRestart } from '@mdi/js'; import { onMount } from 'svelte'; import { linear } from 'svelte/easing'; import { fly } from 'svelte/transition'; @@ -297,7 +296,7 @@ {:else} Date: Thu, 17 Oct 2024 13:17:32 -0400 Subject: [PATCH 09/72] refactor(server): auth enums (#13552) --- server/src/config.ts | 2 +- .../src/controllers/asset-media.controller.ts | 4 ++-- server/src/controllers/auth.controller.ts | 3 +-- server/src/controllers/oauth.controller.ts | 3 +-- .../src/controllers/shared-link.controller.ts | 4 ++-- server/src/dtos/auth.dto.ts | 23 +------------------ server/src/enum.ts | 22 ++++++++++++++++++ .../middleware/asset-upload.interceptor.ts | 2 +- server/src/middleware/auth.guard.ts | 4 ++-- server/src/services/auth.service.ts | 5 +--- server/src/utils/misc.ts | 3 +-- server/src/utils/response.ts | 3 ++- 12 files changed, 37 insertions(+), 41 deletions(-) diff --git a/server/src/config.ts b/server/src/config.ts index ce97e0c2d0..b79643c501 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -3,13 +3,13 @@ import { CronExpression } from '@nestjs/schedule'; import { Request, Response } from 'express'; import Joi, { Root } from 'joi'; import { CLS_ID, ClsModuleOptions } from 'nestjs-cls'; -import { ImmichHeader } from 'src/dtos/auth.dto'; import { AudioCodec, Colorspace, CQMode, ImageFormat, ImmichEnvironment, + ImmichHeader, LogLevel, ToneMapping, TranscodeHWAccel, diff --git a/server/src/controllers/asset-media.controller.ts b/server/src/controllers/asset-media.controller.ts index b2d3933be4..56e793975a 100644 --- a/server/src/controllers/asset-media.controller.ts +++ b/server/src/controllers/asset-media.controller.ts @@ -32,8 +32,8 @@ import { CheckExistingAssetsDto, UploadFieldName, } from 'src/dtos/asset-media.dto'; -import { AuthDto, ImmichHeader } from 'src/dtos/auth.dto'; -import { RouteKey } from 'src/enum'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { ImmichHeader, RouteKey } from 'src/enum'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { AssetUploadInterceptor } from 'src/middleware/asset-upload.interceptor'; import { Auth, Authenticated, FileResponse } from 'src/middleware/auth.guard'; diff --git a/server/src/controllers/auth.controller.ts b/server/src/controllers/auth.controller.ts index 04250f5300..92fa59f6bf 100644 --- a/server/src/controllers/auth.controller.ts +++ b/server/src/controllers/auth.controller.ts @@ -4,7 +4,6 @@ import { Request, Response } from 'express'; import { AuthDto, ChangePasswordDto, - ImmichCookie, LoginCredentialDto, LoginResponseDto, LogoutResponseDto, @@ -12,7 +11,7 @@ import { ValidateAccessTokenResponseDto, } from 'src/dtos/auth.dto'; import { UserAdminResponseDto } from 'src/dtos/user.dto'; -import { AuthType } from 'src/enum'; +import { AuthType, ImmichCookie } from 'src/enum'; import { Auth, Authenticated, GetLoginDetails } from 'src/middleware/auth.guard'; import { AuthService, LoginDetails } from 'src/services/auth.service'; import { respondWithCookie, respondWithoutCookie } from 'src/utils/response'; diff --git a/server/src/controllers/oauth.controller.ts b/server/src/controllers/oauth.controller.ts index 4e626b10f0..b5b94030f2 100644 --- a/server/src/controllers/oauth.controller.ts +++ b/server/src/controllers/oauth.controller.ts @@ -3,14 +3,13 @@ import { ApiTags } from '@nestjs/swagger'; import { Request, Response } from 'express'; import { AuthDto, - ImmichCookie, LoginResponseDto, OAuthAuthorizeResponseDto, OAuthCallbackDto, OAuthConfigDto, } from 'src/dtos/auth.dto'; import { UserAdminResponseDto } from 'src/dtos/user.dto'; -import { AuthType } from 'src/enum'; +import { AuthType, ImmichCookie } from 'src/enum'; import { Auth, Authenticated, GetLoginDetails } from 'src/middleware/auth.guard'; import { AuthService, LoginDetails } from 'src/services/auth.service'; import { respondWithCookie } from 'src/utils/response'; diff --git a/server/src/controllers/shared-link.controller.ts b/server/src/controllers/shared-link.controller.ts index 065e578ec5..59f81068d8 100644 --- a/server/src/controllers/shared-link.controller.ts +++ b/server/src/controllers/shared-link.controller.ts @@ -3,14 +3,14 @@ import { ApiTags } from '@nestjs/swagger'; import { Request, Response } from 'express'; import { AssetIdsResponseDto } from 'src/dtos/asset-ids.response.dto'; import { AssetIdsDto } from 'src/dtos/asset.dto'; -import { AuthDto, ImmichCookie } from 'src/dtos/auth.dto'; +import { AuthDto } from 'src/dtos/auth.dto'; import { SharedLinkCreateDto, SharedLinkEditDto, SharedLinkPasswordDto, SharedLinkResponseDto, } from 'src/dtos/shared-link.dto'; -import { Permission } from 'src/enum'; +import { ImmichCookie, Permission } from 'src/enum'; import { Auth, Authenticated, GetLoginDetails } from 'src/middleware/auth.guard'; import { LoginDetails } from 'src/services/auth.service'; import { SharedLinkService } from 'src/services/shared-link.service'; diff --git a/server/src/dtos/auth.dto.ts b/server/src/dtos/auth.dto.ts index f2d5bd2324..b2bf1b8bcc 100644 --- a/server/src/dtos/auth.dto.ts +++ b/server/src/dtos/auth.dto.ts @@ -5,30 +5,9 @@ import { APIKeyEntity } from 'src/entities/api-key.entity'; import { SessionEntity } from 'src/entities/session.entity'; import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { UserEntity } from 'src/entities/user.entity'; +import { ImmichCookie } from 'src/enum'; import { toEmail } from 'src/validation'; -export enum ImmichCookie { - ACCESS_TOKEN = 'immich_access_token', - AUTH_TYPE = 'immich_auth_type', - IS_AUTHENTICATED = 'immich_is_authenticated', - SHARED_LINK_TOKEN = 'immich_shared_link_token', -} - -export enum ImmichHeader { - API_KEY = 'x-api-key', - USER_TOKEN = 'x-immich-user-token', - SESSION_TOKEN = 'x-immich-session-token', - SHARED_LINK_KEY = 'x-immich-share-key', - CHECKSUM = 'x-immich-checksum', - CID = 'x-immich-cid', -} - -export enum ImmichQuery { - SHARED_LINK_KEY = 'key', - API_KEY = 'apiKey', - SESSION_KEY = 'sessionKey', -} - export type CookieResponse = { isSecure: boolean; values: Array<{ key: ImmichCookie; value: string }>; diff --git a/server/src/enum.ts b/server/src/enum.ts index 109e9a90b7..157fa0003d 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -3,6 +3,28 @@ export enum AuthType { OAUTH = 'oauth', } +export enum ImmichCookie { + ACCESS_TOKEN = 'immich_access_token', + AUTH_TYPE = 'immich_auth_type', + IS_AUTHENTICATED = 'immich_is_authenticated', + SHARED_LINK_TOKEN = 'immich_shared_link_token', +} + +export enum ImmichHeader { + API_KEY = 'x-api-key', + USER_TOKEN = 'x-immich-user-token', + SESSION_TOKEN = 'x-immich-session-token', + SHARED_LINK_KEY = 'x-immich-share-key', + CHECKSUM = 'x-immich-checksum', + CID = 'x-immich-cid', +} + +export enum ImmichQuery { + SHARED_LINK_KEY = 'key', + API_KEY = 'apiKey', + SESSION_KEY = 'sessionKey', +} + export enum AssetType { IMAGE = 'IMAGE', VIDEO = 'VIDEO', diff --git a/server/src/middleware/asset-upload.interceptor.ts b/server/src/middleware/asset-upload.interceptor.ts index 0f38c34259..bc403ee562 100644 --- a/server/src/middleware/asset-upload.interceptor.ts +++ b/server/src/middleware/asset-upload.interceptor.ts @@ -2,7 +2,7 @@ import { CallHandler, ExecutionContext, Injectable, NestInterceptor } from '@nes import { Response } from 'express'; import { of } from 'rxjs'; import { AssetMediaResponseDto, AssetMediaStatus } from 'src/dtos/asset-media-response.dto'; -import { ImmichHeader } from 'src/dtos/auth.dto'; +import { ImmichHeader } from 'src/enum'; import { AuthenticatedRequest } from 'src/middleware/auth.guard'; import { AssetMediaService } from 'src/services/asset-media.service'; import { fromMaybeArray } from 'src/utils/request'; diff --git a/server/src/middleware/auth.guard.ts b/server/src/middleware/auth.guard.ts index 7bc4f41b21..2eaf411475 100644 --- a/server/src/middleware/auth.guard.ts +++ b/server/src/middleware/auth.guard.ts @@ -10,8 +10,8 @@ import { import { Reflector } from '@nestjs/core'; import { ApiBearerAuth, ApiCookieAuth, ApiOkResponse, ApiQuery, ApiSecurity } from '@nestjs/swagger'; import { Request } from 'express'; -import { AuthDto, ImmichQuery } from 'src/dtos/auth.dto'; -import { MetadataKey, Permission } from 'src/enum'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { ImmichQuery, MetadataKey, Permission } from 'src/enum'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { AuthService, LoginDetails } from 'src/services/auth.service'; import { UAParser } from 'ua-parser-js'; diff --git a/server/src/services/auth.service.ts b/server/src/services/auth.service.ts index 8a86ad16d1..00324c909c 100644 --- a/server/src/services/auth.service.ts +++ b/server/src/services/auth.service.ts @@ -8,9 +8,6 @@ import { OnEvent } from 'src/decorators'; import { AuthDto, ChangePasswordDto, - ImmichCookie, - ImmichHeader, - ImmichQuery, LoginCredentialDto, LogoutResponseDto, OAuthAuthorizeResponseDto, @@ -21,7 +18,7 @@ import { } from 'src/dtos/auth.dto'; import { UserAdminResponseDto, mapUserAdmin } from 'src/dtos/user.dto'; import { UserEntity } from 'src/entities/user.entity'; -import { AuthType, Permission } from 'src/enum'; +import { AuthType, ImmichCookie, ImmichHeader, ImmichQuery, Permission } from 'src/enum'; import { OAuthProfile } from 'src/interfaces/oauth.interface'; import { BaseService } from 'src/services/base.service'; import { isGranted } from 'src/utils/access'; diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index e191170e0d..6e435e68a8 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -12,8 +12,7 @@ import { writeFileSync } from 'node:fs'; import path from 'node:path'; import { SystemConfig } from 'src/config'; import { CLIP_MODEL_INFO, serverVersion } from 'src/constants'; -import { ImmichCookie, ImmichHeader } from 'src/dtos/auth.dto'; -import { MetadataKey } from 'src/enum'; +import { ImmichCookie, ImmichHeader, MetadataKey } from 'src/enum'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; export const getExternalDomain = (server: SystemConfig['server'], port: number) => diff --git a/server/src/utils/response.ts b/server/src/utils/response.ts index f318ca3300..679d947afb 100644 --- a/server/src/utils/response.ts +++ b/server/src/utils/response.ts @@ -1,6 +1,7 @@ import { CookieOptions, Response } from 'express'; import { Duration } from 'luxon'; -import { CookieResponse, ImmichCookie } from 'src/dtos/auth.dto'; +import { CookieResponse } from 'src/dtos/auth.dto'; +import { ImmichCookie } from 'src/enum'; export const respondWithCookie = (res: Response, body: T, { isSecure, values }: CookieResponse) => { const defaults: CookieOptions = { From 7011231c4cc605fa07eb8f0710df8cfced6968c5 Mon Sep 17 00:00:00 2001 From: grgergo Date: Thu, 17 Oct 2024 18:07:01 +0000 Subject: [PATCH 10/72] fix(web): improve photosphere viewer settings (#13468) change photosphere viewer settings --- .../asset-viewer/photo-sphere-viewer-adapter.svelte | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte b/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte index da8febc3d9..53a78a346c 100644 --- a/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte +++ b/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte @@ -25,11 +25,12 @@ plugins, container, panorama, - touchmoveTwoFingers: true, + touchmoveTwoFingers: false, mousewheelCtrlKey: false, navbar, - maxFov: 180, - fisheye: true, + minFov: 10, + maxFov: 120, + fisheye: false, }); if (originalImageUrl && !$alwaysLoadOriginalFile) { From 274381deaa03793b3aadf3107e19abaf5d734efc Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Thu, 17 Oct 2024 20:23:51 +0200 Subject: [PATCH 11/72] fix: person update state inconsistencies (#13556) --- .../[personId]/[[photos=photos]]/[[assetId=id]]/+page.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/routes/(user)/people/[personId]/[[photos=photos]]/[[assetId=id]]/+page.svelte b/web/src/routes/(user)/people/[personId]/[[photos=photos]]/[[assetId=id]]/+page.svelte index 037feaf35f..d68367d106 100644 --- a/web/src/routes/(user)/people/[personId]/[[photos=photos]]/[[assetId=id]]/+page.svelte +++ b/web/src/routes/(user)/people/[personId]/[[photos=photos]]/[[assetId=id]]/+page.svelte @@ -201,7 +201,7 @@ return; } try { - await updatePerson({ id: person.id, personUpdateDto: { featureFaceAssetId: asset.id } }); + person = await updatePerson({ id: person.id, personUpdateDto: { featureFaceAssetId: asset.id } }); notificationController.show({ message: $t('feature_photo_updated'), type: NotificationType.Info }); } catch (error) { handleError(error, $t('errors.unable_to_set_feature_photo')); @@ -252,7 +252,7 @@ try { isEditingName = false; - await updatePerson({ id: person.id, personUpdateDto: { name: personName } }); + person = await updatePerson({ id: person.id, personUpdateDto: { name: personName } }); notificationController.show({ message: $t('change_name_successfully'), From 57704ba5a70428fac847525c8f16763812087f66 Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Thu, 17 Oct 2024 21:17:38 +0200 Subject: [PATCH 12/72] fix: gh container registry rate limits (#13554) --- .github/workflows/test.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 064e3c2761..84cc8914dc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -255,6 +255,12 @@ jobs: with: submodules: 'recursive' + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_READ_USERNAME }} + password: ${{ secrets.DOCKERHUB_READ_TOKEN }} + - name: Production build if: ${{ !cancelled() }} run: docker compose -f e2e/docker-compose.yml build @@ -278,6 +284,12 @@ jobs: with: submodules: 'recursive' + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_READ_USERNAME }} + password: ${{ secrets.DOCKERHUB_READ_TOKEN }} + - name: Setup Node uses: actions/setup-node@v4 with: @@ -320,6 +332,12 @@ jobs: with: submodules: 'recursive' + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_READ_USERNAME }} + password: ${{ secrets.DOCKERHUB_READ_TOKEN }} + - name: Setup Node uses: actions/setup-node@v4 with: From 6b2f23b5a3a0a1a4744fc96201af2642c3b69a44 Mon Sep 17 00:00:00 2001 From: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> Date: Thu, 17 Oct 2024 21:53:12 +0200 Subject: [PATCH 13/72] fix: web search add to album reactivity (#13539) --- .../components/photos-page/actions/add-to-album.svelte | 10 +++++++++- web/src/lib/utils/actions.ts | 1 + .../[[photos=photos]]/[[assetId=id]]/+page.svelte | 9 +++++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/web/src/lib/components/photos-page/actions/add-to-album.svelte b/web/src/lib/components/photos-page/actions/add-to-album.svelte index d3998510cd..8c46764408 100644 --- a/web/src/lib/components/photos-page/actions/add-to-album.svelte +++ b/web/src/lib/components/photos-page/actions/add-to-album.svelte @@ -6,8 +6,10 @@ import { getAssetControlContext } from '../asset-select-control-bar.svelte'; import { mdiImageAlbum, mdiShareVariantOutline } from '@mdi/js'; import { t } from 'svelte-i18n'; + import type { OnAddToAlbum } from '$lib/utils/actions'; export let shared = false; + export let onAddToAlbum: OnAddToAlbum = () => {}; let showAlbumPicker = false; @@ -21,13 +23,19 @@ showAlbumPicker = false; const assetIds = [...getAssets()].map((asset) => asset.id); - await addAssetsToNewAlbum(albumName, assetIds); + const album = await addAssetsToNewAlbum(albumName, assetIds); + if (!album) { + return; + } + + onAddToAlbum(assetIds, album.id); }; const handleAddToAlbum = async (album: AlbumResponseDto) => { showAlbumPicker = false; const assetIds = [...getAssets()].map((asset) => asset.id); await addAssetsToAlbum(album.id, assetIds); + onAddToAlbum(assetIds, album.id); }; diff --git a/web/src/lib/utils/actions.ts b/web/src/lib/utils/actions.ts index 291d3926ee..d4715db729 100644 --- a/web/src/lib/utils/actions.ts +++ b/web/src/lib/utils/actions.ts @@ -8,6 +8,7 @@ export type OnDelete = (assetIds: string[]) => void; export type OnRestore = (ids: string[]) => void; export type OnLink = (assets: { still: AssetResponseDto; motion: AssetResponseDto }) => void; export type OnUnlink = (assets: { still: AssetResponseDto; motion: AssetResponseDto }) => void; +export type OnAddToAlbum = (ids: string[], albumId: string) => void; export type OnArchive = (ids: string[], isArchived: boolean) => void; export type OnFavorite = (ids: string[], favorite: boolean) => void; export type OnStack = (ids: string[]) => void; diff --git a/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte b/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte index 9c6a8f9e75..eb0c493204 100644 --- a/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte +++ b/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte @@ -216,6 +216,11 @@ const triggerAssetUpdate = () => (searchResultAssets = searchResultAssets); + const onAddToAlbum = (assetIds: string[]) => { + const assetIdSet = new Set(assetIds); + searchResultAssets = searchResultAssets.filter((a: AssetResponseDto) => !assetIdSet.has(a.id)); + }; + function getObjectKeys(obj: T): (keyof T)[] { return Object.keys(obj) as (keyof T)[]; } @@ -230,8 +235,8 @@ - - + + From 0eabb3ef80bf2b855682f6616a27b958bfc532ce Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:54:40 -0400 Subject: [PATCH 14/72] fix(deps): update dependency sirv to v3 (#13511) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- server/package-lock.json | 16 ++++++++-------- server/package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/package-lock.json b/server/package-lock.json index ac1f2dbe23..7830c22a3b 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -59,7 +59,7 @@ "sanitize-filename": "^1.6.3", "semver": "^7.6.2", "sharp": "^0.33.0", - "sirv": "^2.0.4", + "sirv": "^3.0.0", "tailwindcss-preset-email": "^1.3.2", "thumbhash": "^0.1.1", "typeorm": "^0.3.17", @@ -13341,16 +13341,16 @@ "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, "node_modules/sirv": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/sirv/-/sirv-2.0.4.tgz", - "integrity": "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.0.tgz", + "integrity": "sha512-BPwJGUeDaDCHihkORDchNyyTvWFhcusy1XMmhEVTQTwGeybFbp8YEmB+njbPnth1FibULBSBVwCQni25XlCUDg==", "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", "totalist": "^3.0.0" }, "engines": { - "node": ">= 10" + "node": ">=18" } }, "node_modules/slice-source": { @@ -24493,9 +24493,9 @@ } }, "sirv": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/sirv/-/sirv-2.0.4.tgz", - "integrity": "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.0.tgz", + "integrity": "sha512-BPwJGUeDaDCHihkORDchNyyTvWFhcusy1XMmhEVTQTwGeybFbp8YEmB+njbPnth1FibULBSBVwCQni25XlCUDg==", "requires": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", diff --git a/server/package.json b/server/package.json index 738b43aa54..1ef34647bc 100644 --- a/server/package.json +++ b/server/package.json @@ -84,7 +84,7 @@ "sanitize-filename": "^1.6.3", "semver": "^7.6.2", "sharp": "^0.33.0", - "sirv": "^2.0.4", + "sirv": "^3.0.0", "tailwindcss-preset-email": "^1.3.2", "thumbhash": "^0.1.1", "typeorm": "^0.3.17", From c8f672f494349f8942068e1c0242878a1cf3d374 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 17 Oct 2024 17:00:22 -0400 Subject: [PATCH 15/72] refactor(server): telemetry ports env (#13560) --- server/src/interfaces/config.interface.ts | 15 +++++-- .../repositories/config.repository.spec.ts | 40 +++++++++++++++++++ server/src/repositories/config.repository.ts | 12 ++++++ server/src/workers/api.ts | 16 ++------ server/src/workers/microservices.ts | 6 +-- .../repositories/config.repository.mock.ts | 9 +++++ 6 files changed, 80 insertions(+), 18 deletions(-) diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.ts index 10e9a86aef..9870b86d10 100644 --- a/server/src/interfaces/config.interface.ts +++ b/server/src/interfaces/config.interface.ts @@ -29,6 +29,11 @@ export interface EnvData { thirdPartySupportUrl?: string; }; + bull: { + config: QueueOptions; + queues: RegisterQueueOptions[]; + }; + database: { url?: string; host: string; @@ -45,6 +50,10 @@ export interface EnvData { server: string; }; + network: { + trustedProxies: string[]; + }; + resourcePaths: { lockFile: string; geodata: { @@ -62,9 +71,9 @@ export interface EnvData { redis: RedisOptions; - bull: { - config: QueueOptions; - queues: RegisterQueueOptions[]; + telemetry: { + apiPort: number; + microservicesPort: number; }; storage: { diff --git a/server/src/repositories/config.repository.spec.ts b/server/src/repositories/config.repository.spec.ts index 78b512b2fd..36b7b48062 100644 --- a/server/src/repositories/config.repository.spec.ts +++ b/server/src/repositories/config.repository.spec.ts @@ -9,6 +9,9 @@ const resetEnv = () => { for (const env of [ 'IMMICH_WORKERS_INCLUDE', 'IMMICH_WORKERS_EXCLUDE', + 'IMMICH_TRUSTED_PROXIES', + 'IMMICH_API_METRICS_PORT', + 'IMMICH_MICROSERVICES_METRICS_PORT', 'DB_URL', 'DB_HOSTNAME', @@ -178,4 +181,41 @@ describe('getEnv', () => { expect(getEnv).toThrowError('Invalid worker(s) found: api,microservices,randomservice'); }); }); + + describe('network', () => { + it('should return default network options', () => { + const { network } = getEnv(); + expect(network).toEqual({ + trustedProxies: [], + }); + }); + + it('should parse trusted proxies', () => { + process.env.IMMICH_TRUSTED_PROXIES = '10.1.0.0,10.2.0.0, 169.254.0.0/16'; + const { network } = getEnv(); + expect(network).toEqual({ + trustedProxies: ['10.1.0.0', '10.2.0.0', '169.254.0.0/16'], + }); + }); + }); + + describe('telemetry', () => { + it('should return default ports', () => { + const { telemetry } = getEnv(); + expect(telemetry).toEqual({ + apiPort: 8081, + microservicesPort: 8082, + }); + }); + + it('should parse custom ports', () => { + process.env.IMMICH_API_METRICS_PORT = '2001'; + process.env.IMMICH_MICROSERVICES_METRICS_PORT = '2002'; + const { telemetry } = getEnv(); + expect(telemetry).toEqual({ + apiPort: 2001, + microservicesPort: 2002, + }); + }); + }); }); diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index 585e719e3a..8b511fba5a 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -117,6 +117,13 @@ const getEnv = (): EnvData => { licensePublicKey: isProd ? productionKeys : stagingKeys, + network: { + trustedProxies: (process.env.IMMICH_TRUSTED_PROXIES ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean), + }, + redis: redisConfig, resourcePaths: { @@ -138,6 +145,11 @@ const getEnv = (): EnvData => { ignoreMountCheckErrors: process.env.IMMICH_IGNORE_MOUNT_CHECK_ERRORS === 'true', }, + telemetry: { + apiPort: Number(process.env.IMMICH_API_METRICS_PORT || '') || 8081, + microservicesPort: Number(process.env.IMMICH_MICROSERVICES_METRICS_PORT || '') || 8082, + }, + workers, noColor: !!process.env.NO_COLOR, diff --git a/server/src/workers/api.ts b/server/src/workers/api.ts index 4a7755fa1e..b369b56953 100644 --- a/server/src/workers/api.ts +++ b/server/src/workers/api.ts @@ -10,25 +10,17 @@ import { ImmichEnvironment } from 'src/enum'; import { IConfigRepository } from 'src/interfaces/config.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { WebSocketAdapter } from 'src/middleware/websocket.adapter'; +import { ConfigRepository } from 'src/repositories/config.repository'; import { ApiService } from 'src/services/api.service'; import { isStartUpError } from 'src/services/storage.service'; import { otelStart } from 'src/utils/instrumentation'; import { useSwagger } from 'src/utils/misc'; -function parseTrustedProxy(input?: string) { - if (!input) { - return []; - } - // Split on ',' char to allow multiple IPs - return input.split(','); -} - async function bootstrap() { process.title = 'immich-api'; - const otelPort = Number.parseInt(process.env.IMMICH_API_METRICS_PORT ?? '8081'); - const trustedProxies = parseTrustedProxy(process.env.IMMICH_TRUSTED_PROXIES ?? ''); - otelStart(otelPort); + const { telemetry, network } = new ConfigRepository().getEnv(); + otelStart(telemetry.apiPort); const app = await NestFactory.create(ApiModule, { bufferLogs: true }); const logger = await app.resolve(ILoggerRepository); @@ -39,7 +31,7 @@ async function bootstrap() { logger.setContext('Bootstrap'); app.useLogger(logger); - app.set('trust proxy', ['loopback', 'linklocal', 'uniquelocal', ...trustedProxies]); + app.set('trust proxy', ['loopback', 'linklocal', 'uniquelocal', ...network.trustedProxies]); app.set('etag', 'strong'); app.use(cookieParser()); app.use(json({ limit: '10mb' })); diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.ts index 3cb478057c..7b60fb8db6 100644 --- a/server/src/workers/microservices.ts +++ b/server/src/workers/microservices.ts @@ -5,13 +5,13 @@ import { serverVersion } from 'src/constants'; import { IConfigRepository } from 'src/interfaces/config.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { WebSocketAdapter } from 'src/middleware/websocket.adapter'; +import { ConfigRepository } from 'src/repositories/config.repository'; import { isStartUpError } from 'src/services/storage.service'; import { otelStart } from 'src/utils/instrumentation'; export async function bootstrap() { - const otelPort = Number.parseInt(process.env.IMMICH_MICROSERVICES_METRICS_PORT ?? '8082'); - - otelStart(otelPort); + const { telemetry } = new ConfigRepository().getEnv(); + otelStart(telemetry.microservicesPort); const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true }); const logger = await app.resolve(ILoggerRepository); diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts index 65e419fe36..c7da917cc4 100644 --- a/server/test/repositories/config.repository.mock.ts +++ b/server/test/repositories/config.repository.mock.ts @@ -31,6 +31,10 @@ const envData: EnvData = { server: 'server-public-key', }, + network: { + trustedProxies: [], + }, + redis: { host: 'redis', port: 6379, @@ -56,6 +60,11 @@ const envData: EnvData = { ignoreMountCheckErrors: false, }, + telemetry: { + apiPort: 8081, + microservicesPort: 8082, + }, + workers: [ImmichWorker.API, ImmichWorker.MICROSERVICES], noColor: false, From 23646f0d551d41ef078445180cbd83ab554197ea Mon Sep 17 00:00:00 2001 From: Mert <101130780+mertalev@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:02:41 -0400 Subject: [PATCH 16/72] feat(server): vaapi hardware decoding (#13561) * add hw decoding for vaapi * add tests * update docs --- docs/docs/features/hardware-transcoding.md | 4 +- server/src/services/media.service.spec.ts | 77 ++++++++++++++++++++- server/src/utils/media.ts | 80 ++++++++++++++++++++-- web/src/lib/i18n/en.json | 2 +- 4 files changed, 154 insertions(+), 9 deletions(-) diff --git a/docs/docs/features/hardware-transcoding.md b/docs/docs/features/hardware-transcoding.md index 7f74140ac0..756bb6823c 100644 --- a/docs/docs/features/hardware-transcoding.md +++ b/docs/docs/features/hardware-transcoding.md @@ -23,7 +23,7 @@ You do not need to redo any transcoding jobs after enabling hardware acceleratio - Raspberry Pi is currently not supported. - Two-pass mode is only supported for NVENC. Other APIs will ignore this setting. - By default, only encoding is currently hardware accelerated. This means the CPU is still used for software decoding and tone-mapping. - - NVENC and RKMPP can be fully accelerated by enabling hardware decoding in the video transcoding settings. + - You can benefit from end-to-end acceleration by enabling hardware decoding in the video transcoding settings. - Hardware dependent - Codec support varies, but H.264 and HEVC are usually supported. - Notably, NVIDIA and AMD GPUs do not support VP9 encoding. @@ -66,7 +66,7 @@ For RKMPP to work: 3. Redeploy the `immich-server` container with these updated settings. 4. In the Admin page under `Video transcoding settings`, change the hardware acceleration setting to the appropriate option and save. -5. (Optional) If using a compatible backend, you may enable hardware decoding for optimal performance. +5. (Optional) Enable hardware decoding for optimal performance. #### Single Compose File diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index 703794e8b7..0489169c1a 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -1619,7 +1619,7 @@ describe(MediaService.name, () => { '-refs 5', '-g 256', '-v verbose', - '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720', + '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq', '-preset 7', '-global_quality:v 23', '-maxrate 10000k', @@ -1803,7 +1803,7 @@ describe(MediaService.name, () => { '-strict unofficial', '-g 256', '-v verbose', - '-vf format=nv12,hwupload,scale_vaapi=-2:720', + '-vf format=nv12,hwupload,scale_vaapi=-2:720:mode=hq:out_range=pc', '-compression_level 7', '-rc_mode 1', ]), @@ -1946,6 +1946,79 @@ describe(MediaService.name, () => { ); }); + it('should use hardware decoding for vaapi if enabled', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + systemMock.get.mockResolvedValue({ + ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true }, + }); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + + await sut.handleVideoConversion({ id: assetStub.video.id }); + + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + expect.objectContaining({ + inputOptions: expect.arrayContaining([ + '-hwaccel vaapi', + '-hwaccel_output_format vaapi', + '-noautorotate', + '-threads 1', + ]), + outputOptions: expect.arrayContaining([ + expect.stringContaining('scale_vaapi=-2:720:mode=hq:out_range=pc:format=nv12'), + ]), + twoPass: false, + }), + ); + }); + + it('should use hardware tone-mapping for qsv if hardware decoding is enabled and should tone map', async () => { + storageMock.readdir.mockResolvedValue(['renderD128']); + mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR); + systemMock.get.mockResolvedValue({ + ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true }, + }); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + + await sut.handleVideoConversion({ id: assetStub.video.id }); + + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + expect.objectContaining({ + inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_output_format vaapi', '-threads 1']), + outputOptions: expect.arrayContaining([ + expect.stringContaining( + 'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709,hwmap=derive_device=vaapi:reverse=1,format=vaapi', + ), + ]), + twoPass: false, + }), + ); + }); + + it('should use preferred device for vaapi when hardware decoding', async () => { + storageMock.readdir.mockResolvedValue(['renderD128', 'renderD129', 'renderD130']); + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); + systemMock.get.mockResolvedValue({ + ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true, preferredHwDevice: 'renderD129' }, + }); + assetMock.getByIds.mockResolvedValue([assetStub.video]); + + await sut.handleVideoConversion({ id: assetStub.video.id }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/as/se/asset-id.mp4', + expect.objectContaining({ + inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_device /dev/dri/renderD129']), + outputOptions: expect.any(Array), + twoPass: false, + }), + ); + }); + it('should fallback to sw transcoding if hw transcoding fails', async () => { storageMock.readdir.mockResolvedValue(['renderD128']); mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index e58ca2f470..55f92d109a 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -52,7 +52,9 @@ export class BaseConfig implements VideoCodecSWConfig { break; } case TranscodeHWAccel.VAAPI: { - handler = new VAAPIConfig(config, devices); + handler = config.accelDecode + ? new VaapiHwDecodeConfig(config, devices) + : new VaapiSwDecodeConfig(config, devices); break; } case TranscodeHWAccel.RKMPP: { @@ -688,7 +690,7 @@ export class QsvSwDecodeConfig extends BaseHWConfig { const options = this.getToneMapping(videoStream); options.push('format=nv12', 'hwupload=extra_hw_frames=64'); if (this.shouldScale(videoStream)) { - options.push(`scale_qsv=${this.getScaling(videoStream)}`); + options.push(`scale_qsv=${this.getScaling(videoStream)}:mode=hq`); } return options; } @@ -811,7 +813,7 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig { } } -export class VAAPIConfig extends BaseHWConfig { +export class VaapiSwDecodeConfig extends BaseHWConfig { getBaseInputOptions() { if (this.devices.length === 0) { throw new Error('No VAAPI device found'); @@ -829,7 +831,7 @@ export class VAAPIConfig extends BaseHWConfig { const options = this.getToneMapping(videoStream); options.push('format=nv12', 'hwupload'); if (this.shouldScale(videoStream)) { - options.push(`scale_vaapi=${this.getScaling(videoStream)}`); + options.push(`scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`); } return options; @@ -878,6 +880,76 @@ export class VAAPIConfig extends BaseHWConfig { } } +export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig { + getBaseInputOptions() { + if (this.devices.length === 0) { + throw new Error('No VAAPI device found'); + } + + const options = [ + '-hwaccel vaapi', + '-hwaccel_output_format vaapi', + '-noautorotate', + ...this.getInputThreadOptions(), + ]; + const hwDevice = this.getPreferredHardwareDevice(); + if (hwDevice) { + options.push(`-hwaccel_device ${hwDevice}`); + } + + return options; + } + + getFilterOptions(videoStream: VideoStreamInfo) { + const options = []; + if (this.shouldScale(videoStream) || !this.shouldToneMap(videoStream)) { + let scaling = `scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`; + if (!this.shouldToneMap(videoStream)) { + scaling += ':format=nv12'; + } + options.push(scaling); + } + + options.push(...this.getToneMapping(videoStream)); + return options; + } + + getToneMapping(videoStream: VideoStreamInfo): string[] { + if (!this.shouldToneMap(videoStream)) { + return []; + } + + const colors = this.getColors(); + const tonemapOptions = [ + 'desat=0', + 'format=nv12', + `matrix=${colors.matrix}`, + `primaries=${colors.primaries}`, + 'range=pc', + `tonemap=${this.config.tonemap}`, + `transfer=${colors.transfer}`, + ]; + + return [ + 'hwmap=derive_device=opencl', + `tonemap_opencl=${tonemapOptions.join(':')}`, + 'hwmap=derive_device=vaapi:reverse=1,format=vaapi', + ]; + } + + getInputThreadOptions() { + return [`-threads 1`]; + } + + getColors() { + return { + primaries: 'bt709', + transfer: 'bt709', + matrix: 'bt709', + }; + } +} + export class RkmppSwDecodeConfig extends BaseHWConfig { constructor( protected config: SystemConfigFFmpegDto, diff --git a/web/src/lib/i18n/en.json b/web/src/lib/i18n/en.json index 7b60fcbd2e..730efe4edf 100644 --- a/web/src/lib/i18n/en.json +++ b/web/src/lib/i18n/en.json @@ -274,7 +274,7 @@ "transcoding_hardware_acceleration": "Hardware Acceleration", "transcoding_hardware_acceleration_description": "Experimental; much faster, but will have lower quality at the same bitrate", "transcoding_hardware_decoding": "Hardware decoding", - "transcoding_hardware_decoding_setting_description": "Applies only to NVENC, QSV and RKMPP. Enables end-to-end acceleration instead of only accelerating encoding. May not work on all videos.", + "transcoding_hardware_decoding_setting_description": "Enables end-to-end acceleration instead of only accelerating encoding. May not work on all videos.", "transcoding_hevc_codec": "HEVC codec", "transcoding_max_b_frames": "Maximum B-frames", "transcoding_max_b_frames_description": "Higher values improve compression efficiency, but slow down encoding. May not be compatible with hardware acceleration on older devices. 0 disables B-frames, while -1 sets this value automatically.", From 12628b80bce82cb5f8b5f64de5bc08442ef266c3 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 17 Oct 2024 18:04:25 -0400 Subject: [PATCH 17/72] refactor(server): telemetry env (#13564) --- server/src/app.module.ts | 5 +-- server/src/bin/sync-sql.ts | 6 ++- server/src/interfaces/config.interface.ts | 8 ++++ .../repositories/config.repository.spec.ts | 39 ++++++++++++++++++- server/src/repositories/config.repository.ts | 29 +++++++++++++- server/src/repositories/metric.repository.ts | 16 ++++---- server/src/utils/instrumentation.ts | 33 +++------------- .../repositories/config.repository.mock.ts | 15 +++++++ 8 files changed, 108 insertions(+), 43 deletions(-) diff --git a/server/src/app.module.ts b/server/src/app.module.ts index 43aefbd0f0..fd921150fd 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -23,7 +23,6 @@ import { repositories } from 'src/repositories'; import { ConfigRepository } from 'src/repositories/config.repository'; import { services } from 'src/services'; import { DatabaseService } from 'src/services/database.service'; -import { otelConfig } from 'src/utils/instrumentation'; const common = [...services, ...repositories]; @@ -37,14 +36,14 @@ const middleware = [ ]; const configRepository = new ConfigRepository(); -const { bull } = configRepository.getEnv(); +const { bull, otel } = configRepository.getEnv(); const imports = [ BullModule.forRoot(bull.config), BullModule.registerQueue(...bull.queues), ClsModule.forRoot(clsConfig), ConfigModule.forRoot(immichAppConfig), - OpenTelemetryModule.forRoot(otelConfig), + OpenTelemetryModule.forRoot(otel), TypeOrmModule.forRootAsync({ inject: [ModuleRef], useFactory: (moduleRef: ModuleRef) => { diff --git a/server/src/bin/sync-sql.ts b/server/src/bin/sync-sql.ts index 92c3cc1103..e4f11cc692 100644 --- a/server/src/bin/sync-sql.ts +++ b/server/src/bin/sync-sql.ts @@ -14,8 +14,8 @@ import { entities } from 'src/entities'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { repositories } from 'src/repositories'; import { AccessRepository } from 'src/repositories/access.repository'; +import { ConfigRepository } from 'src/repositories/config.repository'; import { AuthService } from 'src/services/auth.service'; -import { otelConfig } from 'src/utils/instrumentation'; import { Logger } from 'typeorm'; export class SqlLogger implements Logger { @@ -74,6 +74,8 @@ class SqlGenerator { await rm(this.options.targetDir, { force: true, recursive: true }); await mkdir(this.options.targetDir); + const { otel } = new ConfigRepository().getEnv(); + const moduleFixture = await Test.createTestingModule({ imports: [ TypeOrmModule.forRoot({ @@ -84,7 +86,7 @@ class SqlGenerator { logger: this.sqlLogger, }), TypeOrmModule.forFeature(entities), - OpenTelemetryModule.forRoot(otelConfig), + OpenTelemetryModule.forRoot(otel), ], providers: [...repositories, AuthService, SchedulerRegistry], }).compile(); diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.ts index 9870b86d10..4391909df7 100644 --- a/server/src/interfaces/config.interface.ts +++ b/server/src/interfaces/config.interface.ts @@ -1,6 +1,7 @@ import { RegisterQueueOptions } from '@nestjs/bullmq'; import { QueueOptions } from 'bullmq'; import { RedisOptions } from 'ioredis'; +import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces'; import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum'; import { VectorExtension } from 'src/interfaces/database.interface'; @@ -54,6 +55,8 @@ export interface EnvData { trustedProxies: string[]; }; + otel: OpenTelemetryModuleOptions; + resourcePaths: { lockFile: string; geodata: { @@ -74,6 +77,11 @@ export interface EnvData { telemetry: { apiPort: number; microservicesPort: number; + enabled: boolean; + apiMetrics: boolean; + hostMetrics: boolean; + repoMetrics: boolean; + jobMetrics: boolean; }; storage: { diff --git a/server/src/repositories/config.repository.spec.ts b/server/src/repositories/config.repository.spec.ts index 36b7b48062..84da211182 100644 --- a/server/src/repositories/config.repository.spec.ts +++ b/server/src/repositories/config.repository.spec.ts @@ -12,6 +12,11 @@ const resetEnv = () => { 'IMMICH_TRUSTED_PROXIES', 'IMMICH_API_METRICS_PORT', 'IMMICH_MICROSERVICES_METRICS_PORT', + 'IMMICH_METRICS', + 'IMMICH_API_METRICS', + 'IMMICH_HOST_METRICS', + 'IMMICH_IO_METRICS', + 'IMMICH_JOB_METRICS', 'DB_URL', 'DB_HOSTNAME', @@ -200,11 +205,16 @@ describe('getEnv', () => { }); describe('telemetry', () => { - it('should return default ports', () => { + it('should have default values', () => { const { telemetry } = getEnv(); expect(telemetry).toEqual({ apiPort: 8081, microservicesPort: 8082, + enabled: false, + apiMetrics: false, + hostMetrics: false, + jobMetrics: false, + repoMetrics: false, }); }); @@ -212,10 +222,35 @@ describe('getEnv', () => { process.env.IMMICH_API_METRICS_PORT = '2001'; process.env.IMMICH_MICROSERVICES_METRICS_PORT = '2002'; const { telemetry } = getEnv(); - expect(telemetry).toEqual({ + expect(telemetry).toMatchObject({ apiPort: 2001, microservicesPort: 2002, }); }); + + it('should run with telemetry enabled', () => { + process.env.IMMICH_METRICS = 'true'; + const { telemetry } = getEnv(); + expect(telemetry).toMatchObject({ + enabled: true, + apiMetrics: true, + hostMetrics: true, + jobMetrics: true, + repoMetrics: true, + }); + }); + + it('should run with telemetry enabled and jobs disabled', () => { + process.env.IMMICH_METRICS = 'true'; + process.env.IMMICH_JOB_METRICS = 'false'; + const { telemetry } = getEnv(); + expect(telemetry).toMatchObject({ + enabled: true, + apiMetrics: true, + hostMetrics: true, + jobMetrics: false, + repoMetrics: true, + }); + }); }); }); diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index 8b511fba5a..44b8c7b605 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; import { join } from 'node:path'; -import { citiesFile } from 'src/constants'; +import { citiesFile, excludePaths } from 'src/constants'; import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum'; import { EnvData, IConfigRepository } from 'src/interfaces/config.interface'; import { DatabaseExtension } from 'src/interfaces/database.interface'; @@ -30,6 +30,8 @@ const asSet = (value: string | undefined, defaults: ImmichWorker[]) => { return new Set(values.length === 0 ? defaults : (values as ImmichWorker[])); }; +const parseBoolean = (value: string | undefined, defaultValue: boolean) => (value ? value === 'true' : defaultValue); + const getEnv = (): EnvData => { const included = asSet(process.env.IMMICH_WORKERS_INCLUDE, [ImmichWorker.API, ImmichWorker.MICROSERVICES]); const excluded = asSet(process.env.IMMICH_WORKERS_EXCLUDE, []); @@ -66,6 +68,16 @@ const getEnv = (): EnvData => { } } + const globalEnabled = parseBoolean(process.env.IMMICH_METRICS, false); + const hostMetrics = parseBoolean(process.env.IMMICH_HOST_METRICS, globalEnabled); + const apiMetrics = parseBoolean(process.env.IMMICH_API_METRICS, globalEnabled); + const repoMetrics = parseBoolean(process.env.IMMICH_IO_METRICS, globalEnabled); + const jobMetrics = parseBoolean(process.env.IMMICH_JOB_METRICS, globalEnabled); + const telemetryEnabled = globalEnabled || hostMetrics || apiMetrics || repoMetrics || jobMetrics; + if (!telemetryEnabled && process.env.OTEL_SDK_DISABLED === undefined) { + process.env.OTEL_SDK_DISABLED = 'true'; + } + return { host: process.env.IMMICH_HOST, port: Number(process.env.IMMICH_PORT) || 2283, @@ -124,6 +136,16 @@ const getEnv = (): EnvData => { .filter(Boolean), }, + otel: { + metrics: { + hostMetrics, + apiMetrics: { + enable: apiMetrics, + ignoreRoutes: excludePaths, + }, + }, + }, + redis: redisConfig, resourcePaths: { @@ -148,6 +170,11 @@ const getEnv = (): EnvData => { telemetry: { apiPort: Number(process.env.IMMICH_API_METRICS_PORT || '') || 8081, microservicesPort: Number(process.env.IMMICH_MICROSERVICES_METRICS_PORT || '') || 8082, + enabled: telemetryEnabled, + hostMetrics, + apiMetrics, + repoMetrics, + jobMetrics, }, workers, diff --git a/server/src/repositories/metric.repository.ts b/server/src/repositories/metric.repository.ts index 5948e92fa6..b59bcf9ed1 100644 --- a/server/src/repositories/metric.repository.ts +++ b/server/src/repositories/metric.repository.ts @@ -1,11 +1,12 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { MetricOptions } from '@opentelemetry/api'; import { MetricService } from 'nestjs-otel'; +import { IConfigRepository } from 'src/interfaces/config.interface'; import { IMetricGroupRepository, IMetricRepository, MetricGroupOptions } from 'src/interfaces/metric.interface'; -import { apiMetrics, hostMetrics, jobMetrics, repoMetrics } from 'src/utils/instrumentation'; class MetricGroupRepository implements IMetricGroupRepository { private enabled = false; + constructor(private metricService: MetricService) {} addToCounter(name: string, value: number, options?: MetricOptions): void { @@ -39,10 +40,11 @@ export class MetricRepository implements IMetricRepository { jobs: MetricGroupRepository; repo: MetricGroupRepository; - constructor(metricService: MetricService) { - this.api = new MetricGroupRepository(metricService).configure({ enabled: apiMetrics }); - this.host = new MetricGroupRepository(metricService).configure({ enabled: hostMetrics }); - this.jobs = new MetricGroupRepository(metricService).configure({ enabled: jobMetrics }); - this.repo = new MetricGroupRepository(metricService).configure({ enabled: repoMetrics }); + constructor(metricService: MetricService, @Inject(IConfigRepository) configRepository: IConfigRepository) { + const { telemetry } = configRepository.getEnv(); + this.api = new MetricGroupRepository(metricService).configure({ enabled: telemetry.apiMetrics }); + this.host = new MetricGroupRepository(metricService).configure({ enabled: telemetry.hostMetrics }); + this.jobs = new MetricGroupRepository(metricService).configure({ enabled: telemetry.jobMetrics }); + this.repo = new MetricGroupRepository(metricService).configure({ enabled: telemetry.repoMetrics }); } } diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.ts index 484ba5901c..bd522f27b2 100644 --- a/server/src/utils/instrumentation.ts +++ b/server/src/utils/instrumentation.ts @@ -7,32 +7,19 @@ import { PgInstrumentation } from '@opentelemetry/instrumentation-pg'; import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node'; import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import { snakeCase, startCase } from 'lodash'; -import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces'; import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils'; import { performance } from 'node:perf_hooks'; -import { excludePaths, serverVersion } from 'src/constants'; +import { serverVersion } from 'src/constants'; import { DecorateAll } from 'src/decorators'; - -let metricsEnabled = process.env.IMMICH_METRICS === 'true'; -export const hostMetrics = - process.env.IMMICH_HOST_METRICS == null ? metricsEnabled : process.env.IMMICH_HOST_METRICS === 'true'; -export const apiMetrics = - process.env.IMMICH_API_METRICS == null ? metricsEnabled : process.env.IMMICH_API_METRICS === 'true'; -export const repoMetrics = - process.env.IMMICH_IO_METRICS == null ? metricsEnabled : process.env.IMMICH_IO_METRICS === 'true'; -export const jobMetrics = - process.env.IMMICH_JOB_METRICS == null ? metricsEnabled : process.env.IMMICH_JOB_METRICS === 'true'; - -metricsEnabled ||= hostMetrics || apiMetrics || repoMetrics || jobMetrics; -if (!metricsEnabled && process.env.OTEL_SDK_DISABLED === undefined) { - process.env.OTEL_SDK_DISABLED = 'true'; -} +import { ConfigRepository } from 'src/repositories/config.repository'; const aggregation = new metrics.ExplicitBucketHistogramAggregation( [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000], true, ); +const { telemetry } = new ConfigRepository().getEnv(); + let otelSingleton: NodeSDK | undefined; export const otelStart = (port: number) => { @@ -64,23 +51,13 @@ export const otelShutdown = async () => { } }; -export const otelConfig: OpenTelemetryModuleOptions = { - metrics: { - hostMetrics, - apiMetrics: { - enable: apiMetrics, - ignoreRoutes: excludePaths, - }, - }, -}; - function ExecutionTimeHistogram({ description, unit = 'ms', valueType = contextBase.ValueType.DOUBLE, }: contextBase.MetricOptions = {}) { return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => { - if (!repoMetrics || process.env.OTEL_SDK_DISABLED) { + if (!telemetry.repoMetrics || process.env.OTEL_SDK_DISABLED) { return; } diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts index c7da917cc4..bb3cfcebb9 100644 --- a/server/test/repositories/config.repository.mock.ts +++ b/server/test/repositories/config.repository.mock.ts @@ -35,6 +35,16 @@ const envData: EnvData = { trustedProxies: [], }, + otel: { + metrics: { + hostMetrics: false, + apiMetrics: { + enable: false, + ignoreRoutes: [], + }, + }, + }, + redis: { host: 'redis', port: 6379, @@ -63,6 +73,11 @@ const envData: EnvData = { telemetry: { apiPort: 8081, microservicesPort: 8082, + enabled: false, + hostMetrics: false, + apiMetrics: false, + jobMetrics: false, + repoMetrics: false, }, workers: [ImmichWorker.API, ImmichWorker.MICROSERVICES], From b1149881bd7ff2f19cde39cf407512a2854196bc Mon Sep 17 00:00:00 2001 From: pyorot Date: Fri, 18 Oct 2024 13:26:16 +0100 Subject: [PATCH 18/72] feat(server): add pcm_s16le accepted audio codec (#13418) --- docs/docs/install/config-file.md | 2 +- mobile/openapi/lib/model/audio_codec.dart | Bin 2639 -> 2762 bytes open-api/immich-openapi-specs.json | 3 ++- open-api/typescript-sdk/src/fetch-client.ts | 3 ++- server/src/config.ts | 2 +- server/src/enum.ts | 1 + .../services/system-config.service.spec.ts | 2 +- .../settings/ffmpeg/ffmpeg-settings.svelte | 2 ++ 8 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/docs/install/config-file.md b/docs/docs/install/config-file.md index b789d8653f..ed902f39cf 100644 --- a/docs/docs/install/config-file.md +++ b/docs/docs/install/config-file.md @@ -19,7 +19,7 @@ The default configuration looks like this: "targetVideoCodec": "h264", "acceptedVideoCodecs": ["h264"], "targetAudioCodec": "aac", - "acceptedAudioCodecs": ["aac", "mp3", "libopus"], + "acceptedAudioCodecs": ["aac", "mp3", "libopus", "pcm_s16le"], "acceptedContainers": ["mov", "ogg", "webm"], "targetResolution": "720", "maxBitrate": "0", diff --git a/mobile/openapi/lib/model/audio_codec.dart b/mobile/openapi/lib/model/audio_codec.dart index ca195f7d06e1ac1c2fe996511a1f8b8fa1640308..ea1e96f36e00a8b2a81ed980ed593a4274e23514 100644 GIT binary patch delta 118 zcmX>va!Pc=3Z}`WO#I3P$+^LXW;v+}whE4=DVh1s`6;Q%dhr@X>Oh(JVvyA2B}`ph n3JNflo3)q~*mRQldR}D13Z}{TnA$cMG0U)RKFZ0;2mob42j>6) diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 415cc663f4..b99da367b8 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -8507,7 +8507,8 @@ "enum": [ "mp3", "aac", - "libopus" + "libopus", + "pcm_s16le" ], "type": "string" }, diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 2077943bf8..17079c07c3 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -3501,7 +3501,8 @@ export enum TranscodeHWAccel { export enum AudioCodec { Mp3 = "mp3", Aac = "aac", - Libopus = "libopus" + Libopus = "libopus", + PcmS16Le = "pcm_s16le" } export enum VideoContainer { Mov = "mov", diff --git a/server/src/config.ts b/server/src/config.ts index b79643c501..e386c134b4 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -163,7 +163,7 @@ export const defaults = Object.freeze({ targetVideoCodec: VideoCodec.H264, acceptedVideoCodecs: [VideoCodec.H264], targetAudioCodec: AudioCodec.AAC, - acceptedAudioCodecs: [AudioCodec.AAC, AudioCodec.MP3, AudioCodec.LIBOPUS], + acceptedAudioCodecs: [AudioCodec.AAC, AudioCodec.MP3, AudioCodec.LIBOPUS, AudioCodec.PCMS16LE], acceptedContainers: [VideoContainer.MOV, VideoContainer.OGG, VideoContainer.WEBM], targetResolution: '720', maxBitrate: '0', diff --git a/server/src/enum.ts b/server/src/enum.ts index 157fa0003d..8c11834dac 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -278,6 +278,7 @@ export enum AudioCodec { MP3 = 'mp3', AAC = 'aac', LIBOPUS = 'libopus', + PCMS16LE = 'pcm_s16le', } export enum VideoContainer { diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index 807d8299b8..f81abc4795 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -49,7 +49,7 @@ const updatedConfig = Object.freeze({ threads: 0, preset: 'ultrafast', targetAudioCodec: AudioCodec.AAC, - acceptedAudioCodecs: [AudioCodec.AAC, AudioCodec.MP3, AudioCodec.LIBOPUS], + acceptedAudioCodecs: [AudioCodec.AAC, AudioCodec.MP3, AudioCodec.LIBOPUS, AudioCodec.PCMS16LE], targetResolution: '720', targetVideoCodec: VideoCodec.H264, acceptedVideoCodecs: [VideoCodec.H264], diff --git a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte index c048a22207..42cc004c52 100644 --- a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte +++ b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte @@ -102,6 +102,7 @@ onSelect={() => (config.ffmpeg.acceptedVideoCodecs = [config.ffmpeg.targetVideoCodec])} /> + From 3fb5adb31c10a942926143c277271bbe99890a91 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Fri, 18 Oct 2024 14:50:32 -0400 Subject: [PATCH 19/72] refactor(server): rename metrics to telemetry (#13584) --- ...ic.interface.ts => telemetry.interface.ts} | 4 +-- server/src/repositories/index.ts | 6 ++-- ....repository.ts => telemetry.repository.ts} | 14 +++++---- server/src/services/base.service.ts | 4 +-- server/src/services/job.service.ts | 8 ++--- .../repositories/metric.repository.mock.ts | 31 ------------------- .../repositories/telemetry.repository.mock.ts | 20 ++++++++++++ server/test/utils.ts | 8 ++--- 8 files changed, 43 insertions(+), 52 deletions(-) rename server/src/interfaces/{metric.interface.ts => telemetry.interface.ts} (84%) rename server/src/repositories/{metric.repository.ts => telemetry.repository.ts} (80%) delete mode 100644 server/test/repositories/metric.repository.mock.ts create mode 100644 server/test/repositories/telemetry.repository.mock.ts diff --git a/server/src/interfaces/metric.interface.ts b/server/src/interfaces/telemetry.interface.ts similarity index 84% rename from server/src/interfaces/metric.interface.ts rename to server/src/interfaces/telemetry.interface.ts index a87a849833..070014f2e0 100644 --- a/server/src/interfaces/metric.interface.ts +++ b/server/src/interfaces/telemetry.interface.ts @@ -1,6 +1,6 @@ import { MetricOptions } from '@opentelemetry/api'; -export const IMetricRepository = 'IMetricRepository'; +export const ITelemetryRepository = 'ITelemetryRepository'; export interface MetricGroupOptions { enabled: boolean; @@ -13,7 +13,7 @@ export interface IMetricGroupRepository { configure(options: MetricGroupOptions): this; } -export interface IMetricRepository { +export interface ITelemetryRepository { api: IMetricGroupRepository; host: IMetricGroupRepository; jobs: IMetricGroupRepository; diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts index 5bf08d0d78..94a0212204 100644 --- a/server/src/repositories/index.ts +++ b/server/src/repositories/index.ts @@ -17,7 +17,6 @@ import { IMapRepository } from 'src/interfaces/map.interface'; import { IMediaRepository } from 'src/interfaces/media.interface'; import { IMemoryRepository } from 'src/interfaces/memory.interface'; import { IMetadataRepository } from 'src/interfaces/metadata.interface'; -import { IMetricRepository } from 'src/interfaces/metric.interface'; import { IMoveRepository } from 'src/interfaces/move.interface'; import { INotificationRepository } from 'src/interfaces/notification.interface'; import { IOAuthRepository } from 'src/interfaces/oauth.interface'; @@ -31,6 +30,7 @@ import { IStackRepository } from 'src/interfaces/stack.interface'; import { IStorageRepository } from 'src/interfaces/storage.interface'; import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface'; import { ITagRepository } from 'src/interfaces/tag.interface'; +import { ITelemetryRepository } from 'src/interfaces/telemetry.interface'; import { ITrashRepository } from 'src/interfaces/trash.interface'; import { IUserRepository } from 'src/interfaces/user.interface'; import { IVersionHistoryRepository } from 'src/interfaces/version-history.interface'; @@ -54,7 +54,6 @@ import { MapRepository } from 'src/repositories/map.repository'; import { MediaRepository } from 'src/repositories/media.repository'; import { MemoryRepository } from 'src/repositories/memory.repository'; import { MetadataRepository } from 'src/repositories/metadata.repository'; -import { MetricRepository } from 'src/repositories/metric.repository'; import { MoveRepository } from 'src/repositories/move.repository'; import { NotificationRepository } from 'src/repositories/notification.repository'; import { OAuthRepository } from 'src/repositories/oauth.repository'; @@ -68,6 +67,7 @@ import { StackRepository } from 'src/repositories/stack.repository'; import { StorageRepository } from 'src/repositories/storage.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; +import { TelemetryRepository } from 'src/repositories/telemetry.repository'; import { TrashRepository } from 'src/repositories/trash.repository'; import { UserRepository } from 'src/repositories/user.repository'; import { VersionHistoryRepository } from 'src/repositories/version-history.repository'; @@ -93,7 +93,6 @@ export const repositories = [ { provide: IMediaRepository, useClass: MediaRepository }, { provide: IMemoryRepository, useClass: MemoryRepository }, { provide: IMetadataRepository, useClass: MetadataRepository }, - { provide: IMetricRepository, useClass: MetricRepository }, { provide: IMoveRepository, useClass: MoveRepository }, { provide: INotificationRepository, useClass: NotificationRepository }, { provide: IOAuthRepository, useClass: OAuthRepository }, @@ -107,6 +106,7 @@ export const repositories = [ { provide: IStorageRepository, useClass: StorageRepository }, { provide: ISystemMetadataRepository, useClass: SystemMetadataRepository }, { provide: ITagRepository, useClass: TagRepository }, + { provide: ITelemetryRepository, useClass: TelemetryRepository }, { provide: ITrashRepository, useClass: TrashRepository }, { provide: IUserRepository, useClass: UserRepository }, { provide: IVersionHistoryRepository, useClass: VersionHistoryRepository }, diff --git a/server/src/repositories/metric.repository.ts b/server/src/repositories/telemetry.repository.ts similarity index 80% rename from server/src/repositories/metric.repository.ts rename to server/src/repositories/telemetry.repository.ts index b59bcf9ed1..d1dc66ae85 100644 --- a/server/src/repositories/metric.repository.ts +++ b/server/src/repositories/telemetry.repository.ts @@ -2,7 +2,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { MetricOptions } from '@opentelemetry/api'; import { MetricService } from 'nestjs-otel'; import { IConfigRepository } from 'src/interfaces/config.interface'; -import { IMetricGroupRepository, IMetricRepository, MetricGroupOptions } from 'src/interfaces/metric.interface'; +import { IMetricGroupRepository, ITelemetryRepository, MetricGroupOptions } from 'src/interfaces/telemetry.interface'; class MetricGroupRepository implements IMetricGroupRepository { private enabled = false; @@ -34,7 +34,7 @@ class MetricGroupRepository implements IMetricGroupRepository { } @Injectable() -export class MetricRepository implements IMetricRepository { +export class TelemetryRepository implements ITelemetryRepository { api: MetricGroupRepository; host: MetricGroupRepository; jobs: MetricGroupRepository; @@ -42,9 +42,11 @@ export class MetricRepository implements IMetricRepository { constructor(metricService: MetricService, @Inject(IConfigRepository) configRepository: IConfigRepository) { const { telemetry } = configRepository.getEnv(); - this.api = new MetricGroupRepository(metricService).configure({ enabled: telemetry.apiMetrics }); - this.host = new MetricGroupRepository(metricService).configure({ enabled: telemetry.hostMetrics }); - this.jobs = new MetricGroupRepository(metricService).configure({ enabled: telemetry.jobMetrics }); - this.repo = new MetricGroupRepository(metricService).configure({ enabled: telemetry.repoMetrics }); + const { apiMetrics, hostMetrics, jobMetrics, repoMetrics } = telemetry; + + this.api = new MetricGroupRepository(metricService).configure({ enabled: apiMetrics }); + this.host = new MetricGroupRepository(metricService).configure({ enabled: hostMetrics }); + this.jobs = new MetricGroupRepository(metricService).configure({ enabled: jobMetrics }); + this.repo = new MetricGroupRepository(metricService).configure({ enabled: repoMetrics }); } } diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index 2bb717b45b..441a81cf91 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -20,7 +20,6 @@ import { IMapRepository } from 'src/interfaces/map.interface'; import { IMediaRepository } from 'src/interfaces/media.interface'; import { IMemoryRepository } from 'src/interfaces/memory.interface'; import { IMetadataRepository } from 'src/interfaces/metadata.interface'; -import { IMetricRepository } from 'src/interfaces/metric.interface'; import { IMoveRepository } from 'src/interfaces/move.interface'; import { INotificationRepository } from 'src/interfaces/notification.interface'; import { IOAuthRepository } from 'src/interfaces/oauth.interface'; @@ -34,6 +33,7 @@ import { IStackRepository } from 'src/interfaces/stack.interface'; import { IStorageRepository } from 'src/interfaces/storage.interface'; import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface'; import { ITagRepository } from 'src/interfaces/tag.interface'; +import { ITelemetryRepository } from 'src/interfaces/telemetry.interface'; import { ITrashRepository } from 'src/interfaces/trash.interface'; import { IUserRepository } from 'src/interfaces/user.interface'; import { IVersionHistoryRepository } from 'src/interfaces/version-history.interface'; @@ -64,7 +64,6 @@ export class BaseService { @Inject(IMediaRepository) protected mediaRepository: IMediaRepository, @Inject(IMemoryRepository) protected memoryRepository: IMemoryRepository, @Inject(IMetadataRepository) protected metadataRepository: IMetadataRepository, - @Inject(IMetricRepository) protected metricRepository: IMetricRepository, @Inject(IMoveRepository) protected moveRepository: IMoveRepository, @Inject(INotificationRepository) protected notificationRepository: INotificationRepository, @Inject(IOAuthRepository) protected oauthRepository: IOAuthRepository, @@ -78,6 +77,7 @@ export class BaseService { @Inject(IStorageRepository) protected storageRepository: IStorageRepository, @Inject(ISystemMetadataRepository) protected systemMetadataRepository: ISystemMetadataRepository, @Inject(ITagRepository) protected tagRepository: ITagRepository, + @Inject(ITelemetryRepository) protected telemetryRepository: ITelemetryRepository, @Inject(ITrashRepository) protected trashRepository: ITrashRepository, @Inject(IUserRepository) protected userRepository: IUserRepository, @Inject(IVersionHistoryRepository) protected versionRepository: IVersionHistoryRepository, diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index 971509447f..46771ff046 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -124,7 +124,7 @@ export class JobService extends BaseService { throw new BadRequestException(`Job is already running`); } - this.metricRepository.jobs.addToCounter(`immich.queues.${snakeCase(name)}.started`, 1); + this.telemetryRepository.jobs.addToCounter(`immich.queues.${snakeCase(name)}.started`, 1); switch (name) { case QueueName.VIDEO_CONVERSION: { @@ -197,19 +197,19 @@ export class JobService extends BaseService { } const queueMetric = `immich.queues.${snakeCase(queueName)}.active`; - this.metricRepository.jobs.addToGauge(queueMetric, 1); + this.telemetryRepository.jobs.addToGauge(queueMetric, 1); try { const status = await handler(data); const jobMetric = `immich.jobs.${name.replaceAll('-', '_')}.${status}`; - this.metricRepository.jobs.addToCounter(jobMetric, 1); + this.telemetryRepository.jobs.addToCounter(jobMetric, 1); if (status === JobStatus.SUCCESS || status == JobStatus.SKIPPED) { await this.onDone(item); } } catch (error: Error | any) { this.logger.error(`Unable to run job handler (${queueName}/${name}): ${error}`, error?.stack, data); } finally { - this.metricRepository.jobs.addToGauge(queueMetric, -1); + this.telemetryRepository.jobs.addToGauge(queueMetric, -1); } }); } diff --git a/server/test/repositories/metric.repository.mock.ts b/server/test/repositories/metric.repository.mock.ts deleted file mode 100644 index e2c3e2aac1..0000000000 --- a/server/test/repositories/metric.repository.mock.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { IMetricRepository } from 'src/interfaces/metric.interface'; -import { Mocked, vitest } from 'vitest'; - -export const newMetricRepositoryMock = (): Mocked => { - return { - api: { - addToCounter: vitest.fn(), - addToGauge: vitest.fn(), - addToHistogram: vitest.fn(), - configure: vitest.fn(), - }, - host: { - addToCounter: vitest.fn(), - addToGauge: vitest.fn(), - addToHistogram: vitest.fn(), - configure: vitest.fn(), - }, - jobs: { - addToCounter: vitest.fn(), - addToGauge: vitest.fn(), - addToHistogram: vitest.fn(), - configure: vitest.fn(), - }, - repo: { - addToCounter: vitest.fn(), - addToGauge: vitest.fn(), - addToHistogram: vitest.fn(), - configure: vitest.fn(), - }, - }; -}; diff --git a/server/test/repositories/telemetry.repository.mock.ts b/server/test/repositories/telemetry.repository.mock.ts new file mode 100644 index 0000000000..737463065c --- /dev/null +++ b/server/test/repositories/telemetry.repository.mock.ts @@ -0,0 +1,20 @@ +import { ITelemetryRepository } from 'src/interfaces/telemetry.interface'; +import { Mocked, vitest } from 'vitest'; + +const newMetricGroupMock = () => { + return { + addToCounter: vitest.fn(), + addToGauge: vitest.fn(), + addToHistogram: vitest.fn(), + configure: vitest.fn(), + }; +}; + +export const newTelemetryRepositoryMock = (): Mocked => { + return { + api: newMetricGroupMock(), + host: newMetricGroupMock(), + jobs: newMetricGroupMock(), + repo: newMetricGroupMock(), + }; +}; diff --git a/server/test/utils.ts b/server/test/utils.ts index 3b7e80994d..9a40a22c2c 100644 --- a/server/test/utils.ts +++ b/server/test/utils.ts @@ -20,7 +20,6 @@ import { newMapRepositoryMock } from 'test/repositories/map.repository.mock'; import { newMediaRepositoryMock } from 'test/repositories/media.repository.mock'; import { newMemoryRepositoryMock } from 'test/repositories/memory.repository.mock'; import { newMetadataRepositoryMock } from 'test/repositories/metadata.repository.mock'; -import { newMetricRepositoryMock } from 'test/repositories/metric.repository.mock'; import { newMoveRepositoryMock } from 'test/repositories/move.repository.mock'; import { newNotificationRepositoryMock } from 'test/repositories/notification.repository.mock'; import { newOAuthRepositoryMock } from 'test/repositories/oauth.repository.mock'; @@ -34,6 +33,7 @@ import { newStackRepositoryMock } from 'test/repositories/stack.repository.mock' import { newStorageRepositoryMock } from 'test/repositories/storage.repository.mock'; import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock'; import { newTagRepositoryMock } from 'test/repositories/tag.repository.mock'; +import { newTelemetryRepositoryMock } from 'test/repositories/telemetry.repository.mock'; import { newTrashRepositoryMock } from 'test/repositories/trash.repository.mock'; import { newUserRepositoryMock } from 'test/repositories/user.repository.mock'; import { newVersionHistoryRepositoryMock } from 'test/repositories/version-history.repository.mock'; @@ -73,7 +73,6 @@ export const newTestService = ( const mediaMock = newMediaRepositoryMock(); const memoryMock = newMemoryRepositoryMock(); const metadataMock = (metadataRepository || newMetadataRepositoryMock()) as Mocked; - const metricMock = newMetricRepositoryMock(); const moveMock = newMoveRepositoryMock(); const notificationMock = newNotificationRepositoryMock(); const oauthMock = newOAuthRepositoryMock(); @@ -87,6 +86,7 @@ export const newTestService = ( const storageMock = newStorageRepositoryMock(); const systemMock = newSystemMetadataRepositoryMock(); const tagMock = newTagRepositoryMock(); + const telemetryMock = newTelemetryRepositoryMock(); const trashMock = newTrashRepositoryMock(); const userMock = newUserRepositoryMock(); const versionHistoryMock = newVersionHistoryRepositoryMock(); @@ -112,7 +112,6 @@ export const newTestService = ( mediaMock, memoryMock, metadataMock, - metricMock, moveMock, notificationMock, oauthMock, @@ -126,6 +125,7 @@ export const newTestService = ( storageMock, systemMock, tagMock, + telemetryMock, trashMock, userMock, versionHistoryMock, @@ -153,7 +153,6 @@ export const newTestService = ( mediaMock, memoryMock, metadataMock, - metricMock, moveMock, notificationMock, oauthMock, @@ -167,6 +166,7 @@ export const newTestService = ( storageMock, systemMock, tagMock, + telemetryMock, trashMock, userMock, versionHistoryMock, From e1e3ae811dff92f0af2d19a97e6b54e669aa6398 Mon Sep 17 00:00:00 2001 From: akara <55230837+richeyphu@users.noreply.github.com> Date: Sat, 19 Oct 2024 02:41:32 +0700 Subject: [PATCH 20/72] chore(docs): add Thai README (#13591) * chore(docs): add Thai README * chore(docs): add links to Thai README --- README.md | 34 ++++----- readme_i18n/README_ar_JO.md | 1 + readme_i18n/README_ca_ES.md | 1 + readme_i18n/README_de_DE.md | 1 + readme_i18n/README_es_ES.md | 1 + readme_i18n/README_fr_FR.md | 1 + readme_i18n/README_it_IT.md | 1 + readme_i18n/README_ja_JP.md | 1 + readme_i18n/README_ko_KR.md | 1 + readme_i18n/README_nl_NL.md | 1 + readme_i18n/README_pt_BR.md | 1 + readme_i18n/README_ru_RU.md | 1 + readme_i18n/README_sv_SE.md | 1 + readme_i18n/README_th_TH.md | 134 ++++++++++++++++++++++++++++++++++++ readme_i18n/README_tr_TR.md | 1 + readme_i18n/README_vi_VN.md | 1 + readme_i18n/README_zh_CN.md | 3 +- 17 files changed, 167 insertions(+), 18 deletions(-) create mode 100644 readme_i18n/README_th_TH.md diff --git a/README.md b/README.md index 5c4b9c39ed..7ad539c4cd 100644 --- a/README.md +++ b/README.md @@ -17,24 +17,24 @@
+

- -Català -Español -Français -Italiano -日本語 -한국어 -Deutsch -Nederlands -Türkçe -中文 -Русский -Português Brasileiro -Svenska -العربية -Tiếng Việt - + Català + Español + Français + Italiano + 日本語 + 한국어 + Deutsch + Nederlands + Türkçe + 中文 + Русский + Português Brasileiro + Svenska + العربية + Tiếng Việt + ภาษาไทย

## Disclaimer diff --git a/readme_i18n/README_ar_JO.md b/readme_i18n/README_ar_JO.md index 7df39d226b..8fa4ac1195 100644 --- a/readme_i18n/README_ar_JO.md +++ b/readme_i18n/README_ar_JO.md @@ -32,6 +32,7 @@ Русский Português Brasileiro Svenska + ภาษาไทย

## تنصل diff --git a/readme_i18n/README_ca_ES.md b/readme_i18n/README_ca_ES.md index ed14649e0a..66a8b584fd 100644 --- a/readme_i18n/README_ca_ES.md +++ b/readme_i18n/README_ca_ES.md @@ -32,6 +32,7 @@ Português Brasileiro Svenska العربية + ภาษาไทย

## Avís legal diff --git a/readme_i18n/README_de_DE.md b/readme_i18n/README_de_DE.md index 7a59e3444e..d6c69106f3 100644 --- a/readme_i18n/README_de_DE.md +++ b/readme_i18n/README_de_DE.md @@ -32,6 +32,7 @@ Português Brasileiro Svenska العربية + ภาษาไทย

## Warnung diff --git a/readme_i18n/README_es_ES.md b/readme_i18n/README_es_ES.md index 726a504526..0b0dbf919d 100644 --- a/readme_i18n/README_es_ES.md +++ b/readme_i18n/README_es_ES.md @@ -32,6 +32,7 @@ Português Brasileiro Svenska العربية + ภาษาไทย

## Advertencia diff --git a/readme_i18n/README_fr_FR.md b/readme_i18n/README_fr_FR.md index da52fe28a6..e2f979d254 100644 --- a/readme_i18n/README_fr_FR.md +++ b/readme_i18n/README_fr_FR.md @@ -32,6 +32,7 @@ Português Brasileiro Svenska العربية + ภาษาไทย

## Clause de non-responsabilité diff --git a/readme_i18n/README_it_IT.md b/readme_i18n/README_it_IT.md index 1523143f06..7208df7e24 100644 --- a/readme_i18n/README_it_IT.md +++ b/readme_i18n/README_it_IT.md @@ -32,6 +32,7 @@ Português Brasileiro Svenska العربية + ภาษาไทย

## Declino di responsabilità diff --git a/readme_i18n/README_ja_JP.md b/readme_i18n/README_ja_JP.md index 98ff8e68d9..828afa9812 100644 --- a/readme_i18n/README_ja_JP.md +++ b/readme_i18n/README_ja_JP.md @@ -32,6 +32,7 @@ Português Brasileiro Svenska العربية + ภาษาไทย

## 免責事項 diff --git a/readme_i18n/README_ko_KR.md b/readme_i18n/README_ko_KR.md index 66df040d75..8b280e0a9b 100644 --- a/readme_i18n/README_ko_KR.md +++ b/readme_i18n/README_ko_KR.md @@ -33,6 +33,7 @@ Português Brasileiro Svenska العربية +ภาษาไทย

diff --git a/readme_i18n/README_nl_NL.md b/readme_i18n/README_nl_NL.md index 1c877d9d3e..e1cf6d66f5 100644 --- a/readme_i18n/README_nl_NL.md +++ b/readme_i18n/README_nl_NL.md @@ -32,6 +32,7 @@ Português Brasileiro Svenska العربية + ภาษาไทย

## Disclaimer diff --git a/readme_i18n/README_pt_BR.md b/readme_i18n/README_pt_BR.md index 51ea8238da..5468ebb4c4 100644 --- a/readme_i18n/README_pt_BR.md +++ b/readme_i18n/README_pt_BR.md @@ -34,6 +34,7 @@ Svenska العربية Tiếng Việt +ภาษาไทย

diff --git a/readme_i18n/README_ru_RU.md b/readme_i18n/README_ru_RU.md index 11a2a34f33..0ff3e3f08f 100644 --- a/readme_i18n/README_ru_RU.md +++ b/readme_i18n/README_ru_RU.md @@ -32,6 +32,7 @@ Português Brasileiro Svenska العربية + ภาษาไทย

## Предупреждение diff --git a/readme_i18n/README_sv_SE.md b/readme_i18n/README_sv_SE.md index 3673eab57c..29706acb55 100644 --- a/readme_i18n/README_sv_SE.md +++ b/readme_i18n/README_sv_SE.md @@ -33,6 +33,7 @@ Русский Português Brasileiro العربية + ภาษาไทย

## Ansvarsfriskrivning diff --git a/readme_i18n/README_th_TH.md b/readme_i18n/README_th_TH.md new file mode 100644 index 0000000000..6a6b70d435 --- /dev/null +++ b/readme_i18n/README_th_TH.md @@ -0,0 +1,134 @@ +

+
+ License: AGPLv3 + + Discord + +
+
+

+ +

+ +

+ +

โซลูชันการจัดการภาพถ่ายและวิดีโอแบบโฮสต์เองที่มีประสิทธิภาพสูง

+
+ + + + +
+ +

+ English + Català + Español + Français + Italiano + 日本語 + 한국어 + Deutsch + Nederlands + Türkçe + 中文 + Русский + Português Brasileiro + Svenska + العربية + Tiếng Việt +

+ +## ข้อจำกัดความรับผิดชอบ + +- ⚠️ โพรเจกต์นี้กำลังอยู่ระหว่างการพัฒนา**ที่มีการเปลี่ยนแปลงบ่อยมาก** +- ⚠️ คาดว่าจะมีข้อผิดพลาดและการเปลี่ยนแปลงที่ส่งผลเสีย +- ⚠️ **ห้ามใช้แอปนี้เป็นวิธีการเดียวในการจัดเก็บภาพถ่ายและวิดีโอของคุณ** +- ⚠️ ปฏิบัติตามแผนการสำรองข้อมูลแบบ [3-2-1](https://www.backblaze.com/blog/the-3-2-1-backup-strategy/) สำหรับภาพถ่ายและวิดีโอที่สำคัญของคุณอยู่เสมอ! + +> [!NOTE] +> คุณสามารถหาคู่มือหลัก รวมถึงคู่มือการติดตั้ง ได้ที่ https://immich.app/ + +## ลิงก์ + +- [คู่มือ](https://immich.app/docs) +- [เกี่ยวกับ](https://immich.app/docs/overview/introduction) +- [การติดตั้ง](https://immich.app/docs/install/requirements) +- [โรดแมป](https://immich.app/roadmap) +- [สาธิต](#สาธิต) +- [คุณสมบัติ](#คุณสมบัติ) +- [การแปลภาษา](https://immich.app/docs/developer/translations) +- [สนับสนุนโพรเจกต์](https://immich.app/docs/overview/support-the-project) + +## สาธิต + +เข้าถึงการสาธิตได้ [ที่นี่](https://demo.immich.app) โดยการสาธิตนี้ทำงานบน Oracle VM Free-tier ตั้งอยู่ที่อัมสเตอร์ดัม ใช้ซีพียู ARM64 quad-core 2.4Ghz และแรม 24GB + +สำหรับแอปมือถือ คุณสามารถใช้ `https://demo.immich.app/api` เป็น `Server Endpoint URL` + +### ข้อมูลการเข้าสู่ระบบ + +| อีเมล | รหัสผ่าน | +| --------------- | -------- | +| demo@immich.app | demo | + +## คุณสมบัติ + +| คุณสมบัติ | มือถือ | เว็บ | +| :----------------------------------------- | ------ | ------ | +| อัปโหลดและดูวิดีโอและภาพถ่าย | ใช่ | ใช่ | +| การสำรองข้อมูลอัตโนมัติเมื่อเปิดแอป | ใช่ | N/A | +| ป้องกันการซ้ำซ้อนของไฟล์ | ใช่ | ใช่ | +| เลือกอัลบั้มสำหรับสำรองข้อมูล | ใช่ | N/A | +| ดาวน์โหลดภาพถ่ายและวิดีโอไปยังอุปกรณ์ | ใช่ | ใช่ | +| รองรับผู้ใช้หลายคน | ใช่ | ใช่ | +| อัลบั้มและอัลบั้มแชร์ | ใช่ | ใช่ | +| แถบเลื่อนแบบลากได้ | ใช่ | ใช่ | +| รองรับรูปแบบไฟล์ RAW | ใช่ | ใช่ | +| ดูข้อมูลเมตา (EXIF, แผนที่) | ใช่ | ใช่ | +| ค้นหาจากข้อมูลเมตา วัตถุ ใบหน้า และ CLIP | ใช่ | ใช่ | +| ฟังก์ชันการจัดการผู้ดูแลระบบ | ไม่ใช่ | ใช่ | +| การสำรองข้อมูลพื้นหลัง | ใช่ | N/A | +| การเลื่อนแบบเสมือน | ใช่ | ใช่ | +| รองรับ OAuth | ใช่ | ใช่ | +| คีย์ API | N/A | ใช่ | +| การสำรองและเล่น LivePhoto/MotionPhoto | ใช่ | ใช่ | +| รองรับการแสดงภาพ 360 องศา | ไม่ใช่ | ใช่ | +| โครงสร้างการจัดเก็บข้อมูลที่ผู้ใช้กำหนดเอง | ใช่ | ใช่ | +| การแชร์สาธารณะ | ใช่ | ใช่ | +| การจัดเก็บและรายการโปรด | ใช่ | ใช่ | +| แผนที่ทั่วโลก | ใช่ | ใช่ | +| การแชร์กับคู่หู | ใช่ | ใช่ | +| การจดจำใบหน้าและการจัดกลุ่ม | ใช่ | ใช่ | +| ความทรงจำ (x ปีที่แล้ว) | ใช่ | ใช่ | +| รองรับแบบออฟไลน์ | ใช่ | ไม่ใช่ | +| แกลเลอรีแบบอ่านอย่างเดียว | ใช่ | ใช่ | +| ภาพถ่ายซ้อนกัน | ใช่ | ใช่ | + +## การแปลภาษา + +อ่านเพิ่มเติมเกี่ยวกับการแปลภาษา [ที่นี่](https://immich.app/docs/developer/translations) + + + สถานะการแปล + + +## กิจกรรมของคลังเก็บข้อมูล + +![กิจกรรม](https://repobeats.axiom.co/api/embed/9e86d9dc3ddd137161f2f6d2e758d7863b1789cb.svg "ภาพการวิเคราะห์ของ Repobeats") + +## ประวัติการให้ดาว + + + + + + แผนภูมิประวัติการให้ดาว + + + +## ผู้ร่วมพัฒนา + + + + diff --git a/readme_i18n/README_tr_TR.md b/readme_i18n/README_tr_TR.md index f95d914880..6bf23be5f8 100644 --- a/readme_i18n/README_tr_TR.md +++ b/readme_i18n/README_tr_TR.md @@ -32,6 +32,7 @@ Português Brasileiro Svenska العربية + ภาษาไทย

## Feragatname diff --git a/readme_i18n/README_vi_VN.md b/readme_i18n/README_vi_VN.md index 7ec4b9c948..69d7a151be 100644 --- a/readme_i18n/README_vi_VN.md +++ b/readme_i18n/README_vi_VN.md @@ -35,6 +35,7 @@ Svenska العربية Tiếng Việt +ภาษาไทย

diff --git a/readme_i18n/README_zh_CN.md b/readme_i18n/README_zh_CN.md index 6355cd65ed..380dc25992 100644 --- a/readme_i18n/README_zh_CN.md +++ b/readme_i18n/README_zh_CN.md @@ -36,7 +36,8 @@ Português Brasileiro Svenska العربية - + ภาษาไทย +

## 免责声明 From 76c0b964ebcbd8b35f158b3af40f1e4a6ce34cca Mon Sep 17 00:00:00 2001 From: Christian Koch Date: Fri, 18 Oct 2024 21:43:48 +0200 Subject: [PATCH 21/72] chore(docs): update _storage-template.md (#13578) Update _storage-template.md The example for the {{if}} was a little bit confusing. Just a recommendation --- docs/docs/partials/_storage-template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/partials/_storage-template.md b/docs/docs/partials/_storage-template.md index b6dcd5ad77..0c668d0a3e 100644 --- a/docs/docs/partials/_storage-template.md +++ b/docs/docs/partials/_storage-template.md @@ -31,5 +31,5 @@ Immich also provides a mechanism to migrate between templates so that if the tem If you want to store assets in album folders, but you also have assets that do not belong to any album, you can use `{{#if album}}`, `{{else}}` and `{{/if}}` to create a conditional statement. For example, the following template will store assets in album folders if they belong to an album, and in a folder named "Other/Month" if they do not belong to an album: ``` -{{y}}/{{#if album}}{{album}}{{else}}Other/{{MM}}{{/if}}/{{filename}} +{{y}}/{{#if album}}{{album}}{{else}}Other{{/if}}/{{MM}}/{{filename}} ``` From 4a2a7b7735980921d47a8b816757bbc91443c6ce Mon Sep 17 00:00:00 2001 From: Hayden Date: Fri, 18 Oct 2024 13:51:34 -0600 Subject: [PATCH 22/72] feat(server): wait five minutes before sending email on new album item (#12223) Album update jobs will now wait five minutes to send. If a new image is added while that job is pending, the old job will be cancelled, and a new one will be enqueued for a minute. This is to prevent a flood of notifications by dragging in images directly to the album, which adds them to the album one at a time. Album updates now include a list of users to email, which is generally everybody except the updater. If somebody else updates the album within that minute, both people will get an album update email in a minute, as they both added images and the other should be notified. --- server/src/interfaces/event.interface.ts | 2 +- server/src/interfaces/job.interface.ts | 10 +++- server/src/repositories/job.repository.ts | 21 +++++++- server/src/services/album.service.spec.ts | 6 +-- server/src/services/album.service.ts | 8 +++- .../src/services/notification.service.spec.ts | 48 +++++++++---------- server/src/services/notification.service.ts | 36 ++++++++++++-- server/test/fixtures/user.stub.ts | 1 + .../test/repositories/job.repository.mock.ts | 1 + 9 files changed, 93 insertions(+), 40 deletions(-) diff --git a/server/src/interfaces/event.interface.ts b/server/src/interfaces/event.interface.ts index 7ea48faf53..40efaf150c 100644 --- a/server/src/interfaces/event.interface.ts +++ b/server/src/interfaces/event.interface.ts @@ -22,7 +22,7 @@ type EventMap = { 'config.validate': [{ newConfig: SystemConfig; oldConfig: SystemConfig }]; // album events - 'album.update': [{ id: string; updatedBy: string }]; + 'album.update': [{ id: string; recipientIds: string[] }]; 'album.invite': [{ id: string; userId: string }]; // asset events diff --git a/server/src/interfaces/job.interface.ts b/server/src/interfaces/job.interface.ts index aa3090675e..82176ffa93 100644 --- a/server/src/interfaces/job.interface.ts +++ b/server/src/interfaces/job.interface.ts @@ -120,6 +120,11 @@ export interface IBaseJob { force?: boolean; } +export interface IDelayedJob extends IBaseJob { + /** The minimum time to wait to execute this job, in milliseconds. */ + delay?: number; +} + export interface IEntityJob extends IBaseJob { id: string; source?: 'upload' | 'sidecar-write' | 'copy'; @@ -181,8 +186,8 @@ export interface INotifyAlbumInviteJob extends IEntityJob { recipientId: string; } -export interface INotifyAlbumUpdateJob extends IEntityJob { - senderId: string; +export interface INotifyAlbumUpdateJob extends IEntityJob, IDelayedJob { + recipientIds: string[]; } export interface JobCounts { @@ -310,4 +315,5 @@ export interface IJobRepository { getQueueStatus(name: QueueName): Promise; getJobCounts(name: QueueName): Promise; waitForQueueCompletion(...queues: QueueName[]): Promise; + removeJob(jobId: string, name: JobName): Promise; } diff --git a/server/src/repositories/job.repository.ts b/server/src/repositories/job.repository.ts index 3ff26f1ba4..846b6dc9cd 100644 --- a/server/src/repositories/job.repository.ts +++ b/server/src/repositories/job.repository.ts @@ -7,6 +7,7 @@ import { CronJob, CronTime } from 'cron'; import { setTimeout } from 'node:timers/promises'; import { IConfigRepository } from 'src/interfaces/config.interface'; import { + IEntityJob, IJobRepository, JobCounts, JobItem, @@ -252,6 +253,9 @@ export class JobRepository implements IJobRepository { private getJobOptions(item: JobItem): JobsOptions | null { switch (item.name) { + case JobName.NOTIFY_ALBUM_UPDATE: { + return { jobId: item.data.id, delay: item.data?.delay }; + } case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE: { return { jobId: item.data.id }; } @@ -261,7 +265,6 @@ export class JobRepository implements IJobRepository { case JobName.QUEUE_FACIAL_RECOGNITION: { return { jobId: JobName.QUEUE_FACIAL_RECOGNITION }; } - default: { return null; } @@ -271,4 +274,20 @@ export class JobRepository implements IJobRepository { private getQueue(queue: QueueName): Queue { return this.moduleReference.get(getQueueToken(queue), { strict: false }); } + + public async removeJob(jobId: string, name: JobName): Promise { + const existingJob = await this.getQueue(JOBS_TO_QUEUE[name]).getJob(jobId); + if (!existingJob) { + return; + } + try { + await existingJob.remove(); + } catch (error: any) { + if (error.message?.includes('Missing key for job')) { + return; + } + throw error; + } + return existingJob.data; + } } diff --git a/server/src/services/album.service.spec.ts b/server/src/services/album.service.spec.ts index 33c8f5dd7f..12c93ee127 100644 --- a/server/src/services/album.service.spec.ts +++ b/server/src/services/album.service.spec.ts @@ -537,10 +537,6 @@ describe(AlbumService.name, () => { albumThumbnailAssetId: 'asset-1', }); expect(albumMock.addAssetIds).toHaveBeenCalledWith('album-123', ['asset-1', 'asset-2', 'asset-3']); - expect(eventMock.emit).toHaveBeenCalledWith('album.update', { - id: 'album-123', - updatedBy: authStub.admin.user.id, - }); }); it('should not set the thumbnail if the album has one already', async () => { @@ -583,7 +579,7 @@ describe(AlbumService.name, () => { expect(albumMock.addAssetIds).toHaveBeenCalledWith('album-123', ['asset-1', 'asset-2', 'asset-3']); expect(eventMock.emit).toHaveBeenCalledWith('album.update', { id: 'album-123', - updatedBy: authStub.user1.user.id, + recipientIds: ['admin_id'], }); }); diff --git a/server/src/services/album.service.ts b/server/src/services/album.service.ts index e8acce9b6c..2cf83e9b99 100644 --- a/server/src/services/album.service.ts +++ b/server/src/services/album.service.ts @@ -174,7 +174,13 @@ export class AlbumService extends BaseService { albumThumbnailAssetId: album.albumThumbnailAssetId ?? firstNewAssetId, }); - await this.eventRepository.emit('album.update', { id, updatedBy: auth.user.id }); + const allUsersExceptUs = [...album.albumUsers.map(({ user }) => user.id), album.owner.id].filter( + (userId) => userId !== auth.user.id, + ); + + if (allUsersExceptUs.length > 0) { + await this.eventRepository.emit('album.update', { id, recipientIds: allUsersExceptUs }); + } } return results; diff --git a/server/src/services/notification.service.spec.ts b/server/src/services/notification.service.spec.ts index 028e512b39..d07d06443a 100644 --- a/server/src/services/notification.service.spec.ts +++ b/server/src/services/notification.service.spec.ts @@ -7,7 +7,7 @@ import { AssetFileType, UserMetadataKey } from 'src/enum'; import { IAlbumRepository } from 'src/interfaces/album.interface'; import { IAssetRepository } from 'src/interfaces/asset.interface'; import { IEventRepository } from 'src/interfaces/event.interface'; -import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface'; +import { IJobRepository, INotifyAlbumUpdateJob, JobName, JobStatus } from 'src/interfaces/job.interface'; import { EmailTemplate, INotificationRepository } from 'src/interfaces/notification.interface'; import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface'; import { IUserRepository } from 'src/interfaces/user.interface'; @@ -170,10 +170,10 @@ describe(NotificationService.name, () => { describe('onAlbumUpdateEvent', () => { it('should queue notify album update event', async () => { - await sut.onAlbumUpdate({ id: '', updatedBy: '42' }); + await sut.onAlbumUpdate({ id: 'album', recipientIds: ['42'] }); expect(jobMock.queue).toHaveBeenCalledWith({ name: JobName.NOTIFY_ALBUM_UPDATE, - data: { id: '', senderId: '42' }, + data: { id: 'album', recipientIds: ['42'], delay: 300_000 }, }); }); }); @@ -512,34 +512,17 @@ describe(NotificationService.name, () => { describe('handleAlbumUpdate', () => { it('should skip if album could not be found', async () => { - await expect(sut.handleAlbumUpdate({ id: '', senderId: '' })).resolves.toBe(JobStatus.SKIPPED); + await expect(sut.handleAlbumUpdate({ id: '', recipientIds: ['1'] })).resolves.toBe(JobStatus.SKIPPED); expect(userMock.get).not.toHaveBeenCalled(); }); it('should skip if owner could not be found', async () => { albumMock.getById.mockResolvedValue(albumStub.emptyWithValidThumbnail); - await expect(sut.handleAlbumUpdate({ id: '', senderId: '' })).resolves.toBe(JobStatus.SKIPPED); + await expect(sut.handleAlbumUpdate({ id: '', recipientIds: ['1'] })).resolves.toBe(JobStatus.SKIPPED); expect(systemMock.get).not.toHaveBeenCalled(); }); - it('should filter out the sender', async () => { - albumMock.getById.mockResolvedValue({ - ...albumStub.emptyWithValidThumbnail, - albumUsers: [ - { user: { id: userStub.user1.id } } as AlbumUserEntity, - { user: { id: userStub.user2.id } } as AlbumUserEntity, - ], - }); - userMock.get.mockResolvedValue(userStub.user1); - notificationMock.renderEmail.mockResolvedValue({ html: '', text: '' }); - - await sut.handleAlbumUpdate({ id: '', senderId: userStub.user1.id }); - expect(userMock.get).not.toHaveBeenCalledWith(userStub.user1.id, { withDeleted: false }); - expect(userMock.get).toHaveBeenCalledWith(userStub.user2.id, { withDeleted: false }); - expect(notificationMock.renderEmail).toHaveBeenCalledOnce(); - }); - it('should skip recipient that could not be looked up', async () => { albumMock.getById.mockResolvedValue({ ...albumStub.emptyWithValidThumbnail, @@ -548,7 +531,7 @@ describe(NotificationService.name, () => { userMock.get.mockResolvedValueOnce(userStub.user1); notificationMock.renderEmail.mockResolvedValue({ html: '', text: '' }); - await sut.handleAlbumUpdate({ id: '', senderId: '' }); + await sut.handleAlbumUpdate({ id: '', recipientIds: [userStub.user1.id] }); expect(userMock.get).toHaveBeenCalledWith(userStub.user1.id, { withDeleted: false }); expect(notificationMock.renderEmail).not.toHaveBeenCalled(); }); @@ -571,7 +554,7 @@ describe(NotificationService.name, () => { }); notificationMock.renderEmail.mockResolvedValue({ html: '', text: '' }); - await sut.handleAlbumUpdate({ id: '', senderId: '' }); + await sut.handleAlbumUpdate({ id: '', recipientIds: [userStub.user1.id] }); expect(userMock.get).toHaveBeenCalledWith(userStub.user1.id, { withDeleted: false }); expect(notificationMock.renderEmail).not.toHaveBeenCalled(); }); @@ -594,7 +577,7 @@ describe(NotificationService.name, () => { }); notificationMock.renderEmail.mockResolvedValue({ html: '', text: '' }); - await sut.handleAlbumUpdate({ id: '', senderId: '' }); + await sut.handleAlbumUpdate({ id: '', recipientIds: [userStub.user1.id] }); expect(userMock.get).toHaveBeenCalledWith(userStub.user1.id, { withDeleted: false }); expect(notificationMock.renderEmail).not.toHaveBeenCalled(); }); @@ -607,11 +590,24 @@ describe(NotificationService.name, () => { userMock.get.mockResolvedValue(userStub.user1); notificationMock.renderEmail.mockResolvedValue({ html: '', text: '' }); - await sut.handleAlbumUpdate({ id: '', senderId: '' }); + await sut.handleAlbumUpdate({ id: '', recipientIds: [userStub.user1.id] }); expect(userMock.get).toHaveBeenCalledWith(userStub.user1.id, { withDeleted: false }); expect(notificationMock.renderEmail).toHaveBeenCalled(); expect(jobMock.queue).toHaveBeenCalled(); }); + + it('should add new recipients for new images if job is already queued', async () => { + jobMock.removeJob.mockResolvedValue({ id: '1', recipientIds: ['2', '3', '4'] } as INotifyAlbumUpdateJob); + await sut.onAlbumUpdate({ id: '1', recipientIds: ['1', '2', '3'] } as INotifyAlbumUpdateJob); + expect(jobMock.queue).toHaveBeenCalledWith({ + name: JobName.NOTIFY_ALBUM_UPDATE, + data: { + id: '1', + delay: 300_000, + recipientIds: ['1', '2', '3', '4'], + }, + }); + }); }); describe('handleSendEmail', () => { diff --git a/server/src/services/notification.service.ts b/server/src/services/notification.service.ts index 122a09ee2e..c3c7727468 100644 --- a/server/src/services/notification.service.ts +++ b/server/src/services/notification.service.ts @@ -5,9 +5,11 @@ import { AlbumEntity } from 'src/entities/album.entity'; import { ArgOf } from 'src/interfaces/event.interface'; import { IEmailJob, + IEntityJob, INotifyAlbumInviteJob, INotifyAlbumUpdateJob, INotifySignupJob, + JobItem, JobName, JobStatus, } from 'src/interfaces/job.interface'; @@ -21,6 +23,8 @@ import { getPreferences } from 'src/utils/preferences'; @Injectable() export class NotificationService extends BaseService { + private static albumUpdateEmailDelayMs = 300_000; + @OnEvent({ name: 'config.update' }) onConfigUpdate({ oldConfig, newConfig }: ArgOf<'config.update'>) { this.eventRepository.clientBroadcast('on_config_update'); @@ -100,8 +104,30 @@ export class NotificationService extends BaseService { } @OnEvent({ name: 'album.update' }) - async onAlbumUpdate({ id, updatedBy }: ArgOf<'album.update'>) { - await this.jobRepository.queue({ name: JobName.NOTIFY_ALBUM_UPDATE, data: { id, senderId: updatedBy } }); + async onAlbumUpdate({ id, recipientIds }: ArgOf<'album.update'>) { + // if recipientIds is empty, album likely only has one user part of it, don't queue notification if so + if (recipientIds.length === 0) { + return; + } + + const job: JobItem = { + name: JobName.NOTIFY_ALBUM_UPDATE, + data: { id, recipientIds, delay: NotificationService.albumUpdateEmailDelayMs }, + }; + + const previousJobData = await this.jobRepository.removeJob(id, JobName.NOTIFY_ALBUM_UPDATE); + if (previousJobData && this.isAlbumUpdateJob(previousJobData)) { + for (const id of previousJobData.recipientIds) { + if (!recipientIds.includes(id)) { + recipientIds.push(id); + } + } + } + await this.jobRepository.queue(job); + } + + private isAlbumUpdateJob(job: IEntityJob): job is INotifyAlbumUpdateJob { + return 'recipientIds' in job; } @OnEvent({ name: 'album.invite' }) @@ -228,7 +254,7 @@ export class NotificationService extends BaseService { return JobStatus.SUCCESS; } - async handleAlbumUpdate({ id, senderId }: INotifyAlbumUpdateJob) { + async handleAlbumUpdate({ id, recipientIds }: INotifyAlbumUpdateJob) { const album = await this.albumRepository.getById(id, { withAssets: false }); if (!album) { @@ -240,7 +266,9 @@ export class NotificationService extends BaseService { return JobStatus.SKIPPED; } - const recipients = [...album.albumUsers.map((user) => user.user), owner].filter((user) => user.id !== senderId); + const recipients = [...album.albumUsers.map((user) => user.user), owner].filter((user) => + recipientIds.includes(user.id), + ); const attachment = await this.getAlbumThumbnailAttachment(album); const { server } = await this.getConfig({ withCache: false }); diff --git a/server/test/fixtures/user.stub.ts b/server/test/fixtures/user.stub.ts index b65cd6b395..9553b5344a 100644 --- a/server/test/fixtures/user.stub.ts +++ b/server/test/fixtures/user.stub.ts @@ -7,6 +7,7 @@ export const userStub = { ...authStub.admin.user, password: 'admin_password', name: 'admin_name', + id: 'admin_id', storageLabel: 'admin', oauthId: '', shouldChangePassword: false, diff --git a/server/test/repositories/job.repository.mock.ts b/server/test/repositories/job.repository.mock.ts index 871801830a..cfa1826dd8 100644 --- a/server/test/repositories/job.repository.mock.ts +++ b/server/test/repositories/job.repository.mock.ts @@ -16,5 +16,6 @@ export const newJobRepositoryMock = (): Mocked => { getJobCounts: vitest.fn(), clear: vitest.fn(), waitForQueueCompletion: vitest.fn(), + removeJob: vitest.fn(), }; }; From c9c0212ca9aed6ecb1083878d1efa68406ca0030 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 18 Oct 2024 15:53:47 -0500 Subject: [PATCH 23/72] fix(web): intersection observer not triggered to load more people (#13589) --- .../lib/components/faces-page/manage-people-visibility.svelte | 2 +- web/src/lib/components/faces-page/people-infinite-scroll.svelte | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/src/lib/components/faces-page/manage-people-visibility.svelte b/web/src/lib/components/faces-page/manage-people-visibility.svelte index 23a69e7759..a48fd6bf74 100644 --- a/web/src/lib/components/faces-page/manage-people-visibility.svelte +++ b/web/src/lib/components/faces-page/manage-people-visibility.svelte @@ -145,7 +145,7 @@ {@const hidden = personIsHidden[person.id]}