diff --git a/server/src/app.module.ts b/server/src/app.module.ts index fd921150fd..3c26faaca3 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -14,6 +14,7 @@ import { entities } from 'src/entities'; import { ImmichWorker } from 'src/enum'; import { IEventRepository } from 'src/interfaces/event.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; +import { ITelemetryRepository } from 'src/interfaces/telemetry.interface'; import { AuthGuard } from 'src/middleware/auth.guard'; import { ErrorInterceptor } from 'src/middleware/error.interceptor'; import { FileUploadInterceptor } from 'src/middleware/file-upload.interceptor'; @@ -21,6 +22,7 @@ import { GlobalExceptionFilter } from 'src/middleware/global-exception.filter'; import { LoggingInterceptor } from 'src/middleware/logging.interceptor'; import { repositories } from 'src/repositories'; import { ConfigRepository } from 'src/repositories/config.repository'; +import { teardownTelemetry } from 'src/repositories/telemetry.repository'; import { services } from 'src/services'; import { DatabaseService } from 'src/services/database.service'; @@ -66,6 +68,7 @@ abstract class BaseModule implements OnModuleInit, OnModuleDestroy { constructor( @Inject(ILoggerRepository) logger: ILoggerRepository, @Inject(IEventRepository) private eventRepository: IEventRepository, + @Inject(ITelemetryRepository) private telemetryRepository: ITelemetryRepository, ) { logger.setAppName(this.worker); } @@ -73,12 +76,14 @@ abstract class BaseModule implements OnModuleInit, OnModuleDestroy { abstract getWorker(): ImmichWorker; async onModuleInit() { + this.telemetryRepository.setup({ repositories: repositories.map(({ useClass }) => useClass) }); this.eventRepository.setup({ services }); await this.eventRepository.emit('app.bootstrap', this.worker); } async onModuleDestroy() { await this.eventRepository.emit('app.shutdown', this.worker); + await teardownTelemetry(); } } diff --git a/server/src/config.ts b/server/src/config.ts index e386c134b4..fca6719bc0 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -354,9 +354,9 @@ export const immichAppConfig: ConfigModuleOptions = { ), IMMICH_METRICS: Joi.boolean().optional().default(false), - IMMICH_HOST_METRICS: Joi.boolean().optional().default(false), - IMMICH_API_METRICS: Joi.boolean().optional().default(false), - IMMICH_IO_METRICS: Joi.boolean().optional().default(false), + IMMICH_HOST_METRICS: Joi.boolean().optional(), + IMMICH_API_METRICS: Joi.boolean().optional(), + IMMICH_IO_METRICS: Joi.boolean().optional(), }), }; diff --git a/server/src/decorators.ts b/server/src/decorators.ts index 2782368239..db755c5ff9 100644 --- a/server/src/decorators.ts +++ b/server/src/decorators.ts @@ -86,27 +86,6 @@ export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator { return Chunked({ ...options, mergeFn: setUnion }); } -// https://stackoverflow.com/a/74898678 -export function DecorateAll( - decorator: ( - target: any, - propertyKey: string, - descriptor: TypedPropertyDescriptor, - ) => TypedPropertyDescriptor | void, -) { - return (target: any) => { - const descriptors = Object.getOwnPropertyDescriptors(target.prototype); - for (const [propName, descriptor] of Object.entries(descriptors)) { - const isMethod = typeof descriptor.value == 'function' && propName !== 'constructor'; - if (!isMethod) { - continue; - } - decorator({ ...target, constructor: { ...target.constructor, name: target.name } as any }, propName, descriptor); - Object.defineProperty(target.prototype, propName, descriptor); - } - }; -} - const UUID = '00000000-0000-4000-a000-000000000000'; export const DummyValue = { @@ -128,6 +107,9 @@ export interface GenerateSqlQueries { params: unknown[]; } +export const Telemetry = (options: { enabled?: boolean }) => + SetMetadata(MetadataKey.TELEMETRY_ENABLED, options?.enabled ?? true); + /** Decorator to enable versioning/tracking of generated Sql */ export const GenerateSql = (...options: GenerateSqlQueries[]) => SetMetadata(GENERATE_SQL_KEY, options); diff --git a/server/src/enum.ts b/server/src/enum.ts index 8c11834dac..902d6635e7 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -334,6 +334,7 @@ export enum MetadataKey { SHARED_ROUTE = 'shared_route', API_KEY_SECURITY = 'api_key', EVENT_CONFIG = 'event_config', + TELEMETRY_ENABLED = 'telemetry_enabled', } export enum RouteKey { diff --git a/server/src/interfaces/telemetry.interface.ts b/server/src/interfaces/telemetry.interface.ts index 070014f2e0..688e52c21e 100644 --- a/server/src/interfaces/telemetry.interface.ts +++ b/server/src/interfaces/telemetry.interface.ts @@ -1,4 +1,5 @@ import { MetricOptions } from '@opentelemetry/api'; +import { ClassConstructor } from 'class-transformer'; export const ITelemetryRepository = 'ITelemetryRepository'; @@ -14,6 +15,7 @@ export interface IMetricGroupRepository { } export interface ITelemetryRepository { + setup(options: { repositories: ClassConstructor[] }): void; api: IMetricGroupRepository; host: IMetricGroupRepository; jobs: IMetricGroupRepository; diff --git a/server/src/repositories/access.repository.ts b/server/src/repositories/access.repository.ts index f6921ffe27..f3cbf392db 100644 --- a/server/src/repositories/access.repository.ts +++ b/server/src/repositories/access.repository.ts @@ -15,7 +15,6 @@ import { StackEntity } from 'src/entities/stack.entity'; import { TagEntity } from 'src/entities/tag.entity'; import { AlbumUserRole } from 'src/enum'; import { IAccessRepository } from 'src/interfaces/access.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { Brackets, In, Repository } from 'typeorm'; type IActivityAccess = IAccessRepository['activity']; @@ -29,7 +28,6 @@ type IStackAccess = IAccessRepository['stack']; type ITagAccess = IAccessRepository['tag']; type ITimelineAccess = IAccessRepository['timeline']; -@Instrumentation() @Injectable() class ActivityAccess implements IActivityAccess { constructor( diff --git a/server/src/repositories/activity.repository.ts b/server/src/repositories/activity.repository.ts index e21f746483..0f0a0cb60e 100644 --- a/server/src/repositories/activity.repository.ts +++ b/server/src/repositories/activity.repository.ts @@ -3,7 +3,6 @@ import { InjectRepository } from '@nestjs/typeorm'; import { DummyValue, GenerateSql } from 'src/decorators'; import { ActivityEntity } from 'src/entities/activity.entity'; import { IActivityRepository } from 'src/interfaces/activity.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { IsNull, Repository } from 'typeorm'; export interface ActivitySearch { @@ -13,7 +12,6 @@ export interface ActivitySearch { isLiked?: boolean; } -@Instrumentation() @Injectable() export class ActivityRepository implements IActivityRepository { constructor(@InjectRepository(ActivityEntity) private repository: Repository) {} diff --git a/server/src/repositories/album-user.repository.ts b/server/src/repositories/album-user.repository.ts index 7fd18711aa..9328ea8cfc 100644 --- a/server/src/repositories/album-user.repository.ts +++ b/server/src/repositories/album-user.repository.ts @@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { AlbumUserEntity } from 'src/entities/album-user.entity'; import { AlbumPermissionId, IAlbumUserRepository } from 'src/interfaces/album-user.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class AlbumUserRepository implements IAlbumUserRepository { constructor(@InjectRepository(AlbumUserEntity) private repository: Repository) {} diff --git a/server/src/repositories/album.repository.ts b/server/src/repositories/album.repository.ts index f7b4cb44aa..8b7565e318 100644 --- a/server/src/repositories/album.repository.ts +++ b/server/src/repositories/album.repository.ts @@ -4,7 +4,6 @@ import { Chunked, ChunkedArray, ChunkedSet, DummyValue, GenerateSql } from 'src/ import { AlbumEntity } from 'src/entities/album.entity'; import { AssetEntity } from 'src/entities/asset.entity'; import { AlbumAssetCount, AlbumInfoOptions, IAlbumRepository } from 'src/interfaces/album.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { DataSource, EntityManager, @@ -23,7 +22,6 @@ const withoutDeletedUsers = (album: T) => { return album; }; -@Instrumentation() @Injectable() export class AlbumRepository implements IAlbumRepository { constructor( diff --git a/server/src/repositories/api-key.repository.ts b/server/src/repositories/api-key.repository.ts index 5178039177..bb37390de1 100644 --- a/server/src/repositories/api-key.repository.ts +++ b/server/src/repositories/api-key.repository.ts @@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm'; import { DummyValue, GenerateSql } from 'src/decorators'; import { APIKeyEntity } from 'src/entities/api-key.entity'; import { IKeyRepository } from 'src/interfaces/api-key.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class ApiKeyRepository implements IKeyRepository { constructor(@InjectRepository(APIKeyEntity) private repository: Repository) {} diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index fd47a976a5..6080e943e4 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -29,7 +29,6 @@ import { } from 'src/interfaces/asset.interface'; import { AssetSearchOptions, SearchExploreItem } from 'src/interfaces/search.interface'; import { searchAssetBuilder } from 'src/utils/database'; -import { Instrumentation } from 'src/utils/instrumentation'; import { Paginated, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination'; import { Brackets, @@ -54,7 +53,6 @@ const dateTrunc = (options: TimeBucketOptions) => truncateMap[options.size] }', (asset."localDateTime" at time zone 'UTC')) at time zone 'UTC')::timestamptz`; -@Instrumentation() @Injectable() export class AssetRepository implements IAssetRepository { constructor( diff --git a/server/src/repositories/audit.repository.ts b/server/src/repositories/audit.repository.ts index deb0d0f6f1..ac73c3a8b9 100644 --- a/server/src/repositories/audit.repository.ts +++ b/server/src/repositories/audit.repository.ts @@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { AuditEntity } from 'src/entities/audit.entity'; import { AuditSearch, IAuditRepository } from 'src/interfaces/audit.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { In, LessThan, MoreThan, Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class AuditRepository implements IAuditRepository { constructor(@InjectRepository(AuditEntity) private repository: Repository) {} diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index 44b8c7b605..fabccd7846 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -1,6 +1,7 @@ import { Injectable } from '@nestjs/common'; import { join } from 'node:path'; import { citiesFile, excludePaths } from 'src/constants'; +import { Telemetry } from 'src/decorators'; import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum'; import { EnvData, IConfigRepository } from 'src/interfaces/config.interface'; import { DatabaseExtension } from 'src/interfaces/database.interface'; @@ -74,9 +75,6 @@ const getEnv = (): EnvData => { const repoMetrics = parseBoolean(process.env.IMMICH_IO_METRICS, globalEnabled); const jobMetrics = parseBoolean(process.env.IMMICH_JOB_METRICS, globalEnabled); const telemetryEnabled = globalEnabled || hostMetrics || apiMetrics || repoMetrics || jobMetrics; - if (!telemetryEnabled && process.env.OTEL_SDK_DISABLED === undefined) { - process.env.OTEL_SDK_DISABLED = 'true'; - } return { host: process.env.IMMICH_HOST, @@ -186,6 +184,7 @@ const getEnv = (): EnvData => { let cached: EnvData | undefined; @Injectable() +@Telemetry({ enabled: false }) export class ConfigRepository implements IConfigRepository { getEnv(): EnvData { if (!cached) { diff --git a/server/src/repositories/crypto.repository.ts b/server/src/repositories/crypto.repository.ts index 72e75ef174..ee25609fec 100644 --- a/server/src/repositories/crypto.repository.ts +++ b/server/src/repositories/crypto.repository.ts @@ -3,9 +3,7 @@ import { compareSync, hash } from 'bcrypt'; import { createHash, createPublicKey, createVerify, randomBytes, randomUUID } from 'node:crypto'; import { createReadStream } from 'node:fs'; import { ICryptoRepository } from 'src/interfaces/crypto.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; -@Instrumentation() @Injectable() export class CryptoRepository implements ICryptoRepository { randomUUID() { diff --git a/server/src/repositories/database.repository.ts b/server/src/repositories/database.repository.ts index 547f03fc20..b5e2edfdea 100644 --- a/server/src/repositories/database.repository.ts +++ b/server/src/repositories/database.repository.ts @@ -15,11 +15,9 @@ import { VectorUpdateResult, } from 'src/interfaces/database.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { isValidInteger } from 'src/validation'; import { DataSource, EntityManager, QueryRunner } from 'typeorm'; -@Instrumentation() @Injectable() export class DatabaseRepository implements IDatabaseRepository { private vectorExtension: VectorExtension; diff --git a/server/src/repositories/event.repository.ts b/server/src/repositories/event.repository.ts index cb58d56b2a..bb265196f9 100644 --- a/server/src/repositories/event.repository.ts +++ b/server/src/repositories/event.repository.ts @@ -24,7 +24,6 @@ import { } from 'src/interfaces/event.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { AuthService } from 'src/services/auth.service'; -import { Instrumentation } from 'src/utils/instrumentation'; import { handlePromiseError } from 'src/utils/misc'; type EmitHandlers = Partial<{ [T in EmitEvent]: Array> }>; @@ -37,7 +36,6 @@ type Item = { label: string; }; -@Instrumentation() @WebSocketGateway({ cors: true, path: '/api/socket.io', diff --git a/server/src/repositories/job.repository.ts b/server/src/repositories/job.repository.ts index 846b6dc9cd..2b783e7d2f 100644 --- a/server/src/repositories/job.repository.ts +++ b/server/src/repositories/job.repository.ts @@ -17,7 +17,6 @@ import { QueueStatus, } from 'src/interfaces/job.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; export const JOBS_TO_QUEUE: Record = { // misc @@ -99,7 +98,6 @@ export const JOBS_TO_QUEUE: Record = { [JobName.QUEUE_TRASH_EMPTY]: QueueName.BACKGROUND_TASK, }; -@Instrumentation() @Injectable() export class JobRepository implements IJobRepository { private workers: Partial> = {}; diff --git a/server/src/repositories/library.repository.ts b/server/src/repositories/library.repository.ts index 36fb4b9217..1446395854 100644 --- a/server/src/repositories/library.repository.ts +++ b/server/src/repositories/library.repository.ts @@ -4,11 +4,9 @@ import { DummyValue, GenerateSql } from 'src/decorators'; import { LibraryStatsResponseDto } from 'src/dtos/library.dto'; import { LibraryEntity } from 'src/entities/library.entity'; import { ILibraryRepository } from 'src/interfaces/library.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { IsNull, Not } from 'typeorm'; import { Repository } from 'typeorm/repository/Repository.js'; -@Instrumentation() @Injectable() export class LibraryRepository implements ILibraryRepository { constructor(@InjectRepository(LibraryEntity) private repository: Repository) {} diff --git a/server/src/repositories/logger.repository.ts b/server/src/repositories/logger.repository.ts index 2023cd6c43..4f1d3cac22 100644 --- a/server/src/repositories/logger.repository.ts +++ b/server/src/repositories/logger.repository.ts @@ -1,6 +1,7 @@ import { ConsoleLogger, Inject, Injectable, Scope } from '@nestjs/common'; import { isLogLevelEnabled } from '@nestjs/common/services/utils/is-log-level-enabled.util'; import { ClsService } from 'nestjs-cls'; +import { Telemetry } from 'src/decorators'; import { LogLevel } from 'src/enum'; import { IConfigRepository } from 'src/interfaces/config.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; @@ -17,6 +18,7 @@ enum LogColor { } @Injectable({ scope: Scope.TRANSIENT }) +@Telemetry({ enabled: false }) export class LoggerRepository extends ConsoleLogger implements ILoggerRepository { private static logLevels: LogLevel[] = [LogLevel.LOG, LogLevel.WARN, LogLevel.ERROR, LogLevel.FATAL]; private noColor: boolean; diff --git a/server/src/repositories/machine-learning.repository.ts b/server/src/repositories/machine-learning.repository.ts index b9404022ef..74b17ca6a7 100644 --- a/server/src/repositories/machine-learning.repository.ts +++ b/server/src/repositories/machine-learning.repository.ts @@ -12,11 +12,9 @@ import { ModelTask, ModelType, } from 'src/interfaces/machine-learning.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; const errorPrefix = 'Machine learning request'; -@Instrumentation() @Injectable() export class MachineLearningRepository implements IMachineLearningRepository { private async predict(url: string, payload: ModelPayload, config: MachineLearningRequest): Promise { diff --git a/server/src/repositories/map.repository.ts b/server/src/repositories/map.repository.ts index 3e5c499f41..7ad94016e8 100644 --- a/server/src/repositories/map.repository.ts +++ b/server/src/repositories/map.repository.ts @@ -20,11 +20,9 @@ import { } from 'src/interfaces/map.interface'; import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface'; import { OptionalBetween } from 'src/utils/database'; -import { Instrumentation } from 'src/utils/instrumentation'; import { DataSource, In, IsNull, Not, QueryRunner, Repository } from 'typeorm'; import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js'; -@Instrumentation() @Injectable() export class MapRepository implements IMapRepository { constructor( diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts index 0777ca3479..64cfc540e5 100644 --- a/server/src/repositories/media.repository.ts +++ b/server/src/repositories/media.repository.ts @@ -17,7 +17,6 @@ import { TranscodeCommand, VideoInfo, } from 'src/interfaces/media.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { handlePromiseError } from 'src/utils/misc'; const probe = (input: string, options: string[]): Promise => @@ -36,7 +35,6 @@ type ProgressEvent = { percent?: number; }; -@Instrumentation() @Injectable() export class MediaRepository implements IMediaRepository { constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) { diff --git a/server/src/repositories/memory.repository.ts b/server/src/repositories/memory.repository.ts index e9b4532fe9..3c2a1ae191 100644 --- a/server/src/repositories/memory.repository.ts +++ b/server/src/repositories/memory.repository.ts @@ -3,10 +3,8 @@ import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; import { Chunked, ChunkedSet, DummyValue, GenerateSql } from 'src/decorators'; import { MemoryEntity } from 'src/entities/memory.entity'; import { IMemoryRepository } from 'src/interfaces/memory.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { DataSource, In, Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class MemoryRepository implements IMemoryRepository { constructor( diff --git a/server/src/repositories/metadata.repository.ts b/server/src/repositories/metadata.repository.ts index dc2a4cdf9b..81c1b35e15 100644 --- a/server/src/repositories/metadata.repository.ts +++ b/server/src/repositories/metadata.repository.ts @@ -3,9 +3,7 @@ import { DefaultReadTaskOptions, ExifTool, Tags } from 'exiftool-vendored'; import geotz from 'geo-tz'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { IMetadataRepository, ImmichTags } from 'src/interfaces/metadata.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; -@Instrumentation() @Injectable() export class MetadataRepository implements IMetadataRepository { private exiftool = new ExifTool({ diff --git a/server/src/repositories/move.repository.ts b/server/src/repositories/move.repository.ts index 45fd446526..16d9004014 100644 --- a/server/src/repositories/move.repository.ts +++ b/server/src/repositories/move.repository.ts @@ -4,10 +4,8 @@ import { DummyValue, GenerateSql } from 'src/decorators'; import { MoveEntity } from 'src/entities/move.entity'; import { PathType } from 'src/enum'; import { IMoveRepository, MoveCreate } from 'src/interfaces/move.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class MoveRepository implements IMoveRepository { constructor(@InjectRepository(MoveEntity) private repository: Repository) {} diff --git a/server/src/repositories/notification.repository.ts b/server/src/repositories/notification.repository.ts index 9814a7bd5e..293a80576f 100644 --- a/server/src/repositories/notification.repository.ts +++ b/server/src/repositories/notification.repository.ts @@ -15,9 +15,7 @@ import { SendEmailResponse, SmtpOptions, } from 'src/interfaces/notification.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; -@Instrumentation() @Injectable() export class NotificationRepository implements INotificationRepository { constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) { diff --git a/server/src/repositories/oauth.repository.ts b/server/src/repositories/oauth.repository.ts index adde7099d0..ed038f0137 100644 --- a/server/src/repositories/oauth.repository.ts +++ b/server/src/repositories/oauth.repository.ts @@ -2,9 +2,7 @@ import { Inject, Injectable, InternalServerErrorException } from '@nestjs/common import { custom, generators, Issuer } from 'openid-client'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { IOAuthRepository, OAuthConfig, OAuthProfile } from 'src/interfaces/oauth.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; -@Instrumentation() @Injectable() export class OAuthRepository implements IOAuthRepository { constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) { diff --git a/server/src/repositories/partner.repository.ts b/server/src/repositories/partner.repository.ts index e0c8998dbf..6b11a4e31e 100644 --- a/server/src/repositories/partner.repository.ts +++ b/server/src/repositories/partner.repository.ts @@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { PartnerEntity } from 'src/entities/partner.entity'; import { IPartnerRepository, PartnerIds } from 'src/interfaces/partner.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { DeepPartial, Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class PartnerRepository implements IPartnerRepository { constructor(@InjectRepository(PartnerEntity) private repository: Repository) {} diff --git a/server/src/repositories/person.repository.ts b/server/src/repositories/person.repository.ts index c62c4b8739..56116d7b3b 100644 --- a/server/src/repositories/person.repository.ts +++ b/server/src/repositories/person.repository.ts @@ -20,11 +20,9 @@ import { UnassignFacesOptions, UpdateFacesData, } from 'src/interfaces/person.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { Paginated, PaginationOptions, paginate, paginatedBuilder } from 'src/utils/pagination'; import { DataSource, FindManyOptions, FindOptionsRelations, FindOptionsSelect, In, Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class PersonRepository implements IPersonRepository { constructor( diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index 882a2634bd..b5969beecb 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -23,12 +23,10 @@ import { SmartSearchOptions, } from 'src/interfaces/search.interface'; import { asVector, searchAssetBuilder } from 'src/utils/database'; -import { Instrumentation } from 'src/utils/instrumentation'; import { Paginated, PaginationResult, paginatedBuilder } from 'src/utils/pagination'; import { isValidInteger } from 'src/validation'; import { Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class SearchRepository implements ISearchRepository { private vectorExtension: VectorExtension; diff --git a/server/src/repositories/server-info.repository.ts b/server/src/repositories/server-info.repository.ts index 1936ecdb61..b4a4652871 100644 --- a/server/src/repositories/server-info.repository.ts +++ b/server/src/repositories/server-info.repository.ts @@ -7,7 +7,6 @@ import sharp from 'sharp'; import { IConfigRepository } from 'src/interfaces/config.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { GitHubRelease, IServerInfoRepository, ServerBuildVersions } from 'src/interfaces/server-info.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; const exec = promisify(execCallback); const maybeFirstLine = async (command: string): Promise => { @@ -34,7 +33,6 @@ const getLockfileVersion = (name: string, lockfile?: BuildLockfile) => { return item?.version; }; -@Instrumentation() @Injectable() export class ServerInfoRepository implements IServerInfoRepository { constructor( diff --git a/server/src/repositories/session.repository.ts b/server/src/repositories/session.repository.ts index a4b55a19d7..3a0af1ef69 100644 --- a/server/src/repositories/session.repository.ts +++ b/server/src/repositories/session.repository.ts @@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm'; import { DummyValue, GenerateSql } from 'src/decorators'; import { SessionEntity } from 'src/entities/session.entity'; import { ISessionRepository, SessionSearchOptions } from 'src/interfaces/session.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { LessThanOrEqual, Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class SessionRepository implements ISessionRepository { constructor(@InjectRepository(SessionEntity) private repository: Repository) {} diff --git a/server/src/repositories/shared-link.repository.ts b/server/src/repositories/shared-link.repository.ts index 48dbb3ab90..1dfde99a75 100644 --- a/server/src/repositories/shared-link.repository.ts +++ b/server/src/repositories/shared-link.repository.ts @@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm'; import { DummyValue, GenerateSql } from 'src/decorators'; import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { ISharedLinkRepository } from 'src/interfaces/shared-link.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class SharedLinkRepository implements ISharedLinkRepository { constructor(@InjectRepository(SharedLinkEntity) private repository: Repository) {} diff --git a/server/src/repositories/stack.repository.ts b/server/src/repositories/stack.repository.ts index f23a1c9a9c..ae1a7f70d4 100644 --- a/server/src/repositories/stack.repository.ts +++ b/server/src/repositories/stack.repository.ts @@ -3,10 +3,8 @@ import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; import { AssetEntity } from 'src/entities/asset.entity'; import { StackEntity } from 'src/entities/stack.entity'; import { IStackRepository, StackSearch } from 'src/interfaces/stack.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { DataSource, In, Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class StackRepository implements IStackRepository { constructor( diff --git a/server/src/repositories/storage.repository.ts b/server/src/repositories/storage.repository.ts index b957449984..1ef0e9d6bf 100644 --- a/server/src/repositories/storage.repository.ts +++ b/server/src/repositories/storage.repository.ts @@ -14,10 +14,8 @@ import { ImmichZipStream, WatchEvents, } from 'src/interfaces/storage.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { mimeTypes } from 'src/utils/mime-types'; -@Instrumentation() @Injectable() export class StorageRepository implements IStorageRepository { constructor(@Inject(ILoggerRepository) private logger: ILoggerRepository) { diff --git a/server/src/repositories/system-metadata.repository.ts b/server/src/repositories/system-metadata.repository.ts index d4e58bf74a..1c6aaf0517 100644 --- a/server/src/repositories/system-metadata.repository.ts +++ b/server/src/repositories/system-metadata.repository.ts @@ -3,10 +3,8 @@ import { InjectRepository } from '@nestjs/typeorm'; import { readFile } from 'node:fs/promises'; import { SystemMetadata, SystemMetadataEntity } from 'src/entities/system-metadata.entity'; import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class SystemMetadataRepository implements ISystemMetadataRepository { constructor( diff --git a/server/src/repositories/tag.repository.ts b/server/src/repositories/tag.repository.ts index 1a5415b8db..df5f7e6e42 100644 --- a/server/src/repositories/tag.repository.ts +++ b/server/src/repositories/tag.repository.ts @@ -4,10 +4,8 @@ import { Chunked, ChunkedSet, DummyValue, GenerateSql } from 'src/decorators'; import { TagEntity } from 'src/entities/tag.entity'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { AssetTagItem, ITagRepository } from 'src/interfaces/tag.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { DataSource, In, Repository, TreeRepository } from 'typeorm'; -@Instrumentation() @Injectable() export class TagRepository implements ITagRepository { constructor( diff --git a/server/src/repositories/telemetry.repository.ts b/server/src/repositories/telemetry.repository.ts index d1dc66ae85..f450c162dc 100644 --- a/server/src/repositories/telemetry.repository.ts +++ b/server/src/repositories/telemetry.repository.ts @@ -1,7 +1,22 @@ import { Inject, Injectable } from '@nestjs/common'; +import { Reflector } from '@nestjs/core'; import { MetricOptions } from '@opentelemetry/api'; +import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks'; +import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; +import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'; +import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'; +import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core'; +import { PgInstrumentation } from '@opentelemetry/instrumentation-pg'; +import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +import { ClassConstructor } from 'class-transformer'; +import { snakeCase, startCase } from 'lodash'; import { MetricService } from 'nestjs-otel'; +import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils'; +import { serverVersion } from 'src/constants'; +import { MetadataKey } from 'src/enum'; import { IConfigRepository } from 'src/interfaces/config.interface'; +import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { IMetricGroupRepository, ITelemetryRepository, MetricGroupOptions } from 'src/interfaces/telemetry.interface'; class MetricGroupRepository implements IMetricGroupRepository { @@ -33,6 +48,43 @@ class MetricGroupRepository implements IMetricGroupRepository { } } +const aggregation = new metrics.ExplicitBucketHistogramAggregation( + [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000], + true, +); + +let instance: NodeSDK | undefined; + +export const bootstrapTelemetry = (port: number) => { + if (instance) { + throw new Error('OpenTelemetry SDK already started'); + } + instance = new NodeSDK({ + resource: new resources.Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: `immich`, + [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(), + }), + metricReader: new PrometheusExporter({ port }), + contextManager: new AsyncLocalStorageContextManager(), + instrumentations: [ + new HttpInstrumentation(), + new IORedisInstrumentation(), + new NestInstrumentation(), + new PgInstrumentation(), + ], + views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })], + }); + + instance.start(); +}; + +export const teardownTelemetry = async () => { + if (instance) { + await instance.shutdown(); + instance = undefined; + } +}; + @Injectable() export class TelemetryRepository implements ITelemetryRepository { api: MetricGroupRepository; @@ -40,8 +92,13 @@ export class TelemetryRepository implements ITelemetryRepository { jobs: MetricGroupRepository; repo: MetricGroupRepository; - constructor(metricService: MetricService, @Inject(IConfigRepository) configRepository: IConfigRepository) { - const { telemetry } = configRepository.getEnv(); + constructor( + private metricService: MetricService, + private reflect: Reflector, + @Inject(IConfigRepository) private configRepository: IConfigRepository, + @Inject(ILoggerRepository) private logger: ILoggerRepository, + ) { + const { telemetry } = this.configRepository.getEnv(); const { apiMetrics, hostMetrics, jobMetrics, repoMetrics } = telemetry; this.api = new MetricGroupRepository(metricService).configure({ enabled: apiMetrics }); @@ -49,4 +106,61 @@ export class TelemetryRepository implements ITelemetryRepository { this.jobs = new MetricGroupRepository(metricService).configure({ enabled: jobMetrics }); this.repo = new MetricGroupRepository(metricService).configure({ enabled: repoMetrics }); } + + setup({ repositories }: { repositories: ClassConstructor[] }) { + const { telemetry } = this.configRepository.getEnv(); + if (!telemetry.enabled || !telemetry.repoMetrics) { + return; + } + + for (const Repository of repositories) { + const isEnabled = this.reflect.get(MetadataKey.TELEMETRY_ENABLED, Repository) ?? true; + if (!isEnabled) { + this.logger.debug(`Telemetry disabled for ${Repository.name}`); + continue; + } + + this.wrap(Repository); + } + } + + private wrap(Repository: ClassConstructor) { + const className = Repository.name; + const descriptors = Object.getOwnPropertyDescriptors(Repository.prototype); + const unit = 'ms'; + + for (const [propName, descriptor] of Object.entries(descriptors)) { + const isMethod = typeof descriptor.value == 'function' && propName !== 'constructor'; + if (!isMethod) { + continue; + } + + const method = descriptor.value; + const propertyName = snakeCase(String(propName)); + const metricName = `${snakeCase(className).replaceAll(/_(?=(repository)|(controller)|(provider)|(service)|(module))/g, '.')}.${propertyName}.duration`; + + const histogram = this.metricService.getHistogram(metricName, { + prefix: 'immich', + description: `The elapsed time in ${unit} for the ${startCase(className)} to ${propertyName.toLowerCase()}`, + unit, + valueType: contextBase.ValueType.DOUBLE, + }); + + descriptor.value = function (...args: any[]) { + const start = performance.now(); + const result = method.apply(this, args); + + void Promise.resolve(result) + .then(() => histogram.record(performance.now() - start, {})) + .catch(() => { + // noop + }); + + return result; + }; + + copyMetadataFromFunctionToFunction(method, descriptor.value); + Object.defineProperty(Repository.prototype, propName, descriptor); + } + } } diff --git a/server/src/repositories/user.repository.ts b/server/src/repositories/user.repository.ts index c64d5a3655..6ac8536ef8 100644 --- a/server/src/repositories/user.repository.ts +++ b/server/src/repositories/user.repository.ts @@ -10,10 +10,8 @@ import { UserListFilter, UserStatsQueryResponse, } from 'src/interfaces/user.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { IsNull, Not, Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class UserRepository implements IUserRepository { constructor( diff --git a/server/src/repositories/version-history.repository.ts b/server/src/repositories/version-history.repository.ts index 26c638bd76..e32ceaf4e9 100644 --- a/server/src/repositories/version-history.repository.ts +++ b/server/src/repositories/version-history.repository.ts @@ -2,10 +2,8 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; import { VersionHistoryEntity } from 'src/entities/version-history.entity'; import { IVersionHistoryRepository } from 'src/interfaces/version-history.interface'; -import { Instrumentation } from 'src/utils/instrumentation'; import { Repository } from 'typeorm'; -@Instrumentation() @Injectable() export class VersionHistoryRepository implements IVersionHistoryRepository { constructor(@InjectRepository(VersionHistoryEntity) private repository: Repository) {} diff --git a/server/src/services/microservices.service.ts b/server/src/services/microservices.service.ts index d1d2bb8f20..275103d05c 100644 --- a/server/src/services/microservices.service.ts +++ b/server/src/services/microservices.service.ts @@ -20,7 +20,6 @@ import { TagService } from 'src/services/tag.service'; import { TrashService } from 'src/services/trash.service'; import { UserService } from 'src/services/user.service'; import { VersionService } from 'src/services/version.service'; -import { otelShutdown } from 'src/utils/instrumentation'; @Injectable() export class MicroservicesService { @@ -101,8 +100,4 @@ export class MicroservicesService { [JobName.QUEUE_TRASH_EMPTY]: () => this.trashService.handleQueueEmptyTrash(), }); } - - async onShutdown() { - await otelShutdown(); - } } diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.ts deleted file mode 100644 index bd522f27b2..0000000000 --- a/server/src/utils/instrumentation.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks'; -import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; -import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'; -import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'; -import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core'; -import { PgInstrumentation } from '@opentelemetry/instrumentation-pg'; -import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node'; -import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; -import { snakeCase, startCase } from 'lodash'; -import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils'; -import { performance } from 'node:perf_hooks'; -import { serverVersion } from 'src/constants'; -import { DecorateAll } from 'src/decorators'; -import { ConfigRepository } from 'src/repositories/config.repository'; - -const aggregation = new metrics.ExplicitBucketHistogramAggregation( - [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000], - true, -); - -const { telemetry } = new ConfigRepository().getEnv(); - -let otelSingleton: NodeSDK | undefined; - -export const otelStart = (port: number) => { - if (otelSingleton) { - throw new Error('OpenTelemetry SDK already started'); - } - otelSingleton = new NodeSDK({ - resource: new resources.Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: `immich`, - [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(), - }), - metricReader: new PrometheusExporter({ port }), - contextManager: new AsyncLocalStorageContextManager(), - instrumentations: [ - new HttpInstrumentation(), - new IORedisInstrumentation(), - new NestInstrumentation(), - new PgInstrumentation(), - ], - views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })], - }); - otelSingleton.start(); -}; - -export const otelShutdown = async () => { - if (otelSingleton) { - await otelSingleton.shutdown(); - otelSingleton = undefined; - } -}; - -function ExecutionTimeHistogram({ - description, - unit = 'ms', - valueType = contextBase.ValueType.DOUBLE, -}: contextBase.MetricOptions = {}) { - return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => { - if (!telemetry.repoMetrics || process.env.OTEL_SDK_DISABLED) { - return; - } - - const method = descriptor.value; - const className = target.constructor.name as string; - const propertyName = String(propertyKey); - const metricName = `${snakeCase(className).replaceAll(/_(?=(repository)|(controller)|(provider)|(service)|(module))/g, '.')}.${snakeCase(propertyName)}.duration`; - - const metricDescription = - description ?? - `The elapsed time in ${unit} for the ${startCase(className)} to ${startCase(propertyName).toLowerCase()}`; - - let histogram: contextBase.Histogram | undefined; - - descriptor.value = function (...args: any[]) { - const start = performance.now(); - const result = method.apply(this, args); - - void Promise.resolve(result) - .then(() => { - const end = performance.now(); - if (!histogram) { - histogram = contextBase.metrics - .getMeter('immich') - .createHistogram(metricName, { description: metricDescription, unit, valueType }); - } - histogram.record(end - start, {}); - }) - .catch(() => { - // noop - }); - - return result; - }; - - copyMetadataFromFunctionToFunction(method, descriptor.value); - }; -} - -export const Instrumentation = () => DecorateAll(ExecutionTimeHistogram()); diff --git a/server/src/workers/api.ts b/server/src/workers/api.ts index b369b56953..6451f1b792 100644 --- a/server/src/workers/api.ts +++ b/server/src/workers/api.ts @@ -11,16 +11,18 @@ import { IConfigRepository } from 'src/interfaces/config.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { WebSocketAdapter } from 'src/middleware/websocket.adapter'; import { ConfigRepository } from 'src/repositories/config.repository'; +import { bootstrapTelemetry } from 'src/repositories/telemetry.repository'; import { ApiService } from 'src/services/api.service'; import { isStartUpError } from 'src/services/storage.service'; -import { otelStart } from 'src/utils/instrumentation'; import { useSwagger } from 'src/utils/misc'; async function bootstrap() { process.title = 'immich-api'; const { telemetry, network } = new ConfigRepository().getEnv(); - otelStart(telemetry.apiPort); + if (telemetry.enabled) { + bootstrapTelemetry(telemetry.apiPort); + } const app = await NestFactory.create(ApiModule, { bufferLogs: true }); const logger = await app.resolve(ILoggerRepository); diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.ts index 7b60fb8db6..df4abb01da 100644 --- a/server/src/workers/microservices.ts +++ b/server/src/workers/microservices.ts @@ -6,12 +6,14 @@ import { IConfigRepository } from 'src/interfaces/config.interface'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { WebSocketAdapter } from 'src/middleware/websocket.adapter'; import { ConfigRepository } from 'src/repositories/config.repository'; +import { bootstrapTelemetry } from 'src/repositories/telemetry.repository'; import { isStartUpError } from 'src/services/storage.service'; -import { otelStart } from 'src/utils/instrumentation'; export async function bootstrap() { const { telemetry } = new ConfigRepository().getEnv(); - otelStart(telemetry.microservicesPort); + if (telemetry.enabled) { + bootstrapTelemetry(telemetry.microservicesPort); + } const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true }); const logger = await app.resolve(ILoggerRepository); diff --git a/server/test/repositories/telemetry.repository.mock.ts b/server/test/repositories/telemetry.repository.mock.ts index 737463065c..2d537e888a 100644 --- a/server/test/repositories/telemetry.repository.mock.ts +++ b/server/test/repositories/telemetry.repository.mock.ts @@ -12,6 +12,7 @@ const newMetricGroupMock = () => { export const newTelemetryRepositoryMock = (): Mocked => { return { + setup: vitest.fn(), api: newMetricGroupMock(), host: newMetricGroupMock(), jobs: newMetricGroupMock(),