From 2f2aecfb47e8488f5a632425d52cc05e811b98d5 Mon Sep 17 00:00:00 2001 From: Zack Pollard Date: Mon, 10 Jun 2024 17:01:04 +0100 Subject: [PATCH] fix(server): otel not working due to port conflicts after combining containers (#10078) fix: otel not working due to port conflicts after combining containers Fixes #9759 --- docker/prometheus.yml | 6 +-- server/src/config.ts | 3 +- server/src/services/microservices.service.ts | 4 +- server/src/utils/instrumentation.ts | 45 +++++++++++++------- server/src/workers/api.ts | 6 ++- server/src/workers/microservices.ts | 6 ++- 6 files changed, 44 insertions(+), 26 deletions(-) diff --git a/docker/prometheus.yml b/docker/prometheus.yml index e25bb7db6e..3b18e53450 100644 --- a/docker/prometheus.yml +++ b/docker/prometheus.yml @@ -3,10 +3,10 @@ global: evaluation_interval: 15s scrape_configs: - - job_name: immich_server + - job_name: immich_api static_configs: - targets: ['immich-server:8081'] - + - job_name: immich_microservices static_configs: - - targets: ['immich-microservices:8081'] + - targets: ['immich-server:8082'] diff --git a/server/src/config.ts b/server/src/config.ts index f1f89f3e32..624dd385ad 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -374,7 +374,8 @@ export const immichAppConfig: ConfigModuleOptions = { DB_SKIP_MIGRATIONS: Joi.boolean().optional().default(false), IMMICH_PORT: Joi.number().optional(), - IMMICH_METRICS_PORT: Joi.number().optional(), + IMMICH_API_METRICS_PORT: Joi.number().optional(), + IMMICH_MICROSERVICES_METRICS_PORT: Joi.number().optional(), IMMICH_METRICS: Joi.boolean().optional().default(false), IMMICH_HOST_METRICS: Joi.boolean().optional().default(false), diff --git a/server/src/services/microservices.service.ts b/server/src/services/microservices.service.ts index f175ed0459..8e9f6ca082 100644 --- a/server/src/services/microservices.service.ts +++ b/server/src/services/microservices.service.ts @@ -17,7 +17,7 @@ import { StorageService } from 'src/services/storage.service'; import { SystemConfigService } from 'src/services/system-config.service'; import { UserService } from 'src/services/user.service'; import { VersionService } from 'src/services/version.service'; -import { otelSDK } from 'src/utils/instrumentation'; +import { otelShutdown } from 'src/utils/instrumentation'; @Injectable() export class MicroservicesService { @@ -102,6 +102,6 @@ export class MicroservicesService { async teardown() { await this.libraryService.teardown(); await this.metadataService.teardown(); - await otelSDK.shutdown(); + await otelShutdown(); } } diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.ts index 2164fae426..484ba5901c 100644 --- a/server/src/utils/instrumentation.ts +++ b/server/src/utils/instrumentation.ts @@ -33,23 +33,36 @@ const aggregation = new metrics.ExplicitBucketHistogramAggregation( true, ); -const metricsPort = Number.parseInt(process.env.IMMICH_METRICS_PORT ?? '8081'); +let otelSingleton: NodeSDK | undefined; -export const otelSDK = new NodeSDK({ - resource: new resources.Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: `immich`, - [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(), - }), - metricReader: new PrometheusExporter({ port: metricsPort }), - contextManager: new AsyncLocalStorageContextManager(), - instrumentations: [ - new HttpInstrumentation(), - new IORedisInstrumentation(), - new NestInstrumentation(), - new PgInstrumentation(), - ], - views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })], -}); +export const otelStart = (port: number) => { + if (otelSingleton) { + throw new Error('OpenTelemetry SDK already started'); + } + otelSingleton = new NodeSDK({ + resource: new resources.Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: `immich`, + [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(), + }), + metricReader: new PrometheusExporter({ port }), + contextManager: new AsyncLocalStorageContextManager(), + instrumentations: [ + new HttpInstrumentation(), + new IORedisInstrumentation(), + new NestInstrumentation(), + new PgInstrumentation(), + ], + views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })], + }); + otelSingleton.start(); +}; + +export const otelShutdown = async () => { + if (otelSingleton) { + await otelSingleton.shutdown(); + otelSingleton = undefined; + } +}; export const otelConfig: OpenTelemetryModuleOptions = { metrics: { diff --git a/server/src/workers/api.ts b/server/src/workers/api.ts index 6c0bb5e789..dae0f83e86 100644 --- a/server/src/workers/api.ts +++ b/server/src/workers/api.ts @@ -9,14 +9,16 @@ import { envName, excludePaths, isDev, serverVersion, WEB_ROOT } from 'src/const import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { WebSocketAdapter } from 'src/middleware/websocket.adapter'; import { ApiService } from 'src/services/api.service'; -import { otelSDK } from 'src/utils/instrumentation'; +import { otelStart } from 'src/utils/instrumentation'; import { useSwagger } from 'src/utils/misc'; const host = process.env.HOST; async function bootstrap() { process.title = 'immich-api'; - otelSDK.start(); + const otelPort = Number.parseInt(process.env.IMMICH_API_METRICS_PORT ?? '8081'); + + otelStart(otelPort); const port = Number(process.env.IMMICH_PORT) || 3001; const app = await NestFactory.create(ApiModule, { bufferLogs: true }); diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.ts index 50c82ae7d9..f920e8c947 100644 --- a/server/src/workers/microservices.ts +++ b/server/src/workers/microservices.ts @@ -4,10 +4,12 @@ import { MicroservicesModule } from 'src/app.module'; import { envName, serverVersion } from 'src/constants'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { WebSocketAdapter } from 'src/middleware/websocket.adapter'; -import { otelSDK } from 'src/utils/instrumentation'; +import { otelStart } from 'src/utils/instrumentation'; export async function bootstrap() { - otelSDK.start(); + const otelPort = Number.parseInt(process.env.IMMICH_MICROSERVICES_METRICS_PORT ?? '8082'); + + otelStart(otelPort); const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true }); const logger = await app.resolve(ILoggerRepository);