From 2f2aecfb47e8488f5a632425d52cc05e811b98d5 Mon Sep 17 00:00:00 2001
From: Zack Pollard <zackpollard@ymail.com>
Date: Mon, 10 Jun 2024 17:01:04 +0100
Subject: [PATCH] fix(server): otel not working due to port conflicts after
combining containers (#10078)
fix: otel not working due to port conflicts after combining containers
Fixes #9759
---
docker/prometheus.yml | 6 +--
server/src/config.ts | 3 +-
server/src/services/microservices.service.ts | 4 +-
server/src/utils/instrumentation.ts | 45 +++++++++++++-------
server/src/workers/api.ts | 6 ++-
server/src/workers/microservices.ts | 6 ++-
6 files changed, 44 insertions(+), 26 deletions(-)
diff --git a/docker/prometheus.yml b/docker/prometheus.yml
index e25bb7db6e..3b18e53450 100644
--- a/docker/prometheus.yml
+++ b/docker/prometheus.yml
@@ -3,10 +3,10 @@ global:
evaluation_interval: 15s
scrape_configs:
- - job_name: immich_server
+ - job_name: immich_api
static_configs:
- targets: ['immich-server:8081']
-
+
- job_name: immich_microservices
static_configs:
- - targets: ['immich-microservices:8081']
+ - targets: ['immich-server:8082']
diff --git a/server/src/config.ts b/server/src/config.ts
index f1f89f3e32..624dd385ad 100644
--- a/server/src/config.ts
+++ b/server/src/config.ts
@@ -374,7 +374,8 @@ export const immichAppConfig: ConfigModuleOptions = {
DB_SKIP_MIGRATIONS: Joi.boolean().optional().default(false),
IMMICH_PORT: Joi.number().optional(),
- IMMICH_METRICS_PORT: Joi.number().optional(),
+ IMMICH_API_METRICS_PORT: Joi.number().optional(),
+ IMMICH_MICROSERVICES_METRICS_PORT: Joi.number().optional(),
IMMICH_METRICS: Joi.boolean().optional().default(false),
IMMICH_HOST_METRICS: Joi.boolean().optional().default(false),
diff --git a/server/src/services/microservices.service.ts b/server/src/services/microservices.service.ts
index f175ed0459..8e9f6ca082 100644
--- a/server/src/services/microservices.service.ts
+++ b/server/src/services/microservices.service.ts
@@ -17,7 +17,7 @@ import { StorageService } from 'src/services/storage.service';
import { SystemConfigService } from 'src/services/system-config.service';
import { UserService } from 'src/services/user.service';
import { VersionService } from 'src/services/version.service';
-import { otelSDK } from 'src/utils/instrumentation';
+import { otelShutdown } from 'src/utils/instrumentation';
@Injectable()
export class MicroservicesService {
@@ -102,6 +102,6 @@ export class MicroservicesService {
async teardown() {
await this.libraryService.teardown();
await this.metadataService.teardown();
- await otelSDK.shutdown();
+ await otelShutdown();
}
}
diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.ts
index 2164fae426..484ba5901c 100644
--- a/server/src/utils/instrumentation.ts
+++ b/server/src/utils/instrumentation.ts
@@ -33,23 +33,36 @@ const aggregation = new metrics.ExplicitBucketHistogramAggregation(
true,
);
-const metricsPort = Number.parseInt(process.env.IMMICH_METRICS_PORT ?? '8081');
+let otelSingleton: NodeSDK | undefined;
-export const otelSDK = new NodeSDK({
- resource: new resources.Resource({
- [SemanticResourceAttributes.SERVICE_NAME]: `immich`,
- [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),
- }),
- metricReader: new PrometheusExporter({ port: metricsPort }),
- contextManager: new AsyncLocalStorageContextManager(),
- instrumentations: [
- new HttpInstrumentation(),
- new IORedisInstrumentation(),
- new NestInstrumentation(),
- new PgInstrumentation(),
- ],
- views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],
-});
+export const otelStart = (port: number) => {
+ if (otelSingleton) {
+ throw new Error('OpenTelemetry SDK already started');
+ }
+ otelSingleton = new NodeSDK({
+ resource: new resources.Resource({
+ [SemanticResourceAttributes.SERVICE_NAME]: `immich`,
+ [SemanticResourceAttributes.SERVICE_VERSION]: serverVersion.toString(),
+ }),
+ metricReader: new PrometheusExporter({ port }),
+ contextManager: new AsyncLocalStorageContextManager(),
+ instrumentations: [
+ new HttpInstrumentation(),
+ new IORedisInstrumentation(),
+ new NestInstrumentation(),
+ new PgInstrumentation(),
+ ],
+ views: [new metrics.View({ aggregation, instrumentName: '*', instrumentUnit: 'ms' })],
+ });
+ otelSingleton.start();
+};
+
+export const otelShutdown = async () => {
+ if (otelSingleton) {
+ await otelSingleton.shutdown();
+ otelSingleton = undefined;
+ }
+};
export const otelConfig: OpenTelemetryModuleOptions = {
metrics: {
diff --git a/server/src/workers/api.ts b/server/src/workers/api.ts
index 6c0bb5e789..dae0f83e86 100644
--- a/server/src/workers/api.ts
+++ b/server/src/workers/api.ts
@@ -9,14 +9,16 @@ import { envName, excludePaths, isDev, serverVersion, WEB_ROOT } from 'src/const
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
import { ApiService } from 'src/services/api.service';
-import { otelSDK } from 'src/utils/instrumentation';
+import { otelStart } from 'src/utils/instrumentation';
import { useSwagger } from 'src/utils/misc';
const host = process.env.HOST;
async function bootstrap() {
process.title = 'immich-api';
- otelSDK.start();
+ const otelPort = Number.parseInt(process.env.IMMICH_API_METRICS_PORT ?? '8081');
+
+ otelStart(otelPort);
const port = Number(process.env.IMMICH_PORT) || 3001;
const app = await NestFactory.create<NestExpressApplication>(ApiModule, { bufferLogs: true });
diff --git a/server/src/workers/microservices.ts b/server/src/workers/microservices.ts
index 50c82ae7d9..f920e8c947 100644
--- a/server/src/workers/microservices.ts
+++ b/server/src/workers/microservices.ts
@@ -4,10 +4,12 @@ import { MicroservicesModule } from 'src/app.module';
import { envName, serverVersion } from 'src/constants';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
-import { otelSDK } from 'src/utils/instrumentation';
+import { otelStart } from 'src/utils/instrumentation';
export async function bootstrap() {
- otelSDK.start();
+ const otelPort = Number.parseInt(process.env.IMMICH_MICROSERVICES_METRICS_PORT ?? '8082');
+
+ otelStart(otelPort);
const app = await NestFactory.create(MicroservicesModule, { bufferLogs: true });
const logger = await app.resolve(ILoggerRepository);