fix: handle the case when CUSTOM_OTELCOL_CONFIG_FILE is not specified (#1080)

plus fixing startup issue when the team isn't created yet

Author: wrn14897

.changeset/happy-tips-cough.md

@@ -0,0 +1,7 @@
+---
+"@hyperdx/api": patch
+"@hyperdx/app": patch
+"@hyperdx/common-utils": patch
+---
+
+fix: handle the case when `CUSTOM_OTELCOL_CONFIG_FILE` is not specified

docker/otel-collector/Dockerfile

@@ -8,8 +8,10 @@ FROM alpine:latest@sha256:a8560b36e8b8210634f77d9f7f9efd7ffa463e380b75e2e74aff45
 ARG USER_UID=10001
 ARG USER_GID=10001
 
-# Install certs, create user/group, and make the writable data dir
+# Install certs, gomplate for templating, create user/group, and make the writable data dir
 RUN apk add --no-cache ca-certificates && \
+  wget -O /usr/local/bin/gomplate https://github.com/hairyhenderson/gomplate/releases/download/v4.3.3/gomplate_linux-amd64 && \
+  chmod +x /usr/local/bin/gomplate && \
   addgroup -S -g ${USER_GID} otel && \
   adduser  -S -u ${USER_UID} -G otel otel && \
   install -d -m 0777 -o ${USER_UID} -g ${USER_GID} /etc/otel/supervisor-data
@@ -27,7 +29,7 @@ COPY --chmod=755 ./log-rotator.sh /log-rotator.sh
 FROM base AS dev
 
 COPY ./config.yaml /etc/otelcol-contrib/config.yaml
-COPY ./supervisor_docker.yaml /etc/otel/supervisor.yaml
+COPY ./supervisor_docker.yaml.tmpl /etc/otel/supervisor.yaml.tmpl
 
 EXPOSE 4317 4318 13133
 
@@ -38,7 +40,7 @@ CMD ["--config", "/etc/otel/supervisor.yaml"]
 FROM base AS prod
 
 COPY ./config.yaml /etc/otelcol-contrib/config.yaml
-COPY ./supervisor_docker.yaml /etc/otel/supervisor.yaml
+COPY ./supervisor_docker.yaml.tmpl /etc/otel/supervisor.yaml.tmpl
 
 EXPOSE 4317 4318 13133
 

docker/otel-collector/entrypoint.sh

@@ -5,5 +5,19 @@ set -e
 # Arguments: log_file_path [max_size_mb] [max_archives] [check_interval_seconds]
 /log-rotator.sh /etc/otel/supervisor-data/agent.log 16 1 60 &
 
+# Render the supervisor config template using gomplate
+# Write to supervisor-data directory which has proper permissions for otel user
+gomplate -f /etc/otel/supervisor.yaml.tmpl -o /etc/otel/supervisor-data/supervisor-runtime.yaml
+
+# Log the configuration being used
+if [ -n "$CUSTOM_OTELCOL_CONFIG_FILE" ]; then
+    echo "Using custom OTEL config file: $CUSTOM_OTELCOL_CONFIG_FILE"
+else
+    echo "CUSTOM_OTELCOL_CONFIG_FILE not set, using default configuration"
+fi
+
+# Update the command arguments to use the rendered config file
+set -- "$1" --config /etc/otel/supervisor-data/supervisor-runtime.yaml
+
 # Execute the supervisor with all passed arguments
 exec "$@"

…er/otel-collector/supervisor_docker.yaml …el-collector/supervisor_docker.yaml.tmpldocker/otel-collector/supervisor_docker.yaml renamed to docker/otel-collector/supervisor_docker.yaml.tmpl docker/otel-collector/supervisor_docker.yaml renamed to docker/otel-collector/supervisor_docker.yaml.tmpl

@@ -1,6 +1,6 @@
 # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/cmd/opampsupervisor/specification/README.md#supervisor-configuration
 server:
-  endpoint: ${OPAMP_SERVER_URL}/v1/opamp
+  endpoint: {{ getenv "OPAMP_SERVER_URL" }}/v1/opamp
   tls:
     # Disable verification to test locally.
     # Don't do this in production.
@@ -20,10 +20,14 @@ agent:
   executable: /otelcontribcol
   config_files:
     - /etc/otelcol-contrib/config.yaml
-    - ${env:CUSTOM_OTELCOL_CONFIG_FILE}
+{{- if getenv "CUSTOM_OTELCOL_CONFIG_FILE" }}
+    - {{ getenv "CUSTOM_OTELCOL_CONFIG_FILE" }}
+{{- end }}
   args:
-    - ${env:OTEL_AGENT_FEATURE_GATE_ARG}
-  passthrough_logs: ${env:OTEL_SUPERVISOR_PASSTHROUGH_LOGS}
+{{- if getenv "OTEL_AGENT_FEATURE_GATE_ARG" }}
+    - {{ getenv "OTEL_AGENT_FEATURE_GATE_ARG" }}
+{{- end }}
+  passthrough_logs: {{ getenv "OTEL_SUPERVISOR_PASSTHROUGH_LOGS" | default "false" }}
 
 storage:
   directory: /etc/otel/supervisor-data/

packages/api/src/opamp/controllers/opampController.ts

@@ -106,9 +106,9 @@ type CollectorConfig = {
     extensions: string[];
     pipelines: {
       [key: string]: {
-        receivers?: string[];
+        receivers: string[];
         processors?: string[];
-        exporters?: string[];
+        exporters: string[];
       };
     };
   };
@@ -119,129 +119,139 @@ export const buildOtelCollectorConfig = (teams: ITeam[]): CollectorConfig => {
   const collectorAuthenticationEnforced =
     teams[0]?.collectorAuthenticationEnforced;
 
-  if (apiKeys && apiKeys.length > 0) {
-    // Build full configuration with all team API keys
-    const otelCollectorConfig: CollectorConfig = {
-      extensions: {},
-      receivers: {
-        'otlp/hyperdx': {
-          protocols: {
-            grpc: {
-              endpoint: '0.0.0.0:4317',
-              include_metadata: true,
-            },
-            http: {
-              endpoint: '0.0.0.0:4318',
-              cors: {
-                allowed_origins: ['*'],
-                allowed_headers: ['*'],
-              },
-              include_metadata: true,
-            },
+  const otelCollectorConfig: CollectorConfig = {
+    extensions: {},
+    receivers: {
+      nop: null,
+      'otlp/hyperdx': {
+        protocols: {
+          grpc: {
+            endpoint: '0.0.0.0:4317',
+            include_metadata: true,
           },
-        },
-        prometheus: {
-          config: {
-            scrape_configs: [
-              {
-                job_name: 'otelcol',
-                scrape_interval: '30s',
-                static_configs: [
-                  {
-                    targets: [
-                      '0.0.0.0:8888',
-                      '${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}',
-                    ],
-                  },
-                ],
-              },
-            ],
+          http: {
+            endpoint: '0.0.0.0:4318',
+            cors: {
+              allowed_origins: ['*'],
+              allowed_headers: ['*'],
+            },
+            include_metadata: true,
           },
         },
-        fluentforward: {
-          endpoint: '0.0.0.0:24225',
-        },
       },
-      connectors: {
-        'routing/logs': {
-          default_pipelines: ['logs/out-default'],
-          error_mode: 'ignore',
-          table: [
+      prometheus: {
+        config: {
+          scrape_configs: [
             {
-              context: 'log',
-              statement:
-                'route() where IsMatch(attributes["rr-web.event"], ".*")',
-              pipelines: ['logs/out-rrweb'],
+              job_name: 'otelcol',
+              scrape_interval: '30s',
+              static_configs: [
+                {
+                  targets: [
+                    '0.0.0.0:8888',
+                    '${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}',
+                  ],
+                },
+              ],
             },
           ],
         },
       },
-      exporters: {
-        debug: {
-          verbosity: 'detailed',
-          sampling_initial: 5,
-          sampling_thereafter: 200,
-        },
-        'clickhouse/rrweb': {
-          endpoint: '${env:CLICKHOUSE_ENDPOINT}',
-          database: '${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}',
-          username: '${env:CLICKHOUSE_USER}',
-          password: '${env:CLICKHOUSE_PASSWORD}',
-          ttl: '720h',
-          logs_table_name: 'hyperdx_sessions',
-          timeout: '5s',
-          retry_on_failure: {
-            enabled: true,
-            initial_interval: '5s',
-            max_interval: '30s',
-            max_elapsed_time: '300s',
+      fluentforward: {
+        endpoint: '0.0.0.0:24225',
+      },
+    },
+    connectors: {
+      'routing/logs': {
+        default_pipelines: ['logs/out-default'],
+        error_mode: 'ignore',
+        table: [
+          {
+            context: 'log',
+            statement:
+              'route() where IsMatch(attributes["rr-web.event"], ".*")',
+            pipelines: ['logs/out-rrweb'],
           },
+        ],
+      },
+    },
+    exporters: {
+      debug: {
+        verbosity: 'detailed',
+        sampling_initial: 5,
+        sampling_thereafter: 200,
+      },
+      'clickhouse/rrweb': {
+        endpoint: '${env:CLICKHOUSE_ENDPOINT}',
+        database: '${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}',
+        username: '${env:CLICKHOUSE_USER}',
+        password: '${env:CLICKHOUSE_PASSWORD}',
+        ttl: '720h',
+        logs_table_name: 'hyperdx_sessions',
+        timeout: '5s',
+        retry_on_failure: {
+          enabled: true,
+          initial_interval: '5s',
+          max_interval: '30s',
+          max_elapsed_time: '300s',
         },
-        clickhouse: {
-          endpoint: '${env:CLICKHOUSE_ENDPOINT}',
-          database: '${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}',
-          username: '${env:CLICKHOUSE_USER}',
-          password: '${env:CLICKHOUSE_PASSWORD}',
-          ttl: '720h',
-          timeout: '5s',
-          retry_on_failure: {
-            enabled: true,
-            initial_interval: '5s',
-            max_interval: '30s',
-            max_elapsed_time: '300s',
-          },
+      },
+      clickhouse: {
+        endpoint: '${env:CLICKHOUSE_ENDPOINT}',
+        database: '${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}',
+        username: '${env:CLICKHOUSE_USER}',
+        password: '${env:CLICKHOUSE_PASSWORD}',
+        ttl: '720h',
+        timeout: '5s',
+        retry_on_failure: {
+          enabled: true,
+          initial_interval: '5s',
+          max_interval: '30s',
+          max_elapsed_time: '300s',
         },
       },
-      service: {
-        extensions: [],
-        pipelines: {
-          traces: {
-            receivers: ['otlp/hyperdx'],
-            processors: ['memory_limiter', 'batch'],
-            exporters: ['clickhouse'],
-          },
-          metrics: {
-            receivers: ['otlp/hyperdx', 'prometheus'],
-            processors: ['memory_limiter', 'batch'],
-            exporters: ['clickhouse'],
-          },
-          'logs/in': {
-            receivers: ['otlp/hyperdx', 'fluentforward'],
-            exporters: ['routing/logs'],
-          },
-          'logs/out-default': {
-            receivers: ['routing/logs'],
-            processors: ['memory_limiter', 'transform', 'batch'],
-            exporters: ['clickhouse'],
-          },
-          'logs/out-rrweb': {
-            receivers: ['routing/logs'],
-            processors: ['memory_limiter', 'batch'],
-            exporters: ['clickhouse/rrweb'],
-          },
+    },
+    service: {
+      extensions: [],
+      pipelines: {
+        traces: {
+          receivers: ['nop'],
+          processors: ['memory_limiter', 'batch'],
+          exporters: ['clickhouse'],
+        },
+        metrics: {
+          // TODO: prometheus needs to be authenticated
+          receivers: ['prometheus'],
+          processors: ['memory_limiter', 'batch'],
+          exporters: ['clickhouse'],
+        },
+        'logs/in': {
+          // TODO: fluentforward needs to be authenticated
+          receivers: ['fluentforward'],
+          exporters: ['routing/logs'],
+        },
+        'logs/out-default': {
+          receivers: ['routing/logs'],
+          processors: ['memory_limiter', 'transform', 'batch'],
+          exporters: ['clickhouse'],
+        },
+        'logs/out-rrweb': {
+          receivers: ['routing/logs'],
+          processors: ['memory_limiter', 'batch'],
+          exporters: ['clickhouse/rrweb'],
         },
       },
-    };
+    },
+  };
+  if (apiKeys && apiKeys.length > 0) {
+    // attach otlp/hyperdx receiver
+    otelCollectorConfig.service.pipelines.traces.receivers.push('otlp/hyperdx');
+    otelCollectorConfig.service.pipelines.metrics.receivers.push(
+      'otlp/hyperdx',
+    );
+    otelCollectorConfig.service.pipelines['logs/in'].receivers.push(
+      'otlp/hyperdx',
+    );
 
     if (collectorAuthenticationEnforced) {
       if (otelCollectorConfig.receivers['otlp/hyperdx'] == null) {
@@ -261,36 +271,9 @@ export const buildOtelCollectorConfig = (teams: ITeam[]): CollectorConfig => {
       };
       otelCollectorConfig.service.extensions = ['bearertokenauth/hyperdx'];
     }
-
-    return otelCollectorConfig;
   }
 
-  // If no apiKeys are found, return NOP config
-  // This is later merged with otel-collector/config.yaml
-  // we need to instantiate a valid config so the collector
-  // can at least start up
-  return {
-    extensions: {},
-    receivers: {
-      nop: null,
-    },
-    connectors: {},
-    exporters: {},
-    service: {
-      extensions: [],
-      pipelines: {
-        traces: {
-          receivers: ['nop'],
-        },
-        metrics: {
-          receivers: ['nop'],
-        },
-        logs: {
-          receivers: ['nop'],
-        },
-      },
-    },
-  };
+  return otelCollectorConfig;
 };
 
 export class OpampController {