Fixes and tests for telemetry (#4265)

* Add integration tests for telemetry metrics

* Allow enabling metrics in CI

* Try to reduce some flake

* Fix config key used in tracer settings

* Ensure we record profiler configuration in config

* Add configuration key for enabling debug mode

* Allow setting the debug flag on telemetry payloads

Author: andrewlock

.azure-pipelines/steps/run-in-docker.yml

@@ -78,6 +78,7 @@ steps:
         --env DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH \
         --env DD_LOGGER_DD_TAGS \
         --env DD_INTERNAL_TELEMETRY_V2_ENABLED \
+        --env DD_TELEMETRY_METRICS_ENABLED \
         ${{ parameters.extraArgs }} \
         dd-trace-dotnet/${{ parameters.baseImage }}-${{ parameters.target }} \
         dotnet /build/bin/Debug/_build.dll ${{ parameters.command }}

.azure-pipelines/ultimate-pipeline.yml

@@ -117,6 +117,7 @@ variables:
   # Run code coverage on main branches only, on scheduled build only
   runCodeCoverage: $[or(eq(variables['run_code_coverage'], 'true'), and(eq(variables['Build.Reason'], 'Schedule'), or(in(variables['Build.SourceBranch'], 'refs/heads/master', 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/hotfix/'))))]
   DD_INTERNAL_TELEMETRY_V2_ENABLED: $[eq(variables['enable_telemetry_v2'], 'true')]
+  DD_TELEMETRY_METRICS_ENABLED: $[eq(variables['enable_metrics'], 'true')]
   DD_DOTNET_TRACER_MSBUILD:
   NugetPackageDirectory: $(System.DefaultWorkingDirectory)/packages
   relativeNugetPackageDirectory: packages

docker-compose.yml

@@ -366,6 +366,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - DD_INTERNAL_TELEMETRY_V2_ENABLED
+      - DD_TELEMETRY_METRICS_ENABLED
 
   ProfilerIntegrationTests:
     build:
@@ -413,6 +414,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - DD_INTERNAL_TELEMETRY_V2_ENABLED
+      - DD_TELEMETRY_METRICS_ENABLED
     hostname: integrationtests
 
   IntegrationTests:
@@ -487,6 +489,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - DD_INTERNAL_TELEMETRY_V2_ENABLED
+      - DD_TELEMETRY_METRICS_ENABLED
     hostname: integrationtests
     depends_on:
       - aerospike
@@ -557,6 +560,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - DD_INTERNAL_TELEMETRY_V2_ENABLED
+      - DD_TELEMETRY_METRICS_ENABLED
     hostname: integrationtests
 
   IntegrationTests.Serverless:
@@ -611,6 +615,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - DD_INTERNAL_TELEMETRY_V2_ENABLED
+      - DD_TELEMETRY_METRICS_ENABLED
     hostname: integrationtests
 
   ExplorationTests:
@@ -661,6 +666,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - DD_INTERNAL_TELEMETRY_V2_ENABLED
+      - DD_TELEMETRY_METRICS_ENABLED
 
   StartDependencies:
     image: andrewlock/wait-for-dependencies
@@ -752,6 +758,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - DD_INTERNAL_TELEMETRY_V2_ENABLED
+      - DD_TELEMETRY_METRICS_ENABLED
     depends_on:
       - servicestackredis_arm64
       - stackexchangeredis_arm64
@@ -831,6 +838,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - DD_INTERNAL_TELEMETRY_V2_ENABLED
+      - DD_TELEMETRY_METRICS_ENABLED
 
   start-test-agent:
     image: andrewlock/wait-for-dependencies

tracer/build/crank/Samples.AspNetCoreSimpleController.yml

@@ -5,6 +5,7 @@ imports:
 variables:
   commit_hash: 0
   telemetry_v2: 0
+  enable_metrics: 0
 
 jobs:
   server:

tracer/build/crank/Security.Samples.AspNetCoreSimpleController.yml

@@ -5,6 +5,7 @@
 variables:
   commit_hash: 0
   telemetry_v2: 0
+  enable_metrics: 0
 
 jobs:
   server:

tracer/build/crank/os.profiles.yml

@@ -20,6 +20,7 @@ profiles:
           DD_TRACE_LOGGING_RATE: 6
           DD_TRACE_DEBUG: 0
           DD_INTERNAL_TELEMETRY_V2_ENABLED: "{{ telemetry_v2 }}"
+          DD_TELEMETRY_METRICS_ENABLED: "{{ enable_metrics }}"
         options:
           requiredOperatingSystem: windows
           requiredArchitecture: x64
@@ -46,6 +47,7 @@ profiles:
           DD_TRACE_LOGGING_RATE: 6
           DD_TRACE_DEBUG: 0
           DD_INTERNAL_TELEMETRY_V2_ENABLED: "{{ telemetry_v2 }}"
+          DD_TELEMETRY_METRICS_ENABLED: "{{ enable_metrics }}"
         options:
           requiredOperatingSystem: linux
           requiredArchitecture: x64
@@ -72,6 +74,7 @@ profiles:
           DD_TRACE_LOGGING_RATE: 6
           DD_TRACE_DEBUG: 0
           DD_INTERNAL_TELEMETRY_V2_ENABLED: "{{ telemetry_v2 }}"
+          DD_TELEMETRY_METRICS_ENABLED: "{{ enable_metrics }}"
           DD_ENV: throughput-adhoc
         options:
           requiredOperatingSystem: linux
@@ -100,6 +103,7 @@ profiles:
           LD_PRELOAD: "{{ linuxProfilerPath }}/tracer-home-linux/linux-x64/Datadog.Linux.ApiWrapper.x64.so"
           DD_PROFILING_ENABLED: 1
           DD_INTERNAL_TELEMETRY_V2_ENABLED: "{{ telemetry_v2 }}"
+          DD_TELEMETRY_METRICS_ENABLED: "{{ enable_metrics }}"
         options:
           requiredOperatingSystem: linux
           requiredArchitecture: x64
@@ -126,6 +130,7 @@ profiles:
           DD_TRACE_LOGGING_RATE: 6
           DD_TRACE_DEBUG: 0
           DD_INTERNAL_TELEMETRY_V2_ENABLED: "{{ telemetry_v2 }}"
+          DD_TELEMETRY_METRICS_ENABLED: "{{ enable_metrics }}"
         options:
           requiredOperatingSystem: linux
           requiredArchitecture: arm64