[OTEL] Switches OTLP Tests To dd-apm-test-agent Instead of MockTracerAgent (#7656)

## Summary of changes
Making use of the
[dd-apm-test-agent](https://github.com/DataDog/dd-apm-test-agent)
container to enable OTLP metrics support for GRPC and HTTP, then gets
then as JSON the container for snapshot validation.

## Reason for change
To easily and more reliably test GRPC and ongoing OTLP support and
validation in the .NET tracer without needing to build GRPC servers or
change our agent for each new signal we plan to support.

## Implementation details
Added the container to run for the integrations tests in the
corresponding host name making use of either port `4317` or 4318`:

1. Pings `test/session/clear` to clear the session at the start of the
tests
2. Pings `/test/session/metrics` endpoint for the generated metrics

## Test coverage
Updated the snapshots format but the same 4 should pass even for GRPC
now that added to run for OTEL.

## Other details
<!-- Fixes #{issue} -->


<!--  ⚠️ Note:

Where possible, please obtain 2 approvals prior to merging. Unless
CODEOWNERS specifies otherwise, for external teams it is typically best
to have one review from a team member, and one review from apm-dotnet.
Trivial changes do not require 2 reviews.

MergeQueue is NOT enabled in this repository. If you have write access
to the repo, the PR has 1-2 approvals (see above), and all of the
required checks have passed, you can use the Squash and Merge button to
merge the PR. If you don't have write access, or you need help, reach
out in the #apm-dotnet channel in Slack.
-->

Author: link04

.azure-pipelines/ultimate-pipeline.yml

@@ -1683,6 +1683,12 @@ stages:
       displayName: 'Initialize LocalDB'
       workingDirectory: $(Build.Repository.LocalPath)
 
+    - powershell: |
+        mkdir -Force ./artifacts/build_data/snapshots
+        docker compose -f docker-compose.windows.yml run --rm start-test-agent.windows
+      displayName: Start test agent
+      retryCountOnTaskFailure: 3
+
     - script: tracer\build.cmd CompileTrimmingSamples BuildWindowsIntegrationTests BuildWindowsRegressionTests -Framework $(framework) --code-coverage-enabled $(CodeCoverageEnabled)
       displayName: Build integration tests
       retryCountOnTaskFailure: 3
@@ -1700,6 +1706,11 @@ stages:
         Filter: $(IntegrationTestFilter)
         SampleName: $(IntegrationTestSampleName)
 
+    - script: docker compose -f docker-compose.windows.yml down
+      displayName: Stop test agent
+      condition: succeededOrFailed()
+      continueOnError: true
+
     - task: PublishTestResults@2
       displayName: publish test results
       inputs:

docker-compose.windows.yml

@@ -32,6 +32,8 @@ services:
     - ./artifacts/build_data/snapshots:c:/debug_snapshots
     ports:
     - "8126"
+    - "4317:4317"
+    - "4318:4318"
     environment:
     - SNAPSHOT_CI=1
       # api-security attrs are unfortunately ignored because gzip compression generates different bytes per platform windows/linux
@@ -306,4 +308,4 @@ services:
     - dockerTag=${dockerTag:-unset}
     - DD_TRACE_AGENT_URL=http://test-agent.windows:8126
     depends_on:
-    - test-agent.windows
+    - test-agent.windows

docker-compose.yml

@@ -473,6 +473,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - RANDOM_SEED
+      - TEST_AGENT_HOST=test-agent
     hostname: integrationtests
     depends_on:
       - servicestackredis
@@ -494,6 +495,7 @@ services:
       - localstack
       - couchbase
       - azureservicebus-emulator
+      - test-agent
 
   IntegrationTests.Debugger:
     build:
@@ -675,9 +677,10 @@ services:
       - localstack
       - couchbase
       - azureservicebus-emulator
+      - test-agent
     environment:
       - TIMEOUT_LENGTH=120
-    command: servicestackredis:6379 stackexchangeredis:6379 stackexchangeredis-replica:6379 stackexchangeredis-single:6379 elasticsearch5:9200 elasticsearch6:9200 elasticsearch7:9200 sqlserver:1433 mongo:27017 postgres:5432 mysql:3306 mysql57:3306 rabbitmq:5672 kafka-broker:9092 kafka-zookeeper:2181 localstack:4566 couchbase:11210 azureservicebus-emulator:5672  # oracle:1521
+    command: servicestackredis:6379 stackexchangeredis:6379 stackexchangeredis-replica:6379 stackexchangeredis-single:6379 elasticsearch5:9200 elasticsearch6:9200 elasticsearch7:9200 sqlserver:1433 mongo:27017 postgres:5432 mysql:3306 mysql57:3306 rabbitmq:5672 kafka-broker:9092 kafka-zookeeper:2181 localstack:4566 couchbase:11210 azureservicebus-emulator:5672 test-agent:8126  # oracle:1521
 
   IntegrationTests.ARM64:
     build:
@@ -746,6 +749,7 @@ services:
       - DD_LOGGER_SYSTEM_PULLREQUEST_SOURCEBRANCH
       - DD_LOGGER_DD_TAGS
       - RANDOM_SEED
+      - TEST_AGENT_HOST=test-agent
     depends_on:
       - servicestackredis_arm64
       - stackexchangeredis_arm64
@@ -758,6 +762,7 @@ services:
       - mysql_arm64
       - rabbitmq_arm64
       - localstack_arm64
+      - test-agent
 
   StartDependencies.ARM64:
     image: andrewlock/wait-for-dependencies
@@ -773,9 +778,10 @@ services:
       - mysql_arm64
       - rabbitmq_arm64
       - localstack_arm64
+      - test-agent
     environment:
       - TIMEOUT_LENGTH=120
-    command: servicestackredis_arm64:6379 stackexchangeredis_arm64:6379 stackexchangeredis_arm64-replica:6379 stackexchangeredis_arm64-single:6379 elasticsearch7_arm64:9200 sqledge_arm64:1433 mongo_arm64:27017 postgres_arm64:5432 mysql_arm64:3306 rabbitmq_arm64:5672 localstack_arm64:4566
+    command: servicestackredis_arm64:6379 stackexchangeredis_arm64:6379 stackexchangeredis_arm64-replica:6379 stackexchangeredis_arm64-single:6379 elasticsearch7_arm64:9200 sqledge_arm64:1433 mongo_arm64:27017 postgres_arm64:5432 mysql_arm64:3306 rabbitmq_arm64:5672 localstack_arm64:4566 test-agent:8126
 
   IntegrationTests.ARM64.Debugger:
     build:
@@ -841,6 +847,8 @@ services:
     - ./artifacts/build_data/snapshots:/debug_snapshots
     ports:
     - "8126:8126"
+    - "4317:4317"
+    - "4318:4318"
     environment:
     - ENABLED_CHECKS=trace_count_header,meta_tracer_version_header,trace_content_length
     - SNAPSHOT_CI=1

tracer/build/_build/docker/test-agent.windows.dockerfile

@@ -3,6 +3,6 @@
 WORKDIR /
 
 # Pin to older test agent versions to try to avoid breakages in the future
-RUN pip install --no-cache-dir "ddapm-test-agent==1.28.0" "ddsketch==3.0.1" "ddsketch[serialization]==3.0.1"
+RUN pip install --no-cache-dir "ddapm-test-agent==1.32.0" "ddsketch==3.0.1" "ddsketch[serialization]==3.0.1"
 
 ENTRYPOINT [ "ddapm-test-agent", "--port=8126" ]

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/OpenTelemetrySdkTests.cs

@@ -10,6 +10,8 @@
 using System.Threading.Tasks;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.TestHelpers;
+using Datadog.Trace.Vendors.Newtonsoft.Json;
+using Datadog.Trace.Vendors.Newtonsoft.Json.Linq;
 using FluentAssertions;
 using FluentAssertions.Execution;
 using VerifyXunit;
@@ -72,7 +74,6 @@ public class OpenTelemetrySdkTests : TracingIntegrationTest
 
         private readonly Regex _versionRegex = new(@"telemetry.sdk.version: (0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)");
         private readonly Regex _timeUnixNanoRegex = new(@"time_unix_nano"":([0-9]{10}[0-9]+)");
-        private readonly Regex _timeUnixNanoRegexMetrics = new(@"TimeUnixNano: ([0-9]{10}[0-9]+)");
         private readonly Regex _exceptionStacktraceRegex = new(@"exception.stacktrace"":""System.ArgumentException: Example argument exception.*"",""");
 
         public OpenTelemetrySdkTests(ITestOutputHelper output)
@@ -89,8 +90,9 @@ public static IEnumerable<object[]> GetMetricsTestData()
         {
             foreach (var packageVersion in PackageVersions.OpenTelemetry)
             {
-                yield return [packageVersion[0], "false", "true"];
-                yield return [packageVersion[0], "true", "false"];
+                yield return [packageVersion[0], "false", "true", "grpc"];
+                yield return [packageVersion[0], "false", "true", "http/protobuf"];
+                yield return [packageVersion[0], "true", "false", "http/protobuf"];
             }
         }
 #endif
@@ -213,10 +215,11 @@ public async Task IntegrationDisabled(string packageVersion)
         [SkippableTheory]
         [Trait("Category", "EndToEnd")]
         [Trait("RunOnWindows", "True")]
+        [Trait("RequiresDockerDependency", "true")]
         [MemberData(nameof(GetMetricsTestData))]
-        public async Task SubmitsOtlpMetrics(string packageVersion, string datadogMetricsEnabled, string otelMetricsEnabled)
+        public async Task SubmitsOtlpMetrics(string packageVersion, string datadogMetricsEnabled, string otelMetricsEnabled, string protocol)
         {
-            var parsedVersion = Version.Parse(!string.IsNullOrEmpty(packageVersion) ? packageVersion : "1.12.0");
+            var parsedVersion = Version.Parse(!string.IsNullOrEmpty(packageVersion) ? packageVersion : "1.13.1");
             var runtimeMajor = Environment.Version.Major;
 
             var snapshotName = runtimeMajor switch
@@ -227,94 +230,71 @@ public async Task SubmitsOtlpMetrics(string packageVersion, string datadogMetric
                 _ => throw new SkipException($"Skipping test due to irrelevant runtime and OTel versions mix: .NET {runtimeMajor} & Otel v{parsedVersion}")
             };
 
-            var initialAgentPort = TcpPortProvider.GetOpenPort();
+            var testAgentHost = Environment.GetEnvironmentVariable("TEST_AGENT_HOST") ?? "localhost";
+            var otlpPort = protocol == "grpc" ? 4317 : 4318;
+
+            using (var httpClient = new System.Net.Http.HttpClient())
+            {
+                await httpClient.GetAsync($"http://{testAgentHost}:4318/test/session/clear");
+            }
 
             SetEnvironmentVariable("DD_ENV", string.Empty);
             SetEnvironmentVariable("DD_SERVICE", string.Empty);
             SetEnvironmentVariable("DD_METRICS_OTEL_METER_NAMES", "OpenTelemetryMetricsMeter");
             SetEnvironmentVariable("DD_METRICS_OTEL_ENABLED", datadogMetricsEnabled);
             SetEnvironmentVariable("OTEL_METRICS_EXPORTER_ENABLED", otelMetricsEnabled);
-            SetEnvironmentVariable("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf");
-            SetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT", $"http://127.0.0.1:{initialAgentPort}");
+            SetEnvironmentVariable("OTEL_EXPORTER_OTLP_PROTOCOL", protocol);
+            SetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT", $"http://{testAgentHost}:{otlpPort}");
             SetEnvironmentVariable("OTEL_METRIC_EXPORT_INTERVAL", "1000");
 
             // Up until Sdk version 1.6.0 Otel didn't support reading from the env var
             SetEnvironmentVariable("OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE", runtimeMajor >= 9 ? "delta" : "cumulative");
 
-            using var agent = EnvironmentHelper.GetMockAgent(fixedPort: initialAgentPort);
-            using (await RunSampleAndWaitForExit(agent, packageVersion: packageVersion ?? "1.12.0"))
+            using var agent = EnvironmentHelper.GetMockAgent();
+            using (await RunSampleAndWaitForExit(agent, packageVersion: packageVersion ?? "1.13.1"))
             {
-                // Collect requests from both MockTracerAgent and MockOtlpGrpcServer
-                var metricRequests = agent.OtlpRequests
-                                          .Where(r => r.PathAndQuery.StartsWith("/v1/metrics") || r.PathAndQuery.Contains("MetricsService"))
-                                          .ToList();
-
-                metricRequests.Should().NotBeEmpty("Expected OTLP metric requests were not received.");
-
-                // Group the scope metrics by the resource metrics and schema URL (should only be one unique combination)
-                var resourceMetricByResource = metricRequests
-                                        .SelectMany(r => r.MetricsData.ResourceMetrics)
-                                        .GroupBy(r => new Tuple<global::OpenTelemetry.Proto.Resource.V1.Resource, string>(r.Resource, r.SchemaUrl))
-                                        .Should()
-                                        .ContainSingle()
-                                        .Subject;
-
-                // Group the individual metrics by scope metric and schema URL (should only be one unique combination since we're only using one ActivitySource)
-                // This may result in multiple entries for metrics that are repeated multiple times before the test exits
-                var scopeMetricsByResource = resourceMetricByResource
-                                        .SelectMany(r => r.ScopeMetrics)
-                                        .GroupBy(r => new Tuple<global::OpenTelemetry.Proto.Common.V1.InstrumentationScope, string>(r.Scope, r.SchemaUrl))
-                                        .OrderBy(group => group.Key.Item1.Name);
-
-                var scopeMetrics = new List<object>();
-                foreach (var scopeMetricByResource in scopeMetricsByResource)
+                await Task.Delay(2000);
+
+                using var httpClient = new System.Net.Http.HttpClient();
+                var metricsResponse = await httpClient.GetAsync($"http://{testAgentHost}:4318/test/session/metrics");
+                metricsResponse.EnsureSuccessStatusCode();
+
+                var metricsJson = await metricsResponse.Content.ReadAsStringAsync();
+                var metricsData = JToken.Parse(metricsJson);
+
+                metricsData.Should().NotBeNullOrEmpty();
+
+                foreach (var attribute in metricsData.SelectTokens("$..resource.attributes[?(@.key == 'telemetry.sdk.version')]"))
                 {
-                    var metrics = scopeMetricByResource
-                                    .SelectMany(r => r.Metrics)
-                                    .GroupBy(r => r.Name)
-                                    .OrderBy(group => group.Key)
-                                    .Select(group => group.First())
-                                    .ToList();
-
-                    scopeMetrics.Add(new
-                    {
-                        Scope = scopeMetricByResource.Key.Item1,
-                        Metrics = metrics,
-                        SchemaUrl = scopeMetricByResource.Key.Item2
-                    });
+                    attribute["value"]!["string_value"] = "sdk-version";
                 }
 
-                // Filter out the telemetry resource name, if any
-                foreach (var attribute in resourceMetricByResource.Key.Item1.Attributes)
+                foreach (var attribute in metricsData.SelectTokens("$..resource.attributes[?(@.key == 'telemetry.sdk.name')]"))
                 {
-                    if (attribute.Key.Equals("telemetry.sdk.version"))
-                    {
-                        attribute.Value.StringValue = "sdk-version";
-                    }
-                    else if (attribute.Key.Equals("telemetry.sdk.name"))
-                    {
-                        attribute.Value.StringValue = "sdk-name";
-                    }
+                    attribute["value"]!["string_value"] = "sdk-name";
+                }
+
+                foreach (var dataPoint in metricsData.SelectTokens("$..data_points[*]"))
+                {
+                    dataPoint["start_time_unix_nano"] = "0";
+                    dataPoint["time_unix_nano"] = "0";
                 }
 
-                // Although there's only one resource, let's still emit snapshot data in the expected array format
-                var resourceMetrics = new object[]
+                foreach (var scopeMetric in metricsData.SelectTokens("$..scope_metrics[*]"))
                 {
-                    new
+                    if (scopeMetric["metrics"] is JArray metricsArray)
                     {
-                        Resource = resourceMetricByResource.Key.Item1,
-                        ScopeMetrics = scopeMetrics,
-                        SchemaUrl = resourceMetricByResource.Key.Item2,
+                        var sorted = new JArray(metricsArray.OrderBy(m => m["name"]?.ToString()));
+                        scopeMetric["metrics"] = sorted;
                     }
-                };
+                }
 
+                var formattedJson = metricsData.ToString(Formatting.Indented);
                 var settings = VerifyHelper.GetSpanVerifierSettings();
-                settings.AddRegexScrubber(_timeUnixNanoRegexMetrics, @"TimeUnixNano"": <DateTimeOffset.Now>");
-
                 var suffix = GetSuffix(packageVersion);
                 var fileName = $"{nameof(OpenTelemetrySdkTests)}.SubmitsOtlpMetrics{suffix}{snapshotName}";
 
-                await Verifier.Verify(resourceMetrics, settings)
+                await Verifier.Verify(formattedJson, settings)
                               .UseFileName(fileName)
                               .DisableRequireUniquePrefix();
             }