[Tracer] Attribute Schema configuration: new configuration and v1 service naming (#4019)

Adds a new opt-in configuration `DD_TRACE_SPAN_ATTRIBUTE_SCHEMA` whose recognized values are "v1" or "v0" (the default value) and performs one (of several) v1 behaviors: the ServiceName for all spans are set to `$DD_SERVICE` instead of `$DD_SERVICE-<library>`.

Author: zacharycmontoya

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AdoNet/DbScopeFactory.cs

@@ -223,7 +223,7 @@ private static string GetServiceName(Tracer tracer, string dbTypeName)
                     if (DbTypeName != dbTypeName)
                     {
                         // We cannot cache in the base class
-                        return $"{tracer.DefaultServiceName}-{dbTypeName}";
+                        return tracer.Settings.GetServiceName(tracer, dbTypeName);
                     }
 
                     var serviceNameCache = _serviceNameCache;
@@ -239,7 +239,7 @@ private static string GetServiceName(Tracer tracer, string dbTypeName)
                     // We create or replace the cache with the new service name
                     // Slowpath
                     var defaultServiceName = tracer.DefaultServiceName;
-                    serviceName = $"{defaultServiceName}-{dbTypeName}";
+                    serviceName = tracer.Settings.GetServiceName(tracer, dbTypeName);
                     _serviceNameCache = new KeyValuePair<string, string>(defaultServiceName, serviceName);
                 }
 

tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.cs

@@ -404,6 +404,14 @@ internal static partial class ConfigurationKeys
         /// <seealso cref="TracerSettings.DbmPropagationMode"/>
         public const string DbmPropagationMode = "DD_DBM_PROPAGATION_MODE";
 
+        /// <summary>
+        /// Configuration key for setting the schema version for service naming and span attributes
+        /// Accepted values are: "v1", "v0"
+        /// Default value is "v0"
+        /// </summary>
+        /// <seealso cref="TracerSettings.MetadataSchemaVersion"/>
+        public const string MetadataSchemaVersion = "DD_TRACE_SPAN_ATTRIBUTE_SCHEMA";
+
         /// <summary>
         /// String constants for CI Visibility configuration keys.
         /// </summary>

tracer/src/Datadog.Trace/Configuration/ImmutableTracerSettings.cs

@@ -86,6 +86,7 @@ public ImmutableTracerSettings(TracerSettings settings)
             HttpClientExcludedUrlSubstrings = settings.HttpClientExcludedUrlSubstrings;
             HttpServerErrorStatusCodes = settings.HttpServerErrorStatusCodes;
             HttpClientErrorStatusCodes = settings.HttpClientErrorStatusCodes;
+            MetadataSchemaVersion = settings.MetadataSchemaVersion;
             ServiceNameMappings = settings.ServiceNameMappings;
             TraceBufferSize = settings.TraceBufferSize;
             TraceBatchInterval = settings.TraceBatchInterval;
@@ -434,6 +435,11 @@ public ImmutableTracerSettings(TracerSettings settings)
         /// </summary>
         internal ImmutableAzureAppServiceSettings? AzureAppServiceMetadata { get; }
 
+        /// <summary>
+        /// Gets the metadata schema version
+        /// </summary>
+        internal string MetadataSchemaVersion { get; }
+
         /// <summary>
         /// Create a <see cref="ImmutableTracerSettings"/> populated from the default sources
         /// returned by <see cref="GlobalConfigurationSource.Instance"/>.

tracer/src/Datadog.Trace/Configuration/ServiceNames.cs

@@ -13,9 +13,11 @@ internal class ServiceNames
     {
         private readonly object _lock = new object();
         private Dictionary<string, string> _mappings = null;
+        private bool _unifyServiceNames;
 
-        public ServiceNames(IDictionary<string, string> mappings)
+        public ServiceNames(IDictionary<string, string> mappings, string metadataSchemaVersion)
         {
+            _unifyServiceNames = metadataSchemaVersion == "v0" ? false : true;
             if (mappings?.Count > 0)
             {
                 _mappings = new Dictionary<string, string>(mappings);
@@ -28,6 +30,10 @@ public string GetServiceName(string applicationName, string key)
             {
                 return name;
             }
+            else if (_unifyServiceNames)
+            {
+                return applicationName;
+            }
             else
             {
                 return $"{applicationName}-{key}";

tracer/src/Datadog.Trace/Configuration/TracerSettings.cs

@@ -116,12 +116,13 @@ public TracerSettings(IConfigurationSource? source)
             var headerTagsNormalizationFixEnabled = source.GetBool(ConfigurationKeys.FeatureFlags.HeaderTagsNormalizationFixEnabled) ?? true;
             // Filter out tags with empty keys or empty values, and trim whitespaces
             HeaderTags = InitializeHeaderTags(inputHeaderTags, headerTagsNormalizationFixEnabled);
+            MetadataSchemaVersion = ParseMetadataSchemaVersion(source.GetString(ConfigurationKeys.MetadataSchemaVersion));
 
             var serviceNameMappings = source.GetDictionary(ConfigurationKeys.ServiceNameMappings)
                                             ?.Where(kvp => !string.IsNullOrWhiteSpace(kvp.Key) && !string.IsNullOrWhiteSpace(kvp.Value))
                                             ?.ToDictionary(kvp => kvp.Key.Trim(), kvp => kvp.Value.Trim());
 
-            ServiceNameMappings = new ServiceNames(serviceNameMappings);
+            ServiceNameMappings = new ServiceNames(serviceNameMappings, MetadataSchemaVersion);
 
             TracerMetricsEnabled = source.GetBool(ConfigurationKeys.TracerMetricsEnabled) ??
                                    // default value
@@ -609,6 +610,11 @@ public bool DiagnosticSourceEnabled
         /// </summary>
         internal ImmutableAzureAppServiceSettings? AzureAppServiceMetadata { get; set; }
 
+        /// <summary>
+        /// Gets or sets the metadata schema version
+        /// </summary>
+        internal string MetadataSchemaVersion { get; set; }
+
         /// <summary>
         /// Create a <see cref="TracerSettings"/> populated from the default sources
         /// returned by <see cref="GlobalConfigurationSource.Instance"/>.
@@ -701,6 +707,13 @@ private static IDictionary<string, string> InitializeHeaderTags(IDictionary<stri
             return headerTags;
         }
 
+        private static string ParseMetadataSchemaVersion(string? value) =>
+            value switch
+            {
+                "v1" or "V1" => "v1",
+                _ => "v0",
+            };
+
         internal static string[] TrimSplitString(string? textValues, char[] separators)
         {
             if (string.IsNullOrWhiteSpace(textValues))

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsSqsTests.cs

@@ -26,18 +26,22 @@ public AwsSqsTests(ITestOutputHelper output)
         {
         }
 
-        public override Result ValidateIntegrationSpan(MockSpan span) =>
-            span.Name switch
-            {
-                "sqs.request" => span.IsAwsSqs(),
-                _ => Result.DefaultSuccess,
-            };
+        public static IEnumerable<object[]> GetEnabledConfig()
+            => from packageVersionArray in PackageVersions.AwsSqs
+               from metadataSchemaVersion in new[] { "v0", "v1" }
+               select new[] { packageVersionArray[0], metadataSchemaVersion };
+
+        public override Result ValidateIntegrationSpan(MockSpan span) => span.IsAwsSqs();
 
         [SkippableTheory]
-        [MemberData(nameof(PackageVersions.AwsSqs), MemberType = typeof(PackageVersions))]
+        [MemberData(nameof(GetEnabledConfig))]
         [Trait("Category", "EndToEnd")]
-        public async Task SubmitsTraces(string packageVersion)
+        public async Task SubmitsTraces(string packageVersion, string metadataSchemaVersion)
         {
+            SetEnvironmentVariable("DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", metadataSchemaVersion);
+            var isExternalSpan = metadataSchemaVersion == "v0";
+            var clientSpanServiceName = isExternalSpan ? $"{EnvironmentHelper.FullSampleName}-aws-sqs" : EnvironmentHelper.FullSampleName;
+
             using var telemetry = this.ConfigureTelemetry();
             using (var agent = EnvironmentHelper.GetMockAgent())
             using (RunSampleAndWaitForExit(agent, packageVersion: packageVersion))
@@ -50,17 +54,13 @@ public async Task SubmitsTraces(string packageVersion)
                 var frameworkName = "NetCore";
 #endif
                 var spans = agent.WaitForSpans(expectedCount);
-                foreach (var span in spans)
-                {
-                    // TODO: Refactor to use ValidateIntegrationSpans later once we figure out how to best handle the combination of "http-client" and "sqs" service names produced by the integration
-                    var result = ValidateIntegrationSpan(span);
-                    Assert.True(result.Success, result.ToString());
-                }
+                var sqsSpans = spans.Where(span => span.Name == "sqs.request");
+                ValidateIntegrationSpans(sqsSpans, expectedServiceName: clientSpanServiceName, isExternalSpan);
 
                 var host = Environment.GetEnvironmentVariable("AWS_SQS_HOST");
 
                 var settings = VerifyHelper.GetSpanVerifierSettings();
-                settings.UseFileName($"{nameof(AwsSqsTests)}.{frameworkName}");
+                settings.UseFileName($"{nameof(AwsSqsTests)}.{frameworkName}.Schema{metadataSchemaVersion.ToUpper()}");
                 if (!string.IsNullOrWhiteSpace(host))
                 {
                     settings.AddSimpleScrubber(host, "localhost:00000");

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AdoNet/DapperTests.cs

@@ -23,37 +23,28 @@ public DapperTests(ITestOutputHelper output)
 
         [SkippableFact]
         [Trait("Category", "EndToEnd")]
-        public void SubmitsTraces()
-        {
-            const int expectedSpanCount = 17;
-            const string dbType = "postgres";
-            const string expectedOperationName = dbType + ".query";
-            const string expectedServiceName = "Samples.Dapper-" + dbType;
-
-            using (var agent = EnvironmentHelper.GetMockAgent())
-            using (RunSampleAndWaitForExit(agent))
-            {
-                var spans = agent.WaitForSpans(expectedSpanCount, operationName: expectedOperationName);
-                Assert.Equal(expectedSpanCount, spans.Count);
-                ValidateIntegrationSpans(spans, expectedServiceName: expectedServiceName);
-            }
-        }
+        public void SubmitsTracesV0() => RunTest("v0");
 
         [SkippableFact]
         [Trait("Category", "EndToEnd")]
-        public void SubmitsTracesWithNetStandard()
+        public void SubmitsTracesV1() => RunTest("v1");
+
+        private void RunTest(string metadataSchemaVersion)
         {
             const int expectedSpanCount = 17;
             const string dbType = "postgres";
             const string expectedOperationName = dbType + ".query";
-            const string expectedServiceName = "Samples.Dapper-" + dbType;
+
+            SetEnvironmentVariable("DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", metadataSchemaVersion);
+            var isExternalSpan = metadataSchemaVersion == "v0";
+            var clientSpanServiceName = isExternalSpan ? $"{EnvironmentHelper.FullSampleName}-{dbType}" : EnvironmentHelper.FullSampleName;
 
             using (var agent = EnvironmentHelper.GetMockAgent())
             using (RunSampleAndWaitForExit(agent))
             {
                 var spans = agent.WaitForSpans(expectedSpanCount, operationName: expectedOperationName);
                 Assert.Equal(expectedSpanCount, spans.Count);
-                ValidateIntegrationSpans(spans, expectedServiceName: expectedServiceName);
+                ValidateIntegrationSpans(spans, expectedServiceName: clientSpanServiceName, isExternalSpan);
             }
         }
     }

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AdoNet/FakeCommandTests.cs

@@ -23,7 +23,13 @@ public FakeCommandTests(ITestOutputHelper output)
 
         [SkippableFact]
         [Trait("Category", "EndToEnd")]
-        public void SubmitsTraces()
+        public void SubmitsTracesV0() => RunTest("v0");
+
+        [SkippableFact]
+        [Trait("Category", "EndToEnd")]
+        public void SubmitsTracesV1() => RunTest("v1");
+
+        private void RunTest(string metadataSchemaVersion)
         {
             // ALWAYS: 91 spans
             // - FakeCommand: 21 spans (3 groups * 7 spans)
@@ -37,7 +43,10 @@ public void SubmitsTraces()
             const int expectedSpanCount = 91;
             const string dbType = "fake";
             const string expectedOperationName = dbType + ".query";
-            const string expectedServiceName = "Samples.FakeDbCommand-fake";
+
+            SetEnvironmentVariable("DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", metadataSchemaVersion);
+            var isExternalSpan = metadataSchemaVersion == "v0";
+            var clientSpanServiceName = isExternalSpan ? $"{EnvironmentHelper.FullSampleName}-{dbType}" : EnvironmentHelper.FullSampleName;
 
             using var telemetry = this.ConfigureTelemetry();
             using var agent = EnvironmentHelper.GetMockAgent();
@@ -46,7 +55,7 @@ public void SubmitsTraces()
             int actualSpanCount = spans.Count(s => s.ParentId.HasValue); // Remove unexpected DB spans from the calculation
 
             Assert.Equal(expectedSpanCount, actualSpanCount);
-            ValidateIntegrationSpans(spans, expectedServiceName: expectedServiceName);
+            ValidateIntegrationSpans(spans, expectedServiceName: clientSpanServiceName, isExternalSpan);
 
             foreach (var span in spans)
             {

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AdoNet/MicrosoftDataSqlClientTests.cs

@@ -4,6 +4,7 @@
 // </copyright>
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.TestHelpers;
@@ -21,13 +22,18 @@ public MicrosoftDataSqlClientTests(ITestOutputHelper output)
             SetServiceVersion("1.0.0");
         }
 
+        public static IEnumerable<object[]> GetEnabledConfig()
+            => from packageVersionArray in PackageVersions.MicrosoftDataSqlClient
+               from metadataSchemaVersion in new[] { "v0", "v1" }
+               select new[] { packageVersionArray[0], metadataSchemaVersion };
+
         public override Result ValidateIntegrationSpan(MockSpan span) => span.IsSqlClient();
 
         [SkippableTheory]
-        [MemberData(nameof(PackageVersions.MicrosoftDataSqlClient), MemberType = typeof(PackageVersions))]
+        [MemberData(nameof(GetEnabledConfig))]
         [Trait("Category", "EndToEnd")]
         [Trait("RunOnWindows", "True")]
-        public void SubmitsTraces(string packageVersion)
+        public void SubmitsTraces(string packageVersion, string metadataSchemaVersion)
         {
             // ALWAYS: 133 spans
             // - SqlCommand: 21 spans (3 groups * 7 spans)
@@ -60,7 +66,10 @@ public void SubmitsTraces(string packageVersion)
             var expectedSpanCount = isVersion4 ? 91 : 147;
             const string dbType = "sql-server";
             const string expectedOperationName = dbType + ".query";
-            const string expectedServiceName = "Samples.Microsoft.Data.SqlClient-" + dbType;
+
+            SetEnvironmentVariable("DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", metadataSchemaVersion);
+            var isExternalSpan = metadataSchemaVersion == "v0";
+            var clientSpanServiceName = isExternalSpan ? $"{EnvironmentHelper.FullSampleName}-{dbType}" : EnvironmentHelper.FullSampleName;
 
             using var telemetry = this.ConfigureTelemetry();
             using var agent = EnvironmentHelper.GetMockAgent();
@@ -69,7 +78,7 @@ public void SubmitsTraces(string packageVersion)
             int actualSpanCount = spans.Count(s => s.ParentId.HasValue); // Remove unexpected DB spans from the calculation
 
             Assert.Equal(expectedSpanCount, actualSpanCount);
-            ValidateIntegrationSpans(spans, expectedServiceName: expectedServiceName);
+            ValidateIntegrationSpans(spans, expectedServiceName: clientSpanServiceName, isExternalSpan);
             telemetry.AssertIntegrationEnabled(IntegrationId.SqlClient);
         }
 

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AdoNet/MicrosoftDataSqliteTests.cs

@@ -6,6 +6,7 @@
 #if !NETCOREAPP2_1
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.TestHelpers;
@@ -22,14 +23,19 @@ public MicrosoftDataSqliteTests(ITestOutputHelper output)
             SetServiceVersion("1.0.0");
         }
 
+        public static IEnumerable<object[]> GetEnabledConfig()
+            => from packageVersionArray in PackageVersions.MicrosoftDataSqlite
+               from metadataSchemaVersion in new[] { "v0", "v1" }
+               select new[] { packageVersionArray[0], metadataSchemaVersion };
+
         public override Result ValidateIntegrationSpan(MockSpan span) => span.IsSqlite();
 
         [SkippableTheory]
-        [MemberData(nameof(PackageVersions.MicrosoftDataSqlite), MemberType = typeof(PackageVersions))]
+        [MemberData(nameof(GetEnabledConfig))]
         [Trait("Category", "EndToEnd")]
         [Trait("RunOnWindows", "True")]
         [Trait("Category", "ArmUnsupported")]
-        public void SubmitsTraces(string packageVersion)
+        public void SubmitsTraces(string packageVersion, string metadataSchemaVersion)
         {
 #if NETCOREAPP3_0
             if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("IsAlpine")) // set in dockerfile
@@ -43,15 +49,18 @@ public void SubmitsTraces(string packageVersion)
             const int expectedSpanCount = 91;
             const string dbType = "sqlite";
             const string expectedOperationName = dbType + ".query";
-            const string expectedServiceName = "Samples.Microsoft.Data.Sqlite-" + dbType;
+
+            SetEnvironmentVariable("DD_TRACE_SPAN_ATTRIBUTE_SCHEMA", metadataSchemaVersion);
+            var isExternalSpan = metadataSchemaVersion == "v0";
+            var clientSpanServiceName = isExternalSpan ? $"{EnvironmentHelper.FullSampleName}-{dbType}" : EnvironmentHelper.FullSampleName;
 
             using var telemetry = this.ConfigureTelemetry();
             using var agent = EnvironmentHelper.GetMockAgent();
             using var process = RunSampleAndWaitForExit(agent, packageVersion: packageVersion);
             var spans = agent.WaitForSpans(expectedSpanCount, operationName: expectedOperationName);
 
             Assert.Equal(expectedSpanCount, spans.Count);
-            ValidateIntegrationSpans(spans, expectedServiceName: expectedServiceName);
+            ValidateIntegrationSpans(spans, expectedServiceName: clientSpanServiceName, isExternalSpan);
 
             telemetry.AssertIntegrationEnabled(IntegrationId.Sqlite);
         }