Add fix for persistent Process.Start version-conflict issue (#7789)

## Summary of changes

- Adds a workaround for the version-conflict issue that occurs on app
startup on Linux
- Allow doing call target modification of version conflict dll
- Fix a bug in CallTarget instrumentation where we get the assembly
reference wrong

## Reason for change

As part of app startup, on _some_ linux distros, we shell out to `stat`
to build the container tags/entity ID. We don't want to trace this
`Process.Start()` call, so in
#5280 we added a flag to
skip instrumenting these calls. However, this fix relies on a
`[ThreadStatic]` variable, and in version-conflict scenarios (2.x.x
manual, 3.x.x automatic) we end up still instrumenting this call, which
causes recursion in `Tracer` initialization and
[errors](https://app.datadoghq.com/error-tracking?query=service%3Ainstrumentation-telemetry-data%20source%3Adotnet%20%40tracer_version%3A3.29.0.0%20-%40error.is_crash%3Atrue&et-side=activity&order=total_count&refresh_mode=sliding&source=all&sp=%5B%7B%22p%22%3A%7B%22issueId%22%3A%22cbce5fc2-3adf-11f0-a4be-da7ad0900002%22%7D%2C%22i%22%3A%22error-tracking-issue%22%7D%5D&view=spans&from_ts=1761134402500&to_ts=1762344002500&live=true).

> Note that since #7453
we don't do a process start at all, but that doesn't help in this
situation, because it's the 2.x.x library that's doing the
`Process.Start()`

## Implementation details

- Use standard call target instrumentation on the 2.x.x version of
`Datadog.Trace` (i.e. version conflict only)
- Hook the `ProcessHelpers.StartWithDoNotTrace()` method, and set the
3.x.x `_doNotTrace` variable for the duration of the method call
- Tweak the Rejit handler so that we _do_ rejit/call target the
Datadog.Trace 2.x.x module (but not the 3.x.x module)
- Fix a bug in the module builder which was incorrectly injecting a
reference to the 2.x.x assembly instead of the 3.x.x assembly

## Test coverage

We were already working around this issue in our VersionConflict tests,
so I removed the workaround, confirmed that the test failed, then made
the fix, and confirmed the tests pass again.

## Other details

This will only help for customers using a manual version of 2.49.0+
(when we introduced the `StartWithDoNotTrace()` call). I think that's
good enough support.

---------

Co-authored-by: Lucas Pimentel <[email protected]>

Author: andrewlock

tracer/build/_build/Honeypot/IntegrationGroups.cs

@@ -125,6 +125,7 @@ static IntegrationMap()
             NugetPackages.Add("RestSharp", Array.Empty<string>());
 
             // Manual instrumentation
+            NugetPackages.Add("Datadog.Trace", new string[] { });
             NugetPackages.Add("Datadog.Trace.Manual", new string[] { });
             NugetPackages.Add("Datadog.Trace.OpenTracing", new string[] { });
         }

tracer/build/supported_calltargets.g.json

@@ -7197,6 +7197,31 @@
     "IsAdoNetIntegration": false,
     "InstrumentationCategory": 1
   },
+  {
+    "IntegrationName": "DatadogTraceVersionConflict",
+    "AssemblyName": "Datadog.Trace",
+    "TargetTypeName": "Datadog.Trace.Util.ProcessHelpers",
+    "TargetMethodName": "StartWithDoNotTrace",
+    "TargetReturnType": "System.Diagnostics.Process",
+    "TargetParameterTypes": [
+      "System.Diagnostics.ProcessStartInfo",
+      "System.Boolean"
+    ],
+    "MinimumVersion": {
+      "Item1": 2,
+      "Item2": 49,
+      "Item3": 0
+    },
+    "MaximumVersion": {
+      "Item1": 2,
+      "Item2": 65535,
+      "Item3": 65535
+    },
+    "InstrumentationTypeName": "Datadog.Trace.ClrProfiler.AutoInstrumentation.VersionConflict.ProcessHelpersStartWithDoNotTraceIntegration",
+    "IntegrationKind": 0,
+    "IsAdoNetIntegration": false,
+    "InstrumentationCategory": 1
+  },
   {
     "IntegrationName": "DotnetTest",
     "AssemblyName": "coverlet.core",

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/VersionConflict/ProcessHelpersStartWithDoNotTraceIntegration.cs

@@ -0,0 +1,54 @@
+// <copyright file="ProcessHelpersStartWithDoNotTraceIntegration.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+
+using System;
+using System.ComponentModel;
+using Datadog.Trace.ClrProfiler.CallTarget;
+using Datadog.Trace.Configuration;
+using Datadog.Trace.Util;
+
+namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.VersionConflict;
+
+/// <summary>
+/// System.Diagnostics.Process Datadog.Trace.Util.ProcessHelpers::StartWithDoNotTrace(System.Diagnostics.ProcessStartInfo,System.Boolean) calltarget instrumentation
+/// </summary>
+[InstrumentMethod(
+    AssemblyName = "Datadog.Trace",
+    TypeName = "Datadog.Trace.Util.ProcessHelpers",
+    MethodName = "StartWithDoNotTrace",
+    ReturnTypeName = ClrNames.Process,
+    ParameterTypeNames = ["System.Diagnostics.ProcessStartInfo", ClrNames.Bool],
+    MinimumVersion = "2.49.0", // We only introduced ProcessHelpers.StartWithDoNotTrace() in 2.49.0
+    MaximumVersion = "2.*.*",
+    IntegrationName = nameof(IntegrationId.DatadogTraceVersionConflict))]
+[Browsable(false)]
+[EditorBrowsable(EditorBrowsableState.Never)]
+public class ProcessHelpersStartWithDoNotTraceIntegration
+{
+    internal static CallTargetState OnMethodBegin<TTarget, TStartInfo>(ref TStartInfo? startInfo, ref bool doNotTrace)
+    {
+        if (doNotTrace)
+        {
+            // When we run in a version conflict scenario (with a 2.x Datadog.Trace version of manual instrumentation
+            // and a 3.x version of auto-instrumentation), we will incorrectly trace Process.Start() calls
+            // that the 2.x lib is trying to avoid tracing by setting a [ThreadStatic] bool _doNotTrace on ProcessHelpers.
+            // However, because the types aren't shared between 2.x and 3.x, and because it's the 3.x types that are
+            // doing the tracing, this value is not respected. To work around that, we instrument the 2.x libraries
+            // and set the 3.x version, mirroring the desired behaviour.
+            ProcessHelpers.ForceDoNotTrace(true);
+        }
+
+        return CallTargetState.GetDefault();
+    }
+
+    internal static CallTargetReturn<System.Diagnostics.Process?> OnMethodEnd<TTarget>(System.Diagnostics.Process? returnValue, Exception? exception, in CallTargetState state)
+    {
+        // Always reset this back to false
+        ProcessHelpers.ForceDoNotTrace(false);
+        return new CallTargetReturn<System.Diagnostics.Process?>(returnValue);
+    }
+}

tracer/src/Datadog.Trace/Configuration/IntegrationId.cs

@@ -87,6 +87,7 @@ internal enum IntegrationId
         DatadogTraceManual,
         EmailHtmlInjection,
         Protobuf,
-        AzureEventHubs
+        AzureEventHubs,
+        DatadogTraceVersionConflict,
     }
 }

tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/EnumExtensionsGenerator/IntegrationIdExtensions_EnumExtensions.g.cs

@@ -17,7 +17,7 @@ internal static partial class IntegrationIdExtensions
     /// The number of members in the enum.
     /// This is a non-distinct count of defined names.
     /// </summary>
-    public const int Length = 76;
+    public const int Length = 77;
 
     /// <summary>
     /// Returns the string representation of the <see cref="Datadog.Trace.Configuration.IntegrationId"/> value.
@@ -106,6 +106,7 @@ public static string ToStringFast(this Datadog.Trace.Configuration.IntegrationId
             Datadog.Trace.Configuration.IntegrationId.EmailHtmlInjection => nameof(Datadog.Trace.Configuration.IntegrationId.EmailHtmlInjection),
             Datadog.Trace.Configuration.IntegrationId.Protobuf => nameof(Datadog.Trace.Configuration.IntegrationId.Protobuf),
             Datadog.Trace.Configuration.IntegrationId.AzureEventHubs => nameof(Datadog.Trace.Configuration.IntegrationId.AzureEventHubs),
+            Datadog.Trace.Configuration.IntegrationId.DatadogTraceVersionConflict => nameof(Datadog.Trace.Configuration.IntegrationId.DatadogTraceVersionConflict),
             _ => value.ToString(),
         };
 
@@ -195,6 +196,7 @@ public static Datadog.Trace.Configuration.IntegrationId[] GetValues()
             Datadog.Trace.Configuration.IntegrationId.EmailHtmlInjection,
             Datadog.Trace.Configuration.IntegrationId.Protobuf,
             Datadog.Trace.Configuration.IntegrationId.AzureEventHubs,
+            Datadog.Trace.Configuration.IntegrationId.DatadogTraceVersionConflict,
         };
 
     /// <summary>
@@ -284,5 +286,6 @@ public static string[] GetNames()
             nameof(Datadog.Trace.Configuration.IntegrationId.EmailHtmlInjection),
             nameof(Datadog.Trace.Configuration.IntegrationId.Protobuf),
             nameof(Datadog.Trace.Configuration.IntegrationId.AzureEventHubs),
+            nameof(Datadog.Trace.Configuration.IntegrationId.DatadogTraceVersionConflict),
         };
 }

tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/InstrumentationDefinitionsGenerator/InstrumentationDefinitions.g.cs

@@ -327,6 +327,8 @@ internal static bool IsInstrumentedAssembly(string assemblyName)
                     or "Datadog.Trace.ClrProfiler.AutoInstrumentation.ManualInstrumentation.OpenTracing.OpenTracingTracerFactoryCreateTracerIntegration"
                     or "Datadog.Trace.ClrProfiler.AutoInstrumentation.ManualInstrumentation.OpenTracing.OpenTracingTracerFactoryWrapTracerIntegration"
                     => Datadog.Trace.Configuration.IntegrationId.DatadogTraceManual,
+                "Datadog.Trace.ClrProfiler.AutoInstrumentation.VersionConflict.ProcessHelpersStartWithDoNotTraceIntegration"
+                    => Datadog.Trace.Configuration.IntegrationId.DatadogTraceVersionConflict,
                 "Datadog.Trace.ClrProfiler.AutoInstrumentation.Testing.DotnetTest.CoverageGetCoverageResultIntegration"
                     or "Datadog.Trace.ClrProfiler.AutoInstrumentation.Testing.DotnetTest.TestCommand5ctorIntegration"
                     or "Datadog.Trace.ClrProfiler.AutoInstrumentation.Testing.DotnetTest.TestCommandctorIntegration"

tracer/src/Datadog.Trace/Generated/net461/Datadog.Trace.SourceGenerators/TelemetryMetricGenerator/CiVisibilityMetricsTelemetryCollector_CountShared.g.cs

@@ -11,7 +11,7 @@
 namespace Datadog.Trace.Telemetry;
 internal partial class CiVisibilityMetricsTelemetryCollector
 {
-    private const int CountSharedLength = 324;
+    private const int CountSharedLength = 328;
 
     /// <summary>
     /// Creates the buffer for the <see cref="Datadog.Trace.Telemetry.Metrics.CountShared" /> values.
@@ -28,6 +28,10 @@ private static AggregatedMetric[] GetCountSharedBuffer()
             new(new[] { "integration_name:opentracing", "error_type:invoker" }),
             new(new[] { "integration_name:opentracing", "error_type:execution" }),
             new(new[] { "integration_name:opentracing", "error_type:missing_member" }),
+            new(new[] { "integration_name:version_conflict", "error_type:duck_typing" }),
+            new(new[] { "integration_name:version_conflict", "error_type:invoker" }),
+            new(new[] { "integration_name:version_conflict", "error_type:execution" }),
+            new(new[] { "integration_name:version_conflict", "error_type:missing_member" }),
             new(new[] { "integration_name:ciapp", "error_type:duck_typing" }),
             new(new[] { "integration_name:ciapp", "error_type:invoker" }),
             new(new[] { "integration_name:ciapp", "error_type:execution" }),
@@ -352,7 +356,7 @@ private static AggregatedMetric[] GetCountSharedBuffer()
     /// It is equal to the cardinality of the tag combinations (or 1 if there are no tags)
     /// </summary>
     private static int[] CountSharedEntryCounts { get; }
-        = new int[]{ 324, };
+        = new int[]{ 328, };
 
     public void RecordCountSharedIntegrationsError(Datadog.Trace.Telemetry.Metrics.MetricTags.IntegrationName tag1, Datadog.Trace.Telemetry.Metrics.MetricTags.InstrumentationError tag2, int increment = 1)
     {