[Tracer] fix: Re-use runtime metrics writer resources to limit memory growth (#7434)

## Summary of changes
Updates `TracerManagerFactory.CreateTracerManager` to pass and re-use
the previous `RuntimeMetricsWriter`, if runtime metrics is enabled for
the new TracerManager. We still create a new `IDogStatsd` client every
time, so any updated Agent settings are adopted by the DogStatsd client,
but this could be further optimized at a later time to re-use the client
if none of its configuration (e.g. host/port/tags) have changed.

## Reason for change
We've observed a scenario where the number of `RuntimeEventListener`
instances continues to grow, consuming more and more memory. This
happens whenever new Dynamic Configuration settings are received by the
tracer and runtime metrics are enabled. This PR resolves the issue.

## Implementation details
When creating the new `TracerManager`, pass in the previous
`RuntimeMetricsWriter` instance and only update the `IDogStatsd` object
with new settings. This makes sure that we maintain only one
`RuntimeMetricsWriter` instance while getting up-to-date DogStatsD
settings throughout the application lifetime.

## Test coverage
Adds a small unit test to confirm that the previous
`RuntimeMetricsWriter` is re-used. Additionally, local testing was done
to confirm that the number of
`Datadog.Trace.RuntimeMetrics.RuntimeEventListener` objects does not
grow when Dynamic Configuration is updated.

### Without the fix
After some number of Dynamic Configuration settings were made in the
Datadog UI, a dump was taken with the following analysis:
```
> dumpheap -type Datadog.Trace.RuntimeMetrics.RuntimeEventListener
         Address               MT           Size
    0159c3c7fe88     7ffe4e801810             88
    0159c442ed28     7ffe4e801810             88
    0159c44eac60     7ffe4e801810             88
    0159c4500f00     7ffe4e801810             88
    0159c454ea68     7ffe4e801810             88

Statistics:
          MT Count TotalSize Class Name
7ffe4e801810     5       440 Datadog.Trace.RuntimeMetrics.RuntimeEventListener
Total 5 objects, 440 bytes
```

Then after one additional Dynamic Configuration update was made in the
Datadog UI, a dump was taken with the following analysis:
```
> dumpheap -type Datadog.Trace.RuntimeMetrics.RuntimeEventListener
         Address               MT           Size
    0159c3c7fe88     7ffe4e801810             88
    0159c442ed28     7ffe4e801810             88
    0159c44eac60     7ffe4e801810             88
    0159c4500f00     7ffe4e801810             88
    0159c454ea68     7ffe4e801810             88
    0159c61fef48     7ffe4e801810             88

Statistics:
          MT Count TotalSize Class Name
7ffe4e801810     6       528 Datadog.Trace.RuntimeMetrics.RuntimeEventListener
Total 6 objects, 528 bytes
```

### With the fix
After some number of Dynamic Configuration settings were made in the
Datadog UI, a dump was taken with the following analysis:
```
> dumpheap -type Datadog.Trace.RuntimeMetrics.RuntimeEventListener
         Address               MT           Size
    01527107faf0     7ffe4e7acc00             88

Statistics:
          MT Count TotalSize Class Name
7ffe4e7acc00     1        88 Datadog.Trace.RuntimeMetrics.RuntimeEventListener
Total 1 objects, 88 bytes
```

Then after one additional Dynamic Configuration update was made in the
Datadog UI, a dump was taken with the following analysis:
```
> dumpheap -type Datadog.Trace.RuntimeMetrics.RuntimeEventListener
         Address               MT           Size
    01527107faf0     7ffe4e7acc00             88

Statistics:
          MT Count TotalSize Class Name
7ffe4e7acc00     1        88 Datadog.Trace.RuntimeMetrics.RuntimeEventListener
Total 1 objects, 88 bytes
```

## Other details
<!-- Fixes #{issue} -->


<!--  ⚠️ Note:

Where possible, please obtain 2 approvals prior to merging. Unless
CODEOWNERS specifies otherwise, for external teams it is typically best
to have one review from a team member, and one review from apm-dotnet.
Trivial changes do not require 2 reviews.

MergeQueue is NOT enabled in this repository. If you have write access
to the repo, the PR has 1-2 approvals (see above), and all of the
required checks have passed, you can use the Squash and Merge button to
merge the PR. If you don't have write access, or you need help, reach
out in the #apm-dotnet channel in Slack.
-->

---------

Co-authored-by: Andrew Lock <[email protected]>

Author: zacharycmontoya

tracer/src/Datadog.Trace/RuntimeMetrics/RuntimeMetricsWriter.cs

@@ -39,7 +39,6 @@ internal class RuntimeMetricsWriter : IDisposable
 
         private readonly TimeSpan _delay;
 
-        private readonly IDogStatsd _statsd;
         private readonly Timer _timer;
 
         private readonly IRuntimeMetricsListener _listener;
@@ -52,6 +51,7 @@ internal class RuntimeMetricsWriter : IDisposable
 #endif
 
         private readonly ConcurrentDictionary<string, int> _exceptionCounts = new ConcurrentDictionary<string, int>();
+        private IDogStatsd _statsd;
         private int _outOfMemoryCount;
 
         // The time when the runtime metrics were last pushed
@@ -158,6 +158,11 @@ public void Dispose()
             _exceptionCounts.Clear();
         }
 
+        internal void UpdateStatsd(IDogStatsd statsd)
+        {
+            Interlocked.Exchange(ref _statsd, statsd);
+        }
+
         internal void PushEvents()
         {
             if (Volatile.Read(ref _disposed) == 1)

tracer/src/Datadog.Trace/TracerManagerFactory.cs

@@ -60,8 +60,8 @@ internal TracerManager CreateTracerManager(TracerSettings settings, TracerManage
                 agentWriter: null,
                 sampler: null,
                 scopeManager: previous?.ScopeManager, // no configuration, so can always use the same one
-                statsd: null,
-                runtimeMetrics: null,
+                statsd: null, // For now, let's continue to always create a new StatsD instance
+                runtimeMetrics: previous?.RuntimeMetrics,
                 logSubmissionManager: previous?.DirectLogSubmission,
                 telemetry: null,
                 discoveryService: null,
@@ -137,9 +137,13 @@ internal TracerManager CreateTracerManager(
             agentWriter ??= GetAgentWriter(settings, settings.TracerMetricsEnabled ? statsd : null, rates => sampler.SetDefaultSampleRates(rates), discoveryService);
             scopeManager ??= new AsyncLocalScopeManager();
 
-            if (runtimeMetricsEnabled)
+            if (runtimeMetricsEnabled && runtimeMetrics is { })
             {
-                runtimeMetrics ??= new RuntimeMetricsWriter(statsd, TimeSpan.FromSeconds(10), settings.IsRunningInAzureAppService);
+                runtimeMetrics.UpdateStatsd(statsd);
+            }
+            else if (runtimeMetricsEnabled)
+            {
+                runtimeMetrics = new RuntimeMetricsWriter(statsd, TimeSpan.FromSeconds(10), settings.IsRunningInAzureAppService);
             }
             else
             {

tracer/test/Datadog.Trace.Tests/Configuration/DynamicConfigurationTests.cs

@@ -90,6 +90,24 @@ public void DoesNotOverrideDirectLogsTags()
             TracerManager.Instance.DirectLogSubmission.Formatter.Tags.Should().Be("key1:value1");
         }
 
+        [Fact]
+        public void DoesNotReplaceRuntimeMetricsWriter()
+        {
+            var tracerSettings = TracerSettings.Create(new()
+            {
+                { ConfigurationKeys.RuntimeMetricsEnabled, "true" },
+                { ConfigurationKeys.GlobalTags, "key1:value1" },
+            });
+            TracerManager.ReplaceGlobalManager(tracerSettings, TracerManagerFactory.Instance);
+
+            var previousRuntimeMetrics = TracerManager.Instance.RuntimeMetrics;
+
+            var configBuilder = CreateConfig(("tracing_tags", new[] { "key2:value2" }));
+            DynamicConfigurationManager.OnlyForTests_ApplyConfiguration(configBuilder);
+
+            TracerManager.Instance.RuntimeMetrics.Should().Be(previousRuntimeMetrics);
+        }
+
         [Fact]
         public void EnableTracing()
         {