Skip to content

Commit 79b266b

Browse files
[Debugger Default-On] DEBUG-4341 Forward snapshots to debugger intake (#7388)
## Summary of changes Dynamic Instrumentation and Exception Replay are about to be enabled by default with strict redaction rules. To support that, we have to perform redaction in the tracer (already done) _and_ in the backend. Up until now, we uploaded our snapshots through Logs Intake. In Logs Intake there is no built in support for redaction. In order to support redaction at ingestion time, we direct our snapshots through debugger's diagnostics endpoint instead of Logs Intake. ## Reason for change Enable DI and ER features by default. ## Implementation details Direct DI&ER snapshots through Debugger's intake. --------- Co-authored-by: Andrew Lock <[email protected]>
1 parent 69965ac commit 79b266b

File tree

226 files changed

+1215
-190
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

226 files changed

+1215
-190
lines changed

tracer/src/Datadog.Trace/Agent/DiscoveryService/AgentConfiguration.cs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ internal record AgentConfiguration
1212
public AgentConfiguration(
1313
string? configurationEndpoint,
1414
string? debuggerEndpoint,
15+
string? debuggerV2Endpoint,
16+
string? diagnosticsEndpoint,
1517
string? symbolDbEndpoint,
1618
string? agentVersion,
1719
string? statsEndpoint,
@@ -20,12 +22,12 @@ public AgentConfiguration(
2022
string? telemetryProxyEndpoint,
2123
string? tracerFlareEndpoint,
2224
bool clientDropP0,
23-
string? diagnosticsEndpoint,
2425
bool spanMetaStructs,
2526
bool? spanEvents)
2627
{
2728
ConfigurationEndpoint = configurationEndpoint;
2829
DebuggerEndpoint = debuggerEndpoint;
30+
DebuggerV2Endpoint = debuggerV2Endpoint;
2931
DiagnosticsEndpoint = diagnosticsEndpoint;
3032
SymbolDbEndpoint = symbolDbEndpoint;
3133
AgentVersion = agentVersion;
@@ -43,6 +45,8 @@ public AgentConfiguration(
4345

4446
public string? DebuggerEndpoint { get; }
4547

48+
public string? DebuggerV2Endpoint { get; }
49+
4650
public string? DiagnosticsEndpoint { get; }
4751

4852
public string? SymbolDbEndpoint { get; }

tracer/src/Datadog.Trace/Agent/DiscoveryService/DiscoveryService.cs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ namespace Datadog.Trace.Agent.DiscoveryService
2020
internal class DiscoveryService : IDiscoveryService
2121
{
2222
private const string SupportedDebuggerEndpoint = "debugger/v1/input";
23+
private const string SupportedDebuggerV2Endpoint = "debugger/v2/input";
2324
private const string SupportedDiagnosticsEndpoint = "debugger/v1/diagnostics";
2425
private const string SupportedSymbolDbEndpoint = "symdb/v1/input";
2526
private const string SupportedConfigurationEndpoint = "v0.7/config";
@@ -66,6 +67,7 @@ public DiscoveryService(
6667
new[]
6768
{
6869
SupportedDebuggerEndpoint,
70+
SupportedDebuggerV2Endpoint,
6971
SupportedDiagnosticsEndpoint,
7072
SupportedSymbolDbEndpoint,
7173
SupportedConfigurationEndpoint,
@@ -223,6 +225,7 @@ private async Task ProcessDiscoveryResponse(IApiResponse response)
223225
var discoveredEndpoints = (jObject["endpoints"] as JArray)?.Values<string>().ToArray();
224226
string? configurationEndpoint = null;
225227
string? debuggerEndpoint = null;
228+
string? debuggerV2Endpoint = null;
226229
string? diagnosticsEndpoint = null;
227230
string? symbolDbEndpoint = null;
228231
string? statsEndpoint = null;
@@ -246,6 +249,10 @@ private async Task ProcessDiscoveryResponse(IApiResponse response)
246249
{
247250
debuggerEndpoint = endpoint;
248251
}
252+
else if (endpoint.Equals(SupportedDebuggerV2Endpoint, StringComparison.OrdinalIgnoreCase))
253+
{
254+
debuggerV2Endpoint = endpoint;
255+
}
249256
else if (endpoint.Equals(SupportedDiagnosticsEndpoint, StringComparison.OrdinalIgnoreCase))
250257
{
251258
diagnosticsEndpoint = endpoint;
@@ -290,7 +297,8 @@ private async Task ProcessDiscoveryResponse(IApiResponse response)
290297
var newConfig = new AgentConfiguration(
291298
configurationEndpoint: configurationEndpoint,
292299
debuggerEndpoint: debuggerEndpoint,
293-
diagnosticsEndpoint: diagnosticsEndpoint ?? debuggerEndpoint,
300+
debuggerV2Endpoint: debuggerV2Endpoint ?? diagnosticsEndpoint,
301+
diagnosticsEndpoint: diagnosticsEndpoint,
294302
symbolDbEndpoint: symbolDbEndpoint,
295303
agentVersion: agentVersion,
296304
statsEndpoint: statsEndpoint,

tracer/src/Datadog.Trace/Debugger/DebuggerFactory.cs

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,12 @@ internal class DebuggerFactory
3030
internal static DynamicInstrumentation CreateDynamicInstrumentation(IDiscoveryService discoveryService, IRcmSubscriptionManager remoteConfigurationManager, TracerSettings tracerSettings, string serviceName, DebuggerSettings debuggerSettings, IGitMetadataTagsProvider gitMetadataTagsProvider)
3131
{
3232
var snapshotSlicer = SnapshotSlicer.Create(debuggerSettings);
33-
var snapshotStatusSink = SnapshotSink.Create(debuggerSettings, snapshotSlicer);
33+
var snapshotSink = SnapshotSink.Create(debuggerSettings, snapshotSlicer);
34+
var logSink = SnapshotSink.Create(debuggerSettings, snapshotSlicer);
3435
var diagnosticsSink = DiagnosticsSink.Create(serviceName, debuggerSettings);
3536

36-
var debuggerUploader = CreateSnapshotUploader(discoveryService, debuggerSettings, gitMetadataTagsProvider, GetApiFactory(tracerSettings, false), snapshotStatusSink);
37+
var snapshotUploader = CreateSnapshotUploader(discoveryService, debuggerSettings, gitMetadataTagsProvider, GetApiFactory(tracerSettings, true), snapshotSink);
38+
var logUploader = CreateSnapshotUploader(discoveryService, debuggerSettings, gitMetadataTagsProvider, GetApiFactory(tracerSettings, false), logSink);
3739
var diagnosticsUploader = CreateDiagnosticsUploader(discoveryService, debuggerSettings, gitMetadataTagsProvider, GetApiFactory(tracerSettings, true), diagnosticsSink);
3840
var lineProbeResolver = LineProbeResolver.Create(debuggerSettings.ThirdPartyDetectionExcludes, debuggerSettings.ThirdPartyDetectionIncludes);
3941
var probeStatusPoller = ProbeStatusPoller.Create(diagnosticsSink, debuggerSettings);
@@ -46,7 +48,8 @@ internal static DynamicInstrumentation CreateDynamicInstrumentation(IDiscoverySe
4648
discoveryService: discoveryService,
4749
remoteConfigurationManager: remoteConfigurationManager,
4850
lineProbeResolver: lineProbeResolver,
49-
snapshotUploader: debuggerUploader,
51+
snapshotUploader: snapshotUploader,
52+
logUploader: logUploader,
5053
diagnosticsUploader: diagnosticsUploader,
5154
probeStatusPoller: probeStatusPoller,
5255
configurationUpdater: configurationUpdater,
@@ -70,12 +73,22 @@ private static IDogStatsd GetDogStatsd(TracerSettings tracerSettings, string ser
7073
return statsd;
7174
}
7275

73-
private static SnapshotUploader CreateSnapshotUploader(IDiscoveryService discoveryService, DebuggerSettings debuggerSettings, IGitMetadataTagsProvider gitMetadataTagsProvider, IApiRequestFactory apiFactory, SnapshotSink snapshotStatusSink)
76+
private static SnapshotUploader CreateSnapshotUploader(IDiscoveryService discoveryService, DebuggerSettings debuggerSettings, IGitMetadataTagsProvider gitMetadataTagsProvider, IApiRequestFactory apiFactory, SnapshotSink snapshotSink)
7477
{
7578
var snapshotBatchUploadApi = DebuggerUploadApiFactory.CreateSnapshotUploadApi(apiFactory, discoveryService, gitMetadataTagsProvider);
7679
var snapshotBatchUploader = BatchUploader.Create(snapshotBatchUploadApi);
7780

78-
var debuggerSink = SnapshotUploader.Create(snapshotStatusSink, snapshotBatchUploader, debuggerSettings);
81+
var debuggerSink = SnapshotUploader.Create(snapshotSink, snapshotBatchUploader, debuggerSettings);
82+
83+
return debuggerSink;
84+
}
85+
86+
private static SnapshotUploader CreateLogUploader(IDiscoveryService discoveryService, DebuggerSettings debuggerSettings, IGitMetadataTagsProvider gitMetadataTagsProvider, IApiRequestFactory apiFactory, SnapshotSink snapshotSink)
87+
{
88+
var logUploaderApi = DebuggerUploadApiFactory.CreateLogUploadApi(apiFactory, discoveryService, gitMetadataTagsProvider);
89+
var logBatchUploader = BatchUploader.Create(logUploaderApi);
90+
91+
var debuggerSink = SnapshotUploader.Create(snapshotSink, logBatchUploader, debuggerSettings);
7992

8093
return debuggerSink;
8194
}

tracer/src/Datadog.Trace/Debugger/DynamicInstrumentation.cs

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ internal class DynamicInstrumentation : IDisposable
3838
private readonly IRcmSubscriptionManager _subscriptionManager;
3939
private readonly ISubscription _subscription;
4040
private readonly ISnapshotUploader _snapshotUploader;
41+
private readonly ISnapshotUploader _logUploader;
4142
private readonly IDebuggerUploader _diagnosticsUploader;
4243
private readonly ILineProbeResolver _lineProbeResolver;
4344
private readonly List<ProbeDefinition> _unboundProbes;
@@ -54,6 +55,7 @@ internal DynamicInstrumentation(
5455
IRcmSubscriptionManager remoteConfigurationManager,
5556
ILineProbeResolver lineProbeResolver,
5657
ISnapshotUploader snapshotUploader,
58+
ISnapshotUploader logUploader,
5759
IDebuggerUploader diagnosticsUploader,
5860
IProbeStatusPoller probeStatusPoller,
5961
ConfigurationUpdater configurationUpdater,
@@ -65,6 +67,7 @@ internal DynamicInstrumentation(
6567
_discoveryService = discoveryService;
6668
_lineProbeResolver = lineProbeResolver;
6769
_snapshotUploader = snapshotUploader;
70+
_logUploader = logUploader;
6871
_diagnosticsUploader = diagnosticsUploader;
6972
_probeStatusPoller = probeStatusPoller;
7073
_subscriptionManager = remoteConfigurationManager;
@@ -138,6 +141,13 @@ private void StartBackgroundProcess()
138141
CancellationToken.None,
139142
TaskContinuationOptions.OnlyOnFaulted,
140143
TaskScheduler.Default);
144+
145+
_ = _logUploader.StartFlushingAsync()
146+
.ContinueWith(
147+
t => Log.Error(t?.Exception, "Error in log uploader"),
148+
CancellationToken.None,
149+
TaskContinuationOptions.OnlyOnFaulted,
150+
TaskScheduler.Default);
141151
}
142152

143153
internal void UpdateAddedProbeInstrumentations(IReadOnlyList<ProbeDefinition> addedProbes)
@@ -463,10 +473,27 @@ internal void AddSnapshot(ProbeInfo probe, string snapshot)
463473
return;
464474
}
465475

476+
if (!probe.IsFullSnapshot)
477+
{
478+
AddLog(probe, snapshot);
479+
return;
480+
}
481+
466482
_snapshotUploader.Add(probe.ProbeId, snapshot);
467483
SetProbeStatusToEmitting(probe);
468484
}
469485

486+
internal void AddLog(ProbeInfo probe, string log)
487+
{
488+
if (IsDisposed)
489+
{
490+
return;
491+
}
492+
493+
_logUploader.Add(probe.ProbeId, log);
494+
SetProbeStatusToEmitting(probe);
495+
}
496+
470497
internal void SetProbeStatusToEmitting(ProbeInfo probe)
471498
{
472499
if (IsDisposed)

tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/ExceptionReplaySnapshotCreator.cs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,8 @@
44
// </copyright>
55

66
using System;
7-
using System.Collections.Generic;
8-
using System.Linq;
9-
using System.Text;
10-
using System.Threading.Tasks;
117
using Datadog.Trace.Debugger.Expressions;
8+
using Datadog.Trace.Debugger.Models;
129
using Datadog.Trace.Debugger.Snapshots;
1310
using ProbeLocation = Datadog.Trace.Debugger.Expressions.ProbeLocation;
1411

@@ -33,6 +30,8 @@ public ExceptionReplaySnapshotCreator(bool isFullSnapshot, ProbeLocation locatio
3330

3431
internal static string FrameIndex { get; } = Guid.NewGuid().ToString();
3532

33+
internal override string DebuggerProduct => DebuggerTags.DebuggerProduct.ER;
34+
3635
internal override DebuggerSnapshotCreator EndSnapshot()
3736
{
3837
JsonWriter.WritePropertyName("exceptionHash");

tracer/src/Datadog.Trace/Debugger/Models/Snapshot.cs

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,4 +43,32 @@ internal record struct DebuggerSnapshot
4343
{
4444
public SnapshotProbe Snapshot { get; set; }
4545
}
46+
47+
internal static class DebuggerTags
48+
{
49+
/// <summary>
50+
/// ddsource
51+
/// </summary>
52+
internal const string DDSource = "dd_debugger";
53+
54+
/// <summary>
55+
/// Tags for debugger.type
56+
/// </summary>
57+
internal static class DebuggerType
58+
{
59+
internal const string Snapshot = "snapshot";
60+
internal const string Diagnostic = "diagnostic";
61+
internal const string SymDb = "symdb";
62+
}
63+
64+
/// <summary>
65+
/// Tags for debugger.product
66+
/// </summary>
67+
internal static class DebuggerProduct
68+
{
69+
internal const string DI = "di";
70+
internal const string ER = "er";
71+
internal const string LD = "ld";
72+
}
73+
}
4674
}

tracer/src/Datadog.Trace/Debugger/Sink/Models/ProbeStatus.cs

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
// </copyright>
55

66
using System;
7+
using Datadog.Trace.Debugger.Models;
78
using Datadog.Trace.Vendors.Newtonsoft.Json;
89

910
namespace Datadog.Trace.Debugger.Sink.Models
@@ -14,7 +15,6 @@ public ProbeStatus(string service, string probeId, Status status, int probeVersi
1415
{
1516
Message = GetMessage();
1617
Service = service;
17-
1818
DebuggerDiagnostics = new DebuggerDiagnostics(new Diagnostics(probeId, status, probeVersion));
1919

2020
if (status == Status.ERROR)
@@ -37,14 +37,21 @@ string GetMessage()
3737
}
3838

3939
[JsonProperty("ddsource")]
40-
public string DdSource { get; } = "dd_debugger";
40+
public string DdSource { get; } = DebuggerTags.DDSource;
4141

4242
[JsonProperty("service")]
4343
public string Service { get; }
4444

4545
[JsonProperty("message")]
4646
public string Message { get; }
4747

48+
[JsonProperty("debugger.type")]
49+
public string DebuggerType { get; } = DebuggerTags.DebuggerType.Diagnostic;
50+
51+
// in .net tracer, this is always "di" for now
52+
[JsonProperty("debugger.product")]
53+
public string DebuggerProduct { get; } = DebuggerTags.DebuggerProduct.DI;
54+
4855
[JsonProperty("debugger")]
4956
public DebuggerDiagnostics DebuggerDiagnostics { get; }
5057
}

tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotCreator.cs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
using System.Linq;
1212
using System.Text;
1313
using System.Threading;
14-
using Datadog.Trace.ClrProfiler;
1514
using Datadog.Trace.Debugger.Configurations.Models;
1615
using Datadog.Trace.Debugger.Expressions;
1716
using Datadog.Trace.Debugger.Helpers;
@@ -25,7 +24,6 @@ namespace Datadog.Trace.Debugger.Snapshots
2524
internal class DebuggerSnapshotCreator : IDebuggerSnapshotCreator, IDisposable
2625
{
2726
private const string LoggerVersion = "2";
28-
private const string DDSource = "dd_debugger";
2927
private const string UnknownValue = "Unknown";
3028

3129
#pragma warning disable SA1401
@@ -68,6 +66,8 @@ public DebuggerSnapshotCreator(bool isFullSnapshot, ProbeLocation location, bool
6866
MethodScopeMembers = methodScopeMembers;
6967
}
7068

69+
internal virtual string DebuggerProduct => DebuggerTags.DebuggerProduct.DI;
70+
7171
internal string SnapshotId
7272
{
7373
get
@@ -874,14 +874,20 @@ internal DebuggerSnapshotCreator AddGeneralInfo(string service, string traceId,
874874
JsonWriter.WriteValue(service ?? UnknownValue);
875875

876876
JsonWriter.WritePropertyName("ddsource");
877-
JsonWriter.WriteValue(DDSource);
877+
JsonWriter.WriteValue(DebuggerTags.DDSource);
878878

879879
JsonWriter.WritePropertyName("dd.trace_id");
880880
JsonWriter.WriteValue(traceId);
881881

882882
JsonWriter.WritePropertyName("dd.span_id");
883883
JsonWriter.WriteValue(spanId);
884884

885+
JsonWriter.WritePropertyName("debugger.type");
886+
JsonWriter.WriteValue(DebuggerTags.DebuggerType.Snapshot);
887+
888+
JsonWriter.WritePropertyName("debugger.product");
889+
JsonWriter.WriteValue(DebuggerProduct);
890+
885891
return this;
886892
}
887893

tracer/src/Datadog.Trace/Debugger/Upload/DebuggerUploadApiBase.cs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,12 @@
55

66
#nullable enable
77
using System;
8+
using System.CodeDom;
89
using System.Threading;
910
using System.Threading.Tasks;
1011
using System.Web;
1112
using Datadog.Trace.Agent;
13+
using Datadog.Trace.Agent.Transports;
1214
using Datadog.Trace.Ci.Tags;
1315
using Datadog.Trace.Configuration;
1416
using Datadog.Trace.Processors;
@@ -18,6 +20,8 @@ namespace Datadog.Trace.Debugger.Upload;
1820

1921
internal abstract class DebuggerUploadApiBase : IBatchUploadApi
2022
{
23+
protected const string DebuggerV1Endpoint = "debugger/v1/input";
24+
2125
private readonly IApiRequestFactory _apiRequestFactory;
2226
private readonly IGitMetadataTagsProvider? _gitMetadataTagsProvider;
2327

@@ -54,6 +58,16 @@ protected string? Endpoint
5458
return builder.ToString();
5559
}
5660

61+
protected Task<IApiResponse> PostAsync(string uri, ArraySegment<byte> data)
62+
{
63+
var request = _apiRequestFactory.Create(new Uri(uri));
64+
var isDebuggerV1 = uri.Contains(DebuggerV1Endpoint);
65+
66+
return this is DiagnosticsUploadApi && !isDebuggerV1
67+
? request.PostAsync([new("event", MimeTypes.Json, "event.json", data)])
68+
: request.PostAsync(data, MimeTypes.Json);
69+
}
70+
5771
private string GetDefaultTagsMergedWithGlobalTags()
5872
{
5973
var sb = StringBuilderCache.Acquire();

tracer/src/Datadog.Trace/Debugger/Upload/DebuggerUploadApiFactory.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@ internal static IBatchUploadApi CreateSnapshotUploadApi(IApiRequestFactory apiRe
1717
return SnapshotUploadApi.Create(apiRequestFactory, discoveryService, gitMetadataTagsProvider);
1818
}
1919

20+
internal static IBatchUploadApi CreateLogUploadApi(IApiRequestFactory apiRequestFactory, IDiscoveryService discoveryService, IGitMetadataTagsProvider gitMetadataTagsProvider)
21+
{
22+
return LogUploadApi.Create(apiRequestFactory, discoveryService, gitMetadataTagsProvider);
23+
}
24+
2025
internal static IBatchUploadApi CreateDiagnosticsUploadApi(IApiRequestFactory apiRequestFactory, IDiscoveryService discoveryService, IGitMetadataTagsProvider gitMetadataTagsProvider)
2126
{
2227
return DiagnosticsUploadApi.Create(apiRequestFactory, discoveryService, gitMetadataTagsProvider);

0 commit comments

Comments
 (0)