Skip to content

Commit b92236e

Browse files
GreenMatanGreenMatan
authored
[Dynamic Instrumentationm] Added PII redaction support (#4786)
* Added PII redaction based on predefined list of keywords * quarantine failing async span decoration tests
1 parent 37edb88 commit b92236e

File tree

47 files changed

+1037
-294
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1037
-294
lines changed

‎tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,12 @@ internal static class Debugger
5252
/// </summary>
5353
/// <seealso cref="DebuggerSettings.UploadFlushIntervalMilliseconds"/>
5454
public const string UploadFlushInterval = "DD_DEBUGGER_UPLOAD_FLUSH_INTERVAL";
55+
56+
/// <summary>
57+
/// Configuration key for set of identifiers that are used in redaction decisions.
58+
/// </summary>
59+
/// <seealso cref="DebuggerSettings.RedactedIdentifiers"/>
60+
public const string RedactedIdentifiers = "DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS";
5561
}
5662
}
5763
}

‎tracer/src/Datadog.Trace/Debugger/DebuggerSettings.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
// </copyright>
55

66
#nullable enable
7+
using System;
8+
using System.Collections.Generic;
9+
using System.Linq;
710
using Datadog.Trace.Configuration;
811
using Datadog.Trace.Configuration.Telemetry;
912
using Datadog.Trace.Telemetry;
@@ -55,6 +58,14 @@ public DebuggerSettings(IConfigurationSource? source, IConfigurationTelemetry te
5558
.WithKeys(ConfigurationKeys.Debugger.UploadFlushInterval)
5659
.AsInt32(DefaultUploadFlushIntervalMilliseconds, flushInterval => flushInterval >= 0)
5760
.Value;
61+
62+
var redactedIdentifiers = config
63+
.WithKeys(ConfigurationKeys.Debugger.RedactedIdentifiers)
64+
.AsString()?
65+
.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) ??
66+
Enumerable.Empty<string>();
67+
68+
RedactedIdentifiers = new HashSet<string>(redactedIdentifiers, StringComparer.OrdinalIgnoreCase);
5869
}
5970

6071
public bool Enabled { get; }
@@ -69,6 +80,8 @@ public DebuggerSettings(IConfigurationSource? source, IConfigurationTelemetry te
6980

7081
public int UploadFlushIntervalMilliseconds { get; }
7182

83+
public HashSet<string> RedactedIdentifiers { get; }
84+
7285
public static DebuggerSettings FromSource(IConfigurationSource source, IConfigurationTelemetry telemetry)
7386
{
7487
return new DebuggerSettings(source, telemetry);

‎tracer/src/Datadog.Trace/Debugger/Expressions/ProbeExpressionParser.Collection.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using System.Linq;
1010
using System.Linq.Expressions;
1111
using System.Reflection;
12+
using Datadog.Trace.Debugger.Snapshots;
1213
using Datadog.Trace.Vendors.Newtonsoft.Json;
1314
using Type = System.Type;
1415

@@ -88,6 +89,14 @@ private Expression GetItemAtIndex(JsonTextReader reader, List<ParameterExpressio
8889
throw new InvalidOperationException("Source must implement IList or IDictionary");
8990
}
9091

92+
if (indexOrKey.Type == typeof(string) &&
93+
indexOrKey is ConstantExpression expr &&
94+
Redaction.ShouldRedact(expr.Value?.ToString(), expr.Type, out _))
95+
{
96+
AddError($"{source?.ToString() ?? "N/A"}[{indexOrKey?.ToString() ?? "N/A"}]", "The property or field is redacted.");
97+
return RedactedValue();
98+
}
99+
91100
return CallGetItem(source, assignableFrom, indexOrKey);
92101
}
93102
catch (Exception e)

‎tracer/src/Datadog.Trace/Debugger/Expressions/ProbeExpressionParser.Dump.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ internal partial class ProbeExpressionParser<T>
2121
private Expression DumpExpression(Expression expression, List<ParameterExpression> scopeMembers)
2222
{
2323
if (Datadog.Trace.Debugger.Helpers.TypeExtensions.IsSimple(expression.Type) ||
24-
SupportedTypesService.AllowedTypesSafeToCallToString.Contains(expression.Type))
24+
Redaction.AllowedTypesSafeToCallToString.Contains(expression.Type))
2525
{
2626
return Expression.Call(expression, GetMethodByReflection(typeof(object), nameof(object.ToString), Type.EmptyTypes));
2727
}
@@ -211,7 +211,7 @@ private string DumpObject(object value, Type type, string name, int depth = 0)
211211
: $"{name}{ex.GetType().FullName}, {ex.Message}, {ex.StackTrace}";
212212
}
213213

214-
return SupportedTypesService.IsSafeToCallToString(type) ?
214+
return Redaction.IsSafeToCallToString(type) ?
215215
$"{name}{value?.ToString() ?? type?.FullName}" :
216216
$"{name}{type?.FullName}";
217217
}

‎tracer/src/Datadog.Trace/Debugger/Expressions/ProbeExpressionParser.General.cs‎

Lines changed: 55 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88
using System.Globalization;
99
using System.Linq;
1010
using System.Linq.Expressions;
11+
using System.Reflection;
1112
using Datadog.Trace.Debugger.Helpers;
13+
using Datadog.Trace.Debugger.Snapshots;
1214
using Datadog.Trace.Vendors.Newtonsoft.Json;
1315

1416
namespace Datadog.Trace.Debugger.Expressions;
@@ -85,7 +87,7 @@ private Expression GetMember(JsonTextReader reader, List<ParameterExpression> pa
8587
var referralMember = ParseTree(reader, parameters, itParameter);
8688
var refMember = (ConstantExpression)ParseTree(reader, parameters, itParameter);
8789

88-
return MemberPathExpression(referralMember, refMember.Value.ToString());
90+
return MemberPathExpression(referralMember, refMember);
8991
}
9092

9193
private Expression GetReference(JsonTextReader reader, List<ParameterExpression> parameters, ParameterExpression itParameter)
@@ -99,14 +101,22 @@ private Expression GetReference(JsonTextReader reader, List<ParameterExpression>
99101
return refMember;
100102
}
101103

102-
var argOrLocal = parameters.FirstOrDefault(p => p.Name == constant.Value.ToString());
104+
var constantValue = constant.Value.ToString();
105+
106+
if (Redaction.ShouldRedact(constantValue, constant.Type, out _))
107+
{
108+
AddError(reader.Value?.ToString() ?? "N/A", "The property or field is redacted.");
109+
return RedactedValue();
110+
}
111+
112+
var argOrLocal = parameters.FirstOrDefault(p => p.Name == constantValue);
103113
if (argOrLocal != null)
104114
{
105115
return argOrLocal;
106116
}
107117

108118
// will return an instance field\property or an UndefinedValue
109-
return MemberPathExpression(GetParameterExpression(parameters, ScopeMemberKind.This), constant.Value.ToString());
119+
return MemberPathExpression(GetParameterExpression(parameters, ScopeMemberKind.This), constant);
110120
}
111121
catch (Exception e)
112122
{
@@ -115,15 +125,49 @@ private Expression GetReference(JsonTextReader reader, List<ParameterExpression>
115125
}
116126
}
117127

118-
private Expression MemberPathExpression(Expression expression, string field)
128+
private Expression MemberPathExpression(Expression expression, ConstantExpression propertyOrField)
119129
{
130+
var propertyOrFieldValue = propertyOrField.Value.ToString();
131+
120132
try
121133
{
122-
return Expression.PropertyOrField(expression, field);
134+
if (Redaction.ShouldRedact(propertyOrFieldValue, propertyOrField.Type, out _))
135+
{
136+
AddError($"{expression}.{propertyOrFieldValue}", "The property or field is redacted.");
137+
return RedactedValue();
138+
}
139+
140+
var memberInfo = expression.Type.GetMember(
141+
propertyOrFieldValue,
142+
BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance)
143+
.FirstOrDefault();
144+
145+
if (memberInfo == null)
146+
{
147+
AddError($"{expression}.{propertyOrFieldValue}", "The property or field does not exist.");
148+
return UndefinedValue();
149+
}
150+
151+
bool isStatic = (memberInfo is PropertyInfo && ((PropertyInfo)memberInfo).GetGetMethod(true)?.IsStatic == true) ||
152+
(memberInfo is FieldInfo && ((FieldInfo)memberInfo).IsStatic);
153+
154+
if (isStatic)
155+
{
156+
return memberInfo.MemberType switch
157+
{
158+
MemberTypes.Field => Expression.Field(null, (FieldInfo)memberInfo),
159+
MemberTypes.Property => Expression.Property(null, (PropertyInfo)memberInfo),
160+
_ => throw new InvalidOperationException("Unsupported member type for static member access.")
161+
};
162+
}
163+
else
164+
{
165+
return Expression.PropertyOrField(expression, propertyOrFieldValue);
166+
}
123167
}
124168
catch (Exception e)
125169
{
126-
AddError($"{expression}.{field}", e.Message);
170+
AddError($"{expression}.{propertyOrFieldValue}", e.Message);
127171
return UndefinedValue();
128172
}
129173
}
@@ -133,6 +177,11 @@ private Expression UndefinedValue()
133177
return Expression.Constant(Expressions.UndefinedValue.Instance);
134178
}
135179

180+
private Expression RedactedValue()
181+
{
182+
return Expression.Constant("{REDACTED}");
183+
}
184+
136185
private GotoExpression ReturnDefaultValueExpression()
137186
{
138187
if (typeof(T) == typeof(bool))

‎tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,7 @@ internal bool Process<TCapture>(ref CaptureInfo<TCapture> info, DebuggerSnapshot
193193
if (info.IsAsyncCapture())
194194
{
195195
AddAsyncMethodArguments(snapshotCreator, ref info);
196+
AddAsyncMethodLocals(snapshotCreator, ref info);
196197
}
197198

198199
snapshotCreator.AddScopeMember(info.Name, info.Type, info.Value, info.MemberKind);

‎tracer/src/Datadog.Trace/Debugger/Instrumentation/AsyncLineDebuggerInvoker.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ public static void EndLine(ref AsyncLineDebuggerState state)
206206

207207
state.HasLocalsOrReturnValue = false;
208208
var asyncCaptureInfo = new AsyncCaptureInfo(state.MoveNextInvocationTarget, state.KickoffInvocationTarget, state.MethodMetadataInfo.KickoffInvocationTargetType, kickoffMethod: state.MethodMetadataInfo.KickoffMethod, hoistedArgs: state.MethodMetadataInfo.AsyncMethodHoistedArguments, hoistedLocals: state.MethodMetadataInfo.AsyncMethodHoistedLocals);
209-
var captureInfo = new CaptureInfo<object>(memberKind: ScopeMemberKind.This, methodState: MethodState.EndLineAsync, hasLocalOrArgument: hasArgumentsOrLocals, asyncCaptureInfo: asyncCaptureInfo, lineCaptureInfo: new LineCaptureInfo(state.LineNumber, state.ProbeFilePath));
209+
var captureInfo = new CaptureInfo<object>(value: state.KickoffInvocationTarget, type: state.MethodMetadataInfo.KickoffInvocationTargetType, name: "this", memberKind: ScopeMemberKind.This, methodState: MethodState.EndLineAsync, hasLocalOrArgument: hasArgumentsOrLocals, asyncCaptureInfo: asyncCaptureInfo, lineCaptureInfo: new LineCaptureInfo(state.LineNumber, state.ProbeFilePath));
210210
state.ProbeData.Processor.Process(ref captureInfo, state.SnapshotCreator);
211211
}
212212
catch (Exception e)

‎tracer/src/Datadog.Trace/Debugger/LiveDebugger.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ public async Task InitializeAsync()
118118
_subscriptionManager.SubscribeToChanges(_subscription);
119119

120120
DebuggerSnapshotSerializer.SetConfig(_settings);
121+
Redaction.SetConfig(_settings);
121122
AppDomain.CurrentDomain.AssemblyLoad += (sender, args) => CheckUnboundProbes();
122123

123124
await StartAsync().ConfigureAwait(false);

‎tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotSerializer.NotCapturedReason.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@ private enum NotCapturedReason
1414
collectionSize,
1515
depth,
1616
fieldCount,
17-
timeout
17+
timeout,
18+
redactedByIndentifier,
19+
redactedByType
1820
}
1921
}
2022
}

‎tracer/src/Datadog.Trace/Debugger/Snapshots/DebuggerSnapshotSerializer.cs‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -62,30 +62,36 @@ private static bool SerializeInternal(
6262
{
6363
try
6464
{
65-
if (source is IEnumerable enumerable && (SupportedTypesService.IsSupportedCollection(source) ||
66-
SupportedTypesService.IsSupportedDictionary(source)))
65+
if (Redaction.ShouldRedact(variableName, type, out var redactionReason))
6766
{
6867
if (variableName != null)
6968
{
7069
jsonWriter.WritePropertyName(variableName);
7170
}
7271

72+
var notCapturedReason = redactionReason == RedactionReason.Identifier ? NotCapturedReason.redactedByIndentifier : NotCapturedReason.redactedByType;
73+
7374
jsonWriter.WriteStartObject();
74-
SerializeEnumerable(source, type, jsonWriter, enumerable, currentDepth, cts);
75+
jsonWriter.WritePropertyName("type");
76+
jsonWriter.WriteValue(type.Name);
77+
WriteNotCapturedReason(jsonWriter, notCapturedReason);
7578
jsonWriter.WriteEndObject();
7679

7780
return true;
7881
}
7982

80-
if (SupportedTypesService.IsDenied(type))
83+
if (source is IEnumerable enumerable && (Redaction.IsSupportedCollection(source) ||
84+
Redaction.IsSupportedDictionary(source)))
8185
{
82-
jsonWriter.WritePropertyName(variableName);
86+
if (variableName != null)
87+
{
88+
jsonWriter.WritePropertyName(variableName);
89+
}
90+
8391
jsonWriter.WriteStartObject();
84-
jsonWriter.WritePropertyName("type");
85-
jsonWriter.WriteValue(type.Name);
86-
jsonWriter.WritePropertyName("value");
87-
jsonWriter.WriteValue("********");
92+
SerializeEnumerable(source, type, jsonWriter, enumerable, currentDepth, cts);
8893
jsonWriter.WriteEndObject();
94+
8995
return true;
9096
}
9197

@@ -162,7 +168,7 @@ private static void WriteTypeAndValue(
162168
jsonWriter.WritePropertyName("isNull");
163169
jsonWriter.WriteValue("true");
164170
}
165-
else if (SupportedTypesService.IsSafeToCallToString(type))
171+
else if (Redaction.IsSafeToCallToString(type))
166172
{
167173
jsonWriter.WritePropertyName("value");
168174
var stringValue = source.ToString();
@@ -183,7 +189,7 @@ private static void SerializeInstanceFieldsInternal(
183189
CancellationTokenSource cts,
184190
int currentDepth)
185191
{
186-
if (SupportedTypesService.IsSafeToCallToString(type) || source == null)
192+
if (Redaction.IsSafeToCallToString(type) || source == null)
187193
{
188194
return;
189195
}
@@ -270,7 +276,7 @@ private static void SerializeEnumerable(
270276
{
271277
try
272278
{
273-
var isDictionary = SupportedTypesService.IsSupportedDictionary(source);
279+
var isDictionary = Redaction.IsSupportedDictionary(source);
274280
if (source is ICollection collection)
275281
{
276282
jsonWriter.WritePropertyName("type");

0 commit comments

Comments
 (0)