Skip to content

Commit 7e973a6

Browse files
bouwkastandrewlock
bouwkast
and
andrewlock
authored
Fix Isolated Azure Functions performance when using ASP.NET Core Integration (#6567)
## Summary of changes This fixes an application performance issue when using Isolated Azure Functions with the `ASP.NET Core` Integration caused by us modifying a `static` `TypedData` object when requests were being proxied via HTTP. https://learn.microsoft.com/en-us/azure/azure-functions/dotnet-isolated-process-guide?tabs=hostbuilder%2Cwindows#aspnet-core-integration ## Reason for change When using the `ASP.NET Core` integration with an Isolated Azure Functions application (`ConfigureFunctionsWebApplication`) under load the worker `ASP.NET Core` application would start to timeout on requests or hang waiting for responses from the function application. Ultimately it appears that within `GrpcMessageConversionExtensions` a `static` instance of the `TypedData` that we used for injection was added specifically when the requests were being proxied via HTTP requests. Under load we'd end up modifying this shared object with our propagated headers which would ultimately cause requests to start timing out as we shouldn't have been modifying those headers. ## Implementation details We now check for when the requests are being proxied and if so we generate a new, non-`static` `TypedData` instance that we can safely inject headers into. Additionally, injection into these will now correctly adhere to whether or not the Functions automatic instrumentation is enabled or not. I was able to reliably reproduce the issue prior to the fix and haven't seen the issue post-fix. ## Test coverage - Added new Azure Functions Sample projects (and tests): - `Samples.AzureFunctions.V4Isolated.AspNetCore.SdkV1` tests `ASP.NET Core` integration with V1 of the NuGets - `Samples.AzureFunctions.V4Isolated.AspNetCore.Sdk` tests `ASP.NET Core` integration with V2 of the NuGets - Dependabot is now enabled for these sample projects as Azure Functions fell outside of our automated version range testing - Load tested each version of the Azure Functions model locally both pre and post fix to validate that the fix worked. - Manually tested and checked traces / spans emitted from both `ASP.NET Core` and `gRPC` Function models. ## Other details <!-- Fixes #{issue} --> Fixes #6494 Noted some potential improvements in the signal-to-noise ratio of the traces / spans produced by the Functions integration that we'll bring up as future improvements. <!-- ⚠️ Note: where possible, please obtain 2 approvals prior to merging. Unless CODEOWNERS specifies otherwise, for external teams it is typically best to have one review from a team member, and one review from apm-dotnet. Trivial changes do not require 2 reviews. --> --------- Co-authored-by: Andrew Lock <[email protected]>
1 parent 3375c75 commit 7e973a6

28 files changed

+1680
-75
lines changed

‎Datadog.Trace.Samples.g.sln‎

Lines changed: 15 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -129,22 +129,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Generated", "Generated", "{
129129
tracer\build\supported_calltargets.g.json = tracer\build\supported_calltargets.g.json
130130
EndProjectSection
131131
EndProject
132-
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tracer", "tracer", "{82FFBC1A-6B13-4C0A-896A-90306AE4828F}"
133-
ProjectSection(SolutionItems) = preProject
134-
EndProjectSection
135-
EndProject
136-
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{07D12F26-2583-4C6F-AFBB-AA30FF339FC6}"
137-
ProjectSection(SolutionItems) = preProject
138-
EndProjectSection
139-
EndProject
140-
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test-applications", "test-applications", "{F8C637E1-1F4F-4E3B-9E34-AAD61097C3F8}"
141-
ProjectSection(SolutionItems) = preProject
142-
EndProjectSection
143-
EndProject
144-
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "azure-functions", "azure-functions", "{FE9F14E0-8DFF-413B-BB9E-49CEA4115A5D}"
145-
ProjectSection(SolutionItems) = preProject
146-
EndProjectSection
147-
EndProject
148132
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.ExampleLibrary", "tracer\test\test-applications\integrations\dependency-libs\Samples.ExampleLibrary\Samples.ExampleLibrary.csproj", "{FDB5C8D0-018D-4FF9-9680-C6A5078F819B}"
149133
EndProject
150134
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Samples.ExampleLibraryTracer", "tracer\test\test-applications\integrations\dependency-libs\Samples.ExampleLibraryTracer\Samples.ExampleLibraryTracer.csproj", "{4B243CF1-4269-45C6-A238-1A9BFA58B8CC}"
@@ -445,6 +429,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AssemblyLoadContextResolve"
445429
EndProject
446430
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Samples.AzureFunctions.V4Isolated.SdkV1", "tracer\test\test-applications\azure-functions\Samples.AzureFunctions.V4Isolated.SdkV1\Samples.AzureFunctions.V4Isolated.SdkV1.csproj", "{18767A3E-9ADC-485C-A8C7-50660D5B579D}"
447431
EndProject
432+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Samples.AzureFunctions.V4Isolated.AspNetCore.SdkV1", "tracer\test\test-applications\azure-functions\Samples.AzureFunctions.V4Isolated.AspNetCore.SdkV1\Samples.AzureFunctions.V4Isolated.AspNetCore.SdkV1.csproj", "{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C}"
433+
EndProject
434+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Samples.AzureFunctions.V4Isolated.AspNetCore", "tracer\test\test-applications\azure-functions\Samples.AzureFunctions.V4Isolated.AspNetCore\Samples.AzureFunctions.V4Isolated.AspNetCore.csproj", "{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77}"
435+
EndProject
448436
Global
449437
GlobalSection(SolutionConfigurationPlatforms) = preSolution
450438
Debug|Any CPU = Debug|Any CPU
@@ -1043,6 +1031,14 @@ Global
10431031
{18767A3E-9ADC-485C-A8C7-50660D5B579D}.Debug|Any CPU.Build.0 = Debug|Any CPU
10441032
{18767A3E-9ADC-485C-A8C7-50660D5B579D}.Release|Any CPU.ActiveCfg = Release|Any CPU
10451033
{18767A3E-9ADC-485C-A8C7-50660D5B579D}.Release|Any CPU.Build.0 = Release|Any CPU
1034+
{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
1035+
{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C}.Debug|Any CPU.Build.0 = Debug|Any CPU
1036+
{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C}.Release|Any CPU.ActiveCfg = Release|Any CPU
1037+
{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C}.Release|Any CPU.Build.0 = Release|Any CPU
1038+
{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
1039+
{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77}.Debug|Any CPU.Build.0 = Debug|Any CPU
1040+
{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77}.Release|Any CPU.ActiveCfg = Release|Any CPU
1041+
{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77}.Release|Any CPU.Build.0 = Release|Any CPU
10461042
EndGlobalSection
10471043
GlobalSection(NestedProjects) = preSolution
10481044
{9518425A-36A5-4B8F-B0B8-6137DB88441D} = {8CEC2042-F11C-49F5-A674-2355793B600A}
@@ -1063,9 +1059,6 @@ Global
10631059
{16427BFB-B4C6-46A9-A290-8EA51FF73FEA} = {9518425A-36A5-4B8F-B0B8-6137DB88441D}
10641060
{0884B566-D22E-498C-BAA9-26D50ABCAE3A} = {16427BFB-B4C6-46A9-A290-8EA51FF73FEA}
10651061
{E1B0F72C-991A-409D-9266-DE5ED1BD940E} = {A0C5FBBB-CFB2-4FB9-B8F0-55676E9DCF06}
1066-
{07D12F26-2583-4C6F-AFBB-AA30FF339FC6} = {82FFBC1A-6B13-4C0A-896A-90306AE4828F}
1067-
{F8C637E1-1F4F-4E3B-9E34-AAD61097C3F8} = {07D12F26-2583-4C6F-AFBB-AA30FF339FC6}
1068-
{FE9F14E0-8DFF-413B-BB9E-49CEA4115A5D} = {F8C637E1-1F4F-4E3B-9E34-AAD61097C3F8}
10691062
{FDB5C8D0-018D-4FF9-9680-C6A5078F819B} = {8683D82A-2BBE-4199-9C36-C59F48804F90}
10701063
{4B243CF1-4269-45C6-A238-1A9BFA58B8CC} = {8683D82A-2BBE-4199-9C36-C59F48804F90}
10711064
{086FF8A0-9CEE-470A-9751-78B0F1340649} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A}
@@ -1215,6 +1208,8 @@ Global
12151208
{2CA0D70C-DFC1-458A-871B-328AB6E87E3A} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A}
12161209
{D6155F26-8245-4B66-8944-79C3DF9F9DA3} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A}
12171210
{8B1AF6A7-DD41-4347-B637-90C23D69B50E} = {498A300E-D036-49B7-A43D-821D1CAF11A5}
1218-
{18767A3E-9ADC-485C-A8C7-50660D5B579D} = {FE9F14E0-8DFF-413B-BB9E-49CEA4115A5D}
1211+
{18767A3E-9ADC-485C-A8C7-50660D5B579D} = {C4C1E313-C7C1-4490-AECE-0DD0062380A4}
1212+
{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C} = {C4C1E313-C7C1-4490-AECE-0DD0062380A4}
1213+
{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77} = {C4C1E313-C7C1-4490-AECE-0DD0062380A4}
12191214
EndGlobalSection
12201215
EndGlobal

‎Datadog.Trace.sln‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
Microsoft Visual Studio Solution File, Format Version 12.00
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
23
# Visual Studio Version 17
34
VisualStudioVersion = 17.1.31903.286
45
MinimumVisualStudioVersion = 15.0.26124.0
@@ -596,15 +597,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AssemblyLoadContextResolve"
596597
EndProject
597598
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Samples.AspNet.MultipleAppsInDomain", "tracer\test\test-applications\aspnet\Samples.AspNet.MultipleAppsInDomain\Samples.AspNet.MultipleAppsInDomain.csproj", "{A82EB6F8-D8D0-4763-B252-08CA3F39D153}"
598599
EndProject
599-
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tracer", "tracer", "{82FFBC1A-6B13-4C0A-896A-90306AE4828F}"
600-
EndProject
601-
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{07D12F26-2583-4C6F-AFBB-AA30FF339FC6}"
602-
EndProject
603-
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test-applications", "test-applications", "{F8C637E1-1F4F-4E3B-9E34-AAD61097C3F8}"
600+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Samples.AzureFunctions.V4Isolated.SdkV1", "tracer\test\test-applications\azure-functions\Samples.AzureFunctions.V4Isolated.SdkV1\Samples.AzureFunctions.V4Isolated.SdkV1.csproj", "{18767A3E-9ADC-485C-A8C7-50660D5B579D}"
604601
EndProject
605-
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "azure-functions", "azure-functions", "{FE9F14E0-8DFF-413B-BB9E-49CEA4115A5D}"
602+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Samples.AzureFunctions.V4Isolated.AspNetCore.SdkV1", "tracer\test\test-applications\azure-functions\Samples.AzureFunctions.V4Isolated.AspNetCore.SdkV1\Samples.AzureFunctions.V4Isolated.AspNetCore.SdkV1.csproj", "{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C}"
606603
EndProject
607-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Samples.AzureFunctions.V4Isolated.SdkV1", "tracer\test\test-applications\azure-functions\Samples.AzureFunctions.V4Isolated.SdkV1\Samples.AzureFunctions.V4Isolated.SdkV1.csproj", "{18767A3E-9ADC-485C-A8C7-50660D5B579D}"
604+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Samples.AzureFunctions.V4Isolated.AspNetCore", "tracer\test\test-applications\azure-functions\Samples.AzureFunctions.V4Isolated.AspNetCore\Samples.AzureFunctions.V4Isolated.AspNetCore.csproj", "{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77}"
608605
EndProject
609606
Global
610607
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -1442,6 +1439,14 @@ Global
14421439
{18767A3E-9ADC-485C-A8C7-50660D5B579D}.Debug|Any CPU.Build.0 = Debug|Any CPU
14431440
{18767A3E-9ADC-485C-A8C7-50660D5B579D}.Release|Any CPU.ActiveCfg = Release|Any CPU
14441441
{18767A3E-9ADC-485C-A8C7-50660D5B579D}.Release|Any CPU.Build.0 = Release|Any CPU
1442+
{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
1443+
{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C}.Debug|Any CPU.Build.0 = Debug|Any CPU
1444+
{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C}.Release|Any CPU.ActiveCfg = Release|Any CPU
1445+
{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C}.Release|Any CPU.Build.0 = Release|Any CPU
1446+
{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
1447+
{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77}.Debug|Any CPU.Build.0 = Debug|Any CPU
1448+
{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77}.Release|Any CPU.ActiveCfg = Release|Any CPU
1449+
{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77}.Release|Any CPU.Build.0 = Release|Any CPU
14451450
EndGlobalSection
14461451
GlobalSection(SolutionProperties) = preSolution
14471452
HideSolutionNode = FALSE
@@ -1674,10 +1679,9 @@ Global
16741679
{D6155F26-8245-4B66-8944-79C3DF9F9DA3} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A}
16751680
{8B1AF6A7-DD41-4347-B637-90C23D69B50E} = {498A300E-D036-49B7-A43D-821D1CAF11A5}
16761681
{A82EB6F8-D8D0-4763-B252-08CA3F39D153} = {AFA0AB23-64F0-4AC1-9050-6CE8FE06F580}
1677-
{07D12F26-2583-4C6F-AFBB-AA30FF339FC6} = {82FFBC1A-6B13-4C0A-896A-90306AE4828F}
1678-
{F8C637E1-1F4F-4E3B-9E34-AAD61097C3F8} = {07D12F26-2583-4C6F-AFBB-AA30FF339FC6}
1679-
{FE9F14E0-8DFF-413B-BB9E-49CEA4115A5D} = {F8C637E1-1F4F-4E3B-9E34-AAD61097C3F8}
1680-
{18767A3E-9ADC-485C-A8C7-50660D5B579D} = {FE9F14E0-8DFF-413B-BB9E-49CEA4115A5D}
1682+
{18767A3E-9ADC-485C-A8C7-50660D5B579D} = {C4C1E313-C7C1-4490-AECE-0DD0062380A4}
1683+
{5D2C6B9C-FCE2-4E46-B4ED-BC3B11CFBB3C} = {C4C1E313-C7C1-4490-AECE-0DD0062380A4}
1684+
{0F8EAB52-0C5B-4F60-92C5-42FAC21F4E77} = {C4C1E313-C7C1-4490-AECE-0DD0062380A4}
16811685
EndGlobalSection
16821686
GlobalSection(ExtensibilityGlobals) = postSolution
16831687
SolutionGuid = {160A1D00-1F5B-40F8-A155-621B4459D78F}
@@ -1692,7 +1696,9 @@ Global
16921696
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{0e036453-2c80-4fc9-a517-771f0071734b}*SharedItemsImports = 5
16931697
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{0f0f7d45-0e13-42b0-a158-8f303bbe8358}*SharedItemsImports = 5
16941698
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{0f1d9fb5-4415-40f1-b7b0-6dd5a3bab0c4}*SharedItemsImports = 5
1699+
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{0f8eab52-0c5b-4f60-92c5-42fac21f4e77}*SharedItemsImports = 5
16951700
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{10619ba2-aed1-482a-8570-bb7c7b83dddc}*SharedItemsImports = 5
1701+
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{18767a3e-9adc-485c-a8c7-50660d5b579d}*SharedItemsImports = 5
16961702
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{18a6904a-5afd-4816-ac3f-9f5e433720b5}*SharedItemsImports = 5
16971703
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{1a5e9f40-f3a5-4b59-9898-3dcd65c459c3}*SharedItemsImports = 5
16981704
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{1b3e6bee-f7ab-433e-a1d9-e8be3782419b}*SharedItemsImports = 5
@@ -1733,6 +1739,7 @@ Global
17331739
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{56de0d44-e9e5-48da-baea-2934b1e28d4e}*SharedItemsImports = 5
17341740
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{5a806f4b-39e7-4f38-b36f-f5cfc4f8760a}*SharedItemsImports = 13
17351741
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{5c2829c2-ed0d-414c-b5a0-2bfdca07b493}*SharedItemsImports = 5
1742+
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{5d2c6b9c-fce2-4e46-b4ed-bc3b11cfbb3c}*SharedItemsImports = 5
17361743
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{5e290fa1-e87b-4782-b977-eb5fa6c96efe}*SharedItemsImports = 5
17371744
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{5ee6b6eb-b768-47ec-882b-8dcaca2b1360}*SharedItemsImports = 5
17381745
tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{600953c4-bd8f-4a4b-a275-6d6f9ef48342}*SharedItemsImports = 5

‎tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/AzureFunctionsCommon.cs‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -280,26 +280,6 @@ _ when type.StartsWith("eventGrid", StringComparison.OrdinalIgnoreCase) => "Even
280280
return scope;
281281
}
282282

283-
internal static void OverridePropagatedContext<TTarget, TTypeData>(Tracer tracer, TTypeData typedData, string? useNullableHeadersCapability)
284-
where TTypeData : ITypedData
285-
{
286-
if (tracer.Settings.IsIntegrationEnabled(IntegrationId)
287-
&& tracer.ActiveScope is Scope { Span: { OperationName: OperationName } span })
288-
{
289-
// The HTTP request represented by TypedData is a duplicate of the original incoming
290-
// request that was received by func.exe. This is used to create a span representing
291-
// the request from the client. The typed data is then sent by the GRPC connection
292-
// to the functions app and is used to invoke the actual function. We intercept that
293-
// in the functions app and use it to create a span representing the actual work of the app.
294-
// In order for the span hierarchy/parenting to work correctly, we need to replace the parentID
295-
// in the GRPC http request representation, which is what we're doing here by overwriting all
296-
// the existing datadog headers
297-
var useNullableHeaders = !string.IsNullOrEmpty(useNullableHeadersCapability);
298-
var context = new PropagationContext(span.Context, Baggage.Current);
299-
tracer.TracerManager.SpanContextPropagator.Inject(context, new RpcHttpHeadersCollection<TTarget>(typedData.Http, useNullableHeaders));
300-
}
301-
}
302-
303283
private static PropagationContext ExtractPropagatedContextFromHttp<T>(T context, string? bindingName)
304284
where T : IFunctionContext
305285
{

‎tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/GrpcMessageConversionExtensionsToRpcHttpIntegration.cs‎

Lines changed: 54 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
using System;
1010
using System.ComponentModel;
1111
using Datadog.Trace.ClrProfiler.CallTarget;
12+
using Datadog.Trace.DuckTyping;
13+
using Datadog.Trace.Propagators;
1214
using Microsoft.AspNetCore.Http;
1315

1416
namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Azure.Functions;
@@ -31,17 +33,62 @@ namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Azure.Functions;
3133
public class GrpcMessageConversionExtensionsToRpcHttpIntegration
3234
{
3335
internal static CallTargetState OnMethodBegin<TTarget, TLogger, TGrpcCapabilities>(TTarget nullInstance, HttpRequest request, TLogger logger, TGrpcCapabilities capabilities)
34-
where TGrpcCapabilities : IGrpcCapabilities
3536
{
36-
var capability = capabilities.GetCapabilityState("UseNullableValueDictionaryForHttp");
37-
return new CallTargetState(scope: null, state: capability);
37+
return new CallTargetState(scope: null, state: capabilities);
3838
}
3939

40-
internal static TReturn OnAsyncMethodEnd<TTarget, TReturn>(TTarget nullInstance, TReturn returnValue, Exception exception, in CallTargetState state)
41-
where TReturn : ITypedData
40+
internal static TReturn OnAsyncMethodEnd<TTarget, TReturn>(TTarget nullInstance, TReturn returnValue, Exception? exception, in CallTargetState state)
41+
// We can't do this now, as we need to get and return the _real_ underlying type
42+
// where TReturn : ITypedData
4243
{
43-
AzureFunctionsCommon.OverridePropagatedContext<TTarget, TReturn>(Tracer.Instance, returnValue, state.State as string);
44-
return returnValue;
44+
var capabilities = state.State.DuckCast<IGrpcCapabilities>();
45+
if (capabilities is null || returnValue is null)
46+
{
47+
// Something went wrong, this shouldn't happen
48+
return returnValue;
49+
}
50+
51+
var tracer = Tracer.Instance;
52+
if (!tracer.Settings.IsIntegrationEnabled(AzureFunctionsCommon.IntegrationId)
53+
|| tracer.ActiveScope is not Scope { Span: { OperationName: AzureFunctionsCommon.OperationName } span })
54+
{
55+
return returnValue;
56+
}
57+
58+
// The HTTP request represented by TypedData is essentially a duplicate of the original incoming
59+
// request that was received by func.exe. This is used to create a span representing
60+
// the request from the client. The typed data is then sent by the GRPC connection
61+
// to the functions app and is used to invoke the actual function. We intercept that
62+
// in the functions app and use it to create a span representing the actual work of the app.
63+
// In order for the span hierarchy/parenting to work correctly, we need to replace the parentID
64+
// in the GRPC http request representation, which is what we're doing here by overwriting all
65+
// the existing datadog headers
66+
//
67+
// However, when using the AspNetCore integration, things work a bit differently. The
68+
// func.exe app instead primarily _proxies_ the HTTP request (if it is an HTTP trigger) to the functions
69+
// app, instead of sending the bulk of the context as a gRPC message. This means that we _shouldn't_ inject the
70+
// context into the gRPC request, because the context is already present in the incoming HTTP request, and is
71+
// used preferentially.
72+
//
73+
// What's more, in the case of HTTP triggers with proxying enabled, the TypedData returnValue returned from
74+
// this is method is a shared object, so mutating it can cause issues with other parts of the system.
75+
// See https://github.com/Azure/azure-functions-host/blob/420a4686802612857cae35cefea2b685283507c9/src/WebJobs.Script.Grpc/MessageExtensions/GrpcMessageConversionExtensions.cs#L104-L126
76+
77+
var isHttpProxying = !string.IsNullOrEmpty(capabilities.GetCapabilityState("HttpUri"));
78+
var requiresRouteParameters = !string.IsNullOrEmpty(capabilities.GetCapabilityState("RequiresRouteParameters"));
79+
var useNullableHeaders = !string.IsNullOrEmpty(capabilities.GetCapabilityState("UseNullableValueDictionaryForHttp"));
80+
81+
// When proxying, this method returns a singleton value that we must not update, so we create a new one
82+
// If we're not proxying, we can safely inject the context into the provided gRPC request
83+
var typedData = isHttpProxying && !requiresRouteParameters
84+
? TypedDataHelper<TReturn>.CreateTypedData()
85+
: returnValue.DuckCast<ITypedData>();
86+
87+
var context = new PropagationContext(span.Context, Baggage.Current);
88+
tracer.TracerManager.SpanContextPropagator.Inject(context, new RpcHttpHeadersCollection<TTarget>(typedData.Http, useNullableHeaders));
89+
90+
// Get the "real" value back out, whether it's the one we were provided or the new one we created
91+
return (TReturn)typedData.Instance!;
4592
}
4693
}
4794
#endif

‎tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/RpcHttpStruct.cs‎ renamed to ‎tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/IRpcHttp.cs‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// <copyright file="RpcHttpStruct.cs" company="Datadog">
1+
// <copyright file="IRpcHttp.cs" company="Datadog">
22
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
33
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
44
// </copyright>
@@ -13,15 +13,17 @@ namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Azure.Functions;
1313

1414
/// <summary>
1515
/// Duck type for RpcHttp
16+
/// This can't be a [DuckCopy] struct, because we set the Http property on TypedData to an instance of RpcHttp
17+
/// and a [DuckCopy] struct is _purely_ for extracting properites
1618
/// </summary>
17-
[DuckCopy]
18-
internal struct RpcHttpStruct
19+
internal interface IRpcHttp
1920
{
2021
/// <summary>
21-
/// An IDictionary&lt;string, NullableString&gt;
22+
/// Gets an IDictionary&lt;string, NullableString&gt;
2223
/// </summary>
23-
public IDictionary NullableHeaders;
24-
public IDictionary<string, string> Headers;
24+
public IDictionary NullableHeaders { get; }
25+
26+
public IDictionary<string, string> Headers { get; }
2527
}
2628

2729
#endif

‎tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Azure/Functions/Isolated/ITypedData.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
44
// </copyright>
55

6+
using Datadog.Trace.DuckTyping;
7+
68
#if !NETFRAMEWORK
79
#nullable enable
810

@@ -13,8 +15,9 @@ namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Azure.Functions;
1315
/// Interface because used in integration definition
1416
/// https://github.com/Azure/azure-functions-host/blob/8ceb05a89a4337f07264d4991545538a3e8b58a0/src/WebJobs.Script.Grpc/azure-functions-language-worker-protobuf/src/proto/FunctionRpc.proto#L443
1517
/// </summary>
16-
internal interface ITypedData
18+
///
19+
internal interface ITypedData : IDuckType
1720
{
18-
public RpcHttpStruct Http { get; }
21+
public IRpcHttp Http { get; set; }
1922
}
2023
#endif

0 commit comments

Comments
 (0)