[Profiler/Tracer] Bump to libdatadog v23

.azure-pipelines/ultimate-pipeline.yml

@@ -4734,7 +4734,7 @@ stages:
             versionSpec: '3.12'
           displayName: Install python 3.12
 
-        - script: git clone --depth 1 https://github.com/DataDog/system-tests.git
+        - script: git clone -b gleocadie/disable-test-discovery-test-for-dotnet --depth 1 https://github.com/DataDog/system-tests.git
           displayName: Get system tests repo
 
         - script: |
@@ -4836,7 +4836,7 @@ stages:
           versionSpec: '3.12'
         displayName: Install python 3.12
 
-      - script: git clone --depth 1 https://github.com/DataDog/system-tests.git
+      - script: git clone -b gleocadie/disable-test-discovery-test-for-dotnet --depth 1 https://github.com/DataDog/system-tests.git
         displayName: Get system tests repo
 
       - template: steps/download-artifact.yml

build/cmake/FindLibdatadog.cmake

@@ -4,12 +4,12 @@ endif()
 
 include(FetchContent)
 
-set(LIBDATADOG_VERSION "v20.0.0" CACHE STRING "libdatadog version")
+set(LIBDATADOG_VERSION "v23.0.0" CACHE STRING "libdatadog version")
 
 if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     # For Darwin, we'll download both architectures and combine them
-    set(SHA256_LIBDATADOG_ARM64 "916a207ea1343744062ff44f421f198bc9d79ae2be7624f4ba355542ff03b7ec" CACHE STRING "libdatadog arm64 sha256")
-    set(SHA256_LIBDATADOG_X86_64 "879d213a5cbf982500833323828c862a5efc824afdb03628daa8f363e70cbfe9" CACHE STRING "libdatadog x86_64 sha256")
+    set(SHA256_LIBDATADOG_ARM64 "10e14157f51feb3338d6a4338aaa22c68a7f3871408b98a613c9b848bb9f67c9" CACHE STRING "libdatadog arm64 sha256")
+    set(SHA256_LIBDATADOG_X86_64 "9d1596c241e33fde4cc99b5e16bbba85696c02c6d33ee416b0b1267b921a4fb2" CACHE STRING "libdatadog x86_64 sha256")
     set(FILE_TO_DOWNLOAD_ARM64 libdatadog-aarch64-apple-darwin.tar.gz)
     set(FILE_TO_DOWNLOAD_X86_64 libdatadog-x86_64-apple-darwin.tar.gz)
 
@@ -62,18 +62,18 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
 else()
     if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
         if(DEFINED ENV{IsAlpine} AND "$ENV{IsAlpine}" MATCHES "true")
-            set(SHA256_LIBDATADOG "12791239d3d0bfeab8fc659d10dd64e8ccf9ad87aa3f293c3f509a57bf2b3ade" CACHE STRING "libdatadog sha256")
+            set(SHA256_LIBDATADOG "01b4913bbbd3780c813bc1a59f6e5626a12d1846318552fc04e65754a32ccf87" CACHE STRING "libdatadog sha256")
             set(FILE_TO_DOWNLOAD libdatadog-aarch64-alpine-linux-musl.tar.gz)
         else()
-            set(SHA256_LIBDATADOG "368903a0a73b700ba2a6a8aee91b2be49af9ef6077b684b9f1aace408f8917b1" CACHE STRING "libdatadog sha256")
+            set(SHA256_LIBDATADOG "04f243b103817018ea438a3282c7e1e0b8e20fd2297000fb2d7edb6fbff5c3b2" CACHE STRING "libdatadog sha256")
             set(FILE_TO_DOWNLOAD libdatadog-aarch64-unknown-linux-gnu.tar.gz)
         endif()
     else()
         if(DEFINED ENV{IsAlpine} AND "$ENV{IsAlpine}" MATCHES "true")
-            set(SHA256_LIBDATADOG "bcf85135fea719aa192d176b08bb62056beff5d96ddbd114dfc28e50fcfea08c" CACHE STRING "libdatadog sha256")
+            set(SHA256_LIBDATADOG "adc56b0a5f8d7808ca6d9bc59c563fbcaa013a1da4f21db7d7b217bfa2037a01" CACHE STRING "libdatadog sha256")
             set(FILE_TO_DOWNLOAD libdatadog-${CMAKE_SYSTEM_PROCESSOR}-alpine-linux-musl.tar.gz)
         else()
-            set(SHA256_LIBDATADOG "84b779bdaa3da7668795e264a5ac4b015185044d147a142727349026eb49d428" CACHE STRING "libdatadog sha256")
+            set(SHA256_LIBDATADOG "c2db57b42d9808130fd2882fdf30e4fe955cd9dc3c27a7c68f663cfb0e940385" CACHE STRING "libdatadog sha256")
             set(FILE_TO_DOWNLOAD libdatadog-${CMAKE_SYSTEM_PROCESSOR}-unknown-linux-gnu.tar.gz)
         endif()
     endif()

build/vcpkg_local_ports/libdatadog/portfile.cmake

@@ -3,11 +3,11 @@ set(LIBDATADOG_VERSION ${VERSION})
 if(TARGET_TRIPLET STREQUAL "x64-windows" OR
    TARGET_TRIPLET STREQUAL "x64-windows-static")
     set(PLATFORM "x64")
-    set(LIBDATADOG_HASH "f43b4e0a41a4f1d20995e13bc43b7d3502120c2809aefa177f3eea73db1f430e9de350f6f54bdadae09820ad7a492b7f054ec0180d94c33e44fb9a462ab3153a")
+    set(LIBDATADOG_HASH "d9ea259ad2b85bb0e560d4316a410dc0d0b769fe52a7adcabd4226b000e6262159190bb0c3027856e048bba064255b89d0a594b5a414b7bd6b715e72debc393f")
 elseif(TARGET_TRIPLET STREQUAL "x86-windows" OR
        TARGET_TRIPLET STREQUAL "x86-windows-static")
     set(PLATFORM "x86")
-    set(LIBDATADOG_HASH "9cdc780f52edcccf272098f46ee2200e2fa3d56b29be21b5bfc1bcf7943f9a0efd7b7d9e84ca6340e618ec92c4b19d7625d6907592027d4ca67241b08b9063a6")
+    set(LIBDATADOG_HASH "7aaeb29fc9c522dcc8f3d7be90da14ef6052c756a1f162a241b17b109b6e92d0096345e2d22db05083b652e4f06eb29d15d8ba3fd191a1afbf0409da42c95185")
 else()
     message(FATAL_ERROR "Unsupported triplet: ${TARGET_TRIPLET}")
 endif()

build/vcpkg_local_ports/libdatadog/vcpkg.json

@@ -1,6 +1,6 @@
 {
     "name": "libdatadog",
-    "version-string": "20.0.0",
+    "version-string": "23.0.0",
     "description": "Package providing libdatadog prebuilt binaries for Windows only.",
     "dependencies": []
 }

profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.cpp

@@ -3,6 +3,8 @@
 
 // from dotnet coreclr includes
 
+// TO REMOVE: only to trigger profiler tests
+//
 #include "cor.h"
 #include "corprof.h"
 // end

profiler/test/Datadog.Profiler.IntegrationTests/Datadog.Profiler.IntegrationTests.csproj

@@ -86,6 +86,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
+     <PackageReference Include="ZstdSharp.Port" Version="0.8.6" />
   </ItemGroup>
   <ItemGroup Condition=" $(DD_LOGGER_ENABLED) != 'false' ">
     <PackageReference Include="DatadogTestLogger" Version="0.0.38" ExcludeAssets="compile" />

profiler/test/Datadog.Profiler.IntegrationTests/Helpers/SamplesHelper.cs

@@ -11,12 +11,14 @@
 using K4os.Compression.LZ4.Streams;
 using Perftools.Profiles;
 using Xunit;
+using ZstdSharp;
 
 namespace Datadog.Profiler.IntegrationTests.Helpers
 {
     public static class SamplesHelper
     {
         private static readonly byte[] Lz4MagicNumber = BitConverter.GetBytes(0x184D2204);
+        private static readonly byte[] ZstdMagicNumber = BitConverter.GetBytes(0xFD2FB528);
 
         public static int GetSamplesCount(string directory)
         {
@@ -213,6 +215,10 @@ private static Stream GetStream(string filename)
             {
                 return LZ4Stream.Decode(s);
             }
+            else if (ZstdMagicNumber.SequenceEqual(buffer))
+            {
+                return new DecompressionStream(s);
+            }
             else
             {
                 return s;

tracer/src/Datadog.Trace/LibDatadog/HandsOffConfiguration/ConfiguratorHelper.cs

@@ -33,8 +33,7 @@ internal static ConfigurationResult GetConfiguration(string? handsOffLocalConfig
         CString? localPath = null;
         CString? fleetPath = null;
         var configHandle = IntPtr.Zero;
-        Error? resultError = null;
-        LibraryConfigs? libraryConfigs = null;
+        LibraryConfigResult? configurationResult = null;
         try
         {
             languageCs = new CharSlice(TracerConstants.Language);
@@ -55,26 +54,26 @@ internal static ConfigurationResult GetConfiguration(string? handsOffLocalConfig
                 NativeInterop.LibraryConfig.ConfiguratorWithFleetPath(configHandle, fleetPath.Value);
             }
 
-            var configurationResult = NativeInterop.LibraryConfig.ConfiguratorGet(configHandle);
-            var result = configurationResult.Result;
+            configurationResult = NativeInterop.LibraryConfig.ConfiguratorGet(configHandle);
+            var result = configurationResult.Value.Result;
 
-            if (configurationResult.Tag == ResultTag.Err)
+            if (configurationResult.Value.Tag == ResultTag.Err)
             {
-                resultError = result.Error;
-                var error = resultError.Value.Message.ToUtf8String();
+                var resultError = result.Error;
+                var error = resultError.Message.ToUtf8String();
                 return new ConfigurationResult(null, error, Result.LibDatadogCallError);
             }
 
-            libraryConfigs = result.Ok;
-            var configsLength = (int)libraryConfigs.Value.Length;
+            var libraryConfigs = result.Ok;
+            var configsLength = (int)libraryConfigs.Length;
             var configEntriesLocal = new Dictionary<string, string>();
             var configEntriesRemote = new Dictionary<string, string>();
             var structSize = Marshal.SizeOf<LibraryConfig>();
             for (var i = 0; i < configsLength; i++)
             {
                 unsafe
                 {
-                    var ptr = new IntPtr(libraryConfigs.Value.Ptr + (structSize * i));
+                    var ptr = new IntPtr(libraryConfigs.Ptr + (structSize * i));
                     var libraryConfig = (LibraryConfig*)ptr;
                     var name = libraryConfig->Name.ToUtf8String();
                     var value = libraryConfig->Value.ToUtf8String();
@@ -100,15 +99,10 @@ internal static ConfigurationResult GetConfiguration(string? handsOffLocalConfig
             languageCs?.Dispose();
             localPath?.Dispose();
             fleetPath?.Dispose();
-            if (resultError.HasValue)
-            {
-                var resultErrorValue = resultError.Value;
-                NativeInterop.Common.DropError(ref resultErrorValue);
-            }
 
-            if (libraryConfigs.HasValue)
+            if (configurationResult.HasValue)
             {
-                NativeInterop.LibraryConfig.LibraryConfigDrop(libraryConfigs.Value);
+                NativeInterop.LibraryConfig.LibraryConfigDrop(configurationResult.Value);
             }
 
             if (configHandle != IntPtr.Zero)

tracer/src/Datadog.Trace/LibDatadog/HandsOffConfiguration/InteropStructs/LibraryConfigs.cs

@@ -14,4 +14,5 @@ internal struct LibraryConfigs
     public nint Ptr;       // const LibraryConfig*
     public nuint Length;
     public nuint Capacity;
+    public CString Logs; // ffi::CString
 }

tracer/src/Datadog.Trace/LibDatadog/NativeInterop.cs

@@ -116,16 +116,17 @@ internal static class Common
 
     internal static class LibraryConfig
     {
-        [DllImport(DllName, EntryPoint = "ddog_store_tracer_metadata")]
-        internal static extern TracerMemfdHandleResult StoreTracerMetadata(
-            byte schemaVersion,
-            CharSlice runtimeId,
-            CharSlice tracerLanguage,
-            CharSlice tracerVersion,
-            CharSlice hostname,
-            CharSlice serviceName,
-            CharSlice serviceEnv,
-            CharSlice serviceVersion);
+        [DllImport(DllName, EntryPoint = "ddog_tracer_metadata_new")]
+        internal static extern IntPtr TracerMetadataNew();
+
+        [DllImport(DllName, EntryPoint = "ddog_tracer_metadata_free")]
+        internal static extern void TracerMetadataFree(IntPtr metadata);
+
+        [DllImport(DllName, EntryPoint = "ddog_tracer_metadata_set")]
+        internal static extern void TracerMetadataSet(IntPtr metadata, MetadataKind kind, CString value);
+
+        [DllImport(DllName, EntryPoint = "ddog_tracer_metadata_store")]
+        internal static extern TracerMemfdHandleResult StoreTracerMetadata(IntPtr metadata);
 
         [DllImport(DllName, EntryPoint = "ddog_library_configurator_new")]
         internal static extern IntPtr ConfiguratorNew(byte debugLogs, CharSlice language);
@@ -143,6 +144,6 @@ internal static extern TracerMemfdHandleResult StoreTracerMetadata(
         internal static extern void ConfiguratorDrop(IntPtr configurator);
 
         [DllImport(DllName, EntryPoint = "ddog_library_config_drop")]
-        internal static extern void LibraryConfigDrop(LibraryConfigs configs);
+        internal static extern void LibraryConfigDrop(LibraryConfigResult configs);
     }
 }

tracer/src/Datadog.Trace/LibDatadog/ServiceDiscovery/MetadataKind.cs

@@ -0,0 +1,22 @@
+// <copyright file="MetadataKind.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+using System;
+
+namespace Datadog.Trace.LibDatadog.ServiceDiscovery;
+
+internal enum MetadataKind : int
+{
+    RuntimeId = 0,
+    TracerLanguage = 1,
+    TracerVersion = 2,
+    Hostname = 3,
+    ServiceName = 4,
+    ServiceEnvironment = 5,
+    ServiceVersion = 6,
+    ProcessTags = 7,
+    ContainerId = 8,
+}

tracer/src/Datadog.Trace/LibDatadog/ServiceDiscovery/ServiceDiscoveryHelper.cs

@@ -31,7 +31,6 @@ internal static StoreMetadataResult StoreTracerMetadata(TracerSettings tracerSet
             try
             {
                 var result = StoreTracerMetadata(
-                    1,
                     Tracer.RuntimeId,
                     TracerConstants.Language,
                     TracerConstants.ThreePartVersion,
@@ -66,7 +65,6 @@ internal static StoreMetadataResult StoreTracerMetadata(TracerSettings tracerSet
     }
 
     private static TracerMemfdHandleResult StoreTracerMetadata(
-        byte schemaVersion,
         string runtimeId,
         string tracerLanguage,
         string tracerVersion,
@@ -75,14 +73,32 @@ private static TracerMemfdHandleResult StoreTracerMetadata(
         string? serviceEnv,
         string? serviceVersion)
     {
-        using var runtimeIdCharSlice = new CharSlice(runtimeId);
-        using var tracerLanguageCharSlice = new CharSlice(tracerLanguage);
-        using var tracerVersionCharSlice = new CharSlice(tracerVersion);
-        using var hostnameCharSlice = new CharSlice(hostname);
-        using var serviceNameCharSlice = new CharSlice(serviceName);
-        using var serviceEnvCharSlice = new CharSlice(serviceEnv);
-        using var serviceVersionCharSlice = new CharSlice(serviceVersion);
+        IntPtr ptr = IntPtr.Zero;
+        try
+        {
+            ptr = NativeInterop.LibraryConfig.TracerMetadataNew();
+            SetMetadata(ptr, MetadataKind.RuntimeId, runtimeId);
+            SetMetadata(ptr, MetadataKind.TracerLanguage, tracerLanguage);
+            SetMetadata(ptr, MetadataKind.TracerVersion, tracerVersion);
+            SetMetadata(ptr, MetadataKind.Hostname, hostname);
+            SetMetadata(ptr, MetadataKind.ServiceName, serviceName);
+            SetMetadata(ptr, MetadataKind.ServiceEnvironment, serviceEnv);
+            SetMetadata(ptr, MetadataKind.ServiceVersion, serviceVersion);
 
-        return NativeInterop.LibraryConfig.StoreTracerMetadata(schemaVersion, runtimeIdCharSlice, tracerLanguageCharSlice, tracerVersionCharSlice, hostnameCharSlice, serviceNameCharSlice, serviceEnvCharSlice, serviceVersionCharSlice);
+            return NativeInterop.LibraryConfig.StoreTracerMetadata(ptr);
+        }
+        finally
+        {
+            if (ptr != IntPtr.Zero)
+            {
+                NativeInterop.LibraryConfig.TracerMetadataFree(ptr);
+            }
+        }
+
+        void SetMetadata(IntPtr ptr, MetadataKind kind, string? value)
+        {
+            using var valueCharSlice = new CString(value);
+            NativeInterop.LibraryConfig.TracerMetadataSet(ptr, kind, valueCharSlice);
+        }
     }
 }

tracer/test/Datadog.Trace.TestHelpers/MockTracerAgent.cs

@@ -113,7 +113,7 @@ protected MockTracerAgent(bool telemetryEnabled, TestTransports transport)
         /// </summary>
         public bool ShouldDeserializeTraces { get; set; } = true;
 
-        public static TcpUdpAgent Create(ITestOutputHelper output, int? port = null, int retries = 5, bool useStatsd = false, bool doNotBindPorts = false, int? requestedStatsDPort = null, bool useTelemetry = false, AgentConfiguration agentConfiguration = null)
+        public static TcpUdpAgent Create(ITestOutputHelper output, int? port = null, int retries = 5, bool useStatsd = false, bool doNotBindPorts = false, int? requestedStatsDPort = null, bool useTelemetry = true, AgentConfiguration agentConfiguration = null)
             => new TcpUdpAgent(port, retries, useStatsd, doNotBindPorts, requestedStatsDPort, useTelemetry) { Output = output, Configuration = agentConfiguration ?? new() };
 
 #if NETCOREAPP3_1_OR_GREATER