[OTEL] OTLP Logs API Support (#7680)

## Summary of changes
This PR implements OTLP Logs export for ILogger as part of the
OpenTelemetry Logs RFC Key additions:

- Custom OTLP Serializer: Protobuf writer using vendored OpenTelemetry
utilities (no SDK dependency)
- Multi-Protocol Support: gRPC transports and HTTP/protobuf
- Auto-Instrumentation: Zero-touch enablement via
DD_LOGS_OTEL_ENABLED=true
- Configurations: Full support for OTEL_EXPORTER_OTLP_* environment
variables

## Reason for change
The [RFC] OpenTelemetry Logs API support enables customers using
OpenTelemetry standards to migrate to Datadog SDK with minimal friction
while allowing customers to continue to receive their logs as expected.

## Implementation details
✅ Core Components:
- `OtlpLogsSerializer` - Custom protobuf writer using vendored OTLP
field constants (no external dependencies)
- `OtlpExporter` - Multi-protocol exporter (HTTP/protobuf and gRPC) with
configurable endpoint/headers/timeout
- `OtlpSubmissionLogSink` - Batching sink that integrates with existing
Direct Log Submission pipeline
- `LogPoint` - Internal data model representing OTLP log records with
trace/span IDs and attributes

✅ Vendored OpenTelemetry Components:
- `ProtobufSerializer` - Low-level protobuf writer
- `ProtobufOtlpLogFieldNumberConstants` - Official OTLP field numbers
- `OtlpHttpExportClient` - HTTP/protobuf transport client

✅ Integration:
- `DirectSubmissionLogger` auto-instrumentation detects
`OtlpSubmissionLogSink`and creates `LogPoint` objects
- When OTLP logs are enabled, they take precedence over Datadog Direct
Log Submission to send logs over OTLP instead.

## Test coverage

Unit Tests:
- OtlpSinkTests - Validates batching, trace context injection, attribute
handling, severity mapping
- TracerSettingsTests - Configuration parsing (protocol, endpoint,
headers, timeout fallbacks)

Integration Tests:
- OpenTelemetrySdkTests.SubmitsOtlpLogs - End-to-end validation with
snapshot testing
- Verifies OTLP payload structure and resource attributes
- Confirms trace-log correlation with active spans

Author: link04

tracer/build/_build/UpdateVendors/VendoredDependency.cs

@@ -167,11 +167,20 @@ static VendoredDependency()
                     AddOpenTelemetryUsings),
                 relativePathsToExclude: new[]
                 {
-                    // Vendor only the gRPC transport client - exclude everything else
+                    // Vendor gRPC and HTTP export clients for logs and metrics
+                    // Vendor low-level protobuf utilities: ProtobufSerializer, ProtobufWireType
+                    // Vendor ONLY field constants we actually use (Logs and Common)
+                    // EXCLUDE high-level serializers that depend on OpenTelemetry SDK types
                     ".publicApi/",
                     "Builder/",
                     "PersistentStorage/",
-                    "Implementation/Serializer/",
+                    "Implementation/Serializer/ProtobufOtlpLogSerializer.cs",          
+                    "Implementation/Serializer/ProtobufOtlpMetricSerializer.cs",        
+                    "Implementation/Serializer/ProtobufOtlpTraceSerializer.cs",        
+                    "Implementation/Serializer/ProtobufOtlpResourceSerializer.cs",     
+                    "Implementation/Serializer/ProtobufOtlpTagWriter.cs",              
+                    "Implementation/Serializer/ProtobufOtlpMetricFieldNumberConstants.cs",  // Not used - metrics has own FieldNumbers ATM
+                    "Implementation/Serializer/ProtobufOtlpTraceFieldNumberConstants.cs",
                     "Implementation/Transmission/",
                     "Implementation/ActivityExtensions.cs",
                     "Implementation/ExperimentalOptions.cs",
@@ -182,7 +191,6 @@ static VendoredDependency()
                     "Implementation/OtlpExporterOptionsConfigurationType.cs",
                     "Implementation/OtlpSpecConfigDefinitions.cs",
                     "Implementation/TimestampHelpers.cs",
-                    "Implementation/ExportClient/OtlpHttpExportClient.cs",  // We only need gRPC for this PR
                     "Implementation/ExportClient/OtlpRetry.cs",
                     "CHANGELOG.md",
                     "README.md",

tracer/src/Datadog.Trace.Trimming/build/Datadog.Trace.Trimming.xml

@@ -237,6 +237,10 @@
       <type fullname="System.Diagnostics.DebuggerTypeProxyAttribute" />
    </assembly>
    <assembly fullname="System.Diagnostics.DiagnosticSource">
+      <type fullname="System.Diagnostics.Activity" />
+      <type fullname="System.Diagnostics.ActivityIdFormat" />
+      <type fullname="System.Diagnostics.ActivitySpanId" />
+      <type fullname="System.Diagnostics.ActivityTraceId" />
       <type fullname="System.Diagnostics.DiagnosticListener" />
       <type fullname="System.Diagnostics.DistributedContextPropagator" />
       <type fullname="System.Diagnostics.Metrics.Instrument" />

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Logging/ILogger/DirectSubmission/DatadogLogEventCreator.cs

@@ -0,0 +1,62 @@
+// <copyright file="DatadogLogEventCreator.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+#nullable enable
+
+using System;
+using Datadog.Trace.ClrProfiler.AutoInstrumentation.Logging.ILogger.DirectSubmission.Formatting;
+using Datadog.Trace.Logging.DirectSubmission.Formatting;
+
+namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Logging.ILogger.DirectSubmission;
+
+/// <summary>
+/// Creates log events in Datadog format
+/// </summary>
+internal class DatadogLogEventCreator : ILogEventCreator
+{
+    private readonly LogFormatter _logFormatter;
+    private readonly IExternalScopeProvider? _scopeProvider;
+
+    public DatadogLogEventCreator(LogFormatter logFormatter, IExternalScopeProvider? scopeProvider)
+    {
+        _logFormatter = logFormatter;
+        _scopeProvider = scopeProvider;
+    }
+
+    public LoggerDirectSubmissionLogEvent CreateLogEvent<TState>(
+        int logLevel,
+        string categoryName,
+        object eventId,
+        TState state,
+        Exception? exception,
+        Func<TState, Exception?, string> formatter)
+    {
+        var logEntry = new LogEntry<TState>(
+            DateTime.UtcNow,
+            logLevel,
+            categoryName,
+            eventId.GetHashCode(),
+            state,
+            exception,
+            formatter,
+            _scopeProvider);
+
+        var serializedLog = LoggerLogFormatter.FormatLogEvent(_logFormatter, logEntry);
+        return new LoggerDirectSubmissionLogEvent(serializedLog);
+    }
+
+    public IDisposable BeginScope<TState>(TState state)
+    {
+        return _scopeProvider?.Push(state) ?? NullDisposable.Instance;
+    }
+
+    private class NullDisposable : IDisposable
+    {
+        public static readonly NullDisposable Instance = new();
+
+        public void Dispose()
+        {
+        }
+    }
+}

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Logging/ILogger/DirectSubmission/DirectSubmissionLogger.cs

@@ -5,10 +5,8 @@
 #nullable enable
 
 using System;
-using Datadog.Trace.ClrProfiler.AutoInstrumentation.Logging.ILogger.DirectSubmission.Formatting;
 using Datadog.Trace.DuckTyping;
 using Datadog.Trace.Logging.DirectSubmission;
-using Datadog.Trace.Logging.DirectSubmission.Formatting;
 using Datadog.Trace.Logging.DirectSubmission.Sink;
 using Datadog.Trace.Telemetry;
 using Datadog.Trace.Telemetry.Metrics;
@@ -21,22 +19,19 @@ namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Logging.ILogger.DirectSu
     internal class DirectSubmissionLogger
     {
         private readonly string _name;
-        private readonly IExternalScopeProvider? _scopeProvider;
         private readonly IDirectSubmissionLogSink _sink;
-        private readonly LogFormatter? _logFormatter;
+        private readonly ILogEventCreator _logEventCreator;
         private readonly int _minimumLogLevel;
 
         internal DirectSubmissionLogger(
             string name,
-            IExternalScopeProvider? scopeProvider,
             IDirectSubmissionLogSink sink,
-            LogFormatter? logFormatter,
+            ILogEventCreator logEventCreator,
             DirectSubmissionLogLevel minimumLogLevel)
         {
             _name = name;
-            _scopeProvider = scopeProvider;
             _sink = sink;
-            _logFormatter = logFormatter;
+            _logEventCreator = logEventCreator;
             _minimumLogLevel = (int)minimumLogLevel;
         }
 
@@ -57,24 +52,7 @@ public void Log<TState>(int logLevel, object eventId, TState state, Exception? e
                 return;
             }
 
-            // We render the event to a string immediately as we need to capture the properties
-            // This is more expensive from a CPU perspective, but saves having to persist the
-            // properties to a dictionary and rendering later
-
-            var logEntry = new LogEntry<TState>(
-                DateTime.UtcNow,
-                logLevel,
-                _name,
-                eventId.GetHashCode(),
-                state,
-                exception,
-                formatter,
-                _scopeProvider);
-            var logFormatter = _logFormatter ?? TracerManager.Instance.DirectLogSubmission.Formatter;
-            var serializedLog = LoggerLogFormatter.FormatLogEvent(logFormatter, logEntry);
-
-            var log = new LoggerDirectSubmissionLogEvent(serializedLog);
-
+            var log = _logEventCreator.CreateLogEvent(logLevel, _name, eventId, state, exception, formatter);
             TelemetryFactory.Metrics.RecordCountDirectLogLogs(MetricTags.IntegrationName.ILogger);
             _sink.EnqueueLog(log);
         }
@@ -94,15 +72,6 @@ public void Log<TState>(int logLevel, object eventId, TState state, Exception? e
         /// <typeparam name="TState">The type of the state to begin scope for.</typeparam>
         /// <returns>An <see cref="IDisposable"/> that ends the logical operation scope on dispose.</returns>
         [DuckReverseMethod(ParameterTypeNames = new[] { "TState" })]
-        public IDisposable BeginScope<TState>(TState state) => _scopeProvider?.Push(state) ?? NullDisposable.Instance;
-
-        private class NullDisposable : IDisposable
-        {
-            public static readonly NullDisposable Instance = new();
-
-            public void Dispose()
-            {
-            }
-        }
+        public IDisposable BeginScope<TState>(TState state) => _logEventCreator.BeginScope(state);
     }
 }

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Logging/ILogger/DirectSubmission/DirectSubmissionLoggerProvider.cs

@@ -6,7 +6,6 @@
 
 using System;
 using System.Collections.Concurrent;
-using System.ComponentModel;
 using Datadog.Trace.DuckTyping;
 using Datadog.Trace.Logging.DirectSubmission;
 using Datadog.Trace.Logging.DirectSubmission.Formatting;
@@ -23,9 +22,9 @@ internal class DirectSubmissionLoggerProvider
         private readonly Func<string, DirectSubmissionLogger> _createLoggerFunc;
         private readonly ConcurrentDictionary<string, DirectSubmissionLogger> _loggers = new();
         private readonly IDirectSubmissionLogSink _sink;
-        private readonly LogFormatter? _formatter;
         private readonly DirectSubmissionLogLevel _minimumLogLevel;
-        private IExternalScopeProvider? _scopeProvider;
+        private readonly LogFormatter? _logFormatter;
+        private ILogEventCreator _logEventCreator;
 
         internal DirectSubmissionLoggerProvider(IDirectSubmissionLogSink sink, DirectSubmissionLogLevel minimumLogLevel, IExternalScopeProvider? scopeProvider)
             : this(sink, formatter: null, minimumLogLevel, scopeProvider)
@@ -40,10 +39,19 @@ internal DirectSubmissionLoggerProvider(
             IExternalScopeProvider? scopeProvider)
         {
             _sink = sink;
-            _formatter = formatter;
             _minimumLogLevel = minimumLogLevel;
             _createLoggerFunc = CreateLoggerImplementation;
-            _scopeProvider = scopeProvider;
+
+#if NETCOREAPP3_1_OR_GREATER
+            if (sink is OtlpSubmissionLogSink)
+            {
+                _logEventCreator = new OtelLogEventCreator();
+                return;
+            }
+#endif
+
+            _logFormatter = formatter ?? TracerManager.Instance.DirectLogSubmission.Formatter;
+            _logEventCreator = new DatadogLogEventCreator(_logFormatter, scopeProvider);
         }
 
         /// <summary>
@@ -59,7 +67,7 @@ public DirectSubmissionLogger CreateLogger(string categoryName)
 
         private DirectSubmissionLogger CreateLoggerImplementation(string name)
         {
-            return new DirectSubmissionLogger(name, _scopeProvider, _sink, _formatter, _minimumLogLevel);
+            return new DirectSubmissionLogger(name, _sink, _logEventCreator, _minimumLogLevel);
         }
 
         /// <inheritdoc cref="IDisposable.Dispose"/>
@@ -75,7 +83,10 @@ public void Dispose()
         [DuckReverseMethod(ParameterTypeNames = new[] { "Microsoft.Extensions.Logging.IExternalScopeProvider, Microsoft.Extensions.Logging.Abstractions" })]
         public void SetScopeProvider(IExternalScopeProvider scopeProvider)
         {
-            _scopeProvider = scopeProvider;
+            if (_logFormatter != null)
+            {
+                _logEventCreator = new DatadogLogEventCreator(_logFormatter, scopeProvider);
+            }
         }
     }
 }

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Logging/ILogger/DirectSubmission/Formatting/OtlpLogEventBuilder.cs

@@ -0,0 +1,106 @@
+// <copyright file="OtlpLogEventBuilder.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#if NETCOREAPP3_1_OR_GREATER
+#nullable enable
+
+using System;
+using System.Collections.Generic;
+using Datadog.Trace.OpenTelemetry.Logs;
+using Datadog.Trace.Util;
+
+namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Logging.ILogger.DirectSubmission.Formatting
+{
+    /// <summary>
+    /// Helper class to build OTLP log events from ILogger log calls.
+    /// Extracts structured data and trace context for OTLP export.
+    /// </summary>
+    internal static class OtlpLogEventBuilder
+    {
+        /// <summary>
+        /// Creates a LoggerDirectSubmissionLogEvent with OTLP structured data from ILogger state.
+        /// </summary>
+        public static LoggerDirectSubmissionLogEvent CreateLogEvent<TState>(
+            int logLevel,
+            string categoryName,
+            object eventId,
+            TState state,
+            Exception? exception,
+            Func<TState, Exception?, string> formatter)
+        {
+            var message = formatter(state, exception);
+            var attributes = ExtractAttributes(state, eventId);
+            var (traceId, spanId, flags) = ExtractTraceContext();
+
+            return new LoggerDirectSubmissionLogEvent(null)
+            {
+                OtlpLog = new LogPoint
+                {
+                    Message = message,
+                    LogLevel = logLevel,
+                    CategoryName = categoryName,
+                    Timestamp = DateTimeOffset.UtcNow.DateTime,
+                    Exception = exception,
+                    Attributes = attributes,
+                    TraceId = traceId,
+                    SpanId = spanId,
+                    Flags = flags
+                }
+            };
+        }
+
+        private static Dictionary<string, object?> ExtractAttributes<TState>(TState state, object eventId)
+        {
+            var attributes = new Dictionary<string, object?>();
+
+            // Extract structured properties from ILogger state
+            if (state is IReadOnlyList<KeyValuePair<string, object?>> properties)
+            {
+                foreach (var property in properties)
+                {
+                    if (!string.IsNullOrEmpty(property.Key) && property.Key != "{OriginalFormat}")
+                    {
+                        attributes[property.Key] = property.Value;
+                    }
+                }
+            }
+
+            // Add EventId if present
+            if (eventId.GetHashCode() != 0)
+            {
+                attributes["EventId"] = eventId.GetHashCode();
+            }
+
+            return attributes;
+        }
+
+        private static (TraceId TraceId, ulong SpanId, int Flags) ExtractTraceContext()
+        {
+            // Prefer Datadog's trace context for accurate correlation with DD spans
+            // Fallback to Activity if Datadog tracing is not active
+            var ddSpan = Tracer.Instance.ActiveScope?.Span as Span;
+            if (ddSpan != null)
+            {
+                return (ddSpan.TraceId128, ddSpan.SpanId, SamplingPriorityValues.IsKeep(ddSpan.Context.SamplingPriority!.Value) ? 1 : 0);
+            }
+
+            var activity = System.Diagnostics.Activity.Current;
+            if (activity != null && activity.IdFormat == System.Diagnostics.ActivityIdFormat.W3C)
+            {
+                if (HexString.TryParseTraceId(activity.TraceId.ToHexString(), out var activityTraceId) &&
+                    HexString.TryParseUInt64(activity.SpanId.ToHexString(), out var activitySpanId))
+                {
+                    var flags = activity.Recorded ? 1 : 0;
+                    return (activityTraceId, activitySpanId, flags);
+                }
+            }
+
+            return (TraceId.Zero, 0, 0);
+        }
+    }
+}
+
+#endif
+

tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/Logging/ILogger/DirectSubmission/ILogEventCreator.cs

@@ -0,0 +1,25 @@
+// <copyright file="ILogEventCreator.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+#nullable enable
+
+using System;
+
+namespace Datadog.Trace.ClrProfiler.AutoInstrumentation.Logging.ILogger.DirectSubmission;
+
+/// <summary>
+/// Creates DirectSubmissionLogEvent instances from ILogger calls
+/// </summary>
+internal interface ILogEventCreator
+{
+    LoggerDirectSubmissionLogEvent CreateLogEvent<TState>(
+        int logLevel,
+        string categoryName,
+        object eventId,
+        TState state,
+        Exception? exception,
+        Func<TState, Exception?, string> formatter);
+
+    IDisposable BeginScope<TState>(TState state);
+}