Fill gaps in otel exporter implementation

lib/instana/backend/host_agent_reporting_observer.rb

@@ -186,11 +186,12 @@ def initialize_otlp_exporter
 
         endpoint = resolve_otlp_endpoint(config[:endpoint], config[:config_source])
         opts = { endpoint: endpoint, timeout: config[:timeout] / 1000.0 }
-        opts[:compression]            = config[:compression]         if config[:compression]
-        opts[:headers]                = config[:headers]             if config[:headers]&.any?
-        opts[:certificate_file]       = config[:certificate] if config[:certificate]
-        opts[:client_certificate_file] = config[:client_certificate] if config[:client_certificate]
-        opts[:client_key_file] = config[:client_key] if config[:client_key]
+        opts[:compression]             = config[:compression]          if config[:compression]
+        opts[:headers]                 = config[:headers]              if config[:headers]&.any?
+        opts[:certificate_file]        = config[:certificate]          if config[:certificate]
+        opts[:client_certificate_file] = config[:client_certificate]   if config[:client_certificate]
+        opts[:client_key_file]         = config[:client_key]           if config[:client_key]
+        opts[:ssl_verify_mode]         = OpenSSL::SSL::VERIFY_NONE     if config[:insecure]
 
         @otlp_exporter = OpenTelemetry::Exporter::OTLP::Exporter.new(**opts)
       rescue StandardError => e

lib/instana/config.rb

@@ -4,7 +4,7 @@
 require 'yaml'
 
 module Instana
-  class Config
+  class Config # rubocop:disable Metrics/ClassLength
 
     LEGACY_TRACING_KEY = 'com.instana.tracing'.freeze
     TRACING_KEY = 'tracing'.freeze
@@ -63,6 +63,9 @@ def initialize(logger: ::Instana.logger, agent_host: ENV.fetch('INSTANA_AGENT_HO
         certificate: nil,
         client_key: nil,
         client_certificate: nil,
+        protocol: 'http/protobuf',
+        semconv_stability: 'stable',
+        insecure: false,
         config_source: 'default'
       }
       read_otlp_config
@@ -257,6 +260,9 @@ def read_otlp_config_from_agent(tracing_config)
       @config[:otlp][:certificate]        = otlp_config['certificate']              if otlp_config['certificate']
       @config[:otlp][:client_key]         = otlp_config['client_key']               if otlp_config['client_key']
       @config[:otlp][:client_certificate] = otlp_config['client_certificate']       if otlp_config['client_certificate']
+      @config[:otlp][:protocol]           = otlp_config['protocol']                 if otlp_config['protocol']
+      @config[:otlp][:semconv_stability]  = otlp_config['semconv_stability']        if otlp_config['semconv_stability']
+      @config[:otlp][:insecure]           = truthy?(otlp_config['insecure'])        unless otlp_config['insecure'].nil?
       @config[:otlp][:config_source]      = 'agent'
     end
 
@@ -381,6 +387,9 @@ def build_otlp_yaml_result(otlp_yaml)
       result[:certificate]        = otlp_yaml['certificate']        if otlp_yaml['certificate']
       result[:client_key]         = otlp_yaml['client_key']         if otlp_yaml['client_key']
       result[:client_certificate] = otlp_yaml['client_certificate'] if otlp_yaml['client_certificate']
+      result[:protocol]           = otlp_yaml['protocol']           if otlp_yaml['protocol']
+      result[:semconv_stability]  = otlp_yaml['semconv_stability']  if otlp_yaml['semconv_stability']
+      result[:insecure]           = truthy?(otlp_yaml['insecure'])  unless otlp_yaml['insecure'].nil?
       result.empty? ? nil : result
     end
 
@@ -399,6 +408,9 @@ def parse_otlp_config_from_env
       result[:certificate]        = raw[:certificate]                       if raw[:certificate]
       result[:client_key]         = raw[:client_key]                        if raw[:client_key]
       result[:client_certificate] = raw[:client_cert]                       if raw[:client_cert]
+      result[:protocol]           = raw[:protocol]                          if raw[:protocol]
+      result[:semconv_stability]  = raw[:semconv_stability]                 if raw[:semconv_stability]
+      result[:insecure]           = truthy?(raw[:insecure_raw])             unless raw[:insecure_raw].nil?
       result
     end
 
@@ -413,7 +425,10 @@ def otlp_env_vars
         headers_raw: ENV.fetch('OTEL_EXPORTER_OTLP_TRACES_HEADERS', nil) || ENV.fetch('OTEL_EXPORTER_OTLP_HEADERS', nil),
         certificate: ENV.fetch('OTEL_EXPORTER_OTLP_CERTIFICATE', nil),
         client_key: ENV.fetch('OTEL_EXPORTER_OTLP_CLIENT_KEY', nil),
-        client_cert: ENV.fetch('OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE', nil)
+        client_cert: ENV.fetch('OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE', nil),
+        protocol: ENV.fetch('OTEL_EXPORTER_OTLP_PROTOCOL', nil),
+        semconv_stability: ENV.fetch('OTEL_SEMCONV_STABILITY_OPT_IN', nil),
+        insecure_raw: ENV.fetch('OTEL_EXPORTER_OTLP_INSECURE', nil)
       }
     end
 

lib/instana/exporter/otlp/aws_converter.rb

@@ -6,6 +6,9 @@
 require 'opentelemetry/semconv/incubating/messaging'
 require 'opentelemetry/semconv/db'
 require 'opentelemetry/semconv/incubating/db'
+require 'opentelemetry/semconv/incubating/rpc'
+require 'opentelemetry/semconv/incubating/cloud'
+require 'opentelemetry/semconv/incubating/faas'
 
 module Instana
   module Exporter
@@ -128,10 +131,18 @@ def convert_dynamodb_attributes(attributes, dynamodb_data)
         def convert_s3_attributes(attributes, s3_data)
           return unless s3_data
 
-          add_attribute(attributes, 'aws.service', 's3')
+          add_attribute(attributes, Instana::Exporter::Otlp::RPC_SYSTEM_ATTR, 'aws-api')
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::RPC::RPC_METHOD, s3_rpc_method(s3_data[:op]))
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::CLOUD::CLOUD_PROVIDER, 'aws')
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::CLOUD::CLOUD_REGION, s3_data[:region])
           add_attribute(attributes, 'aws.s3.bucket', s3_data[:bucket])
           add_attribute(attributes, 'aws.s3.key', s3_data[:key])
-          add_attribute(attributes, 'aws.s3.operation', s3_data[:op])
+        end
+
+        def s3_rpc_method(operation)
+          return unless operation
+
+          "S3.#{operation}"
         end
 
         def convert_lambda_attributes(attributes, lambda_data)
@@ -140,7 +151,7 @@ def convert_lambda_attributes(attributes, lambda_data)
           add_attribute(attributes, 'aws.service', 'lambda')
           add_attribute(attributes, 'aws.lambda.function_name', lambda_data[:function])
           add_attribute(attributes, 'aws.lambda.invocation_type', lambda_data[:type])
-          add_attribute(attributes, 'faas.invoked_name', lambda_data[:function])
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::FAAS::FAAS_INVOKED_NAME, lambda_data[:function])
         end
       end
     end

lib/instana/exporter/otlp/background_job_converter.rb

@@ -41,6 +41,8 @@ def convert_attributes
             convert_job_attributes(attributes, span[:'resque-client'] || span[:data]&.[](:'resque-client'), 'resque', 'publish')
           elsif span_type == 'resque-worker'
             convert_job_attributes(attributes, span[:'resque-worker'] || span[:data]&.[](:'resque-worker'), 'resque', 'process')
+          elsif span_type == 'activejob'
+            convert_activejob_attributes(attributes, span[:data]&.[](:activejob))
           end
 
           attributes
@@ -53,13 +55,22 @@ def convert_job_attributes(attributes, data, system, operation)
 
           add_attribute(attributes, OpenTelemetry::SemConv::Incubating::MESSAGING::MESSAGING_SYSTEM, system)
           add_attribute(attributes, OpenTelemetry::SemConv::Incubating::MESSAGING::MESSAGING_DESTINATION_NAME, data[:queue] || data['queue'])
-          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::MESSAGING::MESSAGING_OPERATION, operation)
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::MESSAGING::MESSAGING_OPERATION_NAME, operation)
           add_attribute(attributes, OpenTelemetry::SemConv::Incubating::MESSAGING::MESSAGING_MESSAGE_ID, data[:job_id] || data['job_id'])
           add_attribute(attributes, OpenTelemetry::SemConv::Incubating::MESSAGING::MESSAGING_CONSUMER_GROUP_NAME, data[:job] || data['job'])
           add_attribute(attributes, OpenTelemetry::SemConv::SERVER::SERVER_ADDRESS, extract_host(data[:'redis-url'] || data['redis-url']))
           add_attribute(attributes, OpenTelemetry::SemConv::SERVER::SERVER_PORT, extract_port(data[:'redis-url'] || data['redis-url']))
         end
 
+        def convert_activejob_attributes(attributes, data)
+          return unless data
+
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::MESSAGING::MESSAGING_SYSTEM, 'activejob')
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::MESSAGING::MESSAGING_DESTINATION_NAME, data[:queue] || data['queue'])
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::MESSAGING::MESSAGING_OPERATION_NAME, data[:action] || data['action'])
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::MESSAGING::MESSAGING_MESSAGE_ID, data[:job_id] || data['job_id'])
+        end
+
         def extract_host(connection)
           return nil unless connection
 

lib/instana/exporter/otlp/base_converter.rb

@@ -5,10 +5,22 @@
 require_relative 'resource'
 require 'opentelemetry/trace'
 require 'forwardable'
+require 'opentelemetry/semconv/incubating/rpc'
 
 module Instana
   module Exporter
     module Otlp
+      # Compatibility shim: RPC_SYSTEM_NAME was added in a newer release of
+      # opentelemetry-semantic_conventions.  Fall back to the deprecated
+      # RPC_SYSTEM constant ('rpc.system') on older installs (e.g. Ruby 3.2
+      # lockfiles that resolve an earlier gem version).
+      RPC_SYSTEM_ATTR =
+        if OpenTelemetry::SemConv::Incubating::RPC.const_defined?(:RPC_SYSTEM_NAME)
+          OpenTelemetry::SemConv::Incubating::RPC::RPC_SYSTEM_NAME
+        else
+          OpenTelemetry::SemConv::Incubating::RPC::RPC_SYSTEM
+        end
+
       # Base class for all OTLP span converters
       #
       # Provides common interface and shared functionality for converting Instana spans
@@ -312,12 +324,12 @@ def build_error_events(error_count, error_msg, stacktrace)
           timestamp = calculate_end_timestamp
 
           if stacktrace
-            attrs = { 'exception.type' => span_name }
+            attrs = {}
             attrs['exception.message']    = error_msg if error_msg
             attrs['exception.stacktrace'] = stacktrace
             [Event.new('exception', attrs, timestamp)]
           else
-            [Event.new('error', { 'error.type' => span_name }, timestamp)]
+            [Event.new('error', {}, timestamp)]
           end
         end
 

lib/instana/exporter/otlp/database_converter.rb

@@ -82,32 +82,51 @@ def mongo_span_name(mongo)
           parts.empty? ? 'mongodb' : parts.join('.')
         end
 
+        # Normalize an ActiveRecord/Sequel adapter name to OTel db.system.name.
+        # Only adapters that are actively instrumented and tested are listed here.
+        DB_SYSTEM_NORMALIZATION = {
+          'mysql2' => 'mysql',
+          'sqlite3' => 'sqlite',
+          'postgresql' => 'postgresql',
+          'mysql' => 'mysql',
+          'sqlite' => 'sqlite'
+        }.freeze
+
+        def normalize_db_system(adapter)
+          return unless adapter
+
+          DB_SYSTEM_NORMALIZATION.fetch(adapter.to_s.downcase, adapter.to_s.downcase)
+        end
+
         def convert_activerecord_attributes(attributes, ar_data)
           return unless ar_data
 
-          add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_SYSTEM_NAME, ar_data[:adapter])
+          add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_SYSTEM_NAME, normalize_db_system(ar_data[:adapter]))
           add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_NAMESPACE, ar_data[:db])
           add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_QUERY_TEXT, ar_data[:sql])
           add_attribute(attributes, OpenTelemetry::SemConv::Incubating::DB::DB_USER, ar_data[:username])
           add_attribute(attributes, OpenTelemetry::SemConv::SERVER::SERVER_ADDRESS, ar_data[:host])
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::DB::DB_COLLECTION_NAME, ar_data[:table])
         end
 
         def convert_sequel_attributes(attributes, seq_data)
           return unless seq_data
 
-          add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_SYSTEM_NAME, seq_data[:adapter])
+          add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_SYSTEM_NAME, normalize_db_system(seq_data[:adapter]))
           add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_NAMESPACE, seq_data[:db])
           add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_QUERY_TEXT, seq_data[:sql])
           add_attribute(attributes, OpenTelemetry::SemConv::Incubating::DB::DB_USER, seq_data[:username])
           add_attribute(attributes, OpenTelemetry::SemConv::SERVER::SERVER_ADDRESS, seq_data[:host])
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::DB::DB_COLLECTION_NAME, seq_data[:table])
         end
 
         def convert_redis_attributes(attributes, redis_data)
           return unless redis_data
 
           add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_SYSTEM_NAME, 'redis')
+          add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_OPERATION_NAME, redis_data[:command])
           add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_QUERY_TEXT, redis_data[:command])
-          add_attribute(attributes, 'db.redis.database_index', redis_data[:db])
+          add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_NAMESPACE, redis_data[:db]&.to_s)
           add_attribute(attributes, OpenTelemetry::SemConv::SERVER::SERVER_ADDRESS, extract_host(redis_data[:connection]))
           add_attribute(attributes, OpenTelemetry::SemConv::SERVER::SERVER_PORT, extract_port(redis_data[:connection]))
         end
@@ -119,7 +138,7 @@ def convert_memcache_attributes(attributes, mc_data)
           add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_OPERATION_NAME, mc_data[:command])
           add_attribute(attributes, 'db.memcached.key', mc_data[:key])
           add_attribute(attributes, 'db.memcached.keys', mc_data[:keys])
-          add_attribute(attributes, 'db.memcached.namespace', mc_data[:namespace])
+          add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_NAMESPACE, mc_data[:namespace])
           add_attribute(attributes, OpenTelemetry::SemConv::SERVER::SERVER_ADDRESS, extract_host(mc_data[:server]))
           add_attribute(attributes, OpenTelemetry::SemConv::SERVER::SERVER_PORT, extract_port(mc_data[:server]))
         end
@@ -131,6 +150,7 @@ def convert_mongo_attributes(attributes, mongo_data)
           add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_NAMESPACE, mongo_data[:namespace])
           add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_OPERATION_NAME, mongo_data[:command])
           add_attribute(attributes, OpenTelemetry::SemConv::DB::DB_QUERY_TEXT, mongo_data[:json])
+          add_attribute(attributes, OpenTelemetry::SemConv::Incubating::DB::DB_COLLECTION_NAME, mongo_data[:collection])
           add_attribute(attributes, OpenTelemetry::SemConv::SERVER::SERVER_ADDRESS, mongo_data.dig(:peer, :hostname))
           add_attribute(attributes, OpenTelemetry::SemConv::SERVER::SERVER_PORT, mongo_data.dig(:peer, :port))
         end

lib/instana/exporter/otlp/http_converter.rb

@@ -7,6 +7,7 @@
 require 'opentelemetry/semconv/url'
 require 'opentelemetry/semconv/server'
 require 'opentelemetry/semconv/user_agent'
+require 'opentelemetry/semconv/network'
 
 module Instana
   module Exporter
@@ -120,8 +121,8 @@ def add_protocol_attributes(attributes, protocol)
           name    = parts[0].downcase
           version = parts[1]
 
-          add_attribute(attributes, 'network.protocol.name', name)
-          add_attribute(attributes, 'network.protocol.version', version)
+          add_attribute(attributes, OpenTelemetry::SemConv::NETWORK::NETWORK_PROTOCOL_NAME, name)
+          add_attribute(attributes, OpenTelemetry::SemConv::NETWORK::NETWORK_PROTOCOL_VERSION, version)
         end
       end
     end