impl(o11y): introduce resend_count logic

gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java

@@ -84,4 +84,10 @@ public class ObservabilityAttributes {
 
   /** The url template of the request (e.g. /v1/{name}:access). */
   public static final String URL_TEMPLATE_ATTRIBUTE = "url.template";
+
+  /** The resend count of the request. Only used in HTTP transport. */
+  public static final String HTTP_RESEND_COUNT_ATTRIBUTE = "http.request.resend_count";
+
+  /** The resend count of the request. Only used in gRPC transport. */
+  public static final String GRPC_RESEND_COUNT_ATTRIBUTE = "gcp.grpc.resend_count";
 }

gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java

@@ -65,6 +65,8 @@ static Attributes toOtelAttributes(Map<String, Object> attributes) {
         (k, v) -> {
           if (v instanceof String) {
             attributesBuilder.put(k, (String) v);
+          } else if (v instanceof Long) {
+            attributesBuilder.put(k, (Long) v);
           } else if (v instanceof Integer) {
             attributesBuilder.put(k, (long) (Integer) v);
           }

gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java

@@ -105,12 +105,26 @@ private void buildAttributes() {
 
   @Override
   public void attemptStarted(Object request, int attemptNumber) {
+    Map<String, Object> currentAttemptAttributes = new HashMap<>(this.attemptAttributes);
+
+    if (attemptNumber > 0) {
+      ApiTracerContext.Transport transport = apiTracerContext.transport();
+      if (transport == ApiTracerContext.Transport.GRPC) {
+        currentAttemptAttributes.put(
+            ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber);
+      } else if (transport == ApiTracerContext.Transport.HTTP) {
+        currentAttemptAttributes.put(
+            ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber);
+      }
+    }
+
     SpanBuilder spanBuilder = tracer.spanBuilder(attemptSpanName);
 
     // Attempt spans are of the CLIENT kind
     spanBuilder.setSpanKind(SpanKind.CLIENT);
 
-    spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(this.attemptAttributes));
+    // Pass the combined attributes to the new SpanBuilder method
+    spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(currentAttemptAttributes));
 
     this.attemptSpan = spanBuilder.startSpan();
   }
@@ -120,6 +134,26 @@ public void attemptSucceeded() {
     endAttempt();
   }
 
+  @Override
+  public void attemptCancelled() {
+    endAttempt();
+  }
+
+  @Override
+  public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
+    endAttempt();
+  }
+
+  @Override
+  public void attemptFailedRetriesExhausted(Throwable error) {
+    endAttempt();
+  }
+
+  @Override
+  public void attemptPermanentFailure(Throwable error) {
+    endAttempt();
+  }
+
   private void endAttempt() {
     if (attemptSpan != null) {
       attemptSpan.end();

gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java

@@ -86,4 +86,104 @@ void testAttemptStarted_includesLanguageAttribute() {
             io.opentelemetry.api.common.AttributeKey.stringKey(SpanTracer.LANGUAGE_ATTRIBUTE),
             SpanTracer.DEFAULT_LANGUAGE);
   }
+
+  @Test
+  void testAttemptStarted_noRetryAttributes_grpc() {
+    ApiTracerContext grpcContext =
+        ApiTracerContext.newBuilder()
+            .setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty())
+            .setTransport(ApiTracerContext.Transport.GRPC)
+            .build();
+    SpanTracer grpcTracer = new SpanTracer(tracer, grpcContext, ATTEMPT_SPAN_NAME);
+
+    // Initial attempt, attemptNumber is 0
+    grpcTracer.attemptStarted(new Object(), 0);
+    ArgumentCaptor<Attributes> attributesCaptor = ArgumentCaptor.forClass(Attributes.class);
+    verify(spanBuilder).setAllAttributes(attributesCaptor.capture());
+    assertThat(attributesCaptor.getValue().asMap())
+        .doesNotContainKey(
+            io.opentelemetry.api.common.AttributeKey.longKey(
+                ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE));
+    assertThat(attributesCaptor.getValue().asMap())
+        .doesNotContainKey(
+            io.opentelemetry.api.common.AttributeKey.longKey(
+                ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE));
+  }
+
+  @Test
+  void testAttemptStarted_retryAttributes_grpc() {
+    ApiTracerContext grpcContext =
+        ApiTracerContext.newBuilder()
+            .setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty())
+            .setTransport(ApiTracerContext.Transport.GRPC)
+            .build();
+    SpanTracer grpcTracer = new SpanTracer(tracer, grpcContext, ATTEMPT_SPAN_NAME);
+
+    // N-th retry, attemptNumber is 5
+    grpcTracer.attemptStarted(new Object(), 5);
+    ArgumentCaptor<Attributes> attributesCaptor = ArgumentCaptor.forClass(Attributes.class);
+    verify(spanBuilder).setAllAttributes(attributesCaptor.capture());
+    java.util.Map<io.opentelemetry.api.common.AttributeKey<?>, Object> capturedAttributes =
+        attributesCaptor.getValue().asMap();
+    assertThat(capturedAttributes)
+        .containsEntry(
+            io.opentelemetry.api.common.AttributeKey.longKey(
+                ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE),
+            5L);
+    assertThat(capturedAttributes)
+        .doesNotContainKey(
+            io.opentelemetry.api.common.AttributeKey.longKey(
+                ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE));
+  }
+
+  @Test
+  void testAttemptStarted_noRetryAttributes_http() {
+    ApiTracerContext httpContext =
+        ApiTracerContext.newBuilder()
+            .setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty())
+            .setTransport(ApiTracerContext.Transport.HTTP)
+            .build();
+    SpanTracer httpTracer = new SpanTracer(tracer, httpContext, ATTEMPT_SPAN_NAME);
+
+    // Initial attempt, attemptNumber is 0
+    httpTracer.attemptStarted(new Object(), 0);
+    ArgumentCaptor<Attributes> attributesCaptor = ArgumentCaptor.forClass(Attributes.class);
+    verify(spanBuilder).setAllAttributes(attributesCaptor.capture());
+    java.util.Map<io.opentelemetry.api.common.AttributeKey<?>, Object> capturedAttributes =
+        attributesCaptor.getValue().asMap();
+    assertThat(capturedAttributes)
+        .doesNotContainKey(
+            io.opentelemetry.api.common.AttributeKey.longKey(
+                ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE));
+    assertThat(capturedAttributes)
+        .doesNotContainKey(
+            io.opentelemetry.api.common.AttributeKey.longKey(
+                ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE));
+  }
+
+  @Test
+  void testAttemptStarted_retryAttributes_http() {
+    ApiTracerContext httpContext =
+        ApiTracerContext.newBuilder()
+            .setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty())
+            .setTransport(ApiTracerContext.Transport.HTTP)
+            .build();
+    SpanTracer httpTracer = new SpanTracer(tracer, httpContext, ATTEMPT_SPAN_NAME);
+
+    // N-th retry, attemptNumber is 5
+    httpTracer.attemptStarted(new Object(), 5);
+    ArgumentCaptor<Attributes> attributesCaptor = ArgumentCaptor.forClass(Attributes.class);
+    verify(spanBuilder).setAllAttributes(attributesCaptor.capture());
+    java.util.Map<io.opentelemetry.api.common.AttributeKey<?>, Object> capturedAttributes =
+        attributesCaptor.getValue().asMap();
+    assertThat(capturedAttributes)
+        .doesNotContainKey(
+            io.opentelemetry.api.common.AttributeKey.longKey(
+                ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE));
+    assertThat(capturedAttributes)
+        .containsEntry(
+            io.opentelemetry.api.common.AttributeKey.longKey(
+                ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE),
+            5L);
+  }
 }

java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java

@@ -31,13 +31,23 @@
 package com.google.showcase.v1beta1.it;
 
 import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
 
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.gax.core.NoCredentialsProvider;
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.api.gax.rpc.StatusCode;
+import com.google.api.gax.rpc.UnavailableException;
 import com.google.api.gax.tracing.ObservabilityAttributes;
 import com.google.api.gax.tracing.SpanTracer;
 import com.google.api.gax.tracing.SpanTracerFactory;
+import com.google.rpc.Status;
 import com.google.showcase.v1beta1.EchoClient;
 import com.google.showcase.v1beta1.EchoRequest;
+import com.google.showcase.v1beta1.EchoSettings;
 import com.google.showcase.v1beta1.it.util.TestClientInitializer;
+import com.google.showcase.v1beta1.stub.EchoStub;
+import com.google.showcase.v1beta1.stub.EchoStubSettings;
 import io.opentelemetry.api.GlobalOpenTelemetry;
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.trace.SpanKind;
@@ -200,4 +210,146 @@ void testTracing_successfulEcho_httpjson() throws Exception {
           .isEqualTo("v1beta1/echo:echo");
     }
   }
+
+  @Test
+  void testTracing_retry_grpc() throws Exception {
+    final int attempts = 5;
+    final StatusCode.Code statusCode = StatusCode.Code.UNAVAILABLE;
+    // A custom EchoClient is used in this test because retries have jitter, and we cannot
+    // predict the number of attempts that are scheduled for an RPC invocation otherwise.
+    // The custom retrySettings limit to a set number of attempts before the call gives up.
+    RetrySettings retrySettings =
+        RetrySettings.newBuilder()
+            .setTotalTimeout(org.threeten.bp.Duration.ofMillis(5000L))
+            .setMaxAttempts(attempts)
+            .build();
+
+    EchoStubSettings.Builder grpcEchoSettingsBuilder = EchoStubSettings.newBuilder();
+    grpcEchoSettingsBuilder
+        .echoSettings()
+        .setRetrySettings(retrySettings)
+        .setRetryableCodes(statusCode);
+    EchoSettings grpcEchoSettings = EchoSettings.create(grpcEchoSettingsBuilder.build());
+    grpcEchoSettings =
+        grpcEchoSettings.toBuilder()
+            .setCredentialsProvider(NoCredentialsProvider.create())
+            .setTransportChannelProvider(EchoSettings.defaultGrpcTransportProviderBuilder().build())
+            .setEndpoint("localhost:7469")
+            .build();
+
+    SpanTracerFactory tracingFactory = new SpanTracerFactory(openTelemetrySdk);
+
+    EchoStubSettings echoStubSettings =
+        (EchoStubSettings)
+            grpcEchoSettings.getStubSettings().toBuilder().setTracerFactory(tracingFactory).build();
+    EchoStub stub = echoStubSettings.createStub();
+    EchoClient grpcClient = EchoClient.create(stub);
+
+    EchoRequest echoRequest =
+        EchoRequest.newBuilder()
+            .setError(Status.newBuilder().setCode(statusCode.ordinal()).build())
+            .build();
+
+    assertThrows(UnavailableException.class, () -> grpcClient.echo(echoRequest));
+
+    List<SpanData> spans = spanExporter.getFinishedSpanItems();
+    assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry
+
+    // This single span represents the successful retry, which has resend_count=1
+    // The first attempt has no resend_count. The subsequent retries will have a resend_count,
+    // starting from 1.
+    List<Long> resendCounts =
+        spans.stream()
+            .map(
+                span ->
+                    (Long)
+                        span.getAttributes()
+                            .asMap()
+                            .get(
+                                AttributeKey.longKey(
+                                    ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE)))
+            .filter(java.util.Objects::nonNull)
+            .sorted()
+            .collect(java.util.stream.Collectors.toList());
+
+    List<Long> expectedCounts =
+        java.util.stream.LongStream.range(1, attempts)
+            .boxed()
+            .collect(java.util.stream.Collectors.toList());
+    assertThat(resendCounts).containsExactlyElementsIn(expectedCounts).inOrder();
+  }
+
+  @Test
+  void testTracing_retry_httpjson() throws Exception {
+    final int attempts = 5;
+    final StatusCode.Code statusCode = StatusCode.Code.UNAVAILABLE;
+    // A custom EchoClient is used in this test because retries have jitter, and we cannot
+    // predict the number of attempts that are scheduled for an RPC invocation otherwise.
+    // The custom retrySettings limit to a set number of attempts before the call gives up.
+    RetrySettings retrySettings =
+        RetrySettings.newBuilder()
+            .setTotalTimeout(org.threeten.bp.Duration.ofMillis(5000L))
+            .setMaxAttempts(attempts)
+            .build();
+
+    EchoStubSettings.Builder httpJsonEchoSettingsBuilder = EchoStubSettings.newHttpJsonBuilder();
+    httpJsonEchoSettingsBuilder
+        .echoSettings()
+        .setRetrySettings(retrySettings)
+        .setRetryableCodes(statusCode);
+    EchoSettings httpJsonEchoSettings = EchoSettings.create(httpJsonEchoSettingsBuilder.build());
+    httpJsonEchoSettings =
+        httpJsonEchoSettings.toBuilder()
+            .setCredentialsProvider(NoCredentialsProvider.create())
+            .setTransportChannelProvider(
+                EchoSettings.defaultHttpJsonTransportProviderBuilder()
+                    .setHttpTransport(
+                        new NetHttpTransport.Builder().doNotValidateCertificate().build())
+                    .setEndpoint("http://localhost:7469")
+                    .build())
+            .build();
+
+    SpanTracerFactory tracingFactory = new SpanTracerFactory(openTelemetrySdk);
+
+    EchoStubSettings echoStubSettings =
+        (EchoStubSettings)
+            httpJsonEchoSettings.getStubSettings().toBuilder()
+                .setTracerFactory(tracingFactory)
+                .build();
+    EchoStub stub = echoStubSettings.createStub();
+    EchoClient httpClient = EchoClient.create(stub);
+
+    EchoRequest echoRequest =
+        EchoRequest.newBuilder()
+            .setError(Status.newBuilder().setCode(statusCode.ordinal()).build())
+            .build();
+
+    assertThrows(UnavailableException.class, () -> httpClient.echo(echoRequest));
+
+    List<SpanData> spans = spanExporter.getFinishedSpanItems();
+    assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry
+
+    // This single span represents the successful retry, which has resend_count=1
+    // The first attempt has no resend_count. The subsequent retries will have a resend_count,
+    // starting from 1.
+    List<Long> resendCounts =
+        spans.stream()
+            .map(
+                span ->
+                    (Long)
+                        span.getAttributes()
+                            .asMap()
+                            .get(
+                                AttributeKey.longKey(
+                                    ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE)))
+            .filter(java.util.Objects::nonNull)
+            .sorted()
+            .collect(java.util.stream.Collectors.toList());
+
+    List<Long> expectedCounts =
+        java.util.stream.LongStream.range(1, attempts)
+            .boxed()
+            .collect(java.util.stream.Collectors.toList());
+    assertThat(resendCounts).containsExactlyElementsIn(expectedCounts).inOrder();
+  }
 }