diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java index 5ea52462ca..7cb2789a6d 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityAttributes.java @@ -84,4 +84,10 @@ public class ObservabilityAttributes { /** The url template of the request (e.g. /v1/{name}:access). */ public static final String URL_TEMPLATE_ATTRIBUTE = "url.template"; + + /** The resend count of the request. Only used in HTTP transport. */ + public static final String HTTP_RESEND_COUNT_ATTRIBUTE = "http.request.resend_count"; + + /** The resend count of the request. Only used in gRPC transport. */ + public static final String GRPC_RESEND_COUNT_ATTRIBUTE = "gcp.grpc.resend_count"; } diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java index f2a787fc95..2487964370 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/ObservabilityUtils.java @@ -65,6 +65,8 @@ static Attributes toOtelAttributes(Map attributes) { (k, v) -> { if (v instanceof String) { attributesBuilder.put(k, (String) v); + } else if (v instanceof Long) { + attributesBuilder.put(k, (Long) v); } else if (v instanceof Integer) { attributesBuilder.put(k, (long) (Integer) v); } diff --git a/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java b/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java index 5bb10374b4..a2690359dd 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/tracing/SpanTracer.java @@ -105,12 +105,26 @@ private void buildAttributes() { @Override public void attemptStarted(Object request, int attemptNumber) { + Map currentAttemptAttributes = new HashMap<>(this.attemptAttributes); + + if (attemptNumber > 0) { + ApiTracerContext.Transport transport = apiTracerContext.transport(); + if (transport == ApiTracerContext.Transport.GRPC) { + currentAttemptAttributes.put( + ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber); + } else if (transport == ApiTracerContext.Transport.HTTP) { + currentAttemptAttributes.put( + ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber); + } + } + SpanBuilder spanBuilder = tracer.spanBuilder(attemptSpanName); // Attempt spans are of the CLIENT kind spanBuilder.setSpanKind(SpanKind.CLIENT); - spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(this.attemptAttributes)); + // Pass the combined attributes to the new SpanBuilder method + spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(currentAttemptAttributes)); this.attemptSpan = spanBuilder.startSpan(); } @@ -120,6 +134,26 @@ public void attemptSucceeded() { endAttempt(); } + @Override + public void attemptCancelled() { + endAttempt(); + } + + @Override + public void attemptFailedDuration(Throwable error, java.time.Duration delay) { + endAttempt(); + } + + @Override + public void attemptFailedRetriesExhausted(Throwable error) { + endAttempt(); + } + + @Override + public void attemptPermanentFailure(Throwable error) { + endAttempt(); + } + private void endAttempt() { if (attemptSpan != null) { attemptSpan.end(); diff --git a/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java b/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java index 6d20cdfcf4..3e9fc53ce5 100644 --- a/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java +++ b/gax-java/gax/src/test/java/com/google/api/gax/tracing/SpanTracerTest.java @@ -86,4 +86,104 @@ void testAttemptStarted_includesLanguageAttribute() { io.opentelemetry.api.common.AttributeKey.stringKey(SpanTracer.LANGUAGE_ATTRIBUTE), SpanTracer.DEFAULT_LANGUAGE); } + + @Test + void testAttemptStarted_noRetryAttributes_grpc() { + ApiTracerContext grpcContext = + ApiTracerContext.newBuilder() + .setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty()) + .setTransport(ApiTracerContext.Transport.GRPC) + .build(); + SpanTracer grpcTracer = new SpanTracer(tracer, grpcContext, ATTEMPT_SPAN_NAME); + + // Initial attempt, attemptNumber is 0 + grpcTracer.attemptStarted(new Object(), 0); + ArgumentCaptor attributesCaptor = ArgumentCaptor.forClass(Attributes.class); + verify(spanBuilder).setAllAttributes(attributesCaptor.capture()); + assertThat(attributesCaptor.getValue().asMap()) + .doesNotContainKey( + io.opentelemetry.api.common.AttributeKey.longKey( + ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE)); + assertThat(attributesCaptor.getValue().asMap()) + .doesNotContainKey( + io.opentelemetry.api.common.AttributeKey.longKey( + ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE)); + } + + @Test + void testAttemptStarted_retryAttributes_grpc() { + ApiTracerContext grpcContext = + ApiTracerContext.newBuilder() + .setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty()) + .setTransport(ApiTracerContext.Transport.GRPC) + .build(); + SpanTracer grpcTracer = new SpanTracer(tracer, grpcContext, ATTEMPT_SPAN_NAME); + + // N-th retry, attemptNumber is 5 + grpcTracer.attemptStarted(new Object(), 5); + ArgumentCaptor attributesCaptor = ArgumentCaptor.forClass(Attributes.class); + verify(spanBuilder).setAllAttributes(attributesCaptor.capture()); + java.util.Map, Object> capturedAttributes = + attributesCaptor.getValue().asMap(); + assertThat(capturedAttributes) + .containsEntry( + io.opentelemetry.api.common.AttributeKey.longKey( + ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE), + 5L); + assertThat(capturedAttributes) + .doesNotContainKey( + io.opentelemetry.api.common.AttributeKey.longKey( + ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE)); + } + + @Test + void testAttemptStarted_noRetryAttributes_http() { + ApiTracerContext httpContext = + ApiTracerContext.newBuilder() + .setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty()) + .setTransport(ApiTracerContext.Transport.HTTP) + .build(); + SpanTracer httpTracer = new SpanTracer(tracer, httpContext, ATTEMPT_SPAN_NAME); + + // Initial attempt, attemptNumber is 0 + httpTracer.attemptStarted(new Object(), 0); + ArgumentCaptor attributesCaptor = ArgumentCaptor.forClass(Attributes.class); + verify(spanBuilder).setAllAttributes(attributesCaptor.capture()); + java.util.Map, Object> capturedAttributes = + attributesCaptor.getValue().asMap(); + assertThat(capturedAttributes) + .doesNotContainKey( + io.opentelemetry.api.common.AttributeKey.longKey( + ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE)); + assertThat(capturedAttributes) + .doesNotContainKey( + io.opentelemetry.api.common.AttributeKey.longKey( + ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE)); + } + + @Test + void testAttemptStarted_retryAttributes_http() { + ApiTracerContext httpContext = + ApiTracerContext.newBuilder() + .setLibraryMetadata(com.google.api.gax.rpc.LibraryMetadata.empty()) + .setTransport(ApiTracerContext.Transport.HTTP) + .build(); + SpanTracer httpTracer = new SpanTracer(tracer, httpContext, ATTEMPT_SPAN_NAME); + + // N-th retry, attemptNumber is 5 + httpTracer.attemptStarted(new Object(), 5); + ArgumentCaptor attributesCaptor = ArgumentCaptor.forClass(Attributes.class); + verify(spanBuilder).setAllAttributes(attributesCaptor.capture()); + java.util.Map, Object> capturedAttributes = + attributesCaptor.getValue().asMap(); + assertThat(capturedAttributes) + .doesNotContainKey( + io.opentelemetry.api.common.AttributeKey.longKey( + ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE)); + assertThat(capturedAttributes) + .containsEntry( + io.opentelemetry.api.common.AttributeKey.longKey( + ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE), + 5L); + } } diff --git a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java index b43351c2ac..c2e8d63adc 100644 --- a/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java +++ b/java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java @@ -31,13 +31,23 @@ package com.google.showcase.v1beta1.it; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.rpc.StatusCode; +import com.google.api.gax.rpc.UnavailableException; import com.google.api.gax.tracing.ObservabilityAttributes; import com.google.api.gax.tracing.SpanTracer; import com.google.api.gax.tracing.SpanTracerFactory; +import com.google.rpc.Status; import com.google.showcase.v1beta1.EchoClient; import com.google.showcase.v1beta1.EchoRequest; +import com.google.showcase.v1beta1.EchoSettings; import com.google.showcase.v1beta1.it.util.TestClientInitializer; +import com.google.showcase.v1beta1.stub.EchoStub; +import com.google.showcase.v1beta1.stub.EchoStubSettings; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.trace.SpanKind; @@ -200,4 +210,146 @@ void testTracing_successfulEcho_httpjson() throws Exception { .isEqualTo("v1beta1/echo:echo"); } } + + @Test + void testTracing_retry_grpc() throws Exception { + final int attempts = 5; + final StatusCode.Code statusCode = StatusCode.Code.UNAVAILABLE; + // A custom EchoClient is used in this test because retries have jitter, and we cannot + // predict the number of attempts that are scheduled for an RPC invocation otherwise. + // The custom retrySettings limit to a set number of attempts before the call gives up. + RetrySettings retrySettings = + RetrySettings.newBuilder() + .setTotalTimeout(org.threeten.bp.Duration.ofMillis(5000L)) + .setMaxAttempts(attempts) + .build(); + + EchoStubSettings.Builder grpcEchoSettingsBuilder = EchoStubSettings.newBuilder(); + grpcEchoSettingsBuilder + .echoSettings() + .setRetrySettings(retrySettings) + .setRetryableCodes(statusCode); + EchoSettings grpcEchoSettings = EchoSettings.create(grpcEchoSettingsBuilder.build()); + grpcEchoSettings = + grpcEchoSettings.toBuilder() + .setCredentialsProvider(NoCredentialsProvider.create()) + .setTransportChannelProvider(EchoSettings.defaultGrpcTransportProviderBuilder().build()) + .setEndpoint("localhost:7469") + .build(); + + SpanTracerFactory tracingFactory = new SpanTracerFactory(openTelemetrySdk); + + EchoStubSettings echoStubSettings = + (EchoStubSettings) + grpcEchoSettings.getStubSettings().toBuilder().setTracerFactory(tracingFactory).build(); + EchoStub stub = echoStubSettings.createStub(); + EchoClient grpcClient = EchoClient.create(stub); + + EchoRequest echoRequest = + EchoRequest.newBuilder() + .setError(Status.newBuilder().setCode(statusCode.ordinal()).build()) + .build(); + + assertThrows(UnavailableException.class, () -> grpcClient.echo(echoRequest)); + + List spans = spanExporter.getFinishedSpanItems(); + assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry + + // This single span represents the successful retry, which has resend_count=1 + // The first attempt has no resend_count. The subsequent retries will have a resend_count, + // starting from 1. + List resendCounts = + spans.stream() + .map( + span -> + (Long) + span.getAttributes() + .asMap() + .get( + AttributeKey.longKey( + ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE))) + .filter(java.util.Objects::nonNull) + .sorted() + .collect(java.util.stream.Collectors.toList()); + + List expectedCounts = + java.util.stream.LongStream.range(1, attempts) + .boxed() + .collect(java.util.stream.Collectors.toList()); + assertThat(resendCounts).containsExactlyElementsIn(expectedCounts).inOrder(); + } + + @Test + void testTracing_retry_httpjson() throws Exception { + final int attempts = 5; + final StatusCode.Code statusCode = StatusCode.Code.UNAVAILABLE; + // A custom EchoClient is used in this test because retries have jitter, and we cannot + // predict the number of attempts that are scheduled for an RPC invocation otherwise. + // The custom retrySettings limit to a set number of attempts before the call gives up. + RetrySettings retrySettings = + RetrySettings.newBuilder() + .setTotalTimeout(org.threeten.bp.Duration.ofMillis(5000L)) + .setMaxAttempts(attempts) + .build(); + + EchoStubSettings.Builder httpJsonEchoSettingsBuilder = EchoStubSettings.newHttpJsonBuilder(); + httpJsonEchoSettingsBuilder + .echoSettings() + .setRetrySettings(retrySettings) + .setRetryableCodes(statusCode); + EchoSettings httpJsonEchoSettings = EchoSettings.create(httpJsonEchoSettingsBuilder.build()); + httpJsonEchoSettings = + httpJsonEchoSettings.toBuilder() + .setCredentialsProvider(NoCredentialsProvider.create()) + .setTransportChannelProvider( + EchoSettings.defaultHttpJsonTransportProviderBuilder() + .setHttpTransport( + new NetHttpTransport.Builder().doNotValidateCertificate().build()) + .setEndpoint("http://localhost:7469") + .build()) + .build(); + + SpanTracerFactory tracingFactory = new SpanTracerFactory(openTelemetrySdk); + + EchoStubSettings echoStubSettings = + (EchoStubSettings) + httpJsonEchoSettings.getStubSettings().toBuilder() + .setTracerFactory(tracingFactory) + .build(); + EchoStub stub = echoStubSettings.createStub(); + EchoClient httpClient = EchoClient.create(stub); + + EchoRequest echoRequest = + EchoRequest.newBuilder() + .setError(Status.newBuilder().setCode(statusCode.ordinal()).build()) + .build(); + + assertThrows(UnavailableException.class, () -> httpClient.echo(echoRequest)); + + List spans = spanExporter.getFinishedSpanItems(); + assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry + + // This single span represents the successful retry, which has resend_count=1 + // The first attempt has no resend_count. The subsequent retries will have a resend_count, + // starting from 1. + List resendCounts = + spans.stream() + .map( + span -> + (Long) + span.getAttributes() + .asMap() + .get( + AttributeKey.longKey( + ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE))) + .filter(java.util.Objects::nonNull) + .sorted() + .collect(java.util.stream.Collectors.toList()); + + List expectedCounts = + java.util.stream.LongStream.range(1, attempts) + .boxed() + .collect(java.util.stream.Collectors.toList()); + assertThat(resendCounts).containsExactlyElementsIn(expectedCounts).inOrder(); + } }