Improve shutdown time in the first 30 seconds (#5725)

* Improve shutdown time in the first 30 seconds

Trying to shutdown (SIGINT) tempo in the first 30 seconds
would block until we got a stable key which currently takes
30 seconds. This unblock that.

* Updated changelog

Author: ldufr

CHANGELOG.md

@@ -7,6 +7,7 @@
 * [ENHANCEMENT] Add endpoints for partition downscaling for live-store [#5600](https://github.com/grafana/tempo/pull/5600) [#5738](https://github.com/grafana/tempo/pull/5738) (@ruslan-mikhailov, @mapno)
 * [ENHANCEMENT] Add config to enable instance label for spanmetrics series [#5706](https://github.com/grafana/tempo/pull/5706) (@ie-pham)
 * [ENHANCEMENT] Unsafe search hints for frontend performance tests [#5723](https://github.com/grafana/tempo/pull/5723) (@ruslan-mikhailov)
+* [ENHANCEMENT] Improve shutdown time in the first 30 seconds [#5725](https://github.com/grafana/tempo/pull/5725) (@ldufr)
 * [FEATURE] Add `tempo_metrics_generator_registry_active_series_demand_estimate` that estimates metrics-generator active series demand even when the active series limit is reached [#5710](https://github.com/grafana/tempo/pull/5710) (@carles-grafana)
 * [BUGFIX] Fix issues related to integer dedicated columns in vParquet5-preview2 [#5716](https://github.com/grafana/tempo/pull/5716) (@stoewer)
 * [FEATURE] Added validation mode and tests for tempo-vulture [#5605](https://github.com/grafana/tempo/pull/5605)

pkg/usagestats/reporter.go

@@ -102,6 +102,10 @@ func (rep *Reporter) initLeader(ctx context.Context) *ClusterSeed {
 		}
 		// ensure stability of the cluster seed
 		stableSeed := ensureStableKey(ctx, kvClient, rep.logger)
+		if stableSeed == nil {
+			return nil
+		}
+
 		seed = *stableSeed
 		// Fetch the remote cluster seed.
 		remoteSeed, err := rep.fetchSeed(ctx,
@@ -136,7 +140,12 @@ func ensureStableKey(ctx context.Context, kvClient kv.Client, logger log.Logger)
 		stableCount int
 	)
 	for {
-		time.Sleep(stabilityCheckInterval)
+		select {
+		case <-time.After(stabilityCheckInterval):
+		case <-ctx.Done():
+			return nil
+		}
+
 		value, err := kvClient.Get(ctx, seedKey)
 		if err != nil {
 			level.Debug(logger).Log("msg", "failed to get cluster seed key for stability check", "err", err)