Add metric to track added latency to write requests (#5781)

Author: mapno

CHANGELOG.md

@@ -13,6 +13,7 @@
 * [ENHANCEMENT] Unsafe search hints for frontend performance tests [#5723](https://github.com/grafana/tempo/pull/5723) (@ruslan-mikhailov)
 * [ENHANCEMENT] Add new livestore alert to the tempo-mixin [#5752](https://github.com/grafana/tempo/pull/5752) (@javiermolinar)
 * [ENHANCEMENT] Improve shutdown time in the first 30 seconds [#5725](https://github.com/grafana/tempo/pull/5725) (@ldufr)
+* [ENHANCEMENT] Add metric for tracking added latency to write requests [#](https://github.com/grafana/tempo/pull/) (@mapno)
 * [BUGFIX] Fix compactor to properly consider SSE-KMS information during metadata copy [#5774](https://github.com/grafana/tempo/pull/5774) (@steffsas)
 * [BUGFIX] Correctly track and reject too large traces in live stores. [#5757](https://github.com/grafana/tempo/pull/5757) (@joe-elliott)
 * [BUGFIX] Fix issues related to integer dedicated columns in vParquet5-preview2 [#5716](https://github.com/grafana/tempo/pull/5716) (@stoewer)

modules/distributor/artificial_delay.go

@@ -3,8 +3,20 @@ package distributor
 import (
 	"math/rand"
 	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 )
 
+var metricAddedLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{
+	Namespace:                       "tempo",
+	Subsystem:                       "distributor",
+	Name:                            "added_latency_seconds",
+	NativeHistogramBucketFactor:     1.1,
+	NativeHistogramMaxBucketNumber:  100,
+	NativeHistogramMinResetDuration: time.Hour,
+}, []string{"tenant"})
+
 func (d *Distributor) padWithArtificialDelay(reqStart time.Time, userID string) {
 	artificialDelay := d.cfg.ArtificialDelay
 	if artificialDelayOverride, ok := d.overrides.IngestionArtificialDelay(userID); ok {
@@ -19,6 +31,7 @@ func (d *Distributor) padWithArtificialDelay(reqStart time.Time, userID string)
 	// If the request took longer than the target delay, we don't delay at all as sleep will return immediately for a negative value
 	reqDuration := d.now().Sub(reqStart)
 	delay := artificialDelay - reqDuration
+	metricAddedLatency.WithLabelValues(userID).Observe(delay.Seconds())
 	d.sleep(durationWithJitter(delay, 0.10))
 }