Live-store: query live-store flag (grafana#5808)

* Live-store: query live-store flag

* Support querying metrics with live-stores via header

Author: ruslan-mikhailov

cmd/tempo/app/modules.go

@@ -473,7 +473,7 @@ func (t *App) initQuerier() (services.Service, error) {
 		ingesterRings,
 		t.cfg.GeneratorClient,
 		t.readRings[ringMetricsGenerator],
-		t.cfg.PartitionRingLiveStore,
+		t.cfg.Querier.QueryLiveStore,
 		t.cfg.LiveStoreClient,
 		t.readRings[ringLiveStore],
 		t.partitionRing,

example/docker-compose/ingest-storage/tempo.yaml

@@ -41,6 +41,7 @@ compactor:
 querier:
   frontend_worker:
     frontend_address: query-frontend:9095
+  query_live_store: true
 
 metrics_generator:
   registry:
@@ -96,4 +97,4 @@ block_builder:
     block-builder-1: [1]
 
 usage_report:
-  reporting_enabled: false
+  reporting_enabled: false

example/tk/tempo-microservices/main.jsonnet

@@ -93,6 +93,9 @@ minio + metrics + load + kafka + tempo {
         topic: 'tempo-ingest',
       },
     },
+    querier+: {
+      query_live_store: true,
+    },
     block_builder+: {
       consume_cycle_duration: '30s',
       assigned_partitions: {

integration/e2e/ingest/config-live-store.yaml

@@ -72,3 +72,4 @@ ingest:
 querier:
   frontend_worker:
     frontend_address: query-frontend:9095
+  query_live_store: true

modules/querier/config.go

@@ -24,6 +24,7 @@ type Config struct {
 	ShuffleShardingIngestersLookbackPeriod time.Duration `yaml:"shuffle_sharding_ingesters_lookback_period"`
 	QueryRelevantIngesters                 bool          `yaml:"query_relevant_ingesters"`
 	SecondaryIngesterRing                  string        `yaml:"secondary_ingester_ring,omitempty"`
+	QueryLiveStore                         bool          `yaml:"query_live_store,omitempty"` // todo: remove after rhythm migration
 }
 
 type SearchConfig struct {
@@ -81,6 +82,7 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet)
 	cfg.PartitionRing.MinimizeRequests = true
 	cfg.PartitionRing.MinimizeRequestsHedgingDelay = 3 * time.Second
 	cfg.PartitionRing.PreferredZone = ""
+	cfg.QueryLiveStore = false
 
 	f.StringVar(&cfg.Worker.FrontendAddress, prefix+".frontend-address", "", "Address of query frontend service, in host:port format.")
 }

modules/querier/querier.go

@@ -485,7 +485,9 @@ func (q *Querier) forGivenGenerators(ctx context.Context, f forEachGeneratorFn)
 	ctx, span := tracer.Start(ctx, "Querier.forGivenGenerators")
 	defer span.End()
 
-	if q.queryPartitionRing {
+	recentDataTarget := extractRecentDataTarget(ctx)
+
+	if q.queryPartitionRing || recentDataTarget == "live-store" {
 		rs, err := q.partitionRing.GetReplicationSetsForOperation(ring.Read)
 		if err != nil {
 			return nil, fmt.Errorf("error finding partition ring replicas: %w", err)