Fixed profiles, added 1y continous timing and better hashing algo. (#31)

Signed-off-by: Bartlomiej Plotka <[email protected]>

Author: bwplotka

cmd/thanosbench/block.go

@@ -133,7 +133,7 @@ Example plan with generation:
 ./thanosbench block plan -p <profile> --labels 'cluster="one"' --max-time 2019-10-18T00:00:00Z | ./thanosbench block gen --output.dir ./genblocks --workers 20`)
 	profile := cmd.Flag("profile", "Name of the harcoded profile to use").Required().Short('p').Enum(blockgen.Profiles.Keys()...)
 	maxTime := model.TimeOrDuration(cmd.Flag("max-time", "If empty current time - 30m (usual consistency delay) is used.").Default("30m"))
-	extLset := cmd.Flag("labels", "External labels for block stream (repeated).").PlaceHolder("<name>=\"<value>\"").Required().Strings()
+	extLset := cmd.Flag("labels", "External labels for block stream (repeated).").PlaceHolder("<name>=\"<value>\"").Strings()
 	m["block plan"] = func(g *run.Group, _ log.Logger) error {
 		ctx, cancel := context.WithCancel(context.Background())
 		g.Add(func() error {

go.mod

@@ -2,6 +2,7 @@ module github.com/thanos-io/thanosbench
 
 require (
 	github.com/bwplotka/mimic v0.0.0-20190730202618-06ab9976e8ef
+	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/fatih/structtag v1.1.0
 	github.com/go-kit/kit v0.10.0
 	github.com/go-openapi/swag v0.19.9

pkg/blockgen/blockgen.go

@@ -7,6 +7,7 @@ import (
 	"path"
 	"time"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/pkg/errors"
 
 	"github.com/go-kit/kit/log"
@@ -76,7 +77,12 @@ func Generate(ctx context.Context, logger log.Logger, goroutines int, dir string
 	if err != nil {
 		return ulid.ULID{}, err
 	}
-	set := &blockSeriesSet{config: block}
+
+	extLset := block.Thanos.Labels
+	if extLset == nil {
+		extLset = map[string]string{}
+	}
+	set := &blockSeriesSet{config: block, extLset: labels.FromMap(extLset)}
 	if err := seriesgen.Append(ctx, goroutines, w, set); err != nil {
 		return ulid.ULID{}, errors.Wrap(err, "append")
 	}
@@ -98,10 +104,11 @@ func Generate(ctx context.Context, logger log.Logger, goroutines int, dir string
 }
 
 type blockSeriesSet struct {
-	config BlockSpec
-	i      int
-	target int
-	err    error
+	config  BlockSpec
+	extLset labels.Labels
+	i       int
+	target  int
+	err     error
 
 	curr seriesgen.Series
 }
@@ -122,9 +129,23 @@ func (s *blockSeriesSet) Next() bool {
 	series := s.config.Series[s.i-1]
 	lset := labels.Labels(append([]labels.Label{{Name: "__blockgen_target__", Value: fmt.Sprintf("%v", s.target)}}, series.Labels...))
 
+	b := make([]byte, 0, 1024)
+	for _, v := range lset {
+		b = append(b, v.Name...)
+		b = append(b, '\xff')
+		b = append(b, v.Value...)
+		b = append(b, '\xff')
+	}
+	for _, v := range s.extLset {
+		b = append(b, v.Name...)
+		b = append(b, '\xff')
+		b = append(b, v.Value...)
+		b = append(b, '\xff')
+	}
+
 	// Stable random per series name.
 	iter, err := series.Type.Create(
-		rand.New(rand.NewSource(int64(lset.Hash()))),
+		rand.New(rand.NewSource(int64(xxhash.Sum64(b)))),
 		series.MinTime,
 		series.MaxTime,
 		series.Characteristics,

pkg/blockgen/profiles.go

@@ -52,7 +52,34 @@ var (
 			48 * time.Hour,
 			2 * time.Hour,
 		}, 1*time.Hour, 100, 50),
-
+		"realistic-k8s-30d-tiny": realisticK8s([]time.Duration{
+			// 30 days, from newest to oldest.
+			2 * time.Hour,
+			2 * time.Hour,
+			2 * time.Hour,
+			8 * time.Hour,
+			176 * time.Hour,
+			176 * time.Hour,
+			176 * time.Hour,
+			176 * time.Hour,
+			2 * time.Hour,
+		}, 1*time.Hour, 1, 5),
+		"realistic-k8s-365d-tiny": realisticK8s([]time.Duration{
+			// 1y days, from newest to oldest.
+			2 * time.Hour,
+			2 * time.Hour,
+			2 * time.Hour,
+			8 * time.Hour,
+			176 * time.Hour,
+			176 * time.Hour,
+			176 * time.Hour,
+			176 * time.Hour,
+			67 * 24 * time.Hour,
+			67 * 24 * time.Hour,
+			67 * 24 * time.Hour,
+			67 * 24 * time.Hour,
+			67 * 24 * time.Hour,
+		}, 1*time.Hour, 1, 5),
 		"continuous-1w-small": continuous([]time.Duration{
 			// One week, from newest to oldest, in the same way Thanos compactor would do.
 			2 * time.Hour,
@@ -66,8 +93,7 @@ var (
 			2 * time.Hour,
 			// 10,000 series per block.
 		}, 100, 100),
-
-		"key-k8s-30d-tiny": realisticK8s([]time.Duration{
+		"continuous-30d-tiny": continuous([]time.Duration{
 			// 30 days, from newest to oldest.
 			2 * time.Hour,
 			2 * time.Hour,
@@ -78,9 +104,8 @@ var (
 			176 * time.Hour,
 			176 * time.Hour,
 			2 * time.Hour,
-		}, 1*time.Hour, 1, 5),
-
-		"key-k8s-365d-tiny": realisticK8s([]time.Duration{
+		}, 1, 5),
+		"continuous-365d-tiny": continuous([]time.Duration{
 			// 1y days, from newest to oldest.
 			2 * time.Hour,
 			2 * time.Hour,
@@ -93,7 +118,9 @@ var (
 			67 * 24 * time.Hour,
 			67 * 24 * time.Hour,
 			67 * 24 * time.Hour,
-		}, 1*time.Hour, 1, 5),
+			67 * 24 * time.Hour,
+			67 * 24 * time.Hour,
+		}, 1, 5),
 	}
 )
 
@@ -201,7 +228,7 @@ func continuous(ranges []time.Duration, apps int, metricsPerApp int) PlanFn {
 		}
 
 		for _, r := range ranges {
-			mint := maxt - durToMilis(r)
+			mint := maxt - durToMilis(r) + 1
 
 			if ctx.Err() != nil {
 				return ctx.Err()