update README.md

Author: JoaoBraveCoding

README.md

@@ -15,25 +15,43 @@ These benchmarks can be executed on a vanilla Kubernetes or OpenShift cluster. I
   * Non-OpenShift Deployments Only; Optional: [Cadvisor](https://github.com/google/cadvisor)
 
 * Notes
-   * Clone git repositories into sibling directories to the `loki-benchmarks` one.
-   * Recommended cluster size: `m4.16xlarge`
+  * Clone git repositories into sibling directories to the `loki-benchmarks` one.
+  * Recommended cluster size: `m4.16xlarge`
 
 ## Configuring Tests
 
 To change the testing configuration, see the files in the [config](./config) directory.
 
-Use the `scenarios/benchmarks.yaml` file to add, modify, or remove configurations. Modify the `generator.yaml`, `metrics.yaml`, or `querier.yaml` in the prefered deployment method directory to change these soruces.
+Different scenarios can be customized under `config/benchmarks/scenarios/benchmarks`. Current the benchmarks support two testing scenarios:
+
+* Ingestion path scenarios: [suppored configuration](https://github.com/observatorium/loki-benchmarks/blob/1a0a9e8f6190475b6c1bfacb5a31a88bd76cbb36/internal/config/config.go#L76-L81), this test will generate X amount of logs throughout a 30 minute window that's supposed to represent a full day of log ingestion.
+* Query path scenarios: [supported configuration](https://github.com/observatorium/loki-benchmarks/blob/1a0a9e8f6190475b6c1bfacb5a31a88bd76cbb36/internal/config/config.go#L102-L108), the theory behind this test is to generate the amount of data that would be queried before it starts running the queries.
 
 ## Running Benchmarks
 
-Use the `make run-rhobs-benchmarks` or `make run-operator-benchmarks` to execute the benchmark program with the RHOBS or operator deployment styles on OpenShift respectively. Upon successful completion, a JSON and XML file will be created in the `reports/date+time` directory with the results of the tests.
+### Prerequisites
 
-## Troubleshooting
+The `run-operator-benchmarks` expects the following two env vars to be set `LOKI_OPERATOR_REGISTRY` `LOKI_STORAGE_BUCKET`.
+E.g
+
+```shell
+export LOKI_OPERATOR_REGISTRY=jmarcal
+export LOKI_STORAGE_BUCKET=jmarcal-loki-benchmark-storage
+```
 
-During benchmark execution, use [hack/scripts/ocp-deploy-grafana.sh](hack/scripts/ocp-deploy-grafana.sh) to deploy grafna and connect to Loki as a datasource: 
-- Use a web browser to access grafana UI. The URL, username and password are printed by the script 
-- In the UI, under settings -> data-sources hit `Save & test` to verify that Loki data-source is connected and that there are no errors
-- In explore tab change the data-source to `Loki` and use `{client="promtail"}` query to visualize log lines
-- Use additional queries such as `rate({client="promtail"}[1m])` to verify the behaviour of Loki and the benchmark
+### Steps
+
+1. Use the `make run-rhobs-benchmarks` or `make run-operator-benchmarks` to execute the benchmark program with the RHOBS or operator deployment styles on OpenShift respectively.
+Both commands will run all the scenarios under `config/benchmarks/scenarios/benchmarks`.
+2. Upon successful completion of each scenario, a JSON and XML file will be created in the `reports/date+time/schenario_name` directory with the results of the tests.
+3. Once all scenarios have been run we can run `python3 hack/scripts/generate_report.py $PATH_TO_SCENARIO_1 $PATH_TO_SCENARIO_2 $PATH_TO_SCENARIO_...` to compile a report that helps compare the different scenarios.
+4. To share the report on gDoc you can run `pthon3 hack/scripts/create-gdoc.py $PATH_TO_THE_REPORT` this will generate a docx file that can then be shared.
+
+## Troubleshooting
 
+During benchmark execution, use [hack/scripts/ocp-deploy-grafana.sh](hack/scripts/ocp-deploy-grafana.sh) to deploy grafna and connect to Loki as a datasource:
 
+* Use a web browser to access grafana UI. The URL, username and password are printed by the script
+* In the UI, under settings -> data-sources hit `Save & test` to verify that Loki data-source is connected and that there are no errors
+* In explore tab change the data-source to `Loki` and use `{client="promtail"}` query to visualize log lines
+* Use additional queries such as `rate({client="promtail"}[1m])` to verify the behaviour of Loki and the benchmark

internal/metrics/client.go

@@ -252,22 +252,22 @@ func (c *Client) measureCommonRequestMetrics(
 	var name, code, badCode, requestRateName, badRequestRateName string
 
 	if method == GRPCMethod {
-		name = fmt.Sprintf("%s successful GRPC %s", job, route)
+		name = fmt.Sprintf("%s successful GRPC %s", annotation, route)
 		code = "success"
 		requestRateName = name
 		if pathRoutes == GRPCReadPathRoutes {
 			requestRateName = "successful GRPC reads"
 		}
 	} else {
-		name = fmt.Sprintf("%s 2xx %s", job, route)
+		name = fmt.Sprintf("%s 2xx %s", annotation, route)
 		code = "2.*"
 		requestRateName = name
 		if pathRoutes == HTTPReadPathRoutes {
 			requestRateName = "2xx reads"
 		}
 
 		badCode = "5.*"
-		badRequestRateName = fmt.Sprintf("%s 5xx %s", job, route)
+		badRequestRateName = fmt.Sprintf("%s 5xx %s", annotation, route)
 		if pathRoutes == HTTPReadPathRoutes {
 			badRequestRateName = "5xx reads"
 		}

internal/metrics/resources.go

@@ -9,7 +9,7 @@ import (
 
 func ContainerCPU(job string, duration model.Duration, annotation gmeasure.Annotation) Measurement {
 	return Measurement{
-		Name: fmt.Sprintf("%s Sum of Container CPU Usage", annotation),
+		Name: fmt.Sprintf("%s sum of Container CPU Usage", annotation),
 		Query: fmt.Sprintf(
 			`sum(avg_over_time(pod:container_cpu_usage:sum{pod=~".*%s.*"}[%s])) * %d`,
 			job, duration, CoresToMillicores,
@@ -21,7 +21,7 @@ func ContainerCPU(job string, duration model.Duration, annotation gmeasure.Annot
 
 func ContainerMemoryWorkingSetBytes(job string, duration model.Duration, annotation gmeasure.Annotation) Measurement {
 	return Measurement{
-		Name: fmt.Sprintf("%s Sum of Container WorkingSet Memory", annotation),
+		Name: fmt.Sprintf("%s sum of Container WorkingSet Memory", annotation),
 		Query: fmt.Sprintf(
 			`sum(avg_over_time(container_memory_working_set_bytes{pod=~".*%s.*", container=""}[%s]) / %d)`,
 			job, duration, BytesToGigabytesMultiplier,
@@ -33,7 +33,7 @@ func ContainerMemoryWorkingSetBytes(job string, duration model.Duration, annotat
 
 func ContainerGoMemstatsHeapInuse(job string, _ model.Duration, annotation gmeasure.Annotation) Measurement {
 	return Measurement{
-		Name: fmt.Sprintf("%s Sum of Container Go Memstats Heap Inuse", annotation),
+		Name: fmt.Sprintf("%s sum of Container Go Memstats Heap Inuse", annotation),
 		Query: fmt.Sprintf(
 			`sum(go_memstats_heap_inuse_bytes{pod=~".*%s.*"}) / %d`,
 			job, BytesToGigabytesMultiplier,
@@ -45,7 +45,7 @@ func ContainerGoMemstatsHeapInuse(job string, _ model.Duration, annotation gmeas
 
 func PersistentVolumeUsedBytes(job string, duration model.Duration, annotation gmeasure.Annotation) Measurement {
 	return Measurement{
-		Name: fmt.Sprintf("%s Sum of Persistent Volume Used Bytes", annotation),
+		Name: fmt.Sprintf("%s sum of Persistent Volume Used Bytes", annotation),
 		Query: fmt.Sprintf(
 			`sum(avg_over_time(kubelet_volume_stats_used_bytes{persistentvolumeclaim=~".*%s.*"}[%s]) / %d)`,
 			job, duration, BytesToGigabytesMultiplier,