misc: Fix doc naming and use flexible plot in cli (#65)

Co-authored-by: key4ng <[email protected]>

Author: key4ng

.coveragerc

@@ -3,6 +3,8 @@ omit =
     genai_bench/cli/report.py
     genai_bench/analysis/excel_report.py
     genai_bench/analysis/plot_report.py
+    genai_bench/analysis/flexible_plot_report.py
+    genai_bench/analysis/plot_config.py
     genai_bench/ui/*
     genai_bench/logging.py
     tests/*

docs/user-guide/multi-cloud-auth-storage.md

@@ -176,7 +176,7 @@ genai-bench benchmark \
   --max-requests-per-run 100 \
   --max-time-per-run 10
 ```
-**Note:** for Dedicated model, the `--api-model-name` is just a placeholder, the model depends on the the endpointId you provided 
+**Note:** for Dedicated model, the `--api-model-name` is just a placeholder, the model depends on the endpointId you provided
 
 **Advanced features:**
 ```bash
@@ -343,7 +343,7 @@ vLLM and SGLang use OpenAI-compatible APIs with optional authentication.
 **Example:**
 ```bash
 genai-bench benchmark \
-  --api-backend vllm \
+  --api-backend sglang \
   --api-base http://localhost:8000 \
   --api-key optional-key \
   --api-model-name meta-llama/Llama-2-7b-hf \
@@ -657,4 +657,4 @@ The main changes are:
 
 - `--bucket` → `--storage-bucket`
 - `--prefix` → `--storage-prefix`
-- Add `--storage-provider oci` (though OCI is the default for backward compatibility)
+- Add `--storage-provider oci` (though OCI is the default for backward compatibility)

docs/user-guide/multi-cloud-quick-reference.md

@@ -2,7 +2,7 @@
 
 This is a quick reference guide for common multi-cloud scenarios with genai-bench. For detailed information, see the [comprehensive guide](multi-cloud-auth-storage.md).
 
-> **Note**: For OpenAI, vLLM, and SGLang backends, both `--api-key` and `--model-api-key` are supported for backward compatibility.
+> **Note**: For OpenAI, SGLang and vLLM backends, both `--api-key` and `--model-api-key` are supported for backward compatibility.
 
 ## OpenAI Benchmarking
 
@@ -277,4 +277,4 @@ export GITHUB_REPO=benchmarks
 ```bash
 # HuggingFace (for downloading tokenizers)
 export HF_TOKEN=hf_...
-```
+```

docs/user-guide/run-benchmark.md

@@ -21,12 +21,12 @@ export TRANSFORMERS_VERBOSITY=error
 genai-bench benchmark --api-backend openai \
             --api-base "http://localhost:8082" \
             --api-key "your-openai-api-key" \
-            --api-model-name "vllm-model" \
+            --api-model-name "meta-llama/Meta-Llama-3-70B-Instruct" \
             --model-tokenizer "/mnt/data/models/Meta-Llama-3.1-70B-Instruct" \
             --task text-to-text \
             --max-time-per-run 15 \
             --max-requests-per-run 300 \
-            --server-engine "vLLM" \
+            --server-engine "SGLang" \
             --server-gpu-type "H100" \
             --server-version "v0.6.0" \
             --server-gpu-count 4
@@ -119,7 +119,7 @@ genai-bench benchmark --api-backend oci-cohere \
             --api-base "https://inference.generativeai.us-chicago-1.oci.oraclecloud.com" \
             --api-model-name "c4ai-command-r-08-2024" \
             --model-tokenizer "/home/ubuntu/c4ai-command-r-08-2024" \
-            --server-engine "vLLM" \
+            --server-engine "SGLang" \
             --task text-to-text \
             --num-concurrency 1 \
             --server-gpu-type A100-80G \
@@ -344,4 +344,4 @@ If you want to benchmark a specific portion of a vision dataset, you can use the
 - Access to ALL HuggingFace `load_dataset` parameters
 - Reusable and version-controllable
 - Support for complex configurations
-- Future-proof (no CLI updates needed for new HuggingFace features)
+- Future-proof (no CLI updates needed for new HuggingFace features)

docs/user-guide/upload-benchmark-result.md

@@ -16,12 +16,12 @@ To enable result uploading, use the following options with the `benchmark` comma
 genai-bench benchmark \
     --api-base "http://localhost:8082" \
     --api-key "your-openai-api-key" \
-    --api-model-name "vllm-model" \
+    --api-model-name "meta-llama/Meta-Llama-3-70B-Instruct" \
     --model-tokenizer "/mnt/data/models/Meta-Llama-3.1-70B-Instruct" \
     --task text-to-text \
     --max-time-per-run 15 \
     --max-requests-per-run 300 \
-    --server-engine "vLLM" \
+    --server-engine "SGLang" \
     --server-gpu-type "H100" \
     --server-version "v0.6.0" \
     --server-gpu-count 4 \
@@ -44,4 +44,4 @@ GenAI Bench now supports multiple cloud storage providers:
 - **GCP Cloud Storage**: Use `--storage-provider gcp`
 - **GitHub Releases**: Use `--storage-provider github`
 
-For detailed configuration and authentication options for each provider, please refer to the [Multi-Cloud Authentication & Storage Guide](multi-cloud-auth-storage.md).
+For detailed configuration and authentication options for each provider, please refer to the [Multi-Cloud Authentication & Storage Guide](multi-cloud-auth-storage.md).

examples/experiment_excel.py

@@ -11,7 +11,7 @@
 LoggingManager("excel")
 
 
-folder_name = "/Users/changsu/openai_chat_vllm-model_tokenizer__mnt_data_models_Llama-3-70B-Instruct_20240904_003850"  # noqa: E501
+folder_name = "<Path to your experiment folder>"  # noqa: E501
 os.makedirs(folder_name, exist_ok=True)
 experiment_metadata, run_data = load_one_experiment(folder_name)
 create_workbook(

examples/experiment_plots.py

@@ -6,16 +6,16 @@
     load_multiple_experiments,
     load_one_experiment,
 )
-from genai_bench.analysis.plot_report import plot_experiment_data
+from genai_bench.analysis.flexible_plot_report import plot_experiment_data_flexible
 from genai_bench.logging import LoggingManager
 
 LoggingManager("plot")
 
 
 # Example usage with filtering multiple experiments
-folder_name = "/Users/changsu/experiment_plot"
+folder_name = "<Path to the experiment folder>"
 filter_criteria = {
-    "model": "vllm-model",
+    "model": "Llama-4-Scout-17B-16E-Instruct",
 }
 
 os.makedirs(folder_name, exist_ok=True)
@@ -26,20 +26,20 @@
     print("Empty data after filtering")
 else:
     # Plot the data grouped by 'server_version'
-    plot_experiment_data(
+    plot_experiment_data_flexible(
         run_data_list, group_key="server_version", experiment_folder=folder_name
     )
 
 # Plot for one experiment
 experiment_folder = os.path.join(
     folder_name,
-    "openai_chat_vllm-model_tokenizer__mnt_data_models_Llama-3-70B-Instruct_20240904_003850",
+    "openai_SGLang_v0.4.7.post1_text-to-text_Llama-4-Scout-17B-16E-Instruct_20250620_042005",
 )
 experiment_metadata, run_data = load_one_experiment(experiment_folder)
 if not experiment_metadata or not run_data:
     print("Didn't find any experiment data")
 else:
-    plot_experiment_data(
+    plot_experiment_data_flexible(
         [
             [experiment_metadata, run_data],
         ],

genai_bench/cli/cli.py

@@ -11,9 +11,10 @@
 from genai_bench.analysis.excel_report import create_workbook
 from genai_bench.analysis.experiment_loader import load_one_experiment
 from genai_bench.analysis.plot_report import (
-    plot_experiment_data,
     plot_single_scenario_inference_speed_vs_throughput,
 )
+from genai_bench.analysis.flexible_plot_report import plot_experiment_data_flexible
+
 from genai_bench.auth.unified_factory import UnifiedAuthFactory
 from genai_bench.cli.option_groups import (
     api_options,
@@ -506,7 +507,7 @@ def benchmark(
         ),
         percentile="mean",
     )
-    plot_experiment_data(
+    plot_experiment_data_flexible(
         [
             (experiment_metadata, run_data),
         ],

genai_bench/cli/report.py

@@ -71,7 +71,7 @@ def excel(ctx, experiment_folder, excel_name, metric_percentile):
     default=None,
     callback=validate_filter_criteria,
     help="A dictionary containing filter criteria for the plot. Default: {}. "
-    "Example: '{'model': 'vllm-model'}'",
+    "Example: '{'model': 'meta-llama/Meta-Llama-3-70B-Instruct'}'",
 )
 @click.option(
     "--plot-config",

tests/analysis/mock_experiment_data.json

@@ -1,12 +1,12 @@
 {
-    "cmd": "/home/changsu/miniconda3/bin/genai-bench --api_backend openai --api_base http://localhost:8082 --api_key your-openai-api-key --api_model_name vllm-model --model_tokenizer /mnt/data/models/Meta-Llama-3.1-70B-Instruct --task chat --run_time 1 --server_engine vLLM --server_gpu_type H100 --server_version v0.6.0 --server_gpu_count 4 --model Meta-Llama-3.1-70B-Instruct --num_concurrency (1, 2, 4, 8, 16, 32, 64, 128, 256) --traffic_scenario N(480,240)/(300,150) --traffic_scenario D(100,100) --traffic_scenario D(100,1000) --traffic_scenario D(2000,200) --traffic_scenario D(7800,200)",
+    "cmd": "/home/changsu/miniconda3/bin/genai-bench --api_backend openai --api_base http://localhost:8082 --api_key your-openai-api-key --api_model_name meta-llama/Meta-Llama-3-70B-Instruct --model_tokenizer /mnt/data/models/Meta-Llama-3.1-70B-Instruct --task chat --run_time 1 --server_engine SGLang --server_gpu_type H100 --server_version v0.6.0 --server_gpu_count 4 --model Meta-Llama-3.1-70B-Instruct --num_concurrency (1, 2, 4, 8, 16, 32, 64, 128, 256) --traffic_scenario N(480,240)/(300,150) --traffic_scenario D(100,100) --traffic_scenario D(100,1000) --traffic_scenario D(2000,200) --traffic_scenario D(7800,200)",
     "benchmark_version": "",
     "api_backend": "openai",
     "auth_config": {
         "api_base": "http://localhost:8084",
         "api_key": "********_key"
     },
-    "api_model_name": "vllm-model",
+    "api_model_name": "meta-llama/Meta-Llama-3-70B-Instruct",
     "server_model_tokenizer": "/mnt/data/models/Meta-Llama-3.1-70B-Instruct",
     "model": "Meta-Llama-3.1-70B-Instruct",
     "task": "text-to-text",
@@ -29,13 +29,13 @@
         "D(7800,200)"
     ],
     "additional_request_params": {},
-    "server_engine": "vLLM",
+    "server_engine": "SGLang",
     "server_version": "v0.6.0",
     "server_gpu_type": "H100",
     "server_gpu_count": "4",
     "max_time_per_run_s": 60,
     "max_requests_per_run": 300,
-    "experiment_folder_name": "/home/changsu/openai_vLLM_v0.6.0_chat_vllm-model_tokenizer__mnt_data_models_Meta-Llama-3.1-70B-Instruct_20240906_165433",
+    "experiment_folder_name": "/home/changsu/openai_SGLang_v0.6.0_chat_sglang-model_tokenizer__mnt_data_models_Meta-Llama-3.1-70B-Instruct_20240906_165433",
     "dataset_path": null,
     "dataset_prompt_column_index": 0,
     "character_token_ratio": 4.059085841694538