CI: Add CI pipeline and pre-commit hooks (#7)

Author: CatherineSue

.dockerignore

@@ -159,4 +159,3 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
-

.github/CODEOWNERS

@@ -0,0 +1,8 @@
+# Global code owners
+* @slin1237 @CatherineSue
+
+# Specific ownership for critical files
+/.github/ @slin1237 @CatherineSue
+/genai_bench/ @slin1237 @CatherineSue
+/tests/ @slin1237 @CatherineSue
+pyproject.toml @slin1237 @CatherineSue

.github/workflows/ci.yml

@@ -0,0 +1,113 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+
+    - name: Update pip and install uv
+      run: |
+        python -m pip install -U pip hatchling
+        python -m pip install -U uv
+
+    - name: Check uv version
+      run: python -m uv version
+
+    - name: Install dependencies
+      run: |
+        python -m uv pip install --system .
+        python -m uv pip install --system ".[dev]"
+
+    - name: Run tests with coverage
+      run: |
+        uv run --no-reinstall pytest tests --cov --cov-report=xml --cov-config=.coveragerc -v -s
+
+    - name: Check coverage threshold
+      run: |
+        uv run --no-reinstall coverage report --show-missing --fail-under=93
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v3
+      with:
+        file: ./coverage.xml
+        fail_ci_if_error: false
+
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.11"
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+
+    - name: Update pip and install uv
+      run: |
+        python -m pip install -U pip hatchling
+        python -m pip install -U uv
+
+    - name: Install dependencies
+      run: |
+        python -m uv pip install --system .
+        python -m uv pip install --system ".[dev]"
+
+    - name: Check formatting with ruff
+      run: |
+        uv run --no-reinstall ruff format --diff genai_bench tests
+
+    - name: Lint with ruff
+      run: |
+        uv run --no-reinstall ruff check genai_bench tests
+
+  build:
+    runs-on: ubuntu-latest
+    needs: [test, lint]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.11"
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+
+    - name: Update pip and install uv
+      run: |
+        python -m pip install -U pip hatchling
+        python -m pip install -U uv
+
+    - name: Install dependencies
+      run: |
+        python -m uv pip install --no-color -vvv .
+        python -m uv pip install ".[dev]"
+
+    - name: Build package
+      run: |
+        python -m uv build -vvv

.gitignore

@@ -42,4 +42,4 @@ genai_bench*.log
 #  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
 #  refer to https://docs.cursor.com/context/ignore-files
 .cursorignore
-.cursorindexingignore
+.cursorindexingignore

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-merge-conflict
+      - id: check-case-conflict
+      - id: check-json
+      - id: check-toml
+      - id: check-yaml
+      - id: debug-statements
+      - id: check-added-large-files
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.7.4
+    hooks:
+      - id: ruff
+        args: [--fix]
+        files: ^(genai_bench|tests)/
+      - id: ruff-format
+        files: ^(genai_bench|tests)/

CONTRIBUTING.md

@@ -151,7 +151,7 @@ class AudioSampler(Sampler):
     def sample(self, scenario: Scenario) -> UserRequest:
         # Validate scenario
         self._validate_scenario(scenario)
-        
+
         if self.output_modality == "text":
             return self._create_audio_to_text_request(scenario)
         elif self.output_modality == "embeddings":

Dockerfile

@@ -35,4 +35,4 @@ RUN apt-get clean && \
     rm -rf /var/lib/apt/lists/* /var/cache/apt/* /var/log/*
 
 # start poetry shell to activate the virtual environment
-ENTRYPOINT ["genai-bench"]
+ENTRYPOINT ["genai-bench"]

Makefile

@@ -98,5 +98,5 @@ pipx: ## Install pipx package manager.
 	$(PYTHON_CMD) -m pip install -U pipx
 
 .PHONY: uv
-uv: pipx ## Install uv and make a virtual env with 
-	$(PYTHON_CMD) -m pipx ensurepath; $(PYTHON_CMD) -m pipx install uv; uv venv -p $(PYTHON_CMD)
+uv: pipx ## Install uv and make a virtual env with
+	$(PYTHON_CMD) -m pipx ensurepath; $(PYTHON_CMD) -m pipx install uv; uv venv -p $(PYTHON_CMD)

USER_GUIDE.md

@@ -166,10 +166,10 @@ Below is a sample command you can use to start a benchmark. The command will con
 
 ```shell
 # Optional. This is required when you load the tokenizer from huggingface.co with a model-id
-export HF_TOKEN="<your-key>"  
-# HF transformers will log a warning about torch not installed, since benchmark doesn't really need torch 
+export HF_TOKEN="<your-key>"
+# HF transformers will log a warning about torch not installed, since benchmark doesn't really need torch
 # and cuda, we use this env to disable the warning
-export TRANSFORMERS_VERBOSITY=error 
+export TRANSFORMERS_VERBOSITY=error
 
 genai-bench benchmark --api-backend openai \
             --api-base "http://localhost:8082" \
@@ -182,7 +182,7 @@ genai-bench benchmark --api-backend openai \
             --server-engine "vLLM" \
             --server-gpu-type "H100" \
             --server-version "v0.6.0" \
-            --server-gpu-count 4 
+            --server-gpu-count 4
 ```
 
 <!-- TOC --><a name="start-a-vision-benchmark"></a>
@@ -238,7 +238,7 @@ genai-bench benchmark --api-backend openai \
             --traffic-scenario "E(1024)" \
             --server-gpu-type "H100" \
             --server-version "v0.4.2" \
-            --server-gpu-count 1 
+            --server-gpu-count 1
 ```
 
 <!-- TOC --><a name="start-a-rerank-benchmark"></a>
@@ -270,7 +270,7 @@ genai-bench benchmark --api-backend oci-cohere \
 
 ### Start a benchmark against OCI Cohere
 
-Below is a sample command to trigger a benchmark against cohere chat API. 
+Below is a sample command to trigger a benchmark against cohere chat API.
 
 ```shell
 genai-bench benchmark --api-backend oci-cohere \
@@ -311,7 +311,7 @@ genai-bench benchmark \
             --max-requests-per-run 300 \
             --num-concurrency 1 --num-concurrency 2 --num-concurrency 4 \
             --num-concurrency 8 --num-concurrency 16 --num-concurrency 32 \
-            --traffic-scenario "N(480,240)/(300,150)" --traffic-scenario "D(100,100)" 
+            --traffic-scenario "N(480,240)/(300,150)" --traffic-scenario "D(100,100)"
 ```
 
 <!-- TOC --><a name="notes-on-specific-options"></a>
@@ -350,7 +350,7 @@ For heavier traffic scenarios, like `D(16000,200)` or `D(128000,200)`, use the f
 If you see the message below in the genai-bench logs, it indicates that a single process is insufficient to generate the desired load.
 
 ```log
-CPU usage above 90%! This may constrain your throughput and may even give inconsistent response time measurements! 
+CPU usage above 90%! This may constrain your throughput and may even give inconsistent response time measurements!
 ```
 
 To address this, you can increase the number of worker processes using the `--num-workers` option. For example, to spin up 4 worker processes, use:
@@ -364,9 +364,9 @@ This distributes the load across multiple processes on a single machine, improvi
 
 #### Notes on Usage
 
-1. This feature is experimental, so monitor the system’s behavior when enabling multiple workers. 
+1. This feature is experimental, so monitor the system’s behavior when enabling multiple workers.
 2. Recommended Limit: Do **not** set the number of workers to more than 16, as excessive worker processes can lead to resource contention and diminished performance.
-3. Ensure your system has sufficient CPU and memory resources to support the desired number of workers. 
+3. Ensure your system has sufficient CPU and memory resources to support the desired number of workers.
 4. Adjust the number of workers based on your target load and system capacity to achieve optimal results.
 
 <!-- TOC --><a name="using-dataset-configurations"></a>

examples/dataset_configs/config_llava-bench-in-the-wild.json

@@ -8,4 +8,4 @@
   },
   "prompt_column": "question",
   "image_column": "image"
-}
+}