[Benchmarks] Benchmarks scripts cleanup (#20460)

Author: PatKamin

devops/scripts/benchmarks/CONTRIB.md

@@ -18,11 +18,10 @@ The suite is structured around four main components: Suites, Benchmarks, Results
     * Represent a single benchmark, usually mapping to a binary execution.
     * Must implement the `Benchmark` base class (`benches/base.py`).
     * **Required Methods:**
-        * `setup()`: Initializes the benchmark (e.g., build, download data). Use `self.download()` for data dependencies. **Do not** perform setup in `__init__`.
         * `run(env_vars)`: Executes the benchmark binary (use `self.run_bench()`) and returns a list of `Result` objects. Can be called multiple times, must produce consistent results.
-        * `teardown()`: Cleans up resources. Can be empty. No need to remove build artifacts or downloaded datasets.
         * `name()`: Returns a unique identifier string for the benchmark across *all* suites. If a benchmark class is instantiated multiple times with different parameters (e.g., "Submit In Order", "Submit Out Of Order"), the `name()` must reflect this uniqueness.
     * **Optional Methods:**
+        * `setup()`: Initializes the benchmark (e.g., build, download data). Use `self.download()` for data dependencies. **Do not** perform setup in `__init__`.
         * `lower_is_better()`: Returns `True` if lower result values are better (default: `True`).
         * `description()`: Provides a short description about the benchmark.
         * `notes()`: Provides additional commentary about the benchmark results (string).
@@ -163,9 +162,49 @@ The benchmark suite generates an interactive HTML dashboard that visualizes `Res
 **Stability:**
 * Mark unstable benchmarks with `metadata.unstable` to hide them by default.
 
+## Code Style Guidelines
+
+### Benchmark Class Structure
+
+When creating benchmark classes, follow this consistent structure pattern:
+
+**1. Constructor (`__init__`):**
+* Assign all parameters to protected (prefixed with `_`) or private (prefixed with `__`) instance variables.
+* Set `self._iterations_regular` and `self._iterations_trace` BEFORE calling `super().__init__()` (required for subclasses of `ComputeBenchmark`).
+
+**2. Method Order:**
+* Align with methods order as in the abstract base class `Benchmark`. Not all of them are required, but follow the order for consistency.
+* Public methods first, then protected, then private.
+
+### Naming Conventions
+
+**Method Return Values:**
+* `name()`: Unique identifier with underscores, lowercase, includes all distinguishing parameters
+  * Example: `"api_overhead_benchmark_sycl SubmitKernel in order with measure completion"`
+* `display_name()`: User-friendly, uses proper capitalization, commas for readability, used for charts titles
+  * Example: `"SYCL SubmitKernel in order, with measure completion, NumKernels 10"`
+
+**Class method names and variables should follow PEP 8 guidelines.**
+* Use lowercase with underscores for method names and variables.
+* Use single underscores prefixes for protected variables/methods and double underscores for private variables/methods.
+
+### Description Writing
+
+Descriptions should:
+* Clearly state what is being measured
+* Include key parameters and their values
+* Explain the purpose or what the benchmark tests
+* Be 1-3 sentences, clear and concise
+* If not needed, can be omitted
+
+### Tag Selection
+
+* Use predefined tags from `benches/base.py` when available
+* Tags should be lowercase, descriptive, single words
+
 ## Adding New Benchmarks
 
-1. **Create Benchmark Class:** Implement a new class inheriting from `benches.base.Benchmark`. Implement required methods (`setup`, `run`, `teardown`, `name`) and optional ones (`description`, `get_tags`, etc.) as needed.
+1. **Create Benchmark Class:** Implement a new class inheriting from `benches.base.Benchmark`. Implement required methods (`run`, `name`) and optional ones (`description`, `get_tags`, etc.) as needed. Follow the code style guidelines above.
 2. **Add to Suite:**
     * If adding to an existing category, modify the corresponding `Suite` class (e.g., `benches/compute.py`) to instantiate and return your new benchmark in its `benchmarks()` method.
     * If creating a new category, create a new `Suite` class inheriting from `benches.base.Suite`. Implement `name()` and `benchmarks()`. Add necessary `setup()` if the suite requires shared setup. Add group metadata via `additional_metadata()` if needed.

devops/scripts/benchmarks/benches/base.py

@@ -56,6 +56,28 @@ def __init__(self, suite):
     def name(self) -> str:
         pass
 
+    @abstractmethod
+    def run(
+        self,
+        env_vars,
+        run_trace: TracingType = TracingType.NONE,
+        force_trace: bool = False,
+    ) -> list[Result]:
+        """Execute the benchmark with the given environment variables.
+
+        Args:
+            env_vars: Environment variables to use when running the benchmark.
+            run_trace: The type of tracing to run (NONE, UNITRACE, or FLAMEGRAPH).
+            force_trace: If True, ignore the traceable() method and force tracing.
+
+        Returns:
+            A list of Result objects with the benchmark results.
+
+        Raises:
+            Exception: If the benchmark fails for any reason.
+        """
+        pass
+
     def display_name(self) -> str:
         """Returns a user-friendly name for display in charts.
         By default returns the same as name(), but can be overridden.
@@ -87,44 +109,6 @@ def setup(self):
         """Extra setup steps to be performed before running the benchmark."""
         pass
 
-    @abstractmethod
-    def teardown(self):
-        pass
-
-    @abstractmethod
-    def run(
-        self,
-        env_vars,
-        run_trace: TracingType = TracingType.NONE,
-        force_trace: bool = False,
-    ) -> list[Result]:
-        """Execute the benchmark with the given environment variables.
-
-        Args:
-            env_vars: Environment variables to use when running the benchmark.
-            run_trace: The type of tracing to run (NONE, UNITRACE, or FLAMEGRAPH).
-            force_trace: If True, ignore the traceable() method and force tracing.
-
-        Returns:
-            A list of Result objects with the benchmark results.
-
-        Raises:
-            Exception: If the benchmark fails for any reason.
-        """
-        pass
-
-    @staticmethod
-    def get_adapter_full_path():
-        for libs_dir_name in ["lib", "lib64"]:
-            adapter_path = os.path.join(
-                options.ur, libs_dir_name, f"libur_adapter_{options.ur_adapter}.so"
-            )
-            if os.path.isfile(adapter_path):
-                return adapter_path
-        assert (
-            False
-        ), f"could not find adapter file {adapter_path} (and in similar lib paths)"
-
     def run_bench(
         self,
         command,
@@ -268,6 +252,18 @@ def get_metadata(self) -> dict[str, BenchmarkMetadata]:
             )
         }
 
+    @staticmethod
+    def get_adapter_full_path():
+        for libs_dir_name in ["lib", "lib64"]:
+            adapter_path = os.path.join(
+                options.ur, libs_dir_name, f"libur_adapter_{options.ur_adapter}.so"
+            )
+            if os.path.isfile(adapter_path):
+                return adapter_path
+        assert (
+            False
+        ), f"could not find adapter file {adapter_path} (and in similar lib paths)"
+
 
 class Suite(ABC):
     @abstractmethod

devops/scripts/benchmarks/benches/benchdnn.py

@@ -95,9 +95,6 @@ def setup(self) -> None:
             timeout=60 * 20,
         )
 
-    def teardown(self):
-        pass
-
 
 class OneDnnBenchmark(Benchmark):
     def __init__(self, suite, bench_driver, bench_name, bench_args, syclgraph=True):
@@ -211,6 +208,3 @@ def _extract_time(self, output):
         if values:
             return sum(values)
         return 0.0
-
-    def teardown(self):
-        pass