[Automated Commit] Format Codebase

Author: github-actions[bot]

tools/submission/submission_checker/checks/accuracy_check.py

@@ -5,8 +5,10 @@
 import re
 import os
 
+
 class AccuracyCheck(BaseCheck):
-    def __init__(self, log, path, config: Config, submission_logs: SubmissionLogs):
+    def __init__(self, log, path, config: Config,
+                 submission_logs: SubmissionLogs):
         super().__init__(log, path)
         self.name = "accuracy checks"
         self.submission_logs = submission_logs
@@ -15,9 +17,12 @@ def __init__(self, log, path, config: Config, submission_logs: SubmissionLogs):
         self.accuracy_json = self. submission_logs.accuracy_json
         self.config = config
         self.model = self.submission_logs.loader_data.get("benchmark", "")
-        self.model_mapping = self.submission_logs.loader_data.get("model_mapping", {})
-        self.model = self.config.get_mlperf_model(self.model, self.model_mapping)
-        self.scenario_fixed = self.submission_logs.loader_data.get("scenario", "")
+        self.model_mapping = self.submission_logs.loader_data.get(
+            "model_mapping", {})
+        self.model = self.config.get_mlperf_model(
+            self.model, self.model_mapping)
+        self.scenario_fixed = self.submission_logs.loader_data.get(
+            "scenario", "")
         self.scenario = self.mlperf_log["effective_scenario"]
         self.division = self.submission_logs.loader_data.get("division", "")
         self.setup_checks()
@@ -97,7 +102,7 @@ def accuracy_result_check(self):
             is_valid = False
 
         return is_valid
-    
+
     def accuracy_json_check(self):
         if not os.path.exists(self.accuracy_json):
             self.log.error("%s is missing", self.accuracy_json)
@@ -107,7 +112,7 @@ def accuracy_json_check(self):
                 self.log.error("%s is not truncated", self.accuracy_json)
                 return False
         return True
-    
+
     def loadgen_errors_check(self):
         if self.mlperf_log.has_error():
             if self.config.ignore_uncommited:
@@ -125,7 +130,7 @@ def loadgen_errors_check(self):
                 )
                 return False
         return True
-    
+
     def dataset_check(self):
         qsl_total_count = self.mlperf_log["qsl_reported_total_count"]
         expected_qsl_total_count = self.config.get_dataset_size(self.model)
@@ -134,4 +139,4 @@ def dataset_check(self):
                 "%s accurcy run does not cover all dataset, accuracy samples: %s, dataset size: %s", self.path, qsl_total_count, expected_qsl_total_count
             )
             return False
-        return True
+        return True

tools/submission/submission_checker/checks/base.py

@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod
 
+
 class BaseCheck(ABC):
     """
     A generic check class meant to be inherited by concrete check implementations.
@@ -27,16 +28,19 @@ def run_checks(self):
             if not valid:
                 return False
         return valid
-            
+
     def execute(self, check):
         return check()
-    
+
     def __call__(self):
         """Allows the check instance to be called like a function."""
         self.log.info("Starting %s for: %s", self.name, self.path)
         valid = self.run_checks()
         if valid:
             self.log.info("All %s checks passed for: %s", self.name, self.path)
         else:
-            self.log.error("Some %s Checks failed for: %s", self.name, self.path)
-        return valid
+            self.log.error(
+                "Some %s Checks failed for: %s",
+                self.name,
+                self.path)
+        return valid

tools/submission/submission_checker/checks/performance_check.py

@@ -3,18 +3,23 @@
 from ..loader import SubmissionLogs
 from ..configuration.configuration import Config
 
+
 class PerformanceCheck(BaseCheck):
-    def __init__(self, log, path, config: Config, submission_logs: SubmissionLogs):
+    def __init__(self, log, path, config: Config,
+                 submission_logs: SubmissionLogs):
         super().__init__(log, path)
         self.name = "performance checks"
         self.submission_logs = submission_logs
         self.mlperf_log = self.submission_logs.performance_log
         self.system_json = self.submission_logs.system_json
         self.config = config
         self.model = self.submission_logs.loader_data.get("benchmark", "")
-        self.model_mapping = self.submission_logs.loader_data.get("model_mapping", {})
-        self.model = self.config.get_mlperf_model(self.model, self.model_mapping)
-        self.scenario_fixed = self.submission_logs.loader_data.get("scenario", "")
+        self.model_mapping = self.submission_logs.loader_data.get(
+            "model_mapping", {})
+        self.model = self.config.get_mlperf_model(
+            self.model, self.model_mapping)
+        self.scenario_fixed = self.submission_logs.loader_data.get(
+            "scenario", "")
         self.scenario = self.mlperf_log["effective_scenario"]
         self.division = self.submission_logs.loader_data.get("division", "")
         self.setup_checks()
@@ -36,7 +41,7 @@ def missing_check(self):
             self.log.error("Performance log missing at %s", self.path)
             return False
         return True
-    
+
     def loadgen_errors_check(self):
         if self.mlperf_log.has_error():
             if self.config.ignore_uncommited:
@@ -56,13 +61,17 @@ def loadgen_errors_check(self):
         return True
 
     def equal_issue_check(self):
-        if self.config.requires_equal_issue(self.model, self.division) and self.mlperf_log["effective_sample_concatenate_permutation"]:
-            self.log.error("%s requires equal issue mode (sample_concatenate_permutation), expected=true, found=false", self.path)
+        if self.config.requires_equal_issue(
+                self.model, self.division) and self.mlperf_log["effective_sample_concatenate_permutation"]:
+            self.log.error(
+                "%s requires equal issue mode (sample_concatenate_permutation), expected=true, found=false",
+                self.path)
             return False
         return True
-    
+
     def performance_sample_count_check(self):
-        required_performance_sample_count = self.config.get_performance_sample_count(self.model)
+        required_performance_sample_count = self.config.get_performance_sample_count(
+            self.model)
         performance_sample_count = self.mlperf_log["effective_performance_sample_count"]
         if performance_sample_count < required_performance_sample_count:
             self.log.error(
@@ -73,7 +82,7 @@ def performance_sample_count_check(self):
             )
             return False
         return True
-    
+
     def seeds_check(self):
         config_seeds = self.config.seeds
         qsl_rng_seed = self.mlperf_log["effective_qsl_rng_seed"]
@@ -105,7 +114,7 @@ def seeds_check(self):
             )
             is_valid = False
         return is_valid
-    
+
     def latency_check(self):
         uses_early_stopping = self.config.uses_early_stopping(self.scenario)
         if uses_early_stopping:
@@ -165,7 +174,8 @@ def min_query_count_check(self):
         min_query_count = self.mlperf_log["effective_min_query_count"]
         samples_per_query = self.mlperf_log["effective_samples_per_query"]
         if not uses_early_stopping:
-            required_min_query_count = self.config.get_min_query_count(self.model, self.scenario)
+            required_min_query_count = self.config.get_min_query_count(
+                self.model, self.scenario)
             if required_min_query_count and min_query_count < required_min_query_count:
                 self.log.error(
                     "%s Required minimum Query Count not met by user config, Expected=%s, Found=%s",
@@ -184,7 +194,7 @@ def min_query_count_check(self):
             )
             return False
         return True
-    
+
     def min_duration_check(self):
         required_min_duration = TEST_DURATION_MS
         min_duration = self.mlperf_log["effective_min_duration_ms"]
@@ -197,9 +207,10 @@ def min_duration_check(self):
             )
             return False
         return True
-    
+
     def network_check(self):
-        is_network_mode_sys_spec_str = self.system_json.get(SYSTEM_DESC_IS_NETWORK_MODE)
+        is_network_mode_sys_spec_str = self.system_json.get(
+            SYSTEM_DESC_IS_NETWORK_MODE)
         is_network_system = (
             is_network_mode_sys_spec_str.lower() == "true"
             if is_network_mode_sys_spec_str is not None
@@ -216,7 +227,6 @@ def network_check(self):
             )
             return False
 
-
         sut_name = self.mlperf_log["sut_name"]
         if is_network_system:
             # for network mode verify the SUT name is valid, according to the rules
@@ -228,7 +238,6 @@ def network_check(self):
                 return False
 
         return True
-    
 
     def llm_check(self):
         if self.model in self.config.get_llm_models():

tools/submission/submission_checker/configuration/configuration.py

@@ -17,10 +17,9 @@ def __init__(
         self.ignore_uncommited = ignore_uncommited
         self.skip_power_check = skip_power_check
         self.load_config(version)
-        
 
     def load_config(self, version):
-        # TODO: Load values from 
+        # TODO: Load values from
         self.models = self.base["models"]
         self.seeds = self.base["seeds"]
         if self.base.get("test05_seeds"):
@@ -100,7 +99,7 @@ def get_accuracy_target(self, model):
 
     def get_accuracy_upper_limit(self, model):
         return self.accuracy_upper_limit.get(model, None)
-    
+
     def get_accuracy_values(self, model):
         patterns = []
         acc_targets = []
@@ -147,7 +146,7 @@ def get_min_query_count(self, model, scenario):
         if model not in self.min_queries:
             raise ValueError("model not known: " + model)
         return self.min_queries[model].get(scenario)
-    
+
     def get_dataset_size(self, model):
         model = self.get_mlperf_model(model)
         if model not in self.dataset_size:
@@ -187,7 +186,7 @@ def requires_equal_issue(self, model, division):
             ]
             and self.version in ["v4.1"]
         )
-    
+
     def get_llm_models(self):
         return [
             "llama2-70b-99",
@@ -197,6 +196,6 @@ def get_llm_models(self):
             "mixtral-8x7b",
             "llama3.1-405b",
             "llama3.1-8b",
-            "llama3.1-8b-edge", 
+            "llama3.1-8b-edge",
             "deepseek-r1"
         ]

tools/submission/submission_checker/constants.py

@@ -1072,4 +1072,4 @@
     "v5.1": "{division}/{submitter}/systems/{system}.json",
     "v6.0": "{division}/{submitter}/systems/{system}.json",
     "default": "{division}/{submitter}/systems/{system}.json",
-}
+}