Add next batch of checks

Author: pgmpablo157321

tools/submission/submission_checker/checks/performance_check.py

@@ -199,6 +199,12 @@ def min_duration_check(self):
         return True
 
     def network_check(self):
+        if self.system_json is None:
+            self.log.error(
+                "%s system json file not found",
+                self.path
+            )
+            return False
         is_network_mode_sys_spec_str = self.system_json.get(SYSTEM_DESC_IS_NETWORK_MODE)
         is_network_system = (
             is_network_mode_sys_spec_str.lower() == "true"

tools/submission/submission_checker/checks/system_check.py

@@ -0,0 +1,118 @@
+from .base import BaseCheck
+from ..constants import *
+from ..loader import SubmissionLogs
+from ..configuration.configuration import Config
+
+class SystemCheck(BaseCheck):
+    def __init__(self, log, path, config: Config, submission_logs: SubmissionLogs):
+        super().__init__(log, path)
+        self.submission_logs = submission_logs
+        self.system_json = self.submission_logs.system_json
+        self.submitter = self.submission_logs.loader_data.get("submitter", "")
+        self.division = self.submission_logs.loader_data.get("division", "")
+        self.config = config
+        self.setup_checks()
+
+    def setup_checks(self):
+        self.checks.append(self.missing_check)
+        self.checks.append(self.availability_check)
+        self.checks.append(self.system_type_check)
+        self.checks.append(self.network_check)
+        self.checks.append(self.required_fields_check)
+        self.checks.append(self.submitter_check)
+        self.checks.append(self.division_check)
+    
+    def missing_check(self):
+        if self.system_json is None:
+            self.log.error(
+                "%s system json file not found",
+                self.path
+            )
+            return False
+        return True
+    
+    def availability_check(self):
+        availability = self.system_json.get("status").lower()
+        if availability not in VALID_AVAILABILITIES:
+            self.log.error(
+                "%s has invalid status (%s)", self.path, availability
+            )
+            return False
+        return True
+
+    def system_type_check(self):
+        system_type = self.system_json.get("system_type")
+        valid_system_types = [
+            "datacenter", "edge", "datacenter,edge", "edge,datacenter"]
+
+        if system_type not in valid_system_types:
+            self.log.error(
+                "%s has invalid system type (%s)",
+                self.path,
+                system_type,
+            )
+            return False
+        #Maybe add this line if needed
+        #self.config.set_type(system_type)
+        return True
+    
+    def network_check(self):
+        is_network = self.system_json.get(SYSTEM_DESC_IS_NETWORK_MODE)
+        is_network = (
+        is_network.lower() == "true"
+            if is_network is not None
+            else False
+        )
+        expected_state_by_division = {"network": True, "closed": False}
+        if self.division in expected_state_by_division:
+            if expected_state_by_division[self.division] != is_network:
+                self.log.error(
+                    f"{self.path} incorrect network mode (={is_network}) for division '{self.division}'"
+                )
+                return False
+        return False
+    
+    def required_fields_check(self):
+        required_fields = SYSTEM_DESC_REQUIRED_FIELDS.copy()
+        is_network = self.system_json.get(SYSTEM_DESC_IS_NETWORK_MODE)
+        if is_network:
+            required_fields += SYSTEM_DESC_REQUIRED_FIELDS_NETWORK_MODE
+
+        check_empty_fields = True
+        is_valid = True
+        for k in required_fields:
+            if k not in self.system_json:
+                is_valid = False
+                self.log.error("%s, field %s is missing", self.path, k)
+            elif (
+                check_empty_fields
+                and k in SYSTEM_DESC_MEANINGFUL_RESPONSE_REQUIRED_FIELDS
+                and not self.system_json[k]
+            ):
+                is_valid = False
+                self.log.error(
+                    "%s, field %s requires a meaningful response but is empty", self.path, k
+                )
+        return is_valid
+    
+    def submitter_check(self):
+        if self.system_json.get("submitter").lower() != self.submitter.lower():
+            self.log.error(
+                "%s has submitter %s, directory has %s",
+                self.path,
+                self.system_json.get("submitter"),
+                self.submitter,
+            )
+            return False
+        return True
+    
+    def division_check(self):
+        if self.system_json.get("division").lower() != self.division.lower():
+            self.log.error(
+                "%s has division %s, directory has %s",
+                self.path,
+                self.system_json.get("division"),
+                self.division,
+            )
+            return False
+        return True

tools/submission/submission_checker/checks/systems_check.py

@@ -1025,10 +1025,10 @@
 }
 
 MEASUREMENTS_PATH = {
-    "v5.0": "{division}/{submitter}/measurements/{system}/{benchmark}/{scenario}/{system}.json",
-    "v5.1": "{division}/{submitter}/measurements/{system}/{benchmark}/{scenario}/{system}.json",
-    "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/{system}.json",
-    "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/{system}.json",
+    "v5.0": "{division}/{submitter}/measurements/{system}/{benchmark}/{scenario}/{file}.json",
+    "v5.1": "{division}/{submitter}/measurements/{system}/{benchmark}/{scenario}/{file}.json",
+    "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/measurements.json",
+    "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/measurements.json",
 }
 
 TEST01_PERF_PATH = {

tools/submission/submission_checker/constants.py

@@ -34,6 +34,7 @@ def __init__(self, root, version) -> None:
         self.acc_json_path = os.path.join(self.root, ACCURACY_JSON_PATH.get(version, ACCURACY_JSON_PATH["default"]))
         self.system_log_path = os.path.join(self.root, SYSTEM_PATH.get(version, SYSTEM_PATH["default"]))
 
+
     def load_single_log(self, path, log_type: Literal["Performance", "Accuracy", "AccuracyResult", "AccuracyJSON", "Test", "System"]):
         log = None
         if os.path.exists(path):
@@ -89,6 +90,7 @@ def load(self) -> Generator[SubmissionLogs, None, None]:
                                 "scenario": scenario,
                                 "perf_path": perf_path,
                                 "acc_path": acc_path,
+                                "system_path": system_path,
                                 "model_mapping": model_mapping,
                             }
                             yield SubmissionLogs(perf_log, acc_log, acc_result, acc_json, system_json, loader_data)