Quick fixes for loader class

Author: pgmpablo157321

tools/submission/submission_checker/__init__.py

@@ -2,7 +2,13 @@
 from .constants import PERFORMANCE_LOG_PATH, PERFORMANCE_SUMMARY_PATH, ACCURACY_LOG_PATH, VALID_DIVISIONS
 from .utils import list_dir
 from .parsers.loadgen_parser import LoadgenParser
-from typing import Generator
+from typing import Generator, Literal
+import logging
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="[%(asctime)s %(filename)s:%(lineno)d %(levelname)s] %(message)s",
+)
 
 
 class SubmissionLogs:
@@ -15,26 +21,36 @@ class Loader:
     def __init__(self, root, version) -> None:
         self.root = root
         self.version = version
-        self.perf_log_path = PERFORMANCE_LOG_PATH.get(version, PERFORMANCE_LOG_PATH["default"])
-        self.perf_summary_path = PERFORMANCE_SUMMARY_PATH.get(version, PERFORMANCE_SUMMARY_PATH["default"])
-        self.acc_log_path = ACCURACY_LOG_PATH.get(version, ACCURACY_LOG_PATH["default"])
+        self.logger = logging.getLogger("LoadgenParser")
+        self.perf_log_path = os.path.join(self.root, PERFORMANCE_LOG_PATH.get(version, PERFORMANCE_LOG_PATH["default"]))
+        self.perf_summary_path = os.path.join(self.root, PERFORMANCE_SUMMARY_PATH.get(version, PERFORMANCE_SUMMARY_PATH["default"]))
+        self.acc_log_path = os.path.join(self.root, ACCURACY_LOG_PATH.get(version, ACCURACY_LOG_PATH["default"]))
+
+    def load_single_log(self, path, log_type: Literal["Performance", "Accuracy", "Test"]):
+        log = None
+        if os.path.exists(path):
+            self.logger.info("Loading %s log from %s", log_type, path)
+            log = LoadgenParser(path)
+        else:
+            self.logger.info("Could not load %s log from %s, path does not exist", log_type, path)
+        return log
 
 
-    def load(self) -> Generator[SubmissionLogs, None]:
+    def load(self) -> Generator[SubmissionLogs, None, None]:
         for division in list_dir(self.root):
             if division not in VALID_DIVISIONS:
                 continue
             division_path = os.path.join(self.root, division)
-            for submitter in list_dir(division):
+            for submitter in list_dir(division_path):
                 results_path = os.path.join(division_path, submitter, "results")
                 for system in list_dir(results_path):
                     system_path = os.path.join(results_path, system)
                     for benchmark in list_dir(system_path):
                         benchmark_path = os.path.join(system_path, benchmark)
-                        for scenario in benchmark_path:
+                        for scenario in list_dir(benchmark_path):
                             scenario_path = os.path.join(benchmark_path, benchmark)
-                            perf_log = LoadgenParser(self.perf_log_path.format(division = division, submitter = submitter, system = system, benchmark = system, scenario = scenario))
-                            acc_log = LoadgenParser(self.acc_log_path.format(division = division, submitter = submitter, system = system, benchmark = system, scenario = scenario))
+                            perf_path = self.perf_log_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
+                            acc_path = self.acc_log_path.format(division = division, submitter = submitter, system = system, benchmark = benchmark, scenario = scenario)
+                            perf_log = self.load_single_log(perf_path, "Performance")
+                            acc_log = perf_log = self.load_single_log(acc_path, "Accuracy")
                             yield SubmissionLogs(perf_log, acc_log)
-        yield None
-

tools/submission/submission_checker/configuration/__init__.py

@@ -2,7 +2,7 @@
 import logging
 import os
 
-from .constants import MODEL_CONFIG, CSV_HEAD
+from .constants import MODEL_CONFIG
 from .configuration.configuration import Config
 
 logging.basicConfig(level=logging.INFO)
@@ -15,7 +15,7 @@ def get_args():
     parser.add_argument("--input", required=True, help="submission directory")
     parser.add_argument(
         "--version",
-        default="v4.1",
+        default="v5.1",
         choices=list(MODEL_CONFIG.keys()),
         help="mlperf version",
     )

tools/submission/submission_checker/loader.py

@@ -3,14 +3,14 @@
 
 def list_dir(*path):
     path = os.path.join(*path)
-    return [f for f in os.listdir(
-        path) if os.path.isdir(os.path.join(path, f))]
+    return sorted([f for f in os.listdir(
+        path) if os.path.isdir(os.path.join(path, f))])
 
 
 def list_files(*path):
     path = os.path.join(*path)
-    return [f for f in os.listdir(
-        path) if os.path.isfile(os.path.join(path, f))]
+    return sorted([f for f in os.listdir(
+        path) if os.path.isfile(os.path.join(path, f))])
 
 
 def list_empty_dirs_recursively(*path):