Skip to content

Commit 9f7e9f4

Browse files
pgmpablo157321pgmpablo157321
committed
Add initial loader loop
1 parent ce2244c commit 9f7e9f4

File tree

3 files changed

+104
-4
lines changed

3 files changed

+104
-4
lines changed

‎tools/submission/submission_checker/constants.py‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -985,4 +985,68 @@
985985
"starting_weights_filename",
986986
"weight_data_types",
987987
"weight_transformations",
988-
]
988+
]
989+
990+
991+
PERFORMANCE_LOG_PATH = {
992+
"v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/mlperf_log_detail.txt",
993+
"v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/mlperf_log_detail.txt",
994+
"v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/mlperf_log_detail.txt",
995+
"default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/mlperf_log_detail.txt",
996+
}
997+
998+
PERFORMANCE_SUMMARY_PATH = {
999+
"v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/mlperf_log_summary.txt",
1000+
"v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/mlperf_log_summary.txt",
1001+
"v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/mlperf_log_summary.txt",
1002+
"default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/mlperf_log_summary.txt",
1003+
}
1004+
1005+
ACCURACY_LOG_PATH = {
1006+
"v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_detail.txt",
1007+
"v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_detail.txt",
1008+
"v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_detail.txt",
1009+
"default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_detail.txt",
1010+
}
1011+
1012+
MEASUREMENTS_PATH = {
1013+
"v5.0": "{division}/{submitter}/measurements/{system}/{benchmark}/{scenario}/{system}.json",
1014+
"v5.1": "{division}/{submitter}/measurements/{system}/{benchmark}/{scenario}/{system}.json",
1015+
"v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/{system}.json",
1016+
"default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/{system}.json",
1017+
}
1018+
1019+
TEST01_PERF_PATH = {
1020+
"v5.0": "{division}/{submitter}/compliance/{system}/{benchmark}/{scenario}/TEST01/performance/run_1/mlperf_log_detail.txt",
1021+
"v5.1": "{division}/{submitter}/compliance/{system}/{benchmark}/{scenario}/TEST01/performance/run_1/mlperf_log_detail.txt",
1022+
"v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST01/performance/run_1/mlperf_log_detail.txt",
1023+
"default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST01/performance/run_1/mlperf_log_detail.txt",
1024+
}
1025+
1026+
TEST01_ACC_PATH = {
1027+
"v5.0": "{division}/{submitter}/compliance/{system}/{benchmark}/{scenario}/TEST01/accuracy/mlperf_log_detail.txt",
1028+
"v5.1": "{division}/{submitter}/compliance/{system}/{benchmark}/{scenario}/TEST01/accuracy/mlperf_log_detail.txt",
1029+
"v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST01/accuracy/mlperf_log_detail.txt",
1030+
"default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST01/accuracy/mlperf_log_detail.txt",
1031+
}
1032+
1033+
TEST05_PERF_PATH = {
1034+
"v5.0": "{division}/{submitter}/compliance/{system}/{benchmark}/{scenario}/TEST05/performance/run_1/mlperf_log_detail.txt",
1035+
"v5.1": "{division}/{submitter}/compliance/{system}/{benchmark}/{scenario}/TEST05/performance/run_1/mlperf_log_detail.txt",
1036+
"v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST05/performance/run_1/mlperf_log_detail.txt",
1037+
"default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST05/performance/run_1/mlperf_log_detail.txt",
1038+
}
1039+
1040+
TEST05_ACC_PATH = {
1041+
"v5.0": "{division}/{submitter}/compliance/{system}/{benchmark}/{scenario}/TEST05/accuracy/mlperf_log_detail.txt",
1042+
"v5.1": "{division}/{submitter}/compliance/{system}/{benchmark}/{scenario}/TEST05/accuracy/mlperf_log_detail.txt",
1043+
"v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST05/accuracy/mlperf_log_detail.txt",
1044+
"default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST05/accuracy/mlperf_log_detail.txt",
1045+
}
1046+
1047+
TEST06_ACC_PATH = {
1048+
"v5.0": "{division}/{submitter}/compliance/{system}/{benchmark}/{scenario}/TEST06/accuracy/mlperf_log_detail.txt",
1049+
"v5.1": "{division}/{submitter}/compliance/{system}/{benchmark}/{scenario}/TEST06/accuracy/mlperf_log_detail.txt",
1050+
"v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST06/accuracy/mlperf_log_detail.txt",
1051+
"default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/TEST06/accuracy/mlperf_log_detail.txt",
1052+
}

‎tools/submission/submission_checker/loader.py‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,40 @@
1+
import os
2+
from .constants import PERFORMANCE_LOG_PATH, PERFORMANCE_SUMMARY_PATH, ACCURACY_LOG_PATH, VALID_DIVISIONS
3+
from .utils import list_dir
4+
from .parsers.loadgen_parser import LoadgenParser
5+
from typing import Generator
6+
7+
8+
class SubmissionLogs:
9+
def __init__(self, performance_log, accuracy_log) -> None:
10+
self.performance_log = performance_log
11+
self.accuracy_log = accuracy_log
12+
113

214
class Loader:
3-
def __init__(self) -> None:
4-
pass
15+
def __init__(self, root, version) -> None:
16+
self.root = root
17+
self.version = version
18+
self.perf_log_path = PERFORMANCE_LOG_PATH.get(version, PERFORMANCE_LOG_PATH["default"])
19+
self.perf_summary_path = PERFORMANCE_SUMMARY_PATH.get(version, PERFORMANCE_SUMMARY_PATH["default"])
20+
self.acc_log_path = ACCURACY_LOG_PATH.get(version, ACCURACY_LOG_PATH["default"])
21+
22+
23+
def load(self) -> Generator[SubmissionLogs, None]:
24+
for division in list_dir(self.root):
25+
if division not in VALID_DIVISIONS:
26+
continue
27+
division_path = os.path.join(self.root, division)
28+
for submitter in list_dir(division):
29+
results_path = os.path.join(division_path, submitter, "results")
30+
for system in list_dir(results_path):
31+
system_path = os.path.join(results_path, system)
32+
for benchmark in list_dir(system_path):
33+
benchmark_path = os.path.join(system_path, benchmark)
34+
for scenario in benchmark_path:
35+
scenario_path = os.path.join(benchmark_path, benchmark)
36+
perf_log = LoadgenParser(self.perf_log_path.format(division = division, submitter = submitter, system = system, benchmark = system, scenario = scenario))
37+
acc_log = LoadgenParser(self.acc_log_path.format(division = division, submitter = submitter, system = system, benchmark = system, scenario = scenario))
38+
yield SubmissionLogs(perf_log, acc_log)
39+
yield None
40+

‎tools/submission/submission_checker/parsers/loadgen_parser.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ def get_dict(self):
9898
else:
9999
self.logger.warning(
100100
"There are multiple messages with key {:} in the log. Emprically choosing the first one.".format(
101-
key
101+
message["key"]
102102
)
103103
)
104104

0 commit comments

Comments
 (0)