Add results to submission checker

Author: pgmpablo157321

tools/submission/submission_checker/checks/accuracy_check.py

@@ -29,8 +29,6 @@ def setup_checks(self):
         self.checks.append(self.dataset_check)
 
     def accuracy_result_check(self):
-        if self.division.lower() == "open":
-            return True
         patterns, acc_targets, acc_types, acc_limits, up_patterns, acc_upper_limit = self.config.get_accuracy_values(
             self.model
         )
@@ -95,7 +93,9 @@ def accuracy_result_check(self):
         if not hash_val:
             self.log.error("%s not hash value for accuracy.txt", self.path)
             is_valid = False
-
+        self.submission_logs.loader_data["accuracy_metrics"] = result_acc
+        if self.division.lower() == "open":
+            return True
         return is_valid
 
     def accuracy_json_check(self):

tools/submission/submission_checker/checks/performance_check.py

@@ -2,6 +2,7 @@
 from ..constants import *
 from ..loader import SubmissionLogs
 from ..configuration.configuration import Config
+import os
 
 class PerformanceCheck(BaseCheck):
     def __init__(self, log, path, config: Config, submission_logs: SubmissionLogs):
@@ -31,6 +32,7 @@ def setup_checks(self):
         self.checks.append(self.network_check)
         self.checks.append(self.llm_check)
         self.checks.append(self.inferred_check)
+        self.checks.append(self.get_performance_metric_check)
 
     def missing_check(self):
         if self.mlperf_log is None:
@@ -282,3 +284,78 @@ def inferred_check(self):
                 self.log.error("Result for scenario %s can not be inferred from %s for: %s", self.scenario_fixed, self.scenario, self.path)
                 return False
         return True
+    
+    def get_performance_metric_check(self):
+        # Assumes new logging format
+        is_valid = True
+        version = self.config.version
+        if (
+            "result_validity" in self.mlperf_log.get_keys()
+            and self.mlperf_log["result_validity"] == "VALID"
+        ):
+            is_valid = True
+        scenario = self.mlperf_log["effective_scenario"]
+
+        res = float(self.mlperf_log[RESULT_FIELD_NEW[version][scenario]])
+        if (
+            version in RESULT_FIELD_BENCHMARK_OVERWRITE
+            and self.model in RESULT_FIELD_BENCHMARK_OVERWRITE[version]
+            and scenario in RESULT_FIELD_BENCHMARK_OVERWRITE[version][self.model]
+        ):
+            res = float(
+                self.mlperf_log[RESULT_FIELD_BENCHMARK_OVERWRITE[version]
+                        [self.model][scenario]]
+            )
+
+        inferred = False
+        if self.scenario.lower() != self.scenario_fixed.lower() and (self.scenario.lower(), self.scenario_fixed.lower()) != ("server", "interactive"):
+            res, is_valid = self.get_inferred_result(res)
+        self.submission_logs.loader_data["performance_metric"] = res
+        return is_valid
+    
+    def get_inferred_result(self, res):
+
+        inferred = False
+        is_valid = True
+        # Check if current scenario (and version) uses early stopping
+        uses_early_stopping = self.config.uses_early_stopping(self.scenario)
+
+        latency_mean = self.mlperf_log["result_mean_latency_ns"]
+        if self.scenario in ["MultiStream"]:
+            latency_99_percentile = self.mlperf_log[
+                "result_99.00_percentile_per_query_latency_ns"
+            ]
+            latency_mean = self.mlperf_log["result_mean_query_latency_ns"]
+        samples_per_query = self.mlperf_log["effective_samples_per_query"]
+        if self.scenario == "SingleStream":
+            # qps_wo_loadgen_overhead is only used for inferring Offline from
+            # SingleStream; only for old submissions
+            qps_wo_loadgen_overhead = self.mlperf_log["result_qps_without_loadgen_overhead"]
+
+        # special case for results inferred from different scenario
+        if self.scenario_fixed in ["Offline"] and self.scenario in ["SingleStream"]:
+            inferred = True
+            res = qps_wo_loadgen_overhead
+
+        if (self.scenario_fixed in ["Offline"]) and self.scenario in ["MultiStream"]:
+            inferred = True
+            res = samples_per_query * S_TO_MS / (latency_mean / MS_TO_NS)
+
+        if (self.scenario_fixed in ["MultiStream"]) and self.scenario in ["SingleStream"]:
+            inferred = True
+            # samples_per_query does not match with the one reported in the logs
+            # when inferring MultiStream from SingleStream
+            samples_per_query = 8
+            if uses_early_stopping:
+                early_stopping_latency_ms = self.mlperf_log["early_stopping_latency_ms"]
+                if early_stopping_latency_ms == 0:
+                    self.log.error(
+                        "Not enough samples were processed for early stopping to make an estimate"
+                    )
+                    is_valid = False
+                res = (early_stopping_latency_ms * samples_per_query) / MS_TO_NS
+            else:
+                res = (latency_99_percentile * samples_per_query) / MS_TO_NS
+        if (self.scenario_fixed in ["Interactive"]) and self.scenario not in ["Server"]:
+            is_valid = False
+        return res, is_valid

tools/submission/submission_checker/constants.py

@@ -969,7 +969,7 @@
     "sw_notes",
     "host_network_card_count",
     "system_type_detail",
-    "network_speed_mbit",
+    #"network_speed_mbit",
 ]
 
 SYSTEM_DESC_MEANINGFUL_RESPONSE_REQUIRED_FIELDS = [
@@ -1002,7 +1002,7 @@
 ]
 
 SYSTEM_DESC_NUMERIC_RESPONSE_REQUIRED_FIELDS = [
-    "network_speed_mbit"
+    #"network_speed_mbit"
 ]
 
 
@@ -1052,6 +1052,67 @@
     "weight_transformations",
 ]
 
+SPECIAL_UNIT_DICT = {
+    "llama3.1-8b": {
+        "Offline": "Tokens/s",
+        "Server": "Tokens/s",
+    },
+    "llama3.1-8b-edge": {
+        "Offline": "Tokens/s",
+    },
+    "llama2-70b-99": {
+        "Offline": "Tokens/s",
+        "Server": "Tokens/s",
+        "Interactive": "Tokens/s",
+    },
+    "llama2-70b-99.9": {
+        "Offline": "Tokens/s",
+        "Server": "Tokens/s",
+        "Interactive": "Tokens/s",
+    },
+    "mixtral-8x7b": {
+        "Offline": "Tokens/s",
+        "Server": "Tokens/s",
+        "Interactive": "Tokens/s",
+    },
+    "llama3.1-405b": {
+        "Offline": "Tokens/s",
+        "Server": "Tokens/s",
+        "Interactive": "Tokens/s",
+    },
+    "deepseek-r1": {
+        "Offline": "Tokens/s",
+        "Server": "Tokens/s",
+        "Interactive": "Tokens/s",
+    },
+}
+UNIT_DICT = {
+    "SingleStream": "Latency (ms)",
+    "MultiStream": "Latency (ms)",
+    "Offline": "Samples/s",
+    "Server": "Queries/s",
+    "Interactive": "Queries/s",
+
+    "singlestream": "Latency (ms)",
+    "multistream": "Latency (ms)",
+    "offline": "Samples/s",
+    "server": "Queries/s",
+    "interactive": "Queries/s",
+}
+POWER_UNIT_DICT = {
+    "SingleStream": "millijoules",
+    "MultiStream": "millijoules",
+    "Offline": "Watts",
+    "Server": "Watts",
+    "Interactive": "Watts",
+
+    "singlestream": "millijoules",
+    "multistream": "millijoules",
+    "offline": "Watts",
+    "server": "Watts",
+    "interactive": "Watts",
+}
+
 
 PERFORMANCE_LOG_PATH = {
     "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/mlperf_log_detail.txt",

tools/submission/submission_checker/main.py

@@ -11,6 +11,7 @@
 from .checks.measurements_checks import MeasurementsCheck
 from .checks.compliance_check import ComplianceCheck
 from .checks.power_check import PowerCheck
+from .results import ResultExporter
 
 logging.basicConfig(level=logging.INFO)
 log = logging.getLogger("main")
@@ -101,6 +102,7 @@ def main():
         scenarios_to_skip = []
 
     loader = Loader(args.input, args.version)
+    exporter = ResultExporter(args.csv, config)
     for logs in loader.load():
         # Initialize check classes
         performance_checks = PerformanceCheck(log, logs.loader_data["perf_path"], config, logs)
@@ -110,15 +112,17 @@ def main():
         measurements_checks = ComplianceCheck(log, logs.loader_data["compliance_path"], config, logs)
         power_checks = PowerCheck(log, logs.loader_data["power_dir_path"], config, logs)
         # Run checks
-        performance_checks()
-        accuracy_checks()
-        system_checks()
-        measurements_checks()
-        power_checks()
-
-    with open(args.csv, "w") as csv:
-        # Output summary
-        pass
+        valid = True
+        valid &= performance_checks()
+        valid &= accuracy_checks()
+        valid &= system_checks()
+        valid &= measurements_checks()
+        valid &= power_checks()
+        # Add results to summary
+        if valid:
+            exporter.add_result(logs)
+    # Export results
+    exporter.export()
 
     # log results
     results = {}

tools/submission/submission_checker/results.py

@@ -0,0 +1,103 @@
+
+from .loader import SubmissionLogs
+from .configuration.configuration import Config
+import os
+import csv
+from .constants import *
+import json
+
+class ResultExporter:
+    def __init__(self, csv_path, config: Config) -> None:
+        self.head = [
+            "Organization",
+            "Availability",
+            "Division",
+            "SystemType",
+            "SystemName",
+            "Platform",
+            "Model",
+            "MlperfModel",
+            "Scenario",
+            "Result",
+            "Accuracy",
+            "number_of_nodes",
+            "host_processor_model_name",
+            "host_processors_per_node",
+            "host_processor_core_count",
+            "accelerator_model_name",
+            "accelerators_per_node",
+            "Location",
+            "framework",
+            "operating_system",
+            "notes",
+            "compliance",
+            "errors",
+            "version",
+            "inferred",
+            "has_power",
+            "Units",
+            "weight_data_types",
+        ]
+        self.rows = []
+        self.csv_path = csv_path
+        self.config = config
+
+    def add_result(self, submission_logs: SubmissionLogs):
+        row = {key: "" for key in self.head}
+        row["Organization"] = submission_logs.loader_data["submitter"]
+        row["Availability"] = submission_logs.system_json["status"]
+        row["Division"] = submission_logs.loader_data["division"]
+        row["SystemType"] = submission_logs.system_json["system_type"]
+        row["SystemName"] = submission_logs.system_json["system_name"]
+        row["Platform"] = submission_logs.loader_data["system"]
+        row["Model"] = submission_logs.loader_data["benchmark"]
+        row["MlperfModel"] = self.config.get_mlperf_model(row["Model"], submission_logs.loader_data.get("model_mapping", {}))
+        row["Scenario"] = submission_logs.loader_data["scenario"]
+        row["Result"] = submission_logs.loader_data["performance_metric"]
+        row["Accuracy"] = json.dumps(submission_logs.loader_data["accuracy_metrics"]).replace(",", " ").replace('"', "").replace("{", "").replace("}", "").strip()
+        row["number_of_nodes"] = submission_logs.system_json["number_of_nodes"]
+        row["host_processor_model_name"] = submission_logs.system_json["host_processor_model_name"]
+        row["host_processors_per_node"] = submission_logs.system_json["host_processors_per_node"]
+        row["host_processor_core_count"] = submission_logs.system_json["host_processor_core_count"]
+        row["accelerator_model_name"] = submission_logs.system_json["accelerator_model_name"]
+        row["accelerators_per_node"] = submission_logs.system_json["accelerators_per_node"]
+        row["Location"] = os.path.dirname(submission_logs.loader_data["perf_path"])
+        row["framework"] = submission_logs.system_json["framework"]
+        row["operating_system"] = submission_logs.system_json["operating_system"]
+        notes = submission_logs.system_json.get("hw_notes", "")
+        if submission_logs.system_json.get("sw_notes"):
+            notes = notes + ". " if notes else ""
+            notes = notes + submission_logs.system_json.get("sw_notes")
+        row["notes"] = notes
+        row["compliance"] = submission_logs.loader_data["division"] # TODO
+        row["errors"] = 0
+        row["version"] = self.config.version        
+        row["inferred"] = 1 if row["Scenario"] != submission_logs.performance_log["effective_scenario"] and (submission_logs.performance_log["effective_scenario"], row["Scenario"]) != ("server", "interactive") else 0
+        row["has_power"] = os.path.exists(submission_logs.loader_data["power_dir_path"])
+        unit = SPECIAL_UNIT_DICT.get(
+            row["MlperfModel"], UNIT_DICT).get(
+            row["Scenario"], UNIT_DICT[row["Scenario"]]
+        )
+        row["Units"] = unit
+        row["weight_data_types"] = submission_logs.measurements_json["weight_data_types"]
+        self.rows.append(row.copy())
+        if row["has_power"]:
+            row["Result"] = submission_logs.loader_data["division"] #TODO
+            power_unit = POWER_UNIT_DICT[row["Scenario"]]
+            row["Units"] = power_unit
+            self.rows.append(row.copy())
+
+
+    def export_row(self, row: dict):
+        values = [str(row.get(key, "")) for key in self.head]
+        csv_row = ",".join(values) + "\n"
+        with open(self.csv_path, "+a") as csv:
+            csv.write(csv_row)
+
+
+    def export(self):
+        csv_header = ",".join(self.head) + "\n"
+        with open(self.csv_path, "w") as csv:
+            csv.write(csv_header)
+        for row in self.rows:
+            self.export_row(row)