Add callback to set rel_threshold for metric results

Author: pwolnows

tools/accuracy_checker/accuracy_checker/evaluators/base_evaluator.py

@@ -42,7 +42,7 @@ def print_metrics_results(self, ignore_results_formatting=False, ignore_metric_r
 
     # extract metrics results values prepared for printing
     def extract_metrics_results(self, print_results=True, ignore_results_formatting=False,
-                                ignore_metric_reference=False):
+                                ignore_metric_reference=False, threshold_callback=None):
         raise NotImplementedError
 
     # destruction for entity, which can not be deleted automatically

tools/accuracy_checker/accuracy_checker/evaluators/model_evaluator.py

@@ -579,13 +579,15 @@ def compute_metrics(self, print_results=True, ignore_results_formatting=False, i
         return self._metrics_results
 
     def extract_metrics_results(self, print_results=True, ignore_results_formatting=False,
-                                ignore_metric_reference=False):
+                                ignore_metric_reference=False, threshold_callback=None):
         if not self._metrics_results:
             self.compute_metrics(False, ignore_results_formatting, ignore_metric_reference)
 
         result_presenters = self.metric_executor.get_metric_presenters()
         extracted_results, extracted_meta = [], []
         for presenter, metric_result in zip(result_presenters, self._metrics_results):
+            if threshold_callback:
+                metric_result = metric_result._replace(rel_threshold=threshold_callback(metric_result))
             result, metadata = presenter.extract_result(metric_result)
             if isinstance(result, list):
                 extracted_results.extend(result)

tools/accuracy_checker/accuracy_checker/evaluators/module_evaluator.py

@@ -51,9 +51,9 @@ def print_metrics_results(self, ignore_results_formatting=False, ignore_metric_r
         self._internal_module.print_metrics(ignore_results_formatting, ignore_metric_reference)
 
     def extract_metrics_results(self, print_results=True, ignore_results_formatting=False,
-                                ignore_metric_reference=False):
+                                ignore_metric_reference=False, threshold_callback=None):
         return self._internal_module.extract_metrics_results(print_results, ignore_results_formatting,
-                                                             ignore_metric_reference)
+                                                             ignore_metric_reference, threshold_callback)
 
     def release(self):
         self._internal_module.release()

tools/accuracy_checker/accuracy_checker/evaluators/quantization_model_evaluator.py

@@ -464,13 +464,15 @@ def print_metrics_results(self, ignore_results_formatting=False, ignore_metric_r
             presenter.write_result(metric_result, ignore_results_formatting, ignore_metric_reference)
 
     def extract_metrics_results(self, print_results=True, ignore_results_formatting=False,
-                                ignore_metric_reference=False):
+                                ignore_metric_reference=False, threshold_callback=None):
         if not self._metrics_results:
             self.compute_metrics(False, ignore_results_formatting, ignore_metric_reference)
 
         result_presenters = self.metric_executor.get_metric_presenters()
         extracted_results, extracted_meta = [], []
         for presenter, metric_result in zip(result_presenters, self._metrics_results):
+            if threshold_callback:
+                metric_result = metric_result._replace(rel_threshold=threshold_callback(metric_result))
             result, metadata = presenter.extract_result(metric_result)
             if isinstance(result, list):
                 extracted_results.extend(result)