Allow dumping models first inference input/output data to pickle files (#4027)

* Add possibility to dump model first inference input ouput data
* Present model status based on abs_threshold
* Make pickle inder data independent from accuracy_checker

Author: pwolnows

tools/accuracy_checker/accuracy_checker/config/config_validator.py

@@ -84,7 +84,8 @@ class ConfigValidator(BaseValidator):
     WARN_ON_EXTRA_ARGUMENT = _ExtraArgumentBehaviour.WARN
     ERROR_ON_EXTRA_ARGUMENT = _ExtraArgumentBehaviour.ERROR
     IGNORE_ON_EXTRA_ARGUMENT = _ExtraArgumentBehaviour.IGNORE
-    acceptable_unknown_options = ['connector', '_command_line_mapping', 'model']
+    acceptable_unknown_options = ['connector', '_command_line_mapping', 'model',
+                                  '_dump_first_infer_data', '_list_processed_image_infos']
 
     def __init__(self, config_uri, on_extra_argument=WARN_ON_EXTRA_ARGUMENT, fields=None, **kwargs):
         super().__init__(**kwargs)

tools/accuracy_checker/accuracy_checker/dataset.py

@@ -373,6 +373,10 @@ def validation_scheme(cls):
         })
         return [scheme]
 
+    def store_first_annotation(self, dump_infer_data_path):
+        if self.data_provider:
+            self.data_provider.store_first_annotation(dump_infer_data_path)
+
     @property
     def metadata(self):
         return self.data_provider.metadata
@@ -595,6 +599,18 @@ def ibl_subset(pairs_set, subsample_set, ids):
 
         return list(subsample_set)
 
+    def store_first_annotation(self, annotation, identifier, dump_infer_data_path):
+        plain_annotation = {
+            'class_name': annotation.__class__.__name__,
+            **{
+                key: value.tolist() if isinstance(value, np.ndarray) else value
+                for key, value in annotation.__dict__.items()
+            },
+        }
+        print_info(f"Storing first annotation to {dump_infer_data_path}")
+        with open(dump_infer_data_path, mode='wb') as content:
+            pickle.dump({'annotation': plain_annotation, 'identifier': identifier}, content)
+
     @property
     def metadata(self):
         return deepcopy(self._meta)  # read-only
@@ -834,6 +850,16 @@ def send_annotation_info(self, config):
 
         return info
 
+    def store_first_annotation(self, dump_infer_data_path):
+        if not self.annotation_provider or self.size == 0:
+            return
+        _, batch_annotations, _, batch_identifiers = self[0]
+        if not batch_annotations:
+            raise ConfigError('first dataset item does not contain annotations')
+        self.annotation_provider.store_first_annotation(
+            batch_annotations[0], batch_identifiers[0], dump_infer_data_path
+        )
+
 
 class DatasetWrapper(DataProvider):
     pass

tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/base_custom_evaluator.py

@@ -91,6 +91,10 @@ def process_dataset(self, subset=None, num_images=None, check_progress=False, da
                 check_progress, self.dataset.size
             )
 
+        dump_infer_data = kwargs.get('_dump_first_infer_data', None)
+        if dump_infer_data:
+            self.dataset.store_first_annotation(dump_infer_data)
+
         self._process(output_callback, calculate_metrics, _progress_reporter, metric_config, kwargs.get('csv_result'))
 
         if _progress_reporter:
@@ -178,6 +182,7 @@ def extract_metrics_results(self, print_results=True, ignore_results_formatting=
             if threshold_callback:
                 abs_threshold, rel_threshold = threshold_callback(metric_result)
                 metric_result = metric_result._replace(abs_threshold=abs_threshold, rel_threshold=rel_threshold)
+
             result, metadata = presenter.extract_result(metric_result)
             if isinstance(result, list):
                 extracted_results.extend(result)

tools/accuracy_checker/accuracy_checker/evaluators/custom_evaluators/base_models.py

@@ -16,7 +16,7 @@
 from pathlib import Path
 from collections import OrderedDict
 import numpy as np
-
+import pickle  # nosec B403  # disable import-pickle check
 from ...config import ConfigError
 from ...utils import get_path, parse_partial_shape, contains_any
 from ...logging import print_info
@@ -137,6 +137,7 @@ def __init__(self, network_info, launcher, suffix=None, delayed_model_loading=Fa
             self.input_blob = None
         self.with_prefix = False
         self.is_dynamic = False
+        self._dump_first_infer_data = network_info.get('_dump_first_infer_data', None)
         if not delayed_model_loading:
             self.load_model(network_info, launcher, log=True)
 
@@ -406,6 +407,12 @@ def infer(self, input_data, raw_results=False):
         feed_dict = {tensors_mapping[name]: data for name, data in input_data.items()}
         outputs = self.infer_request.infer(feed_dict)
         res_outputs = {out_node.get_node().friendly_name: out_res for out_node, out_res in outputs.items()}
+        if self._dump_first_infer_data:
+            with open(self._dump_first_infer_data, 'wb') as file:
+                print_info(f'Storing first inference data to {self._dump_first_infer_data}')
+                dump_inf_data = {'input': feed_dict, 'output': res_outputs}
+                pickle.dump(dump_inf_data, file)
+                self._dump_first_infer_data = None
         if raw_results:
             return res_outputs, outputs
         return res_outputs

tools/accuracy_checker/accuracy_checker/evaluators/model_evaluator.py

@@ -410,6 +410,10 @@ def process_dataset_sync(self, stored_predictions, progress_reporter, *args, **k
         (enable_profiling, compute_intermediate_metric_res, metric_interval, ignore_results_formatting,
          ignore_metric_reference) = metric_config
         self._resolve_undefined_shapes()
+        dump_first_infer_data = kwargs.get('_dump_first_infer_data', None)
+        if dump_first_infer_data:
+            self.dataset.store_first_annotation(dump_first_infer_data)
+
         for batch_id, (batch_input_ids, batch_annotation, batch_input, batch_identifiers) in enumerate(self.dataset):
             filled_inputs, batch_meta, _ = self._get_batch_input(batch_annotation, batch_input)
             batch_predictions = self.launcher.predict(filled_inputs, batch_meta, **kwargs)

tools/accuracy_checker/accuracy_checker/launcher/openvino_launcher.py

@@ -21,6 +21,7 @@
 from pathlib import Path
 import re
 import numpy as np
+import pickle  # nosec B403  # disable import-pickle check
 from openvino import Core, AsyncInferQueue, get_version, PartialShape, Type, Dimension
 from openvino.preprocess import PrePostProcessor
 from .dlsdk_launcher_config import (
@@ -115,6 +116,13 @@ def __init__(self, config_entry, model_name='', delayed_model_loading=False,
         if '_list_lstm_inputs' in self.config:
             self._configure_lstm_inputs()
         self.reset_memory_state = self.get_value_from_config('reset_memory_state')
+        self._dump_first_infer_data = self.config.get('_dump_first_infer_data', None)
+        if self._dump_first_infer_data:
+            self.async_mode = False
+            warning(
+                "Async mode has been disabled because inference data dumping is enabled "
+                "(_dump_first_infer_data is set). This may affect performance results."
+            )
 
     @classmethod
     def validate_config(cls, config, delayed_model_loading=False, fetch_only=False, uri_prefix=''):
@@ -181,7 +189,10 @@ def predict(self, inputs, metadata=None, return_raw=False, **kwargs):
             feed_dict = {self.input_to_tensor_name[layer_name]: data for layer_name, data in infer_inputs.items()}
             outputs = self.infer_request.infer(inputs=feed_dict)
             raw_results.append(outputs)
-            results.append({out_node.get_node().friendly_name: out_res for out_node, out_res in outputs.items()})
+            friendly_outputs = {out_node.get_node().friendly_name: out_res for out_node, out_res in outputs.items()}
+            results.append(friendly_outputs)
+            self._dump_first_inference(feed_dict, friendly_outputs)
+
         if self.reset_memory_state:
             for state in self.infer_request.query_state():
                 state.reset()
@@ -205,6 +216,7 @@ def _predict_sequential(self, inputs, metadata=None, return_raw=False, **kwargs)
             out_tensors = self.infer_request.infer(infer_inputs)
             output_result = {
                 out_node.get_node().friendly_name: out_tensor for out_node, out_tensor in out_tensors.items()}
+            self._dump_first_inference(infer_inputs, output_result)
             lstm_inputs_feed = self._fill_lstm_inputs(output_result)
             results.append(output_result)
             if return_raw:
@@ -1030,3 +1042,13 @@ def release(self):
             del self.exec_network
         if 'ie_core' in self.__dict__:
             del self.ie_core
+
+    def _dump_first_inference(self, input_data, output_data):
+        if not self._dump_first_infer_data:
+            return
+        print_info(f'Storing first inference data to {self._dump_first_infer_data}')
+        dump_inf_data = {'input': input_data, 'output': output_data}
+        with open(self._dump_first_infer_data, 'wb') as file:
+            pickle.dump(dump_inf_data, file)
+        self._dump_first_infer_data = None
+

tools/accuracy_checker/accuracy_checker/presenters.py

@@ -207,18 +207,20 @@ def write_scalar_result(
     message = '{}: {}{}'.format(display_name, display_result, postfix)
 
     if diff_with_ref and (diff_with_ref[0] or diff_with_ref[1]):
-        abs_error = diff_with_ref[0] * scale
-        rel_error = diff_with_ref[1]
-        error_text = "abs error = {:.4} | relative error = {:.4}".format(abs_error, rel_error)
-
-        if not abs_threshold or not rel_threshold:
-            result_message = "[RESULT:  {}]".format(error_text)
+        if abs_threshold is None:
+            result_message = "[abs error = {:.4} | relative error = {:.4}]".format(
+                diff_with_ref[0] * scale, diff_with_ref[1]
+            )
             message = "{} {}".format(message, result_message)
-        elif abs_threshold <= diff_with_ref[0] or rel_threshold <= diff_with_ref[1]:
-            fail_message = "[FAILED:  {}]".format(error_text)
+        elif abs_threshold <= diff_with_ref[0] or (rel_threshold and rel_threshold <= diff_with_ref[1]):
+            fail_message = "FAILED: [abs error = {:.4} | relative error = {:.4}]".format(
+                diff_with_ref[0] * scale, diff_with_ref[1]
+            )
             message = "{} {}".format(message, color_format(fail_message, Color.FAILED))
         else:
-            pass_message = "[PASS:  {}]".format(error_text)
+            pass_message = "PASSED: [abs error = {:.4} | relative error = {:.4}]".format(
+                diff_with_ref[0] * scale, diff_with_ref[1]
+            )
             message = "{} {}".format(message, color_format(pass_message, Color.PASSED))
 
     print_info(message)