From 3a8765348378e490671903e3f76f9fd51c1d52d2 Mon Sep 17 00:00:00 2001 From: Adam Osewski Date: Wed, 29 Jul 2020 09:34:35 +0200 Subject: [PATCH] ONNX Model runner (#1415) --- ngraph/python/tests/__init__.py | 6 + ngraph/python/tests/conftest.py | 7 ++ .../tests/test_onnx/test_additional_models.py | 55 ++++++++ .../python/tests/test_onnx/utils/model_importer.py | 140 +++++++++++++++++++++ .../python/tests/test_onnx/utils/onnx_backend.py | 9 +- 5 files changed, 214 insertions(+), 3 deletions(-) create mode 100644 ngraph/python/tests/test_onnx/test_additional_models.py create mode 100644 ngraph/python/tests/test_onnx/utils/model_importer.py diff --git a/ngraph/python/tests/__init__.py b/ngraph/python/tests/__init__.py index e5d2471..e20b483 100644 --- a/ngraph/python/tests/__init__.py +++ b/ngraph/python/tests/__init__.py @@ -18,3 +18,9 @@ # nGraph backend tests will use. It's set during pytest configuration time. # See `pytest_configure` hook in `conftest.py` for more details. BACKEND_NAME = None + +# test.ADDITIONAL_MODELS_DIR is a configuration variable providing the path +# with additional ONNX models to load and test import. It's set during pytest +# configuration time. See `pytest_configure` hook in `conftest.py` for more +# details. +ADDITIONAL_MODELS_DIR = None diff --git a/ngraph/python/tests/conftest.py b/ngraph/python/tests/conftest.py index eab2ca0..905b458 100644 --- a/ngraph/python/tests/conftest.py +++ b/ngraph/python/tests/conftest.py @@ -25,11 +25,17 @@ def pytest_addoption(parser): choices=["CPU", "GPU", "FPGA", "HDDL", "MYRIAD", "HETERO"], help="Select target device", ) + parser.addoption( + "--additional_models", + default="", + type=str, + ) def pytest_configure(config): backend_name = config.getvalue("backend") tests.BACKEND_NAME = backend_name + tests.ADDITIONAL_MODELS_DIR = config.getvalue("additional_models") # register additional markers config.addinivalue_line("markers", "skip_on_cpu: Skip test on CPU") @@ -43,6 +49,7 @@ def pytest_configure(config): def pytest_collection_modifyitems(config, items): backend_name = config.getvalue("backend") + tests.ADDITIONAL_MODELS_DIR = config.getvalue("additional_models") keywords = { "CPU": "skip_on_cpu", diff --git a/ngraph/python/tests/test_onnx/test_additional_models.py b/ngraph/python/tests/test_onnx/test_additional_models.py new file mode 100644 index 0000000..bcc342e --- /dev/null +++ b/ngraph/python/tests/test_onnx/test_additional_models.py @@ -0,0 +1,55 @@ +# ****************************************************************************** +# Copyright 2018-2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ****************************************************************************** + +import tests +from operator import itemgetter +from pathlib import Path +import os + +from tests.test_onnx.utils import OpenVinoOnnxBackend +from tests.test_onnx.utils.model_importer import ModelImportRunner + + +def _get_default_additional_models_dir(): + onnx_home = os.path.expanduser(os.getenv("ONNX_HOME", os.path.join("~", ".onnx"))) + return os.path.join(onnx_home, "additonal_models") + + +MODELS_ROOT_DIR = tests.ADDITIONAL_MODELS_DIR +if len(MODELS_ROOT_DIR) == 0: + MODELS_ROOT_DIR = _get_default_additional_models_dir() + +zoo_models = [] +for path in Path(MODELS_ROOT_DIR).rglob("*.onnx"): + mdir, file = os.path.split(path) + if not file.startswith("."): + zoo_models.append({"model_name": path, "model_file": file, "dir": str(mdir)}) + +if len(zoo_models) > 0: + sorted(zoo_models, key=itemgetter("model_name")) + + # Set backend device name to be used instead of hardcoded by ONNX BackendTest class ones. + OpenVinoOnnxBackend.backend_name = tests.BACKEND_NAME + + # import all test cases at global scope to make them visible to pytest + backend_test = ModelImportRunner(OpenVinoOnnxBackend, zoo_models, __name__) + test_cases = backend_test.test_cases["OnnxBackendValidationModelImportTest"] + del test_cases + + test_cases = backend_test.test_cases["OnnxBackendValidationModelExecutionTest"] + del test_cases + + globals().update(backend_test.enable_report().test_cases) diff --git a/ngraph/python/tests/test_onnx/utils/model_importer.py b/ngraph/python/tests/test_onnx/utils/model_importer.py new file mode 100644 index 0000000..b7c5dd8 --- /dev/null +++ b/ngraph/python/tests/test_onnx/utils/model_importer.py @@ -0,0 +1,140 @@ +# ****************************************************************************** +# Copyright 2018-2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ****************************************************************************** + +import glob +import numpy as np +import onnx +import onnx.backend.test +import os +import unittest + +from collections import defaultdict +from onnx import numpy_helper, NodeProto, ModelProto +from onnx.backend.base import Backend, BackendRep +from onnx.backend.test.case.test_case import TestCase as OnnxTestCase +from onnx.backend.test.runner import TestItem +from typing import Any, Dict, List, Optional, Pattern, Set, Text, Type, Union + + +class ModelImportRunner(onnx.backend.test.BackendTest): + def __init__( + self, + backend: Type[Backend], + models: List[Dict[str, str]], + parent_module: Optional[str] = None, + ) -> None: + self.backend = backend + self._parent_module = parent_module + self._include_patterns: Set[Pattern[Text]] = set() + self._exclude_patterns: Set[Pattern[Text]] = set() + self._test_items: Dict[Text, Dict[Text, TestItem]] = defaultdict(dict) + + for model in models: + test_name = "test_{}".format(model["model_name"]) + + test_case = OnnxTestCase( + name=test_name, + url=None, + model_name=model["model_name"], + model_dir=model["dir"], + model=model["model_file"], + data_sets=None, + kind="OnnxBackendRealModelTest", + rtol=model.get("rtol", 0.001), + atol=model.get("atol", 1e-07), + ) + self._add_model_import_test(test_case, "Validation") + self._add_model_execution_test(test_case, "Validation") + + @staticmethod + def _load_onnx_model(model_dir: str, filename: str) -> ModelProto: + if model_dir is None: + raise unittest.SkipTest("Model directory not provided") + + model_pb_path = os.path.join(model_dir, filename) + return onnx.load(model_pb_path) + + def _add_model_import_test(self, model_test: OnnxTestCase, kind: Text) -> None: + # model is loaded at runtime, note sometimes it could even + # never loaded if the test skipped + model_marker: List[Optional[Union[ModelProto, NodeProto]]] = [None] + + def run_import(test_self: Any, device: Text) -> None: + model = ModelImportRunner._load_onnx_model(model_test.model_dir, model_test.model) + model_marker[0] = model + if not hasattr(self.backend, "is_compatible") and not callable( + self.backend.is_compatible + ): + raise unittest.SkipTest("Provided backend does not provide is_compatible method") + self.backend.is_compatible(model) + + self._add_test(kind + "ModelImport", model_test.name, run_import, model_marker) + + @classmethod + def _execute_npz_data( + cls, model_dir: str, prepared_model: BackendRep, result_rtol: float, result_atol: float, + ) -> None: + for test_data_npz in glob.glob(os.path.join(model_dir, "test_data_*.npz")): + test_data = np.load(test_data_npz, encoding="bytes") + inputs = list(test_data["inputs"]) + outputs = list(prepared_model.run(inputs)) + ref_outputs = test_data["outputs"] + cls.assert_similar_outputs(ref_outputs, outputs, result_rtol, result_atol) + + @classmethod + def _execute_pb_data( + cls, model_dir: str, prepared_model: BackendRep, result_rtol: float, result_atol: float, + ) -> None: + for test_data_dir in glob.glob(os.path.join(model_dir, "test_data_set*")): + inputs = [] + inputs_num = len(glob.glob(os.path.join(test_data_dir, "input_*.pb"))) + for i in range(inputs_num): + input_file = os.path.join(test_data_dir, "input_{}.pb".format(i)) + tensor = onnx.TensorProto() + with open(input_file, "rb") as f: + tensor.ParseFromString(f.read()) + inputs.append(numpy_helper.to_array(tensor)) + ref_outputs = [] + ref_outputs_num = len(glob.glob(os.path.join(test_data_dir, "output_*.pb"))) + for i in range(ref_outputs_num): + output_file = os.path.join(test_data_dir, "output_{}.pb".format(i)) + tensor = onnx.TensorProto() + with open(output_file, "rb") as f: + tensor.ParseFromString(f.read()) + ref_outputs.append(numpy_helper.to_array(tensor)) + outputs = list(prepared_model.run(inputs)) + cls.assert_similar_outputs(ref_outputs, outputs, result_rtol, result_atol) + + def _add_model_execution_test(self, model_test: OnnxTestCase, kind: Text) -> None: + # model is loaded at runtime, note sometimes it could even + # never loaded if the test skipped + model_marker: List[Optional[Union[ModelProto, NodeProto]]] = [None] + + def run_execution(test_self: Any, device: Text) -> None: + model = ModelImportRunner._load_onnx_model(model_test.model_dir, model_test.model) + model_marker[0] = model + prepared_model = self.backend.prepare(model, device) + assert prepared_model is not None + + ModelImportRunner._execute_npz_data( + model_test.model_dir, prepared_model, model_test.rtol, model_test.atol + ) + + ModelImportRunner._execute_pb_data( + model_test.model_dir, prepared_model, model_test.rtol, model_test.atol + ) + + self._add_test(kind + "ModelExecution", model_test.name, run_execution, model_marker) diff --git a/ngraph/python/tests/test_onnx/utils/onnx_backend.py b/ngraph/python/tests/test_onnx/utils/onnx_backend.py index 7513fc4..4414d30 100644 --- a/ngraph/python/tests/test_onnx/utils/onnx_backend.py +++ b/ngraph/python/tests/test_onnx/utils/onnx_backend.py @@ -67,7 +67,6 @@ class OpenVinoOnnxBackend(Backend): device="CPU", # type: Text **kwargs # type: Any ): # type: (...) -> OpenVinoOnnxBackendRep - onnx.checker.check_model(onnx_model) super().prepare(onnx_model, device, **kwargs) ng_model_function = import_onnx_model(onnx_model) return OpenVinoOnnxBackendRep(ng_model_function, cls.backend_name) @@ -98,7 +97,9 @@ class OpenVinoOnnxBackend(Backend): output_tensor_shapes = [()] # type: List[Tuple[int, ...]] if outputs_info is not None: - output_tensor_types = [np_dtype_to_tensor_type(dtype) for (dtype, shape) in outputs_info] + output_tensor_types = [ + np_dtype_to_tensor_type(dtype) for (dtype, shape) in outputs_info + ] output_tensor_shapes = [shape for (dtype, shape) in outputs_info] input_tensors = [ @@ -107,7 +108,9 @@ class OpenVinoOnnxBackend(Backend): ] output_tensors = [ make_tensor_value_info(name, tensor_type, shape) - for name, shape, tensor_type in zip(node.output, output_tensor_shapes, output_tensor_types) + for name, shape, tensor_type in zip( + node.output, output_tensor_shapes, output_tensor_types + ) ] graph = make_graph([node], "compute_graph", input_tensors, output_tensors) -- 2.7.4