From d14d09e796023b603271121597137d190f87d758 Mon Sep 17 00:00:00 2001 From: Rafal Blaczkowski Date: Tue, 4 Aug 2020 12:26:32 +0200 Subject: [PATCH] Update ONNX Python tests (#1514) --- .../python/tests/test_onnx/test_ops_batchnorm.py | 3 +- ngraph/python/tests/test_onnx/test_ops_binary.py | 10 +- ngraph/python/tests/test_onnx/test_ops_convpool.py | 15 +- ngraph/python/tests/test_onnx/test_ops_logical.py | 15 +- ngraph/python/tests/test_onnx/test_ops_matmul.py | 116 +++++----- .../python/tests/test_onnx/test_ops_nonlinear.py | 27 ++- .../python/tests/test_onnx/test_ops_reduction.py | 234 ++++----------------- ngraph/python/tests/test_onnx/test_ops_reshape.py | 23 +- ngraph/python/tests/test_onnx/test_ops_unary.py | 36 +++- ngraph/python/tests/test_onnx/test_ops_variadic.py | 4 +- ngraph/python/tests/test_onnx/utils/__init__.py | 37 +++- ngraph/python/tox.ini | 7 +- 12 files changed, 223 insertions(+), 304 deletions(-) diff --git a/ngraph/python/tests/test_onnx/test_ops_batchnorm.py b/ngraph/python/tests/test_onnx/test_ops_batchnorm.py index f941e8a..9a87c42 100644 --- a/ngraph/python/tests/test_onnx/test_ops_batchnorm.py +++ b/ngraph/python/tests/test_onnx/test_ops_batchnorm.py @@ -17,7 +17,7 @@ import numpy as np import onnx -from tests.test_onnx.utils import run_node +from tests.test_onnx.utils import run_node, xfail_issue_35893 def make_batch_norm_node(**node_attributes): @@ -26,6 +26,7 @@ def make_batch_norm_node(**node_attributes): ) +@xfail_issue_35893 def test_batch_norm_test_node(): data = np.arange(48).reshape((1, 3, 4, 4)).astype(np.float32) scale = np.ones((3,)).astype(np.float32) # Gamma diff --git a/ngraph/python/tests/test_onnx/test_ops_binary.py b/ngraph/python/tests/test_onnx/test_ops_binary.py index 0b5464d..d8d1a16 100644 --- a/ngraph/python/tests/test_onnx/test_ops_binary.py +++ b/ngraph/python/tests/test_onnx/test_ops_binary.py @@ -18,7 +18,7 @@ import onnx import pytest from onnx.helper import make_graph, make_model, make_tensor_value_info -from tests.test_onnx.utils import run_model +from tests.test_onnx.utils import run_model, skip_segfault def import_and_compute(op_type, input_data_left, input_data_right, opset=7, **node_attributes): @@ -37,7 +37,7 @@ def import_and_compute(op_type, input_data_left, input_data_right, opset=7, **no return run_model(model, inputs)[0] -@pytest.mark.skip(reason="Causes segmentation fault") +@skip_segfault def test_add_opset4(): assert np.array_equal(import_and_compute("Add", 1, 2, opset=4), np.array(3, dtype=np.float32)) @@ -110,7 +110,7 @@ def test_add_opset7(left_shape, right_shape): assert np.array_equal(import_and_compute("Add", left_input, right_input), left_input + right_input) -@pytest.mark.skip(reason="Causes segmentation fault") +@skip_segfault def test_sub(): assert np.array_equal(import_and_compute("Sub", 20, 1), np.array(19, dtype=np.float32)) @@ -124,7 +124,7 @@ def test_sub(): ) -@pytest.mark.skip(reason="Causes segmentation fault") +@skip_segfault def test_mul(): assert np.array_equal(import_and_compute("Mul", 2, 3), np.array(6, dtype=np.float32)) @@ -138,7 +138,7 @@ def test_mul(): ) -@pytest.mark.skip(reason="Causes segmentation fault") +@skip_segfault def test_div(): assert np.array_equal(import_and_compute("Div", 6, 3), np.array(2, dtype=np.float32)) diff --git a/ngraph/python/tests/test_onnx/test_ops_convpool.py b/ngraph/python/tests/test_onnx/test_ops_convpool.py index bb55ccc..d964590 100644 --- a/ngraph/python/tests/test_onnx/test_ops_convpool.py +++ b/ngraph/python/tests/test_onnx/test_ops_convpool.py @@ -19,7 +19,13 @@ import pytest from onnx.helper import make_graph, make_model, make_node, make_tensor_value_info from tests.runtime import get_runtime -from tests.test_onnx.utils import get_node_model, import_onnx_model, run_model, run_node +from tests.test_onnx.utils import (get_node_model, + import_onnx_model, + run_model, + run_node, + xfail_issue_35911, + xfail_issue_35912 + ) @pytest.fixture @@ -268,6 +274,7 @@ def test_2d_conv_transpose(): ) +@xfail_issue_35911 def test_pad_opset_1(): x = np.ones((2, 2), dtype=np.float32) y = np.pad(x, pad_width=1, mode="constant") @@ -317,8 +324,7 @@ def test_pad_opset_2(): run_model(model, [x]) -# Error of validate layer: B with type: Pad. Cannot parse parameter pads_begin -# from IR for layer B. Value -1,0 cannot be casted to int. +@xfail_issue_35912 def test_pad_negative_values_begin(): x = np.ones((2, 2), dtype=np.float32) @@ -333,8 +339,7 @@ def test_pad_negative_values_begin(): assert np.array_equal(ng_result, np.array([[1], [1]])) -# Error of validate layer: B with type: Pad. Cannot parse parameter pads_begin -# from IR for layer B. Value -1,0 cannot be casted to int. +@xfail_issue_35912 def test_pad_negative_values_end(): x = np.ones((2, 2), dtype=np.float32) diff --git a/ngraph/python/tests/test_onnx/test_ops_logical.py b/ngraph/python/tests/test_onnx/test_ops_logical.py index f1963ba..f01af58 100644 --- a/ngraph/python/tests/test_onnx/test_ops_logical.py +++ b/ngraph/python/tests/test_onnx/test_ops_logical.py @@ -17,19 +17,18 @@ import numpy as np import onnx import pytest -# [PARAMETER_MISMATCH] Failed to set Blob with precision FP32 -from tests.test_onnx.utils import run_node +from tests.test_onnx.utils import run_node, xfail_issue_35914, xfail_issue_35915 @pytest.mark.parametrize( "onnx_op, numpy_func, data_type", [ - ("And", np.logical_and, np.bool), - ("Or", np.logical_or, np.bool), - ("Xor", np.logical_xor, np.bool), - ("Equal", np.equal, np.int32), - ("Greater", np.greater, np.int32), - ("Less", np.less, np.int32), + pytest.param("And", np.logical_and, np.bool, marks=xfail_issue_35914), + pytest.param("Or", np.logical_or, np.bool, marks=xfail_issue_35914), + pytest.param("Xor", np.logical_xor, np.bool, marks=xfail_issue_35914), + pytest.param("Equal", np.equal, np.int32, marks=xfail_issue_35915), + pytest.param("Greater", np.greater, np.int32, marks=xfail_issue_35915), + pytest.param("Less", np.less, np.int32, marks=xfail_issue_35915), ], ) def test_logical(onnx_op, numpy_func, data_type): diff --git a/ngraph/python/tests/test_onnx/test_ops_matmul.py b/ngraph/python/tests/test_onnx/test_ops_matmul.py index 34061ae..76da7ed 100644 --- a/ngraph/python/tests/test_onnx/test_ops_matmul.py +++ b/ngraph/python/tests/test_onnx/test_ops_matmul.py @@ -18,7 +18,14 @@ import onnx from onnx.helper import make_graph, make_model, make_node, make_tensor_value_info from tests.runtime import get_runtime -from tests.test_onnx.utils import import_onnx_model +from tests.test_onnx.utils import (import_onnx_model, + xfail_issue_35916, + xfail_issue_35917, + xfail_issue_35918, + xfail_issue_35921 + ) + +import pytest def make_onnx_model_for_matmul_op(input_left, input_right): @@ -99,33 +106,20 @@ def import_and_compute_gemm(input_a, input_b, input_c, **kwargs): return computation(input_a, input_b, input_c)[0] -def test_op_matmul(): - # vector @ vector - data = ([1, 2], [1, 3]) - assert np.array_equal(import_and_compute_matmul(*data), np.matmul(*data)) - - data = ([1, 2, 3], [[4], [5], [6]]) - assert np.array_equal(import_and_compute_matmul(*data), np.matmul(*data)) - - data = ([[1, 2, 3]], [1, 2, 3]) - assert np.array_equal(import_and_compute_matmul(*data), np.matmul(*data)) - - # vector @ matrix - data = ([1, 2, 3], [[4, 5], [6, 7], [8, 9]]) - assert np.array_equal(import_and_compute_matmul(*data), np.matmul(*data)) - - # matrix @ vector - data = ([[1, 2, 3], [4, 5, 6]], [[7], [8], [9]]) - assert np.array_equal(import_and_compute_matmul(*data), np.matmul(*data)) - - # matrix @ matrix - data = ([[1, 2], [3, 4]], [[5, 6], [7, 8]]) - assert np.array_equal(import_and_compute_matmul(*data), np.matmul(*data)) - - data = ([[1, 2, 3], [4, 5, 6]], [[7, 8], [9, 10], [11, 12]]) - assert np.array_equal(import_and_compute_matmul(*data), np.matmul(*data)) - - data = ([[1, 2], [3, 4], [5, 6]], [[7, 8, 9], [10, 11, 12]]) +@pytest.mark.parametrize( + "data, description", + [ + pytest.param(([1, 2], [1, 3]), "vector and vector 1", marks=xfail_issue_35916), + (([1, 2, 3], [[4], [5], [6]]), "vector and vector 2"), + (([[1, 2, 3]], [1, 2, 3]), "vector and vector 3"), + (([1, 2, 3], [[4, 5], [6, 7], [8, 9]]), "vector and matrix"), + (([[1, 2, 3], [4, 5, 6]], [[7], [8], [9]]), "matrix and vector"), + (([[1, 2], [3, 4]], [[5, 6], [7, 8]]), "matrix and matrix 1"), + (([[1, 2, 3], [4, 5, 6]], [[7, 8], [9, 10], [11, 12]]), "matrix and matrix 2"), + (([[1, 2], [3, 4], [5, 6]], [[7, 8, 9], [10, 11, 12]]), "matrix and matrix 3") + ], +) +def test_op_matmul(data, description): assert np.array_equal(import_and_compute_matmul(*data), np.matmul(*data)) @@ -138,43 +132,41 @@ def test_op_matmul_3d(): assert np.array_equal(import_and_compute_matmul(*data), np.matmul(*data)) -def test_gemm(): - data = ([1, 2], [1, 3], [1, 4]) - assert np.array_equal(import_and_compute_gemm(*data), numpy_gemm(*data)) - - data = ([1, 2], [1, 3], 1) - assert np.array_equal(import_and_compute_gemm(*data), numpy_gemm(*data)) - - data = ([1, 2], [1, 3], [1]) - assert np.array_equal(import_and_compute_gemm(*data), numpy_gemm(*data)) - - data = ([1, 2], [1, 3], [1, 4]) - kwargs = {"alpha": 7, "beta": 9} - assert np.array_equal(import_and_compute_gemm(*data, **kwargs), numpy_gemm(*data, **kwargs)) - - data = ([1, 2, 3, 4], [1, 3, 5, 7], [1, 4]) - kwargs = {"alpha": 7, "beta": 9} - assert np.array_equal(import_and_compute_gemm(*data, **kwargs), numpy_gemm(*data, **kwargs)) - - -def test_gemm_transpositions(): - data = ([1, 2], [1, 3], [1, 4]) - kwargs = {"trans_a": True, "trans_b": True} - assert np.array_equal(import_and_compute_gemm(*data, **kwargs), numpy_gemm(*data, **kwargs)) - - data = ([[1, 2], [1, 2]], [[1, 3], [1, 3]], [4, 1]) - kwargs = {"trans_a": True, "trans_b": True, "alpha": 7, "beta": 9} - assert np.array_equal(import_and_compute_gemm(*data, **kwargs), numpy_gemm(*data, **kwargs)) - - data = ([[1, 2]], [[1, 3]], 1) - kwargs = {"trans_b": True, "alpha": 7, "beta": 9} - assert np.array_equal(import_and_compute_gemm(*data, **kwargs), numpy_gemm(*data, **kwargs)) - - data = ([[1], [2]], [[1], [3]], 1) - kwargs = {"trans_a": True, "alpha": 7, "beta": 9} +@pytest.mark.parametrize( + "data, kwargs, description", + [ + pytest.param(([1, 2], [1, 3], [1, 4]), {}, "vectors", marks=xfail_issue_35917), + pytest.param(([1, 2], [1, 3], 1), {}, "vectors and scalar", marks=xfail_issue_35917), + pytest.param(([1, 2], [1, 3], [1]), {}, "vectors and identity vector", marks=xfail_issue_35917), + pytest.param(([1, 2], [1, 3], [1, 4]), {"alpha": 7, "beta": 9}, + "vectors with alpha and beta", marks=xfail_issue_35918), + pytest.param(([1, 2, 3, 4], [1, 3, 5, 7], [1, 4]), {"alpha": 7, "beta": 9}, + "longer vectors with alpha and beta", marks=xfail_issue_35918) + ], +) +def test_gemm(data, kwargs, description): + assert np.array_equal(import_and_compute_gemm(*data, **kwargs), numpy_gemm(*data)) + + +@pytest.mark.parametrize( + "data, kwargs, description", + [ + pytest.param(([1, 2], [1, 3], [1, 4]), {"trans_a": True, "trans_b": True}, + "vectors with trans_a/trans_b", marks=xfail_issue_35917), + pytest.param(([[1, 2], [1, 2]], [[1, 3], [1, 3]], [4, 1]), + {"trans_a": True, "trans_b": True, "alpha": 7, "beta": 9}, + "matrices and vector with trans_b and alpha/beta", marks=xfail_issue_35918), + pytest.param(([[1, 2]], [[1, 3]], 1), {"trans_b": True, "alpha": 7, "beta": 9}, + "matrices and scalar with trans_b and alpha/beta", marks=xfail_issue_35918), + pytest.param(([[1], [2]], [[1], [3]], 1), {"trans_a": True, "alpha": 7, "beta": 9}, + "matrices and scalar with trans_a and alpha/beta", marks=xfail_issue_35918), + ], +) +def test_gemm_transpositions(data, kwargs, description): assert np.array_equal(import_and_compute_gemm(*data, **kwargs), numpy_gemm(*data, **kwargs)) +@xfail_issue_35921 def test_gemm_flatten(): # input_a.shape is (4,1,1) data = ([[[1]], [[2]], [[3]], [[4]]], [1, 3, 5, 7], [1, 4]) diff --git a/ngraph/python/tests/test_onnx/test_ops_nonlinear.py b/ngraph/python/tests/test_onnx/test_ops_nonlinear.py index 9ed74ed..c30275e 100644 --- a/ngraph/python/tests/test_onnx/test_ops_nonlinear.py +++ b/ngraph/python/tests/test_onnx/test_ops_nonlinear.py @@ -17,7 +17,7 @@ import numpy as np import onnx import pytest -from tests.test_onnx.utils import run_node +from tests.test_onnx.utils import run_node, xfail_issue_35918, xfail_issue_35923, xfail_issue_35924 def import_and_compute(op_type, input_data, **node_attrs): @@ -70,15 +70,16 @@ def test_leaky_relu(): assert_onnx_import_equals_callable("LeakyRelu", leaky_relu, [[-3, -2, -1], [1, 2, 3]]) +@xfail_issue_35923 @pytest.mark.parametrize( - "x,slope", + "x, slope", [ ([-2, -1.0, 0.0, 1.0, 2.0], 0.5), ([0.0], 1), ([-0.9, -0.8, -0.7, -0.4, -0.3, -0.2, -0.1], 1), ([[1, 2, 3], [4, 5, 6]], 0.5), ([[-3, -2, -1], [1, 2, 3]], 1), - ], + ] ) def test_parametric_relu(x, slope): def parametic_relu(x, slope): @@ -103,13 +104,19 @@ def test_selu(): assert_onnx_import_equals_callable("Selu", selu, [-2, -1.0, 0.0, 1.0, 2.0], gamma=0.5, alpha=0.5) -def test_elu(): +@pytest.mark.parametrize( + "data, alpha_value", + [ + pytest.param([-2, -1.0, 0.0, 1.0, 2.0], 1, marks=xfail_issue_35918), + pytest.param([0.0], 1, marks=xfail_issue_35918), + pytest.param([-0.9, -0.8, -0.7, -0.4, -0.3, -0.2, -0.1], 1, marks=xfail_issue_35918), + pytest.param([[1, 2, 3], [4, 5, 6]], 1, marks=xfail_issue_35918), + pytest.param([-2, -1.0, 0.0, 1.0, 2.0], 0.5, marks=xfail_issue_35924) + ] +) +def test_elu(data, alpha_value): # f(x) = alpha * (exp(x) - 1) for x < 0, f(x) = x for x >= 0 - def elu(x, alpha=1): + def elu(x, alpha): return np.where(x < 0, alpha * (np.exp(x) - 1), x) - assert_onnx_import_equals_callable("Elu", elu, [-2, -1.0, 0.0, 1.0, 2.0]) - assert_onnx_import_equals_callable("Elu", elu, [0.0]) - assert_onnx_import_equals_callable("Elu", elu, [-0.9, -0.8, -0.7, -0.4, -0.3, -0.2, -0.1]) - assert_onnx_import_equals_callable("Elu", elu, [[1, 2, 3], [4, 5, 6]]) - assert_onnx_import_equals_callable("Elu", elu, [-2, -1.0, 0.0, 1.0, 2.0], alpha=0.5) + assert_onnx_import_equals_callable("Elu", elu, data, alpha=alpha_value) diff --git a/ngraph/python/tests/test_onnx/test_ops_reduction.py b/ngraph/python/tests/test_onnx/test_ops_reduction.py index da23f38..d76635f 100644 --- a/ngraph/python/tests/test_onnx/test_ops_reduction.py +++ b/ngraph/python/tests/test_onnx/test_ops_reduction.py @@ -17,7 +17,30 @@ import numpy as np import onnx import pytest -from tests.test_onnx.utils import run_node +from tests.test_onnx.utils import (run_node, + unstrict_xfail_issue_35925, + strict_xfail_issue_35925, + xfail_issue_36437) + +reduce_data = np.array([[[5, 1], [20, 2]], [[30, 1], [40, 2]], [[55, 1], [60, 2]]], dtype=np.float32) +reduce_axis_parameters = [ + None, + (0,), + (1,), + (2,), + (0, 1), + (0, 2), + (1, 2), + (0, 1, 2) +] + +reduce_operation_parameters = [ + ("ReduceMax", np.max), + ("ReduceMin", np.min), + ("ReduceMean", np.mean), + ("ReduceSum", np.sum), + ("ReduceProd", np.prod) +] def import_and_compute(op_type, input_data, **node_attrs): @@ -26,196 +49,17 @@ def import_and_compute(op_type, input_data, **node_attrs): return run_node(node, data_inputs).pop() -def test_reduce_max(): - data = np.array([[[5, 1], [20, 2]], [[30, 1], [40, 2]], [[55, 1], [60, 2]]], dtype=np.float32) - - assert np.array_equal(import_and_compute("ReduceMax", data, keepdims=0), np.max(data, keepdims=False)) - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(0,), keepdims=0), np.max(data, keepdims=False, axis=(0,)) - ) - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(1,), keepdims=0), np.max(data, keepdims=False, axis=(1,)) - ) - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(2,), keepdims=0), np.max(data, keepdims=False, axis=(2,)) - ) - - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(0, 1), keepdims=0), - np.max(data, keepdims=False, axis=(0, 1)), - ) - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(0, 2), keepdims=0), - np.max(data, keepdims=False, axis=(0, 2)), - ) - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(1, 2), keepdims=0), - np.max(data, keepdims=False, axis=(1, 2)), - ) - - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(0, 1, 2), keepdims=0), - np.max(data, keepdims=False, axis=(0, 1, 2)), - ) - - -def test_reduce_max_keepdims(): - data = np.array([[[5, 1], [20, 2]], [[30, 1], [40, 2]], [[55, 1], [60, 2]]], dtype=np.float32) - - assert np.array_equal(import_and_compute("ReduceMax", data), np.max(data, keepdims=True)) - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(0,)), np.max(data, keepdims=True, axis=(0,)) - ) - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(1,)), np.max(data, keepdims=True, axis=(1,)) - ) - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(2,)), np.max(data, keepdims=True, axis=(2,)) - ) - - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(0, 1)), np.max(data, keepdims=True, axis=(0, 1)) - ) - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(0, 2)), np.max(data, keepdims=True, axis=(0, 2)) - ) - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(1, 2)), np.max(data, keepdims=True, axis=(1, 2)) - ) - - assert np.array_equal( - import_and_compute("ReduceMax", data, axes=(0, 1, 2)), np.max(data, keepdims=True, axis=(0, 1, 2)) - ) - - -def test_reduce_min(): - data = np.array([[[5, 1], [20, 2]], [[30, 1], [40, 2]], [[55, 1], [60, 2]]], dtype=np.float32) - - assert np.array_equal(import_and_compute("ReduceMin", data), np.min(data, keepdims=True)) - assert np.array_equal(import_and_compute("ReduceMin", data, keepdims=0), np.min(data, keepdims=False)) - - assert np.array_equal( - import_and_compute("ReduceMin", data, axes=(1,)), np.min(data, keepdims=True, axis=(1,)) - ) - assert np.array_equal( - import_and_compute("ReduceMin", data, axes=(1,), keepdims=0), np.min(data, keepdims=False, axis=(1,)) - ) - - assert np.array_equal( - import_and_compute("ReduceMin", data, axes=(0, 2)), np.min(data, keepdims=True, axis=(0, 2)) - ) - assert np.array_equal( - import_and_compute("ReduceMin", data, axes=(0, 2), keepdims=0), - np.min(data, keepdims=False, axis=(0, 2)), - ) - - assert np.array_equal( - import_and_compute("ReduceMin", data, axes=(0, 1, 2)), np.min(data, keepdims=True, axis=(0, 1, 2)) - ) - assert np.array_equal( - import_and_compute("ReduceMin", data, axes=(0, 1, 2), keepdims=0), - np.min(data, keepdims=False, axis=(0, 1, 2)), - ) - - -def test_reduce_mean(): - data = np.array([[[5, 1], [20, 2]], [[30, 1], [40, 2]], [[55, 1], [60, 2]]], dtype=np.float32) - - assert np.array_equal(import_and_compute("ReduceMean", data), np.mean(data, keepdims=True)) - assert np.array_equal(import_and_compute("ReduceMean", data, keepdims=0), np.mean(data, keepdims=False)) - - assert np.array_equal( - import_and_compute("ReduceMean", data, axes=(1,)), np.mean(data, keepdims=True, axis=(1,)) - ) - assert np.array_equal( - import_and_compute("ReduceMean", data, axes=(1,), keepdims=0), - np.mean(data, keepdims=False, axis=(1,)), - ) - - assert np.array_equal( - import_and_compute("ReduceMean", data, axes=(0, 2)), np.mean(data, keepdims=True, axis=(0, 2)) - ) - assert np.array_equal( - import_and_compute("ReduceMean", data, axes=(0, 2), keepdims=0), - np.mean(data, keepdims=False, axis=(0, 2)), - ) - - assert np.array_equal( - import_and_compute("ReduceMean", data, axes=(0, 1, 2)), np.mean(data, keepdims=True, axis=(0, 1, 2)) - ) - assert np.array_equal( - import_and_compute("ReduceMean", data, axes=(0, 1, 2), keepdims=0), - np.mean(data, keepdims=False, axis=(0, 1, 2)), - ) - - -def test_reduce_sum(): - data = np.array([[[5, 1], [20, 2]], [[30, 1], [40, 2]], [[55, 1], [60, 2]]], dtype=np.float32) - - assert np.array_equal(import_and_compute("ReduceSum", data, keepdims=0), np.sum(data, keepdims=False)) - - assert np.array_equal( - import_and_compute("ReduceSum", data, axes=(1,), keepdims=0), np.sum(data, keepdims=False, axis=(1,)) - ) - - assert np.array_equal( - import_and_compute("ReduceSum", data, axes=(0, 2), keepdims=0), - np.sum(data, keepdims=False, axis=(0, 2)), - ) - - assert np.array_equal( - import_and_compute("ReduceSum", data, axes=(0, 1, 2), keepdims=0), - np.sum(data, keepdims=False, axis=(0, 1, 2)), - ) - - -def test_reduce_sum_keepdims(): - data = np.array([[[5, 1], [20, 2]], [[30, 1], [40, 2]], [[55, 1], [60, 2]]], dtype=np.float32) - - assert np.array_equal(import_and_compute("ReduceSum", data), np.sum(data, keepdims=True)) - - assert np.array_equal( - import_and_compute("ReduceSum", data, axes=(1,)), np.sum(data, keepdims=True, axis=(1,)) - ) - - assert np.array_equal( - import_and_compute("ReduceSum", data, axes=(0, 2)), np.sum(data, keepdims=True, axis=(0, 2)) - ) - - assert np.array_equal( - import_and_compute("ReduceSum", data, axes=(0, 1, 2)), np.sum(data, keepdims=True, axis=(0, 1, 2)) - ) - - -def test_reduce_prod(): - data = np.array([[[5, 1], [20, 2]], [[30, 1], [40, 2]], [[55, 1], [60, 2]]], dtype=np.float32) - - assert np.array_equal(import_and_compute("ReduceProd", data), np.prod(data, keepdims=True)) - assert np.array_equal(import_and_compute("ReduceProd", data, keepdims=0), np.prod(data, keepdims=False)) - - assert np.array_equal( - import_and_compute("ReduceProd", data, axes=(1,)), np.prod(data, keepdims=True, axis=(1,)) - ) - assert np.array_equal( - import_and_compute("ReduceProd", data, axes=(1,), keepdims=0), - np.prod(data, keepdims=False, axis=(1,)), - ) - - assert np.array_equal( - import_and_compute("ReduceProd", data, axes=(0, 2)), np.prod(data, keepdims=True, axis=(0, 2)) - ) - assert np.array_equal( - import_and_compute("ReduceProd", data, axes=(0, 2), keepdims=0), - np.prod(data, keepdims=False, axis=(0, 2)), - ) - - assert np.array_equal( - import_and_compute("ReduceProd", data, axes=(0, 1, 2)), np.prod(data, keepdims=True, axis=(0, 1, 2)) - ) - assert np.array_equal( - import_and_compute("ReduceProd", data, axes=(0, 1, 2), keepdims=0), - np.prod(data, keepdims=False, axis=(0, 1, 2)), - ) +@unstrict_xfail_issue_35925 +@pytest.mark.parametrize("operation, ref_operation", reduce_operation_parameters) +@pytest.mark.parametrize("keepdims", [True, False]) +@pytest.mark.parametrize("axes", reduce_axis_parameters) +def test_reduce_operation(operation, ref_operation, keepdims, axes): + if axes: + assert np.array_equal(import_and_compute(operation, reduce_data, axes=axes, keepdims=keepdims), + ref_operation(reduce_data, keepdims=keepdims, axis=axes)) + else: + assert np.array_equal(import_and_compute(operation, reduce_data, keepdims=keepdims), + ref_operation(reduce_data, keepdims=keepdims)) @pytest.mark.parametrize("reduction_axes", [(0,), (0, 2), (0, 1, 2)]) @@ -237,6 +81,7 @@ def test_reduce_l1(reduction_axes): assert np.allclose(expected, ng_result) +@strict_xfail_issue_35925 def test_reduce_l1_default_axes(): shape = [2, 4, 3, 2] np.random.seed(133391) @@ -275,6 +120,7 @@ def test_reduce_l2(reduction_axes): assert np.allclose(expected, ng_result) +@strict_xfail_issue_35925 def test_reduce_l2_default_axes(): shape = [2, 4, 3, 2] np.random.seed(133391) @@ -293,6 +139,7 @@ def test_reduce_l2_default_axes(): assert np.allclose(expected, ng_result) +@unstrict_xfail_issue_35925 @pytest.mark.parametrize("reduction_axes", [(0,), (0, 2), (0, 1, 2)]) def test_reduce_log_sum(reduction_axes): shape = [2, 4, 3, 2] @@ -312,6 +159,7 @@ def test_reduce_log_sum(reduction_axes): assert np.allclose(expected, ng_result) +@strict_xfail_issue_35925 def test_reduce_log_sum_default_axes(): shape = [2, 4, 3, 2] np.random.seed(133391) @@ -330,6 +178,7 @@ def test_reduce_log_sum_default_axes(): assert np.allclose(expected, ng_result) +@strict_xfail_issue_35925 def test_reduce_log_sum_exp(): def logsumexp(data, axis=None, keepdims=True): return np.log(np.sum(np.exp(data), axis=axis, keepdims=keepdims)) @@ -388,6 +237,7 @@ def test_reduce_sum_square(reduction_axes): assert np.allclose(expected, ng_result) +@strict_xfail_issue_35925 def test_reduce_sum_square_default_axes(): shape = [2, 4, 3, 2] np.random.seed(133391) @@ -406,6 +256,7 @@ def test_reduce_sum_square_default_axes(): assert np.allclose(expected, ng_result) +@xfail_issue_36437 def test_reduce_argmin(): def argmin(ndarray, axis, keepdims=False): res = np.argmin(ndarray, axis=axis) @@ -429,6 +280,7 @@ def test_reduce_argmin(): ) +@xfail_issue_36437 def test_reduce_argmax(): def argmax(ndarray, axis, keepdims=False): res = np.argmax(ndarray, axis=axis) diff --git a/ngraph/python/tests/test_onnx/test_ops_reshape.py b/ngraph/python/tests/test_onnx/test_ops_reshape.py index c18aad7..f6f5d9a 100644 --- a/ngraph/python/tests/test_onnx/test_ops_reshape.py +++ b/ngraph/python/tests/test_onnx/test_ops_reshape.py @@ -19,15 +19,17 @@ import pytest from onnx.helper import make_graph, make_model, make_node, make_tensor_value_info from tests.runtime import get_runtime -from tests.test_onnx.utils import ( - all_arrays_equal, - get_node_model, - import_onnx_model, - run_model, - run_node, -) +from tests.test_onnx.utils import (all_arrays_equal, + get_node_model, + import_onnx_model, + run_model, + run_node, + xfail_issue_35926, + xfail_issue_35927 + ) +@xfail_issue_35926 def test_reshape(): input_data = np.arange(2560).reshape([16, 4, 4, 10]) reshape_node = onnx.helper.make_node("Reshape", inputs=["x"], outputs=["y"], shape=(256, 10)) @@ -76,6 +78,7 @@ def test_reshape_opset5(): assert np.array_equal(ng_results[0], expected_output) +@xfail_issue_35926 def test_reshape_opset5_param_err(): original_shape = [2, 3, 4] output_shape = np.array([4, 2, 3], dtype=np.int64) @@ -85,6 +88,7 @@ def test_reshape_opset5_param_err(): assert ng_result[0].shape == output_shape +@xfail_issue_35926 @pytest.mark.parametrize( "axis,expected_output", [ @@ -110,6 +114,7 @@ def test_flatten_exception(): run_node(node, [data]) +@xfail_issue_35926 def test_transpose(): data = np.arange(120).reshape([2, 3, 4, 5]) @@ -124,6 +129,7 @@ def test_transpose(): assert np.array_equal(ng_results, [expected_output]) +@xfail_issue_35927 def test_slice_opset1(): data = np.array([[1, 2, 3, 4], [5, 6, 7, 8]]) @@ -172,6 +178,7 @@ def test_slice_opset1(): assert np.array_equal(ng_results, [expected_output]) +@xfail_issue_35926 def test_concat(): a = np.array([[1, 2], [3, 4]]) b = np.array([[5, 6]]) @@ -208,6 +215,7 @@ def test_concat(): assert np.array_equal(ng_results, [expected_output]) +@xfail_issue_35926 def test_squeeze(): data = np.arange(6).reshape([1, 2, 3, 1]) expected_output = data.reshape([2, 3]) @@ -241,6 +249,7 @@ def test_unsqueeze(): assert np.array_equal(ng_results, [expected_output]) +@xfail_issue_35926 @pytest.mark.parametrize( "node, expected_output", [ diff --git a/ngraph/python/tests/test_onnx/test_ops_unary.py b/ngraph/python/tests/test_onnx/test_ops_unary.py index a696e28..7cc79f1 100644 --- a/ngraph/python/tests/test_onnx/test_ops_unary.py +++ b/ngraph/python/tests/test_onnx/test_ops_unary.py @@ -21,9 +21,18 @@ from onnx.helper import make_graph, make_model, make_node, make_tensor_value_inf from ngraph.exceptions import NgraphTypeError from tests.runtime import get_runtime -from tests.test_onnx.utils import get_node_model, import_onnx_model, run_model, run_node - - +from tests.test_onnx.utils import (get_node_model, + import_onnx_model, + run_model, run_node, + xfail_issue_35926, + xfail_issue_35929, + xfail_issue_34323, + xfail_issue_35930, + xfail_issue_35932 + ) + + +@xfail_issue_35926 @pytest.mark.parametrize( "input_data", [ @@ -87,6 +96,7 @@ def test_log(input_data): assert np.allclose(ng_results, [expected_output]) +@xfail_issue_35926 @pytest.mark.parametrize( "input_data", [ @@ -102,6 +112,7 @@ def test_neg(input_data): assert np.array_equal(ng_results, [expected_output]) +@xfail_issue_35929 @pytest.mark.parametrize( "input_data", [ @@ -117,6 +128,7 @@ def test_floor(input_data): assert np.array_equal(ng_results, [expected_output]) +@xfail_issue_35929 @pytest.mark.parametrize( "input_data", [ @@ -160,6 +172,7 @@ def test_clip_default(): assert np.allclose(result, [expected]) +@xfail_issue_35929 @pytest.mark.parametrize( "input_data", [ @@ -278,6 +291,7 @@ def test_softmax(): ng_results = run_node(node, [data]) +@xfail_issue_35932 def test_logsoftmax(): def logsoftmax_2d(x): max_x = np.max(x, axis=1).reshape((-1, 1)) @@ -370,6 +384,7 @@ def test_identity(): assert np.array_equal(ng_results[0], expected_result) +@xfail_issue_34323 @pytest.mark.parametrize("val_type, input_data", [(np.dtype(bool), np.zeros((2, 2), dtype=int))]) def test_cast_to_bool(val_type, input_data): expected = np.array(input_data, dtype=val_type) @@ -382,8 +397,8 @@ def test_cast_to_bool(val_type, input_data): @pytest.mark.parametrize( "val_type, range_start, range_end, in_dtype", [ - pytest.param(np.dtype(np.float32), -8, 8, np.dtype(np.int32)), - pytest.param(np.dtype(np.float64), -16383, 16383, np.dtype(np.int64)), + pytest.param(np.dtype(np.float32), -8, 8, np.dtype(np.int32), marks=xfail_issue_34323), + pytest.param(np.dtype(np.float64), -16383, 16383, np.dtype(np.int64), marks=xfail_issue_35929), ], ) def test_cast_to_float(val_type, range_start, range_end, in_dtype): @@ -397,7 +412,10 @@ def test_cast_to_float(val_type, range_start, range_end, in_dtype): @pytest.mark.parametrize( - "val_type", [np.dtype(np.int8), np.dtype(np.int16), np.dtype(np.int32), np.dtype(np.int64)] + "val_type", [pytest.param(np.dtype(np.int8), marks=xfail_issue_34323), + pytest.param(np.dtype(np.int16), marks=xfail_issue_34323), + np.dtype(np.int32), + np.dtype(np.int64)] ) def test_cast_to_int(val_type): np.random.seed(133391) @@ -409,6 +427,7 @@ def test_cast_to_int(val_type): assert np.allclose(result, expected) +@xfail_issue_34323 @pytest.mark.parametrize( "val_type", [np.dtype(np.uint8), np.dtype(np.uint16), np.dtype(np.uint32), np.dtype(np.uint64)] ) @@ -422,6 +441,7 @@ def test_cast_to_uint(val_type): assert np.allclose(result, expected) +@xfail_issue_35930 def test_cast_errors(): np.random.seed(133391) input_data = np.ceil(np.random.rand(2, 3, 4) * 16) @@ -491,7 +511,9 @@ def test_cast_errors(): import_onnx_model(model) -@pytest.mark.parametrize("value_type", [np.float32, np.float64]) +@pytest.mark.parametrize("value_type", + [pytest.param(np.float32, marks=xfail_issue_34323), + pytest.param(np.float64, marks=xfail_issue_35929)]) def test_constant(value_type): values = np.random.randn(5, 5).astype(value_type) node = onnx.helper.make_node( diff --git a/ngraph/python/tests/test_onnx/test_ops_variadic.py b/ngraph/python/tests/test_onnx/test_ops_variadic.py index e1238b7..1a1639d 100644 --- a/ngraph/python/tests/test_onnx/test_ops_variadic.py +++ b/ngraph/python/tests/test_onnx/test_ops_variadic.py @@ -19,9 +19,10 @@ import numpy as np import onnx import pytest -from tests.test_onnx.utils import run_node +from tests.test_onnx.utils import run_node, xfail_issue_35926 +@xfail_issue_35926 @pytest.mark.parametrize("onnx_op,numpy_func", [("Sum", np.add), ("Min", np.minimum), ("Max", np.maximum)]) def test_variadic(onnx_op, numpy_func): data = [np.array([1, 2, 3]), np.array([4, 5, 6]), np.array([7, 8, 9])] @@ -32,6 +33,7 @@ def test_variadic(onnx_op, numpy_func): assert np.array_equal(ng_results, [expected_output]) +@xfail_issue_35926 def test_mean(): data = [np.array([1, 2, 3]), np.array([4, 5, 6]), np.array([7, 8, 9])] node = onnx.helper.make_node("Mean", inputs=["data_0", "data_1", "data_2"], outputs=["y"]) diff --git a/ngraph/python/tests/test_onnx/utils/__init__.py b/ngraph/python/tests/test_onnx/utils/__init__.py index 54ba480..a5478c1 100644 --- a/ngraph/python/tests/test_onnx/utils/__init__.py +++ b/ngraph/python/tests/test_onnx/utils/__init__.py @@ -28,8 +28,41 @@ from tests.test_onnx.utils.onnx_backend import OpenVinoOnnxBackend from tests.test_onnx.utils.onnx_helpers import import_onnx_model -def xfail_test(*backends, reason="Mark the test as expected to fail"): - return pytest.mark.xfail(condition=tests.BACKEND_NAME in backends, reason=reason, strict=True) +def xfail_test(reason="Mark the test as expected to fail", strict=True): + return pytest.mark.xfail(reason=reason, strict=strict) + + +skip_segfault = pytest.mark.skip(reason="Segmentation fault error") +xfail_issue_35893 = xfail_test(reason="ValueError: could not broadcast input array") +xfail_issue_35911 = xfail_test(reason="Assertion error: Pad model mismatch error") +xfail_issue_35912 = xfail_test(reason="RuntimeError: Error of validate layer: B with type: " + "Pad. Cannot parse parameter pads_end from IR for layer B. " + "Value -1,0 cannot be casted to int.") +xfail_issue_35914 = xfail_test(reason="IndexError: too many indices for array: " + "array is 0-dimensional, but 1 were indexed") +xfail_issue_35915 = xfail_test(reason="RuntimeError: Eltwise node with unsupported combination " + "of input and output types") +xfail_issue_35916 = xfail_test(reason="RuntimeError: Unsupported input dims count for layer Z") +xfail_issue_35917 = xfail_test(reason="RuntimeError: Unsupported input dims count for " + "layer MatMul") +xfail_issue_35918 = xfail_test(reason="onnx.onnx_cpp2py_export.checker.ValidationError: " + "Mismatched attribute type in 'test_node : alpha'") +xfail_issue_35921 = xfail_test(reason="ValueError - shapes mismatch in gemm") + +xfail_issue_35923 = xfail_test(reason="RuntimeError: PReLU without weights is not supported") +xfail_issue_35924 = xfail_test(reason="Assertion error - elu results mismatch") +unstrict_xfail_issue_35925 = xfail_test(reason="Assertion error - reduction ops results mismatch", + strict=False) +strict_xfail_issue_35925 = xfail_test(reason="Assertion error - reduction ops results mismatch") +xfail_issue_35926 = xfail_test(reason="RuntimeError: [NOT_IMPLEMENTED] Input image format I64 is " + "not supported yet...") +xfail_issue_35927 = xfail_test(reason="RuntimeError: B has zero dimension that is not allowable") +xfail_issue_35929 = xfail_test(reason="RuntimeError: Incorrect precision f64!") +xfail_issue_34323 = xfail_test(reason="RuntimeError: data [value] doesn't exist") +xfail_issue_35930 = xfail_test(reason="onnx.onnx_cpp2py_export.checker.ValidationError: " + "Required attribute 'to' is missing.") +xfail_issue_35932 = xfail_test(reason="Assertion error - logsoftmax results mismatch") +xfail_issue_36437 = xfail_test(reason="RuntimeError: Cannot find blob with name: y") def run_node(onnx_node, data_inputs, **kwargs): diff --git a/ngraph/python/tox.ini b/ngraph/python/tox.ini index 5ff98e3..e3a0312 100644 --- a/ngraph/python/tox.ini +++ b/ngraph/python/tox.ini @@ -23,11 +23,8 @@ commands= flake8 {posargs:src/ setup.py} flake8 --ignore=D100,D101,D102,D103,D104,D105,D107,W503 tests/ # ignore lack of docs in tests mypy --config-file=tox.ini {posargs:src/} - ; TODO: uncomment the line below when all test are ready (and delete the following line) - ; pytest --backend={env:NGRAPH_BACKEND} {posargs:tests/} - pytest --backend={env:NGRAPH_BACKEND} tests/test_ngraph/test_core.py - pytest --backend={env:NGRAPH_BACKEND} tests/test_onnx/test_onnx_import.py tests/test_onnx/test_ops_binary.py - + pytest --backend={env:NGRAPH_BACKEND} tests/test_ngraph/test_core.py -v + pytest --backend={env:NGRAPH_BACKEND} tests/test_onnx -v -k 'not test_zoo_models.py' [testenv:devenv] envdir = devenv usedevelop = True -- 2.7.4