ngraph/python/tests/__init__.py

   1 # ******************************************************************************
   2 # Copyright 2017-2020 Intel Corporation
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 # ******************************************************************************
  16 import pytest
  17
  18 # test.BACKEND_NAME is a configuration variable determining which
  19 # nGraph backend tests will use. It's set during pytest configuration time.
  20 # See `pytest_configure` hook in `conftest.py` for more details.
  21 BACKEND_NAME = None
  22
  23 # test.MODEL_ZOO_DIR is a configuration variable providing the path
  24 # to the ZOO of ONNX models to test. It's set during pytest configuration time.
  25 # See `pytest_configure` hook in `conftest.py` for more
  26 # details.
  27 MODEL_ZOO_DIR = None
  28
  29 # test.MODEL_ZOO_XFAIL is a configuration variable which enable xfails for model zoo.
  30 MODEL_ZOO_XFAIL = False
  31
  32
  33 def xfail_test(reason="Mark the test as expected to fail", strict=True):
  34     return pytest.mark.xfail(reason=reason, strict=strict)
  35
  36
  37 skip_segfault = pytest.mark.skip(reason="Segmentation fault error")
  38 xfail_issue_33488 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
  39                                       "MaxUnpool")
  40 xfail_issue_33512 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
  41                                       "Einsum")
  42 xfail_issue_33515 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
  43                                       "BitShift")
  44 xfail_issue_33535 = xfail_test(reason="nGraph does not support the following ONNX operations:"
  45                                       "DynamicQuantizeLinear")
  46 xfail_issue_33538 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
  47                                       "Scan")
  48 xfail_issue_33540 = xfail_test(reason="RuntimeError: GRUCell operation has a form that is not supported "
  49                                       "GRUCell_<number> should be converted to GRUCellIE operation")
  50 xfail_issue_33589 = xfail_test(reason="nGraph does not support the following ONNX operations:"
  51                                       "IsNaN and isInf")
  52 xfail_issue_33595 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
  53                                       "Unique")
  54 xfail_issue_33596 = xfail_test(reason="RuntimeError: nGraph does not support different sequence operations:"
  55                                       "ConcatFromSequence, SequenceConstruct, SequenceAt, SplitToSequence,"
  56                                       "SequenceEmpty, SequenceInsert, SequenceErase, SequenceLength ")
  57 xfail_issue_33606 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
  58                                       "Det")
  59 xfail_issue_33616 = xfail_test(reason="Add ceil_mode for Max and Avg pooling (reference implementation)")
  60 xfail_issue_33644 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
  61                                       "Compress")
  62 xfail_issue_33651 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
  63                                       "TfIdfVectorizer")
  64 xfail_issue_34310 = xfail_test(reason="RuntimeError: Error of validate layer: LSTMSequence_<number> with "
  65                                       "type: LSTMSequence. Layer is not instance of RNNLayer class")
  66 xfail_issue_34314 = xfail_test(reason="RuntimeError: RNNCell operation has a form that is not "
  67                                "supported.RNNCell_<number> should be converted to RNNCellIE operation")
  68 xfail_issue_34323 = xfail_test(reason="RuntimeError: data [value] doesn't exist")
  69 xfail_issue_34327 = xfail_test(reason="RuntimeError: '<value>' layer has different "
  70                                       "IN and OUT channels number")
  71 xfail_issue_33581 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
  72                                       "GatherElements")
  73 xfail_issue_35911 = xfail_test(reason="Assertion error: Pad model mismatch error")
  74 xfail_issue_35912 = xfail_test(reason="RuntimeError: Error of validate layer: B with type: "
  75                                       "Pad. Cannot parse parameter pads_end  from IR for layer B. "
  76                                       "Value -1,0 cannot be casted to int.")
  77 xfail_issue_35915 = xfail_test(reason="RuntimeError: Eltwise node with unsupported combination "
  78                                       "of input and output types")
  79 xfail_issue_35916 = xfail_test(reason="RuntimeError: Unsupported input dims count for layer Z")
  80 xfail_issue_35917 = xfail_test(reason="RuntimeError: Unsupported input dims count for "
  81                                       "layer MatMul")
  82 xfail_issue_35918 = xfail_test(reason="onnx.onnx_cpp2py_export.checker.ValidationError: "
  83                                       "Mismatched attribute type in 'test_node : alpha'")
  84 xfail_issue_35921 = xfail_test(reason="ValueError - shapes mismatch in gemm")
  85 xfail_issue_35923 = xfail_test(reason="RuntimeError: PReLU without weights is not supported")
  86 xfail_issue_35924 = xfail_test(reason="Assertion error - elu results mismatch")
  87 xfail_issue_35925 = xfail_test(reason="Assertion error - reduction ops results mismatch")
  88 xfail_issue_35926 = xfail_test(reason="RuntimeError: [NOT_IMPLEMENTED] Input image format I64 is "
  89                                       "not supported yet...")
  90 xfail_issue_35927 = xfail_test(reason="RuntimeError: B has zero dimension that is not allowable")
  91 xfail_issue_35929 = xfail_test(reason="RuntimeError: Incorrect precision f64!")
  92 xfail_issue_35930 = xfail_test(reason="onnx.onnx_cpp2py_export.checker.ValidationError: "
  93                                       "Required attribute 'to' is missing.")
  94 xfail_issue_35932 = xfail_test(reason="Assertion error - logsoftmax results mismatch")
  95 xfail_issue_36437 = xfail_test(reason="RuntimeError: Cannot find blob with name: <value>")
  96 xfail_issue_36476 = xfail_test(reason="RuntimeError: [NOT_IMPLEMENTED] Input image format U32 is "
  97                                "not supported yet...")
  98 xfail_issue_36478 = xfail_test(reason="RuntimeError: [NOT_IMPLEMENTED] Input image format U64 is "
  99                                "not supported yet...")
 100 xfail_issue_36479 = xfail_test(reason="Assertion error - basic computation on ndarrays mismatch")
 101 xfail_issue_36480 = xfail_test(reason="RuntimeError: [NOT_FOUND] Unsupported property dummy_option "
 102                                "by CPU plugin")
 103 xfail_issue_36483 = xfail_test(reason="RuntimeError: Unsupported primitive of type: "
 104                                "Ceiling name: <value>")
 105 xfail_issue_36485 = xfail_test(reason="RuntimeError: Check 'm_group >= 1' failed at "
 106                                "/openvino/ngraph/core/src/op/shuffle_channels.cpp:77:")
 107 xfail_issue_36486 = xfail_test(reason="RuntimeError: HardSigmoid operation should be converted "
 108                                       "to HardSigmoid_IE")
 109 xfail_issue_36487 = xfail_test(reason="Assertion error - mvn operator computation mismatch")
 110 xfail_issue_38084 = xfail_test(reason="RuntimeError: AssertionFailed: layer->get_output_partial_shape(i)"
 111                                       "is_static() nGraph <value> operation with name: <value> cannot be"
 112                                       "converted to <value> layer with name: <value> because output"
 113                                       "with index 0 contains dynamic shapes: {<value>}. Try to use "
 114                                       "CNNNetwork::reshape() method in order to specialize shapes "
 115                                       "before the conversion.")
 116 xfail_issue_38085 = xfail_test(reason="RuntimeError: Interpolate operation should be converted to Interp")
 117 xfail_issue_38086 = xfail_test(reason="RuntimeError: Quantize layer input '<value>' doesn't have blobs")
 118 xfail_issue_38087 = xfail_test(reason="RuntimeError: Cannot cast to tensor desc. Format is unsupported!")
 119 xfail_issue_38088 = xfail_test(reason="RuntimeError: Check '((axis >= axis_range_min) && "
 120                                       "(axis <= axis_range_max))' failed at "
 121                                       "/openvino/ngraph/core/src/validation_util.cpp:913: "
 122                                       "Split Parameter axis <value> out of the tensor rank range <value>.")
 123 xfail_issue_38089 = xfail_test(reason="RuntimeError: Node 2 contains empty child edge for index 0")
 124 xfail_issue_38090 = xfail_test(reason="AssertionError: Items types are not equal")
 125 xfail_issue_38091 = xfail_test(reason="AssertionError: Mismatched elements")
 126 xfail_issue_38699 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 127                                       "ai.onnx.preview.training.Gradient")
 128 xfail_issue_38701 = xfail_test(reason="RuntimeError: unsupported element type: STRING")
 129 xfail_issue_38705 = xfail_test(reason="IndexError: deque::_M_range_check: __n (which is 0)"
 130                                       ">= this->size() (which is 0)")
 131 xfail_issue_38706 = xfail_test(reason="RuntimeError: output_3.0 has zero dimension which is not allowed")
 132 xfail_issue_38707 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 133                                       "SoftmaxCrossEntropyLoss")
 134 xfail_issue_38708 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Slice): y>': "
 135                                       "Axes input must be constant")
 136 xfail_issue_38710 = xfail_test(reason="RuntimeError: roi has zero dimension which is not allowed")
 137 xfail_issue_38712 = xfail_test(reason="RuntimeError: Check '(fmod == 1) "
 138                                       "While validating ONNX node '<Node(Mod): z>': "
 139                                       "Only 'fmod=1' mode is supported for mod operator.")
 140 xfail_issue_38713 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 141                                       "ai.onnx.preview.training.Momentum")
 142 xfail_issue_38714 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Resize): Y>'"
 143                                       "Check 'element::Type::merge(element_type, element_type,"
 144                                       "node->get_input_element_type(i))' "
 145                                       "While validating node 'v1::<name> (sizes[0]:i64{4},"
 146                                       "Convert_29306[0]:f32{4}) -> (dynamic?)' with friendly_name '<name>':"
 147                                       "Argument element types are inconsistent.")
 148 xfail_issue_38715 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(OneHot): y>':"
 149                                       "While validating node 'v1::OneHot OneHot_<number>"
 150                                       "(Convert_13525[0]:i64{3}, depth[0]:f32{},"
 151                                       "Squeeze_13532[0]:i32{}, Squeeze_13529[0]:i32{}) -> (dynamic?)'"
 152                                       "with friendly_name 'OneHot_13534':"
 153                                       "Depth must be integral element type.")
 154 xfail_issue_38717 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 155                                       "GreaterOrEqual")
 156 xfail_issue_38719 = xfail_test(reason="nGraph does not support the following ONNX operations: GatherND")
 157 xfail_issue_38722 = xfail_test(reason="RuntimeError: While validating ONNX nodes MatMulInteger"
 158                                       "and QLinearMatMul"
 159                                       "Input0 scale and input0 zero point shape must be same and 1")
 160 xfail_issue_38723 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 161                                       "QLinearConv")
 162 xfail_issue_38724 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Resize): Y>':"
 163                                       "tf_crop_and_resize - this type of coordinate transformation mode"
 164                                       "is not supported. Choose one of the following modes:"
 165                                       "tf_half_pixel_for_nn, asymmetric, align_corners, pytorch_half_pixel,"
 166                                       "half_pixel")
 167 xfail_issue_38725 = xfail_test(reason="RuntimeError: While validating ONNX node '<Node(Loop):"
 168                                       "value info has no element type specified")
 169 xfail_issue_38726 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 170                                       "LessOrEqual")
 171 xfail_issue_38732 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 172                                       "ConvInteger")
 173 xfail_issue_38733 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 174                                       "Celu")
 175 xfail_issue_38734 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 176                                       "ai.onnx.preview.training.Adam")
 177 xfail_issue_38735 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 178                                       "ai.onnx.preview.training.Adagrad")
 179 xfail_issue_38736 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 180                                       "NegativeLogLikelihoodLoss")
 181
 182 # Model ONNX Zoo issues:
 183 xfail_issue_36533 = xfail_test(reason="AssertionError: zoo models results mismatch")
 184 xfail_issue_36537 = xfail_test(reason="ngraph.exceptions.UserInputError: (Provided tensor's shape:"
 185                                       "%s does not match the expected: %s."
 186                                       "<PartialShape: <value>>, <PartialShape: <value>>)")
 187 xfail_issue_37687 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 188                                       "Loop")
 189 xfail_issue_39682 = xfail_test(reason="model with IR version >= 3 must specify opset_import for ONNX")
 190 xfail_issue_39683 = xfail_test(reason="convolution.W in initializer but not in graph input")
 191 xfail_issue_39684 = xfail_test(reason="ngraph.exceptions.UserInputError:"
 192                                       "('Expected %s parameters, received %s.', 1, 3)")
 193 xfail_issue_39685 = xfail_test(reason="RuntimeError: While validating node 'v1::Transpose 315,"
 194                                       "Constant_9353 -> (f32{?,?,?,?})' with friendly_name '315':"
 195                                       "Input order must have shape [n], where n is the rank of arg.")
 196
 197 # Model MSFT issues:
 198 xfail_issue_36465 = xfail_test(reason="LSTM_Seq_lens: RuntimeError: get_shape was called on a "
 199                                       "descriptor::Tensor with dynamic shape")
 200 xfail_issue_37957 = xfail_test(reason="RuntimeError: nGraph does not support the following ONNX operations:"
 201                                       "com.microsoft.CropAndResize, com.microsoft.GatherND,"
 202                                       "com.microsoft.Pad, com.microsoft.Range")
 203 xfail_issue_39669 = xfail_test(reason="AssertionError: This model has no test data")