model-optimizer/extensions/middle/EltwiseInputReshape.py

   1 """
   2  Copyright (c) 2018-2019 Intel Corporation
   3
   4  Licensed under the Apache License, Version 2.0 (the "License");
   5  you may not use this file except in compliance with the License.
   6  You may obtain a copy of the License at
   7
   8       http://www.apache.org/licenses/LICENSE-2.0
   9
  10  Unless required by applicable law or agreed to in writing, software
  11  distributed under the License is distributed on an "AS IS" BASIS,
  12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  See the License for the specific language governing permissions and
  14  limitations under the License.
  15 """
  16
  17 from copy import deepcopy
  18
  19 import numpy as np
  20
  21 from mo.front.common.layout import get_features_dim, shape_for_layout
  22 from mo.graph.graph import Node, Graph
  23 from mo.middle.passes.fusing.helpers import get_value_id
  24 from mo.middle.replacement import MiddleReplacementPattern
  25 from mo.ops.op import Op
  26 from mo.ops.reshape import Reshape
  27
  28
  29 class Eltwise1DInputReshape(MiddleReplacementPattern):
  30     """
  31     Inserts Reshape before 1-D input to Eltwise if another input of Eltwise is multi-dimensional tensor with the
  32     same feature size as 1-D input
  33
  34     Replacer is useful in cases of layout change in MO (for example NHWC-> NCHW translation of TensorFlow models)
  35
  36     Example:
  37     Eltwise Mul operation in TF multiplies Tensors by feature dimension with shapes [1,375,500,24] and [24].
  38     After layout change in MO Eltwise Mul have input shapes [1,24,375,500] and [24]. It is a problem (500!=24).
  39     We have to insert Reshape layer for Tensor with [24] shape to correspond the feature dimension of
  40     Tensor [1,24,375,500] shape
  41
  42     change of graph.graph['layout'] may cause an issue
  43     change in re-layout function: convert_nhwc_to_nchw(graph) may cause an issue
  44     """
  45     enabled = True
  46
  47     def run_after(self):
  48         return [EltwiseInputReshape]
  49
  50     def find_and_replace_pattern(self, graph: Graph):
  51         layout = graph.graph['layout']
  52         for n in list(graph.nodes()):
  53             if 'type' in graph.node[n] and graph.node[n]['type'] == 'Eltwise' and get_value_id(Node(graph, n)) is None:
  54                 eltwise_op_node = Node(graph, n)
  55                 out_shape = eltwise_op_node.out_node().shape
  56                 if 4 <= len(out_shape) <= 5:
  57                     out_features = out_shape[get_features_dim(layout, len(out_shape))]
  58                     for port, node in eltwise_op_node.in_nodes().items():
  59                         if len(node.shape) != len(out_shape) and len(node.shape) == 1 and out_features == node.shape[0]:
  60                             in_atts = deepcopy(graph.get_edge_data(node.id, n)[0])
  61                             graph.remove_edge(node.id, n)
  62                             new_shape = shape_for_layout(layout, batch=1, features=out_features, height=1, width=1,
  63                                                          depth=1 if len(out_shape) == 5 else None)
  64                             reshape_data_op = Reshape(graph, attrs={'dim': new_shape, 'name': node.id + '/Broadcast'})
  65                             reshape_data_node = reshape_data_op.create_node_with_data([node])
  66                             graph.add_edge(reshape_data_node.id, eltwise_op_node.id, **in_atts)
  67
  68
  69 class EltwiseInputReshape(MiddleReplacementPattern):
  70     enabled = True
  71
  72     def run_after(self):
  73         from extensions.middle.pass_separator import MiddleStart
  74         return [MiddleStart]
  75
  76     def find_and_replace_pattern(self, graph: Graph):
  77         data_nodes = [Node(graph, node) for node in graph.node if Node(graph, node).kind == 'data']
  78         for node in data_nodes:
  79             # Get all requested shapes for current node
  80             # This mapping will contain pairs like {shape:[list of consumers nodes]}
  81             mapping = {}
  82             for consumer in node.out_nodes():
  83                 edge_attrs = graph.get_edge_data(node.id, consumer.id)[0]
  84                 if 'new_shape' in edge_attrs:
  85                     if np.array_equal(edge_attrs['new_shape'], node.shape):
  86                         continue
  87                     new_shape = tuple([x for x in edge_attrs['new_shape']])
  88                     if not new_shape in mapping:
  89                         mapping.update({new_shape: [consumer]})
  90                     else:
  91                         mapping[new_shape].append(consumer)
  92
  93             if node.has_valid('value'):
  94                 # Check that requested shape are the same
  95                 # In case if they are different, we duplicate them
  96                 for shape_key in mapping.keys():
  97                     shape = list(shape_key)
  98                     new_value = np.reshape(node.value, shape)
  99                     node_copy = Op.create_input_data_node(graph, node.id + '/copy', value=np.array(new_value))
 100                     for consumer in mapping[shape_key]:
 101                         edge_attrs = graph.get_edge_data(node.id, consumer.id)[0]
 102                         del edge_attrs['new_shape']
 103
 104                         # Remove edge from previous data node and connect new data node with its consumer
 105                         graph.remove_edge(node.id, consumer.id)
 106                         graph.add_edge(node_copy.id, consumer.id, **edge_attrs)
 107             else:
 108                 # Insert Reshape layer between data node and consumer
 109                 for shape_key in mapping.keys():
 110                     shape = list(shape_key)
 111                     reshape = Reshape(graph, attrs={'dim': shape, 'name': 'EltwiseReshapeNormalization'})
 112                     reshape_data = reshape.create_node_with_data(inputs=[node])
 113
 114                     # Iterate over consumers and reconnect them to Reshape layer output
 115                     for consumer in mapping[shape_key]:
 116                         edge_attrs = graph.get_edge_data(node.id, consumer.id)[0]
 117                         del edge_attrs['new_shape']
 118
 119                         # Reconnect edge from original data node to Reshape output datanode
 120                         graph.remove_edge(node.id, consumer.id)
 121                         graph.add_edge(reshape_data.id, consumer.id, **edge_attrs)