inference-engine/thirdparty/clDNN/api_extension/CPP/fused_conv_eltwise.hpp

   1 /*
   2 // Copyright (c) 2018 Intel Corporation
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //      http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15 */
  16
  17 ///////////////////////////////////////////////////////////////////////////////////////////////////
  18 #pragma once
  19 #include "../C/fused_conv_eltwise.h"
  20 #include "api/CPP/primitive.hpp"
  21 #include "api/CPP/eltwise.hpp"
  22
  23 namespace cldnn
  24 {
  25 /// @addtogroup cpp_api C++ API
  26 /// @{
  27 /// @addtogroup cpp_topology Network Topology
  28 /// @{
  29 /// @addtogroup cpp_primitives Primitives
  30 /// @{
  31
  32 /// @brief Performs forward spatial convolution with fused eltwise and optionally Relu.
  33 struct fused_conv_eltwise : public primitive_base<fused_conv_eltwise, CLDNN_PRIMITIVE_DESC(fused_conv_eltwise)>
  34 {
  35     CLDNN_DECLARE_PRIMITIVE(fused_conv_eltwise)
  36
  37     /// @brief Constructs fused_conv_eltwise primitive.
  38     /// @param id This primitive id.
  39     /// @param input Input primitive id.
  40     /// @param weights List of primitive ids containing weights data.
  41     /// @param bias List of primitive ids containing bias data.
  42     /// @param w_quantization_factor List of primitive ids containing weights quanitization factors per output feature map.
  43     /// @param output_calibration_factors List of primitive ids output containing calibration factors per output feature map.
  44     /// @param i_quantization_factor Input quantization factor
  45     /// @param input_offset Defines a shift, relative to (0,0) position of the input buffer, where (0,0) point of the convolution window should start calculations.
  46     /// @param stride Defines shift in input buffer between adjacent calculations of output values.
  47     /// @param dilation Defines gaps in the input - dilation rate k=1 is normal convolution, k=2 means skipping one pixel per input, k=4 means skipping 3 pixels.
  48     /// As an example in one dimension, a filter w of size 3 would compute over input x the following: w[0]*x[0] + w[1]*x[1] + w[2]*x[2] for dilation of 1.
  49     /// For dilation 2 the filter would instead compute w[0]*x[0] + w[1]*x[2] + w[2]*x[4].
  50     /// @param with_activation Enable Relu activation.
  51     /// @param activation_slp Relu activation slope.
  52     fused_conv_eltwise(
  53         const primitive_id& id,
  54         const primitive_id& input,
  55         const primitive_id& input2,
  56         eltwise_mode mode,
  57         const std::vector<primitive_id>& weights,
  58         const std::vector<primitive_id>& bias,
  59         const std::vector<primitive_id>& conv_w_quantization_factor,
  60         const std::vector<primitive_id>& conv_output_calibration_factors,
  61         const float conv_i_quantization_factor,
  62         const primitive_id& eltw_output_calibration_factors,
  63         const std::vector<tensor>& eltw_stride,
  64         tensor stride = { 1, 1, 1, 1 },
  65         tensor input_offset = { 0,0,0,0 },
  66         tensor dilation = { 1, 1, 1, 1 },
  67         bool conv_with_activation = false,
  68         float conv_activation_slp = 0.0f,
  69         bool eltw_with_activation = false,
  70         float eltw_activation_slp = 0.0f,
  71         const padding& output_padding = padding()
  72     )
  73         :primitive_base(id, { input, input2 }, output_padding)
  74         , conv(_conv_weights.cpp_ids, _conv_bias.cpp_ids, _conv_weights_quantization_factors.cpp_ids, _conv_output_calibration_factors.cpp_ids)
  75         , eltw(eltw_output_calibration_factors)
  76         , _conv_weights(weights)
  77         , _conv_bias(bias)
  78         , _conv_weights_quantization_factors(conv_w_quantization_factor)
  79         , _conv_output_calibration_factors(conv_output_calibration_factors)
  80     {
  81
  82         conv.input_quantization_factor = conv_i_quantization_factor;
  83         conv.output_quantization_factor = 1.0f;
  84
  85         conv.input_offset = input_offset;
  86         conv.stride = stride;
  87         conv.dilation = dilation;
  88         conv.with_activation = conv_with_activation;
  89         conv.activation_negative_slope = conv_activation_slp;
  90         conv.with_output_size = false;
  91
  92         eltw.mode = mode;
  93         eltw.with_activation = eltw_with_activation;
  94         eltw.activation_negative_slope = eltw_activation_slp;
  95         eltw.stride = eltw_stride;
  96
  97         if ((bias.size() != 0) && (weights.size() != bias.size()))
  98             throw std::runtime_error("convolution's weights/bias count does not match");
  99         if (conv.output_calibration_factors.size())
 100         {
 101             if ((weights.size() != 0) && (weights.size() != conv.weights_quantization_factors.size()))
 102                 throw std::runtime_error("convolution's weights count does not match quantization factors count");
 103         }
 104     }
 105
 106     /// @brief Constructs a copy from C API @CLDNN_PRIMITIVE_DESC{convolution}
 107     fused_conv_eltwise(const dto* dto)
 108         :primitive_base(dto)
 109         , conv(_conv_weights.cpp_ids, _conv_bias.cpp_ids, _conv_weights_quantization_factors.cpp_ids, _conv_output_calibration_factors.cpp_ids)
 110         , eltw(dto->eltw.output_calibration_factors)
 111         , _conv_weights(dto->conv.weights)
 112         , _conv_bias(dto->conv.bias)
 113         , _conv_weights_quantization_factors(dto->conv.weights_quantization_factors)
 114         , _conv_output_calibration_factors(dto->conv.output_calibration_factors)
 115         , _eltw_stride(tensor_vector_to_cldnn_vector(eltw.stride))
 116     {
 117         conv.input_quantization_factor = dto->conv.input_quantization_factor;
 118         conv.output_quantization_factor = dto->conv.output_quantization_factor;
 119         conv.input_offset = dto->conv.input_offset;
 120         conv.stride = dto->conv.stride;
 121         conv.dilation = dto->conv.dilation;
 122         conv.with_activation = dto->conv.with_activation != 0;
 123         conv.activation_negative_slope = dto->conv.activation_negative_slope;
 124         conv.with_output_size = dto->conv.with_output_size != 0;
 125         conv.output_size = dto->conv.output_size;
 126
 127         second_input_in_output = dto->second_input_in_output;
 128
 129         if (!dto->conv.split || (conv.weights.size() != conv.bias.size() && conv.bias.size() != 0) || dto->conv.split != conv.weights.size())
 130             throw std::invalid_argument("Invalid convolution dto: bad split value");
 131     }
 132
 133     struct conv_data
 134     {
 135         /// @brief List of primitive ids containing weights data.
 136         fixed_size_vector_ref weights;
 137         /// @brief List of primitive ids containing bias data.
 138         fixed_size_vector_ref bias;
 139         /// @brief List of primitive ids containing weights quanitization factors per output feature map.
 140         fixed_size_vector_ref weights_quantization_factors;
 141         /// @brief List of primitive ids containing output quanitization factors per output feature map for convolution.
 142         fixed_size_vector_ref output_calibration_factors;
 143         /// @brief Input quantization factor for convolution
 144         float input_quantization_factor;
 145         /// @brief Output quantization factor for convolution
 146         float output_quantization_factor;
 147         /// @brief Defines a shift, relative to (0,0) position of the input buffer, where (0,0) point of the convolution window should start calculations.
 148         tensor input_offset;
 149         /// @brief Defines shift in input buffer between adjacent calculations of output values.
 150         tensor stride;
 151         /// @brief Defines gaps in the input - dilation rate k=1 is normal convolution, k=2 means skipping one pixel per input, k=4 means skipping 3 pixels.
 152         /// As an example in one dimension, a filter w of size 3 would compute over input x the following: w[0]*x[0] + w[1]*x[1] + w[2]*x[2] for dilation of 1.
 153         /// For dilation 2 the filter would instead compute w[0]*x[0] + w[1]*x[2] + w[2]*x[4].
 154         tensor dilation;
 155         /// @brief Enable Relu activation.
 156         bool with_activation;
 157         /// @brief Relu activation slope.
 158         float activation_negative_slope;
 159         /// @brief Indicates that the primitive has user-defined output size (non-zero value).
 160         bool with_output_size;
 161         /// @brief User-defined output data size of the primitive (w/o padding).
 162         tensor output_size;
 163
 164         conv_data(const fixed_size_vector_ref& weights,
 165             const fixed_size_vector_ref& bias,
 166             const fixed_size_vector_ref& weights_quantization_factors,
 167             const fixed_size_vector_ref& output_calibration_factors
 168         ) : weights(weights),
 169             bias(bias),
 170             weights_quantization_factors(weights_quantization_factors),
 171             output_calibration_factors(output_calibration_factors)
 172         {}
 173     } conv;
 174
 175     struct eltw_data
 176     {
 177         /// @brief Primitive id containing output quanitization factors per output feature map.
 178         primitive_id output_calibration_factors;
 179         /// @brief Output quantization factor for eltwise
 180         float output_quantization_factor;
 181         /// @param mode Eltwise mode.
 182         eltwise_mode mode;
 183         /// @brief Enable Relu activation.
 184         bool with_activation;
 185         /// @brief Relu activation slope.
 186         float activation_negative_slope;
 187         /// @brief Defines shift in input buffers between adjacent calculations of output values.
 188         std::vector<tensor> stride;
 189
 190         eltw_data(const primitive_id& output_calibration_factors)
 191             : output_calibration_factors(output_calibration_factors)
 192         {}
 193     } eltw;
 194
 195     /// @brief On how many cards split the computation to.
 196     int32_t split() const { return static_cast<int32_t>(conv.weights.size()); }
 197
 198     /// @brief Is optimization that output contains data from second input ON ?
 199     bool second_input_in_output = false;
 200 protected:
 201     primitive_id_arr _conv_weights;
 202     primitive_id_arr _conv_bias;
 203     primitive_id_arr _conv_weights_quantization_factors;
 204     primitive_id_arr _conv_output_calibration_factors;
 205
 206     std::vector<cldnn_tensor> _eltw_stride;
 207
 208     std::vector<std::reference_wrapper<const primitive_id>> get_dependencies() const override
 209     {
 210         std::vector<std::reference_wrapper<const primitive_id>> ret;
 211         ret.reserve(conv.weights.size()
 212             + conv.bias.size()
 213             + conv.weights_quantization_factors.size()
 214             + conv.output_calibration_factors.size()
 215             + (eltw.output_calibration_factors.empty() ? 0 : 1));
 216
 217         for (auto& w : conv.weights)
 218             ret.push_back(w);
 219         for (auto& b : conv.bias)
 220             ret.push_back(b);
 221         for (auto& q : conv.weights_quantization_factors)
 222             ret.push_back(q);
 223         for (auto& q : conv.output_calibration_factors)
 224             ret.push_back(q);
 225
 226         if (!eltw.output_calibration_factors.empty())
 227             ret.push_back(eltw.output_calibration_factors);
 228
 229         return ret;
 230     }
 231
 232     void update_dto(dto& dto) const override
 233     {
 234         dto.conv.weights = _conv_weights.ref();
 235         dto.conv.bias = _conv_bias.ref();
 236         dto.conv.weights_quantization_factors = _conv_weights_quantization_factors.ref();
 237         dto.conv.output_calibration_factors = _conv_output_calibration_factors.ref();
 238         dto.conv.input_quantization_factor = conv.input_quantization_factor;
 239         dto.conv.output_quantization_factor = conv.output_quantization_factor;
 240         dto.conv.input_offset = conv.input_offset;
 241         dto.conv.stride = conv.stride;
 242         dto.conv.split = split();
 243         dto.conv.with_activation = conv.with_activation;
 244         dto.conv.activation_negative_slope = conv.activation_negative_slope;
 245         dto.conv.dilation = conv.dilation;
 246         dto.conv.with_output_size = conv.with_output_size;
 247         dto.conv.output_size = conv.output_size;
 248
 249         dto.eltw.output_calibration_factors = eltw.output_calibration_factors.c_str();
 250         dto.eltw.output_quantization_factor = eltw.output_quantization_factor;
 251         dto.eltw.mode = static_cast<cldnn_eltwise_mode>(eltw.mode);
 252         dto.eltw.with_activation = eltw.with_activation;
 253         dto.eltw.activation_negative_slope = eltw.activation_negative_slope;
 254         dto.eltw.stride = tensor_vector_to_arr(_eltw_stride);
 255
 256         dto.second_input_in_output = second_input_in_output;
 257     }
 258 };
 259 /// @}
 260 /// @}
 261 /// @}
 262 }