runtime/onert/backend/train/ops/FullyConnectedLayer.cc

   1 /*
   2  * Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *      http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include "FullyConnectedLayer.h"
  18
  19 #include "OperationUtils.h"
  20
  21 #include <cker/operation/FullyConnected.h>
  22 #include <cker/operation/Transpose.h>
  23 #include <cker/train/operation/FullyConnected.h>
  24 #include <cker/train/operation/ReLU.h>
  25
  26 namespace
  27 {
  28
  29 using namespace onert;
  30
  31 std::unique_ptr<backend::train::Tensor>
  32 createTransposedTensor(const backend::IPortableTensor *origin_tensor)
  33 {
  34   const auto &origin_shape = origin_tensor->getShape();
  35   assert(origin_shape.rank() == 2);
  36
  37   auto transposed_info = origin_tensor->get_info();
  38   auto transposed_shape = ir::Shape{origin_shape.dim(1), origin_shape.dim(0)};
  39   transposed_info.shape(transposed_shape);
  40
  41   return std::make_unique<backend::train::Tensor>(transposed_info, origin_tensor->layout());
  42 }
  43
  44 } // namespace
  45
  46 namespace onert
  47 {
  48 namespace backend
  49 {
  50 namespace train
  51 {
  52 namespace ops
  53 {
  54
  55 FullyConnectedLayer::FullyConnectedLayer()
  56   : cpu::ops::FullyConnectedLayer{}, _grad_weights{nullptr}, _grad_bias{nullptr},
  57     _deriv_input{nullptr}, _deriv_output{nullptr}, _transposed_weights{nullptr},
  58     _transposed_input{nullptr}, _transposed_deriv_output{nullptr}, _act_deriv_output{nullptr}
  59 {
  60   // DO NOTHING
  61 }
  62
  63 FullyConnectedLayer::~FullyConnectedLayer() = default;
  64
  65 void FullyConnectedLayer::configure(const IPortableTensor *input, const IPortableTensor *weights,
  66                                     const IPortableTensor *bias, IPortableTensor *output,
  67                                     IPortableTensor *deriv_input, IPortableTensor *grad_weights,
  68                                     IPortableTensor *grad_bias, const IPortableTensor *deriv_output,
  69                                     ir::Activation activation,
  70                                     ir::FullyConnectedWeightsFormat weights_format,
  71                                     const std::shared_ptr<train::ExternalContext> &external_context)
  72 {
  73   cpu::ops::FullyConnectedLayer::configure(input, weights, bias, activation, weights_format, output,
  74                                            external_context);
  75
  76   _deriv_input = deriv_input;
  77   _grad_weights = grad_weights;
  78   _grad_bias = grad_bias;
  79   _deriv_output = deriv_output;
  80
  81   if (weights_format != ir::FullyConnectedWeightsFormat::Default)
  82     throw std::runtime_error{
  83       "train FullyConnectedLayer: Weight formats other than default are not supported."};
  84
  85   if (input->get_info().shape().rank() != 2 || weights->get_info().shape().rank() != 2 ||
  86       output->get_info().shape().rank() != 2 || deriv_input->get_info().shape().rank() != 2 ||
  87       grad_weights->get_info().shape().rank() != 2 || deriv_output->get_info().shape().rank() != 2)
  88     throw std::runtime_error{
  89       "train FullyConnectedLayer: Input other ranks than 2 are not supported."};
  90
  91   _transposed_weights = createTransposedTensor(weights);
  92   _transposed_weights->setBuffer(std::make_shared<basic::Allocator>(weights->total_size()));
  93
  94   _transposed_input = createTransposedTensor(input);
  95   _transposed_input->setBuffer(std::make_shared<basic::Allocator>(input->total_size()));
  96
  97   _transposed_deriv_output = createTransposedTensor(deriv_output);
  98   _transposed_deriv_output->setBuffer(
  99     std::make_shared<basic::Allocator>(deriv_output->total_size()));
 100
 101   if (activation != ir::Activation::NONE)
 102   {
 103     _act_deriv_output =
 104       std::make_unique<Tensor>(_deriv_output->get_info(), _deriv_output->layout());
 105     _act_deriv_output->setBuffer(std::make_shared<basic::Allocator>(_deriv_output->total_size()));
 106   }
 107 }
 108
 109 void FullyConnectedLayer::forward(bool) { cpu::ops::FullyConnectedLayer::run(); }
 110
 111 void FullyConnectedLayer::backward()
 112 {
 113   const auto data_type = _deriv_output->data_type();
 114   assert(data_type == _input->data_type());
 115   switch (data_type)
 116   {
 117     case OperandType::FLOAT32:
 118     {
 119       assert(data_type == _grad_weights->data_type());
 120       assert(data_type == _grad_bias->data_type());
 121       backwardFloat32();
 122       break;
 123     }
 124     default:
 125       throw std::runtime_error{"train FullyConnectedLayer: unsupported data type"};
 126   }
 127 }
 128
 129 void FullyConnectedLayer::backwardFloat32()
 130 {
 131   // Calculate gradient for activation
 132   const IPortableTensor *backprop_act;
 133   switch (_activation)
 134   {
 135     case ir::Activation::NONE:
 136       backprop_act = _deriv_output;
 137       break;
 138     case ir::Activation::RELU:
 139       nnfw::cker::train::ReLUGrad(getShape(_output), getBuffer<float>(_output),
 140                                   getShape(_deriv_output), getBuffer<float>(_deriv_output),
 141                                   getShape(_act_deriv_output.get()),
 142                                   getBuffer<float>(_act_deriv_output.get()));
 143       backprop_act = _act_deriv_output.get();
 144       break;
 145     default:
 146       throw std::runtime_error("train FullyConnectedLayer: Unsupported activation type yet");
 147   }
 148
 149   // Initialize TransposeParams
 150   nnfw::cker::TransposeParams transpose_param;
 151   transpose_param.perm_count = 2;
 152   transpose_param.perm[0] = 1;
 153   transpose_param.perm[1] = 0;
 154
 155   // Initialize FullyConnectedParams
 156   nnfw::cker::FullyConnectedParams op_params;
 157   float output_activation_min = 0;
 158   float output_activation_max = 0;
 159   CalculateActivationRange(ir::Activation::NONE, &output_activation_min, &output_activation_max);
 160   op_params.activation = nnfw::cker::FusedActivationFunctionType::kNone;
 161   op_params.float_activation_min = output_activation_min;
 162   op_params.float_activation_max = output_activation_max;
 163   op_params.lhs_cacheable = false;
 164   op_params.rhs_cacheable = false;
 165
 166   // Transpose and compute gradient for input
 167   // ∂L/∂X = fc(Incoming gradient, transposed W)
 168   auto transposed_weights = _transposed_weights.get();
 169   assert(transposed_weights->getShape().rank() == 2);
 170   nnfw::cker::Transpose(transpose_param, getShape(_weights), getBuffer<float>(_weights),
 171                         getShape(transposed_weights), getBuffer<float>(transposed_weights));
 172
 173   nnfw::cker::FullyConnected(op_params, getShape(backprop_act), getBuffer<float>(backprop_act),
 174                              getShape(transposed_weights), getBuffer<float>(transposed_weights),
 175                              getShape(nullptr), nullptr, getShape(_deriv_input),
 176                              getBuffer<float>(_deriv_input));
 177
 178   // Transpose and compute gradient for weights
 179   // ∂L/∂W = fc(transposed incomming gradient, transposed X)
 180   auto transposed_input = _transposed_input.get();
 181   assert(transposed_input->getShape().rank() == 2);
 182   nnfw::cker::Transpose(transpose_param, getShape(_input), getBuffer<float>(_input),
 183                         getShape(transposed_input), getBuffer<float>(transposed_input));
 184
 185   auto transposed_deriv_output = _transposed_deriv_output.get();
 186   assert(transposed_deriv_output->getShape().rank() == 2);
 187   nnfw::cker::Transpose(transpose_param, getShape(backprop_act), getBuffer<float>(backprop_act),
 188                         getShape(transposed_deriv_output),
 189                         getBuffer<float>(transposed_deriv_output));
 190
 191   nnfw::cker::FullyConnected(op_params, getShape(transposed_deriv_output),
 192                              getBuffer<float>(transposed_deriv_output), getShape(transposed_input),
 193                              getBuffer<float>(transposed_input), getShape(nullptr), nullptr,
 194                              getShape(_grad_weights), getBuffer<float>(_grad_weights));
 195
 196   // Compute gradient for bias
 197   if (_bias)
 198   {
 199     assert(_grad_bias);
 200     nnfw::cker::train::FullyConnectedBiasGrad(getShape(backprop_act),
 201                                               getBuffer<float>(backprop_act), getShape(_grad_bias),
 202                                               getBuffer<float>(_grad_bias));
 203   }
 204 }
 205
 206 } // namespace ops
 207 } // namespace train
 208 } // namespace backend
 209 } // namespace onert