onert-micro/luci-interpreter/src/kernels/UnidirectionalSequenceLSTM.h

   1 /*
   2  * Copyright (c) 2023 Samsung Electronics Co., Ltd. All Rights Reserved
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  *    http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #ifndef LUCI_INTERPRETER_KERNELS_UNIDIRECTIONAL_SEQUENCE_LSTM_H
  18 #define LUCI_INTERPRETER_KERNELS_UNIDIRECTIONAL_SEQUENCE_LSTM_H
  19
  20 #include "PALUtils.h"
  21
  22 namespace luci_interpreter
  23 {
  24 namespace lstm
  25 {
  26
  27 struct LSTMStruct
  28 {
  29   LSTMStruct() = delete;
  30   LSTMStruct(const LSTMStruct &) = delete;
  31
  32   explicit LSTMStruct(const circle::Operator *cur_op,
  33                       luci_interpreter::BaseRuntimeGraph *runtime_graph)
  34   {
  35     const auto input_index = cur_op->inputs()->operator[](0);
  36     const auto input_to_input_weights_index = cur_op->inputs()->operator[](1);
  37     const auto input_to_forget_weights_index = cur_op->inputs()->operator[](2);
  38     const auto input_to_cell_weights_index = cur_op->inputs()->operator[](3);
  39     const auto input_to_output_weights_index = cur_op->inputs()->operator[](4);
  40     assert(input_index != -1);
  41     // input_to_input_weights_index - optional
  42     assert(input_to_forget_weights_index != -1);
  43     assert(input_to_cell_weights_index != -1);
  44     assert(input_to_output_weights_index != -1);
  45     internal_tensors[0] = runtime_graph->getCircleTensorByIndex(input_index);
  46     internal_tensors[1] = runtime_graph->getCircleTensorByIndex(input_to_input_weights_index);
  47     internal_tensors[2] = runtime_graph->getCircleTensorByIndex(input_to_forget_weights_index);
  48     internal_tensors[3] = runtime_graph->getCircleTensorByIndex(input_to_cell_weights_index);
  49     internal_tensors[4] = runtime_graph->getCircleTensorByIndex(input_to_output_weights_index);
  50
  51     const auto recurrent_to_input_weights_index = cur_op->inputs()->operator[](5);
  52     const auto recurrent_to_forget_weights_index = cur_op->inputs()->operator[](6);
  53     const auto recurrent_to_cell_weights_index = cur_op->inputs()->operator[](7);
  54     const auto recurrent_to_output_weights_index = cur_op->inputs()->operator[](8);
  55     // recurrent_to_input_weights_index - optional
  56     assert(recurrent_to_forget_weights_index != -1);
  57     assert(recurrent_to_cell_weights_index != -1);
  58     assert(recurrent_to_output_weights_index != -1);
  59     internal_tensors[5] = runtime_graph->getCircleTensorByIndex(recurrent_to_input_weights_index);
  60     internal_tensors[6] = runtime_graph->getCircleTensorByIndex(recurrent_to_forget_weights_index);
  61     internal_tensors[7] = runtime_graph->getCircleTensorByIndex(recurrent_to_cell_weights_index);
  62     internal_tensors[8] = runtime_graph->getCircleTensorByIndex(recurrent_to_output_weights_index);
  63
  64     const auto cell_to_input_weights_index = cur_op->inputs()->operator[](9);
  65     const auto cell_to_forget_weights_index = cur_op->inputs()->operator[](10);
  66     const auto cell_to_output_weights_index = cur_op->inputs()->operator[](11);
  67     // optional cell_to_input_weights_index
  68     // optional cell_to_forget_weights_index
  69     // optional cell_to_output_weights_index
  70     internal_tensors[9] = runtime_graph->getCircleTensorByIndex(cell_to_input_weights_index);
  71     internal_tensors[10] = runtime_graph->getCircleTensorByIndex(cell_to_forget_weights_index);
  72     internal_tensors[11] = runtime_graph->getCircleTensorByIndex(cell_to_output_weights_index);
  73
  74     const auto input_gate_bias_index = cur_op->inputs()->operator[](12);
  75     const auto forget_gate_bias_index = cur_op->inputs()->operator[](13);
  76     const auto cell_gate_bias_index = cur_op->inputs()->operator[](14);
  77     const auto output_gate_bias_index = cur_op->inputs()->operator[](15);
  78     // optional input_gate_bias_index
  79     assert(forget_gate_bias_index != -1);
  80     assert(cell_gate_bias_index != -1);
  81     assert(output_gate_bias_index != -1);
  82     internal_tensors[12] = runtime_graph->getCircleTensorByIndex(input_gate_bias_index);
  83     internal_tensors[13] = runtime_graph->getCircleTensorByIndex(forget_gate_bias_index);
  84     internal_tensors[14] = runtime_graph->getCircleTensorByIndex(cell_gate_bias_index);
  85     internal_tensors[15] = runtime_graph->getCircleTensorByIndex(output_gate_bias_index);
  86
  87     const auto projection_weights_index = cur_op->inputs()->operator[](16);
  88     const auto projection_bias_index = cur_op->inputs()->operator[](17);
  89     // optional projection_weights_index
  90     // optional projection_bias_index
  91     internal_tensors[16] = runtime_graph->getCircleTensorByIndex(projection_weights_index);
  92     internal_tensors[17] = runtime_graph->getCircleTensorByIndex(projection_bias_index);
  93
  94     const auto output_state_index = cur_op->inputs()->operator[](18);
  95     const auto cell_state_index = cur_op->inputs()->operator[](19);
  96     assert(output_state_index != -1);
  97     assert(cell_state_index != -1);
  98     internal_tensors[18] = runtime_graph->getCircleTensorByIndex(output_state_index);
  99     internal_tensors[19] = runtime_graph->getCircleTensorByIndex(cell_state_index);
 100
 101     const auto input_layer_norm_coefficients_index = cur_op->inputs()->operator[](20);
 102     const auto forget_layer_norm_coefficients_index = cur_op->inputs()->operator[](21);
 103     const auto cell_layer_norm_coefficients_index = cur_op->inputs()->operator[](22);
 104     const auto output_layer_norm_coefficients_index = cur_op->inputs()->operator[](23);
 105     // optional input_layer_norm_coefficients_index
 106     // optional forget_layer_norm_coefficients_index
 107     // optional cell_layer_norm_coefficients_index
 108     // optional output_layer_norm_coefficients_index
 109     internal_tensors[20] =
 110       runtime_graph->getCircleTensorByIndex(input_layer_norm_coefficients_index);
 111     internal_tensors[21] =
 112       runtime_graph->getCircleTensorByIndex(forget_layer_norm_coefficients_index);
 113     internal_tensors[22] =
 114       runtime_graph->getCircleTensorByIndex(cell_layer_norm_coefficients_index);
 115     internal_tensors[23] =
 116       runtime_graph->getCircleTensorByIndex(output_layer_norm_coefficients_index);
 117
 118     const auto output_index = cur_op->outputs()->operator[](0);
 119     assert(output_index != -1);
 120     output_internal = runtime_graph->getCircleTensorByIndex(output_index);
 121
 122     options = cur_op->builtin_options_as_UnidirectionalSequenceLSTMOptions();
 123   }
 124
 125   void validateTensorTypes()
 126   {
 127     LUCI_INTERPRETER_CHECK(Tensor::element_type(input()) == Tensor::element_type(output_state()));
 128     LUCI_INTERPRETER_CHECK(Tensor::element_type(output()) == Tensor::element_type(input()));
 129
 130     for (int32_t i = 1; i < 9; ++i)
 131     {
 132       LUCI_INTERPRETER_CHECK(internal_tensors[i] == nullptr or
 133                              Tensor::element_type(input_to_forget_weights()) ==
 134                                Tensor::element_type(internal_tensors[i]));
 135     }
 136
 137     for (int32_t i = 12; i < 16; ++i)
 138     {
 139       LUCI_INTERPRETER_CHECK(internal_tensors[i] == nullptr or
 140                              Tensor::element_type(forget_gate_bias()) ==
 141                                Tensor::element_type(internal_tensors[i]));
 142     }
 143   }
 144
 145   const circle::Tensor *input() { return internal_tensors[0]; };
 146
 147   const circle::Tensor *input_to_input_weights() { return internal_tensors[1]; };
 148   const circle::Tensor *input_to_forget_weights() { return internal_tensors[2]; };
 149   const circle::Tensor *input_to_cell_weights() { return internal_tensors[3]; };
 150   const circle::Tensor *input_to_output_weights() { return internal_tensors[4]; };
 151
 152   const circle::Tensor *recurrent_to_input_weights() { return internal_tensors[5]; };
 153   const circle::Tensor *recurrent_to_forget_weights() { return internal_tensors[6]; };
 154   const circle::Tensor *recurrent_to_cell_weights() { return internal_tensors[7]; };
 155   const circle::Tensor *recurrent_to_output_weights() { return internal_tensors[8]; };
 156
 157   const circle::Tensor *cell_to_input_weights() { return internal_tensors[9]; };
 158   const circle::Tensor *cell_to_forget_weights() { return internal_tensors[10]; };
 159   const circle::Tensor *cell_to_output_weights() { return internal_tensors[11]; };
 160
 161   const circle::Tensor *input_gate_bias() { return internal_tensors[12]; };
 162   const circle::Tensor *forget_gate_bias() { return internal_tensors[13]; };
 163   const circle::Tensor *cell_gate_bias() { return internal_tensors[14]; };
 164   const circle::Tensor *output_gate_bias() { return internal_tensors[15]; };
 165
 166   const circle::Tensor *projection_weights() { return internal_tensors[16]; };
 167   const circle::Tensor *projection_bias() { return internal_tensors[17]; };
 168
 169   const circle::Tensor *output_state() { return internal_tensors[18]; };
 170   const circle::Tensor *cell_state() { return internal_tensors[19]; };
 171
 172   const circle::Tensor *input_layer_norm_coefficients() { return internal_tensors[20]; };
 173   const circle::Tensor *forget_layer_norm_coefficients() { return internal_tensors[21]; };
 174   const circle::Tensor *cell_layer_norm_coefficients() { return internal_tensors[22]; };
 175   const circle::Tensor *output_layer_norm_coefficients() { return internal_tensors[23]; };
 176   const circle::Tensor *output() { return output_internal; };
 177
 178   const circle::UnidirectionalSequenceLSTMOptions *options;
 179
 180   const circle::Tensor *get_internal_tensor(int i) { return internal_tensors[i]; }
 181
 182 private:
 183   const circle::Tensor *output_internal;
 184   const circle::Tensor *internal_tensors[24];
 185 };
 186
 187 struct GateParameters
 188 {
 189   luci_interpreter_pal::FullyConnectedParams input_fc_params;
 190   luci_interpreter_pal::FullyConnectedParams recurrent_fc_params;
 191 };
 192
 193 struct InterGateParameters
 194 {
 195   luci_interpreter_pal::ArithmeticParams forget_cell_mul_params;
 196   luci_interpreter_pal::ArithmeticParams input_mul_params;
 197   luci_interpreter_pal::ArithmeticParams output_mul_params;
 198 };
 199
 200 struct CellStateInfo
 201 {
 202   float cell_clip;
 203   // clipping range for cell state only 16 bits cell is supported (could be
 204   // generalized through templatation)
 205   int16_t quantized_cell_clip;
 206   // 2^-cell_state_scale_power = cell state scale, required by integer tanh
 207   // computation
 208   int32_t cell_state_scale_power;
 209 };
 210
 211 struct LSTMParameters
 212 {
 213   GateParameters forget_gate_parameters;
 214   GateParameters input_gate_parameters;
 215   GateParameters cell_gate_parameters;
 216   GateParameters output_gate_parameters;
 217   InterGateParameters inter_gate_parameters;
 218 };
 219
 220 } // namespace lstm
 221 } // namespace luci_interpreter
 222
 223 #endif // LUCI_INTERPRETER_KERNELS_UNIDIRECTIONAL_SEQUENCE_LSTM_H