src/armnn/layers/LstmLayer.cpp

   1 //
   2 // Copyright © 2017 Arm Ltd. All rights reserved.
   3 // SPDX-License-Identifier: MIT
   4 //
   5 #include "LstmLayer.hpp"
   6
   7 #include "LayerCloneBase.hpp"
   8
   9 #include <armnn/TypesUtils.hpp>
  10 #include <backendsCommon/CpuTensorHandle.hpp>
  11 #include <backendsCommon/WorkloadFactory.hpp>
  12
  13 namespace armnn
  14 {
  15
  16 LstmLayer::LstmLayer(const LstmDescriptor& param, const char* name)
  17         : LayerWithParameters(3, 4, LayerType::Lstm, param, name)
  18 {
  19 }
  20
  21 std::unique_ptr<IWorkload> LstmLayer::CreateWorkload(const Graph& graph, const IWorkloadFactory& factory) const
  22 {
  23     LstmQueueDescriptor descriptor;
  24
  25     // Basic parameters
  26     descriptor.m_InputToForgetWeights = m_BasicParameters.m_InputToForgetWeights.get();
  27     descriptor.m_InputToCellWeights = m_BasicParameters.m_InputToCellWeights.get();
  28     descriptor.m_InputToOutputWeights = m_BasicParameters.m_InputToOutputWeights.get();
  29     descriptor.m_RecurrentToForgetWeights = m_BasicParameters.m_RecurrentToForgetWeights.get();
  30     descriptor.m_RecurrentToCellWeights = m_BasicParameters.m_RecurrentToCellWeights.get();
  31     descriptor.m_RecurrentToOutputWeights = m_BasicParameters.m_RecurrentToOutputWeights.get();
  32     descriptor.m_ForgetGateBias = m_BasicParameters.m_ForgetGateBias.get();
  33     descriptor.m_CellBias = m_BasicParameters.m_CellBias.get();
  34     descriptor.m_OutputGateBias = m_BasicParameters.m_OutputGateBias.get();
  35
  36     // Cifg parameters
  37     if (!m_Param.m_CifgEnabled)
  38     {
  39         descriptor.m_InputToInputWeights = m_CifgParameters.m_InputToInputWeights.get();
  40         descriptor.m_RecurrentToInputWeights = m_CifgParameters.m_RecurrentToInputWeights.get();
  41         descriptor.m_CellToInputWeights = m_CifgParameters.m_CellToInputWeights.get();
  42         descriptor.m_InputGateBias = m_CifgParameters.m_InputGateBias.get();
  43     }
  44
  45     // Projection parameters
  46     if (m_Param.m_ProjectionEnabled)
  47     {
  48         descriptor.m_ProjectionWeights = m_ProjectionParameters.m_ProjectionWeights.get();
  49         descriptor.m_ProjectionBias    = m_ProjectionParameters.m_ProjectionBias.get();
  50     }
  51
  52     // Peephole parameters
  53     if (m_Param.m_PeepholeEnabled)
  54     {
  55         descriptor.m_CellToForgetWeights = m_PeepholeParameters.m_CellToForgetWeights.get();
  56         descriptor.m_CellToOutputWeights = m_PeepholeParameters.m_CellToOutputWeights.get();
  57     }
  58     return factory.CreateLstm(descriptor, PrepInfoAndDesc(descriptor, graph));
  59 }
  60
  61 LstmLayer* LstmLayer::Clone(Graph& graph) const
  62 {
  63     auto layer = CloneBase<LstmLayer>(graph, m_Param, GetName());
  64
  65     layer->m_BasicParameters.m_InputToForgetWeights = m_BasicParameters.m_InputToForgetWeights ?
  66             std::make_unique<ScopedCpuTensorHandle>(*m_BasicParameters.m_InputToForgetWeights)
  67                 : nullptr;
  68     layer->m_BasicParameters.m_InputToCellWeights = m_BasicParameters.m_InputToCellWeights ?
  69             std::make_unique<ScopedCpuTensorHandle>(*m_BasicParameters.m_InputToCellWeights) : nullptr;
  70     layer->m_BasicParameters.m_InputToOutputWeights = m_BasicParameters.m_InputToOutputWeights ?
  71             std::make_unique<ScopedCpuTensorHandle>(*m_BasicParameters.m_InputToOutputWeights) : nullptr;
  72     layer->m_BasicParameters.m_RecurrentToForgetWeights = m_BasicParameters.m_RecurrentToForgetWeights ?
  73             std::make_unique<ScopedCpuTensorHandle>(*m_BasicParameters.m_RecurrentToForgetWeights) : nullptr;
  74     layer->m_BasicParameters.m_RecurrentToCellWeights = m_BasicParameters.m_RecurrentToCellWeights ?
  75             std::make_unique<ScopedCpuTensorHandle>(*m_BasicParameters.m_RecurrentToCellWeights) : nullptr;
  76     layer->m_BasicParameters.m_RecurrentToOutputWeights = m_BasicParameters.m_RecurrentToOutputWeights ?
  77             std::make_unique<ScopedCpuTensorHandle>(*m_BasicParameters.m_RecurrentToOutputWeights) : nullptr;
  78     layer->m_BasicParameters.m_ForgetGateBias = m_BasicParameters.m_ForgetGateBias ?
  79             std::make_unique<ScopedCpuTensorHandle>(*m_BasicParameters.m_ForgetGateBias) : nullptr;
  80     layer->m_BasicParameters.m_CellBias = m_BasicParameters.m_CellBias ?
  81             std::make_unique<ScopedCpuTensorHandle>(*m_BasicParameters.m_CellBias) : nullptr;
  82     layer->m_BasicParameters.m_OutputGateBias = m_BasicParameters.m_OutputGateBias ?
  83             std::make_unique<ScopedCpuTensorHandle>(*m_BasicParameters.m_OutputGateBias) : nullptr;
  84
  85     if (!m_Param.m_CifgEnabled)
  86     {
  87         layer->m_CifgParameters.m_InputToInputWeights = m_CifgParameters.m_InputToInputWeights ?
  88                 std::make_unique<ScopedCpuTensorHandle>(*m_CifgParameters.m_InputToInputWeights) : nullptr;
  89         layer->m_CifgParameters.m_RecurrentToInputWeights = m_CifgParameters.m_RecurrentToInputWeights ?
  90                 std::make_unique<ScopedCpuTensorHandle>(*m_CifgParameters.m_RecurrentToInputWeights) : nullptr;
  91         layer->m_CifgParameters.m_CellToInputWeights = m_CifgParameters.m_CellToInputWeights ?
  92                 std::make_unique<ScopedCpuTensorHandle>(*m_CifgParameters.m_CellToInputWeights) : nullptr;
  93         layer->m_CifgParameters.m_InputGateBias = m_CifgParameters.m_InputGateBias ?
  94                 std::make_unique<ScopedCpuTensorHandle>(*m_CifgParameters.m_InputGateBias) : nullptr;
  95     }
  96
  97     if (m_Param.m_ProjectionEnabled)
  98     {
  99         layer->m_ProjectionParameters.m_ProjectionWeights = m_ProjectionParameters.m_ProjectionWeights ?
 100                std::make_unique<ScopedCpuTensorHandle>(*m_ProjectionParameters.m_ProjectionWeights) : nullptr;
 101         layer->m_ProjectionParameters.m_ProjectionBias = m_ProjectionParameters.m_ProjectionBias ?
 102                std::make_unique<ScopedCpuTensorHandle>(*m_ProjectionParameters.m_ProjectionBias) : nullptr;
 103     }
 104
 105     if (m_Param.m_PeepholeEnabled)
 106     {
 107         layer->m_PeepholeParameters.m_CellToForgetWeights = m_PeepholeParameters.m_CellToForgetWeights ?
 108                std::make_unique<ScopedCpuTensorHandle>(*m_PeepholeParameters.m_CellToForgetWeights) : nullptr;
 109         layer->m_PeepholeParameters.m_CellToOutputWeights = m_PeepholeParameters.m_CellToOutputWeights ?
 110                std::make_unique<ScopedCpuTensorHandle>(*m_PeepholeParameters.m_CellToOutputWeights) : nullptr;
 111     }
 112
 113     return std::move(layer);
 114 }
 115
 116 std::vector<TensorShape> LstmLayer::InferOutputShapes(const std::vector<TensorShape>& inputShapes) const
 117 {
 118     BOOST_ASSERT(inputShapes.size() == 3);
 119
 120     // Get input values for validation
 121     unsigned int batchSize = inputShapes[0][0];
 122     unsigned int outputSize = inputShapes[1][1];
 123     unsigned int numUnits = inputShapes[2][1];
 124
 125     std::vector<TensorShape> outShapes;
 126     if (!m_Param.m_CifgEnabled)
 127     {
 128         outShapes.push_back(TensorShape({batchSize, numUnits*3}));
 129     }
 130     else
 131     {
 132         outShapes.push_back(TensorShape({batchSize, numUnits*4}));
 133     }
 134     outShapes.push_back(TensorShape({batchSize, outputSize}));
 135     outShapes.push_back(TensorShape({batchSize, numUnits}));
 136     outShapes.push_back(TensorShape({batchSize, outputSize}));
 137
 138     return outShapes;
 139 }
 140
 141 void LstmLayer::ValidateTensorShapesFromInputs()
 142 {
 143     VerifyLayerConnections(3, CHECK_LOCATION());
 144
 145     auto inferredShapes = InferOutputShapes( {
 146         GetInputSlot(0).GetConnection()->GetTensorInfo().GetShape(),
 147         GetInputSlot(1).GetConnection()->GetTensorInfo().GetShape(),
 148         GetInputSlot(2).GetConnection()->GetTensorInfo().GetShape()}
 149     );
 150
 151     BOOST_ASSERT(inferredShapes.size() == 4);
 152
 153     // Check if the weights are nullptr
 154     BOOST_ASSERT_MSG(m_BasicParameters.m_InputToForgetWeights != nullptr,
 155                      "LstmLayer: m_BasicParameters.m_InputToForgetWeights should not be null.");
 156     BOOST_ASSERT_MSG(m_BasicParameters.m_InputToCellWeights != nullptr,
 157                      "LstmLayer: m_BasicParameters.m_InputToCellWeights should not be null.");
 158     BOOST_ASSERT_MSG(m_BasicParameters.m_InputToOutputWeights != nullptr,
 159                      "LstmLayer: m_BasicParameters.m_InputToOutputWeights should not be null.");
 160     BOOST_ASSERT_MSG(m_BasicParameters.m_RecurrentToForgetWeights != nullptr,
 161                      "LstmLayer: m_BasicParameters.m_RecurrentToForgetWeights should not be null.");
 162     BOOST_ASSERT_MSG(m_BasicParameters.m_RecurrentToCellWeights != nullptr,
 163                      "LstmLayer: m_BasicParameters.m_RecurrentToCellWeights should not be null.");
 164     BOOST_ASSERT_MSG(m_BasicParameters.m_RecurrentToOutputWeights != nullptr,
 165                      "LstmLayer: m_BasicParameters.m_RecurrentToOutputWeights should not be null.");
 166     BOOST_ASSERT_MSG(m_BasicParameters.m_ForgetGateBias != nullptr,
 167                      "LstmLayer: m_BasicParameters.m_ForgetGateBias should not be null.");
 168     BOOST_ASSERT_MSG(m_BasicParameters.m_CellBias != nullptr,
 169                      "LstmLayer: m_BasicParameters.m_CellBias should not be null.");
 170     BOOST_ASSERT_MSG(m_BasicParameters.m_OutputGateBias != nullptr,
 171                      "LstmLayer: m_BasicParameters.m_OutputGateBias should not be null.");
 172
 173     if (!m_Param.m_CifgEnabled)
 174     {
 175         BOOST_ASSERT_MSG(m_CifgParameters.m_InputToInputWeights != nullptr,
 176                          "LstmLayer: m_CifgParameters.m_InputToInputWeights should not be null.");
 177         BOOST_ASSERT_MSG(m_CifgParameters.m_RecurrentToInputWeights != nullptr,
 178                          "LstmLayer: m_CifgParameters.m_RecurrentToInputWeights should not be null.");
 179         BOOST_ASSERT_MSG(m_CifgParameters.m_InputGateBias != nullptr,
 180                          "LstmLayer: m_CifgParameters.m_InputGateBias should not be null.");
 181
 182         ConditionalThrowIfNotEqual<LayerValidationException>(
 183                 "LstmLayer: TensorShape set on OutputSlot[0] does not match the inferred shape.",
 184                 GetOutputSlot(0).GetTensorInfo().GetShape(),
 185                 inferredShapes[0]);
 186     }
 187     else
 188     {
 189         BOOST_ASSERT_MSG(m_CifgParameters.m_InputToInputWeights == nullptr,
 190             "LstmLayer: m_CifgParameters.m_InputToInputWeights should not have a value when CIFG is enabled.");
 191         BOOST_ASSERT_MSG(m_CifgParameters.m_RecurrentToInputWeights == nullptr,
 192             "LstmLayer: m_CifgParameters.m_RecurrentToInputWeights should not have a value when CIFG is enabled.");
 193         BOOST_ASSERT_MSG(m_CifgParameters.m_CellToInputWeights == nullptr,
 194              "LstmLayer: m_CifgParameters.m_CellToInputWeights should not have a value when CIFG is enabled.");
 195         BOOST_ASSERT_MSG(m_CifgParameters.m_InputGateBias == nullptr,
 196             "LstmLayer: m_CifgParameters.m_InputGateBias should not have a value when CIFG is enabled.");
 197
 198         ConditionalThrowIfNotEqual<LayerValidationException>(
 199                 "LstmLayer: TensorShape set on OutputSlot[0] does not match the inferred shape.",
 200                 GetOutputSlot(0).GetTensorInfo().GetShape(),
 201                 inferredShapes[0]);
 202     }
 203
 204     if (m_Param.m_ProjectionEnabled)
 205     {
 206         BOOST_ASSERT_MSG(m_ProjectionParameters.m_ProjectionWeights != nullptr,
 207                          "LstmLayer: m_ProjectionParameters.m_ProjectionWeights should not be null.");
 208     }
 209
 210     if (m_Param.m_PeepholeEnabled)
 211     {
 212         BOOST_ASSERT_MSG(m_PeepholeParameters.m_CellToForgetWeights != nullptr,
 213                          "LstmLayer: m_PeepholeParameters.m_CellToForgetWeights should not be null.");
 214         BOOST_ASSERT_MSG(m_PeepholeParameters.m_CellToOutputWeights != nullptr,
 215                          "LstmLayer: m_PeepholeParameters.m_CellToOutputWeights should not be null.");
 216     }
 217
 218     ConditionalThrowIfNotEqual<LayerValidationException>(
 219             "LstmLayer: TensorShape set on OutputSlot[1] does not match the inferred shape.",
 220             GetOutputSlot(1).GetTensorInfo().GetShape(),
 221             inferredShapes[1]);
 222     ConditionalThrowIfNotEqual<LayerValidationException>(
 223             "LstmLayer: TensorShape set on OutputSlot[2] does not match the inferred shape.",
 224             GetOutputSlot(2).GetTensorInfo().GetShape(),
 225             inferredShapes[2]);
 226     ConditionalThrowIfNotEqual<LayerValidationException>(
 227             "LstmLayer: TensorShape set on OutputSlot[3] does not match the inferred shape.",
 228             GetOutputSlot(3).GetTensorInfo().GetShape(),
 229             inferredShapes[3]);
 230 }
 231
 232 Layer::ConstantTensors LstmLayer::GetConstantTensorsByRef()
 233 {
 234     return {m_BasicParameters.m_InputToForgetWeights,
 235             m_BasicParameters.m_InputToCellWeights,
 236             m_BasicParameters.m_InputToOutputWeights,
 237             m_BasicParameters.m_RecurrentToForgetWeights,
 238             m_BasicParameters.m_RecurrentToCellWeights,
 239             m_BasicParameters.m_RecurrentToOutputWeights,
 240             m_BasicParameters.m_ForgetGateBias,
 241             m_BasicParameters.m_CellBias,
 242             m_BasicParameters.m_OutputGateBias,
 243
 244             // Cifg parameters
 245             m_CifgParameters.m_InputToInputWeights,
 246             m_CifgParameters.m_RecurrentToInputWeights,
 247             m_CifgParameters.m_CellToInputWeights,
 248             m_CifgParameters.m_InputGateBias,
 249
 250             // Projection parameters
 251             m_ProjectionParameters.m_ProjectionWeights,
 252             m_ProjectionParameters.m_ProjectionBias,
 253
 254             // Peephole parameters
 255             m_PeepholeParameters.m_CellToForgetWeights,
 256             m_PeepholeParameters.m_CellToOutputWeights};
 257 }
 258
 259 } // namespace armnn