Support RNN operation (#5214)

This commit supports RNN operation by using acl cl.

Signed-off-by: jiseob.jang <jiseob.jang@samsung.com>

diff --git a/libs/ARMComputeEx/arm_compute/runtime/CL/CLFunctionsEx.h b/libs/ARMComputeEx/arm_compute/runtime/CL/CLFunctionsEx.h
index d9376fa..bc14406 100644 (file)
--- a/libs/ARMComputeEx/arm_compute/runtime/CL/CLFunctionsEx.h
+++ b/libs/ARMComputeEx/arm_compute/runtime/CL/CLFunctionsEx.h
@@ -35,6 +35,7 @@
 #include <arm_compute/runtime/CL/functions/CLPixelWiseDivision.h>
 #include <arm_compute/runtime/CL/functions/CLPReLU.h>
 #include <arm_compute/runtime/CL/functions/CLReduceOperation.h>
+#include <arm_compute/runtime/CL/functions/CLRNNLayerEx.h>
 #include <arm_compute/runtime/CL/functions/CLSpaceToBatchND.h>
 #include <arm_compute/runtime/CL/functions/CLSpaceToDepth.h>
 #include <arm_compute/runtime/CL/functions/CLSplit.h>

diff --git a/runtimes/neurun/backend/acl_cl/StageGenerator.cc b/runtimes/neurun/backend/acl_cl/StageGenerator.cc
index 3a50ed2..88eef88 100644 (file)
--- a/runtimes/neurun/backend/acl_cl/StageGenerator.cc
+++ b/runtimes/neurun/backend/acl_cl/StageGenerator.cc
@@ -1890,6 +1890,72 @@ void StageGenerator::visit(const model::operation::ReLU6Node &node)
   });
 }
 
+void StageGenerator::visit(const model::operation::RNNNode &node)
+{
+  const auto output_index{node.getOutputs().at(model::operation::RNNNode::Output::OUTPUT)};
+  const auto hidden_state_out_index{
+      node.getOutputs().at(model::operation::RNNNode::Output::HIDDEN_STATE_OUT)};
+
+  const auto input_index{node.getInputs().at(model::operation::RNNNode::Input::INPUT)};
+  const auto weights_index{node.getInputs().at(model::operation::RNNNode::Input::WEIGHTS)};
+  const auto recurrent_weights_index{
+      node.getInputs().at(model::operation::RNNNode::Input::RECURRENT_WEIGHTS)};
+  const auto bias_index{node.getInputs().at(model::operation::RNNNode::Input::BIAS)};
+  const auto hidden_state_in_index{
+      node.getInputs().at(model::operation::RNNNode::Input::HIDDEN_STATE_IN)};
+
+  struct Param
+  {
+    model::OperandIndex output_index;
+    model::OperandIndex hidden_state_out_index;
+
+    model::OperandIndex input_index;
+    model::OperandIndex weights_index;
+    model::OperandIndex recurrent_weights_index;
+    model::OperandIndex bias_index;
+    model::OperandIndex hidden_state_in_index;
+    model::Activation activation;
+  };
+
+  Param param;
+
+  param.output_index = output_index;
+  param.hidden_state_out_index = hidden_state_out_index;
+
+  param.input_index = input_index;
+  param.weights_index = weights_index;
+  param.recurrent_weights_index = recurrent_weights_index;
+  param.bias_index = bias_index;
+  param.hidden_state_in_index = hidden_state_in_index;
+  param.activation = node.param().activation;
+
+  auto tensors = _tensor_builder;
+
+  returnStage([tensors, param](IExecutionBuilder &builder) {
+    auto output_alloc = tensors->at(param.output_index).get();
+    auto hidden_state_out_alloc = tensors->at(param.hidden_state_out_index).get();
+
+    auto input_alloc = tensors->at(param.input_index).get();
+    auto weights_alloc = tensors->at(param.weights_index).get();
+    auto recurrent_weights_alloc = tensors->at(param.recurrent_weights_index).get();
+    auto bias_alloc = tensors->at(param.bias_index).get();
+    auto hidden_state_in_alloc = tensors->at(param.hidden_state_in_index).get();
+    auto act_info = ::neurun::backend::acl_common::asActivationLayerInfo(param.activation);
+
+    auto copy_layer = make_layer<::arm_compute::CLCopy>();
+    copy_layer->configure(hidden_state_in_alloc->handle(), hidden_state_out_alloc->handle());
+    builder.append(asAclFunction(std::move(copy_layer)));
+
+    std::unique_ptr<::arm_compute::IFunction> fn;
+    auto rnn_layer = make_layer<::arm_compute::CLRNNLayerEx>();
+    rnn_layer->configure(input_alloc->handle(), weights_alloc->handle(),
+                         recurrent_weights_alloc->handle(), bias_alloc->handle(),
+                         hidden_state_out_alloc->handle(), output_alloc->handle(), act_info);
+    fn = std::move(rnn_layer);
+    builder.append(asAclFunction(std::move(fn)));
+  });
+}
+
 void StageGenerator::visit(const model::operation::FloorNode &node)
 {
   const auto ofm_index{node.getOutputs().at(0)};

diff --git a/runtimes/neurun/backend/acl_cl/StageGenerator.h b/runtimes/neurun/backend/acl_cl/StageGenerator.h
index 5ae6796..9f93a70 100644 (file)
--- a/runtimes/neurun/backend/acl_cl/StageGenerator.h
+++ b/runtimes/neurun/backend/acl_cl/StageGenerator.h
@@ -65,6 +65,7 @@ public:
   void visit(const model::operation::ResizeBilinearNode &) override;
   void visit(const model::operation::ReLU1Node &) override;
   void visit(const model::operation::ReLU6Node &) override;
+  void visit(const model::operation::RNNNode &) override;
   void visit(const model::operation::FloorNode &) override;
   void visit(const model::operation::SpaceToDepthNode &) override;
   void visit(const model::operation::L2Pool2DNode &) override;

diff --git a/runtimes/neurun/backend/acl_common/Convert.cc b/runtimes/neurun/backend/acl_common/Convert.cc
index b2e6633..9787ccb 100644 (file)
--- a/runtimes/neurun/backend/acl_common/Convert.cc
+++ b/runtimes/neurun/backend/acl_common/Convert.cc
@@ -130,7 +130,8 @@ namespace acl_common
                                       ::arm_compute::DimensionRoundingType::FLOOR};
 }
 
-::arm_compute::ActivationLayerInfo asActivationLayerInfo(::neurun::model::Activation &act_code)
+::arm_compute::ActivationLayerInfo
+asActivationLayerInfo(const ::neurun::model::Activation &act_code)
 {
   switch (act_code)
   {

diff --git a/runtimes/neurun/backend/acl_common/Convert.h b/runtimes/neurun/backend/acl_common/Convert.h
index 97106be..40ff4c4 100644 (file)
--- a/runtimes/neurun/backend/acl_common/Convert.h
+++ b/runtimes/neurun/backend/acl_common/Convert.h
@@ -52,7 +52,8 @@ namespace acl_common
 ::arm_compute::PadStrideInfo asPadStrideInfo(const neurun::util::Padding &padding,
                                              const neurun::util::Stride &stride);
 
-::arm_compute::ActivationLayerInfo asActivationLayerInfo(::neurun::model::Activation act_code);
+::arm_compute::ActivationLayerInfo
+asActivationLayerInfo(const ::neurun::model::Activation &act_code);
 
 std::unique_ptr<AclFunction> asAclFunction(std::unique_ptr<::arm_compute::IFunction> &&layer);
 

diff --git a/runtimes/neurun/core/include/model/Operations.Include.h b/runtimes/neurun/core/include/model/Operations.Include.h
index e0f13df..591cf2f 100644 (file)
--- a/runtimes/neurun/core/include/model/Operations.Include.h
+++ b/runtimes/neurun/core/include/model/Operations.Include.h
@@ -47,6 +47,7 @@
 #include "operation/ResizeBilinearNode.h"
 #include "operation/ReLU1Node.h"
 #include "operation/ReLU6Node.h"
+#include "operation/RNNNode.h"
 #include "operation/FloorNode.h"
 #include "operation/SpaceToDepthNode.h"
 #include "operation/L2Pool2DNode.h"

diff --git a/runtimes/neurun/core/include/model/Operations.lst b/runtimes/neurun/core/include/model/Operations.lst
index fd87d55..472d57f 100644 (file)
--- a/runtimes/neurun/core/include/model/Operations.lst
+++ b/runtimes/neurun/core/include/model/Operations.lst
@@ -51,6 +51,7 @@ OP(ReLUNode                       , true)
 OP(ResizeBilinearNode             , true)
 OP(ReLU1Node                      , true)
 OP(ReLU6Node                      , true)
+OP(RNNNode                        , true)
 OP(FloorNode                      , true)
 OP(SpaceToDepthNode               , true)
 OP(L2Pool2DNode                   , true)

diff --git a/runtimes/neurun/core/include/model/operation/RNNNode.h b/runtimes/neurun/core/include/model/operation/RNNNode.h
new file mode 100644 (file)
index 0000000..a40597f
--- /dev/null
+++ b/runtimes/neurun/core/include/model/operation/RNNNode.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef __NEURUN_MODEL_OPERATION_RNN_NODE_H__
+#define __NEURUN_MODEL_OPERATION_RNN_NODE_H__
+
+#include "model/InternalType.h"
+#include "model/Operation.h"
+
+namespace neurun
+{
+namespace model
+{
+namespace operation
+{
+
+class RNNNode : public model::Operation
+{
+public:
+  enum Input
+  {
+    INPUT = 0,
+    WEIGHTS = 1,
+    RECURRENT_WEIGHTS = 2,
+    BIAS = 3,
+    HIDDEN_STATE_IN = 4
+  };
+
+  enum Output
+  {
+    OUTPUT = 0,
+    HIDDEN_STATE_OUT = 1
+  };
+
+  struct Param
+  {
+    Activation activation;
+  };
+
+public:
+  RNNNode(const OperandIndexSequence &inputs, const OperandIndexSequence &outputs,
+          const Param &param);
+
+public:
+  virtual void accept(OperationVisitor &&) const override;
+  virtual std::string getName() const override { return "RNN"; }
+
+public:
+  const Param &param() const { return _param; }
+
+private:
+  Param _param;
+};
+
+} // namespace operation
+} // namespace model
+} // namespace neurun
+
+#endif // __NEURUN_MODEL_OPERATION_RNN_NODE_H__

diff --git a/runtimes/neurun/core/src/compiler/OperationValidator.cc b/runtimes/neurun/core/src/compiler/OperationValidator.cc
index 927f7dc..209b0e4 100644 (file)
--- a/runtimes/neurun/core/src/compiler/OperationValidator.cc
+++ b/runtimes/neurun/core/src/compiler/OperationValidator.cc
@@ -208,6 +208,56 @@ void OperationValidator::visit(const model::operation::ReduceMaxNode &node)
   }
 }
 
+void OperationValidator::visit(const model::operation::RNNNode &node)
+{
+  // NOTE This validation is for static rnn(non-dynamic shape), but not for dynamic rnn
+  // TODO Support dynamic rnn
+  const auto output_index{node.getOutputs().at(model::operation::RNNNode::Output::OUTPUT)};
+  const auto hidden_state_out_index{
+      node.getOutputs().at(model::operation::RNNNode::Output::HIDDEN_STATE_OUT)};
+
+  const auto input_index{node.getInputs().at(model::operation::RNNNode::Input::INPUT)};
+  const auto weights_index{node.getInputs().at(model::operation::RNNNode::Input::WEIGHTS)};
+  const auto recurrent_weights_index{
+      node.getInputs().at(model::operation::RNNNode::Input::RECURRENT_WEIGHTS)};
+  const auto bias_index{node.getInputs().at(model::operation::RNNNode::Input::BIAS)};
+  const auto hidden_state_in_index{
+      node.getInputs().at(model::operation::RNNNode::Input::HIDDEN_STATE_IN)};
+
+  const auto batch_size = _ctx.at(output_index).shape().dim(0);
+  const auto num_units = _ctx.at(output_index).shape().dim(1);
+
+  UNUSED_RELEASE(output_index);
+  UNUSED_RELEASE(hidden_state_out_index);
+  UNUSED_RELEASE(input_index);
+  UNUSED_RELEASE(weights_index);
+  UNUSED_RELEASE(recurrent_weights_index);
+  UNUSED_RELEASE(bias_index);
+  UNUSED_RELEASE(hidden_state_in_index);
+  UNUSED_RELEASE(batch_size);
+  UNUSED_RELEASE(num_units);
+
+  assert(_ctx.at(output_index).shape().rank() == 2 &&
+         _ctx.at(hidden_state_out_index).shape().rank() == 2 &&
+         _ctx.at(input_index).shape().rank() == 2 && _ctx.at(weights_index).shape().rank() == 2 &&
+         _ctx.at(recurrent_weights_index).shape().rank() == 2 &&
+         _ctx.at(hidden_state_in_index).shape().rank() == 2);
+  assert(_ctx.at(bias_index).shape().rank() == 1);
+
+  assert(batch_size == _ctx.at(input_index).shape().dim(0) &&
+         batch_size == _ctx.at(hidden_state_in_index).shape().dim(0) &&
+         batch_size == _ctx.at(hidden_state_out_index).shape().dim(0));
+  assert(_ctx.at(input_index).shape().dim(1) == _ctx.at(weights_index).shape().dim(1));
+
+  assert(num_units == _ctx.at(weights_index).shape().dim(0) &&
+         num_units == _ctx.at(recurrent_weights_index).shape().dim(0) &&
+         num_units == _ctx.at(bias_index).shape().dim(0));
+  assert(num_units == _ctx.at(output_index).shape().dim(1) &&
+         num_units == _ctx.at(recurrent_weights_index).shape().dim(1) &&
+         num_units == _ctx.at(hidden_state_in_index).shape().dim(1) &&
+         num_units == _ctx.at(hidden_state_out_index).shape().dim(1));
+}
+
 void OperationValidator::visit(const model::operation::SpaceToDepthNode &node)
 {
   const auto ofm_index{node.getOutputs().at(0)};

diff --git a/runtimes/neurun/core/src/compiler/OperationValidator.h b/runtimes/neurun/core/src/compiler/OperationValidator.h
index 4f474f1..769190e 100644 (file)
--- a/runtimes/neurun/core/src/compiler/OperationValidator.h
+++ b/runtimes/neurun/core/src/compiler/OperationValidator.h
@@ -45,6 +45,7 @@ public:
   void visit(const model::operation::ReduceSumNode &node) override;
   void visit(const model::operation::TransposeNode &node) override;
   void visit(const model::operation::ReduceMaxNode &node) override;
+  void visit(const model::operation::RNNNode &node) override;
   void visit(const model::operation::SpaceToDepthNode &node) override;
   void visit(const model::operation::EmbeddingLookupNode &node) override;
   void visit(const model::operation::HashtableLookupNode &node) override;

diff --git a/runtimes/neurun/core/src/graph/dumper/Dumper.cc b/runtimes/neurun/core/src/graph/dumper/Dumper.cc
index 92514a4..372f9d4 100644 (file)
--- a/runtimes/neurun/core/src/graph/dumper/Dumper.cc
+++ b/runtimes/neurun/core/src/graph/dumper/Dumper.cc
@@ -373,6 +373,20 @@ void Dumper::visit(const ResizeBilinearNode &node)
   VERBOSE(LIR) << "  - Output : Output(" << node.getOutputs().at(0).value() << ")" << std::endl;
 }
 
+void Dumper::visit(const RNNNode &node)
+{
+  VERBOSE(LIR) << "* RNN" << std::endl;
+  VERBOSE(LIR) << "  - Inputs : Input(" << node.getInputs().at(RNNNode::Input::INPUT).value()
+               << ") Weights" << node.getInputs().at(RNNNode::Input::WEIGHTS).value()
+               << ") Recurrent Weights"
+               << node.getInputs().at(RNNNode::Input::RECURRENT_WEIGHTS).value() << ") Bias"
+               << node.getInputs().at(RNNNode::Input::BIAS).value() << ") Hidden State"
+               << node.getInputs().at(RNNNode::Input::HIDDEN_STATE_IN).value() << ")" << std::endl;
+  VERBOSE(LIR) << "  - Output : Output(" << node.getOutputs().at(RNNNode::Output::OUTPUT).value()
+               << ") Hidden State" << node.getInputs().at(RNNNode::Output::HIDDEN_STATE_OUT).value()
+               << ")" << std::endl;
+}
+
 void Dumper::visit(const RSQRTNode &node)
 {
   VERBOSE(LIR) << "* RSQRT" << std::endl;

diff --git a/runtimes/neurun/core/src/graph/dumper/Dumper.h b/runtimes/neurun/core/src/graph/dumper/Dumper.h
index 1bb1cfd..3462417 100644 (file)
--- a/runtimes/neurun/core/src/graph/dumper/Dumper.h
+++ b/runtimes/neurun/core/src/graph/dumper/Dumper.h
@@ -71,6 +71,7 @@ public:
   void visit(const model::operation::ReLU6Node &) override;
   void visit(const model::operation::ReshapeNode &node) override;
   void visit(const model::operation::ResizeBilinearNode &) override;
+  void visit(const model::operation::RNNNode &) override;
   void visit(const model::operation::RSQRTNode &) override;
   void visit(const model::operation::SoftmaxNode &node) override;
   void visit(const model::operation::SpaceToDepthNode &) override;

diff --git a/runtimes/neurun/core/src/model/operation/RNNNode.cc b/runtimes/neurun/core/src/model/operation/RNNNode.cc
new file mode 100644 (file)
index 0000000..3e47605
--- /dev/null
+++ b/runtimes/neurun/core/src/model/operation/RNNNode.cc
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "model/operation/RNNNode.h"
+
+#include <cassert>
+
+#include "model/OperationVisitor.h"
+
+namespace neurun
+{
+namespace model
+{
+namespace operation
+{
+
+void RNNNode::accept(OperationVisitor &&v) const { v.visit(*this); }
+
+RNNNode::RNNNode(const OperandIndexSequence &inputs, const OperandIndexSequence &outputs,
+                 const Param &param)
+    : model::Operation{OperandConstraint::createExact(5u), inputs, outputs}, _param{param}
+{
+}
+
+} // namespace operation
+} // namespace model
+} // namespace neurun

diff --git a/runtimes/neurun/frontend/nnapi/wrapper/OperationFactory.cc b/runtimes/neurun/frontend/nnapi/wrapper/OperationFactory.cc
index a59854a..cd0c9ac 100644 (file)
--- a/runtimes/neurun/frontend/nnapi/wrapper/OperationFactory.cc
+++ b/runtimes/neurun/frontend/nnapi/wrapper/OperationFactory.cc
@@ -826,6 +826,38 @@ OperationFactory::OperationFactory()
     return new operation::ReLU6Node{inputs, outputs};
   };
 
+  _map[ANEURALNETWORKS_RNN] = [](const OperationFactory::Param &init_param,
+                                 neurun::model::Operands &operands) {
+    assert(init_param.input_count == 6 && init_param.output_count == 2);
+
+    // Each input should be interpreted as follows:
+    //
+    // 0 -> Input Tensor Index
+    // 1 -> Weights Tensor Index
+    // 2 -> Recurrent Weights Tensor Index
+    // 3 -> Bias Tensor Index
+    // 4 -> Hidden state (in) Index
+    // 5 -> Activation Index
+
+    OperandIndexSequence inputs;
+    for (uint32_t n = 0; n < init_param.input_count - 1; ++n)
+    {
+      inputs.append(OperandIndex{init_param.inputs[n]});
+    }
+    OperandIndexSequence outputs;
+    for (uint32_t n = 0; n < init_param.output_count; ++n)
+    {
+      outputs.append(OperandIndex{init_param.outputs[n]});
+    }
+
+    operation::RNNNode::Param param;
+    const auto activation_index = OperandIndex{init_param.inputs[5]};
+    param.activation =
+        NNAPIConvert::getFusedActivation(operands.at(activation_index).asScalar<FuseCode>());
+
+    return new operation::RNNNode{inputs, outputs, param};
+  };
+
   _map[ANEURALNETWORKS_FLOOR] = [](const OperationFactory::Param &init_param,
                                    neurun::model::Operands &) {
     assert(init_param.input_count == 1 && init_param.output_count == 1);

diff --git a/tests/nnapi/nnapi_gtest.skip.armv7l-linux b/tests/nnapi/nnapi_gtest.skip.armv7l-linux
index 1b1e995..a177150 100644 (file)
--- a/tests/nnapi/nnapi_gtest.skip.armv7l-linux
+++ b/tests/nnapi/nnapi_gtest.skip.armv7l-linux
@@ -5,7 +5,6 @@
 GeneratedTests.lsh_projection*
 GeneratedTests.lstm*
 GeneratedTests.mobilenet*
-GeneratedTests.rnn*
 GeneratedTests.pad*
 GeneratedTests.svdf*
 GeneratedTests.batch_to_space*