From 72032552b84d538809773dcd3e7c71b33617453d Mon Sep 17 00:00:00 2001 From: =?utf8?q?Vladimir=20Plazun/AI=20Tools=20Lab=20/SRR/Engineer/?= =?utf8?q?=EC=82=BC=EC=84=B1=EC=A0=84=EC=9E=90?= Date: Wed, 9 Jan 2019 14:46:20 +0300 Subject: [PATCH] [nnc] Add LeakyRelu activation (#2757) * [nnc] Add LeakyRelu activation Implement leaky relu activation function( relu with negative slope ) Signed-off-by: Vladimir Plazun --- contrib/nnc/core/modelIR/IrDotDumper.cpp | 10 +++++ contrib/nnc/core/modelIR/Operation.cpp | 1 + contrib/nnc/include/core/modelIR/IrDotDumper.h | 1 + .../include/core/modelIR/operations/LeakyReluOp.h | 46 ++++++++++++++++++++++ .../core/modelIR/operations/operations.lst.h | 1 + .../nnc/include/passes/interpreter/Interpreter.h | 1 + .../passes/acl_soft_backend/AclCppOpGenerator.cpp | 5 +++ .../passes/acl_soft_backend/AclCppOpGenerator.h | 1 + .../nnc/passes/caffe_frontend/caffe_op_creator.cpp | 13 ++++-- contrib/nnc/passes/interpreter/Interpreter.cpp | 14 +++++++ contrib/nnc/passes/soft_backend/CPPGenerator.cpp | 2 + contrib/nnc/passes/soft_backend/ModelAnalyzer.cpp | 5 +++ contrib/nnc/passes/soft_backend/ModelAnalyzer.h | 1 + contrib/nnc/passes/soft_backend/SBSerializer.cpp | 7 ++++ contrib/nnc/passes/soft_backend/SBSerializer.h | 1 + .../soft_backend/code_snippets/cpp_leaky_relu.def | 30 ++++++++++++++ .../nnc/passes/tflite_frontend/tflite_importer.cpp | 5 +++ .../passes/tflite_frontend/tflite_op_creator.cpp | 9 +++++ .../nnc/passes/tflite_frontend/tflite_op_creator.h | 7 ++++ .../nnc/unittests/soft_backend/CPPOperations.cpp | 15 +++++++ 20 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 contrib/nnc/include/core/modelIR/operations/LeakyReluOp.h create mode 100644 contrib/nnc/passes/soft_backend/code_snippets/cpp_leaky_relu.def diff --git a/contrib/nnc/core/modelIR/IrDotDumper.cpp b/contrib/nnc/core/modelIR/IrDotDumper.cpp index b430c72..4ab778f 100644 --- a/contrib/nnc/core/modelIR/IrDotDumper.cpp +++ b/contrib/nnc/core/modelIR/IrDotDumper.cpp @@ -31,6 +31,7 @@ #include "core/modelIR/operations/FullyConnectedOp.h" #include "core/modelIR/operations/GatherOp.h" #include "core/modelIR/operations/GemmOp.h" +#include "core/modelIR/operations/LeakyReluOp.h" #include "core/modelIR/operations/PadOp.h" #include "core/modelIR/operations/PoolOp.h" #include "core/modelIR/operations/ReduceFOp.h" @@ -349,6 +350,15 @@ void IrDotDumper::visit(ops::SigmoidOp& op) { dotBuilder.updateWithOp(&op, node_info); } +void IrDotDumper::visit(mir::ops::LeakyReluOp& op) { + auto node_info = DotIrNodeInfo().withType("LeakyReluOp", op.getName()) + .withInShapes(getInputShapes(op)) + .withOutShapes(getOutputShapes(op)) + .withMisc("alpha", op.getAlpha()); + + dotBuilder.updateWithOp(&op, node_info); +} + } // namespace mir } // namespace nnc diff --git a/contrib/nnc/core/modelIR/Operation.cpp b/contrib/nnc/core/modelIR/Operation.cpp index 2eb6eda..50dd174 100644 --- a/contrib/nnc/core/modelIR/Operation.cpp +++ b/contrib/nnc/core/modelIR/Operation.cpp @@ -29,6 +29,7 @@ #include "core/modelIR/operations/FullyConnectedOp.h" #include "core/modelIR/operations/GatherOp.h" #include "core/modelIR/operations/GemmOp.h" +#include "core/modelIR/operations/LeakyReluOp.h" #include "core/modelIR/operations/PadOp.h" #include "core/modelIR/operations/PoolOp.h" #include "core/modelIR/operations/ReduceFOp.h" diff --git a/contrib/nnc/include/core/modelIR/IrDotDumper.h b/contrib/nnc/include/core/modelIR/IrDotDumper.h index 0b3128e..ec67fda 100644 --- a/contrib/nnc/include/core/modelIR/IrDotDumper.h +++ b/contrib/nnc/include/core/modelIR/IrDotDumper.h @@ -45,6 +45,7 @@ public: void visit(ops::FullyConnectedOp& op) override; void visit(ops::GatherOp& op) override; void visit(ops::GemmOp& op) override; + void visit(mir::ops::LeakyReluOp& op) override; void visit(ops::PadOp& op) override; void visit(ops::PoolOp& op) override; void visit(ops::ReduceFOp& op) override; diff --git a/contrib/nnc/include/core/modelIR/operations/LeakyReluOp.h b/contrib/nnc/include/core/modelIR/operations/LeakyReluOp.h new file mode 100644 index 0000000..b968a07 --- /dev/null +++ b/contrib/nnc/include/core/modelIR/operations/LeakyReluOp.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _NNC_CORE_IR_MODEL_LEAKY_RELU_H_ +#define _NNC_CORE_IR_MODEL_LEAKY_RELU_H_ + +#include "core/modelIR/Operation.h" + +namespace nnc { +namespace mir { +namespace ops { + +class LeakyReluOp : public Operation { +public: + explicit LeakyReluOp(const IODescriptor& arg, float alpha) + : Operation(Type::ReLU, {arg}), _alpha(alpha) { + // Infer output shape. + setOutputShape(0, getInputShape(0)); + } + + float getAlpha() const { + return _alpha; + } + +private: + float _alpha; +}; + +} // namespace ops +} // namespace mir +} // namespace nnc + +#endif //_NNC_CORE_IR_MODEL_LEAKY_RELU_H_ diff --git a/contrib/nnc/include/core/modelIR/operations/operations.lst.h b/contrib/nnc/include/core/modelIR/operations/operations.lst.h index c3fe3d3..85b4372 100644 --- a/contrib/nnc/include/core/modelIR/operations/operations.lst.h +++ b/contrib/nnc/include/core/modelIR/operations/operations.lst.h @@ -47,3 +47,4 @@ HANDLE_OP(pad, PadOp) HANDLE_OP(sqrt, SqrtOp) HANDLE_OP(reduceF, ReduceFOp) HANDLE_OP(transpose, TransposeOp) +HANDLE_OP(leakyReLU, LeakyReluOp) diff --git a/contrib/nnc/include/passes/interpreter/Interpreter.h b/contrib/nnc/include/passes/interpreter/Interpreter.h index e6f10d6..23dfaf4 100644 --- a/contrib/nnc/include/passes/interpreter/Interpreter.h +++ b/contrib/nnc/include/passes/interpreter/Interpreter.h @@ -51,6 +51,7 @@ public: void visit(ops::EluOp& op) override; void visit(ops::FullyConnectedOp& op) override; void visit(ops::GatherOp& op) override; + void visit(ops::LeakyReluOp& op) override; void visit(ops::PadOp& op) override; void visit(ops::PoolOp& op) override; void visit(ops::ReduceFOp& op) override; diff --git a/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.cpp b/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.cpp index ea381f3..f7698ee 100644 --- a/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.cpp +++ b/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.cpp @@ -34,6 +34,7 @@ #include "core/modelIR/operations/ElementwiseOp.h" #include "core/modelIR/operations/FullyConnectedOp.h" #include "core/modelIR/operations/GemmOp.h" +#include "core/modelIR/operations/LeakyReluOp.h" #include "core/modelIR/operations/PoolOp.h" #include "core/modelIR/operations/ReduceFOp.h" #include "core/modelIR/operations/ReluOp.h" @@ -944,5 +945,9 @@ void AclCppOpGenerator::visit(ops::SigmoidOp& op) { genActivation(op, "LOGISTIC"); } +void AclCppOpGenerator::visit(mir::ops::LeakyReluOp& op) { + genActivation(op, "LEAKY_RELU", op.getAlpha()); +} + } // namespace nnc diff --git a/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.h b/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.h index f48eb3d..c306a2b 100644 --- a/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.h +++ b/contrib/nnc/passes/acl_soft_backend/AclCppOpGenerator.h @@ -61,6 +61,7 @@ public: void visit(mir::ops::FullyConnectedOp& op) override; void visit(mir::ops::GatherOp& op) override; void visit(mir::ops::GemmOp& op) override; + void visit(mir::ops::LeakyReluOp& op) override; void visit(mir::ops::PadOp& op) override; void visit(mir::ops::PoolOp& op) override; void visit(mir::ops::ReduceFOp& op) override; diff --git a/contrib/nnc/passes/caffe_frontend/caffe_op_creator.cpp b/contrib/nnc/passes/caffe_frontend/caffe_op_creator.cpp index 58c7c27..74b1570 100644 --- a/contrib/nnc/passes/caffe_frontend/caffe_op_creator.cpp +++ b/contrib/nnc/passes/caffe_frontend/caffe_op_creator.cpp @@ -26,6 +26,7 @@ #include "core/modelIR/operations/EluOp.h" #include "core/modelIR/operations/FullyConnectedOp.h" #include "core/modelIR/operations/GatherOp.h" +#include "core/modelIR/operations/LeakyReluOp.h" #include "core/modelIR/operations/PoolOp.h" #include "core/modelIR/operations/ReluOp.h" #include "core/modelIR/operations/ReshapeOp.h" @@ -457,14 +458,20 @@ CaffeOpCreator::convertReshape(const caffe::LayerParameter& layer, void CaffeOpCreator::checkReLU(const ReLUParameter& opts, std::set& problems_op_set) { - if (opts.has_negative_slope()) - problems_op_set.insert("ReLU layer negative_slope param is not supported yet."); } std::vector CaffeOpCreator::convertReLU(const caffe::LayerParameter& layer, const std::vector& inputs) { - auto relu = createOp(layer.name(), inputs[0]); + mir::Operation* relu; + if (layer.relu_param().has_negative_slope()) { + float alpha = layer.relu_param().negative_slope(); + relu = createOp(layer.name(), inputs[0], alpha); + } else { + relu = createOp(layer.name(), inputs[0]); + } + + return {relu->getOutput(0)}; } diff --git a/contrib/nnc/passes/interpreter/Interpreter.cpp b/contrib/nnc/passes/interpreter/Interpreter.cpp index 8e195d4..f0fd1c8 100644 --- a/contrib/nnc/passes/interpreter/Interpreter.cpp +++ b/contrib/nnc/passes/interpreter/Interpreter.cpp @@ -30,6 +30,7 @@ #include "core/modelIR/operations/FullyConnectedOp.h" #include "core/modelIR/operations/GatherOp.h" #include "core/modelIR/operations/GemmOp.h" +#include "core/modelIR/operations/LeakyReluOp.h" #include "core/modelIR/operations/PadOp.h" #include "core/modelIR/operations/PoolOp.h" #include "core/modelIR/operations/ReduceFOp.h" @@ -438,4 +439,17 @@ void NNInterpreter::visit(ops::GatherOp& op) { var(op.getId()) = Gather(data, indices, op)(); } +void NNInterpreter::visit(ops::LeakyReluOp& op) { + auto operand = op.getPrevNodes()[0]; + float alpha = op.getAlpha(); + Tensor input(var(operand.op->getId())[operand.index]); + var(op.getId()) = Fill( + op.getOutputShape(0), [&input, alpha](const Index& id) { + float val = input.at(id); + return val > 0.0f ? val : val * alpha; + })(); + + DUMP(op, false); +} + } // namespace nnc diff --git a/contrib/nnc/passes/soft_backend/CPPGenerator.cpp b/contrib/nnc/passes/soft_backend/CPPGenerator.cpp index 9ab20a7..da2a9b1 100644 --- a/contrib/nnc/passes/soft_backend/CPPGenerator.cpp +++ b/contrib/nnc/passes/soft_backend/CPPGenerator.cpp @@ -39,6 +39,7 @@ using namespace std; #include "cpp_sigmoid.generated.h" #include "cpp_sqrt.generated.h" #include "cpp_relu.generated.h" +#include "cpp_leaky_relu.generated.h" #include "cpp_reduce.generated.h" #include "cpp_resize.generated.h" #include "cpp_softmax.generated.h" @@ -307,6 +308,7 @@ void CPPCodeGenerator::materializeCode(ostream &out, const ModelAnalyzer &ma, co out.write(cpp_scale, sizeof(cpp_scale)); out.write(cpp_dropout, sizeof(cpp_dropout)); out.write(cpp_batchnorm, sizeof(cpp_batchnorm)); + out.write(cpp_leaky_relu, sizeof(cpp_leaky_relu)); // gen NN constructor out << className << "::" << className << "(const string ¶metersPath)\n" diff --git a/contrib/nnc/passes/soft_backend/ModelAnalyzer.cpp b/contrib/nnc/passes/soft_backend/ModelAnalyzer.cpp index b4fb578..1127e36 100644 --- a/contrib/nnc/passes/soft_backend/ModelAnalyzer.cpp +++ b/contrib/nnc/passes/soft_backend/ModelAnalyzer.cpp @@ -37,6 +37,7 @@ #include "core/modelIR/operations/FullyConnectedOp.h" #include "core/modelIR/operations/GatherOp.h" #include "core/modelIR/operations/GemmOp.h" +#include "core/modelIR/operations/LeakyReluOp.h" #include "core/modelIR/operations/PadOp.h" #include "core/modelIR/operations/PoolOp.h" #include "core/modelIR/operations/ReduceFOp.h" @@ -345,4 +346,8 @@ void ModelAnalyzer::visit(mir::ops::SigmoidOp& op) { addOpDescr(&op, "sigmoid"); } +void ModelAnalyzer::visit(mir::ops::LeakyReluOp& op) { + addOpDescr(&op, "leakyRelu"); +} + } // namespace nnc diff --git a/contrib/nnc/passes/soft_backend/ModelAnalyzer.h b/contrib/nnc/passes/soft_backend/ModelAnalyzer.h index 04a56cb..702e905 100644 --- a/contrib/nnc/passes/soft_backend/ModelAnalyzer.h +++ b/contrib/nnc/passes/soft_backend/ModelAnalyzer.h @@ -100,6 +100,7 @@ public: void visit(mir::ops::EluOp& op) override; void visit(mir::ops::FullyConnectedOp& op) override; void visit(mir::ops::GatherOp& op) override; + void visit(mir::ops::LeakyReluOp& op) override; void visit(mir::ops::GemmOp& op) override; void visit(mir::ops::PadOp& op) override; void visit(mir::ops::PoolOp& op) override; diff --git a/contrib/nnc/passes/soft_backend/SBSerializer.cpp b/contrib/nnc/passes/soft_backend/SBSerializer.cpp index 9eeace8..ffd6c20 100644 --- a/contrib/nnc/passes/soft_backend/SBSerializer.cpp +++ b/contrib/nnc/passes/soft_backend/SBSerializer.cpp @@ -34,6 +34,7 @@ #include "core/modelIR/operations/FullyConnectedOp.h" #include "core/modelIR/operations/GatherOp.h" #include "core/modelIR/operations/GemmOp.h" +#include "core/modelIR/operations/LeakyReluOp.h" #include "core/modelIR/operations/PadOp.h" #include "core/modelIR/operations/PoolOp.h" #include "core/modelIR/operations/ReduceFOp.h" @@ -380,4 +381,10 @@ void Serializer::visit(mir::ops::SigmoidOp& op) { _curOp->_paramStartOffset = _buffer.size(); } +void Serializer::visit(mir::ops::LeakyReluOp& op) { + _curOp->_paramStartOffset = _buffer.size(); + serializeT(op.getAlpha()); + serializeShape(op.getOutputShape(0)); +} + } // namespace nnc diff --git a/contrib/nnc/passes/soft_backend/SBSerializer.h b/contrib/nnc/passes/soft_backend/SBSerializer.h index 789be93..6e5c7cc 100644 --- a/contrib/nnc/passes/soft_backend/SBSerializer.h +++ b/contrib/nnc/passes/soft_backend/SBSerializer.h @@ -53,6 +53,7 @@ public: void visit(mir::ops::ElementwiseOp& op) override; void visit(mir::ops::EluOp& op) override; void visit(mir::ops::FullyConnectedOp& op) override; + void visit(mir::ops::LeakyReluOp& op) override; void visit(mir::ops::GatherOp& op) override; void visit(mir::ops::GemmOp& op) override; void visit(mir::ops::PadOp& op) override; diff --git a/contrib/nnc/passes/soft_backend/code_snippets/cpp_leaky_relu.def b/contrib/nnc/passes/soft_backend/code_snippets/cpp_leaky_relu.def new file mode 100644 index 0000000..3526004 --- /dev/null +++ b/contrib/nnc/passes/soft_backend/code_snippets/cpp_leaky_relu.def @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +void leakyRelu(Tensor& out, const char* params, const Tensor& in) { + const float* input = in.getData(); + out.reShape(in.getShape()); + float* output = out.getData(); + const float alpha = deserializeT(params); + + size_t data_length = in.getShape().getNumElems(); + + for( int i = 0; i < data_length; ++i ) { + float val = input[i]; + float res = val > 0 ? val : val * alpha; + output[i] = res; + } +} diff --git a/contrib/nnc/passes/tflite_frontend/tflite_importer.cpp b/contrib/nnc/passes/tflite_frontend/tflite_importer.cpp index e55e068..de060f0 100644 --- a/contrib/nnc/passes/tflite_frontend/tflite_importer.cpp +++ b/contrib/nnc/passes/tflite_frontend/tflite_importer.cpp @@ -112,6 +112,7 @@ void TfliteImporter::processUnsupportedOp(const Operator* op) { case BuiltinOperator_RELU6: case BuiltinOperator_TRANSPOSE: case BuiltinOperator_STRIDED_SLICE: + case BuiltinOperator_LEAKY_RELU: // No checks break; default: @@ -276,6 +277,10 @@ void TfliteImporter::walkOperator(const Operator* op) { outputs = _opCreator->createStridedSlice( inputs, params, op->builtin_options_as()); break; + case BuiltinOperator_LEAKY_RELU: + outputs = _opCreator->createLeakyRelu(inputs, params, + op->builtin_options_as()); + break; default: assert(false && "All unsupported types should have been found before this pass."); } diff --git a/contrib/nnc/passes/tflite_frontend/tflite_op_creator.cpp b/contrib/nnc/passes/tflite_frontend/tflite_op_creator.cpp index be8a0ae..6d039b3 100644 --- a/contrib/nnc/passes/tflite_frontend/tflite_op_creator.cpp +++ b/contrib/nnc/passes/tflite_frontend/tflite_op_creator.cpp @@ -25,6 +25,7 @@ #include "core/modelIR/operations/DepthwiseConv2DOp.h" #include "core/modelIR/operations/ElementwiseOp.h" #include "core/modelIR/operations/FullyConnectedOp.h" +#include "core/modelIR/operations/LeakyReluOp.h" #include "core/modelIR/operations/PadOp.h" #include "core/modelIR/operations/PoolOp.h" #include "core/modelIR/operations/ReduceFOp.h" @@ -485,4 +486,12 @@ TFLiteOpCreator::createStridedSlice(InputOps& inputs, const InputParams& params, slice_outputs[0]->getOutput(0), squeeze_dims); } +std::vector +TFLiteOpCreator::createLeakyRelu(TFLiteOpCreator::InputOps& inputs, const TFLiteOpCreator::InputParams&, + const ::tflite::LeakyReluOptions* opts) { + float alpha = opts->alpha(); + + return createOp(ActivationFunctionType_NONE, inputs[0]->getOutput(0), alpha); +} + } // namespace nnc diff --git a/contrib/nnc/passes/tflite_frontend/tflite_op_creator.h b/contrib/nnc/passes/tflite_frontend/tflite_op_creator.h index 482e2e3..de79a64 100644 --- a/contrib/nnc/passes/tflite_frontend/tflite_op_creator.h +++ b/contrib/nnc/passes/tflite_frontend/tflite_op_creator.h @@ -136,6 +136,13 @@ public: std::vector createStridedSlice(InputOps&, const InputParams&, const ::tflite::StridedSliceOptions*); + /** + * @brief Create leaky relu activation + * @return + */ + std::vector createLeakyRelu(InputOps&, const InputParams&, + const ::tflite::LeakyReluOptions*); + void checkPool2D(const ::tflite::Pool2DOptions*, std::set&); void checkConcatenation(const ::tflite::ConcatenationOptions*, std::set&); diff --git a/contrib/nnc/unittests/soft_backend/CPPOperations.cpp b/contrib/nnc/unittests/soft_backend/CPPOperations.cpp index 1007db6..da93b33 100644 --- a/contrib/nnc/unittests/soft_backend/CPPOperations.cpp +++ b/contrib/nnc/unittests/soft_backend/CPPOperations.cpp @@ -51,6 +51,7 @@ #include "code_snippets/cpp_operations.def" #include "code_snippets/cpp_scale.def" +#include "code_snippets/cpp_leaky_relu.def" // soft backend part @@ -71,6 +72,7 @@ #include "core/modelIR/operations/PoolOp.h" #include "core/modelIR/operations/ReduceFOp.h" #include "core/modelIR/operations/ReluOp.h" +#include "core/modelIR/operations/LeakyReluOp.h" #include "core/modelIR/operations/ReshapeOp.h" #include "core/modelIR/operations/ResizeOp.h" #include "core/modelIR/operations/ScaleOp.h" @@ -798,6 +800,19 @@ TEST(cpp_operations_test, relu) { createAndRunTestGraph(op_generator, relu, input_ntensors, input_atensor); } +TEST(cpp_operations_test, leaky_relu) { + // test prerequisites + vector shape_data{2, 3, 4, 5}; + Tensor input_atensor; + vector> input_ntensors(1); + fillTensors(input_ntensors[0], input_atensor, shape_data, 1.0f); + auto op_generator = [](mir::Graph& g, const std::vector& inputs) { + return g.create("y", inputs[0], 0.1); + }; + + createAndRunTestGraph(op_generator, relu, input_ntensors, input_atensor); +} + TEST(cpp_operations_test, sigmoid) { // test prerequisites vector shape_data{2, 3, 4, 5}; -- 2.7.4