From: Павел Ильютченко/AI Tools Lab /SRR/Engineer/삼성전자 Date: Mon, 28 Jan 2019 13:16:51 +0000 (+0300) Subject: [nnc] Eigen realisation of some activation functions of SoftBackend (#2926) X-Git-Tag: nncc_backup~919 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=1b6fc3d80ee891459c0ec1cc8a6f20e49af6aff3;p=platform%2Fcore%2Fml%2Fnnfw.git [nnc] Eigen realisation of some activation functions of SoftBackend (#2926) New realisations for activation functions: - CappedRelu - LeakyRelu - Sigmoid Signed-off-by: Pavel Iliutchenko --- diff --git a/contrib/nnc/passes/soft_backend/code_snippets/cpp_capped_relu.def b/contrib/nnc/passes/soft_backend/code_snippets/cpp_capped_relu.def index 314ea2b..52e657f 100644 --- a/contrib/nnc/passes/soft_backend/code_snippets/cpp_capped_relu.def +++ b/contrib/nnc/passes/soft_backend/code_snippets/cpp_capped_relu.def @@ -1,25 +1,24 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ inline void CappedRelu(const float* input_data, const Dims<4>& input_dims, float cap, float* output_data, const Dims<4>& output_dims) { - const int flat_size = MatchingFlatSize(input_dims, output_dims); - for (int i = 0; i < flat_size; ++i) { - const float val = input_data[i]; - const float lower = 0; - const float clamped = val > cap ? cap : val < lower ? lower : val; - output_data[i] = clamped; - } + + const auto input = MapAsVector(input_data, input_dims); + auto output = MapAsVector(output_data, output_dims); + + output = input.cwiseMax(0.0f).cwiseMin(cap); } diff --git a/contrib/nnc/passes/soft_backend/code_snippets/cpp_common_funcs.def b/contrib/nnc/passes/soft_backend/code_snippets/cpp_common_funcs.def index 30e490c..ff42e58 100644 --- a/contrib/nnc/passes/soft_backend/code_snippets/cpp_common_funcs.def +++ b/contrib/nnc/passes/soft_backend/code_snippets/cpp_common_funcs.def @@ -487,6 +487,12 @@ VectorMap MapAsVector(Scalar* data, const size_t size) { } template +VectorMap MapAsVector(Scalar* data, const RuntimeShape& shape) { + const int size = shape.FlatSize(); + return VectorMap(data, size, 1); +} + +template using MatrixMap = typename std::conditional< std::is_const::value, Eigen::Map::type, diff --git a/contrib/nnc/passes/soft_backend/code_snippets/cpp_leaky_relu.def b/contrib/nnc/passes/soft_backend/code_snippets/cpp_leaky_relu.def index 3526004..a37bcfd 100644 --- a/contrib/nnc/passes/soft_backend/code_snippets/cpp_leaky_relu.def +++ b/contrib/nnc/passes/soft_backend/code_snippets/cpp_leaky_relu.def @@ -15,16 +15,11 @@ */ void leakyRelu(Tensor& out, const char* params, const Tensor& in) { - const float* input = in.getData(); - out.reShape(in.getShape()); - float* output = out.getData(); const float alpha = deserializeT(params); + out.reShape(in.getShape()); + + const auto input = MapAsVector(in.getData(), static_cast(in.getShape().getNumElems())); + auto output = MapAsVector(out.getData(), static_cast(in.getShape().getNumElems())); - size_t data_length = in.getShape().getNumElems(); - - for( int i = 0; i < data_length; ++i ) { - float val = input[i]; - float res = val > 0 ? val : val * alpha; - output[i] = res; - } + output = (alpha * input).cwiseMax(input); } diff --git a/contrib/nnc/passes/soft_backend/code_snippets/cpp_sigmoid.def b/contrib/nnc/passes/soft_backend/code_snippets/cpp_sigmoid.def index 96d72f9..a67d6fd 100644 --- a/contrib/nnc/passes/soft_backend/code_snippets/cpp_sigmoid.def +++ b/contrib/nnc/passes/soft_backend/code_snippets/cpp_sigmoid.def @@ -1,22 +1,24 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ +/* + * Copyright (c) 2019 Samsung Electronics Co., Ltd. All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ inline void Logistic(const RuntimeShape& input_shape, const float* input_data, const RuntimeShape& output_shape, float* output_data) { - const int flat_size = MatchingFlatSize(input_shape, output_shape); - for (int i = 0; i < flat_size; i++) { - float val = input_data[i]; - float result = 1.f / (1.f + std::exp(-val)); - output_data[i] = result; - } + const auto input = MapAsVector(input_data, input_shape); + auto output = MapAsVector(output_data, output_shape); + + output.array() = input.array().unaryExpr(Eigen::internal::scalar_logistic_op()); } diff --git a/contrib/nnc/passes/soft_backend/code_snippets/eigen.def b/contrib/nnc/passes/soft_backend/code_snippets/eigen.def index b871953..b02f84b 100644 --- a/contrib/nnc/passes/soft_backend/code_snippets/eigen.def +++ b/contrib/nnc/passes/soft_backend/code_snippets/eigen.def @@ -16155,6 +16155,81 @@ struct functor_traits > PacketAccess = packet_traits::HasSign }; }; + +template +struct scalar_logistic_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_logistic_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T operator()(const T& x) const { + const T one = T(1); + return one / (one + numext::exp(-x)); + } + + template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Packet packetOp(const Packet& x) const { + const Packet one = pset1(T(1)); + return pdiv(one, padd(one, pexp(pnegate(x)))); + } +}; +template +struct functor_traits > { + enum { + Cost = NumTraits::AddCost * 2 + NumTraits::MulCost * 6, + PacketAccess = packet_traits::HasAdd && packet_traits::HasDiv && + packet_traits::HasNegate && packet_traits::HasExp + }; +}; + +template <> +struct scalar_logistic_op { + EIGEN_EMPTY_STRUCT_CTOR(scalar_logistic_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float operator()(const float& x) const { + const float one = 1.0f; + return one / (one + numext::exp(-x)); + } + + template EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + Packet packetOp(const Packet& _x) const { + // Clamp the inputs to the range [-18, 18] since anything outside + // this range is 0.0f or 1.0f in single-precision. + const Packet x = pmax(pmin(_x, pset1(18.0)), pset1(-18.0)); + + // The monomial coefficients of the numerator polynomial (odd). + const Packet alpha_1 = pset1(2.48287947061529e-01); + const Packet alpha_3 = pset1(8.51377133304701e-03); + const Packet alpha_5 = pset1(6.08574864600143e-05); + const Packet alpha_7 = pset1(1.15627324459942e-07); + const Packet alpha_9 = pset1(4.37031012579801e-11); + + // The monomial coefficients of the denominator polynomial (even). + const Packet beta_0 = pset1(9.93151921023180e-01); + const Packet beta_2 = pset1(1.16817656904453e-01); + const Packet beta_4 = pset1(1.70198817374094e-03); + const Packet beta_6 = pset1(6.29106785017040e-06); + const Packet beta_8 = pset1(5.76102136993427e-09); + const Packet beta_10 = pset1(6.10247389755681e-13); + + // Since the polynomials are odd/even, we need x^2. + const Packet x2 = pmul(x, x); + + // Evaluate the numerator polynomial p. + Packet p = pmadd(x2, alpha_9, alpha_7); + p = pmadd(x2, p, alpha_5); + p = pmadd(x2, p, alpha_3); + p = pmadd(x2, p, alpha_1); + p = pmul(x, p); + + // Evaluate the denominator polynomial p. + Packet q = pmadd(x2, beta_10, beta_8); + q = pmadd(x2, q, beta_6); + q = pmadd(x2, q, beta_4); + q = pmadd(x2, q, beta_2); + q = pmadd(x2, q, beta_0); + + // Divide the numerator by the denominator and shift it up. + return pmax(pmin(padd(pdiv(p, q), pset1(0.5)), pset1(1.0)), + pset1(0.0)); + } +}; } } #endif @@ -20507,6 +20582,7 @@ typedef CwiseUnaryOp, const Derived> AcosReturn typedef CwiseUnaryOp, const Derived> AsinReturnType; typedef CwiseUnaryOp, const Derived> AtanReturnType; typedef CwiseUnaryOp, const Derived> TanhReturnType; +typedef CwiseUnaryOp, const Derived> LogisticReturnType; typedef CwiseUnaryOp, const Derived> SinhReturnType; typedef CwiseUnaryOp, const Derived> CoshReturnType; typedef CwiseUnaryOp, const Derived> SquareReturnType; @@ -20632,6 +20708,12 @@ cosh() const return CoshReturnType(derived()); } EIGEN_DEVICE_FUNC +inline const LogisticReturnType +logistic() const +{ + return LogisticReturnType(derived()); +} +EIGEN_DEVICE_FUNC inline const InverseReturnType inverse() const {