From dfa42452d30006f5de58f8b7e98bbf490abb88c7 Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Mon, 12 Jul 2021 03:44:29 +0900 Subject: [PATCH] [layer] Update embedding for layerv2 Update embedding layer for layerv2 design. Signed-off-by: Parichay Kapoor --- nntrainer/layers/embedding.cpp | 119 ++++++++++----------- nntrainer/layers/embedding.h | 62 ++++++----- test/unittest/layers/meson.build | 1 + test/unittest/layers/unittest_layers_embedding.cpp | 25 +++++ 4 files changed, 119 insertions(+), 88 deletions(-) create mode 100644 test/unittest/layers/unittest_layers_embedding.cpp diff --git a/nntrainer/layers/embedding.cpp b/nntrainer/layers/embedding.cpp index 220393d..a6cf5ce 100644 --- a/nntrainer/layers/embedding.cpp +++ b/nntrainer/layers/embedding.cpp @@ -21,83 +21,88 @@ namespace nntrainer { +static constexpr size_t SINGLE_INOUT_IDX = 0; + enum EmbeddingParams { weight }; -int EmbeddingLayer::initialize(Manager &manager) { - int status = ML_ERROR_NONE; - if (getNumInputs() != 1) { +void EmbeddingLayer::finalize(InitLayerContext &context) { + if (context.getNumInputs() != 1) { throw std::invalid_argument("Embedding layer takes only one input"); } - if (input_dim[0].channel() != 1) { + const TensorDim &input_dim = context.getInputDimensions()[SINGLE_INOUT_IDX]; + if (input_dim.channel() != 1) { throw std::invalid_argument( "Embedding layer takes only one for channel size"); } - output_dim[0] = input_dim[0]; - - output_dim[0].height(in_length); - output_dim[0].width(out_dim); - input_dim[0].width(in_length); - input_dim[0].height(1); + TensorDim output_dim = input_dim; - TensorDim dim = output_dim[0]; + output_dim.height(in_length); + output_dim.width(out_dim); + context.setOutputDimensions({output_dim}); + TensorDim dim = output_dim; dim.height(in_dim); dim.width(out_dim); dim.batch(1); - if (weights.empty()) { - weights.reserve(1); - weights.emplace_back(dim, weight_initializer, weight_regularizer, - weight_regularizer_constant, true, "Embedding"); - manager.trackWeights(weights); - } else { - weights[EmbeddingParams::weight].reset(dim, weight_initializer, - weight_regularizer, - weight_regularizer_constant, true); - } + weight_idx = + context.requestWeight(dim, weight_initializer, weight_regularizer, + weight_regularizer_constant, "Embedding", true); +} + +void EmbeddingLayer::setProperty(const std::vector &values) { + /// @todo: deprecate this in favor of loadProperties + for (unsigned int i = 0; i < values.size(); ++i) { + std::string key; + std::string value; + std::stringstream ss; + + if (getKeyValue(values[i], key, value) != ML_ERROR_NONE) { + throw std::invalid_argument("Error parsing the property: " + values[i]); + } + + if (value.empty()) { + ss << "value is empty: key: " << key << ", value: " << value; + throw std::invalid_argument(ss.str()); + } - return status; + /// @note this calls derived setProperty if available + setProperty(key, value); + } } -void EmbeddingLayer::setProperty(const PropertyType type, +void EmbeddingLayer::setProperty(const std::string &type_str, const std::string &value) { + using PropertyType = LayerV1::PropertyType; int status = ML_ERROR_NONE; + LayerV1::PropertyType type = + static_cast(parseLayerProperty(type_str)); + switch (type) { case PropertyType::in_dim: { - if (!value.empty()) { - status = setUint(in_dim, value); - throw_status(status); - input_dim[0].width(in_dim); - } + status = setUint(in_dim, value); + throw_status(status); } break; case PropertyType::out_dim: { - if (!value.empty()) { - status = setUint(out_dim, value); - throw_status(status); - output_dim[0].width(out_dim); - } + status = setUint(out_dim, value); + throw_status(status); } break; case PropertyType::in_length: { - if (!value.empty()) { - status = setUint(in_length, value); - throw_status(status); - output_dim[0].height(in_length); - input_dim[0].height(in_length); - } + status = setUint(in_length, value); + throw_status(status); } break; default: - LayerV1::setProperty(type, value); + LayerImpl::setProperty(type_str, value); break; } } -void EmbeddingLayer::forwarding(bool training) { - Tensor &weight = - weightAt(static_cast(EmbeddingParams::weight)).getVariableRef(); - Tensor &hidden_ = net_hidden[0]->getVariableRef(); - Tensor &input_ = net_input[0]->getVariableRef(); +void EmbeddingLayer::forwarding(RunLayerContext &context, bool training) { + Tensor &weight = context.getWeight(weight_idx); + Tensor &hidden_ = context.getOutput(SINGLE_INOUT_IDX); + Tensor &input_ = context.getInput(SINGLE_INOUT_IDX); for (unsigned int b = 0; b < input_.batch(); ++b) { float *in_data = input_.getAddress(b * input_.getDim().getFeatureLen()); @@ -120,22 +125,9 @@ void EmbeddingLayer::forwarding(bool training) { std::copy(weight_data, weight_data + out_dim, out_data); } } - - loss = - weightAt(static_cast(EmbeddingParams::weight)).getRegularizationLoss(); -} - -void EmbeddingLayer::copy(std::shared_ptr l) { - LayerV1::copy(l); - - std::shared_ptr from = - std::static_pointer_cast(l); - this->in_dim = from->in_dim; - this->out_dim = from->out_dim; - this->in_length = from->in_length; } -void EmbeddingLayer::calcDerivative() { +void EmbeddingLayer::calcDerivative(RunLayerContext &context) { // Uncomment this after fixing issues backwarding of first layer. (Issues // #1017) // throw exception::not_supported( @@ -143,11 +135,10 @@ void EmbeddingLayer::calcDerivative() { return; // intended } -void EmbeddingLayer::calcGradient() { - Tensor &djdw = - weightAt(static_cast(EmbeddingParams::weight)).getGradientRef(); - Tensor &derivative_ = net_hidden[0]->getGradientRef(); - Tensor &input_ = net_input[0]->getVariableRef(); +void EmbeddingLayer::calcGradient(RunLayerContext &context) { + Tensor &djdw = context.getWeightGrad(weight_idx); + Tensor &derivative_ = context.getIncomingDerivative(SINGLE_INOUT_IDX); + Tensor &input_ = context.getInput(SINGLE_INOUT_IDX); djdw.setZero(); diff --git a/nntrainer/layers/embedding.h b/nntrainer/layers/embedding.h index 41b3e23..826cc80 100644 --- a/nntrainer/layers/embedding.h +++ b/nntrainer/layers/embedding.h @@ -15,8 +15,7 @@ #define __EMBEDDING_H__ #ifdef __cplusplus -#include -#include +#include namespace nntrainer { @@ -24,23 +23,23 @@ namespace nntrainer { * @class EmbeddingLayer * @brief EmbeddingLayer */ -class EmbeddingLayer : public LayerV1 { +class EmbeddingLayer : public LayerImpl { public: /** * @brief Constructor of Embedding Layer */ - template EmbeddingLayer(unsigned int in_dim_ = 0, unsigned int out_dim_ = 0, - unsigned int in_length_ = 0, Args... args) : - LayerV1(args...), + unsigned int in_length_ = 0) : + LayerImpl(), in_dim(in_dim_), out_dim(out_dim_), - in_length(in_length_) {} + in_length(in_length_), + weight_idx(0) {} /** * @brief Destructor of Embedding Layer */ - ~EmbeddingLayer(){}; + ~EmbeddingLayer() = default; /** * @brief Move constructor. @@ -55,46 +54,50 @@ public: EmbeddingLayer &operator=(EmbeddingLayer &&rhs) = default; /** - * @copydoc Layer::forwarding(bool training) + * @copydoc Layer::finalize(InitLayerContext &context) */ - void forwarding(bool training = true) override; + void finalize(InitLayerContext &context) override; /** - * @copydoc Layer::calcDerivative() + * @copydoc Layer::forwarding(RunLayerContext &context, bool training) */ - void calcDerivative() override; + void forwarding(RunLayerContext &context, bool training) override; /** - * @copydoc Layer::calcGradient() + * @copydoc Layer::calcDerivative(RunLayerContext &context) */ - void calcGradient() override; + void calcDerivative(RunLayerContext &context) override; /** - * @brief copy layer - * @param[in] l layer to copy + * @copydoc Layer::calcGradient(RunLayerContext &context) */ - void copy(std::shared_ptr l) override; + void calcGradient(RunLayerContext &context) override; /** - * @brief initialize layer - * @retval #ML_ERROR_NONE Successful. - * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. + * @copydoc Layer::exportTo(Exporter &exporter, ExportMethods method) */ - int initialize(Manager &manager) override; + void exportTo(Exporter &exporter, + const ExportMethods &method) const override { + LayerImpl::exportTo(exporter, method); + } /** * @copydoc Layer::getType() */ const std::string getType() const override { return EmbeddingLayer::type; }; - using LayerV1::setProperty; + /** + * @copydoc Layer::supportBackwarding() + */ + bool supportBackwarding() const { return true; } + + using Layer::setProperty; /** * @copydoc Layer::setProperty(const PropertyType type, const std::string * &value) */ - void setProperty(const PropertyType type, - const std::string &value = "") override; + void setProperty(const std::vector &values) override; inline static const std::string type = "embedding"; @@ -102,6 +105,17 @@ private: unsigned int in_dim; unsigned int out_dim; unsigned int in_length; + unsigned int weight_idx; + + /** + * @brief setProperty by type and value separated + * @param[in] type property type to be passed + * @param[in] value value to be passed + * @exception exception::not_supported when property type is not valid for + * the particular layer + * @exception std::invalid_argument invalid argument + */ + void setProperty(const std::string &type_str, const std::string &value); }; } // namespace nntrainer diff --git a/test/unittest/layers/meson.build b/test/unittest/layers/meson.build index 93255ed..2c5a211 100644 --- a/test/unittest/layers/meson.build +++ b/test/unittest/layers/meson.build @@ -17,6 +17,7 @@ test_target = [ 'unittest_layers_gru.cpp', 'unittest_layers_preprocess.cpp', 'unittest_layers_split.cpp', + 'unittest_layers_embedding.cpp', ] exe = executable( diff --git a/test/unittest/layers/unittest_layers_embedding.cpp b/test/unittest/layers/unittest_layers_embedding.cpp new file mode 100644 index 0000000..5e54feb --- /dev/null +++ b/test/unittest/layers/unittest_layers_embedding.cpp @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2021 Parichay Kapoor + * + * @file unittest_layers_embedding.cpp + * @date 12 June 2021 + * @brief Embedding Layer Test + * @see https://github.com/nnstreamer/nntrainer + * @author Parichay Kapoor + * @bug No known bugs except for NYI items + */ +#include + +#include + +#include +#include + +auto semantic_embedding = LayerSemanticsParamType( + nntrainer::createLayer, + nntrainer::EmbeddingLayer::type, {"in_length=1", "out_dim=1", "in_dim=1"}, {}, + 0, false); + +INSTANTIATE_TEST_CASE_P(Embedding, LayerSemantics, + ::testing::Values(semantic_embedding)); -- 2.7.4