From 651cc07ad771d0c84de63e0dd1b02495167302b0 Mon Sep 17 00:00:00 2001 From: Parichay Kapoor Date: Tue, 22 Sep 2020 21:51:45 +0900 Subject: [PATCH] [weight] weight class to simplify headers Added a weight class to simplify headers All weight related enums and properties go to weight header rather being dumped in layer or optimizer V2: Update swap to swap gradients even if not trainable Update swap of Tensor, TensorDim to be friend functions Copy and move assignements are now default **Self evaluation:** 1. Build test: [x]Passed [ ]Failed [ ]Skipped 2. Run test: [x]Passed [ ]Failed [ ]Skipped Signed-off-by: Parichay Kapoor --- jni/Android.mk | 3 +- nntrainer/include/layer.h | 84 +++++-------- nntrainer/include/neuralnet.h | 3 + nntrainer/include/optimizer.h | 51 ++------ nntrainer/include/tensor.h | 7 +- nntrainer/include/tensor_dim.h | 7 +- nntrainer/include/weight.h | 178 ++++++++++++++++++++++++++++ nntrainer/meson.build | 6 +- nntrainer/src/bn_layer.cpp | 42 +++---- nntrainer/src/conv2d_layer.cpp | 50 +++----- nntrainer/src/fc_layer.cpp | 28 ++--- nntrainer/src/layer.cpp | 81 +++---------- nntrainer/src/optimizer.cpp | 41 +++---- nntrainer/src/tensor.cpp | 7 -- nntrainer/src/tensor_dim.cpp | 15 +-- nntrainer/src/weight.cpp | 70 +++++++++++ packaging/nntrainer.spec | 1 + test/unittest/unittest_nntrainer_layers.cpp | 62 +++++----- 18 files changed, 436 insertions(+), 300 deletions(-) create mode 100644 nntrainer/include/weight.h create mode 100644 nntrainer/src/weight.cpp diff --git a/jni/Android.mk b/jni/Android.mk index 0134d33..5923411 100644 --- a/jni/Android.mk +++ b/jni/Android.mk @@ -41,7 +41,8 @@ NNTRAINER_SRCS := $(NNTRAINER_ROOT)/nntrainer/src/neuralnet.cpp \ $(NNTRAINER_ROOT)/nntrainer/src/flatten_layer.cpp \ $(NNTRAINER_ROOT)/nntrainer/src/model_loader.cpp \ $(NNTRAINER_ROOT)/nntrainer/src/addition_layer.cpp \ - $(NNTRAINER_ROOT)/nntrainer/src/blas_interface.cpp + $(NNTRAINER_ROOT)/nntrainer/src/blas_interface.cpp \ + $(NNTRAINER_ROOT)/nntrainer/src/weight.cpp NNTRAINER_INCLUDES := $(NNTRAINER_ROOT)/nntrainer/include \ $(NNTRAINER_ROOT)/api \ diff --git a/nntrainer/include/layer.h b/nntrainer/include/layer.h index 53bcd91..1e5246b 100644 --- a/nntrainer/include/layer.h +++ b/nntrainer/include/layer.h @@ -30,6 +30,7 @@ #include #include #include +#include namespace nntrainer { @@ -62,21 +63,6 @@ enum class LayerType { }; /** - * @brief Enumeration of Weight Initialization Type - */ -enum class WeightInitializer { - WEIGHT_ZEROS, /** Zero initialization */ - WEIGHT_ONES, /** One initialization */ - WEIGHT_LECUN_NORMAL, /** LeCun normal initialization */ - WEIGHT_LECUN_UNIFORM, /** uniform initialization */ - WEIGHT_XAVIER_NORMAL, /** Xavier normal initialization */ - WEIGHT_XAVIER_UNIFORM, /** Xavier uniform initialization */ - WEIGHT_HE_NORMAL, /** He normal initialization */ - WEIGHT_HE_UNIFORM, /** He uniform initialization */ - WEIGHT_UNKNOWN /** Unknown */ -}; - -/** * @brief Print Options when printing layer info */ typedef enum { @@ -102,12 +88,13 @@ public: type(LayerType::LAYER_UNKNOWN), loss(0.0f), activation_type(ActivationType::ACT_NONE), - weight_regularizer(), + weight_regularizer(WeightRegularizerType::unknown), + weight_regularizer_constant(0.0f), weight_initializer(WeightInitializer::WEIGHT_XAVIER_UNIFORM), bias_initializer(WeightInitializer::WEIGHT_ZEROS), flatten(false), trainable(true), - param_size(0), + num_weights(0), num_inputs(1), num_outputs(1) {} @@ -154,14 +141,14 @@ public: /** * @brief read layer Weight & Bias data from file - * @note derived class can call this to get/save updatableParams + * @note derived class can call this to get/save weights * @param[in] file input file stream */ virtual void read(std::ifstream &file); /** * @brief save layer Weight & Bias data from file - * @note derived class can call this to get/save updatableParams + * @note derived class can call this to get/save weights * @param[in] file output file stream */ virtual void save(std::ofstream &file); @@ -297,9 +284,9 @@ public: * @brief set weight decay parameters * @param[in] w struct for weight decay */ - // void setWeightRegularizer(WeightRegularizerParam w) { - // weight_regularizer = w; - // } + void setWeightRegularizer(WeightRegularizerType type) { + weight_regularizer = type; + } /** * @brief set Weight Initialization Type @@ -349,10 +336,10 @@ public: void setTrainable(bool train) { trainable = train; } /** - * @brief get updatable params of all + * @brief get all weights of the layer * @retval vector of all params */ - std::shared_ptr getParams() { return params; } + std::shared_ptr getWeights() { return weight_list; } /** * @brief get if the output of this layer must be flatten @@ -394,12 +381,12 @@ public: * @brief get data alias at param position. * @exception std::out_of_range for index out of range */ - UpdatableParam ¶msAt(const unsigned int position) { - if (position >= param_size) { + Weight &weightAt(const unsigned int position) { + if (position >= num_weights) { throw std::out_of_range("index out of range"); } - return params.get()[position]; + return weight_list.get()[position]; } protected: @@ -413,7 +400,7 @@ protected: * @return bool is weightdecay type is L2 Norm */ bool isWeightRegularizerL2Norm() { - return weight_regularizer.type == WeightRegularizerType::l2norm; + return weight_regularizer == WeightRegularizerType::l2norm; } /** * @brief Input Tensor @@ -453,7 +440,9 @@ protected: ActivationType activation_type; - WeightRegularizerParam weight_regularizer; + WeightRegularizerType weight_regularizer; + + float weight_regularizer_constant; WeightInitializer weight_initializer; /** initializer for weights */ @@ -470,33 +459,33 @@ protected: bool trainable; /** - * @brief reserve memory for @a params and set @a param_size - * @exception std::invalid_argument when param_size is already set and + * @brief reserve memory for @a weight_list and set @a num_weights + * @exception std::invalid_argument when num_weights is already set and * shouldn't be changed again. */ - void setParamSize(unsigned int psize) { - if (psize == param_size) + void setNumWeights(unsigned int psize) { + if (psize == num_weights) return; - if (param_size > 0) { + if (num_weights > 0) { throw std::invalid_argument("param size can't be set once it is set"); } - param_size = psize; - params = std::shared_ptr( - new UpdatableParam[psize], std::default_delete()); + num_weights = psize; + weight_list = std::shared_ptr(new Weight[num_weights], + std::default_delete()); } /** - * @brief updatable params in this layer. This contains params of layers. - * @note UpdatableParam has weights and gradients paired. + * @brief weight_list in this layer. This contains trainable weights of + * layers. */ - std::shared_ptr params; + std::shared_ptr weight_list; - unsigned int param_size; /**< length of UpdatableParam * params. + unsigned int num_weights; /**< length of weights. This shouldn't be changed after initiation - use setParamSize() to avoid + use setNumWeights() to avoid setting parameters twice */ /** @@ -569,17 +558,6 @@ std::ostream &operator<<(std::ostream &out, T &l) { return out; } -/** - * @brief initialize Weight - * @param[in] w_dim TensorDim - * @param[in] initializer Weight Initializer - * @param[out] status Status - * @retval Tensor Initialized Tensor - */ -// TODO: move out -Tensor getInitializedTensor(const TensorDim &w_dim, - WeightInitializer initializer); - } // namespace nntrainer #endif /* __cplusplus */ diff --git a/nntrainer/include/neuralnet.h b/nntrainer/include/neuralnet.h index 3b58cbe..32b3f8b 100644 --- a/nntrainer/include/neuralnet.h +++ b/nntrainer/include/neuralnet.h @@ -389,6 +389,9 @@ private: */ void ensureName(std::shared_ptr layer, std::string prefix = ""); + /** + * @brief Swap function for the class + */ friend void swap(NeuralNetwork &lhs, NeuralNetwork &rhs) { using std::swap; diff --git a/nntrainer/include/optimizer.h b/nntrainer/include/optimizer.h index bcd15b1..0bb3ec0 100644 --- a/nntrainer/include/optimizer.h +++ b/nntrainer/include/optimizer.h @@ -25,21 +25,11 @@ #include #include +#include namespace nntrainer { /** - * @brief UpdatableParam that could be updated thorugh optimizer - */ -// TODO: move this out from here -struct UpdatableParam { - Tensor weight; /**< weight to be updated and used */ - Tensor grad; /**< gradient for the weight */ - std::string name; /**< name of the parameter */ - bool updatable = true; /**< if this param is updatable */ -}; - -/** * @brief Enumeration of Optimizer * 0. SGD * 1. ADAM @@ -48,27 +38,6 @@ struct UpdatableParam { enum class OptType { sgd = 0, adam = 1, unknown = 2 }; /** - * @brief Enumeration of Weight Decay type - * 0. L2Norm - * 1. Regression - * 2. Unknown (equivalent to none) - */ -// TODO: move this out of here -enum class WeightRegularizerType { l2norm = 0, regression = 1, unknown = 2 }; - -/** - * @brief type for the Weight Decay hyper-parameter - */ -typedef struct WeightRegularizerParam_ { - WeightRegularizerType type; - float constant; - - WeightRegularizerParam_() : - type(WeightRegularizerType::unknown), - constant(0.0f) {} -} WeightRegularizerParam; - -/** * @brief type for the Optimizor to save hyper-parameter */ typedef struct _OptParam { @@ -188,25 +157,25 @@ public: /** * @brief initialize optimizer. Initialize Weight if it is adam - * @param[in] params UpdatableParam list - * @param[in] param_size size of the array + * @param[in] params Weight list + * @param[in] num_weights size of the array * @param[in] setTensor true if the layer need weight update. * Input Layer and Batch Normalization layer won't need it. * Therefore, it sets false. * @retval #ML_ERROR_NONE Successful. * @retval #ML_ERROR_INVALID_PARAMETER invalid parameter. */ - int initialize(std::shared_ptr params, - unsigned int param_size, bool setTensor); + int initialize(std::shared_ptr params, unsigned int num_weights, + bool setTensor); /** - * @brief apply gradient to weights - * @param[in] params array of updatable params. - * @param[in] param_size size of the array + * @brief apply gradient to weight_list + * @param[in] params Weight list + * @param[in] num_weights size of the array * @param[in] iteration nth epoch number */ - void apply_gradients(std::shared_ptr params, - unsigned int param_size, int iteration); + void apply_gradients(std::shared_ptr params, unsigned int num_weights, + int iteration); /** * @brief Property Enumeration diff --git a/nntrainer/include/tensor.h b/nntrainer/include/tensor.h index ed149e0..6d1a44a 100644 --- a/nntrainer/include/tensor.h +++ b/nntrainer/include/tensor.h @@ -158,7 +158,12 @@ public: */ Tensor &operator=(Tensor &&rhs) noexcept = default; - void swap(Tensor &lhs, Tensor &rhs) noexcept; + friend void swap(Tensor &lhs, Tensor &rhs) noexcept { + std::swap(lhs.dim, rhs.dim); + std::swap(lhs.data, rhs.data); + std::swap(lhs.strides, rhs.strides); + std::swap(lhs.is_contiguous, rhs.is_contiguous); + } /** * @brief Comparison operator overload diff --git a/nntrainer/include/tensor_dim.h b/nntrainer/include/tensor_dim.h index 0719e1e..e2e6a6e 100644 --- a/nntrainer/include/tensor_dim.h +++ b/nntrainer/include/tensor_dim.h @@ -64,7 +64,12 @@ public: * @parma[out] lhs Optimizer * @parma[in] rhs Optimizer */ - void swap(TensorDim &lhs, TensorDim &rhs) noexcept; + friend void swap(TensorDim &lhs, TensorDim &rhs) noexcept { + std::swap_ranges(std::begin(lhs.dim), std::begin(lhs.dim) + MAXDIM, + std::begin(rhs.dim)); + std::swap(lhs.len, rhs.len); + std::swap(lhs.feature_len, rhs.feature_len); + } unsigned int batch() const { return dim[0]; }; unsigned int channel() const { return dim[1]; }; diff --git a/nntrainer/include/weight.h b/nntrainer/include/weight.h new file mode 100644 index 0000000..af0714b --- /dev/null +++ b/nntrainer/include/weight.h @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: Apache-2.0-only +/** + * Copyright (C) 2020 Parichay Kapoor + * + * @file weight.h + * @date 22 September 2020 + * @see https://github.com/nnstreamer/nntrainer + * @author Parichay Kapoor + * @bug No known bugs except for NYI items + * @brief This is Weight Class for Neural Network + * + */ + +#ifndef __WEIGHT_H__ +#define __WEIGHT_H__ + +#include + +namespace nntrainer { + +/** + * @brief Enumeration of Weight Decay type + */ +enum class WeightRegularizerType { + l2norm, /** L2 norm regularizer */ + unknown /** Unknown */ +}; + +/** + * @brief Enumeration of Weight Initialization Type + */ +enum class WeightInitializer { + WEIGHT_ZEROS, /** Zero initialization */ + WEIGHT_ONES, /** One initialization */ + WEIGHT_LECUN_NORMAL, /** LeCun normal initialization */ + WEIGHT_LECUN_UNIFORM, /** uniform initialization */ + WEIGHT_XAVIER_NORMAL, /** Xavier normal initialization */ + WEIGHT_XAVIER_UNIFORM, /** Xavier uniform initialization */ + WEIGHT_HE_NORMAL, /** He normal initialization */ + WEIGHT_HE_UNIFORM, /** He uniform initialization */ + WEIGHT_UNKNOWN /** Unknown */ +}; + +/** + * @class Weight + * @brief Weight with gradient, and its corresponding trainable property + */ +class Weight { + + /** Declare layers as friend to get variable/gradient reference */ + friend class Layer; + friend class Conv2DLayer; + friend class FullyConnectedLayer; + friend class BatchNormalizationLayer; + + /** Declare opitmizer as friend to get variable/gradient reference */ + friend class Optimizer; + +public: + /** + * @brief Weight default constructor + */ + Weight() : initializer(WeightInitializer::WEIGHT_UNKNOWN), trainable(false) {} + + /** + * @brief Construct a new Weight object + * + * @param dim Variable and gradient tensor dimension + * @param init Initializer for the tensor + * @param train If the variable is trainable + * @param name Name for this weight + */ + Weight( + const TensorDim &dim, + const WeightInitializer init = WeightInitializer::WEIGHT_XAVIER_UNIFORM, + bool train = true, std::string name = ""); + + /** + * @brief Allocate and initialize the variable + * + * @param dim Dimension for the variable + */ + void initializeVar(const TensorDim &dim); + + /** + * @brief Swap for weight + * + * @param lhs Swap to + * @param rhs Swap from + * @note Only swap gradient if trainable + */ + friend void swap(Weight &lhs, Weight &rhs) noexcept { + using std::swap; + + swap(lhs.var, rhs.var); + swap(lhs.initializer, rhs.initializer); + swap(lhs.trainable, rhs.trainable); + swap(lhs.grad, rhs.grad); + swap(lhs.name, rhs.name); + } + + /** + * @brief copy assigment + * + * @param rhs copy from + * @return Weight& Updated weight + */ + Weight &operator=(const Weight &rhs) = default; + + /** + * @brief move assignment + * + * @param rhs move from + * @return Weight& Updated weight + */ + Weight &operator=(Weight &&rhs) = default; + + /** + * @brief Get the TensorDim + * + * @return TensorDim Dimension + */ + TensorDim getDim() { return var.getDim(); } + + /** + * @brief Get if the weight is trainable + * + * @return true if trainable + * @return false is not trainable + */ + bool getTrainable() { return trainable; } + + /** + * @brief Get the name of the weight + * + * @return std::string name + */ + std::string getName() { return name; } + + /** + * @brief Get the variable tensor (by name) + * + * @return Tensor Variable tensor + */ + Tensor getVariable() { return var; } + + /** + * @brief Get the Gradient tensor (by name) + * + * @return Tensor Gradient tensor + */ + Tensor getGradient() { return grad; } + +private: + /** + * @brief Get the variable tensor (by reference) + * + * @return Tensor Variable tensor + */ + Tensor &getVariableRef() { return var; } + + /** + * @brief Get the Gradient tensor (by reference) + * + * @return Tensor Gradient tensor + */ + Tensor &getGradientRef() { return grad; } + + Tensor var; /**< variable to be updated and used */ + Tensor grad; /**< gradient for the variable */ + WeightInitializer initializer; /**< initializer for this variable */ + bool trainable; /**< if this variable is trainable */ + std::string name; /**< name of the parameter */ +}; + +} // namespace nntrainer + +#endif /** __WEIGHT_H__ */ diff --git a/nntrainer/meson.build b/nntrainer/meson.build index 611aebb..9366592 100644 --- a/nntrainer/meson.build +++ b/nntrainer/meson.build @@ -27,6 +27,7 @@ endif nntrainer_sources = [ 'src/activation_layer.cpp', 'src/addition_layer.cpp', + 'src/blas_interface.cpp', 'src/bn_layer.cpp', 'src/conv2d_layer.cpp', 'src/databuffer.cpp', @@ -47,12 +48,13 @@ nntrainer_sources = [ 'src/tensor.cpp', 'src/tensor_dim.cpp', 'src/util_func.cpp', - 'src/blas_interface.cpp' + 'src/weight.cpp' ] nntrainer_headers = [ 'include/activation_layer.h', 'include/addition_layer.h', + 'include/blas_interface.h', 'include/bn_layer.h', 'include/conv2d_layer.h', 'include/databuffer.h', @@ -74,7 +76,7 @@ nntrainer_headers = [ 'include/tensor.h', 'include/tensor_dim.h', 'include/util_func.h', - 'include/blas_interface.h', + 'include/weight.h', '../api/nntrainer-api-common.h' ] diff --git a/nntrainer/src/bn_layer.cpp b/nntrainer/src/bn_layer.cpp index dd66746..828a3cb 100644 --- a/nntrainer/src/bn_layer.cpp +++ b/nntrainer/src/bn_layer.cpp @@ -52,22 +52,18 @@ int BatchNormalizationLayer::initialize() { axes_to_reduce.push_back(i); } - Tensor mu = - getInitializedTensor(dim, initializers[static_cast(BNParams::mu)]); - Tensor var = - getInitializedTensor(dim, initializers[static_cast(BNParams::var)]); - - Tensor gamma = - getInitializedTensor(dim, initializers[static_cast(BNParams::gamma)]); - Tensor beta = - getInitializedTensor(dim, initializers[static_cast(BNParams::beta)]); - - setParamSize(4); - paramsAt(0) = {std::move(mu), Tensor(), "BN:moving_mean", false}; + setNumWeights(4); + weightAt(0) = + std::move(Weight(dim, initializers[static_cast(BNParams::mu)], false, + "BN:moving_mean")); ///@todo shift var to std to save computation - paramsAt(1) = {std::move(var), Tensor(), "BN:moving_variance", false}; - paramsAt(2) = {std::move(gamma), Tensor(gamma.getDim()), "BN:gamma"}; - paramsAt(3) = {std::move(beta), Tensor(beta.getDim()), "BN:beta"}; + weightAt(1) = + std::move(Weight(dim, initializers[static_cast(BNParams::var)], false, + "BN:moving_variance")); + weightAt(2) = std::move(Weight( + dim, initializers[static_cast(BNParams::gamma)], true, "BN:gamma")); + weightAt(3) = std::move(Weight( + dim, initializers[static_cast(BNParams::beta)], true, "BN:beta")); return status; } @@ -119,10 +115,10 @@ void BatchNormalizationLayer::setProperty(const PropertyType type, } sharedConstTensor BatchNormalizationLayer::forwarding(sharedConstTensor in) { - Tensor &mu = paramsAt(static_cast(BNParams::mu)).weight; - Tensor &var = paramsAt(static_cast(BNParams::var)).weight; - Tensor &gamma = paramsAt(static_cast(BNParams::gamma)).weight; - Tensor &beta = paramsAt(static_cast(BNParams::beta)).weight; + Tensor &mu = weightAt(static_cast(BNParams::mu)).getVariableRef(); + Tensor &var = weightAt(static_cast(BNParams::var)).getVariableRef(); + Tensor &gamma = weightAt(static_cast(BNParams::gamma)).getVariableRef(); + Tensor &beta = weightAt(static_cast(BNParams::beta)).getVariableRef(); input = *in; /// @todo change trainable #524 @@ -155,9 +151,9 @@ sharedConstTensor BatchNormalizationLayer::forwarding(sharedConstTensor in) { sharedConstTensor BatchNormalizationLayer::backwarding(sharedConstTensor derivative, int iteration) { - Tensor &gamma = paramsAt(static_cast(BNParams::gamma)).weight; - Tensor &dgamma = paramsAt(static_cast(BNParams::gamma)).grad; - Tensor &dbeta = paramsAt(static_cast(BNParams::beta)).grad; + Tensor &gamma = weightAt(static_cast(BNParams::gamma)).getVariableRef(); + Tensor &dgamma = weightAt(static_cast(BNParams::gamma)).getGradientRef(); + Tensor &dbeta = weightAt(static_cast(BNParams::beta)).getGradientRef(); Tensor dx_normalized; Tensor deriv = *derivative; @@ -180,7 +176,7 @@ BatchNormalizationLayer::backwarding(sharedConstTensor derivative, Tensor dx = dx_2.multiply(dx_1); dx.divide_i(N); - opt.apply_gradients(params, param_size, iteration); + opt.apply_gradients(weight_list, num_weights, iteration); return MAKE_SHARED_TENSOR(std::move(dx)); } diff --git a/nntrainer/src/conv2d_layer.cpp b/nntrainer/src/conv2d_layer.cpp index bdec87c..8a7590c 100644 --- a/nntrainer/src/conv2d_layer.cpp +++ b/nntrainer/src/conv2d_layer.cpp @@ -37,28 +37,16 @@ int Conv2DLayer::initialize() { std::string kernelPrefix = "Conv2d:filter"; std::string biasPrefix = "Conv2d:bias"; - setParamSize(filter_size * 2); + setNumWeights(filter_size * 2); for (unsigned int i = 0; i < filter_size; ++i) { - Tensor Knl = getInitializedTensor(dim, weight_initializer); - NN_RETURN_STATUS(); - - Tensor bias = getInitializedTensor(bias_dim, bias_initializer); - NN_RETURN_STATUS(); - - Tensor delK(dim); - delK.setZero(); - - Tensor delBias(bias_dim); - delBias.setZero(); - /*< @note: order of weight and bias are: w0 w1 w2 ... w3 */ - paramsAt(i) = {std::move(Knl), std::move(delK), - kernelPrefix + std::to_string(i)}; - paramsAt(i + filter_size) = {std::move(bias), std::move(delBias), - biasPrefix + std::to_string(i)}; + weightAt(i) = std::move( + Weight(dim, weight_initializer, true, kernelPrefix + std::to_string(i))); + weightAt(i + filter_size) = std::move( + Weight(bias_dim, bias_initializer, true, biasPrefix + std::to_string(i))); } // this output_dim should be the same with dimension of hidden @@ -136,7 +124,7 @@ sharedConstTensor Conv2DLayer::forwarding(sharedConstTensor in) { std::vector imkernel(kdim.getFeatureLen() * filter_size); for (unsigned int i = 0; i < filter_size; ++i) { - Tensor &filters = paramsAt(i).weight; + Tensor &filters = weightAt(i).getVariableRef(); float *d = imkernel.data(); memcpy(&d[i * kdim.getFeatureLen()], filters.getData(), kdim.getFeatureLen() * sizeof(float)); @@ -155,7 +143,7 @@ sharedConstTensor Conv2DLayer::forwarding(sharedConstTensor in) { out.size() * sizeof(float)); for (unsigned int i = 0; i < filter_size; i++) { - Tensor &bias = paramsAt(i + filter_size).weight; + Tensor &bias = weightAt(i + filter_size).getVariableRef(); Tensor tmp(1, 1, hidden.height(), hidden.width()); tmp.setValue(bias.getValue(0, 0, 0, 0)); saxpy(hidden.height() * hidden.width(), 1, tmp.getData(), 1, @@ -166,10 +154,10 @@ sharedConstTensor Conv2DLayer::forwarding(sharedConstTensor in) { } loss = 0.0f; - if (weight_regularizer.type == WeightRegularizerType::l2norm) { + if (weight_regularizer == WeightRegularizerType::l2norm) { for (unsigned int i = 0; i < filter_size; ++i) { - Tensor &weight = paramsAt(i).weight; - loss += weight_regularizer.constant * 0.5f * (weight.l2norm()); + Tensor &weight = weightAt(i).getVariableRef(); + loss += weight_regularizer_constant * 0.5f * (weight.l2norm()); } loss /= filter_size; } @@ -186,8 +174,8 @@ sharedConstTensor Conv2DLayer::backwarding(sharedConstTensor derivative, same_pad[1] = kernel_size[1] - 1; for (unsigned int i = 0; i < filter_size; ++i) { - Tensor &delK = paramsAt(i).grad; - Tensor &delBias = paramsAt(i + filter_size).grad; + Tensor &delK = weightAt(i).getGradientRef(); + Tensor &delBias = weightAt(i + filter_size).getGradientRef(); delK.setZero(); delBias.setZero(); } @@ -253,8 +241,8 @@ sharedConstTensor Conv2DLayer::backwarding(sharedConstTensor derivative, throw std::runtime_error("Backwarding Convolution failed."); for (unsigned int i = 0; i < filter_size; ++i) { - Tensor &delK = paramsAt(i).grad; - Tensor &delBias = paramsAt(i + filter_size).grad; + Tensor &delK = weightAt(i).getGradientRef(); + Tensor &delBias = weightAt(i + filter_size).getGradientRef(); float *del = delK.getData(); unsigned int s = kernel_size[0] * kernel_size[1] * input_dim.channel(); @@ -323,7 +311,7 @@ sharedConstTensor Conv2DLayer::backwarding(sharedConstTensor derivative, for (unsigned int j = 0; j < ret.channel(); ++j) { for (unsigned int i = 0; i < filter_size; ++i) { - Tensor &filters = paramsAt(i).weight; + Tensor &filters = weightAt(i).getVariableRef(); for (unsigned int k = 0; k < kernel_size[0] * kernel_size[1]; ++k) { d[count++] = filters.getData()[j * kernel_size[0] * kernel_size[1] + k]; } @@ -351,17 +339,17 @@ sharedConstTensor Conv2DLayer::backwarding(sharedConstTensor derivative, if (trainable) { // Update K / bias for (unsigned int i = 0; i < filter_size; ++i) { - Tensor &delK = paramsAt(i).grad; - Tensor &filters = paramsAt(i).weight; + Tensor &delK = weightAt(i).getGradientRef(); + Tensor &filters = weightAt(i).getVariableRef(); if (isWeightRegularizerL2Norm()) { - status = delK.add_i(filters, weight_regularizer.constant); + status = delK.add_i(filters, weight_regularizer_constant); if (status != ML_ERROR_NONE) throw std::runtime_error("Weight regularization failed"); } } - opt.apply_gradients(params, param_size, iteration); + opt.apply_gradients(weight_list, num_weights, iteration); } return MAKE_SHARED_TENSOR(std::move(strip_pad(ret, padding))); diff --git a/nntrainer/src/fc_layer.cpp b/nntrainer/src/fc_layer.cpp index 5e90fe3..7db4f7a 100644 --- a/nntrainer/src/fc_layer.cpp +++ b/nntrainer/src/fc_layer.cpp @@ -46,12 +46,9 @@ int FullyConnectedLayer::initialize() { dim.height(input_dim.width()); dim.batch(1); - Tensor weight = getInitializedTensor(dim, weight_initializer); - Tensor bias = getInitializedTensor(bias_dim, bias_initializer); - - setParamSize(2); - paramsAt(0) = {std::move(weight), Tensor(weight.getDim()), "FC:weight"}; - paramsAt(1) = {std::move(bias), Tensor(bias.getDim()), "FC:bias"}; + setNumWeights(2); + weightAt(0) = std::move(Weight(dim, weight_initializer, true, "FC:weight")); + weightAt(1) = std::move(Weight(bias_dim, bias_initializer, true, "FC:bias")); return status; } @@ -74,15 +71,16 @@ void FullyConnectedLayer::setProperty(const PropertyType type, } sharedConstTensor FullyConnectedLayer::forwarding(sharedConstTensor in) { - Tensor &weight = paramsAt(static_cast(FCParams::weight)).weight; - Tensor &bias = paramsAt(static_cast(FCParams::bias)).weight; + Tensor &weight = + weightAt(static_cast(FCParams::weight)).getVariableRef(); + Tensor &bias = weightAt(static_cast(FCParams::bias)).getVariableRef(); input = *in; hidden = input.dot(weight); hidden.add_i(bias); - if (weight_regularizer.type == WeightRegularizerType::l2norm) { - loss = weight_regularizer.constant * 0.5f * (weight.l2norm()); + if (weight_regularizer == WeightRegularizerType::l2norm) { + loss = weight_regularizer_constant * 0.5f * (weight.l2norm()); } return MAKE_SHARED_TENSOR(hidden); @@ -116,20 +114,20 @@ sharedConstTensor FullyConnectedLayer::backwarding(sharedConstTensor derivative, int iteration) { unsigned int weight_idx = static_cast(FCParams::weight); unsigned int bias_idx = static_cast(FCParams::bias); - Tensor &weight = paramsAt(weight_idx).weight; - Tensor &djdw = paramsAt(weight_idx).grad; - Tensor &djdb = paramsAt(bias_idx).grad; + Tensor &weight = weightAt(weight_idx).getVariableRef(); + Tensor &djdw = weightAt(weight_idx).getGradientRef(); + Tensor &djdb = weightAt(bias_idx).getGradientRef(); Tensor ret = derivative->dot(weight, false, true); djdb = derivative->sum(0); djdw = input.dot(*derivative, true, false); if (isWeightRegularizerL2Norm()) - djdw.add_i(weight, weight_regularizer.constant); + djdw.add_i(weight, weight_regularizer_constant); djdw = djdw.sum(0); if (trainable) { - opt.apply_gradients(params, param_size, iteration); + opt.apply_gradients(weight_list, num_weights, iteration); } return MAKE_SHARED_TENSOR(std::move(ret)); diff --git a/nntrainer/src/layer.cpp b/nntrainer/src/layer.cpp index 19b64a9..e3849b7 100644 --- a/nntrainer/src/layer.cpp +++ b/nntrainer/src/layer.cpp @@ -44,7 +44,7 @@ int Layer::setOptimizer(Optimizer &opt) { this->opt.setType(opt.getType()); this->opt.setOptParam(opt.getOptParam()); - return this->opt.initialize(params, param_size, true); + return this->opt.initialize(weight_list, num_weights, true); } int Layer::checkValidation() { @@ -63,69 +63,24 @@ int Layer::checkValidation() { } void Layer::copy(std::shared_ptr l) { - setParamSize(l->param_size); - for (unsigned int i = 0; i < l->param_size; ++i) { - paramsAt(i) = l->paramsAt(i); + setNumWeights(l->num_weights); + for (unsigned int i = 0; i < num_weights; ++i) { + weightAt(i) = l->weightAt(i); } } void Layer::read(std::ifstream &file) { - for (unsigned int i = 0; i < param_size; ++i) { - paramsAt(i).weight.read(file); + for (unsigned int i = 0; i < num_weights; ++i) { + weightAt(i).getVariableRef().read(file); } } void Layer::save(std::ofstream &file) { - for (unsigned int i = 0; i < param_size; ++i) { - paramsAt(i).weight.save(file); + for (unsigned int i = 0; i < num_weights; ++i) { + weightAt(i).getVariableRef().save(file); } } -Tensor getInitializedTensor(const TensorDim &w_dim, - WeightInitializer initializer) { - Tensor w = Tensor(w_dim); - - if (initializer == WeightInitializer::WEIGHT_UNKNOWN) { - ml_logw("Warning: Weight Initalization Type is not set. " - "WEIGHT_XAVIER_NORMAL is used by default"); - initializer = WeightInitializer::WEIGHT_XAVIER_NORMAL; - } - - switch (initializer) { - case WeightInitializer::WEIGHT_ZEROS: - w.setZero(); - break; - case WeightInitializer::WEIGHT_ONES: - w.setValue(1.0f); - break; - case WeightInitializer::WEIGHT_LECUN_NORMAL: - w.setRandNormal(0.0f, sqrtFloat(1.0f / w_dim.height())); - break; - case WeightInitializer::WEIGHT_XAVIER_NORMAL: - w.setRandNormal(0.0f, sqrtFloat(2.0f / (w_dim.width() + w_dim.height()))); - break; - case WeightInitializer::WEIGHT_HE_NORMAL: - w.setRandNormal(0.0f, sqrtFloat(2.0f / (w_dim.height()))); - break; - case WeightInitializer::WEIGHT_LECUN_UNIFORM: - w.setRandUniform(-1.0f * sqrtFloat(1.0f / w_dim.height()), - sqrtFloat(1.0f / w_dim.height())); - break; - case WeightInitializer::WEIGHT_XAVIER_UNIFORM: - w.setRandUniform(-1.0f * sqrtFloat(6.0f / (w_dim.height() + w_dim.width())), - sqrtFloat(6.0 / (w_dim.height() + w_dim.width()))); - break; - case WeightInitializer::WEIGHT_HE_UNIFORM: - w.setRandUniform(-1.0f * sqrtFloat(6.0f / (w_dim.height())), - sqrtFloat(6.0 / (w_dim.height()))); - break; - default: - break; - } - - return w; -} - int Layer::setProperty(std::vector values) { int status = ML_ERROR_NONE; @@ -203,16 +158,16 @@ void Layer::setProperty(const PropertyType type, const std::string &value) { break; case PropertyType::weight_regularizer: if (!value.empty()) { - weight_regularizer.type = + weight_regularizer = (WeightRegularizerType)parseType(value, TOKEN_WEIGHT_REGULARIZER); - if (weight_regularizer.type == WeightRegularizerType::unknown) { + if (weight_regularizer == WeightRegularizerType::unknown) { throw std::invalid_argument("[Layer] Unknown Weight decay"); } } break; case PropertyType::weight_regularizer_constant: if (!value.empty()) { - status = setFloat(weight_regularizer.constant, value); + status = setFloat(weight_regularizer_constant, value); throw_status(status); } break; @@ -257,8 +212,8 @@ void Layer::printIfValid(std::ostream &out, const PropertyType type, void Layer::printShapeInfo(std::ostream &out) { out << "input " << input_dim; - for (unsigned int i = 0; i < param_size; i++) - out << "inner" << i << " " << paramsAt(i).weight.getDim(); + for (unsigned int i = 0; i < num_weights; i++) + out << "inner" << i << " " << weightAt(i).var.getDim(); out << "output " << output_dim; } @@ -272,9 +227,9 @@ void Layer::printPropertiesMeta(std::ostream &out) { void Layer::printProperties(std::ostream &out) { out << "Trainable: " << trainable << std::endl; printIfValid(out, PropertyType::weight_regularizer, - static_cast(weight_regularizer.type)); + static_cast(weight_regularizer)); printIfValid(out, PropertyType::weight_regularizer_constant, - weight_regularizer.constant); + weight_regularizer_constant); } void Layer::printMetric(std::ostream &out) { @@ -310,9 +265,9 @@ void Layer::print(std::ostream &out, unsigned int flags) { if (flags & PRINT_WEIGHTS) { out << "======weights: " << std::endl; - for (unsigned int i = 0; i < param_size; ++i) { - out << '[' << paramsAt(i).name << ']' << std::endl; - out << paramsAt(i).weight; + for (unsigned int i = 0; i < num_weights; ++i) { + out << '[' << weightAt(i).getName() << ']' << std::endl; + out << weightAt(i).var; } } diff --git a/nntrainer/src/optimizer.cpp b/nntrainer/src/optimizer.cpp index eec4a5b..4fe43d7 100644 --- a/nntrainer/src/optimizer.cpp +++ b/nntrainer/src/optimizer.cpp @@ -61,37 +61,32 @@ int Optimizer::setOptParam(OptParam p) { return status; } -int Optimizer::initialize(std::shared_ptr params, - unsigned int param_size, bool set_tensor) { +int Optimizer::initialize(std::shared_ptr weight_list, + unsigned int num_weights, bool set_tensor) { int status = ML_ERROR_NONE; if (type == OptType::adam && set_tensor) { - UpdatableParam *param_data = params.get(); + for (unsigned int i = 0; i < num_weights; ++i) { + Weight &w = weight_list.get()[i]; - for (unsigned int i = 0; i < param_size; ++i) { - UpdatableParam ¶m = param_data[i]; - - if (!param.updatable) + // TODO: only trainable weights must be sent to optimizer + if (!w.getTrainable()) continue; - Tensor &weight = param.weight; - Tensor &grad = param.grad; - Tensor w = Tensor(weight.getDim()); - w.setZero(); - Tensor g = Tensor(grad.getDim()); - g.setZero(); + Tensor m = Tensor(w.getDim()); + m.setZero(); + Tensor v = Tensor(w.getDim()); + v.setZero(); std::pair p = - std::pair(std::move(w), std::move(g)); + std::pair(std::move(m), std::move(v)); weight_mv.push_back(std::move(p)); } } return status; } -void Optimizer::apply_gradients(std::shared_ptr params, - unsigned int param_size, int iteration) { - - UpdatableParam *param_data = params.get(); +void Optimizer::apply_gradients(std::shared_ptr weight_list, + unsigned int num_weights, int iteration) { double ll = popt.learning_rate; @@ -108,14 +103,14 @@ void Optimizer::apply_gradients(std::shared_ptr params, } int idx = 0; - for (unsigned int i = 0; i < param_size; ++i) { - UpdatableParam ¶m = param_data[i]; + for (unsigned int i = 0; i < num_weights; ++i) { + Weight &weight = weight_list.get()[i]; - if (!param.updatable) + if (!weight.getTrainable()) continue; - Tensor &x = param.weight; - const Tensor &x_grad = param.grad; + Tensor &x = weight.getVariableRef(); + const Tensor &x_grad = weight.getGradientRef(); switch (type) { case OptType::sgd: x.add_i(x_grad, -ll); diff --git a/nntrainer/src/tensor.cpp b/nntrainer/src/tensor.cpp index eaffb70..4674910 100644 --- a/nntrainer/src/tensor.cpp +++ b/nntrainer/src/tensor.cpp @@ -90,13 +90,6 @@ Tensor::Tensor(const TensorDim &d, const float *buf) : Tensor() { } } -void Tensor::swap(Tensor &lhs, Tensor &rhs) noexcept { - std::swap(lhs.dim, rhs.dim); - std::swap(lhs.data, rhs.data); - std::swap(lhs.strides, rhs.strides); - std::swap(lhs.is_contiguous, rhs.is_contiguous); -} - bool Tensor::operator==(const Tensor &rhs) const { if (this->dim != rhs.dim) return false; diff --git a/nntrainer/src/tensor_dim.cpp b/nntrainer/src/tensor_dim.cpp index 40279d4..b3b6f8c 100644 --- a/nntrainer/src/tensor_dim.cpp +++ b/nntrainer/src/tensor_dim.cpp @@ -24,21 +24,18 @@ namespace nntrainer { TensorDim &TensorDim::operator=(const TensorDim &rhs) { + using std::swap; + TensorDim tmp(rhs.batch(), rhs.channel(), rhs.height(), rhs.width()); - this->swap(*this, tmp); + swap(*this, tmp); return *this; } TensorDim &TensorDim::operator=(TensorDim &&rhs) noexcept { - this->swap(*this, rhs); - return *this; -} + using std::swap; -void TensorDim::swap(TensorDim &lhs, TensorDim &rhs) noexcept { - std::swap_ranges(std::begin(lhs.dim), std::begin(lhs.dim) + MAXDIM, - std::begin(rhs.dim)); - std::swap(lhs.len, rhs.len); - std::swap(lhs.feature_len, rhs.feature_len); + swap(*this, rhs); + return *this; } void TensorDim::resetLen() { diff --git a/nntrainer/src/weight.cpp b/nntrainer/src/weight.cpp new file mode 100644 index 0000000..afb8164 --- /dev/null +++ b/nntrainer/src/weight.cpp @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0-only +/** + * Copyright (C) 2020 Parichay Kapoor + * + * @file weight.cpp + * @date 22 September 2020 + * @see https://github.com/nnstreamer/nntrainer + * @author Parichay Kapoor + * @bug No known bugs except for NYI items + * @brief This is Weight Class for Neural Network + * + */ + +#include +#include + +namespace nntrainer { + +Weight::Weight(const TensorDim &dim, const WeightInitializer init, bool train, + std::string name) : + initializer(init), + trainable(train), + name(name) { + if (initializer == WeightInitializer::WEIGHT_UNKNOWN) + throw std::invalid_argument("Weight initializer unknown"); + + initializeVar(dim); + if (trainable) { + grad = Tensor(dim); + grad.setZero(); + } else + grad = Tensor(); +} + +void Weight::initializeVar(const TensorDim &dim) { + var = Tensor(dim); + switch (initializer) { + case WeightInitializer::WEIGHT_ZEROS: + var.setZero(); + break; + case WeightInitializer::WEIGHT_ONES: + var.setValue(1.0f); + break; + case WeightInitializer::WEIGHT_LECUN_NORMAL: + var.setRandNormal(0.0f, sqrtFloat(1.0f / dim.height())); + break; + case WeightInitializer::WEIGHT_XAVIER_NORMAL: + var.setRandNormal(0.0f, sqrtFloat(2.0f / (dim.width() + dim.height()))); + break; + case WeightInitializer::WEIGHT_HE_NORMAL: + var.setRandNormal(0.0f, sqrtFloat(2.0f / (dim.height()))); + break; + case WeightInitializer::WEIGHT_LECUN_UNIFORM: + var.setRandUniform(-1.0f * sqrtFloat(1.0f / dim.height()), + sqrtFloat(1.0f / dim.height())); + break; + case WeightInitializer::WEIGHT_XAVIER_UNIFORM: + var.setRandUniform(-1.0f * sqrtFloat(6.0f / (dim.height() + dim.width())), + sqrtFloat(6.0 / (dim.height() + dim.width()))); + break; + case WeightInitializer::WEIGHT_HE_UNIFORM: + var.setRandUniform(-1.0f * sqrtFloat(6.0f / (dim.height())), + sqrtFloat(6.0 / (dim.height()))); + break; + default: + break; + } +} + +} // namespace nntrainer diff --git a/packaging/nntrainer.spec b/packaging/nntrainer.spec index 9cbd930..ae9e16e 100644 --- a/packaging/nntrainer.spec +++ b/packaging/nntrainer.spec @@ -321,6 +321,7 @@ cp -r result %{buildroot}%{_datadir}/nntrainer/unittest/ %{_includedir}/nntrainer/addition_layer.h %{_includedir}/nntrainer/nntrainer-api-common.h %{_includedir}/nntrainer/blas_interface.h +%{_includedir}/nntrainer/weight.h %{_libdir}/pkgconfig/nntrainer.pc %files devel-static diff --git a/test/unittest/unittest_nntrainer_layers.cpp b/test/unittest/unittest_nntrainer_layers.cpp index a979b3f..44d1fd9 100644 --- a/test/unittest/unittest_nntrainer_layers.cpp +++ b/test/unittest/unittest_nntrainer_layers.cpp @@ -526,8 +526,10 @@ protected: void loadUpdatedWeightsGradients(const char *file_uw, const char *file_g) { for (int idx = 0; idx < 2; ++idx) { - new_w.push_back(nntrainer::Tensor(layer.paramsAt(idx).weight.getDim())); - grad.push_back(nntrainer::Tensor(layer.paramsAt(idx).grad.getDim())); + new_w.push_back( + nntrainer::Tensor(layer.weightAt(idx).getVariable().getDim())); + grad.push_back( + nntrainer::Tensor(layer.weightAt(idx).getGradient().getDim())); } loadFile(file_uw, new_w); @@ -542,12 +544,12 @@ protected: } void matchUpdatedWeightsGradients() { - std::shared_ptr params = layer.getParams(); + std::shared_ptr params = layer.getWeights(); /** Match gradients and updated weights */ for (int idx = 0; idx < 2; ++idx) { - matchOutput(params.get()[idx].grad, grad[idx]); - matchOutput(params.get()[idx].weight, new_w[idx]); + matchOutput(params.get()[idx].getGradient(), grad[idx]); + matchOutput(params.get()[idx].getVariable(), new_w[idx]); } } @@ -583,14 +585,14 @@ TEST_F(nntrainer_FullyConnectedLayer_TFmatch, matchOutput(result, "tc_fc_1_goldenFCGradientAdam.out"); - nntrainer::UpdatableParam *param_data = layer.getParams().get(); + nntrainer::Weight *param_data = layer.getWeights().get(); - nntrainer::UpdatableParam ¶m = param_data[0]; - nntrainer::Tensor &weight = param.weight; + nntrainer::Weight ¶m = param_data[0]; + nntrainer::Tensor weight = param.getVariable(); matchOutput(weight, "tc_fc_1_goldenFCUpdatedWeightAdam.out"); - nntrainer::UpdatableParam &bias_param = param_data[1]; - nntrainer::Tensor &bias = bias_param.weight; + nntrainer::Weight &bias_param = param_data[1]; + nntrainer::Tensor bias = bias_param.getVariable(); matchOutput(bias, "tc_fc_1_goldenFCUpdatedBiasAdam.out"); } @@ -1077,11 +1079,11 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_01_p) { EXPECT_NO_THROW( result = layer.backwarding(MAKE_SHARED_TENSOR(derivatives), 1).get()[0]); - nntrainer::UpdatableParam *param_data = layer.getParams().get(); + nntrainer::Weight *param_data = layer.getWeights().get(); for (unsigned int i = 0; i < filter_size * 2; ++i) { - nntrainer::UpdatableParam ¶m = param_data[i]; - nntrainer::Tensor &grad = param.grad; + nntrainer::Weight ¶m = param_data[i]; + nntrainer::Tensor grad = param.getGradient(); const float *gdata = grad.getData(); if (i < filter_size) { for (unsigned int j = 0; j < grad.length(); ++j) { @@ -1130,11 +1132,11 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_04_p) { EXPECT_NO_THROW( result = layer.backwarding(MAKE_SHARED_TENSOR(derivatives), 1).get()[0]); - nntrainer::UpdatableParam *param_data = layer.getParams().get(); + nntrainer::Weight *param_data = layer.getWeights().get(); for (unsigned int i = 0; i < filter_size * 2; ++i) { - nntrainer::UpdatableParam ¶m = param_data[i]; - nntrainer::Tensor &grad = param.grad; + nntrainer::Weight ¶m = param_data[i]; + nntrainer::Tensor grad = param.getGradient(); const float *gdata = grad.getData(); if (i < filter_size) { for (unsigned int j = 0; j < grad.length(); ++j) { @@ -1170,7 +1172,7 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_02_p) { std::vector weight_data; std::vector bias_grad; std::vector bias_weight; - nntrainer::UpdatableParam *param_data; + nntrainer::Weight *param_data; nntrainer::Tensor derivatives(2, 3, 5, 5); @@ -1184,11 +1186,11 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_02_p) { } EXPECT_NO_THROW( result = layer.backwarding(MAKE_SHARED_TENSOR(derivatives), 1).get()[0]); - param_data = layer.getParams().get(); + param_data = layer.getWeights().get(); for (unsigned int i = 0; i < filter_size * 2; ++i) { - nntrainer::UpdatableParam ¶m = param_data[i]; - nntrainer::Tensor &grad = param.grad; + nntrainer::Weight ¶m = param_data[i]; + nntrainer::Tensor grad = param.getGradient(); const float *gdata = grad.getData(); if (i < filter_size) { @@ -1213,11 +1215,11 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_02_p) { result = layer.backwarding(MAKE_SHARED_TENSOR(derivatives), 1).get()[0]); } - param_data = layer.getParams().get(); + param_data = layer.getWeights().get(); for (unsigned int i = 0; i < filter_size * 2; ++i) { - nntrainer::UpdatableParam ¶m = param_data[i]; - nntrainer::Tensor &grad = param.grad; + nntrainer::Weight ¶m = param_data[i]; + nntrainer::Tensor grad = param.getGradient(); const float *gdata = grad.getData(); if (i < filter_size) { @@ -1288,7 +1290,7 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_03_p) { std::vector weight_data; std::vector bias_grad; std::vector bias_weight; - nntrainer::UpdatableParam *param_data; + nntrainer::Weight *param_data; nntrainer::Tensor derivatives(1, 12, 24, 24); @@ -1318,13 +1320,13 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_03_p) { result = layer1.backwarding(MAKE_SHARED_TENSOR(result2), 1).get()[0]); /** Compare second conv */ - param_data = layer2.getParams().get(); + param_data = layer2.getWeights().get(); filter_size = 12; grad_data.clear(); bias_grad.clear(); for (unsigned int i = 0; i < filter_size * 2; ++i) { - nntrainer::UpdatableParam ¶m = param_data[i]; - nntrainer::Tensor &grad = param.grad; + nntrainer::Weight ¶m = param_data[i]; + nntrainer::Tensor grad = param.getGradient(); const float *gdata = grad.getData(); if (i < filter_size) { @@ -1342,13 +1344,13 @@ TEST_F(nntrainer_Conv2DLayer, backwarding_03_p) { matchOutput(bias_grad, "tc_conv2d_int_goldenBias2Grad.out"); /** Compare first conv */ - param_data = layer1.getParams().get(); + param_data = layer1.getWeights().get(); filter_size = 6; grad_data.clear(); bias_grad.clear(); for (unsigned int i = 0; i < filter_size * 2; ++i) { - nntrainer::UpdatableParam ¶m = param_data[i]; - nntrainer::Tensor &grad = param.grad; + nntrainer::Weight ¶m = param_data[i]; + nntrainer::Tensor grad = param.getGradient(); const float *gdata = grad.getData(); if (i < filter_size) { -- 2.7.4